diff --git a/obp-api/src/main/resources/props/sample.props.template b/obp-api/src/main/resources/props/sample.props.template index 8be6c442ad..19e30fd05a 100644 --- a/obp-api/src/main/resources/props/sample.props.template +++ b/obp-api/src/main/resources/props/sample.props.template @@ -157,6 +157,21 @@ long_endpoint_timeout = 55000 ## always runs. A message that cannot be delivered is retried with backoff, on top of this interval. #message_outbox.relay_interval_seconds=10 +## Webhook requests sent through the message outbox (Dynamic Entity webhooks). +## Attempts after which a webhook request that keeps failing (5xx, 408, 429, timeout, no connection) +## stops being retried and is left STICKY. With the relay's backoff, 144 is roughly a day. +#webhook.outbox_max_attempts=144 +## Allow webhook URLs with plain http. For local development only. +#webhook.allow_http=false +## Allow webhook URLs whose host is a loopback, private or link-local address, such as localhost or +## 10.0.0.5. For local development and tests only: in production this would let a webhook reach +## services inside OBP's own network. +#webhook.allow_private_addresses=false +## A webhook that has had this many requests queued within one minute is turned off with +## deactivated_reason CIRCUIT_BREAKER instead of firing again: the backstop against a receiver whose +## reply writes a record that fires the same webhook again. +#webhook.circuit_breaker_per_minute=60 + ## Enable user authentication via the connector #connector.user.authentication=true diff --git a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala index 8edce53e85..8572bd7e32 100644 --- a/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala +++ b/obp-api/src/main/scala/bootstrap/liftweb/Boot.scala @@ -1158,6 +1158,7 @@ object ToSchemify extends MdcLoggable { code.platformapp.PlatformApp, code.platformapp.PlatformAppRequiredScope, code.domainapi.DomainApi, + code.dynamicentitywebhook.DynamicEntityWebhook, PayeeLookup, UtilityPaymentCallback, BulkPayment, diff --git a/obp-api/src/main/scala/code/api/dynamic/endpoint/APIMethodsDynamicEndpoint.scala b/obp-api/src/main/scala/code/api/dynamic/endpoint/APIMethodsDynamicEndpoint.scala index de120098b2..e3a52d2735 100644 --- a/obp-api/src/main/scala/code/api/dynamic/endpoint/APIMethodsDynamicEndpoint.scala +++ b/obp-api/src/main/scala/code/api/dynamic/endpoint/APIMethodsDynamicEndpoint.scala @@ -208,13 +208,18 @@ trait APIMethodsDynamicEndpoint { dynamicData = DynamicDataProvider.connectorMethodProvider.vend.getAll(bankId, entityName, None, false) dynamicJsonData = JArray(dynamicData.map(it => com.openbankproject.commons.util.JsonAliases.parse(it.dataJson)).map(_.asInstanceOf[JObject])) entityObject = DynamicEndpointHelper.getObjectByKeyValuePair(dynamicJsonData, entityIdKey, entityIdValueFromUrl.get) - _ <- NewStyle.function.tryons(s"$InvalidEndpointMapping `response_mapping` must be linked to at least one valid dynamic entity!", 400, cc.callContext) { - val entityIdName = DynamicEntityHelper.createEntityId(entityName) - val entityIdValue = (entityObject \ entityIdName).asInstanceOf[JString].s - DynamicDataProvider.connectorMethodProvider.vend.delete(bankId, entityName, entityIdValue, None, false).head + entityIdName = DynamicEntityHelper.createEntityId(entityName) + entityIdValue <- NewStyle.function.tryons(s"$InvalidEndpointMapping `response_mapping` must be linked to at least one valid dynamic entity!", 400, cc.callContext) { + (entityObject \ entityIdName).asInstanceOf[JString].s } - entityBody = JsonUtils.buildJson(json, requestMappingJvalue) - (box, _) <- NewStyle.function.invokeDynamicConnector(CREATE, entityName, Some(entityBody.asInstanceOf[JObject]), None, bankId, None, None, false, Some(cc)) + // The record keeps its id: a PUT changes the record the URL names, and it is an update, so + // Dynamic Entity webhooks see UPDATED and can compare old and new values. The id field of the + // body is set to the record's id, so the stored values can never name a different record. + entityBody = JsonUtils.buildJson(json, requestMappingJvalue) match { + case body: JObject => JObject(body.obj.filterNot(_.name == entityIdName) :+ JField(entityIdName, JString(entityIdValue))) + case other => other + } + (box, _) <- NewStyle.function.invokeDynamicConnector(UPDATE, entityName, Some(entityBody.asInstanceOf[JObject]), Some(entityIdValue), bankId, None, None, false, Some(cc)) singleObject: JValue = unboxResult(box.asInstanceOf[Box[JValue]], entityName) responseBodyScheme = DynamicEndpointHelper.prepareMappingFields(responseMappingJvalue) responseBody = JsonUtils.buildJson(singleObject, responseBodyScheme) diff --git a/obp-api/src/main/scala/code/api/util/ApiRole.scala b/obp-api/src/main/scala/code/api/util/ApiRole.scala index 52a9aaa861..e45bd657bc 100644 --- a/obp-api/src/main/scala/code/api/util/ApiRole.scala +++ b/obp-api/src/main/scala/code/api/util/ApiRole.scala @@ -623,6 +623,20 @@ object ApiRole extends MdcLoggable{ case class CanDeleteDomainApi(requiresBankId: Boolean = true) extends ApiRole lazy val canDeleteDomainApi = CanDeleteDomainApi() + // Dynamic Entity webhooks: requests OBP sends to a URL when records of a Dynamic Entity change. Held at a + // bank id, or at SYS for the system space, like the other Roles of the dynamic spaces. + case class CanCreateDynamicEntityWebhook(requiresBankId: Boolean = true) extends ApiRole + lazy val canCreateDynamicEntityWebhook = CanCreateDynamicEntityWebhook() + + case class CanGetDynamicEntityWebhooks(requiresBankId: Boolean = true) extends ApiRole + lazy val canGetDynamicEntityWebhooks = CanGetDynamicEntityWebhooks() + + case class CanUpdateDynamicEntityWebhook(requiresBankId: Boolean = true) extends ApiRole + lazy val canUpdateDynamicEntityWebhook = CanUpdateDynamicEntityWebhook() + + case class CanDeleteDynamicEntityWebhook(requiresBankId: Boolean = true) extends ApiRole + lazy val canDeleteDynamicEntityWebhook = CanDeleteDynamicEntityWebhook() + // Shows which Consumers and client IP addresses are sending the most traffic to the instance // (TrafficSources). About the instance, so held at the empty bank id. It names Consumers and IP // addresses, which is why it is a Role of its own and not part of CanGetTelemetry. diff --git a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala index b5fc4e34c6..b3b26adb26 100644 --- a/obp-api/src/main/scala/code/api/util/ErrorMessages.scala +++ b/obp-api/src/main/scala/code/api/util/ErrorMessages.scala @@ -120,6 +120,15 @@ object ErrorMessages { val DomainApiPathClash = "OBP-09031: Endpoints of this space are ambiguous with each other, so a Domain API cannot publish the space until one of them is renamed or removed: " val DynamicPathAmbiguous = "OBP-09032: This would make a path ambiguous in its space. The Dynamic Entities and Dynamic Resource Docs of a space share one set of paths, which a Domain API can publish under its base path at any time, and a Domain API reads a hyphen in an entity name as an underscore, so a Dynamic Resource Doc's path may not start with a path variable, with my, public, community, openapi.json or openapi.yaml, or with the name of one of the space's Dynamic Entities in either spelling (like-this or like_this); a Dynamic Entity may not be named my, public, community, openapi.json or openapi.yaml, nor after the first segment of one of the space's Dynamic Resource Docs, nor so that it differs from another of the space's Dynamic Entities only by hyphens for underscores; and two Dynamic Resource Docs of one verb may not both match one request. Rename one of them: " val DynamicEntityNameHasHyphenOrSpace = "OBP-09033: Invalid Dynamic Entity name. A Dynamic Entity name may not contain a hyphen or a space; use an underscore between words, e.g. 'customer_preferences'. A Domain API serves the entity at the hyphenated form of its name as well. " + // Dynamic Entity webhooks: requests OBP sends to a URL when records of a Dynamic Entity change. + val DynamicEntityWebhookNotFound = "OBP-09034: Dynamic Entity webhook not found for this Dynamic Entity. Please specify a valid value for WEBHOOK_ID." + val InvalidDynamicEntityWebhookEvents = "OBP-09035: Invalid events. events must name at least one of CREATED, UPDATED and DELETED, each at most once. " + val InvalidDynamicEntityWebhookFieldCondition = "OBP-09036: Invalid field_conditions. Field conditions apply to updates only, so events must include UPDATED; each condition must name a field of the Dynamic Entity, at most once, and its from and to values, where given, must fit that field's type and differ from each other. " + val InvalidDynamicEntityWebhookUrl = "OBP-09037: Invalid url. A webhook url must be an absolute https URL with a host name, at most 1024 characters, and must not point at this OBP instance or at a loopback, private, link-local or cloud metadata address. " + val DynamicEntityWebhookUrlAlreadyExists = "OBP-09038: This Dynamic Entity already has a webhook with this url. One webhook per url is enough, because a webhook fires when any of its field conditions matches; change that webhook instead. " + val InvalidDynamicEntityWebhookDescription = "OBP-09039: Invalid description. description must be at most 2000 characters." + val DynamicEntityWebhookDeliveryNotFound = "OBP-09040: Dynamic Entity webhook delivery not found for this webhook. Please specify a valid value for EVENT_ID." + val DynamicEntityWebhookDeliveryNotSticky = "OBP-09041: Only a STICKY delivery can be retried. A PENDING delivery is retried automatically, and a DELIVERED one is done. " // General messages (OBP-10XXX) diff --git a/obp-api/src/main/scala/code/api/util/Glossary.scala b/obp-api/src/main/scala/code/api/util/Glossary.scala index 9b67523603..3467c5b399 100644 --- a/obp-api/src/main/scala/code/api/util/Glossary.scala +++ b/obp-api/src/main/scala/code/api/util/Glossary.scala @@ -4365,6 +4365,52 @@ object Glossary extends MdcLoggable { | |**Resource Docs.** `/obp/v7.0.0/banks/BANK_ID/domain-apis/resource-docs?base_path=BASE_PATH` gives the same endpoints as the OpenAPI document as OBP Resource Docs, each at its full published path, for API Explorer and other OBP tools. No login is needed. | +""".stripMargin) + + glossaryItems += GlossaryItem( + title = "Dynamic Entity Webhooks", + description = + s""" +|# Dynamic Entity Webhooks +| +|A **Dynamic Entity webhook** makes OBP send a POST request to a URL when records of one Dynamic Entity (${getGlossaryItemLink("Dynamic-Entities")}) are created, updated or deleted, so that a system outside OBP learns of a change without polling for it. +| +|**What fires.** A record written through the endpoints: the Dynamic Entity endpoints, Domain APIs (${getGlossaryItemLink("Domain APIs")}), Dynamic Endpoints backed by a Dynamic Entity, and Dynamic Resource Docs whose code writes records. Changes to an entity's definition, backups, restores and rows written straight into the database fire nothing. Neither do records a connector stores in a bank's own system instead of OBP's database. +| +|**Identifiers only.** The request names the event, the bank, the entity and the record, never the record's values or any user id. A receiver that needs the record fetches it with its own credentials, so the normal access checks apply when it reads. +| +|**The request.** One POST per event, with these headers: `OBP-Webhook-Id`, `OBP-Event-Id`, `OBP-Delivery-Attempt` (1 on the first attempt), `OBP-Signature-Status` and, when requests are signed, `OBP-Signature`. The body: +| +|```json +|{ +| "event_name": "dynamic_entity.record.updated", +| "event_id": "0f8e4a71-5ff1-4006-95bb-7fd9e4adaef9", +| "occurred_at": "2026-10-14T09:12:44Z", +| "webhook_id": "7d3c1c2e-5b7a-4f0e-9a51-2f1f6f0c9b11", +| "bank_id": "gh.29.uk", +| "entity_name": "loan_application", +| "record_id": "a1b2c3d4-0000-4000-8000-000000000001", +| "record_updated_at": "2026-10-14T09:12:44Z", +| "matched_conditions": [{"field": "approved", "from": false, "to": true}] +|} +|``` +| +|`event_name` is `dynamic_entity.record.created`, `dynamic_entity.record.updated`, `dynamic_entity.record.deleted`, or `webhook.test` for a test request, which has no `record_id`. `bank_id` is `SYS` for an entity of the system space. `matched_conditions` is present only on an update sent because of field conditions, and lists the conditions that matched as the webhook's owner wrote them. +| +|**Receiving a request.** Reply 2xx as soon as the request is stored, and do the work afterwards: a slow reply is retried. Drop a request whose `event_id` was handled already, and one whose `record_updated_at` is older than the last handled for the same record, since requests can repeat and arrive out of order. To check a signature, read `kid` from the `OBP-Signature` header (a JSON Web Signature with a detached payload, RFC 7515 Appendix F), find the key with that `kid` at `GET /obp/v3.1.0/certs`, and verify the signature against the exact bytes of the body with any JOSE library. Then fetch the record, if it is needed, with the receiver's own credentials. +| +|**Events and field conditions.** A webhook subscribes to any of `CREATED`, `UPDATED` and `DELETED`. Its optional `field_conditions` narrow `UPDATED`: each names a field of the entity and, optionally, the value it must change `from` and the value it must change `to`. `{"field": "approved", "from": false, "to": true}` fires only on that change, `{"field": "status", "to": "CLOSED"}` on any change into `CLOSED`, and `{"field": "amount"}` on any change to `amount`. An update fires the webhook when at least one condition matches. The fields and values are checked against the entity's definition when the webhook is created or changed. A record created with `approved` already true is a `CREATED` event, not a match. +| +|**Delivery.** At least once, in no particular order. Each event carries an `event_id`, the same on every retry, to drop duplicates by, and the record's `record_updated_at`, to drop an event older than one already handled. Requests are queued in the same database transaction as the change, so a change that rolls back sends nothing, and are then sent by the message outbox. A 2xx reply counts as delivered; a 5xx, 408 or 429 reply, a timeout or a failed connection is retried with growing intervals; any other reply, a redirect included, is not retried. `POST .../web-hooks/WEBHOOK_ID/test` queues a `webhook.test` request, to check a receiver before relying on it. `GET .../web-hooks/WEBHOOK_ID/deliveries` lists a webhook's requests, newest first, each `PENDING`, `DELIVERED` or `STICKY` (no longer tried), with its attempts, the last reply's HTTP status and the last error; `POST .../deliveries/EVENT_ID/retry` sends a STICKY one again with the full number of attempts. Deleting a webhook drops its requests not yet delivered. +| +|**Signing.** When this instance has a stable server key (`jwt.use.ssl=true`), each request carries `OBP-Signature`, a JSON Web Signature over the exact request body made with the key published at `GET /obp/v3.1.0/certs`. Otherwise requests go out unsigned, with `OBP-Signature-Status: not_configured`. A webhook's `signature_status` says which; on this instance it is `${code.messageoutbox.WebhookOutbox.signatureStatus}`. +| +|**Where a webhook may send.** Only to an absolute `https` URL, never to this OBP instance, and never to a loopback, private, link-local or cloud metadata address, so that a webhook cannot be used to reach services inside OBP's own network. An entity has at most one webhook per URL. +| +|**Deactivation.** `is_active` false stops a webhook. `deactivated_reason` says why: `TURNED_OFF` when someone set it, `CIRCUIT_BREAKER` when it had ${code.dynamicentitywebhook.DynamicEntityWebhookFiring.circuitBreakerPerMinute} requests queued within a minute (on this instance), the backstop against a receiver whose reply fires the webhook again, `ENTITY_FIELD_REMOVED` or `ENTITY_FIELD_TYPE_CHANGED` when the entity's definition changed under one of its field conditions. Deleting a Dynamic Entity deletes its webhooks. +| +|**Managing.** `/obp/v7.0.0/management/banks/BANK_ID/dynamic-entities/DYNAMIC_ENTITY_ID/web-hooks` creates and lists an entity's webhooks, and `.../web-hooks/WEBHOOK_ID` reads, updates and deletes one. `/obp/v7.0.0/management/banks/BANK_ID/dynamic-entity-web-hooks` lists every webhook of a space. BANK_ID is a bank's id or `SYS` for the system space. The Roles are CanCreateDynamicEntityWebhook, CanGetDynamicEntityWebhooks, CanUpdateDynamicEntityWebhook and CanDeleteDynamicEntityWebhook, held at that BANK_ID. +| """.stripMargin) glossaryItems += GlossaryItem( diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala index c2380db454..f7dfde3ddf 100644 --- a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala +++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700.scala @@ -7775,6 +7775,9 @@ object Http4s700 { // Domain APIs: a space's Dynamic Entities and Dynamic Resource Docs published under a base path of its own. resourceDocs ++= Http4s700DomainApis.resourceDocs + // Dynamic Entity webhooks: URLs OBP sends a request to when records of a Dynamic Entity change. + resourceDocs ++= Http4s700DynamicEntityWebhooks.resourceDocs + // Groups: bring the members of a Group in line with its current Roles. resourceDocs ++= Http4s700Groups.resourceDocs diff --git a/obp-api/src/main/scala/code/api/v7_0_0/Http4s700DynamicEntityWebhooks.scala b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700DynamicEntityWebhooks.scala new file mode 100644 index 0000000000..367918d6e2 --- /dev/null +++ b/obp-api/src/main/scala/code/api/v7_0_0/Http4s700DynamicEntityWebhooks.scala @@ -0,0 +1,502 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ +package code.api.v7_0_0 + +import cats.effect.IO +import code.api.Constant.ApiPathZero +import code.api.dynamic.entity.helper.DynamicEntitySpace +import code.api.util.APIUtil.{EmptyBody, ResourceDoc} +import code.api.util.ApiRole._ +import code.api.util.ApiTag._ +import code.api.util.ErrorMessages._ +import code.api.util.http4s.Http4sRequestAttributes.{EndpointHelpers, RequestOps} +import code.api.util.{CallContext, CustomJsonFormats, Glossary} +import code.dynamicEntity.{DynamicEntityProvider, DynamicEntityT} +import code.dynamicentitywebhook.{DynamicEntityWebhookConditions, DynamicEntityWebhookEvents, DynamicEntityWebhookFieldCondition, DynamicEntityWebhookTrait, DynamicEntityWebhooks} +import code.messageoutbox.{MessageOutbox, WebhookOutbox, WebhookUrlCheck} +import code.util.Helper +import com.github.dwickern.macros.NameOf.nameOf +import com.openbankproject.commons.ExecutionContext.Implicits.global +import com.openbankproject.commons.util.ApiVersion +import net.liftweb.common.Box +import org.http4s._ +import org.http4s.dsl.io._ +import org.json4s.Formats + +import scala.collection.mutable.ArrayBuffer +import scala.concurrent.Future + +/** + * This object holds the v7.0.0 endpoints that manage Dynamic Entity webhooks: URLs OBP sends a request to + * when records of one Dynamic Entity are created, updated or deleted through the endpoints (see + * [[code.dynamicentitywebhook.DynamicEntityWebhooks]] and the Glossary item "Dynamic Entity Webhooks"). + * + * As for the v7.0.0 Dynamic Entity definitions, BANK_ID is a bank's id or SYS for the system space; the + * ResourceDocs declare allowSystemSpace() so the middleware lets SYS through and checks each Role at the + * BANK_ID in the URL. A webhook belongs to one Dynamic Entity of that space, named by DYNAMIC_ENTITY_ID; an + * entity of another space is not found, whatever its id. + * + * Declared in its own object to keep Http4s700's initialiser under the JVM's 64KB method limit. + */ +object Http4s700DynamicEntityWebhooks { + import DynamicEntityWebhooks._ + + implicit val formats: Formats = CustomJsonFormats.formats + + private val implementedInApiVersion = ApiVersion.v7_0_0 + private val prefixPath = Root / ApiPathZero.toString / implementedInApiVersion.toString + + val resourceDocs = ArrayBuffer[ResourceDoc]() + + private val MaxDescriptionLength = 2000 + + private def provider = DynamicEntityWebhooks.dynamicEntityWebhookProvider.vend + + /** The space as webhooks store it: a bank id, or SYS for the system space. */ + private def space(bankIdInUrl: String): String = + DynamicEntitySpace.bankIdOrSystem(DynamicEntitySpace.bankIdOrNoneForSystem(bankIdInUrl)) + + private def fullOrFail[T: Manifest](box: Box[T], cc: CallContext): T = + code.api.util.APIUtil.unboxFullOrFail(box, Some(cc), UnknownError, 400) + + /** The Dynamic Entity DYNAMIC_ENTITY_ID of the space BANK_ID, or a 404. */ + private def entityFound(bankIdInUrl: String, dynamicEntityId: String, cc: CallContext): Future[DynamicEntityT] = + Future(DynamicEntityProvider.connectorMethodProvider.vend.getById(DynamicEntitySpace.bankIdOrNoneForSystem(bankIdInUrl), dynamicEntityId)) + .map(box => code.api.util.APIUtil.unboxFullOrFail(box, Some(cc), + s"$DynamicEntityNotFoundByDynamicEntityId Current DYNAMIC_ENTITY_ID is $dynamicEntityId.", 404)) + + /** The webhook WEBHOOK_ID of that Dynamic Entity, or a 404. */ + private def webhookFound(spaceId: String, dynamicEntityId: String, webhookId: String, cc: CallContext): Future[DynamicEntityWebhookTrait] = + Future(provider.get(spaceId, dynamicEntityId, webhookId)) + .map(box => code.api.util.APIUtil.unboxFullOrFail(box, Some(cc), s"$DynamicEntityWebhookNotFound Current WEBHOOK_ID is $webhookId.", 404)) + + private case class Checked(url: String, events: List[String], conditions: List[DynamicEntityWebhookFieldCondition], description: String) + + /** + * The checks a webhook's settings must pass, on create and on update (`webhookId` is the one being + * updated, which may keep its own URL): the events, the field conditions against the entity's current + * fields, the description's length, the URL (see WebhookUrlCheck), and that no other webhook + * of the entity has the same URL. + */ + private def check(spaceId: String, entity: DynamicEntityT, url: String, events: List[String], + conditions: List[DynamicEntityWebhookFieldCondition], description: String, + webhookId: Option[String], cc: CallContext): Future[Checked] = { + val trimmedUrl = Option(url).map(_.trim).getOrElse("") + val eventsProblem = DynamicEntityWebhookConditions.eventsProblem(events) + lazy val conditionsProblem = DynamicEntityWebhookConditions.conditionsProblem(conditions, events, entity) + lazy val urlProblem = WebhookUrlCheck.problem(trimmedUrl) + lazy val sameUrl = provider.getByUrl(spaceId, entity.dynamicEntityId.getOrElse(""), trimmedUrl).toOption + .filterNot(other => webhookId.contains(other.webhookId)) + for { + _ <- Helper.booleanToFuture(s"$InvalidDynamicEntityWebhookEvents${eventsProblem.getOrElse("")}", 400, Some(cc)) { + eventsProblem.isEmpty + } + _ <- Helper.booleanToFuture(s"$InvalidDynamicEntityWebhookFieldCondition${conditionsProblem.getOrElse("")}", 400, Some(cc)) { + conditionsProblem.isEmpty + } + _ <- Helper.booleanToFuture(InvalidDynamicEntityWebhookDescription, 400, Some(cc)) { + description.length <= MaxDescriptionLength + } + _ <- Helper.booleanToFuture(s"$InvalidDynamicEntityWebhookUrl${urlProblem.getOrElse("")}", 400, Some(cc)) { + urlProblem.isEmpty + } + _ <- Helper.booleanToFuture(s"$DynamicEntityWebhookUrlAlreadyExists Current WEBHOOK_ID with this url is ${sameUrl.map(_.webhookId).getOrElse("")}.", 409, Some(cc)) { + sameUrl.isEmpty + } + } yield Checked(trimmedUrl, events, conditions, description) + } + + private val webhooksDescription = + s"""A Dynamic Entity webhook makes OBP send a POST request to a URL when records of one Dynamic Entity are + |created, updated or deleted through the endpoints: the Dynamic Entity endpoints, Domain APIs, Dynamic + |Endpoints and Dynamic Resource Docs. Changes to the entity's definition, backups, restores and writes + |that do not go through the endpoints send nothing. + | + |The request carries identifiers only (the event, the bank, the entity name and the record id), never the + |record's values: a receiver that needs the record fetches it with its own credentials, so the normal access + |checks apply when it reads. Delivery is at least once and in no particular order; each event carries an + |`event_id` to drop duplicates by. + | + |BANK_ID is the space: a bank's id, or `SYS` for the system space. The Role is checked at that BANK_ID. + | + |`events` names any of `CREATED`, `UPDATED` and `DELETED`. `field_conditions` narrows `UPDATED`: each + |condition names a field of the entity and, optionally, the value it must change `from` and the value it + |must change `to`, and an update fires the webhook when at least one condition matches. `{"field": "amount"}` + |matches any change to `amount`. Conditions do not apply to `CREATED` or `DELETED`. + | + |`url` must be an absolute `https` URL. It may not point at this OBP instance, nor at a loopback, private, + |link-local or cloud metadata address. An entity has at most one webhook per URL. + | + |Each request carries the headers `OBP-Webhook-Id`, `OBP-Event-Id`, `OBP-Delivery-Attempt` (1 on the + |first attempt) and `OBP-Signature-Status`. A 2xx reply counts as delivered. A 5xx, 408 or 429 reply, a + |timeout or a failed connection is retried with growing intervals, up to + |${WebhookOutbox.maxAttemptsInEffect} attempts on this instance; any other reply, a redirect included, + |is not retried. + | + |`signature_status` says whether deliveries from this instance are signed: on this instance it is + |`${WebhookOutbox.signatureStatus}`. A signed delivery carries `OBP-Signature`, a JSON Web Signature over + |the exact request body with a detached payload (RFC 7515, Appendix F), made with the key published at + |`GET /obp/v3.1.0/certs` and named by its `kid`. An unsigned one has `OBP-Signature-Status: not_configured`. + | + |`deactivated_reason` is present only while the webhook is not active: `TURNED_OFF` when someone set + |`is_active` false, `CIRCUIT_BREAKER` when it had ${code.dynamicentitywebhook.DynamicEntityWebhookFiring.circuitBreakerPerMinute} + |requests queued within a minute, and `ENTITY_FIELD_REMOVED` or + |`ENTITY_FIELD_TYPE_CHANGED` when the entity's definition changed under one of its field conditions. + | + |For more information see ${Glossary.getGlossaryItemLink("Dynamic Entity Webhooks")}""".stripMargin + + private val errorsOnWrite = List($BankNotFound, $AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidJsonFormat, + DynamicEntityNotFoundByDynamicEntityId, InvalidDynamicEntityWebhookEvents, InvalidDynamicEntityWebhookFieldCondition, + InvalidDynamicEntityWebhookDescription, InvalidDynamicEntityWebhookUrl, DynamicEntityWebhookUrlAlreadyExists, UnknownError) + + // Route: POST /obp/v7.0.0/management/banks/BANK_ID/dynamic-entities/DYNAMIC_ENTITY_ID/web-hooks (201) + lazy val createDynamicEntityWebhook: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ POST -> `prefixPath` / "management" / "banks" / bankIdInUrl / "dynamic-entities" / dynamicEntityId / "web-hooks" => + EndpointHelpers.withUserAndBodyCreated[PostDynamicEntityWebhookJsonV700, DynamicEntityWebhookJsonV700](req) { (user, body, cc) => + val spaceId = space(bankIdInUrl) + for { + entity <- entityFound(bankIdInUrl, dynamicEntityId, cc) + checked <- check(spaceId, entity, body.url, Option(body.events).getOrElse(Nil), + body.field_conditions.getOrElse(Nil).map(JSONFactory700DynamicEntityWebhooks.toCondition), + body.description.map(_.trim).getOrElse(""), None, cc) + created <- Future(provider.create(spaceId, dynamicEntityId, entity.entityName, checked.url, checked.events, + checked.conditions, body.is_active.getOrElse(true), checked.description, user.userId)) + .map(fullOrFail(_, cc)) + } yield JSONFactory700DynamicEntityWebhooks.createWebhookJson(created) + } + } + + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(createDynamicEntityWebhook), + "POST", + "/management/banks/BANK_ID/dynamic-entities/DYNAMIC_ENTITY_ID/web-hooks", + "Create Dynamic Entity Webhook", + s"""Register a URL that OBP sends a request to when records of a Dynamic Entity change. + | + |Only `url` and `events` are required. `field_conditions` defaults to none, `is_active` to true and + |`description` to empty. + | + |$webhooksDescription""".stripMargin, + JSONFactory700DynamicEntityWebhooks.postDynamicEntityWebhookJsonV700Example, + JSONFactory700DynamicEntityWebhooks.dynamicEntityWebhookJsonV700Example, + errorsOnWrite, + apiTagManageDynamicEntity :: apiTagWebhook :: apiTagApi :: Nil, + Some(canCreateDynamicEntityWebhook :: Nil), + http4sPartialFunction = Some(createDynamicEntityWebhook) + ).allowSystemSpace() + + // Route: GET /obp/v7.0.0/management/banks/BANK_ID/dynamic-entities/DYNAMIC_ENTITY_ID/web-hooks + lazy val getDynamicEntityWebhooks: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ GET -> `prefixPath` / "management" / "banks" / bankIdInUrl / "dynamic-entities" / dynamicEntityId / "web-hooks" => + EndpointHelpers.withUser(req) { (_, cc) => + for { + _ <- entityFound(bankIdInUrl, dynamicEntityId, cc) + webhooks <- Future(provider.getByDynamicEntity(space(bankIdInUrl), dynamicEntityId)).map(fullOrFail(_, cc)) + } yield DynamicEntityWebhooksJsonV700(webhooks.map(JSONFactory700DynamicEntityWebhooks.createWebhookJson)) + } + } + + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(getDynamicEntityWebhooks), + "GET", + "/management/banks/BANK_ID/dynamic-entities/DYNAMIC_ENTITY_ID/web-hooks", + "Get Dynamic Entity Webhooks", + s"""The webhooks of one Dynamic Entity, oldest first. + | + |$webhooksDescription""".stripMargin, + EmptyBody, + JSONFactory700DynamicEntityWebhooks.dynamicEntityWebhooksJsonV700Example, + List($BankNotFound, $AuthenticatedUserIsRequired, UserHasMissingRoles, DynamicEntityNotFoundByDynamicEntityId, UnknownError), + apiTagManageDynamicEntity :: apiTagWebhook :: apiTagApi :: Nil, + Some(canGetDynamicEntityWebhooks :: Nil), + http4sPartialFunction = Some(getDynamicEntityWebhooks) + ).allowSystemSpace() + + // Route: GET /obp/v7.0.0/management/banks/BANK_ID/dynamic-entity-web-hooks + lazy val getDynamicEntityWebhooksAtBank: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ GET -> `prefixPath` / "management" / "banks" / bankIdInUrl / "dynamic-entity-web-hooks" => + EndpointHelpers.withUser(req) { (_, cc) => + Future(provider.getByBank(space(bankIdInUrl))).map(box => + DynamicEntityWebhooksJsonV700(fullOrFail(box, cc).map(JSONFactory700DynamicEntityWebhooks.createWebhookJson))) + } + } + + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(getDynamicEntityWebhooksAtBank), + "GET", + "/management/banks/BANK_ID/dynamic-entity-web-hooks", + "Get Dynamic Entity Webhooks at Bank", + s"""The webhooks of every Dynamic Entity of a space, ordered by entity name. + | + |$webhooksDescription""".stripMargin, + EmptyBody, + JSONFactory700DynamicEntityWebhooks.dynamicEntityWebhooksJsonV700Example, + List($BankNotFound, $AuthenticatedUserIsRequired, UserHasMissingRoles, UnknownError), + apiTagManageDynamicEntity :: apiTagWebhook :: apiTagApi :: Nil, + Some(canGetDynamicEntityWebhooks :: Nil), + http4sPartialFunction = Some(getDynamicEntityWebhooksAtBank) + ).allowSystemSpace() + + // Route: GET /obp/v7.0.0/management/banks/BANK_ID/dynamic-entities/DYNAMIC_ENTITY_ID/web-hooks/WEBHOOK_ID + lazy val getDynamicEntityWebhook: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ GET -> `prefixPath` / "management" / "banks" / bankIdInUrl / "dynamic-entities" / dynamicEntityId / "web-hooks" / webhookId => + EndpointHelpers.withUser(req) { (_, cc) => + for { + _ <- entityFound(bankIdInUrl, dynamicEntityId, cc) + webhook <- webhookFound(space(bankIdInUrl), dynamicEntityId, webhookId, cc) + } yield JSONFactory700DynamicEntityWebhooks.createWebhookJson(webhook) + } + } + + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(getDynamicEntityWebhook), + "GET", + "/management/banks/BANK_ID/dynamic-entities/DYNAMIC_ENTITY_ID/web-hooks/WEBHOOK_ID", + "Get Dynamic Entity Webhook", + s"""One webhook of a Dynamic Entity. + | + |$webhooksDescription""".stripMargin, + EmptyBody, + JSONFactory700DynamicEntityWebhooks.dynamicEntityWebhookJsonV700Example, + List($BankNotFound, $AuthenticatedUserIsRequired, UserHasMissingRoles, DynamicEntityNotFoundByDynamicEntityId, + DynamicEntityWebhookNotFound, UnknownError), + apiTagManageDynamicEntity :: apiTagWebhook :: apiTagApi :: Nil, + Some(canGetDynamicEntityWebhooks :: Nil), + http4sPartialFunction = Some(getDynamicEntityWebhook) + ).allowSystemSpace() + + // Route: PUT /obp/v7.0.0/management/banks/BANK_ID/dynamic-entities/DYNAMIC_ENTITY_ID/web-hooks/WEBHOOK_ID + lazy val updateDynamicEntityWebhook: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ PUT -> `prefixPath` / "management" / "banks" / bankIdInUrl / "dynamic-entities" / dynamicEntityId / "web-hooks" / webhookId => + EndpointHelpers.withUserAndBody[PutDynamicEntityWebhookJsonV700, DynamicEntityWebhookJsonV700](req) { (_, body, cc) => + val spaceId = space(bankIdInUrl) + for { + entity <- entityFound(bankIdInUrl, dynamicEntityId, cc) + existing <- webhookFound(spaceId, dynamicEntityId, webhookId, cc) + // The conditions are checked against the entity on every update, kept or not, so a webhook the + // entity's definition deactivated cannot be turned back on until its conditions fit again. + checked <- check(spaceId, entity, body.url.getOrElse(existing.url), body.events.getOrElse(existing.events), + body.field_conditions.map(_.map(JSONFactory700DynamicEntityWebhooks.toCondition)).getOrElse(existing.fieldConditions), + body.description.map(_.trim).getOrElse(existing.description), Some(webhookId), cc) + isActive = body.is_active.getOrElse(existing.isActive) + // Active: no reason. Turned off now: TURNED_OFF. Already inactive and left so: the reason it had. + deactivatedReason = if (isActive) None + else if (existing.isActive) Some(DeactivatedTurnedOff) + else existing.deactivatedReason.orElse(Some(DeactivatedTurnedOff)) + updated <- Future(provider.update(spaceId, dynamicEntityId, webhookId, checked.url, checked.events, + checked.conditions, isActive, deactivatedReason, checked.description)) + .map(fullOrFail(_, cc)) + } yield JSONFactory700DynamicEntityWebhooks.createWebhookJson(updated) + } + } + + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(updateDynamicEntityWebhook), + "PUT", + "/management/banks/BANK_ID/dynamic-entities/DYNAMIC_ENTITY_ID/web-hooks/WEBHOOK_ID", + "Update Dynamic Entity Webhook", + s"""Change a webhook of a Dynamic Entity. Every field is optional, and a field left out keeps its current + |value; an empty `field_conditions` list removes the conditions. The same checks as on create apply, and + |the field conditions are checked against the entity's current definition even when they are left out. + | + |Setting `is_active` to true turns the webhook back on and clears `deactivated_reason`; setting it to false + |turns it off with `deactivated_reason` `TURNED_OFF`. + | + |$webhooksDescription""".stripMargin, + JSONFactory700DynamicEntityWebhooks.putDynamicEntityWebhookJsonV700Example, + JSONFactory700DynamicEntityWebhooks.dynamicEntityWebhookJsonV700Example, + DynamicEntityWebhookNotFound :: errorsOnWrite, + apiTagManageDynamicEntity :: apiTagWebhook :: apiTagApi :: Nil, + Some(canUpdateDynamicEntityWebhook :: Nil), + http4sPartialFunction = Some(updateDynamicEntityWebhook) + ).allowSystemSpace() + + // Route: DELETE /obp/v7.0.0/management/banks/BANK_ID/dynamic-entities/DYNAMIC_ENTITY_ID/web-hooks/WEBHOOK_ID (204) + lazy val deleteDynamicEntityWebhook: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ DELETE -> `prefixPath` / "management" / "banks" / bankIdInUrl / "dynamic-entities" / dynamicEntityId / "web-hooks" / webhookId => + EndpointHelpers.withUserDelete(req) { (_, cc) => + val spaceId = space(bankIdInUrl) + for { + _ <- entityFound(bankIdInUrl, dynamicEntityId, cc) + _ <- webhookFound(spaceId, dynamicEntityId, webhookId, cc) + deleted <- Future(provider.delete(spaceId, dynamicEntityId, webhookId)).map(fullOrFail(_, cc)) + } yield deleted + } + } + + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(deleteDynamicEntityWebhook), + "DELETE", + "/management/banks/BANK_ID/dynamic-entities/DYNAMIC_ENTITY_ID/web-hooks/WEBHOOK_ID", + "Delete Dynamic Entity Webhook", + s"""Delete a webhook of a Dynamic Entity. No further events are queued for it. + | + |$webhooksDescription""".stripMargin, + EmptyBody, + EmptyBody, + List($BankNotFound, $AuthenticatedUserIsRequired, UserHasMissingRoles, DynamicEntityNotFoundByDynamicEntityId, + DynamicEntityWebhookNotFound, UnknownError), + apiTagManageDynamicEntity :: apiTagWebhook :: apiTagApi :: Nil, + Some(canDeleteDynamicEntityWebhook :: Nil), + http4sPartialFunction = Some(deleteDynamicEntityWebhook) + ).allowSystemSpace() + + // Route: POST /obp/v7.0.0/management/banks/BANK_ID/dynamic-entities/DYNAMIC_ENTITY_ID/web-hooks/WEBHOOK_ID/test (201) + lazy val testDynamicEntityWebhook: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ POST -> `prefixPath` / "management" / "banks" / bankIdInUrl / "dynamic-entities" / dynamicEntityId / "web-hooks" / webhookId / "test" => + EndpointHelpers.executeFutureCreated(req) { + implicit val cc: CallContext = req.callContext + for { + _ <- entityFound(bankIdInUrl, dynamicEntityId, cc) + webhook <- webhookFound(space(bankIdInUrl), dynamicEntityId, webhookId, cc) + queued <- Future(DynamicEntityWebhookEvents.enqueueTest(webhook)) + } yield JSONFactory700DynamicEntityWebhooks.createDeliveryJson(queued) + } + } + + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(testDynamicEntityWebhook), + "POST", + "/management/banks/BANK_ID/dynamic-entities/DYNAMIC_ENTITY_ID/web-hooks/WEBHOOK_ID/test", + "Test Dynamic Entity Webhook", + s"""Queue a test request for a webhook, so its owner can check that the receiver gets OBP's requests, + |and can verify their signature, before relying on it. The request is sent like any other: to the + |webhook's url, signed the same way, retried the same way. Its body has `event_name` `webhook.test` and + |no `record_id`. A webhook that is not active can be tested too. + | + |The response is the queued delivery. It is sent within moments; follow it with + |`GET .../web-hooks/WEBHOOK_ID/deliveries`. + | + |$webhooksDescription""".stripMargin, + EmptyBody, + JSONFactory700DynamicEntityWebhooks.dynamicEntityWebhookDeliveryJsonV700Example, + List($BankNotFound, $AuthenticatedUserIsRequired, UserHasMissingRoles, DynamicEntityNotFoundByDynamicEntityId, + DynamicEntityWebhookNotFound, UnknownError), + apiTagManageDynamicEntity :: apiTagWebhook :: apiTagApi :: Nil, + Some(canUpdateDynamicEntityWebhook :: Nil), + http4sPartialFunction = Some(testDynamicEntityWebhook) + ).allowSystemSpace() + + private val deliveryStatuses = List(MessageOutbox.STATUS_PENDING, MessageOutbox.STATUS_DELIVERED, MessageOutbox.STATUS_STICKY) + + // Route: GET /obp/v7.0.0/management/banks/BANK_ID/dynamic-entities/DYNAMIC_ENTITY_ID/web-hooks/WEBHOOK_ID/deliveries + lazy val getDynamicEntityWebhookDeliveries: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ GET -> `prefixPath` / "management" / "banks" / bankIdInUrl / "dynamic-entities" / dynamicEntityId / "web-hooks" / webhookId / "deliveries" => + EndpointHelpers.withUser(req) { (_, cc) => + val query = req.uri.query.params + val limit = query.get("limit").flatMap(value => scala.util.Try(value.toInt).toOption).getOrElse(50).max(1).min(500) + val offset = query.get("offset").flatMap(value => scala.util.Try(value.toInt).toOption).getOrElse(0).max(0) + val status = query.get("status").map(_.trim.toUpperCase).filter(_.nonEmpty) + for { + _ <- Helper.booleanToFuture(s"$InvalidUrlParameters status must be one of ${deliveryStatuses.mkString(", ")}.", 400, Some(cc)) { + status.forall(deliveryStatuses.contains) + } + _ <- entityFound(bankIdInUrl, dynamicEntityId, cc) + _ <- webhookFound(space(bankIdInUrl), dynamicEntityId, webhookId, cc) + rows <- Future(WebhookOutbox.deliveries(webhookId, status, limit, offset)) + } yield DynamicEntityWebhookDeliveriesJsonV700(rows.map(JSONFactory700DynamicEntityWebhooks.createDeliveryJson)) + } + } + + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(getDynamicEntityWebhookDeliveries), + "GET", + "/management/banks/BANK_ID/dynamic-entities/DYNAMIC_ENTITY_ID/web-hooks/WEBHOOK_ID/deliveries", + "Get Dynamic Entity Webhook Deliveries", + s"""The requests queued for a webhook, newest first, and how sending each has gone, so its owner can see + |whether the receiver is getting them. + | + |Each delivery has its `event_id` and `event_name`, the `record_id` it is about (none for a `webhook.test` + |request), and its `status`: `PENDING` (not delivered yet; it is being retried), `DELIVERED` (the receiver + |replied 2xx) or `STICKY` (OBP has stopped trying; see `last_error`). `attempts` counts the attempts made, + |`last_http_status` is the status of the last reply, left out while no attempt has had one. + | + |Query parameters: `status` (one of `PENDING`, `DELIVERED`, `STICKY`), `limit` (default 50, at most 500) + |and `offset` (default 0). + | + |Delivered requests are kept, so this shows past deliveries as well as current ones. + | + |$webhooksDescription""".stripMargin, + EmptyBody, + JSONFactory700DynamicEntityWebhooks.dynamicEntityWebhookDeliveriesJsonV700Example, + List($BankNotFound, $AuthenticatedUserIsRequired, UserHasMissingRoles, InvalidUrlParameters, + DynamicEntityNotFoundByDynamicEntityId, DynamicEntityWebhookNotFound, UnknownError), + apiTagManageDynamicEntity :: apiTagWebhook :: apiTagApi :: Nil, + Some(canGetDynamicEntityWebhooks :: Nil), + http4sPartialFunction = Some(getDynamicEntityWebhookDeliveries) + ).allowSystemSpace() + + // Route: POST /obp/v7.0.0/management/banks/BANK_ID/dynamic-entities/DYNAMIC_ENTITY_ID/web-hooks/WEBHOOK_ID/deliveries/EVENT_ID/retry + lazy val retryDynamicEntityWebhookDelivery: HttpRoutes[IO] = HttpRoutes.of[IO] { + case req @ POST -> `prefixPath` / "management" / "banks" / bankIdInUrl / "dynamic-entities" / dynamicEntityId / "web-hooks" / webhookId / "deliveries" / eventId / "retry" => + EndpointHelpers.executeAndRespond(req) { implicit cc => + for { + _ <- entityFound(bankIdInUrl, dynamicEntityId, cc) + _ <- webhookFound(space(bankIdInUrl), dynamicEntityId, webhookId, cc) + row <- Future(WebhookOutbox.delivery(webhookId, eventId)).map(found => + code.api.util.APIUtil.unboxFullOrFail(Box(found), Some(cc), s"$DynamicEntityWebhookDeliveryNotFound Current EVENT_ID is $eventId.", 404)) + _ <- Helper.booleanToFuture(s"$DynamicEntityWebhookDeliveryNotSticky Current status is ${row.status}.", 400, Some(cc)) { + row.status == MessageOutbox.STATUS_STICKY + } + requeued <- Future(WebhookOutbox.requeue(row)) + } yield JSONFactory700DynamicEntityWebhooks.createDeliveryJson(requeued) + } + } + + resourceDocs += ResourceDoc( + implementedInApiVersion, + nameOf(retryDynamicEntityWebhookDelivery), + "POST", + "/management/banks/BANK_ID/dynamic-entities/DYNAMIC_ENTITY_ID/web-hooks/WEBHOOK_ID/deliveries/EVENT_ID/retry", + "Retry Dynamic Entity Webhook Delivery", + s"""Send a STICKY delivery again, once whatever stopped it has been fixed (the receiver was down for longer + |than the retries last, or refused the request). It goes back to `PENDING` with its `attempts` reset, so + |it gets the full number of attempts again, and keeps its `event_id` and its body, so a receiver that + |did get it before drops it as a duplicate. + | + |Only a STICKY delivery can be retried: a PENDING one is being retried already, and a DELIVERED one is done. + | + |The response is the delivery, now `PENDING`. + | + |$webhooksDescription""".stripMargin, + EmptyBody, + JSONFactory700DynamicEntityWebhooks.dynamicEntityWebhookDeliveryJsonV700Example, + List($BankNotFound, $AuthenticatedUserIsRequired, UserHasMissingRoles, DynamicEntityNotFoundByDynamicEntityId, + DynamicEntityWebhookNotFound, DynamicEntityWebhookDeliveryNotFound, DynamicEntityWebhookDeliveryNotSticky, UnknownError), + apiTagManageDynamicEntity :: apiTagWebhook :: apiTagApi :: Nil, + Some(canUpdateDynamicEntityWebhook :: Nil), + http4sPartialFunction = Some(retryDynamicEntityWebhookDelivery) + ).allowSystemSpace() +} diff --git a/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory700DynamicEntityWebhooks.scala b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory700DynamicEntityWebhooks.scala new file mode 100644 index 0000000000..763eafda7b --- /dev/null +++ b/obp-api/src/main/scala/code/api/v7_0_0/JSONFactory700DynamicEntityWebhooks.scala @@ -0,0 +1,206 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ +package code.api.v7_0_0 + +import java.util.Date + +import code.dynamicentitywebhook.{DynamicEntityWebhookFieldCondition, DynamicEntityWebhookTrait} +import code.messageoutbox.{MessageOutbox, WebhookOutbox} +import org.json4s.{JBool, JValue} + +/* + * The JSON of the v7.0.0 Dynamic Entity webhook endpoints. Package-level case classes, for the reason given in + * JSONFactory700Operations. + */ + +/** + * One field condition. `from` and `to` are optional: left out, any old (or new) value matches. + */ +case class DynamicEntityWebhookFieldConditionJsonV700(field: String, from: Option[JValue] = None, to: Option[JValue] = None) + +/** + * Create a webhook. `field_conditions` defaults to none, `is_active` to true and `description` to empty. + */ +case class PostDynamicEntityWebhookJsonV700( + url: String, + events: List[String], + field_conditions: Option[List[DynamicEntityWebhookFieldConditionJsonV700]] = None, + is_active: Option[Boolean] = None, + description: Option[String] = None +) + +/** + * Change a webhook. The same fields as on create, all optional: a field left out keeps its current value. + * An empty `field_conditions` list removes the conditions. + */ +case class PutDynamicEntityWebhookJsonV700( + url: Option[String] = None, + events: Option[List[String]] = None, + field_conditions: Option[List[DynamicEntityWebhookFieldConditionJsonV700]] = None, + is_active: Option[Boolean] = None, + description: Option[String] = None +) + +case class DynamicEntityWebhookJsonV700( + webhook_id: String, + bank_id: String, + dynamic_entity_id: String, + entity_name: String, + url: String, + events: List[String], + field_conditions: List[DynamicEntityWebhookFieldConditionJsonV700], + is_active: Boolean, + /** Why the webhook is not active; left out while it is. */ + deactivated_reason: Option[String], + description: String, + /** Whether deliveries on this instance are signed: signed or not_configured. */ + signature_status: String, + created_by_user_id: String, + created_at: Date, + updated_at: Date +) + +case class DynamicEntityWebhooksJsonV700(web_hooks: List[DynamicEntityWebhookJsonV700]) + +/** + * One delivery of a webhook: one event queued for it, and how sending it has gone. `status` is the + * message outbox's own: PENDING, DELIVERED or STICKY. `last_http_status` is left out while no attempt + * has had a reply, `record_id` for the test endpoint's ping, and `last_error` while there is none. + */ +case class DynamicEntityWebhookDeliveryJsonV700( + event_id: String, + event_name: String, + record_id: Option[String], + status: String, + attempts: Int, + last_http_status: Option[Int], + last_error: Option[String], + created_at: Date, + updated_at: Date +) + +case class DynamicEntityWebhookDeliveriesJsonV700(deliveries: List[DynamicEntityWebhookDeliveryJsonV700]) + +object JSONFactory700DynamicEntityWebhooks { + + def toCondition(json: DynamicEntityWebhookFieldConditionJsonV700): DynamicEntityWebhookFieldCondition = + DynamicEntityWebhookFieldCondition(field = Option(json.field).map(_.trim).orNull, from = json.from, to = json.to) + + def createConditionJson(condition: DynamicEntityWebhookFieldCondition): DynamicEntityWebhookFieldConditionJsonV700 = + DynamicEntityWebhookFieldConditionJsonV700(condition.field, condition.from, condition.to) + + def createDeliveryJson(row: MessageOutbox): DynamicEntityWebhookDeliveryJsonV700 = DynamicEntityWebhookDeliveryJsonV700( + event_id = row.subjectId, + event_name = row.operationName, + record_id = scala.util.Try(com.openbankproject.commons.util.JsonAliases.parse(row.payloadJson) \ "record_id") + .toOption.collect { case org.json4s.JString(id) => id }, + status = row.status, + attempts = row.attempts, + last_http_status = WebhookOutbox.lastReply(row).map(_.http_status), + last_error = Option(row.LastError.get).filter(_.trim.nonEmpty), + created_at = row.CreatedAt.get, + updated_at = row.UpdatedAt.get + ) + + def createWebhookJson(webhook: DynamicEntityWebhookTrait): DynamicEntityWebhookJsonV700 = DynamicEntityWebhookJsonV700( + webhook_id = webhook.webhookId, + bank_id = webhook.bankId, + dynamic_entity_id = webhook.dynamicEntityId, + entity_name = webhook.entityName, + url = webhook.url, + events = webhook.events, + field_conditions = webhook.fieldConditions.map(createConditionJson), + is_active = webhook.isActive, + deactivated_reason = webhook.deactivatedReason, + description = webhook.description, + signature_status = WebhookOutbox.signatureStatus, + created_by_user_id = webhook.createdByUserId, + created_at = webhook.createdAt, + updated_at = webhook.updatedAt + ) + + val postDynamicEntityWebhookJsonV700Example = PostDynamicEntityWebhookJsonV700( + url = "https://receiver.example.com/hooks/obp", + events = List("CREATED", "UPDATED", "DELETED"), + field_conditions = Some(List( + DynamicEntityWebhookFieldConditionJsonV700("approved", Some(JBool(false)), Some(JBool(true))), + DynamicEntityWebhookFieldConditionJsonV700("amount") + )), + is_active = Some(true), + description = Some("Tell the loan system when an application is approved or its amount changes") + ) + + val putDynamicEntityWebhookJsonV700Example = PutDynamicEntityWebhookJsonV700( + url = Some(postDynamicEntityWebhookJsonV700Example.url), + events = Some(postDynamicEntityWebhookJsonV700Example.events), + field_conditions = postDynamicEntityWebhookJsonV700Example.field_conditions, + is_active = Some(true), + description = postDynamicEntityWebhookJsonV700Example.description + ) + + val dynamicEntityWebhookJsonV700Example = DynamicEntityWebhookJsonV700( + webhook_id = "7d3c1c2e-5b7a-4f0e-9a51-2f1f6f0c9b11", + bank_id = "gh.29.uk", + dynamic_entity_id = "f6c2a8d4-1b3e-4c5f-8a7b-9d0e1f2a3b4c", + entity_name = "loan_application", + url = "https://receiver.example.com/hooks/obp", + events = List("CREATED", "UPDATED", "DELETED"), + field_conditions = postDynamicEntityWebhookJsonV700Example.field_conditions.getOrElse(Nil), + is_active = true, + deactivated_reason = None, + description = "Tell the loan system when an application is approved or its amount changes", + signature_status = WebhookOutbox.SignatureStatusSigned, + created_by_user_id = "9ca9a7e4-6d02-40e3-a129-0b2bf89de9b1", + created_at = new Date(), + updated_at = new Date() + ) + + val dynamicEntityWebhooksJsonV700Example = DynamicEntityWebhooksJsonV700(List(dynamicEntityWebhookJsonV700Example)) + + val dynamicEntityWebhookDeliveryJsonV700Example = DynamicEntityWebhookDeliveryJsonV700( + event_id = "0f8e4a71-5ff1-4006-95bb-7fd9e4adaef9", + event_name = "webhook.test", + record_id = None, + status = "PENDING", + attempts = 0, + last_http_status = None, + last_error = None, + created_at = new Date(), + updated_at = new Date() + ) + + val dynamicEntityWebhookDeliveriesJsonV700Example = DynamicEntityWebhookDeliveriesJsonV700(List( + dynamicEntityWebhookDeliveryJsonV700Example.copy( + event_name = "dynamic_entity.record.updated", + record_id = Some("a1b2c3d4-0000-4000-8000-000000000001"), + status = "STICKY", + attempts = 8, + last_http_status = Some(503), + last_error = Some("The receiver replied 503 Service Unavailable.") + ) + )) +} diff --git a/obp-api/src/main/scala/code/dynamicEntity/MapppedDynamicDataProvider.scala b/obp-api/src/main/scala/code/dynamicEntity/MapppedDynamicDataProvider.scala index f01743340f..d60cb9af16 100644 --- a/obp-api/src/main/scala/code/dynamicEntity/MapppedDynamicDataProvider.scala +++ b/obp-api/src/main/scala/code/dynamicEntity/MapppedDynamicDataProvider.scala @@ -50,6 +50,13 @@ import java.util.Date * records belonging to the current user (filtered by UserId), regardless of IsPersonalEntity value. * This provides a unified view of a user's data whether it was created via /my/ or non-/my/ endpoints. */ +/* + * Every write below also fires the entity's Dynamic Entity webhooks (DynamicEntityWebhookFiring), in the + * write's own transaction. A write that is not made by a caller fires nothing without special handling: + * a backup writes into a ZZ_BAK_ entity it has just created, which has no webhooks, and a cascade delete + * deletes the definition in the same transaction as its records, which deletes the webhooks together + * with the requests just queued for them. + */ object MappedDynamicDataProvider extends DynamicDataProvider with CustomJsonFormats{ /** @@ -182,6 +189,7 @@ object MappedDynamicDataProvider extends DynamicDataProvider with CustomJsonForm val result = d.asInstanceOf[DynamicData].delete_! // DE_indexing: remove the projection row in the same transaction (no-op unless projection enabled+ready). code.api.dynamic.entity.projection.ProjectionDualWrite.onDelete(bankId, entityName, id) + if (result) code.dynamicentitywebhook.DynamicEntityWebhookFiring.recordDeleted(bankId, entityName, id) result } } @@ -221,6 +229,7 @@ object MappedDynamicDataProvider extends DynamicDataProvider with CustomJsonForm val result = d.asInstanceOf[DynamicData].delete_! // DE_indexing: remove the projection row in the same transaction (no-op unless projection enabled+ready). code.api.dynamic.entity.projection.ProjectionDualWrite.onDelete(bankId, entityName, id) + if (result) code.dynamicentitywebhook.DynamicEntityWebhookFiring.recordDeleted(bankId, entityName, id) result } } @@ -260,6 +269,8 @@ object MappedDynamicDataProvider extends DynamicDataProvider with CustomJsonForm tryo { val dataStr = json.compactRender(requestBody) val isNew = !data.saved_? + // The stored values before this write, which webhook field conditions compare against. + val before = if (isNew) None else scala.util.Try(json.parse(data.DataJson.get).asInstanceOf[JObject]).toOption.orElse(Some(JObject(Nil))) val (writerUserId, writerOnBehalfOfUserId) = writersOf(callerUserId, if (isNew) code.users.UserReference.DynamicData_CreatedByUserId else code.users.UserReference.DynamicData_UpdatedByUserId) if (isNew) data.CreatedByUserId(writerUserId).CreatedByOnBehalfOfUserId(writerOnBehalfOfUserId) @@ -272,6 +283,9 @@ object MappedDynamicDataProvider extends DynamicDataProvider with CustomJsonForm .saveMe() // DE_indexing: keep the projection in sync in the same transaction (no-op unless projection enabled+ready). code.api.dynamic.entity.projection.ProjectionDualWrite.onSave(bankId, entityName, saved.DynamicDataId.get, requestBody) + // Dynamic Entity webhooks: queued in this same transaction, so they go out only if the write commits. + code.dynamicentitywebhook.DynamicEntityWebhookFiring.recordSaved(bankId, entityName, saved.DynamicDataId.get, before, + requestBody, Option(saved.updatedAt.get).getOrElse(new Date())) saved } } diff --git a/obp-api/src/main/scala/code/dynamicEntity/MapppedDynamicEntityProvider.scala b/obp-api/src/main/scala/code/dynamicEntity/MapppedDynamicEntityProvider.scala index ea34e82808..8cbf8a001d 100644 --- a/obp-api/src/main/scala/code/dynamicEntity/MapppedDynamicEntityProvider.scala +++ b/obp-api/src/main/scala/code/dynamicEntity/MapppedDynamicEntityProvider.scala @@ -90,6 +90,8 @@ object MappedDynamicEntityProvider extends DynamicEntityProvider with CustomJson // ยง8.4: switching useRowLevelAccess on for an entity that already has rows makes those // rows admin-only (no backfill). Warn so the operator grants access deliberately. val wasRowLevel = existsDynamicEntity.map(_.useRowLevelAccess).getOrElse(false) + // Read before the save changes the row: the webhooks compare their field conditions against it. + val fieldTypesBefore = existsDynamicEntity.map(code.dynamicentitywebhook.DynamicEntityWebhookConditions.fieldTypes) if (!wasRowLevel && dynamicEntity.useRowLevelAccess) { // The data table stores the system space as the sentinel, never as a SQL NULL, so both spaces // are one comparison. Looking for NULL here counted zero rows for every system level entity, @@ -120,6 +122,7 @@ object MappedDynamicEntityProvider extends DynamicEntityProvider with CustomJson .AuthMode(DynamicEntityAuthMode.normalise(dynamicEntity.authMode)) .saveMe() code.api.dynamic.entity.helper.DynamicEntityHelper.forgetDefinitions() + fieldTypesBefore.foreach(code.dynamicentitywebhook.DynamicEntityWebhookDefinitionChanges.afterUpdate(saved, _)) // DE_indexing: provision/refresh the projection for this definition's indexed scalar fields. // Guarded by projectionEnabled (default off); best-effort (a failure leaves the definition saved // and queries reporting pending, not a broken create). Fields passed explicitly because the new @@ -149,6 +152,13 @@ object MappedDynamicEntityProvider extends DynamicEntityProvider with CustomJson override def delete(dynamicEntity: DynamicEntityT): Box[Boolean] = Box.tryo{ + // The entity's webhooks go with it, whichever branch below deletes it. + val dynamicEntityIds = dynamicEntity match { + case v: DynamicEntity => v.dynamicEntityId.toList + case v => DynamicEntity.findAll(By(DynamicEntity.BankId, storedBankId(v.bankId)), By(DynamicEntity.EntityName, v.entityName)) + .flatMap(_.dynamicEntityId) + } + dynamicEntityIds.filter(_.nonEmpty).foreach(code.dynamicentitywebhook.DynamicEntityWebhookDefinitionChanges.beforeDelete) try dynamicEntity match { case v: DynamicEntity => DynamicEntity.delete_!(v) // Anything that is not one of our own rows is matched by name, and a name identifies an entity diff --git a/obp-api/src/main/scala/code/dynamicentitywebhook/DynamicEntityWebhook.scala b/obp-api/src/main/scala/code/dynamicentitywebhook/DynamicEntityWebhook.scala new file mode 100644 index 0000000000..daad596894 --- /dev/null +++ b/obp-api/src/main/scala/code/dynamicentitywebhook/DynamicEntityWebhook.scala @@ -0,0 +1,333 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ + +package code.dynamicentitywebhook + +import java.util.Date + +import code.api.Constant.DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID +import code.api.util.APIUtil +import code.users.{UserReference, Users} +import net.liftweb.common.{Box, Full} +import net.liftweb.mapper._ +import net.liftweb.util.Helpers.tryo +import net.liftweb.util.SimpleInjector +import org.json4s.{DefaultFormats, Formats, JValue} +import org.json4s.native.Serialization + +/** + * A Dynamic Entity webhook asks OBP to send a request to a URL when records of one Dynamic Entity + * are created, updated or deleted through the endpoints. The request carries identifiers only, never + * record values: a receiver that needs the record fetches it with its own credentials, so the normal + * access checks apply when it reads. See the Glossary item "Dynamic Entity Webhooks". + */ +object DynamicEntityWebhooks extends SimpleInjector { + val dynamicEntityWebhookProvider = new Inject(() => buildOne) {} + def buildOne: DynamicEntityWebhookProvider = DynamicEntityWebhookDbProvider + + /** The record changes a webhook can subscribe to. */ + val EventCreated = "CREATED" + val EventUpdated = "UPDATED" + val EventDeleted = "DELETED" + val allEvents: List[String] = List(EventCreated, EventUpdated, EventDeleted) + + /** Why a webhook is not active. Null (None) while it is active. */ + val DeactivatedTurnedOff = "TURNED_OFF" + val DeactivatedCircuitBreaker = "CIRCUIT_BREAKER" + val DeactivatedEntityFieldRemoved = "ENTITY_FIELD_REMOVED" + val DeactivatedEntityFieldTypeChanged = "ENTITY_FIELD_TYPE_CHANGED" +} + +/** + * This is one field condition of a webhook. An update matches it when the named field's value + * changed, and, where they are given, the old value equals `from` and the new value equals `to`. + * A webhook with conditions fires on an update when at least one of them matches. + */ +case class DynamicEntityWebhookFieldCondition(field: String, from: Option[JValue] = None, to: Option[JValue] = None) + +object DynamicEntityWebhookFieldCondition { + private implicit val formats: Formats = DefaultFormats + + def toJsonString(conditions: List[DynamicEntityWebhookFieldCondition]): String = + Serialization.write(conditions) + + def fromJsonString(json: String): List[DynamicEntityWebhookFieldCondition] = + Option(json).filter(_.trim.nonEmpty) + .flatMap(s => tryo(Serialization.read[List[DynamicEntityWebhookFieldCondition]](s)).toOption) + .getOrElse(Nil) +} + +trait DynamicEntityWebhookTrait { + def webhookId: String + /** A bank id, or SYS for the system space. Never empty. */ + def bankId: String + def dynamicEntityId: String + def entityName: String + def url: String + /** Some of CREATED, UPDATED and DELETED. */ + def events: List[String] + def fieldConditions: List[DynamicEntityWebhookFieldCondition] + def isActive: Boolean + /** Why the webhook is not active; None while it is. */ + def deactivatedReason: Option[String] + def description: String + /** The user who made the call that created the webhook, an agent's own id when an agent made it. */ + def createdByUserId: String + /** The user that call was made for; the same as createdByUserId when nobody was delegating. */ + def createdByOnBehalfOfUserId: String + def createdAt: Date + def updatedAt: Date +} + +trait DynamicEntityWebhookProvider { + def create(bankId: String, dynamicEntityId: String, entityName: String, url: String, events: List[String], + fieldConditions: List[DynamicEntityWebhookFieldCondition], isActive: Boolean, description: String, + createdByUserId: String): Box[DynamicEntityWebhookTrait] + def get(bankId: String, dynamicEntityId: String, webhookId: String): Box[DynamicEntityWebhookTrait] + /** The webhooks of one Dynamic Entity. */ + def getByDynamicEntity(bankId: String, dynamicEntityId: String): Box[List[DynamicEntityWebhookTrait]] + /** The webhooks of every Dynamic Entity of a space. */ + def getByBank(bankId: String): Box[List[DynamicEntityWebhookTrait]] + /** The webhook of a Dynamic Entity with this URL, if there is one. */ + def getByUrl(bankId: String, dynamicEntityId: String, url: String): Box[DynamicEntityWebhookTrait] + /** + * This replaces a webhook's settings. `deactivatedReason` is stored as given, so the caller decides + * it: None when the webhook is active, a reason when it is not. + */ + def update(bankId: String, dynamicEntityId: String, webhookId: String, url: String, events: List[String], + fieldConditions: List[DynamicEntityWebhookFieldCondition], isActive: Boolean, + deactivatedReason: Option[String], description: String): Box[DynamicEntityWebhookTrait] + def delete(bankId: String, dynamicEntityId: String, webhookId: String): Box[Boolean] + /** The active webhooks of the entity named `entityName` in a space: what a record write fires. */ + def getActiveByEntityName(bankId: String, entityName: String): Box[List[DynamicEntityWebhookTrait]] + /** The fields of a webhook's Dynamic Entity and their types, as its conditions are compared against. */ + def entityFieldTypes(dynamicEntityId: String): Map[String, String] = + code.dynamicEntity.DynamicEntity.find(net.liftweb.mapper.By(code.dynamicEntity.DynamicEntity.DynamicEntityId, dynamicEntityId)) + .map(DynamicEntityWebhookConditions.fieldTypes).openOr(Map.empty) + /** The webhooks of a Dynamic Entity, by its id alone, which is unique across spaces. */ + def getByDynamicEntityId(dynamicEntityId: String): Box[List[DynamicEntityWebhookTrait]] + /** This turns a webhook off for `reason`, without changing anything else about it. */ + def deactivate(webhookId: String, reason: String): Box[DynamicEntityWebhookTrait] + /** This records a Dynamic Entity's new name on its webhooks, which look records up by name. */ + def renameEntity(dynamicEntityId: String, entityName: String): Box[Int] + /** This deletes every webhook of a Dynamic Entity, with their undelivered requests. */ + def deleteByDynamicEntityId(dynamicEntityId: String): Box[Int] +} + +object DynamicEntityWebhookDbProvider extends DynamicEntityWebhookProvider { + import DynamicEntityWebhooks._ + + /** + * This works out the two user ids to record as the webhook's creator: the caller itself, which is + * an agent's own id when an agent made the call, and the user the call was made for. Both are kept + * because a webhook sends data out of OBP and outlives the Consent that created it, so "which agent + * set this up?" and "on whose authority?" must both be answerable from the row. + */ + private def creatorsOf(createdByUserId: String): (String, String) = + if (createdByUserId == null || createdByUserId.isEmpty) (createdByUserId, createdByUserId) + else Users.users.vend.attributionOf(createdByUserId, UserReference.DynamicEntityWebhook_CreatedByUserId) match { + case Full(attribution) => (attribution.userId, attribution.onBehalfOfUserId) + case _ => (createdByUserId, createdByUserId) + } + + private def find(bankId: String, dynamicEntityId: String, webhookId: String): Box[DynamicEntityWebhook] = + DynamicEntityWebhook.find( + By(DynamicEntityWebhook.BankId, bankId), + By(DynamicEntityWebhook.DynamicEntityId, dynamicEntityId), + By(DynamicEntityWebhook.WebhookId, webhookId)) + + override def create(bankId: String, dynamicEntityId: String, entityName: String, url: String, events: List[String], + fieldConditions: List[DynamicEntityWebhookFieldCondition], isActive: Boolean, description: String, + createdByUserId: String): Box[DynamicEntityWebhookTrait] = { + val (creator, creatorOnBehalfOf) = creatorsOf(createdByUserId) + tryo { + DynamicEntityWebhook.create + .WebhookId(APIUtil.generateUUID()) + .BankId(bankId) + .DynamicEntityId(dynamicEntityId) + .EntityName(entityName) + .Url(url) + .Events(events.mkString(",")) + .FieldConditions(DynamicEntityWebhookFieldCondition.toJsonString(fieldConditions)) + .IsActive(isActive) + .DeactivatedReason(if (isActive) null else DeactivatedTurnedOff) + .Description(description) + .CreatedByUserId(creator) + .CreatedByOnBehalfOfUserId(creatorOnBehalfOf) + .saveMe() + } + } + + override def get(bankId: String, dynamicEntityId: String, webhookId: String): Box[DynamicEntityWebhookTrait] = + find(bankId, dynamicEntityId, webhookId) + + override def getByDynamicEntity(bankId: String, dynamicEntityId: String): Box[List[DynamicEntityWebhookTrait]] = + tryo(DynamicEntityWebhook.findAll( + By(DynamicEntityWebhook.BankId, bankId), + By(DynamicEntityWebhook.DynamicEntityId, dynamicEntityId), + OrderBy(DynamicEntityWebhook.CreatedAt, Ascending))) + + override def getByBank(bankId: String): Box[List[DynamicEntityWebhookTrait]] = + tryo(DynamicEntityWebhook.findAll( + By(DynamicEntityWebhook.BankId, bankId), + OrderBy(DynamicEntityWebhook.EntityName, Ascending))) + + override def getByUrl(bankId: String, dynamicEntityId: String, url: String): Box[DynamicEntityWebhookTrait] = + DynamicEntityWebhook.find( + By(DynamicEntityWebhook.BankId, bankId), + By(DynamicEntityWebhook.DynamicEntityId, dynamicEntityId), + By(DynamicEntityWebhook.Url, url)) + + override def update(bankId: String, dynamicEntityId: String, webhookId: String, url: String, events: List[String], + fieldConditions: List[DynamicEntityWebhookFieldCondition], isActive: Boolean, + deactivatedReason: Option[String], description: String): Box[DynamicEntityWebhookTrait] = + find(bankId, dynamicEntityId, webhookId).flatMap { row => + tryo { + row.Url(url) + .Events(events.mkString(",")) + .FieldConditions(DynamicEntityWebhookFieldCondition.toJsonString(fieldConditions)) + .IsActive(isActive) + .DeactivatedReason(deactivatedReason.orNull) + .Description(description) + .UpdatedAt(new Date()) + .saveMe() + } + } + + // Requests still waiting to be delivered go with the webhook: nobody asked for them any more. + override def delete(bankId: String, dynamicEntityId: String, webhookId: String): Box[Boolean] = + find(bankId, dynamicEntityId, webhookId).flatMap { row => + tryo { + code.messageoutbox.WebhookOutbox.discardPending(webhookId) + row.delete_! + } + } + override def getActiveByEntityName(bankId: String, entityName: String): Box[List[DynamicEntityWebhookTrait]] = + tryo(DynamicEntityWebhook.findAll( + By(DynamicEntityWebhook.BankId, bankId), + By(DynamicEntityWebhook.EntityName, entityName), + By(DynamicEntityWebhook.IsActive, true))) + + override def getByDynamicEntityId(dynamicEntityId: String): Box[List[DynamicEntityWebhookTrait]] = + tryo(DynamicEntityWebhook.findAll(By(DynamicEntityWebhook.DynamicEntityId, dynamicEntityId))) + + override def deactivate(webhookId: String, reason: String): Box[DynamicEntityWebhookTrait] = + DynamicEntityWebhook.find(By(DynamicEntityWebhook.WebhookId, webhookId)).flatMap { row => + tryo(row.IsActive(false).DeactivatedReason(reason).UpdatedAt(new Date()).saveMe()) + } + + override def renameEntity(dynamicEntityId: String, entityName: String): Box[Int] = + tryo(DynamicEntityWebhook.findAll(By(DynamicEntityWebhook.DynamicEntityId, dynamicEntityId)) + .filter(_.entityName != entityName) + .map(_.EntityName(entityName).UpdatedAt(new Date()).saveMe()) + .size) + + override def deleteByDynamicEntityId(dynamicEntityId: String): Box[Int] = + tryo(DynamicEntityWebhook.findAll(By(DynamicEntityWebhook.DynamicEntityId, dynamicEntityId)).count { row => + code.messageoutbox.WebhookOutbox.discardPending(row.webhookId) + row.delete_! + }) +} + +class DynamicEntityWebhook extends DynamicEntityWebhookTrait with LongKeyedMapper[DynamicEntityWebhook] with IdPK { + def getSingleton = DynamicEntityWebhook + + object WebhookId extends MappedString(this, 36) { + override def dbColumnName = "webhook_id" + } + // SYS for the system space, never NULL: see DynamicEntitySpace. + object BankId extends MappedString(this, 255) { + override def dbColumnName = "bank_id" + override def defaultValue = DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID + } + object DynamicEntityId extends MappedString(this, 255) { + override def dbColumnName = "dynamic_entity_id" + } + // Kept beside the id because a record write knows the entity by name, and looks webhooks up by it. + object EntityName extends MappedString(this, 255) { + override def dbColumnName = "entity_name" + } + object Url extends MappedString(this, 1024) { + override def dbColumnName = "url" + } + // A comma separated list of CREATED, UPDATED and DELETED. + object Events extends MappedString(this, 64) { + override def dbColumnName = "events" + } + // A JSON array of DynamicEntityWebhookFieldCondition. + object FieldConditions extends MappedText(this) { + override def dbColumnName = "field_conditions" + } + object IsActive extends MappedBoolean(this) { + override def dbColumnName = "is_active" + override def defaultValue = true + } + object DeactivatedReason extends MappedString(this, 64) { + override def dbColumnName = "deactivated_reason" + } + object Description extends MappedString(this, 2000) { + override def dbColumnName = "description" + } + object CreatedByUserId extends MappedString(this, 255) { + override def dbColumnName = "created_by_user_id" + } + object CreatedByOnBehalfOfUserId extends MappedString(this, 255) { + override def dbColumnName = "created_by_on_behalf_of_user_id" + } + object CreatedAt extends MappedDateTime(this) { + override def dbColumnName = "created_at" + override def defaultValue = new Date() + } + object UpdatedAt extends MappedDateTime(this) { + override def dbColumnName = "updated_at" + override def defaultValue = new Date() + } + + override def webhookId: String = WebhookId.get + override def bankId: String = BankId.get + override def dynamicEntityId: String = DynamicEntityId.get + override def entityName: String = EntityName.get + override def url: String = Url.get + override def events: List[String] = Option(Events.get).toList.flatMap(_.split(",")).map(_.trim).filter(_.nonEmpty) + override def fieldConditions: List[DynamicEntityWebhookFieldCondition] = DynamicEntityWebhookFieldCondition.fromJsonString(FieldConditions.get) + override def isActive: Boolean = IsActive.get + override def deactivatedReason: Option[String] = Option(DeactivatedReason.get).filter(_.nonEmpty) + override def description: String = Option(Description.get).getOrElse("") + override def createdByUserId: String = CreatedByUserId.get + override def createdByOnBehalfOfUserId: String = CreatedByOnBehalfOfUserId.get + override def createdAt: Date = CreatedAt.get + override def updatedAt: Date = UpdatedAt.get +} + +object DynamicEntityWebhook extends DynamicEntityWebhook with LongKeyedMetaMapper[DynamicEntityWebhook] { + override def dbTableName = "dynamic_entity_webhook" + override def dbIndexes = + UniqueIndex(WebhookId) :: + UniqueIndex(DynamicEntityId, Url) :: + Index(BankId, EntityName) :: + super.dbIndexes +} diff --git a/obp-api/src/main/scala/code/dynamicentitywebhook/DynamicEntityWebhookConditions.scala b/obp-api/src/main/scala/code/dynamicentitywebhook/DynamicEntityWebhookConditions.scala new file mode 100644 index 0000000000..b732c03246 --- /dev/null +++ b/obp-api/src/main/scala/code/dynamicentitywebhook/DynamicEntityWebhookConditions.scala @@ -0,0 +1,107 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ + +package code.dynamicentitywebhook + +import code.dynamicEntity.DynamicEntityT +import com.openbankproject.commons.model.enums.DynamicEntityFieldType +import org.json4s._ + +import scala.util.Try + +/** + * This object checks a webhook's events and field conditions against the Dynamic Entity they are for. + * + * A field condition names a field of the entity and, optionally, the value it must change from and + * the value it must change to. Those values are only useful when they could actually be stored in the + * field, so they are checked against the field's declared type, the same way a record's values are + * checked when it is written. + */ +object DynamicEntityWebhookConditions { + import DynamicEntityWebhooks._ + + /** The entity's fields and their declared type names, e.g. "approved" -> "boolean". */ + def fieldTypes(entity: DynamicEntityT): Map[String, String] = + Try(com.openbankproject.commons.util.JsonAliases.parse(entity.metadataJson) \ entity.entityName \ "properties").toOption match { + case Some(JObject(fields)) => + fields.collect { case JField(name, definition: JObject) => + name -> ((definition \ "type") match { case JString(t) => t; case _ => "" }) + }.toMap + case _ => Map.empty + } + + /** Whether `value` could be stored in a field of this declared type. */ + private def fitsType(typeName: String, value: JValue): Boolean = + value match { + case JNothing | JNull => false + case _ => DynamicEntityFieldType.withNameOption(typeName) match { + case Some(fieldType) => fieldType.isJValueValid(value) + // A reference to another entity, or a type this check does not know: its values are strings. + case None => value.isInstanceOf[JString] + } + } + + /** Why `events` is not acceptable, or None when it is. */ + def eventsProblem(events: List[String]): Option[String] = + if (events.isEmpty) Some("events is empty.") + else if (events.exists(e => !allEvents.contains(e))) Some(s"Unknown events: ${events.filterNot(allEvents.contains).mkString(", ")}.") + else if (events.distinct.size != events.size) Some("events names an event more than once.") + else None + + /** Why these field conditions are not acceptable for `entity` and `events`, or None when they are. */ + def conditionsProblem(conditions: List[DynamicEntityWebhookFieldCondition], events: List[String], entity: DynamicEntityT): Option[String] = { + val types = fieldTypes(entity) + val fields = conditions.map(_.field) + if (conditions.isEmpty) None + else if (!events.contains(EventUpdated)) Some("field_conditions is given but events does not include UPDATED.") + else if (fields.exists(f => f == null || f.trim.isEmpty)) Some("A condition has no field.") + else if (fields.distinct.size != fields.size) Some("A field is named by more than one condition.") + else conditions.collectFirst { + case c if !types.contains(c.field) => + s"${entity.entityName} has no field ${c.field}." + case c if c.from.exists(v => !fitsType(types(c.field), v)) => + s"The from value of ${c.field} does not fit its type ${types(c.field)}." + case c if c.to.exists(v => !fitsType(types(c.field), v)) => + s"The to value of ${c.field} does not fit its type ${types(c.field)}." + case c if c.from.isDefined && c.from == c.to => + s"The from and to values of ${c.field} are the same, so no change could match." + } + } + + /** + * Why a webhook's existing conditions no longer fit its entity, as a deactivation reason, or None + * when they still do. Used when an entity definition changes. + */ + def deactivationReasonFor(conditions: List[DynamicEntityWebhookFieldCondition], entity: DynamicEntityT, + typesBefore: Map[String, String]): Option[String] = { + val typesAfter = fieldTypes(entity) + val named = conditions.map(_.field) + if (named.exists(f => !typesAfter.contains(f))) Some(DeactivatedEntityFieldRemoved) + else if (named.exists(f => typesBefore.get(f).exists(_ != typesAfter(f)))) Some(DeactivatedEntityFieldTypeChanged) + else None + } +} diff --git a/obp-api/src/main/scala/code/dynamicentitywebhook/DynamicEntityWebhookDefinitionChanges.scala b/obp-api/src/main/scala/code/dynamicentitywebhook/DynamicEntityWebhookDefinitionChanges.scala new file mode 100644 index 0000000000..03ae8ac4eb --- /dev/null +++ b/obp-api/src/main/scala/code/dynamicentitywebhook/DynamicEntityWebhookDefinitionChanges.scala @@ -0,0 +1,71 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ + +package code.dynamicentitywebhook + +import code.dynamicEntity.DynamicEntityT +import code.util.Helper.MdcLoggable + +/** + * This object keeps a Dynamic Entity's webhooks in step with its definition. The definition provider + * calls it after it saves or deletes a definition, in the same transaction, so every way of changing a + * definition is covered, whichever endpoint or version made the change. + * + * A change to the definition always goes ahead. When it removes a field that an active webhook's field + * conditions name, or changes such a field's type, the webhook is turned off with `deactivated_reason` + * ENTITY_FIELD_REMOVED or ENTITY_FIELD_TYPE_CHANGED: its conditions no longer mean what its owner wrote, + * so it stops firing until the owner fixes them and turns it back on. A webhook that is already off keeps + * the reason it has. A renamed entity's name is updated on its webhooks, which find records by it, and a + * deleted entity's webhooks are deleted with their undelivered requests. + */ +object DynamicEntityWebhookDefinitionChanges extends MdcLoggable { + + private def provider = DynamicEntityWebhooks.dynamicEntityWebhookProvider.vend + + /** + * Called after an existing definition is saved. `fieldTypesBefore` are its fields and their types as + * they were before the save (see DynamicEntityWebhookConditions.fieldTypes). + */ + def afterUpdate(saved: DynamicEntityT, fieldTypesBefore: Map[String, String]): Unit = + saved.dynamicEntityId.filter(_.nonEmpty).foreach { dynamicEntityId => + provider.renameEntity(dynamicEntityId, saved.entityName).openOrThrowException("rename the entity on its webhooks") + provider.getByDynamicEntityId(dynamicEntityId).openOrThrowException("the entity's webhooks") + .filter(webhook => webhook.isActive && webhook.fieldConditions.nonEmpty) + .foreach { webhook => + DynamicEntityWebhookConditions.deactivationReasonFor(webhook.fieldConditions, saved, fieldTypesBefore).foreach { reason => + provider.deactivate(webhook.webhookId, reason).openOrThrowException("deactivate the webhook") + logger.info(s"afterUpdate says: webhook ${webhook.webhookId} of ${saved.entityName} turned off, $reason") + } + } + } + + /** Called before a definition is deleted. */ + def beforeDelete(dynamicEntityId: String): Unit = { + val deleted = provider.deleteByDynamicEntityId(dynamicEntityId).openOrThrowException("delete the entity's webhooks") + if (deleted > 0) logger.info(s"beforeDelete says: deleted $deleted webhook(s) of Dynamic Entity $dynamicEntityId") + } +} diff --git a/obp-api/src/main/scala/code/dynamicentitywebhook/DynamicEntityWebhookEvents.scala b/obp-api/src/main/scala/code/dynamicentitywebhook/DynamicEntityWebhookEvents.scala new file mode 100644 index 0000000000..d93b5a5862 --- /dev/null +++ b/obp-api/src/main/scala/code/dynamicentitywebhook/DynamicEntityWebhookEvents.scala @@ -0,0 +1,101 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ + +package code.dynamicentitywebhook + +import java.time.Instant +import java.time.temporal.ChronoUnit +import java.util.Date + +import code.api.util.APIUtil +import code.messageoutbox.{MessageOutbox, WebhookOutbox} +import org.json4s._ +import org.json4s.native.JsonMethods.{compact, render} + +/** + * This object builds the request a Dynamic Entity webhook sends, and queues it through + * [[code.messageoutbox.WebhookOutbox]]. + * + * The body carries identifiers only, never the record's values or any user id: a receiver that needs + * the record fetches it with its own credentials, so the normal access checks apply when it reads. + * `matched_conditions` lists the conditions an update matched, as the subscriber wrote them, so it + * says nothing the subscriber did not already ask about. + */ +object DynamicEntityWebhookEvents { + + val EventNameCreated = "dynamic_entity.record.created" + val EventNameUpdated = "dynamic_entity.record.updated" + val EventNameDeleted = "dynamic_entity.record.deleted" + /** The ping the test endpoint sends. */ + val EventNameTest = "webhook.test" + + /** The event name of a record change, e.g. UPDATED -> dynamic_entity.record.updated. */ + def eventNameOf(event: String): String = event match { + case DynamicEntityWebhooks.EventCreated => EventNameCreated + case DynamicEntityWebhooks.EventUpdated => EventNameUpdated + case DynamicEntityWebhooks.EventDeleted => EventNameDeleted + } + + /** A time as the body gives it: UTC, to the second, e.g. 2026-10-14T09:12:44Z. */ + private def timestamp(date: Date): String = date.toInstant.truncatedTo(ChronoUnit.SECONDS).toString + + /** The body of one event, exactly as it is posted and signed. */ + def body(eventName: String, eventId: String, occurredAt: Date, webhook: DynamicEntityWebhookTrait, + recordId: Option[String], recordUpdatedAt: Option[Date], + matchedConditions: List[DynamicEntityWebhookFieldCondition]): String = { + def condition(c: DynamicEntityWebhookFieldCondition): JValue = + JObject(List(JField("field", JString(c.field))) ++ c.from.map(JField("from", _)) ++ c.to.map(JField("to", _))) + compact(render(JObject( + List( + JField("event_name", JString(eventName)), + JField("event_id", JString(eventId)), + JField("occurred_at", JString(timestamp(occurredAt))), + JField("webhook_id", JString(webhook.webhookId)), + JField("bank_id", JString(webhook.bankId)), + JField("entity_name", JString(webhook.entityName)) + ) ++ + recordId.map(id => JField("record_id", JString(id))) ++ + recordUpdatedAt.map(at => JField("record_updated_at", JString(timestamp(at)))) ++ + (if (matchedConditions.isEmpty) None else Some(JField("matched_conditions", JArray(matchedConditions.map(condition))))) + ))) + } + + /** + * This queues one event for one webhook, in the current transaction. The event id is new: the same + * event queued for two webhooks gets two ids, one per delivery. + */ + def enqueue(webhook: DynamicEntityWebhookTrait, eventName: String, recordId: Option[String], + recordUpdatedAt: Option[Date], matchedConditions: List[DynamicEntityWebhookFieldCondition]): MessageOutbox = { + val eventId = APIUtil.generateUUID() + WebhookOutbox.enqueue(webhook.webhookId, eventId, eventName, webhook.url, + body(eventName, eventId, Date.from(Instant.now()), webhook, recordId, recordUpdatedAt, matchedConditions)) + } + + /** This queues the test endpoint's ping for a webhook. */ + def enqueueTest(webhook: DynamicEntityWebhookTrait): MessageOutbox = + enqueue(webhook, EventNameTest, None, None, Nil) +} diff --git a/obp-api/src/main/scala/code/dynamicentitywebhook/DynamicEntityWebhookFiring.scala b/obp-api/src/main/scala/code/dynamicentitywebhook/DynamicEntityWebhookFiring.scala new file mode 100644 index 0000000000..38798f1b50 --- /dev/null +++ b/obp-api/src/main/scala/code/dynamicentitywebhook/DynamicEntityWebhookFiring.scala @@ -0,0 +1,131 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ + +package code.dynamicentitywebhook + +import java.util.Date + +import code.api.Constant.DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID +import code.api.util.APIUtil +import code.messageoutbox.WebhookOutbox +import code.util.Helper.MdcLoggable +import org.json4s._ + +/** + * This object fires Dynamic Entity webhooks when a record is written. The record data provider calls it + * from inside each write, in the same transaction, so the queued requests commit or roll back with the + * record, and every route that writes records through that provider fires them: the Dynamic Entity + * endpoints, Domain APIs, Dynamic Endpoints and Dynamic Resource Docs. + * + * For each active webhook of the record's entity that subscribes to the event, one request is queued + * through [[code.messageoutbox.WebhookOutbox]]. An update only fires a webhook with field conditions + * when at least one of them matches the change; the old values are the stored record's as it was + * before the write. + * + * A webhook that has had more than `webhook.circuit_breaker_per_minute` requests queued in the last + * minute is turned off with `deactivated_reason` CIRCUIT_BREAKER instead of being fired again. This is + * the backstop against a receiver whose reply writes a record that fires the webhook again. + */ +object DynamicEntityWebhookFiring extends MdcLoggable { + import DynamicEntityWebhooks._ + + private def provider = DynamicEntityWebhooks.dynamicEntityWebhookProvider.vend + + val defaultCircuitBreakerPerMinute = 60 + def circuitBreakerPerMinute: Int = + APIUtil.getPropsAsIntValue("webhook.circuit_breaker_per_minute", defaultCircuitBreakerPerMinute) + + /** A record was created (`before` None) or updated (`before` its stored values before the write). */ + def recordSaved(bankId: Option[String], entityName: String, recordId: String, before: Option[JObject], + after: JObject, updatedAt: Date): Unit = + before match { + case None => fire(bankId, entityName, EventCreated, recordId, updatedAt, _ => Some(Nil)) + case Some(old) => fire(bankId, entityName, EventUpdated, recordId, updatedAt, webhook => + if (webhook.fieldConditions.isEmpty) Some(Nil) + else { + val types = provider.entityFieldTypes(webhook.dynamicEntityId) + val matched = webhook.fieldConditions.filter(matches(_, old, after, types)) + if (matched.isEmpty) None else Some(matched) + }) + } + + /** A record was deleted. */ + def recordDeleted(bankId: Option[String], entityName: String, recordId: String): Unit = + fire(bankId, entityName, EventDeleted, recordId, new Date(), _ => Some(Nil)) + + /** + * This queues `event` for each active webhook of the entity that subscribes to it. `decide` says, + * for a webhook, whether it fires (Some) and with which matched conditions, or not (None). + */ + private def fire(bankId: Option[String], entityName: String, event: String, recordId: String, at: Date, + decide: DynamicEntityWebhookTrait => Option[List[DynamicEntityWebhookFieldCondition]]): Unit = { + val space = bankId.filter(_.nonEmpty).getOrElse(DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID) + provider.getActiveByEntityName(space, entityName).openOrThrowException("the entity's webhooks") + .filter(_.events.contains(event)) + .foreach { webhook => + decide(webhook).foreach { matched => + if (tripsCircuitBreaker(webhook)) { + provider.deactivate(webhook.webhookId, DeactivatedCircuitBreaker).openOrThrowException("deactivate the webhook") + logger.warn(s"fire says: webhook ${webhook.webhookId} of $entityName turned off, more than " + + s"$circuitBreakerPerMinute requests in a minute ($DeactivatedCircuitBreaker)") + } else + DynamicEntityWebhookEvents.enqueue(webhook, DynamicEntityWebhookEvents.eventNameOf(event), Some(recordId), Some(at), matched) + } + } + } + + private def tripsCircuitBreaker(webhook: DynamicEntityWebhookTrait): Boolean = + WebhookOutbox.queuedSince(webhook.webhookId, new Date(System.currentTimeMillis() - 60 * 1000L)) >= circuitBreakerPerMinute + + /** + * A value as conditions compare it, for a field of `typeName`. A boolean field also accepts the + * strings "true" and "false", and a number may arrive as an integer or a decimal, so both are brought + * to one form; a missing field and null are the same. + */ + private def normalised(value: JValue, typeName: Option[String]): JValue = (value, typeName) match { + case (JNothing, _) => JNull + case (JString(text), Some("boolean")) if text.equalsIgnoreCase("true") || text.equalsIgnoreCase("false") => JBool(text.toBoolean) + case (JInt(number), _) => JDecimal(BigDecimal(number)) + case (JDouble(number), _) => JDecimal(BigDecimal(number)) + case (JDecimal(number), _) => JDecimal(number.underlying.stripTrailingZeros) + case _ => value + } + + /** Whether an update from `old` to `after` matches a condition: the field changed, from `from` to `to` where given. */ + private def matches(condition: DynamicEntityWebhookFieldCondition, old: JObject, after: JObject, types: Map[String, String]): Boolean = { + val typeName = types.get(condition.field) + def same(left: JValue, right: JValue): Boolean = (normalised(left, typeName), normalised(right, typeName)) match { + case (JDecimal(a), JDecimal(b)) => a.compare(b) == 0 + case (a, b) => a == b + } + val oldValue = old \ condition.field + val newValue = after \ condition.field + !same(oldValue, newValue) && + condition.from.forall(same(_, oldValue)) && + condition.to.forall(same(_, newValue)) + } +} diff --git a/obp-api/src/main/scala/code/messageoutbox/MessageOutbox.scala b/obp-api/src/main/scala/code/messageoutbox/MessageOutbox.scala index 3e94ae4c78..7ebb20b15a 100644 --- a/obp-api/src/main/scala/code/messageoutbox/MessageOutbox.scala +++ b/obp-api/src/main/scala/code/messageoutbox/MessageOutbox.scala @@ -66,6 +66,9 @@ case class OutboxEmailPayload( * SMTP by the relay. Queued rather than sent in the request * so that a rolled back request emails nobody, and so a burst * of emails never blocks request threads. + * WEBHOOK โ€” a webhook request: `payload_json` is the body to POST, + * `metadata_json` the URL, `target_id` the webhook id and + * `subject_id` the event id. Sent by WebhookOutbox. * * Row lifecycle: * PENDING โ€” not yet delivered; the relay keeps publishing with backoff. @@ -125,7 +128,7 @@ class MessageOutbox extends LongKeyedMapper[MessageOutbox] with IdPK { object LastReplyJson extends MappedText(this) { override def dbColumnName = "last_reply_json" } - /** Per-type optional extras; empty for OPEN_CORRIDOR. */ + /** Per-type optional extras; empty for OPEN_CORRIDOR, the URL for WEBHOOK. */ object MetadataJson extends MappedText(this) { override def dbColumnName = "metadata_json" } @@ -161,6 +164,7 @@ object MessageOutbox extends MessageOutbox with LongKeyedMetaMapper[MessageOutbo val TYPE_OPEN_CORRIDOR = "OPEN_CORRIDOR" val TYPE_EMAIL = "EMAIL" + val TYPE_WEBHOOK = "WEBHOOK" val OPERATION_ROLE_GRANTED_EMAIL = "role_granted_email" @@ -176,7 +180,7 @@ object MessageOutbox extends MessageOutbox with LongKeyedMetaMapper[MessageOutbo override def dbTableName = "message_outbox" override def dbIndexes: List[BaseIndex[MessageOutbox]] = - Index(Status) :: Index(SubjectId) :: Index(OutboxType) :: super.dbIndexes + Index(Status) :: Index(SubjectId) :: Index(OutboxType) :: Index(TargetId) :: super.dbIndexes def enqueue( outboxType: String, @@ -184,7 +188,8 @@ object MessageOutbox extends MessageOutbox with LongKeyedMetaMapper[MessageOutbo subjectIdType: String, operationName: String, targetId: String, - payloadJson: String + payloadJson: String, + metadataJson: String = "" ): MessageOutbox = MessageOutbox.create .OutboxType(outboxType) @@ -193,6 +198,7 @@ object MessageOutbox extends MessageOutbox with LongKeyedMetaMapper[MessageOutbo .OperationName(operationName) .TargetId(targetId) .PayloadJson(payloadJson) + .MetadataJson(metadataJson) .Status(STATUS_PENDING) .saveMe() diff --git a/obp-api/src/main/scala/code/messageoutbox/MessageOutboxRelay.scala b/obp-api/src/main/scala/code/messageoutbox/MessageOutboxRelay.scala index 5befab528c..b0b90003d5 100644 --- a/obp-api/src/main/scala/code/messageoutbox/MessageOutboxRelay.scala +++ b/obp-api/src/main/scala/code/messageoutbox/MessageOutboxRelay.scala @@ -79,6 +79,9 @@ import scala.concurrent.duration._ * several instances only one sends it. Sent โ†’ DELIVERED. Not sent โ†’ stays * PENDING and is retried with the same backoff, until maxEmailAttempts, then * STICKY (an address or mail server an operator must look at). + * + * WEBHOOK: claimed the same way, then posted by WebhookOutbox.deliver, which + * says which replies are retried and which go STICKY at once. */ object MessageOutboxRelay extends MdcLoggable { @@ -150,6 +153,7 @@ object MessageOutboxRelay extends MdcLoggable { def relayRow(row: MessageOutbox): Unit = row.outboxType match { case MessageOutbox.TYPE_OPEN_CORRIDOR => relayOpenCorridorRow(row) case MessageOutbox.TYPE_EMAIL => relayEmailRow(row) + case MessageOutbox.TYPE_WEBHOOK => WebhookOutbox.deliver(row) case other => row.Status(MessageOutbox.STATUS_STICKY).Attempts(row.attempts + 1) .LastError(s"no publisher registered for outbox_type '$other'").saveMe() diff --git a/obp-api/src/main/scala/code/messageoutbox/WebhookOutbox.scala b/obp-api/src/main/scala/code/messageoutbox/WebhookOutbox.scala new file mode 100644 index 0000000000..e4e7b37b29 --- /dev/null +++ b/obp-api/src/main/scala/code/messageoutbox/WebhookOutbox.scala @@ -0,0 +1,245 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ + +package code.messageoutbox + +import java.nio.charset.StandardCharsets +import java.security.interfaces.{RSAPrivateKey, RSAPublicKey} +import java.util.concurrent.TimeUnit + +import code.api.util.{APIUtil, CertificateUtil} +import code.util.Helper.MdcLoggable +import com.nimbusds.jose.crypto.RSASSASigner +import com.nimbusds.jose.jwk.RSAKey +import com.nimbusds.jose.{JOSEObjectType, JWSAlgorithm, JWSHeader, JWSObject, Payload} +import net.liftweb.mapper.{By, By_>, Descending, MaxRows, OrderBy, QueryParam, StartAt} +import okhttp3.{MediaType, OkHttpClient, Request, RequestBody} +import org.json4s.native.Serialization +import org.json4s.{DefaultFormats, Formats} + +import scala.util.Try + +/** Where a WEBHOOK outbox row is sent, stored in its metadata_json. */ +case class OutboxWebhookMetadata(url: String) + +/** + * The reply a WEBHOOK delivery last got, stored in its last_reply_json: the HTTP status and the start + * of the body. Absent when the request got no reply at all (a timeout, a refused connection). + */ +case class OutboxWebhookReply(http_status: Int, body: String) + +/** + * This object queues and delivers webhook requests through the message outbox, as rows of type + * WEBHOOK. It is not tied to one kind of webhook: whoever queues a request decides its body, and this + * object posts it. + * + * A row is written in the same database transaction as the change it reports, so a request that rolls + * back queues nothing. The relay then posts it (see [[deliver]]): + * - The body is the JSON queued, sent as it is. The URL is the one the webhook had when the request + * was queued, and it is checked again before every attempt (see [[WebhookUrlCheck]]). Redirects are + * not followed, because one could lead to an address the check would refuse. + * - Every attempt is signed afresh, when this instance has a stable key (see [[signatureStatus]]). + * - A 2xx reply means DELIVERED. A 5xx, 408 or 429 reply, a timeout or a connection that fails is + * retried with the relay's backoff, until `webhook.outbox_max_attempts` attempts, then STICKY. Any + * other reply, and a URL that is now refused, is STICKY at once, because retrying cannot change it. + * + * The row's columns: subject_id is the event id (the same on every attempt, so a receiver drops + * duplicates by it), operation_name the event name, target_id the webhook id. + */ +object WebhookOutbox extends MdcLoggable { + + private implicit val formats: Formats = DefaultFormats + + val SubjectIdTypeEventId = "event_id" + + /** Attempts after which a WEBHOOK row that could not be delivered goes STICKY. With the relay's + * backoff, which reaches its 10 minute cap after 6 attempts, the default of 144 gives up after + * roughly a day. Read on every row so a change applies without a restart. */ + val defaultMaxAttempts = 144 + def maxAttemptsInEffect: Int = APIUtil.getPropsAsIntValue("webhook.outbox_max_attempts", defaultMaxAttempts) + + /** How much of a receiver's reply body is kept. */ + private val MaxReplyBodyLength = 2000L + + val SignatureStatusSigned = "signed" + val SignatureStatusNotConfigured = "not_configured" + + /** + * Whether deliveries are signed. A delivery is signed with this instance's server key, the one + * published at GET /obp/v3.1.0/certs, so a receiver can check it came from OBP. That only works when + * the key is stable: with `jwt.use.ssl=true` it is read from the configured keystore and every node + * shares it. Otherwise each node generates a key pair of its own at startup, which differs between + * nodes and changes on every restart, so a receiver could not verify against it; deliveries then go + * out unsigned and say so. + */ + def isSigningConfigured: Boolean = APIUtil.getPropsAsBoolValue("jwt.use.ssl", false) + + def signatureStatus: String = if (isSigningConfigured) SignatureStatusSigned else SignatureStatusNotConfigured + + /** + * This signs a request body as a JSON Web Signature with a detached payload (RFC 7515, Appendix F): + * the protected header and the signature, with the payload part left empty, because the payload is + * the body itself. The header carries `kid`, the key id GET /obp/v3.1.0/certs gives the public key, + * and `typ` `obp-webhook+jwt`, so the signature cannot be taken for a token. + */ + def sign(body: Array[Byte], privateKey: RSAPrivateKey, publicKey: RSAPublicKey, issuedAtSeconds: Long): String = { + val keyId = new RSAKey.Builder(publicKey).keyIDFromThumbprint().build().getKeyID + val header = new JWSHeader.Builder(JWSAlgorithm.RS256) + .keyID(keyId) + .`type`(new JOSEObjectType("obp-webhook+jwt")) + .customParam("iat", issuedAtSeconds) + .build() + val jws = new JWSObject(header, new Payload(body)) + jws.sign(new RSASSASigner(privateKey)) + jws.serialize(true) + } + + /** + * This queues one webhook request. Call it inside the transaction of the change it reports. + * + * `body` is the exact JSON to post; it must carry `eventId` for receivers to drop duplicates by. + */ + def enqueue(webhookId: String, eventId: String, eventName: String, url: String, body: String): MessageOutbox = + MessageOutbox.enqueue(MessageOutbox.TYPE_WEBHOOK, eventId, SubjectIdTypeEventId, eventName, webhookId, body, + metadataJson = Serialization.write(OutboxWebhookMetadata(url))) + + /** This drops the requests of a webhook that have not been delivered yet, when the webhook is deleted. */ + def discardPending(webhookId: String): Int = + MessageOutbox.findAll( + By(MessageOutbox.OutboxType, MessageOutbox.TYPE_WEBHOOK), + By(MessageOutbox.TargetId, webhookId), + By(MessageOutbox.Status, MessageOutbox.STATUS_PENDING) + ).count(_.delete_!) + + /** A webhook's requests, newest first; `status` narrows them to PENDING, DELIVERED or STICKY. */ + def deliveries(webhookId: String, status: Option[String], limit: Int, offset: Int): List[MessageOutbox] = + MessageOutbox.findAll( + List[QueryParam[MessageOutbox]]( + By(MessageOutbox.OutboxType, MessageOutbox.TYPE_WEBHOOK), + By(MessageOutbox.TargetId, webhookId), + OrderBy(MessageOutbox.id, Descending), + StartAt(offset), + MaxRows(limit) + ) ++ status.map(By(MessageOutbox.Status, _)): _*) + + /** How many requests were queued for a webhook since `since`. */ + def queuedSince(webhookId: String, since: java.util.Date): Long = + MessageOutbox.count( + By(MessageOutbox.OutboxType, MessageOutbox.TYPE_WEBHOOK), + By(MessageOutbox.TargetId, webhookId), + By_>(MessageOutbox.CreatedAt, since)) + + /** One request of a webhook, by its event id. */ + def delivery(webhookId: String, eventId: String): Option[MessageOutbox] = + MessageOutbox.find( + By(MessageOutbox.OutboxType, MessageOutbox.TYPE_WEBHOOK), + By(MessageOutbox.TargetId, webhookId), + By(MessageOutbox.SubjectId, eventId)).toOption + + /** + * This sends a STICKY request again: it goes back to PENDING with its attempts reset, so it gets the + * full number of attempts again, and keeps its event id and body. + */ + def requeue(row: MessageOutbox): MessageOutbox = + row.Status(MessageOutbox.STATUS_PENDING).Attempts(0).LastError("").saveMe() + + /** The reply a row last got, if it got one. */ + def lastReply(row: MessageOutbox): Option[OutboxWebhookReply] = + Option(row.LastReplyJson.get).filter(_.trim.nonEmpty) + .flatMap(json => Try(Serialization.read[OutboxWebhookReply](json)).toOption) + + private lazy val client: OkHttpClient = new OkHttpClient.Builder() + .dns(WebhookUrlCheck.deliveryDns) + .followRedirects(false) + .followSslRedirects(false) + .connectTimeout(10, TimeUnit.SECONDS) + .readTimeout(30, TimeUnit.SECONDS) + .callTimeout(30, TimeUnit.SECONDS) + .build() + + private val jsonMediaType = MediaType.get("application/json; charset=utf-8") + + private def isRetryable(httpStatus: Int): Boolean = httpStatus >= 500 || httpStatus == 408 || httpStatus == 429 + + /** One delivery attempt for a PENDING WEBHOOK row; called by the relay. */ + def deliver(read: MessageOutbox): Unit = + // Another instance's relay may have claimed it first; then it is theirs to send. + if (MessageOutbox.claimForDelivery(read)) { + val row = MessageOutbox.find(By(MessageOutbox.id, read.id.get)).openOrThrowException("claimed row") + val url = Try(Serialization.read[OutboxWebhookMetadata](row.MetadataJson.get).url).toOption.getOrElse("") + WebhookUrlCheck.problem(url) match { + case Some(problem) => giveUp(row, s"The url is refused: $problem") + case None => post(row, url) + } + } + + private def post(row: MessageOutbox, url: String): Unit = { + val body = row.payloadJson.getBytes(StandardCharsets.UTF_8) + val request = { + val builder = new Request.Builder() + .url(url) + .post(RequestBody.create(body, jsonMediaType)) + .header("OBP-Webhook-Id", row.targetId) + .header("OBP-Event-Id", row.subjectId) + .header("OBP-Delivery-Attempt", row.attempts.toString) + .header("OBP-Signature-Status", signatureStatus) + if (isSigningConfigured) + builder.header("OBP-Signature", sign(body, CertificateUtil.privateKey, CertificateUtil.publicKey, System.currentTimeMillis() / 1000)) + builder.build() + } + try { + val response = client.newCall(request).execute() + try { + val httpStatus = response.code + val reply = Serialization.write(OutboxWebhookReply(httpStatus, Try(response.peekBody(MaxReplyBodyLength).string()).getOrElse(""))) + row.LastReplyJson(reply) + if (response.isSuccessful) { + row.Status(MessageOutbox.STATUS_DELIVERED).LastError("").saveMe() + logger.info(s"message outbox row ${row.id.get}: ${row.operationName} for webhook ${row.targetId} DELIVERED ($httpStatus)") + } else if (isRetryable(httpStatus)) retryOrGiveUp(row, s"The receiver replied $httpStatus ${response.message}.") + else giveUp(row, s"The receiver replied $httpStatus ${response.message}.") + } finally response.close() + } catch { + case e: ForbiddenWebhookAddress => giveUp(row.LastReplyJson(""), s"The url is refused: ${e.getMessage}") + case e: java.io.IOException => retryOrGiveUp(row.LastReplyJson(""), s"No reply: ${e.getClass.getSimpleName} ${Option(e.getMessage).getOrElse("")}".trim) + } + } + + // The claim already counted this attempt. + private def retryOrGiveUp(row: MessageOutbox, error: String): Unit = + if (row.attempts >= maxAttemptsInEffect) giveUp(row, s"Gave up after ${row.attempts} attempts. $error") + else { + row.Status(MessageOutbox.STATUS_PENDING).LastError(error.take(2000)).saveMe() + logger.warn(s"message outbox row ${row.id.get}: ${row.operationName} for webhook ${row.targetId} " + + s"not delivered (attempt ${row.attempts}), will retry: $error") + } + + private def giveUp(row: MessageOutbox, error: String): Unit = { + row.Status(MessageOutbox.STATUS_STICKY).LastError(error.take(2000)).saveMe() + logger.warn(s"message outbox row ${row.id.get}: ${row.operationName} for webhook ${row.targetId} STICKY: $error") + } +} diff --git a/obp-api/src/main/scala/code/messageoutbox/WebhookUrlCheck.scala b/obp-api/src/main/scala/code/messageoutbox/WebhookUrlCheck.scala new file mode 100644 index 0000000000..f36abf595f --- /dev/null +++ b/obp-api/src/main/scala/code/messageoutbox/WebhookUrlCheck.scala @@ -0,0 +1,134 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ + +package code.messageoutbox + +import java.net.{InetAddress, URI, UnknownHostException} + +import code.api.util.APIUtil + +import scala.collection.JavaConverters._ +import scala.util.Try + +/** + * This object decides whether OBP may send a webhook request to a URL. + * + * A webhook makes OBP send requests to an address someone else chose. Without a check, that address + * could be a service inside OBP's own network, which outsiders cannot otherwise reach (this is called + * Server-Side Request Forgery), or OBP itself, where the webhook's request could write a record that + * fires the webhook again. So a URL is refused when it is not HTTPS, when its host resolves to a + * loopback, private, link-local (which includes the cloud metadata address 169.254.169.254) or + * unique local address, or when it names this instance's own host and port. + * + * The check runs when a webhook is created or changed, and again before each delivery, because what + * a host name resolves to can change after it was registered. A host name that does not resolve is + * accepted at registration, since the delivery check refuses it if it later resolves somewhere + * forbidden. At delivery the connection itself resolves the host through [[deliveryDns]], which + * refuses the same addresses, so a name cannot pass the check and then resolve somewhere else when + * the connection is opened. + */ +object WebhookUrlCheck { + + val MaxUrlLength = 1024 + + /** Plain http is for local development only. */ + def allowHttp: Boolean = APIUtil.getPropsAsBoolValue("webhook.allow_http", false) + + /** Private and loopback addresses are for local development and tests only. */ + def allowPrivateAddresses: Boolean = APIUtil.getPropsAsBoolValue("webhook.allow_private_addresses", false) + + private def effectivePort(uri: URI): Int = + if (uri.getPort != -1) uri.getPort + else if ("https".equalsIgnoreCase(uri.getScheme)) 443 + else 80 + + /** This instance's own scheme, host and port, from the hostname prop. */ + private lazy val ownUri: Option[URI] = + APIUtil.getPropsValue("hostname").toOption.flatMap(h => Try(new URI(h.trim)).toOption).filter(_.getHost != null) + + private lazy val ownAddresses: Set[InetAddress] = + ownUri.toList.flatMap(u => Try(InetAddress.getAllByName(u.getHost).toList).getOrElse(Nil)).toSet + + private def defaultResolve(host: String): List[InetAddress] = Try(InetAddress.getAllByName(host).toList).getOrElse(Nil) + + /** Whether OBP must not connect to this address: loopback, private, link-local and the like. */ + def isForbiddenAddress(address: InetAddress): Boolean = + !allowPrivateAddresses && ( + address.isLoopbackAddress || address.isSiteLocalAddress || address.isLinkLocalAddress || + address.isAnyLocalAddress || address.isMulticastAddress || + (address.getAddress.length == 16 && (address.getAddress()(0) & 0xfe) == 0xfc)) // IPv6 unique local, fc00::/7 + + /** + * This returns why OBP must not send requests to `url`, or None when it may. + * + * `resolve` looks a host name up; it is a parameter so that tests can say what a name resolves to. + * `requireResolvable` refuses a host that resolves to nothing, as the delivery check does. + */ + def problem(url: String, resolve: String => List[InetAddress] = defaultResolve, + requireResolvable: Boolean = false): Option[String] = { + val trimmed = Option(url).map(_.trim).getOrElse("") + Try(new URI(trimmed)).toOption match { + case _ if trimmed.isEmpty => Some("The url is empty.") + case _ if trimmed.length > MaxUrlLength => Some(s"The url is longer than $MaxUrlLength characters.") + case None => Some("The url cannot be parsed.") + case Some(uri) if !uri.isAbsolute || uri.getHost == null || uri.getHost.isEmpty => Some("The url must be absolute and name a host.") + case Some(uri) if uri.getUserInfo != null => Some("The url must not carry a user name or password.") + case Some(uri) if !("https".equalsIgnoreCase(uri.getScheme) || (allowHttp && "http".equalsIgnoreCase(uri.getScheme))) => + Some(if (allowHttp) "The url must use https or http." else "The url must use https.") + case Some(uri) => + val host = uri.getHost.stripPrefix("[").stripSuffix("]") + val port = effectivePort(uri) + val addresses = resolve(host) + val isThisInstance = ownUri.exists { own => + effectivePort(own) == port && + (own.getHost.equalsIgnoreCase(host) || addresses.exists(ownAddresses.contains)) + } + if (isThisInstance) Some("The url points at this OBP instance.") + else if (addresses.exists(isForbiddenAddress)) + Some(s"The url's host $host resolves to a loopback, private, link-local or unique local address.") + else if (requireResolvable && addresses.isEmpty) Some(s"The url's host $host does not resolve.") + else None + } + } + + /** + * The name lookup a webhook delivery connects with. It refuses a host when any of its addresses is + * forbidden, so the addresses the connection uses are the ones that were checked. + */ + val deliveryDns: okhttp3.Dns = new okhttp3.Dns { + override def lookup(hostname: String): java.util.List[InetAddress] = { + val addresses = okhttp3.Dns.SYSTEM.lookup(hostname).asScala.toList + addresses.find(isForbiddenAddress).foreach { address => + throw new ForbiddenWebhookAddress(s"$hostname resolves to ${address.getHostAddress}, which a webhook may not be sent to") + } + addresses.asJava + } + } +} + +/** A webhook's host resolved to an address OBP may not send to. Retrying cannot fix it. */ +class ForbiddenWebhookAddress(message: String) extends java.net.UnknownHostException(message) diff --git a/obp-api/src/main/scala/code/users/UserReference.scala b/obp-api/src/main/scala/code/users/UserReference.scala index 85f7ba7e4b..a88539d14d 100644 --- a/obp-api/src/main/scala/code/users/UserReference.scala +++ b/obp-api/src/main/scala/code/users/UserReference.scala @@ -200,6 +200,7 @@ object UserReference { case object ApiProductSubscription_CreatedByUserId extends UserReference(UseOnBehalfOfUserId , "code.apiproductsubscription.ApiProductSubscription", List("CreatedByUserId"), "a subscription outlives the Consent that took it out") case object DynamicGlossaryItem_CreatedByUserId extends UserReference(UseOnBehalfOfUserId , "code.glossaryitem.DynamicGlossaryItem", List("CreatedByUserId"), "outlives the Consent that created it") case object DomainApi_CreatedByUserId extends UserReference(UseOnBehalfOfUserId , "code.domainapi.DomainApi", List("CreatedByUserId"), "the Domain API's creator; a published API outlives the Consent that created it") + case object DynamicEntityWebhook_CreatedByUserId extends UserReference(UseOnBehalfOfUserId , "code.dynamicentitywebhook.DynamicEntityWebhook", List("CreatedByUserId", "CreatedByOnBehalfOfUserId"), "record both, like Counterparty: a webhook sends data out of OBP and outlives the Consent that made it, so the agent stays on CreatedByUserId (published as created_by_user_id) and the person it acted for goes to CreatedByOnBehalfOfUserId") case object Bank_CreatedByUserId extends UserReference(UseOnBehalfOfUserId , "code.model.dataAccess.MappedBank", List("CreatedByUserId"), "creator grant already resolved at the endpoint") case object Organisation_CreatedByUserId extends UserReference(UseOnBehalfOfUserId , "code.organisation.Organisation", List("CreatedByUserId"), "outlives the Consent that created it") case object Asset_CreatedByUserId extends UserReference(UseOnBehalfOfUserId , "code.asset.Asset", List("CreatedByUserId"), "outlives the Consent that created it, as RoutingScheme_CreatedByUserId; seeded rows hold system:asset-seed") @@ -296,6 +297,7 @@ object UserReference { ApiProductSubscription_CreatedByUserId, DynamicGlossaryItem_CreatedByUserId, DomainApi_CreatedByUserId, + DynamicEntityWebhook_CreatedByUserId, Bank_CreatedByUserId, Organisation_CreatedByUserId, Asset_CreatedByUserId, diff --git a/obp-api/src/test/scala/code/api/v4_0_0/DynamicEndpointBackedByDynamicEntityTest.scala b/obp-api/src/test/scala/code/api/v4_0_0/DynamicEndpointBackedByDynamicEntityTest.scala new file mode 100644 index 0000000000..77f60c7ea7 --- /dev/null +++ b/obp-api/src/test/scala/code/api/v4_0_0/DynamicEndpointBackedByDynamicEntityTest.scala @@ -0,0 +1,195 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ +package code.api.v4_0_0 + +import code.DynamicData.DynamicDataProvider +import code.api.Constant.DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID +import code.api.dynamic.endpoint.helper.DynamicEndpointHelper +import code.api.util.APIUtil.OAuth._ +import code.api.util.ApiRole.{canCreateDynamicEndpoint, canCreateEndpointMapping} +import code.dynamicEntity.{DynamicEntityCommons, DynamicEntityProvider} +import code.dynamicentitywebhook.DynamicEntityWebhooks +import code.entitlement.Entitlement +import code.messageoutbox.MessageOutbox +import com.openbankproject.commons.util.ApiVersion +import net.liftweb.mapper.By +import org.json4s.JsonAST._ +import org.json4s.JsonDSL._ +import org.json4s.native.JsonMethods.{compact, render} +import org.scalatest.Tag + +/** + * This suite drives a Dynamic Endpoint whose host is `dynamic_entity` end to end: a swagger whose + * operations are served from a Dynamic Entity's records through Endpoint Mappings, called at + * /obp/dynamic-endpoint/... like any client would call it. + * + * It checks that a PUT updates the record its URL names, in place: the record keeps its id, its values + * change, no second record appears, and the entity's webhooks see one UPDATED. A PUT used to delete the + * record and create a new one with a new id, which a client holding the id lost track of, and which + * webhooks saw as DELETED then CREATED. + */ +class DynamicEndpointBackedByDynamicEntityTest extends V400ServerSetup { + + object VersionOfApi extends Tag(ApiVersion.v4_0_0.toString) + object DynamicEndpointPut extends Tag("dynamicEndpoint PUT backed by a Dynamic Entity") + + private val suffix = java.util.UUID.randomUUID().toString.replace("-", "").take(8) + private val entityName = s"gadget_$suffix" + private val idField = s"${entityName}_id" + private val collectionPath = s"gadgets-$suffix" + + /** A swagger served from the Dynamic Entity: create, read one, update one. */ + private val swagger: String = + s"""{ + | "swagger": "2.0", + | "info": {"title": "Gadgets $suffix", "version": "1.0.0"}, + | "definitions": { + | "Gadget": { + | "type": "object", + | "properties": { + | "gadget_id": {"type": "string", "example": "a1"}, + | "name": {"type": "string", "example": "lamp"}, + | "status": {"type": "string", "example": "OPEN"} + | } + | } + | }, + | "paths": { + | "/$collectionPath": { + | "post": { + | "operationId": "POST_gadget_$suffix", + | "consumes": ["application/json"], "produces": ["application/json"], + | "parameters": [{"in": "body", "name": "body", "required": true, "schema": {"$$ref": "#/definitions/Gadget"}}], + | "responses": {"201": {"description": "Created", "schema": {"$$ref": "#/definitions/Gadget"}}}, + | "summary": "Create a gadget $suffix" + | } + | }, + | "/$collectionPath/{gadget_id}": { + | "get": { + | "operationId": "GET_gadget_$suffix", + | "produces": ["application/json"], + | "parameters": [{"in": "path", "name": "gadget_id", "required": true, "type": "string"}], + | "responses": {"200": {"description": "OK", "schema": {"$$ref": "#/definitions/Gadget"}}}, + | "summary": "Get a gadget $suffix" + | }, + | "put": { + | "operationId": "PUT_gadget_$suffix", + | "consumes": ["application/json"], "produces": ["application/json"], + | "parameters": [ + | {"in": "path", "name": "gadget_id", "required": true, "type": "string"}, + | {"in": "body", "name": "body", "required": true, "schema": {"$$ref": "#/definitions/Gadget"}} + | ], + | "responses": {"200": {"description": "OK", "schema": {"$$ref": "#/definitions/Gadget"}}}, + | "summary": "Update a gadget $suffix" + | } + | } + | }, + | "host": "dynamic_entity", + | "schemes": ["https"] + |}""".stripMargin + + /** Each response field read from the entity, the record found by its id field. */ + private val responseMapping: JValue = { + def from(field: String): JValue = ("entity" -> entityName) ~ ("field" -> field) ~ ("query" -> idField) + ("gadget_id" -> from(idField)) ~ ("name" -> from("name")) ~ ("status" -> from("status")) + } + + /** The entity's fields taken from the request body. */ + private val requestMapping: JValue = ("name" -> "name") ~ ("status" -> "status") + + /** The ResourceDoc OBP generated for one operation of the swagger. */ + private def generatedDoc(verb: String, url: String) = + DynamicEndpointHelper.doc.find(doc => doc.requestVerb == verb && doc.requestUrl.endsWith(url)) + .getOrElse(fail(s"no generated ResourceDoc for $verb $url")) + + private def dynamicEndpoint(segments: String*) = segments.foldLeft(baseRequest / "obp" / "dynamic-endpoint")(_ / _) + + feature("A Dynamic Endpoint backed by a Dynamic Entity") { + + scenario("a PUT updates the record its URL names in place: same id, new values, one UPDATED", DynamicEndpointPut, VersionOfApi) { + Given("a system level Dynamic Entity, and a webhook on it") + val entityId = DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(DynamicEntityCommons( + entityName, + // With `required`, as every definition made through the API has it: record validation reads it. + s"""{"$entityName":{"required":[],"properties":{"name":{"type":"string","example":"lamp"},"status":{"type":"string","example":"OPEN"}}}}""", + None, resourceUser1.userId, None, hasPersonalEntity = false)).openOrThrowException("entity").dynamicEntityId.get + val webhook = DynamicEntityWebhooks.dynamicEntityWebhookProvider.vend.create(DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID, + entityId, entityName, s"https://receiver-$suffix.invalid/hook", List("CREATED", "UPDATED", "DELETED"), Nil, + isActive = true, "", resourceUser1.userId).openOrThrowException("webhook") + + And("a Dynamic Endpoint served from it, with an Endpoint Mapping for each operation") + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, canCreateDynamicEndpoint.toString) + val createdEndpoint = makePostRequest((v4_0_0_Request / "management" / "dynamic-endpoints").POST <@ (user1), swagger) + withClue(createdEndpoint.body) { createdEndpoint.code should equal(201) } + + val postDoc = generatedDoc("POST", s"/$collectionPath") + val getDoc = generatedDoc("GET", s"/$collectionPath/GADGET_ID") + val putDoc = generatedDoc("PUT", s"/$collectionPath/GADGET_ID") + + Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, canCreateEndpointMapping.toString) + List(postDoc, getDoc, putDoc).foreach { doc => + val mapping = compact(render(("operation_id" -> doc.operationId) ~ + ("request_mapping" -> requestMapping) ~ ("response_mapping" -> responseMapping))) + val created = makePostRequest((v4_0_0_Request / "management" / "endpoint-mappings").POST <@ (user1), mapping) + withClue(created.body) { created.code should equal(201) } + doc.roles.toList.flatten.foreach(role => Entitlement.entitlement.vend.addEntitlement("", resourceUser1.userId, role.toString)) + } + + When("a gadget is created through the endpoint") + val created = makePostRequest(dynamicEndpoint(collectionPath).POST <@ (user1), + compact(render(("name" -> "lamp") ~ ("status" -> "OPEN")))) + // Success, whichever code: the dispatcher answers 200 for every operation, not the swagger's 201. + withClue(created.body) { List(200, 201) should contain(created.code) } + val gadgetId = (created.body \ "gadget_id").extract[String] + gadgetId should not be empty + + When("it is changed with a PUT") + val updated = makePutRequest(dynamicEndpoint(collectionPath, gadgetId).PUT <@ (user1), + compact(render(("name" -> "brighter lamp") ~ ("status" -> "CLOSED")))) + withClue(updated.body) { updated.code should equal(200) } + + Then("the response is the same record, with the new values") + (updated.body \ "gadget_id").extract[String] should equal(gadgetId) + (updated.body \ "name").extract[String] should equal("brighter lamp") + (updated.body \ "status").extract[String] should equal("CLOSED") + + And("reading it by its id gives the new values") + val read = makeGetRequest(dynamicEndpoint(collectionPath, gadgetId).GET <@ (user1)) + withClue(read.body) { read.code should equal(200) } + (read.body \ "status").extract[String] should equal("CLOSED") + + And("the entity still holds exactly one record, under the same id") + val records = DynamicDataProvider.connectorMethodProvider.vend.getAllCommunity(None, entityName) + records.flatMap(_.dynamicDataId) should equal(List(gadgetId)) + + And("the webhook saw it created and then updated, never deleted") + val queued = MessageOutbox.findAll(By(MessageOutbox.OutboxType, MessageOutbox.TYPE_WEBHOOK), By(MessageOutbox.TargetId, webhook.webhookId)) + .sortBy(_.id.get) + queued.map(_.operationName) should equal(List("dynamic_entity.record.created", "dynamic_entity.record.updated")) + queued.foreach(_.payloadJson should include(gadgetId)) + } + } +} diff --git a/obp-api/src/test/scala/code/api/v7_0_0/DynamicEntityWebhooksTest.scala b/obp-api/src/test/scala/code/api/v7_0_0/DynamicEntityWebhooksTest.scala new file mode 100644 index 0000000000..1a48a7e0ec --- /dev/null +++ b/obp-api/src/test/scala/code/api/v7_0_0/DynamicEntityWebhooksTest.scala @@ -0,0 +1,770 @@ +/** +Open Bank Project - API +Copyright (C) 2011-2026, TESOBE GmbH. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . + +Email: contact@tesobe.com +TESOBE GmbH. +Osloer Strasse 16/17 +Berlin 13359, Germany + +This product includes software developed at +TESOBE (http://www.tesobe.com/) + + */ +package code.api.v7_0_0 + +import code.api.Constant.DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID +import code.api.util.APIUtil.OAuth._ +import code.api.util.ApiRole.{canCreateDynamicEntityWebhook, canDeleteDynamicEntityWebhook, canGetDynamicEntityWebhooks, canUpdateDynamicEntityWebhook} +import code.api.util.ErrorMessages._ +import code.api.v6_0_0.V600ServerSetup +import code.DynamicData.DynamicDataProvider +import code.dynamicEntity.{DynamicEntityCommons, DynamicEntityProvider} +import code.entitlement.Entitlement +import code.messageoutbox.{MessageOutbox, MessageOutboxRelay, WebhookOutbox} +import com.nimbusds.jose.crypto.RSASSAVerifier +import com.nimbusds.jose.jwk.RSAKey +import com.nimbusds.jose.{JWSObject, Payload} +import com.sun.net.httpserver.{HttpExchange, HttpServer} +import net.liftweb.mapper.By +import com.openbankproject.commons.model.ErrorMessage +import com.openbankproject.commons.util.ApiVersion +import org.json4s.JsonAST._ +import org.json4s.JsonDSL._ +import org.json4s.native.JsonMethods.{compact, render} +import org.scalatest.Tag + +import java.net.InetSocketAddress +import java.nio.charset.StandardCharsets +import java.security.KeyPairGenerator +import java.security.interfaces.{RSAPrivateKey, RSAPublicKey} +import scala.collection.JavaConverters._ + +/** + * This suite checks the v7.0.0 endpoints that manage Dynamic Entity webhooks: the Roles at SYS and at a + * bank, create, read, list, update and delete, the checks on events, field conditions, description and url, + * the one-webhook-per-url rule, and how `is_active` and `deactivated_reason` move together on update. + * It also delivers requests through the test endpoint and the message outbox to a receiver run by the + * suite on this machine, to check the request OBP sends and which replies are retried, and checks that a + * signature verifies against the public key the way a receiver would verify it. + */ +class DynamicEntityWebhooksTest extends V600ServerSetup { + + object VersionOfApi extends Tag(ApiVersion.v7_0_0.toString) + object ApiEndpoint1 extends Tag("createDynamicEntityWebhook") + object ApiEndpoint2 extends Tag("getDynamicEntityWebhooks") + object ApiEndpoint3 extends Tag("getDynamicEntityWebhooksAtBank") + object ApiEndpoint4 extends Tag("getDynamicEntityWebhook") + object ApiEndpoint5 extends Tag("updateDynamicEntityWebhook") + object ApiEndpoint6 extends Tag("deleteDynamicEntityWebhook") + object ApiEndpoint7 extends Tag("testDynamicEntityWebhook") + object ApiEndpoint8 extends Tag("getDynamicEntityWebhookDeliveries") + object ApiEndpoint9 extends Tag("retryDynamicEntityWebhookDelivery") + + private val SYS = DYNAMIC_ENTITY_SYSTEM_LEVEL_BANK_ID + private val owner = "dynamic-entity-webhook-owner" + private val suffix = java.util.UUID.randomUUID().toString.replace("-", "").take(8) + + // A host that does not resolve is accepted at registration (the delivery check refuses it later if it + // resolves somewhere forbidden), so the test does not depend on the network. + private def receiverUrl(path: String) = s"https://receiver-$suffix.invalid/$path" + + private def v7 = baseRequest / "obp" / "v7.0.0" + private def webhooks(bankId: String, dynamicEntityId: String) = + v7 / "management" / "banks" / bankId / "dynamic-entities" / dynamicEntityId / "web-hooks" + private def webhooksAtBank(bankId: String) = v7 / "management" / "banks" / bankId / "dynamic-entity-web-hooks" + + private def message(response: code.setup.APIResponse) = response.body.extract[ErrorMessage].message + private def webhookIdsIn(response: code.setup.APIResponse): List[String] = + (response.body \ "web_hooks").children.map(webhook => (webhook \ "webhook_id").extract[String]) + private def grantAllAt(bankId: String): Unit = + List(canCreateDynamicEntityWebhook, canGetDynamicEntityWebhooks, canUpdateDynamicEntityWebhook, canDeleteDynamicEntityWebhook) + .foreach(role => Entitlement.entitlement.vend.addEntitlement(bankId, resourceUser1.userId, role.toString)) + + /** A Dynamic Entity in a space (None = system) with a boolean, a number and a string field; its id. */ + private def entityIn(space: Option[String], entityName: String): String = + DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(DynamicEntityCommons( + entityName, + s"""{"$entityName":{"properties":{ + |"approved":{"type":"boolean","example":"false"}, + |"amount":{"type":"number","example":"10.5"}, + |"status":{"type":"string","example":"OPEN"}}}}""".stripMargin, + None, owner, space, hasPersonalEntity = false)).openOrThrowException("definition").dynamicEntityId.get + + private def webhookBody(url: String, events: List[String], conditions: Option[JValue] = None, + isActive: Option[Boolean] = None, description: Option[String] = None): String = + compact(render(("url" -> url) ~ ("events" -> events) ~ ("field_conditions" -> conditions) ~ + ("is_active" -> isActive) ~ ("description" -> description))) + + /** A request the suite's receiver got. */ + private case class Received(headers: Map[String, String], body: String) + + /** + * A receiver on this machine that answers every request with `status` (and `location`, for a redirect) + * and keeps what it got. Webhooks may only reach it while `webhook.allow_http` and + * `webhook.allow_private_addresses` are set. + */ + private class Receiver(status: Int, location: Option[String] = None) { + val received = new java.util.concurrent.ConcurrentLinkedQueue[Received]() + private val server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0) + server.createContext("/", (exchange: HttpExchange) => { + val body = new String(exchange.getRequestBody.readAllBytes(), StandardCharsets.UTF_8) + val headers = exchange.getRequestHeaders.asScala.map { case (name, values) => name.toLowerCase -> values.asScala.mkString(",") }.toMap + received.add(Received(headers, body)) + location.foreach(exchange.getResponseHeaders.add("Location", _)) + val reply = s"""{"answer":$status}""".getBytes(StandardCharsets.UTF_8) + exchange.sendResponseHeaders(status, reply.length.toLong) + exchange.getResponseBody.write(reply) + exchange.close() + }) + server.start() + val url = s"http://127.0.0.1:${server.getAddress.getPort}/hooks/obp" + def stop(): Unit = server.stop(0) + } + + private def allowLocalReceivers(): Unit = { + setPropsValues("webhook.allow_http" -> "true", "webhook.allow_private_addresses" -> "true") + } + + private def outboxRow(eventId: String): MessageOutbox = + MessageOutbox.find(By(MessageOutbox.SubjectId, eventId)).openOrThrowException(s"outbox row $eventId") + + /** + * One delivery attempt of the row, as the relay makes it. The relay running in the test server may have + * made it already; the claim in the outbox lets only one of them send, so the result is the same. + */ + private def deliverOnce(eventId: String): MessageOutbox = { + val row = outboxRow(eventId) + if (row.status == MessageOutbox.STATUS_PENDING && row.attempts == 0) MessageOutboxRelay.relayRow(row) + outboxRow(eventId) + } + + /** A webhook at SYS sending to `url`, and the event id of the test request queued for it. */ + private def webhookWithTestRequest(entityId: String, url: String): (String, String) = { + val created = makePostRequest(webhooks(SYS, entityId).POST <@ (user1), webhookBody(url, List("CREATED"))) + withClue(created.body) { created.code should equal(201) } + val webhookId = (created.body \ "webhook_id").extract[String] + val queued = makePostRequest((webhooks(SYS, entityId) / webhookId / "test").POST <@ (user1), "") + withClue(queued.body) { queued.code should equal(201) } + (webhookId, (queued.body \ "event_id").extract[String]) + } + + private val approvedCondition: JValue = JArray(List(("field" -> "approved") ~ ("from" -> false) ~ ("to" -> true))) + + feature("Managing Dynamic Entity webhooks") { + + scenario("create, read, list, update and delete a webhook at SYS, with the Roles held at SYS", + ApiEndpoint1, ApiEndpoint2, ApiEndpoint3, ApiEndpoint4, ApiEndpoint5, ApiEndpoint6, VersionOfApi) { + val entityId = entityIn(None, s"loan_$suffix") + val url = receiverUrl("managed") + val body = webhookBody(url, List("CREATED", "UPDATED"), Some(approvedCondition), description = Some("For the test.")) + + When("an anonymous caller creates one") + makePostRequest(webhooks(SYS, entityId).POST, body).code should equal(401) + + When("user1 creates one without the Role") + val refused = makePostRequest(webhooks(SYS, entityId).POST <@ (user1), body) + refused.code should equal(403) + message(refused) should include(UserHasMissingRoles) + + grantAllAt(SYS) + + When("user1 creates one with the Role at SYS") + val created = makePostRequest(webhooks(SYS, entityId).POST <@ (user1), body) + withClue(created.body) { created.code should equal(201) } + val webhookId = (created.body \ "webhook_id").extract[String] + (created.body \ "bank_id").extract[String] should equal(SYS) + (created.body \ "dynamic_entity_id").extract[String] should equal(entityId) + (created.body \ "entity_name").extract[String] should equal(s"loan_$suffix") + (created.body \ "url").extract[String] should equal(url) + (created.body \ "events").extract[List[String]] should equal(List("CREATED", "UPDATED")) + (created.body \ "field_conditions") should equal(approvedCondition) + (created.body \ "is_active").extract[Boolean] should equal(true) + (created.body \ "deactivated_reason") should equal(JNothing) + (created.body \ "created_by_user_id").extract[String] should equal(resourceUser1.userId) + List("signed", "not_configured") should contain((created.body \ "signature_status").extract[String]) + + Then("it is listed for its entity and for its space, and can be read") + val listed = makeGetRequest(webhooks(SYS, entityId).GET <@ (user1)) + listed.code should equal(200) + webhookIdsIn(listed) should contain(webhookId) + val atBank = makeGetRequest(webhooksAtBank(SYS).GET <@ (user1)) + atBank.code should equal(200) + webhookIdsIn(atBank) should contain(webhookId) + makeGetRequest((webhooks(SYS, entityId) / webhookId).GET <@ (user1)).code should equal(200) + + When("only its description is changed") + val updated = makePutRequest((webhooks(SYS, entityId) / webhookId).PUT <@ (user1), + compact(render("description" -> "Changed."))) + withClue(updated.body) { updated.code should equal(200) } + Then("everything left out keeps its value") + (updated.body \ "description").extract[String] should equal("Changed.") + (updated.body \ "url").extract[String] should equal(url) + (updated.body \ "events").extract[List[String]] should equal(List("CREATED", "UPDATED")) + (updated.body \ "field_conditions") should equal(approvedCondition) + + When("it is deleted") + makeDeleteRequest((webhooks(SYS, entityId) / webhookId).DELETE <@ (user1)).code should equal(204) + Then("it is gone") + val gone = makeGetRequest((webhooks(SYS, entityId) / webhookId).GET <@ (user1)) + gone.code should equal(404) + message(gone) should include(DynamicEntityWebhookNotFound) + } + + scenario("a Role held at a bank covers that bank's entities, not the system space's", ApiEndpoint1, VersionOfApi) { + val bankId = testBankId1.value + val bankEntityId = entityIn(Some(bankId), s"bank_loan_$suffix") + val systemEntityId = entityIn(None, s"sys_loan_$suffix") + grantAllAt(bankId) + + val atBank = makePostRequest(webhooks(bankId, bankEntityId).POST <@ (user1), webhookBody(receiverUrl("bank"), List("CREATED"))) + withClue(atBank.body) { atBank.code should equal(201) } + (atBank.body \ "bank_id").extract[String] should equal(bankId) + + Then("the bank's Role does not reach the system space") + val atSystem = makePostRequest(webhooks(SYS, systemEntityId).POST <@ (user1), webhookBody(receiverUrl("bank"), List("CREATED"))) + atSystem.code should equal(403) + + Then("the system entity is not found through the bank") + val throughBank = makePostRequest(webhooks(bankId, systemEntityId).POST <@ (user1), webhookBody(receiverUrl("other"), List("CREATED"))) + throughBank.code should equal(404) + message(throughBank) should include(DynamicEntityNotFoundByDynamicEntityId) + } + + scenario("an unknown entity or webhook is a 404", ApiEndpoint1, ApiEndpoint4, VersionOfApi) { + grantAllAt(SYS) + val unknownEntity = makePostRequest(webhooks(SYS, "no-such-entity").POST <@ (user1), webhookBody(receiverUrl("x"), List("CREATED"))) + unknownEntity.code should equal(404) + message(unknownEntity) should include(DynamicEntityNotFoundByDynamicEntityId) + + val entityId = entityIn(None, s"lookup_$suffix") + val unknownWebhook = makeGetRequest((webhooks(SYS, entityId) / "no-such-webhook").GET <@ (user1)) + unknownWebhook.code should equal(404) + message(unknownWebhook) should include(DynamicEntityWebhookNotFound) + } + + scenario("bad events, field conditions, description and urls are refused with a 400", ApiEndpoint1, VersionOfApi) { + grantAllAt(SYS) + val entityId = entityIn(None, s"checks_$suffix") + def create(body: String) = makePostRequest(webhooks(SYS, entityId).POST <@ (user1), body) + def refusedWith(body: String, error: String): Unit = { + val response = create(body) + withClue(response.body) { response.code should equal(400) } + message(response) should include(error) + } + + When("the body is not JSON") + refusedWith("not json", InvalidJsonFormat) + + When("events is empty, unknown or repeated") + refusedWith(webhookBody(receiverUrl("e1"), Nil), InvalidDynamicEntityWebhookEvents) + refusedWith(webhookBody(receiverUrl("e2"), List("READ")), InvalidDynamicEntityWebhookEvents) + refusedWith(webhookBody(receiverUrl("e3"), List("CREATED", "CREATED")), InvalidDynamicEntityWebhookEvents) + + When("field conditions are given without UPDATED") + refusedWith(webhookBody(receiverUrl("c1"), List("CREATED"), Some(approvedCondition)), InvalidDynamicEntityWebhookFieldCondition) + When("a condition names a field the entity does not have") + refusedWith(webhookBody(receiverUrl("c2"), List("UPDATED"), Some(JArray(List("field" -> "colour")))), + InvalidDynamicEntityWebhookFieldCondition) + When("a value does not fit the field's type") + refusedWith(webhookBody(receiverUrl("c3"), List("UPDATED"), Some(JArray(List(("field" -> "amount") ~ ("to" -> "lots"))))), + InvalidDynamicEntityWebhookFieldCondition) + When("from and to are the same") + refusedWith(webhookBody(receiverUrl("c4"), List("UPDATED"), Some(JArray(List(("field" -> "status") ~ ("from" -> "OPEN") ~ ("to" -> "OPEN"))))), + InvalidDynamicEntityWebhookFieldCondition) + When("a field is named twice") + refusedWith(webhookBody(receiverUrl("c5"), List("UPDATED"), Some(JArray(List("field" -> "amount", "field" -> "amount")))), + InvalidDynamicEntityWebhookFieldCondition) + + When("the description is too long") + refusedWith(webhookBody(receiverUrl("d1"), List("CREATED"), description = Some("x" * 2001)), InvalidDynamicEntityWebhookDescription) + + When("the url is not https, carries a password, is relative, or points at a loopback or private address") + refusedWith(webhookBody(s"http://receiver-$suffix.invalid/hook", List("CREATED")), InvalidDynamicEntityWebhookUrl) + refusedWith(webhookBody(s"https://user:secret@receiver-$suffix.invalid/hook", List("CREATED")), InvalidDynamicEntityWebhookUrl) + refusedWith(webhookBody("/hook", List("CREATED")), InvalidDynamicEntityWebhookUrl) + refusedWith(webhookBody("https://127.0.0.1/hook", List("CREATED")), InvalidDynamicEntityWebhookUrl) + refusedWith(webhookBody("https://10.1.2.3/hook", List("CREATED")), InvalidDynamicEntityWebhookUrl) + refusedWith(webhookBody("https://169.254.169.254/latest/meta-data", List("CREATED")), InvalidDynamicEntityWebhookUrl) + refusedWith(webhookBody("https://[::1]/hook", List("CREATED")), InvalidDynamicEntityWebhookUrl) + + Then("a condition with only a field, or only a to value of the right type, is accepted") + val accepted = create(webhookBody(receiverUrl("ok"), List("UPDATED"), + Some(JArray(List("field" -> "amount", ("field" -> "status") ~ ("to" -> "CLOSED")))))) + withClue(accepted.body) { accepted.code should equal(201) } + } + + scenario("an entity has at most one webhook per url", ApiEndpoint1, ApiEndpoint5, VersionOfApi) { + grantAllAt(SYS) + val entityId = entityIn(None, s"unique_$suffix") + val first = makePostRequest(webhooks(SYS, entityId).POST <@ (user1), webhookBody(receiverUrl("one"), List("CREATED"))) + withClue(first.body) { first.code should equal(201) } + + When("a second webhook has the same url") + val duplicate = makePostRequest(webhooks(SYS, entityId).POST <@ (user1), webhookBody(receiverUrl("one"), List("DELETED"))) + duplicate.code should equal(409) + message(duplicate) should include(DynamicEntityWebhookUrlAlreadyExists) + + When("another webhook is moved onto that url") + val second = makePostRequest(webhooks(SYS, entityId).POST <@ (user1), webhookBody(receiverUrl("two"), List("DELETED"))) + withClue(second.body) { second.code should equal(201) } + val secondId = (second.body \ "webhook_id").extract[String] + val moved = makePutRequest((webhooks(SYS, entityId) / secondId).PUT <@ (user1), compact(render("url" -> receiverUrl("one")))) + moved.code should equal(409) + + Then("a webhook may keep its own url on update") + val kept = makePutRequest((webhooks(SYS, entityId) / secondId).PUT <@ (user1), compact(render("url" -> receiverUrl("two")))) + withClue(kept.body) { kept.code should equal(200) } + + And("another entity may use the same url") + val otherEntityId = entityIn(None, s"unique_other_$suffix") + makePostRequest(webhooks(SYS, otherEntityId).POST <@ (user1), webhookBody(receiverUrl("one"), List("CREATED"))).code should equal(201) + } + + scenario("turning a webhook off and on sets and clears its deactivated_reason", ApiEndpoint1, ApiEndpoint5, VersionOfApi) { + grantAllAt(SYS) + val entityId = entityIn(None, s"toggle_$suffix") + def put(webhookId: String, body: JValue) = makePutRequest((webhooks(SYS, entityId) / webhookId).PUT <@ (user1), compact(render(body))) + + When("a webhook is created inactive") + val created = makePostRequest(webhooks(SYS, entityId).POST <@ (user1), webhookBody(receiverUrl("toggle"), List("CREATED"), isActive = Some(false))) + withClue(created.body) { created.code should equal(201) } + val webhookId = (created.body \ "webhook_id").extract[String] + (created.body \ "is_active").extract[Boolean] should equal(false) + (created.body \ "deactivated_reason").extract[String] should equal("TURNED_OFF") + + When("it is turned on") + val on = put(webhookId, "is_active" -> true) + withClue(on.body) { on.code should equal(200) } + (on.body \ "is_active").extract[Boolean] should equal(true) + (on.body \ "deactivated_reason") should equal(JNothing) + + When("it is turned off") + val off = put(webhookId, "is_active" -> false) + (off.body \ "is_active").extract[Boolean] should equal(false) + (off.body \ "deactivated_reason").extract[String] should equal("TURNED_OFF") + + When("something else changes while it is off") + val stillOff = put(webhookId, "description" -> "Still off.") + (stillOff.body \ "is_active").extract[Boolean] should equal(false) + (stillOff.body \ "deactivated_reason").extract[String] should equal("TURNED_OFF") + } + + scenario("an empty field_conditions list removes the conditions on update", ApiEndpoint5, VersionOfApi) { + grantAllAt(SYS) + val entityId = entityIn(None, s"clear_$suffix") + val created = makePostRequest(webhooks(SYS, entityId).POST <@ (user1), + webhookBody(receiverUrl("clear"), List("UPDATED"), Some(approvedCondition))) + withClue(created.body) { created.code should equal(201) } + val webhookId = (created.body \ "webhook_id").extract[String] + + Then("events cannot drop UPDATED while conditions remain") + val dropUpdated = makePutRequest((webhooks(SYS, entityId) / webhookId).PUT <@ (user1), compact(render("events" -> List("CREATED")))) + dropUpdated.code should equal(400) + message(dropUpdated) should include(InvalidDynamicEntityWebhookFieldCondition) + + When("the conditions are cleared at the same time") + val cleared = makePutRequest((webhooks(SYS, entityId) / webhookId).PUT <@ (user1), + compact(render(("events" -> List("CREATED")) ~ ("field_conditions" -> JArray(Nil))))) + withClue(cleared.body) { cleared.code should equal(200) } + (cleared.body \ "field_conditions") should equal(JArray(Nil)) + } + } + feature("Delivering Dynamic Entity webhook requests") { + + scenario("the test endpoint queues a webhook.test request, which the relay posts with the OBP headers", + ApiEndpoint7, VersionOfApi) { + allowLocalReceivers() + grantAllAt(SYS) + val entityName = s"delivered_$suffix" + val entityId = entityIn(None, entityName) + val receiver = new Receiver(200) + try { + When("an anonymous caller asks for a test request") + val created = makePostRequest(webhooks(SYS, entityId).POST <@ (user1), webhookBody(receiver.url, List("CREATED"))) + withClue(created.body) { created.code should equal(201) } + val webhookId = (created.body \ "webhook_id").extract[String] + makePostRequest((webhooks(SYS, entityId) / webhookId / "test").POST, "").code should equal(401) + + When("user1 asks for one") + val queued = makePostRequest((webhooks(SYS, entityId) / webhookId / "test").POST <@ (user1), "") + withClue(queued.body) { queued.code should equal(201) } + val eventId = (queued.body \ "event_id").extract[String] + (queued.body \ "event_name").extract[String] should equal("webhook.test") + (queued.body \ "record_id") should equal(JNothing) + + When("the relay sends it") + val row = deliverOnce(eventId) + Then("it is delivered, once") + withClue(row.LastError.get) { row.status should equal(MessageOutbox.STATUS_DELIVERED) } + row.attempts should equal(1) + WebhookOutbox.lastReply(row).map(_.http_status) should equal(Some(200)) + receiver.received.size should equal(1) + + And("the request carries the OBP headers and identifiers only") + val request = receiver.received.peek() + request.headers("obp-webhook-id") should equal(webhookId) + request.headers("obp-event-id") should equal(eventId) + request.headers("obp-delivery-attempt") should equal("1") + request.headers("obp-signature-status") should equal(WebhookOutbox.signatureStatus) + request.headers.get("obp-signature").isDefined should equal(WebhookOutbox.isSigningConfigured) + request.headers("content-type") should startWith("application/json") + val body = org.json4s.native.JsonMethods.parse(request.body) + (body \ "event_name").extract[String] should equal("webhook.test") + (body \ "event_id").extract[String] should equal(eventId) + (body \ "webhook_id").extract[String] should equal(webhookId) + (body \ "bank_id").extract[String] should equal(SYS) + (body \ "entity_name").extract[String] should equal(entityName) + (body \ "occurred_at").extract[String] should fullyMatch regex """\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z""" + (body \ "record_id") should equal(JNothing) + } finally receiver.stop() + } + + scenario("a 503 reply is retried, and deleting the webhook drops the request", ApiEndpoint6, ApiEndpoint7, VersionOfApi) { + allowLocalReceivers() + grantAllAt(SYS) + val entityId = entityIn(None, s"unavailable_$suffix") + val receiver = new Receiver(503) + try { + val (webhookId, eventId) = webhookWithTestRequest(entityId, receiver.url) + val row = deliverOnce(eventId) + Then("it stays PENDING, with the attempt and the reply recorded") + row.status should equal(MessageOutbox.STATUS_PENDING) + row.attempts should equal(1) + WebhookOutbox.lastReply(row).map(_.http_status) should equal(Some(503)) + row.LastError.get should include("503") + + When("the webhook is deleted") + makeDeleteRequest((webhooks(SYS, entityId) / webhookId).DELETE <@ (user1)).code should equal(204) + Then("the request is no longer queued") + MessageOutbox.find(By(MessageOutbox.SubjectId, eventId)).isDefined should equal(false) + } finally receiver.stop() + } + + scenario("a 400 reply or a redirect is not retried", ApiEndpoint7, VersionOfApi) { + allowLocalReceivers() + grantAllAt(SYS) + val entityId = entityIn(None, s"refusing_$suffix") + val refusing = new Receiver(400) + val redirecting = new Receiver(302, location = Some("http://169.254.169.254/latest/meta-data")) + try { + val (_, refusedEventId) = webhookWithTestRequest(entityId, refusing.url) + val refused = deliverOnce(refusedEventId) + refused.status should equal(MessageOutbox.STATUS_STICKY) + WebhookOutbox.lastReply(refused).map(_.http_status) should equal(Some(400)) + + val (_, redirectedEventId) = webhookWithTestRequest(entityId, redirecting.url) + val redirected = deliverOnce(redirectedEventId) + redirected.status should equal(MessageOutbox.STATUS_STICKY) + WebhookOutbox.lastReply(redirected).map(_.http_status) should equal(Some(302)) + Then("the redirect was not followed") + redirecting.received.size should equal(1) + } finally { refusing.stop(); redirecting.stop() } + } + + scenario("a url that is refused by the time of delivery is not sent to", ApiEndpoint7, VersionOfApi) { + allowLocalReceivers() + grantAllAt(SYS) + val entityId = entityIn(None, s"refused_later_$suffix") + val receiver = new Receiver(200) + try { + val (_, eventId) = webhookWithTestRequest(entityId, receiver.url) + When("private addresses are no longer allowed when the request is sent") + setPropsValues("webhook.allow_private_addresses" -> "false") + val row = deliverOnce(eventId) + Then("the request is not sent, and is not retried") + row.status should equal(MessageOutbox.STATUS_STICKY) + row.LastError.get should include("refused") + receiver.received.size should equal(0) + } finally receiver.stop() + } + } + + feature("Following and retrying Dynamic Entity webhook deliveries") { + + scenario("a STICKY delivery is listed, can be retried once, and goes back to PENDING with its attempts reset", + ApiEndpoint8, ApiEndpoint9, VersionOfApi) { + allowLocalReceivers() + grantAllAt(SYS) + val entityId = entityIn(None, s"followed_$suffix") + val refusing = new Receiver(400) + try { + val (webhookId, eventId) = webhookWithTestRequest(entityId, refusing.url) + deliverOnce(eventId).status should equal(MessageOutbox.STATUS_STICKY) + val deliveries = webhooks(SYS, entityId) / webhookId / "deliveries" + + When("an anonymous caller lists the deliveries") + makeGetRequest(deliveries.GET).code should equal(401) + + When("user1 lists them") + val listed = makeGetRequest(deliveries.GET <@ (user1)) + withClue(listed.body) { listed.code should equal(200) } + val delivery = (listed.body \ "deliveries").children.find(d => (d \ "event_id").extract[String] == eventId) + .getOrElse(fail(s"delivery $eventId is not listed")) + (delivery \ "event_name").extract[String] should equal("webhook.test") + (delivery \ "status").extract[String] should equal("STICKY") + (delivery \ "attempts").extract[Int] should equal(1) + (delivery \ "last_http_status").extract[Int] should equal(400) + (delivery \ "last_error").extract[String] should include("400") + (delivery \ "record_id") should equal(JNothing) + + Then("the status filter narrows the list, and an unknown status is refused") + (makeGetRequest(deliveries.GET < "STICKY") <@ (user1)).body \ "deliveries").children should have size 1 + (makeGetRequest(deliveries.GET < "DELIVERED") <@ (user1)).body \ "deliveries").children should have size 0 + val badStatus = makeGetRequest(deliveries.GET < "LOST") <@ (user1)) + badStatus.code should equal(400) + message(badStatus) should include(InvalidUrlParameters) + + When("it is retried") + val retried = makePostRequest((deliveries / eventId / "retry").POST <@ (user1), "") + withClue(retried.body) { retried.code should equal(200) } + Then("it is PENDING again, with its attempts reset and the same event id") + (retried.body \ "status").extract[String] should equal("PENDING") + (retried.body \ "attempts").extract[Int] should equal(0) + (retried.body \ "event_id").extract[String] should equal(eventId) + + Then("a delivery that is not STICKY cannot be retried") + // The relay running in the test server may have sent it again already; either way it is not STICKY + // until that attempt has been made and refused, so retry the row only while it is PENDING. + if (outboxRow(eventId).status == MessageOutbox.STATUS_PENDING) { + val again = makePostRequest((deliveries / eventId / "retry").POST <@ (user1), "") + again.code should equal(400) + message(again) should include(DynamicEntityWebhookDeliveryNotSticky) + } + + And("an unknown delivery is a 404") + val unknown = makePostRequest((deliveries / "no-such-event" / "retry").POST <@ (user1), "") + unknown.code should equal(404) + message(unknown) should include(DynamicEntityWebhookDeliveryNotFound) + } finally refusing.stop() + } + } + + feature("Keeping webhooks in step with their Dynamic Entity's definition") { + + /** This saves the definition `dynamicEntityId` again, as `entityName` with `properties`, as every definition endpoint does. */ + def redefine(dynamicEntityId: String, entityName: String, properties: String): Unit = + DynamicEntityProvider.connectorMethodProvider.vend.createOrUpdate(DynamicEntityCommons( + entityName, s"""{"$entityName":{"properties":$properties}}""", Some(dynamicEntityId), owner, None, + hasPersonalEntity = false)).openOrThrowException("redefinition") + + def webhookAt(entityId: String, webhookId: String): JValue = { + val response = makeGetRequest((webhooks(SYS, entityId) / webhookId).GET <@ (user1)) + withClue(response.body) { response.code should equal(200) } + response.body + } + + def create(entityId: String, body: String): String = { + val created = makePostRequest(webhooks(SYS, entityId).POST <@ (user1), body) + withClue(created.body) { created.code should equal(201) } + (created.body \ "webhook_id").extract[String] + } + + scenario("removing or retyping a field a condition names turns the webhook off; other webhooks are left alone", + ApiEndpoint4, ApiEndpoint5, VersionOfApi) { + grantAllAt(SYS) + val entityName = s"redefined_$suffix" + val entityId = entityIn(None, entityName) + val onApproved = create(entityId, webhookBody(receiverUrl("approved"), List("UPDATED"), Some(approvedCondition))) + val onAmount = create(entityId, webhookBody(receiverUrl("amount"), List("UPDATED"), Some(JArray(List("field" -> "amount"))))) + val unconditional = create(entityId, webhookBody(receiverUrl("all"), List("CREATED", "UPDATED"))) + val alreadyOff = create(entityId, webhookBody(receiverUrl("off"), List("UPDATED"), Some(approvedCondition), isActive = Some(false))) + + When("the definition drops approved") + redefine(entityId, entityName, """{"amount":{"type":"number","example":"1"},"status":{"type":"string","example":"OPEN"}}""") + Then("the active webhook whose condition names approved is turned off, saying why") + (webhookAt(entityId, onApproved) \ "is_active").extract[Boolean] should equal(false) + (webhookAt(entityId, onApproved) \ "deactivated_reason").extract[String] should equal("ENTITY_FIELD_REMOVED") + And("the others are as they were") + (webhookAt(entityId, onAmount) \ "is_active").extract[Boolean] should equal(true) + (webhookAt(entityId, unconditional) \ "is_active").extract[Boolean] should equal(true) + (webhookAt(entityId, alreadyOff) \ "deactivated_reason").extract[String] should equal("TURNED_OFF") + + When("the definition changes amount from a number to a string") + redefine(entityId, entityName, """{"amount":{"type":"string","example":"1"},"status":{"type":"string","example":"OPEN"}}""") + Then("the webhook whose condition names amount is turned off, saying why") + (webhookAt(entityId, onAmount) \ "deactivated_reason").extract[String] should equal("ENTITY_FIELD_TYPE_CHANGED") + (webhookAt(entityId, unconditional) \ "is_active").extract[Boolean] should equal(true) + + When("the owner turns the first back on without fixing its condition") + val unfixed = makePutRequest((webhooks(SYS, entityId) / onApproved).PUT <@ (user1), compact(render("is_active" -> true))) + Then("it is refused") + unfixed.code should equal(400) + message(unfixed) should include(InvalidDynamicEntityWebhookFieldCondition) + + When("the owner fixes the condition and turns it back on") + val fixed = makePutRequest((webhooks(SYS, entityId) / onApproved).PUT <@ (user1), + compact(render(("is_active" -> true) ~ ("field_conditions" -> JArray(List(("field" -> "status") ~ ("to" -> "CLOSED"))))))) + withClue(fixed.body) { fixed.code should equal(200) } + (fixed.body \ "is_active").extract[Boolean] should equal(true) + (fixed.body \ "deactivated_reason") should equal(JNothing) + } + + scenario("renaming an entity renames it on its webhooks, and deleting it deletes them", ApiEndpoint3, VersionOfApi) { + grantAllAt(SYS) + val entityId = entityIn(None, s"renamed_$suffix") + val webhookId = create(entityId, webhookBody(receiverUrl("renamed"), List("CREATED"))) + + When("the entity is renamed") + redefine(entityId, s"renamed_again_$suffix", + """{"approved":{"type":"boolean","example":"false"},"amount":{"type":"number","example":"1"},"status":{"type":"string","example":"OPEN"}}""") + Then("its webhook carries the new name") + (webhookAt(entityId, webhookId) \ "entity_name").extract[String] should equal(s"renamed_again_$suffix") + + When("the entity is deleted") + val entity = DynamicEntityProvider.connectorMethodProvider.vend.getById(None, entityId).openOrThrowException("entity") + DynamicEntityProvider.connectorMethodProvider.vend.delete(entity).openOrThrowException("delete") should equal(true) + Then("its webhook is gone") + webhookIdsIn(makeGetRequest(webhooksAtBank(SYS).GET <@ (user1))) should not contain webhookId + } + } + + feature("Firing Dynamic Entity webhooks when records are written") { + + def createWebhook(entityId: String, body: String): String = { + val created = makePostRequest(webhooks(SYS, entityId).POST <@ (user1), body) + withClue(created.body) { created.code should equal(201) } + (created.body \ "webhook_id").extract[String] + } + + /** The requests queued for a webhook, oldest first, with their bodies parsed. */ + def queuedFor(webhookId: String): List[(MessageOutbox, JValue)] = + MessageOutbox.findAll(By(MessageOutbox.OutboxType, MessageOutbox.TYPE_WEBHOOK), By(MessageOutbox.TargetId, webhookId)) + .sortBy(_.id.get) + .map(row => row -> org.json4s.native.JsonMethods.parse(row.payloadJson)) + + def record(entityName: String, id: String, fields: (String, JValue)*): JObject = + JObject(JField(s"${entityName}_id", JString(id)) :: fields.map { case (name, value) => JField(name, value) }.toList) + + def save(entityName: String, body: JObject): Unit = + DynamicDataProvider.connectorMethodProvider.vend.save(None, entityName, body, Some(resourceUser1.userId), false) + .openOrThrowException("save the record") + + def update(entityName: String, id: String, body: JObject): Unit = + DynamicDataProvider.connectorMethodProvider.vend.update(None, entityName, body, id, Some(resourceUser1.userId), false) + .openOrThrowException("update the record") + + scenario("creating, updating and deleting a record queues one request each, carrying identifiers only", VersionOfApi) { + grantAllAt(SYS) + val entityName = s"fired_$suffix" + val entityId = entityIn(None, entityName) + val webhookId = createWebhook(entityId, webhookBody(receiverUrl("fired"), List("CREATED", "UPDATED", "DELETED"))) + val createdOnly = createWebhook(entityId, webhookBody(receiverUrl("created-only"), List("CREATED"))) + val recordId = java.util.UUID.randomUUID().toString + + When("a record is created, updated and deleted") + save(entityName, record(entityName, recordId, "status" -> JString("OPEN"), "amount" -> JDouble(10.5))) + update(entityName, recordId, record(entityName, recordId, "status" -> JString("CLOSED"), "amount" -> JDouble(10.5))) + DynamicDataProvider.connectorMethodProvider.vend.delete(None, entityName, recordId, Some(resourceUser1.userId), false) + .openOrThrowException("delete the record") should equal(true) + + Then("the webhook subscribed to all three has one request for each, in order") + val queued = queuedFor(webhookId) + queued.map(_._1.operationName) should equal(List( + "dynamic_entity.record.created", "dynamic_entity.record.updated", "dynamic_entity.record.deleted")) + queued.map(_._1.status).distinct.foreach(status => List("PENDING", "STICKY") should contain(status)) + And("each body names the record and carries none of its values") + queued.foreach { case (row, body) => + (body \ "record_id").extract[String] should equal(recordId) + (body \ "event_id").extract[String] should equal(row.subjectId) + (body \ "entity_name").extract[String] should equal(entityName) + (body \ "bank_id").extract[String] should equal(SYS) + (body \ "record_updated_at").extract[String] should fullyMatch regex """\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z""" + row.payloadJson should not include "CLOSED" + row.payloadJson should not include "10.5" + (body \ "matched_conditions") should equal(JNothing) + } + And("the webhook subscribed to CREATED only has one request") + queuedFor(createdOnly).map(_._1.operationName) should equal(List("dynamic_entity.record.created")) + } + + scenario("an update fires a webhook with field conditions only when one of them matches", VersionOfApi) { + grantAllAt(SYS) + val entityName = s"conditional_$suffix" + val entityId = entityIn(None, entityName) + val onApproval = createWebhook(entityId, webhookBody(receiverUrl("approval"), List("CREATED", "UPDATED"), Some(approvedCondition))) + val recordId = java.util.UUID.randomUUID().toString + + When("a record is created") + save(entityName, record(entityName, recordId, "approved" -> JBool(false), "status" -> JString("OPEN"))) + Then("it is a CREATED request, not a match") + queuedFor(onApproval).map(_._1.operationName) should equal(List("dynamic_entity.record.created")) + + When("an update leaves approved false") + update(entityName, recordId, record(entityName, recordId, "approved" -> JBool(false), "status" -> JString("REVIEW"))) + Then("nothing more is queued") + queuedFor(onApproval) should have size 1 + + When("an update turns approved from false to true, given as the string \"true\"") + update(entityName, recordId, record(entityName, recordId, "approved" -> JString("true"), "status" -> JString("REVIEW"))) + Then("an UPDATED request is queued, naming the condition that matched") + val (row, body) = queuedFor(onApproval).last + row.operationName should equal("dynamic_entity.record.updated") + (body \ "matched_conditions") should equal(approvedCondition) + + When("approved is set to true again") + update(entityName, recordId, record(entityName, recordId, "approved" -> JBool(true), "status" -> JString("DONE"))) + Then("nothing more is queued: approved did not change") + queuedFor(onApproval) should have size 2 + } + + scenario("a webhook queued for more than the per-minute limit is turned off by the circuit breaker", VersionOfApi) { + grantAllAt(SYS) + setPropsValues("webhook.circuit_breaker_per_minute" -> "2") + val entityName = s"busy_$suffix" + val entityId = entityIn(None, entityName) + val webhookId = createWebhook(entityId, webhookBody(receiverUrl("busy"), List("CREATED"))) + + When("three records are created within the minute") + (1 to 3).foreach(_ => save(entityName, record(entityName, java.util.UUID.randomUUID().toString, "status" -> JString("OPEN")))) + Then("two requests are queued, and the webhook is off with reason CIRCUIT_BREAKER") + queuedFor(webhookId) should have size 2 + val webhook = makeGetRequest((webhooks(SYS, entityId) / webhookId).GET <@ (user1)) + (webhook.body \ "is_active").extract[Boolean] should equal(false) + (webhook.body \ "deactivated_reason").extract[String] should equal("CIRCUIT_BREAKER") + + When("a fourth record is created") + save(entityName, record(entityName, java.util.UUID.randomUUID().toString, "status" -> JString("OPEN"))) + Then("nothing more is queued for the webhook that is off") + queuedFor(webhookId) should have size 2 + } + } + + feature("Signing Dynamic Entity webhook requests") { + + scenario("a signature verifies against the public key and the exact body, and nothing else", VersionOfApi) { + val generator = KeyPairGenerator.getInstance("RSA") + generator.initialize(2048) + val keyPair = generator.generateKeyPair() + val publicKey = keyPair.getPublic.asInstanceOf[RSAPublicKey] + val body = """{"event_name":"webhook.test","event_id":"e1"}""".getBytes(StandardCharsets.UTF_8) + + val signature = WebhookOutbox.sign(body, keyPair.getPrivate.asInstanceOf[RSAPrivateKey], publicKey, 1760000000L) + + Then("its payload part is empty: the body is the payload") + signature.split("\\.", -1).toList.map(_.isEmpty) should equal(List(false, true, false)) + And("it verifies against the body, and names the key and its own type") + val verified = JWSObject.parse(signature, new Payload(body)) + verified.verify(new RSASSAVerifier(publicKey)) should equal(true) + verified.getHeader.getKeyID should equal(new RSAKey.Builder(publicKey).keyIDFromThumbprint().build().getKeyID) + verified.getHeader.getType.getType should equal("obp-webhook+jwt") + verified.getHeader.getCustomParam("iat").toString should equal("1760000000") + And("it does not verify against any other body") + val tampered = JWSObject.parse(signature, new Payload("""{"event_name":"webhook.test","event_id":"e2"}""")) + tampered.verify(new RSASSAVerifier(publicKey)) should equal(false) + } + } +}