From db61309972f03c038123cc87b5418acde788e932 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Fri, 14 Aug 2026 10:06:59 +0200 Subject: [PATCH 01/26] protocol: add v2 protocol design --- doc/design/v2/README.md | 13 + doc/design/v2/udf_protocol_call_lifecycle.md | 96 +++++++ doc/design/v2/udf_protocol_call_lifecycle.mmd | 21 ++ doc/design/v2/udf_protocol_call_lifecycle.svg | 1 + doc/design/v2/udf_protocol_data_stream.md | 130 +++++++++ .../v2/udf_protocol_data_stream_flow.mmd | 35 +++ .../v2/udf_protocol_data_stream_flow.svg | 1 + doc/design/v2/udf_protocol_design_draft.md | 249 ++++++++++++++++++ .../v2/udf_protocol_high_level_call_model.mmd | 16 ++ .../v2/udf_protocol_high_level_call_model.svg | 1 + .../v2/udf_protocol_high_level_calls.md | 127 +++++++++ ...rotocol_high_level_endpoint_scheduling.mmd | 22 ++ ...rotocol_high_level_endpoint_scheduling.svg | 1 + .../udf_protocol_high_level_nested_calls.mmd | 17 ++ .../udf_protocol_high_level_nested_calls.svg | 1 + .../udf_protocol_high_level_run_sequence.mmd | 13 + .../udf_protocol_high_level_run_sequence.svg | 1 + doc/design/v2/udf_protocol_low_level.md | 148 +++++++++++ ...rotocol_low_level_connection_lifecycle.mmd | 16 ++ ...rotocol_low_level_connection_lifecycle.svg | 1 + .../v2/udf_transport_technology_overview.md | 113 ++++++++ udf-runner-cpp/v2/udf_protocol.fbs | 227 ++++++++++++++++ 22 files changed, 1250 insertions(+) create mode 100644 doc/design/v2/README.md create mode 100644 doc/design/v2/udf_protocol_call_lifecycle.md create mode 100644 doc/design/v2/udf_protocol_call_lifecycle.mmd create mode 100644 doc/design/v2/udf_protocol_call_lifecycle.svg create mode 100644 doc/design/v2/udf_protocol_data_stream.md create mode 100644 doc/design/v2/udf_protocol_data_stream_flow.mmd create mode 100644 doc/design/v2/udf_protocol_data_stream_flow.svg create mode 100644 doc/design/v2/udf_protocol_design_draft.md create mode 100644 doc/design/v2/udf_protocol_high_level_call_model.mmd create mode 100644 doc/design/v2/udf_protocol_high_level_call_model.svg create mode 100644 doc/design/v2/udf_protocol_high_level_calls.md create mode 100644 doc/design/v2/udf_protocol_high_level_endpoint_scheduling.mmd create mode 100644 doc/design/v2/udf_protocol_high_level_endpoint_scheduling.svg create mode 100644 doc/design/v2/udf_protocol_high_level_nested_calls.mmd create mode 100644 doc/design/v2/udf_protocol_high_level_nested_calls.svg create mode 100644 doc/design/v2/udf_protocol_high_level_run_sequence.mmd create mode 100644 doc/design/v2/udf_protocol_high_level_run_sequence.svg create mode 100644 doc/design/v2/udf_protocol_low_level.md create mode 100644 doc/design/v2/udf_protocol_low_level_connection_lifecycle.mmd create mode 100644 doc/design/v2/udf_protocol_low_level_connection_lifecycle.svg create mode 100644 doc/design/v2/udf_transport_technology_overview.md create mode 100644 udf-runner-cpp/v2/udf_protocol.fbs diff --git a/doc/design/v2/README.md b/doc/design/v2/README.md new file mode 100644 index 0000000..aa05f0c --- /dev/null +++ b/doc/design/v2/README.md @@ -0,0 +1,13 @@ +# UDF Protocol v2 Design Documents + +This directory contains the split protocol design for the new UDF protocol. + +- [udf_protocol_design_draft.md](udf_protocol_design_draft.md) is the umbrella design draft. +- [udf_protocol_low_level.md](udf_protocol_low_level.md) describes the wire-level and control-stream rules. +- [udf_protocol_call_lifecycle.md](udf_protocol_call_lifecycle.md) describes the generic call abstraction. +- [udf_protocol_data_stream.md](udf_protocol_data_stream.md) is the low-level companion for call-attached + data-stream behavior. +- [udf_protocol_high_level_calls.md](udf_protocol_high_level_calls.md) describes high-level calls such as `Run`, + `Function`, `Script`, `get_connection`, `get_script`, and `execute_query`, plus DB/UDFRunner scheduling policy. + +Mermaid sources and rendered SVGs use matching names and scopes so the textual and visual material stays aligned. diff --git a/doc/design/v2/udf_protocol_call_lifecycle.md b/doc/design/v2/udf_protocol_call_lifecycle.md new file mode 100644 index 0000000..0031764 --- /dev/null +++ b/doc/design/v2/udf_protocol_call_lifecycle.md @@ -0,0 +1,96 @@ +# UDF Protocol v2: Generic Call Lifecycle + +This document describes the generic `Call` abstraction shared by all protocol interactions. It intentionally +excludes wire-level framing details and command-specific behavior. + +## Scope + +This document covers: + +- the generic `Call` abstraction +- normalized message labels +- generic call state transitions +- nested calls as a protocol mechanism + +Related diagram: + +- [udf_protocol_call_lifecycle.svg](udf_protocol_call_lifecycle.svg) + +## Call Abstraction + +A `Call` is the protocol's unit of interaction between two endpoints. + +- a call is opened with an open message +- a call is closed with a close message +- named payloads or other call-scoped metadata may flow while the call is active +- a call may spawn nested calls while remaining active +- a call may have at most one bidirectional data stream attached to it + +Either side may open a call. Transport-level client/server roles do not restrict call initiation. +Each call's stream is identified by `(connection, stream_id)`; the same numeric ID on another connection identifies +a different stream. + +## Normalized Documentation Labels + +The current docs use these normalized labels to describe call behavior without freezing final wire names: + +- `CallOpen(name, metadata)` +- `first call-scoped message` +- `CallClose`, optionally with `Error` +- `Payloads(...)` + +These are documentation aliases, not a second schema. + +## Endpoint State Models + +The state models use one local endpoint's perspective. A locally opened call and a peer-opened call have separate +opening paths; the peer runs the complementary model. + +### Locally Opened Call + +| State | Meaning | +| --- | --- | +| `LocallyOpenedIdle` | The local endpoint has not opened the call. | +| `WaitingForPeerCallTraffic` | The local endpoint sent `CallOpen(...)` and awaits the first call-scoped message. | +| `LocallyOpenedActive` | The call is active at the local endpoint. | +| `LocallyOpenedClosed` | The local endpoint has handled `CallClose`. | + +| From | Local event | To | Notes | +| --- | --- | --- | --- | +| `LocallyOpenedIdle` | Send `CallOpen(name, metadata)` | `WaitingForPeerCallTraffic` | Either endpoint may open a call. | +| `WaitingForPeerCallTraffic` | Receive first call traffic | `LocallyOpenedActive` | No dedicated accept frame is required. | +| `LocallyOpenedActive` | Handle call traffic | `LocallyOpenedActive` | Send or receive `Payloads(...)` and nested `CallOpen(...)` traffic. | +| `LocallyOpenedActive` | Handle `CallClose` | `LocallyOpenedClosed` | Sending or receiving it closes the call locally. | + +### Peer-Opened Call + +| State | Meaning | +| --- | --- | +| `PeerOpenedIdle` | The local endpoint has not received a call open. | +| `PeerOpenedActive` | The call is active at the local endpoint. | +| `PeerOpenedClosed` | The local endpoint has handled `CallClose`. | + +| From | Local event | To | Notes | +| --- | --- | --- | --- | +| `PeerOpenedIdle` | Receive `CallOpen(name, metadata)` | `PeerOpenedActive` | Acceptance may be explicit or inferred later. | +| `PeerOpenedActive` | Handle call traffic | `PeerOpenedActive` | Send or receive `Payloads(...)` and nested `CallOpen(...)` traffic. | +| `PeerOpenedActive` | Handle `CallClose` | `PeerOpenedClosed` | Sending or receiving it closes the call locally. | + +## Nested Calls + +Nested calls are a generic mechanism, not a special-case callback transport: + +- the parent call stays active while nested calls execute +- nested calls use their own connection-scoped `stream_id` +- ordering and priority between a parent call and its nested calls are not fully defined beyond the scheduling policy + documented in the high-level calls document + +## Relationship To Data Streams + +This document only records that a call may carry at most one bidirectional data stream. The detailed data-stream +rules are defined in [udf_protocol_data_stream.md](udf_protocol_data_stream.md). + +## Open Questions + +- precise command-selection fields carried in opening metadata +- stronger ordering guarantees between a call and the nested calls it spawns diff --git a/doc/design/v2/udf_protocol_call_lifecycle.mmd b/doc/design/v2/udf_protocol_call_lifecycle.mmd new file mode 100644 index 0000000..ece9703 --- /dev/null +++ b/doc/design/v2/udf_protocol_call_lifecycle.mmd @@ -0,0 +1,21 @@ +stateDiagram-v2 + state "Locally opened call" as LocallyOpened { + [*] --> LocallyOpenedIdle + + LocallyOpenedIdle --> WaitingForPeerCallTraffic: Send CallOpen + WaitingForPeerCallTraffic --> LocallyOpenedActive: Receive first call traffic + LocallyOpenedActive --> LocallyOpenedActive: Handle call traffic + LocallyOpenedActive --> LocallyOpenedClosed: Handle CallClose + + LocallyOpenedClosed --> [*] + } + + state "Peer-opened call" as PeerOpened { + [*] --> PeerOpenedIdle + + PeerOpenedIdle --> PeerOpenedActive: Receive CallOpen + PeerOpenedActive --> PeerOpenedActive: Handle call traffic + PeerOpenedActive --> PeerOpenedClosed: Handle CallClose + + PeerOpenedClosed --> [*] + } diff --git a/doc/design/v2/udf_protocol_call_lifecycle.svg b/doc/design/v2/udf_protocol_call_lifecycle.svg new file mode 100644 index 0000000..7fc2b39 --- /dev/null +++ b/doc/design/v2/udf_protocol_call_lifecycle.svg @@ -0,0 +1 @@ +

Locally opened call

Send CallOpen

Receive first call traffic

Handle call traffic

Handle CallClose

LocallyOpenedIdle

WaitingForPeerCallTraffic

LocallyOpenedActive

LocallyOpenedClosed

Peer-opened call

Receive CallOpen

Handle call traffic

Handle CallClose

PeerOpenedIdle

PeerOpenedActive

PeerOpenedClosed

 \ No newline at end of file diff --git a/doc/design/v2/udf_protocol_data_stream.md b/doc/design/v2/udf_protocol_data_stream.md new file mode 100644 index 0000000..e2b1599 --- /dev/null +++ b/doc/design/v2/udf_protocol_data_stream.md @@ -0,0 +1,130 @@ +# UDF Protocol v2: Data Stream + +This document describes the generic low-level data-stream behavior attached to a call. It excludes framing details +and high-level command semantics. + +## Scope + +This document covers: + +- the one-data-stream-per-call model +- per-direction schema behavior +- `Next(byte_budget, reset, row_id)` flow control +- `RecordBatch` sequencing rules +- completion behavior and open questions + +Related diagram: + +- [udf_protocol_data_stream_flow.svg](udf_protocol_data_stream_flow.svg) + +## Model + +- a data stream exists only attached to a call +- a call may have at most one bidirectional data stream +- each side owns one logical direction within that data stream +- ordering is total within one direction +- the two directions do not need to share a schema + +## Schema and First Batch Behavior + +Each direction announces its own schema. The normative schema and first-batch requirements are defined in +[Rules](#rules). + +## `Next(byte_budget, reset, row_id)` + +`Next(...)` is the transfer-credit mechanism. + +- `byte_budget` is a byte budget, not a row count +- `reset` indicates that transfer should resume from `row_id` +- `row_id` is a seek position for resumed transfer, not a per-batch correlation field + +This `row_id` usage is distinct from any row correlation carried inside the data itself for high-level call +semantics such as scalar-return `Run`. + +## Endpoint State Models + +The state models use one local endpoint's perspective. That endpoint has one outbound direction and one inbound +direction; its peer runs the same two models with the directions reversed. + +### Outbound Direction + +The outbound model tracks data sent by the local endpoint and `Next(...)` credit received from its peer. + +| State | Meaning | +| --- | --- | +| `OutboundIdle` | The local endpoint has sent neither schema nor batch data. | +| `SchemaAnnounced` | The local endpoint sent its schema, but not its first batch. | +| `FirstBatchCreditGranted` | The peer granted credit before the local endpoint announced its schema. | +| `CreditGranted` | The peer granted credit after the schema or a prior batch. | +| `WaitingForNext` | The local endpoint needs additional credit before sending another batch. | +| `OutboundCompleted` | The local endpoint has no more outbound data. | + +| From | Local event | To | +| --- | --- | --- | +| `OutboundIdle` | Send schema | `SchemaAnnounced` | +| `OutboundIdle` | Send schema + first batch | `WaitingForNext` | +| `SchemaAnnounced` | Send first batch | `WaitingForNext` | +| `OutboundIdle` | Receive `Next(...)` | `FirstBatchCreditGranted` | +| `SchemaAnnounced` | Receive `Next(...)` | `CreditGranted` | +| `FirstBatchCreditGranted` | Send schema + first batch | `WaitingForNext` | +| `CreditGranted` | Send batch | `WaitingForNext` | +| `WaitingForNext` | Receive `Next(...)` | `CreditGranted` | +| `WaitingForNext` | Finish stream | `OutboundCompleted` | + +### Inbound Direction + +The inbound model tracks data received by the local endpoint and `Next(...)` credit it sends to its peer. + +| State | Meaning | +| --- | --- | +| `InboundIdle` | The local endpoint received neither schema nor batch data. | +| `SchemaReceived` | The local endpoint received the peer's schema, but not its first batch. | +| `WaitingForFirstBatch` | The local endpoint sent `Next(...)` and awaits the first batch. | +| `ReadyToRequest` | The local endpoint received a batch and may request another. | +| `WaitingForBatch` | The local endpoint sent `Next(...)` and awaits a non-first batch. | +| `InboundCompleted` | The local endpoint expects no more inbound data. | + +| From | Local event | To | +| --- | --- | --- | +| `InboundIdle` | Receive schema | `SchemaReceived` | +| `InboundIdle` | Receive schema + first batch | `ReadyToRequest` | +| `InboundIdle` | Send `Next(...)` | `WaitingForFirstBatch` | +| `SchemaReceived` | Receive first batch | `ReadyToRequest` | +| `SchemaReceived` | Send `Next(...)` | `WaitingForFirstBatch` | +| `WaitingForFirstBatch` | Receive schema | `WaitingForFirstBatch` | +| `WaitingForFirstBatch` | Receive schema + first batch | `ReadyToRequest` | +| `WaitingForFirstBatch` | Receive first batch | `ReadyToRequest` | +| `ReadyToRequest` | Send `Next(...)` | `WaitingForBatch` | +| `WaitingForBatch` | Receive batch | `ReadyToRequest` | +| `WaitingForFirstBatch` | Finish stream | `InboundCompleted` | +| `WaitingForBatch` | Finish stream | `InboundCompleted` | + +## Rules + +1. Each direction sends exactly one `DataSchema`. +2. A direction sends its schema before, or in the same `StreamMessage` as, its first `DataRecordBatch`. +3. A standalone schema may precede `Next(...)`; no batch may precede its schema. +4. Only the first batch in a direction may precede `Next(...)`. +5. Non-first batches require prior transfer credit. +6. Except for the permitted first batch, a sender must not exceed its peer's granted `Next(...)` credit. +7. `byte_budget` bounds the transfer window. +8. `reset` and `row_id` describe resume position only. + +## Buffer Transfer + +`DataRecordBatch.buffer_transport` selects how the batch buffers identified by its metadata are delivered. + +- `Inline` is supported by every transport binding and is the only permitted mode on TCP/TLS. +- `Memfd` and `OutOfBand` are reserved for future Unix-domain-socket modes that require file-descriptor passing. +- An implementation that cannot establish compatible local handoff must select `Inline`; no fallback is implied + after a batch has been announced with another mode. +- The metadata and any referenced local buffer must be validated as one integrity boundary before the batch is + consumed. + +## Open Questions + +- exact end-of-stream marker +- exact semantics of `reset` +- exact encoding of schema plus first-batch bundling +- descriptor-to-batch correlation, ownership, lifetime, and cleanup for Unix file-descriptor handoff +- GPU-memory handle types and validation rules for future `OutOfBand` transfer diff --git a/doc/design/v2/udf_protocol_data_stream_flow.mmd b/doc/design/v2/udf_protocol_data_stream_flow.mmd new file mode 100644 index 0000000..ea5f879 --- /dev/null +++ b/doc/design/v2/udf_protocol_data_stream_flow.mmd @@ -0,0 +1,35 @@ +stateDiagram-v2 + state "Outbound direction" as Outbound { + [*] --> OutboundIdle + + OutboundIdle --> SchemaAnnounced: Send schema + OutboundIdle --> WaitingForNext: Send schema + first batch + SchemaAnnounced --> WaitingForNext: Send first batch + OutboundIdle --> FirstBatchCreditGranted: Receive Next + SchemaAnnounced --> CreditGranted: Receive Next + FirstBatchCreditGranted --> WaitingForNext: Send schema + first batch + CreditGranted --> WaitingForNext: Send batch + WaitingForNext --> CreditGranted: Receive Next + WaitingForNext --> OutboundCompleted: Finish stream + + OutboundCompleted --> [*] + } + + state "Inbound direction" as Inbound { + [*] --> InboundIdle + + InboundIdle --> SchemaReceived: Receive schema + InboundIdle --> ReadyToRequest: Receive schema + first batch + InboundIdle --> WaitingForFirstBatch: Send Next + SchemaReceived --> ReadyToRequest: Receive first batch + SchemaReceived --> WaitingForFirstBatch: Send Next + WaitingForFirstBatch --> WaitingForFirstBatch: Receive schema + WaitingForFirstBatch --> ReadyToRequest: Receive schema + first batch + WaitingForFirstBatch --> ReadyToRequest: Receive first batch + ReadyToRequest --> WaitingForBatch: Send Next + WaitingForBatch --> ReadyToRequest: Receive batch + WaitingForFirstBatch --> InboundCompleted: Finish stream + WaitingForBatch --> InboundCompleted: Finish stream + + InboundCompleted --> [*] + } diff --git a/doc/design/v2/udf_protocol_data_stream_flow.svg b/doc/design/v2/udf_protocol_data_stream_flow.svg new file mode 100644 index 0000000..51527cf --- /dev/null +++ b/doc/design/v2/udf_protocol_data_stream_flow.svg @@ -0,0 +1 @@ +

Outbound direction

Send schema

Send schema + first batch

Send first batch

Receive Next

Receive Next

Send schema + first batch

Send batch

Receive Next

Finish stream

OutboundIdle

SchemaAnnounced

WaitingForNext

FirstBatchCreditGranted

CreditGranted

OutboundCompleted

Inbound direction

Receive schema

Receive schema + first batch

Send Next

Receive first batch

Send Next

Receive schema

Receive schema + first batch

Receive first batch

Send Next

Receive batch

Finish stream

Finish stream

InboundIdle

SchemaReceived

ReadyToRequest

WaitingForFirstBatch

WaitingForBatch

InboundCompleted

 \ No newline at end of file diff --git a/doc/design/v2/udf_protocol_design_draft.md b/doc/design/v2/udf_protocol_design_draft.md new file mode 100644 index 0000000..5b009f6 --- /dev/null +++ b/doc/design/v2/udf_protocol_design_draft.md @@ -0,0 +1,249 @@ +# New UDF Protocol Design Draft + +Detailed protocol references in this directory: + +- [udf_protocol_low_level.md](udf_protocol_low_level.md) +- [udf_protocol_call_lifecycle.md](udf_protocol_call_lifecycle.md) +- [udf_protocol_data_stream.md](udf_protocol_data_stream.md) +- [udf_protocol_high_level_calls.md](udf_protocol_high_level_calls.md) + +# General Information + +| **Status** | Draft | + +**Design Workflow Steps**: Brainstorming (Team + Architects) -> Write/Update Design -> First Feedback Round (Team + Architects) -> Incorporate Feedback -> Design Feedback Group + +# Input From User Perspective + +This design introduces a new UDF protocol for communication between `DB` and `UDFRunner`. The user-visible goal is to remove current protocol limitations that make UDF execution slower, harder to evolve, and difficult to secure for remote execution scenarios. + +## Problem Statement + +The current UDF protocol has several practical problems: + +- It is synchronous and therefore inefficient for workloads that would benefit from independent send and receive activity. +- It is hard to evolve because changes to call metadata, callback metadata, import/export specification, and connection information tend to require recompilation. +- It forces a request/response pattern that creates unnecessary round trips. For example, small scalar-return batches can require four round trips. +- It uses serialization that is too slow for the intended workload, especially for table data. +- It only supports a single connection and therefore does not scale well to more advanced execution patterns such as multiple streams and `pquery`. +- It is not sufficiently secure for remote connections. + +These problems matter most for containerized or remote UDF execution, where protocol overhead, security requirements, and extensibility all become first-order concerns. + +## User Requirements + +The new UDF protocol should satisfy the following requirements. + +### Core Functional Requirements + +- Support `Run`, `Function`, and `Script` execution modes. +- Support metadata exchange for protocol, script, call, and callback interactions. +- Support DB callback operations such as connection lookup, script lookup, and query execution. +- Support multiple logical streams per physical connection, with total ordering within a stream and partial ordering across streams. +- Support flow-controlled table transfer with explicit request semantics such as `Next(...)`. + +### Performance Requirements + +- Be faster than the current protocol for both small and large batches. +- Reduce unnecessary round trips, especially for scalar or small-batch results. +- Support asynchronous communication so each side can send and receive independently. +- Support efficient transport of table data with minimal copying where practical. +- Handle large rows and large values, including potentially unbounded binary values, without forcing unsafe or impractical buffering strategies. + +### Extensibility Requirements + +- Make it easy to add new calls. +- Make it easy to add new callbacks. +- Allow call and callback metadata to evolve without forcing frequent recompilation of the DB or UDF client base implementation. +- Keep the wire model open enough to support additional metadata fields and future protocol evolution. + +### Deployment and Connectivity Requirements + +- Support multiple connections for the `UDFRunner`. +- Support multiple streams per connection to enable more advanced execution and parallel-query scenarios. +- Scope each logical stream to one connection; use `(connection, stream_id)` as its identity and do not reuse an ID on that connection. +- Support both local and remote communication. +- Use Unix-domain stream sockets as the initial binding while preserving an extension path for TCP/TLS deployments. +- Be implementable in multiple programming languages used by UDF runtimes and clients. + +### Operational Requirements + +- Be reasonably easy to implement for the client side. +- Be reasonably easy to set up and operate. +- Avoid deadlock-prone communication patterns. +- Allow the system to detect stalled or unresponsive peers. + +# Security Requirements + +The new protocol changes a security-sensitive interface between the database and an external or semi-external execution environment. The design therefore needs explicit requirements for authentication, encryption, integrity, isolation, and availability. + +## Trust Model + +- Treat `UDFRunner` as untrusted input from the protocol perspective, even if it is started by Exasol-controlled infrastructure. +- Validate received metadata, stream control messages, schemas, and table-buffer references before use. +- Do not rely on a client to behave correctly with respect to message counts, flow control, or buffer ownership. + +## Authentication and Authorization + +- Remote connections must support authentication. +- The protocol must make it possible to distinguish authorized DB peers from unauthorized remote clients. +- Callback capabilities such as `get_connection`, `get_script`, and `execute_query` must remain DB-controlled capabilities and must not become arbitrary privilege-escalation channels for the client. +- Connection information returned by callbacks must be limited to the data required for the requested operation. + +## Transport Security and Confidentiality + +- Remote connections must support encryption, preferably TLS-based. +- TCP deployments must use TLS, endpoint authentication, and peer identity validation. +- Sensitive metadata and table data must be protected in transit against passive observation. +- The design should allow secure local and remote deployment modes without weakening remote security to fit local-only mechanisms. + +## Integrity Requirements + +- The DB must be protected against malicious or malformed stream, batch, and metadata input. +- Buffer-transfer mechanisms must preserve data integrity after handoff. If a mechanism allows a client to mutate data after the DB starts consuming it, the design must either prevent that behavior or treat the mechanism as unsuitable. +- Any out-of-band buffer transport must preserve the integrity relationship between control metadata and the referenced payload. + +## Availability and Resource-Safety Requirements + +- The protocol must avoid deadlocks caused by both sides blocking on send operations. +- The design must bound or control resource usage such as send buffers, file descriptors, and out-of-band buffer handles. +- The protocol must support keepalive or heartbeat behavior so the DB can detect stalled peers and terminate unhealthy sessions. +- The design must ensure one side cannot force the other into unbounded buffering by sending more data than was requested. + +## Segregation and Deployment Constraints + +- Local-only transports such as shared memory are not sufficient for the general design because the protocol must also support remote connections. +- If multiple transport implementations are allowed, the security properties of each must be documented separately. +- Any local optimization such as `memfd` or shared-memory-style transfer must be evaluated against accidental mutation, crash risk, and cleanup behavior. +- File-descriptor-based buffer handoff is limited to Unix-domain sockets and must fall back to inline buffers when it is unavailable. + +## Legislative and Standards Alignment + +- The design should align with the existing company expectation that remote communication uses standard, reviewable transport-security mechanisms rather than custom cryptography. +- Security controls should be chosen so they can be verified and tested consistently across supported runtime languages. + +# How Other Databases Solve That Problem + +TODO: Add comparative research for Oracle, PostgreSQL, MySQL, SQL Server, and other relevant systems once the transport direction is narrowed. + +# Solution Approaches + +At a high level, the existing material suggests at least two families of approaches: + +- use a higher-level framework that already supports RPC with streaming, such as Arrow Flight +- implement the protocol on a lower-level transport such as sockets or a messaging library + +The initial implementation binds the protocol to Unix-domain stream sockets. The framed protocol remains transport +independent so a future TCP/TLS binding can carry the same byte stream. Unix-only file-descriptor handoff remains +an optional local optimization rather than a requirement for all bindings. + +# Proposition For Exasol + +## Design + +Exasol should introduce a new UDF protocol with the following high-level properties: + +- `DB` acts as the client and `UDFRunner` acts as the server. +- The protocol supports `Run`, `Function`, and `Script` call types. +- The protocol supports multiple logical streams with partial ordering across streams and total ordering within each stream. +- Table transfer is explicitly flow controlled, with `Next(...)`-style requests authorizing batch transfer. +- The protocol supports callback interactions from `UDFRunner` to `DB`, including connection lookup, script lookup, and query execution. +- The protocol includes `ServerCapabilities`, framing, stream identifiers, and keepalive behavior. + +The first implementation uses Unix-domain stream sockets. A future TCP/TLS binding uses the same framing and must +provide authenticated, encrypted peer communication. `Inline` buffers work on every binding; Unix-domain sockets +may later add file-descriptor transfer for `memfd` buffers or GPU-memory handles. + +## Security risks + +The main security risks introduced or highlighted by this design are: + +- exposing a remote protocol surface between `DB` and `UDFRunner` +- transporting sensitive metadata and possibly credentials across that surface +- handling untrusted or malformed batch metadata and buffer references +- deadlock or denial-of-service risk from incorrect flow control or blocked send paths +- resource leakage or cleanup failures for buffer transports such as file-descriptor-backed mechanisms + +These risks are manageable only if authentication, encryption, validation, flow control, and health-check behavior are part of the protocol design rather than optional implementation details. + +## Design Decisions & Limitations + +Current draft decisions: + +- use the new protocol draft as a transport-agnostic design artifact first +- bind the first implementation to Unix-domain stream sockets +- preserve TCP/TLS as a future transport binding with inline buffer transfer only +- reserve Unix-domain file-descriptor handoff for optional `memfd` and GPU-memory optimizations +- treat remote security as mandatory, not optional +- treat extensibility of call/callback metadata as a first-class requirement + +Current draft limitations: + +- no final metadata serialization format has been selected for all message classes +- handling of very large values is acknowledged as a requirement but not yet fully designed +- no capability negotiation or descriptor-to-batch correlation is defined for optional Unix FD handoff + +## Open Questions + +- Which metadata formats should be used for protocol metadata, call metadata, and callback metadata? +- How should very large values be accessed safely if they do not fit well into the normal batch-transfer path? +- Which local-only optimizations, if any, are worth supporting in addition to a remote-safe baseline? +- How should Unix file descriptors be correlated with batch metadata, validated, owned, and released? +- Which GPU-memory handle types, if any, are safe to support through Unix descriptor passing? + +## Technical Design + +The current technical direction from the repo material is: + +- a `ServerCapabilities` startup message and framed message protocol +- Unix-domain stream sockets as the initial byte-stream binding, with a future TCP/TLS binding using the same framing +- a connection-scoped `stream_id` model where `0` is the per-connection control stream, odd/even ownership is assigned to the generic Client and Server roles, and IDs are not reused per connection +- composite `StreamMessage` fields so related control data, such as a close and `Error`, can share one frame +- unilateral `CloseCall` and two-way `CloseConnection` termination, with `Error` indicating abnormal closure +- explicit control messages for metadata, start/call behavior, `Next`, data, callbacks, and heartbeat +- table transfer based on Arrow-compatible batch metadata plus inline buffers on all bindings; Unix FD handoff is optional + +Detailed technical design is still required for optional Unix FD handoff and final message-format decisions. + +## Documentation Input + +Likely documentation targets: + +- developer-facing protocol documentation for UDF runtime implementers +- architecture/operation documentation for remote UDF deployments +- security documentation for authentication, encryption, and trust assumptions +- internal implementation notes for stream control, callbacks, and failure handling + +## Checklist: Changed Behavior & Side Effects + +_**Always review all entries in the list**, if you reviewed check the first checkbox and if it applies, check the second checkbox and describe how it applies._ +_Feel free to add new rows, if there is any side effect to be considered._ + +| **Item** | **Does not apply** | **Applies (please describe how)** | +| --- | --- | --- | +| **Changed behavior** | | Yes. This is a protocol redesign for UDF execution and changes communication behavior between `DB` and `UDFRunner`. | +| Does this have a **medium or high risk** of introducing new bugs? (e.g. when it touches critical code, or many existing code paths) | | Yes. It affects core execution, streaming, callback, and remote-communication behavior. | +| Does this introduce **security risks**? If so make them explicit, explain the reasoning, mitigation and further plans. | | Yes. New/changed remote communication, credential handling, and untrusted payload processing require explicit security design and validation. | +| Potentially affects security (e.g. new or changed public interface, communication channels, new/changed protocols, etc.)? If yes, consider in the design and test plan. | | Yes. This is a new/changed protocol surface. | +| Profiling/Auditing changes/additions needed? (new query elements/combinations => new tests recommended) | | Possibly. Protocol events, callbacks, and failure/timeout behavior may need observability updates. | +| Any other side effects or anything special about this feature? | | Performance, deadlock behavior, multi-stream scheduling, and remote deployment characteristics all change. | +| 3rd party components were updated or introduced? | | Flatbuffer and Arrow | + +## Test Plan + +High-level verification required for the final design and implementation: + +| **Test Short Description** | **Assignee** | +| --- | --- | +| Validate protocol correctness for `Run`, `Function`, and `Script`, including callbacks and stream ordering. | TODO | +| Verify remote-security behavior: authentication, encryption, malformed-input handling, and unauthorized access rejection. | TODO | +| Benchmark small-batch and large-batch performance against the current protocol, including deadlock/resource-stress scenarios. | TODO | + +## Possible problems caused by the new code + +| **Possible Problems** | **Symptoms** | **How to Identify** | **Mitigation** | +| --- | --- | --- | --- | +| Deadlock or stalled communication | Hung UDF execution, blocked send/receive loops, timeout-triggered termination | Inspect protocol logs, thread state, and keepalive/flow-control traces | Prioritize receive on `UDFRunner`, prioritize callback handling on `DB`, add timeout and health-check coverage | +| Malformed or malicious batch metadata / buffer references | Crashes, rejected requests, memory-safety issues, corrupted results | Validate protocol input paths, fuzz metadata parsing, monitor error patterns | Strict validation, bounds checks, defensive parsing, reject invalid payload references | +| Poor small-batch performance despite redesign | No measurable improvement or worse latency than current protocol | Benchmark scalar and small-batch workloads against baseline | Revisit batching strategy, framing, callback round trips, and transport selection | +| Resource leakage in out-of-band buffer transport | File-descriptor growth, memory pressure, cleanup failures | Observe FD counts, memory usage, and long-running worker behavior | Constrain handle lifetime, add cleanup guarantees, prefer safer transport defaults | diff --git a/doc/design/v2/udf_protocol_high_level_call_model.mmd b/doc/design/v2/udf_protocol_high_level_call_model.mmd new file mode 100644 index 0000000..6482bb3 --- /dev/null +++ b/doc/design/v2/udf_protocol_high_level_call_model.mmd @@ -0,0 +1,16 @@ +flowchart TD + Call["Call\n(CallOpen / first peer call-scoped message / CallClose)"] + Call -->|opened by DB or UDFRunner| Opener{{"Either side may open a call"}} + Call -->|optional, at most one| DataStream["Bidirectional Data Stream\n(independent schema per direction)"] + Call -->|optional| Nested["Nested Call(s)\n(started by nested CallOpen traffic)"] + + Run["Run (DB-opened)"] -.->|instance of| Call + Function["Function (DB-opened)"] -.->|instance of| Call + Script["Script (DB-opened)"] -.->|instance of| Call + GetConnection["get_connection (UDFRunner-opened)"] -.->|instance of| Call + GetScript["get_script (UDFRunner-opened)"] -.->|instance of| Call + ExecuteQuery["execute_query / pquery (UDFRunner-opened)"] -.->|instance of| Call + + Run --> RunData["Input/Output Data Stream"] + Script --> ScriptData["Parameter Input / Table Output Data Stream"] + ExecuteQuery --> QueryData["Optional Result-Row Data Stream"] diff --git a/doc/design/v2/udf_protocol_high_level_call_model.svg b/doc/design/v2/udf_protocol_high_level_call_model.svg new file mode 100644 index 0000000..775dddd --- /dev/null +++ b/doc/design/v2/udf_protocol_high_level_call_model.svg @@ -0,0 +1 @@ +

opened by DB or UDFRunner

optional, at most one

optional

instance of

instance of

instance of

instance of

instance of

instance of

Call
(CallOpen / first peer call-scoped message / CallClose)

Either side may open a call

Bidirectional Data Stream
(independent schema per direction)

Nested Call(s)
(started by nested CallOpen traffic)

Run (DB-opened)

Function (DB-opened)

Script (DB-opened)

get_connection (UDFRunner-opened)

get_script (UDFRunner-opened)

execute_query / pquery (UDFRunner-opened)

Input/Output Data Stream

Parameter Input / Table Output Data Stream

Optional Result-Row Data Stream

 \ No newline at end of file diff --git a/doc/design/v2/udf_protocol_high_level_calls.md b/doc/design/v2/udf_protocol_high_level_calls.md new file mode 100644 index 0000000..67615a4 --- /dev/null +++ b/doc/design/v2/udf_protocol_high_level_calls.md @@ -0,0 +1,127 @@ +# UDF Protocol v2: High-Level Calls + +This document describes the high-level protocol calls built on top of the generic call and data-stream mechanisms. + +## Scope + +This document covers: + +- `Run`, `Function`, and `Script` +- `get_connection`, `get_script`, and `execute_query` +- which calls carry data streams +- representative message sequences +- DB/UDFRunner scheduling policy + +Related diagrams: + +- [udf_protocol_high_level_call_model.svg](udf_protocol_high_level_call_model.svg) +- [udf_protocol_high_level_nested_calls.svg](udf_protocol_high_level_nested_calls.svg) +- [udf_protocol_high_level_run_sequence.svg](udf_protocol_high_level_run_sequence.svg) +- [udf_protocol_high_level_endpoint_scheduling.svg](udf_protocol_high_level_endpoint_scheduling.svg) + +## Call Families + +### `DB`-opened calls + +| Call | Data stream | Notes | +| --- | --- | --- | +| `Run` | Yes, bidirectional | Input and output belong to the same call. Scalar-return correlation is carried in the data itself. | +| `Function` | No | Callback-driven work only. | +| `Script` | Yes, bidirectional | One direction carries input parameters, the other direction carries table output. | + +### `UDFRunner`-opened calls + +| Call | Data stream | Notes | +| --- | --- | --- | +| `get_connection` | No | Returns connection information. | +| `get_script` | No | Returns script content. | +| `execute_query` | Optional | Returns query status and may carry result rows. | + +These `UDFRunner`-opened calls are ordinary nested calls, not a separate callback transport. + +## Call-Specific Result Payloads + +Each high-level call defines the names and bodies of its own result payloads, carried by `Payloads(...)`. A result +payload may be sent while the call remains active or together with `CloseCall` in the same composite +`StreamMessage`. Calls that have no result payload may still close normally. + +## Typical Semantics + +### `Run` + +- opened by `DB` +- may carry opening payloads and the first input batch together +- may stay active while nested calls such as `get_script` or `get_connection` execute +- output row correlation for scalar returns belongs in the data, not in `Next(...)` + +### `Function` + +- opened by `DB` +- has no attached data stream in the current model +- relies on nested calls for callback-style interaction + +### `Script` + +- opened by `DB` +- uses one bidirectional data stream +- input and output directions may use different schemas + +### `get_connection` + +- opened by `UDFRunner` +- returns `Payloads(connection_information, ...)` +- may still carry additional named payload traffic while active + +### `get_script` + +- opened by `UDFRunner` +- returns `Payloads(script, ...)` +- may still carry additional named payload traffic while active + +### `execute_query` + +- opened by `UDFRunner` +- returns `Payloads(result_set_status, ...)` +- may optionally carry a result-row data stream + +## Representative Sequences + +The current design keeps the high-level sequences intentionally simple: + +- nested callback-style calls execute while a parent `Run`, `Function`, or `Script` call remains active +- `Run` may combine opening call metadata, input schema announcement, and the first input batch +- `execute_query` may or may not open a result-row data stream after its status result + +See [udf_protocol_high_level_nested_calls.svg](udf_protocol_high_level_nested_calls.svg) and +[udf_protocol_high_level_run_sequence.svg](udf_protocol_high_level_run_sequence.svg). + +## Scheduling Policy + +The source material implies the following DB/UDFRunner deadlock-avoidance rules. These rules govern high-level +call orchestration and do not alter the generic Client/Server stream rules in the low-level protocol. + +### `UDFRunner` + +1. prioritize receiving over sending +2. only block during receive +3. do not receive more messages than were requested +4. send regular `KeepAlive` messages so `DB` can continue housekeeping + +### `DB` + +1. prioritize nested-call responses before data-stream work +2. if nothing is ready to send, block waiting for new incoming messages +3. monitor peer liveness and terminate unhealthy sessions when needed + +See [udf_protocol_high_level_endpoint_scheduling.svg](udf_protocol_high_level_endpoint_scheduling.svg). + +## Forward-Looking Ideas Still Open + +- whether `UDFRunner` may open its own pquery-style call to `DB` beyond the current `execute_query` shape +- whether table-prefetch-like declarations should be added for future call setup + +## Relationship To Other Docs + +- low-level framing and generic stream rules live in [udf_protocol_low_level.md](udf_protocol_low_level.md) +- generic open/close call behavior lives in [udf_protocol_call_lifecycle.md](udf_protocol_call_lifecycle.md) +- generic data-stream behavior lives in [udf_protocol_data_stream.md](udf_protocol_data_stream.md) diff --git a/doc/design/v2/udf_protocol_high_level_endpoint_scheduling.mmd b/doc/design/v2/udf_protocol_high_level_endpoint_scheduling.mmd new file mode 100644 index 0000000..faaea84 --- /dev/null +++ b/doc/design/v2/udf_protocol_high_level_endpoint_scheduling.mmd @@ -0,0 +1,22 @@ +stateDiagram-v2 + state "UDFRunner" as UDF { + [*] --> UdfIdle + UdfIdle --> UdfInitializing: Accept + UdfInitializing --> UdfActive: Capabilities + UdfActive --> UdfWaitingForInboundData: Receive + UdfWaitingForInboundData --> UdfSendingData: Next credit + UdfSendingData --> UdfActive: Transfer done + UdfWaitingForInboundData --> UdfActive: Message + } + + state "DB" as DB { + [*] --> DbIdle + DbIdle --> DbInitializing: Connect + DbInitializing --> DbActive: Capabilities + DbActive --> DbHandlingNestedCall: Call + DbHandlingNestedCall --> DbActive: Reply + DbActive --> DbWaitingForUdfMessage: Wait + DbWaitingForUdfMessage --> DbActive: Message + DbActive --> DbSendingData: Data transfer + DbSendingData --> DbActive: Transfer done + } diff --git a/doc/design/v2/udf_protocol_high_level_endpoint_scheduling.svg b/doc/design/v2/udf_protocol_high_level_endpoint_scheduling.svg new file mode 100644 index 0000000..398d351 --- /dev/null +++ b/doc/design/v2/udf_protocol_high_level_endpoint_scheduling.svg @@ -0,0 +1 @@ +

UDFRunner

Accept

Capabilities

Receive

Next credit

Transfer done

Message

UdfIdle

UdfInitializing

UdfActive

UdfWaitingForInboundData

UdfSendingData

DB

Connect

Capabilities

Call

Reply

Wait

Message

Data transfer

Transfer done

DbIdle

DbInitializing

DbActive

DbHandlingNestedCall

DbWaitingForUdfMessage

DbSendingData

 \ No newline at end of file diff --git a/doc/design/v2/udf_protocol_high_level_nested_calls.mmd b/doc/design/v2/udf_protocol_high_level_nested_calls.mmd new file mode 100644 index 0000000..76f2bf7 --- /dev/null +++ b/doc/design/v2/udf_protocol_high_level_nested_calls.mmd @@ -0,0 +1,17 @@ +sequenceDiagram + participant UDFRunner + participant DB + + Note over UDFRunner,DB: Nested calls opened by UDFRunner while a parent DB-opened call remains active + + UDFRunner->>DB: CallOpen(get_connection, {connection_name}) + DB-->>UDFRunner: Payloads(connection_information, ...) + UDFRunner->>DB: CloseCall + + UDFRunner->>DB: CallOpen(get_script, {script_name}) + DB-->>UDFRunner: Payloads(script, ...) + CloseCall + + UDFRunner->>DB: CallOpen(execute_query, {query}) + DB-->>UDFRunner: Payloads(result_set_status, ...) + DB-->>UDFRunner: DataMetaData + RecordBatch(...) on the call's data stream + UDFRunner->>DB: CloseCall diff --git a/doc/design/v2/udf_protocol_high_level_nested_calls.svg b/doc/design/v2/udf_protocol_high_level_nested_calls.svg new file mode 100644 index 0000000..57b0557 --- /dev/null +++ b/doc/design/v2/udf_protocol_high_level_nested_calls.svg @@ -0,0 +1 @@ +DBUDFRunnerDBUDFRunnerNested calls opened by UDFRunner while a parent DB-opened call remains activeCallOpen(get_connection, {connection_name})Payloads(connection_information, ...)CloseCallCallOpen(get_script, {script_name})Payloads(script, ...) + CloseCallCallOpen(execute_query, {query})Payloads(result_set_status, ...)DataMetaData + RecordBatch(...) on the call's data streamCloseCall \ No newline at end of file diff --git a/doc/design/v2/udf_protocol_high_level_run_sequence.mmd b/doc/design/v2/udf_protocol_high_level_run_sequence.mmd new file mode 100644 index 0000000..9036794 --- /dev/null +++ b/doc/design/v2/udf_protocol_high_level_run_sequence.mmd @@ -0,0 +1,13 @@ +sequenceDiagram + participant DB + participant UDFRunner + + Note over DB,UDFRunner: Run call opened by DB with flow-controlled bidirectional data exchange + + DB->>UDFRunner: CallOpen(Run, call_metadata) + DataMetaData + first input RecordBatch + UDFRunner-->>DB: Next(byte_budget, reset=false, row_id=n) + DB-->>UDFRunner: RecordBatch(input rows starting at n) + UDFRunner-->>DB: DataMetaData + first output RecordBatch with row ids for scalar correlation + DB->>UDFRunner: Next(byte_budget, reset=false, row_id=m) + UDFRunner-->>DB: RecordBatch(output rows starting at m) + DB->>UDFRunner: CallClose(Run) diff --git a/doc/design/v2/udf_protocol_high_level_run_sequence.svg b/doc/design/v2/udf_protocol_high_level_run_sequence.svg new file mode 100644 index 0000000..742aafd --- /dev/null +++ b/doc/design/v2/udf_protocol_high_level_run_sequence.svg @@ -0,0 +1 @@ +UDFRunnerDBUDFRunnerDBRun call opened by DB with flow-controlled bidirectional data exchangeCallOpen(Run, call_metadata) + DataMetaData + first input RecordBatchNext(byte_budget, reset=false, row_id=n)RecordBatch(input rows starting at n)DataMetaData + first output RecordBatch with row ids for scalar correlationNext(byte_budget, reset=false, row_id=m)RecordBatch(output rows starting at m)CallClose(Run) \ No newline at end of file diff --git a/doc/design/v2/udf_protocol_low_level.md b/doc/design/v2/udf_protocol_low_level.md new file mode 100644 index 0000000..574d0f0 --- /dev/null +++ b/doc/design/v2/udf_protocol_low_level.md @@ -0,0 +1,148 @@ +# UDF Protocol v2: Low-Level Protocol + +This document captures the low-level wire rules for the new UDF protocol. It intentionally excludes +high-level call semantics and scheduling policy; those live in +[udf_protocol_high_level_calls.md](udf_protocol_high_level_calls.md). + +## Scope + +This document covers: + +- framing and message layering +- control-stream traffic on `stream_id = 0` +- stream ownership rules +- generic call-attached data-stream rules +- transport binding and buffer handoff + +The generic data-stream rules are defined in the companion +[udf_protocol_data_stream.md](udf_protocol_data_stream.md). They apply below the high-level `DB`/`UDFRunner` +protocol layer. + +Related diagrams: + +- [udf_protocol_low_level_connection_lifecycle.svg](udf_protocol_low_level_connection_lifecycle.svg) + +## Roles + +- `Server` accepts the transport-level connection. +- `Client` initiates the transport-level connection. + +These roles define connection establishment and stream ownership only. A higher layer maps concrete components to +these roles. Transport-level roles do not limit which side may later open a call. + +## Message Layering + +The protocol uses two layers that should remain distinct: + +- `Frame` is the low-level length-framed unit on the transport connection. +- `StreamMessage` is the typed protocol payload carried inside a `Frame`. + +Receive path: + +1. bytes on the socket +2. one decoded `Frame` +3. `stream_id` selection +4. one decoded `StreamMessage` + +This separation is important because the transport boundary and the typed protocol payload evolve independently. + +## Transport Binding + +The initial protocol binding uses Unix-domain stream sockets. The transport carries the length-framed `Frame` byte +stream unchanged; it does not alter logical stream ownership, call semantics, control traffic, or flow control. + +Future bindings may use TCP with the same framing. Any remote TCP deployment must use TLS together with endpoint +authentication and peer identity validation. TCP/TLS supports only inline record-batch buffers; it has no portable +file-descriptor transfer mechanism. + +Unix-domain sockets may later add descriptor passing for local buffer handoff. This is an optional local +optimization and is not required to implement the base protocol. + +## Control Stream + +`stream_id = 0` is reserved for traffic that is independent of any active call. + +The current control-stream message set is: + +- `ServerCapabilities(version)` sent by the `Server` during initialization +- `KeepAlive` +- `Payloads(...)` when named payloads need to be exchanged without any active call +- `CloseConnection` for orderly connection shutdown + +`StreamMessage` is a composite message with independently optional fields. This allows related fields, such as a +close field and `Error`, to travel together in one `Frame`. + +## Stream Ownership + +Each physical connection multiplexes multiple logical streams. A logical stream belongs to exactly one connection; +its protocol identity is the pair `(connection, stream_id)`. The connection is implicit in the transport session and +is therefore not carried in `Frame`. + +The stream-ID rules are: + +- `stream_id` is 64-bit +- odd `stream_id` values belong to the `Client` +- even `stream_id` values belong to the `Server` +- `stream_id = 0` is the control stream which can be used by the `Client` and `Server` +- a `stream_id` is unique for the lifetime of its connection and is not reused after its logical stream closes +- different connections may use the same numeric `stream_id`, including `0`; those pairs identify different + logical streams + +These rules allocate ownership and connection scope, but they do not yet fully define stream creation or teardown +beyond call-scoped usage. + +## Connection Lifecycle + +At the low level, a session progresses through: + +1. connection establishment +2. control-stream initialization, including the `Server` advertising `ServerCapabilities(version)` on + `stream_id = 0` +3. normal call/control traffic +4. a two-way `CloseConnection` exchange on stream `0`, followed by transport close; or abort + +See [udf_protocol_low_level_connection_lifecycle.svg](udf_protocol_low_level_connection_lifecycle.svg). + +## Close Semantics + +`CloseCall` closes the call on its non-zero stream. Either peer may send it; no call-close acknowledgement is +required. A `CloseCall` with `Error` is an abnormal call termination. Without `Error`, it is normal termination. +After sending or receiving `CloseCall`, neither peer sends further call-scoped traffic on that stream. + +`CloseConnection` closes the entire connection and is valid only on stream `0`. Either peer may initiate shutdown +by sending it. The receiver sends `CloseConnection` in reply, optionally with its own `Error`, then closes the +underlying transport. The initiator closes the transport after receiving that reply. A `CloseConnection` with +`Error` is abnormal termination; without `Error`, it is normal termination. If both peers initiate shutdown at the +same time, each treats the received `CloseConnection` as the reply and does not send another one. + +After sending or receiving `CloseConnection`, neither peer opens a stream or sends ordinary call, data, or +control traffic. Every active non-control stream is terminated when the underlying transport closes. + +`Error` without either close field is a non-terminal diagnostic for the enclosing stream. It does not by itself +close a call or connection; peers may continue processing when the error is recoverable. + +## Buffer Handoff By Binding + +`Inline` buffers are supported by every binding. Their bytes follow the `DataRecordBatch` metadata on the framed +connection according to the data-stream rules. + +`Memfd` and `OutOfBand` buffers are reserved for Unix-domain sockets using file-descriptor passing. `Memfd` +denotes file-descriptor-backed shared buffers. `OutOfBand` may later cover GPU-memory handles transferred through +the same Unix-only mechanism; it is not a portable GPU-memory transport. + +The initial binding does not define descriptor-to-batch correlation, supported GPU handle types, buffer ownership, +lifetime, cleanup, or capability negotiation. An implementation that cannot establish a compatible local handoff +must use `Inline` buffers. + +## Serialization Mapping + +- low-frequency call/control metadata: FlatBuffers +- record batches: Apache Arrow IPC-compatible payloads +- named metadata payload bodies such as connection objects or script content: typically JSON + +This document only records the mapping at a protocol level. It does not redefine the `.fbs` schema itself. + +## Open Questions + +- descriptor-to-batch correlation, ownership, lifetime, cleanup, and capability negotiation for Unix FD handoff +- GPU-memory handle types and validation requirements for any future `OutOfBand` binding diff --git a/doc/design/v2/udf_protocol_low_level_connection_lifecycle.mmd b/doc/design/v2/udf_protocol_low_level_connection_lifecycle.mmd new file mode 100644 index 0000000..c663da8 --- /dev/null +++ b/doc/design/v2/udf_protocol_low_level_connection_lifecycle.mmd @@ -0,0 +1,16 @@ +stateDiagram-v2 + [*] --> Idle + + Idle --> Initializing: Connect + Initializing --> ReadyForTraffic: Capabilities + ReadyForTraffic --> Active: Control traffic + Active --> Closing: CloseConnection + Closing --> Completed: Close reply + Completed --> ClosedOrAborted: Transport close + + Idle --> ClosedOrAborted: Setup failure + Initializing --> ClosedOrAborted: Setup failure + ReadyForTraffic --> ClosedOrAborted: Session abort + Active --> ClosedOrAborted: Transport abort + + ClosedOrAborted --> [*] diff --git a/doc/design/v2/udf_protocol_low_level_connection_lifecycle.svg b/doc/design/v2/udf_protocol_low_level_connection_lifecycle.svg new file mode 100644 index 0000000..ef55dfe --- /dev/null +++ b/doc/design/v2/udf_protocol_low_level_connection_lifecycle.svg @@ -0,0 +1 @@ +

Connect

Capabilities

Control traffic

CloseConnection

Close reply

Transport close

Setup failure

Setup failure

Session abort

Transport abort

Idle

Initializing

ReadyForTraffic

Active

Closing

Completed

ClosedOrAborted

 \ No newline at end of file diff --git a/doc/design/v2/udf_transport_technology_overview.md b/doc/design/v2/udf_transport_technology_overview.md new file mode 100644 index 0000000..694b3d3 --- /dev/null +++ b/doc/design/v2/udf_transport_technology_overview.md @@ -0,0 +1,113 @@ +# UDF Transport Technology Overview + +This document summarizes technology choices for moving calls and tabular data +between a database engine and user-defined function (UDF) code. It is a +technology overview, not a protocol specification. + +## Execution Boundaries + +| Boundary | Description | Strengths | Tradeoffs | +| --- | --- | --- | --- | +| In-process ABI | UDF code runs in the database process and receives engine-owned values or vectors through an application binary interface. | Lowest transfer overhead and direct access to engine-native vectors. | A faulty UDF can affect the database process; no isolation or remote deployment boundary. | +| Side process or container | UDF code runs in a separate local process or container. | Isolates language runtimes and failures; supports independent dependencies and long-lived workers. | Requires IPC, serialization, lifecycle management, and resource limits. | +| Remote service | UDF code runs on another host or platform. | Independent scaling, deployment, and fault domain. | Adds network latency and requires authentication, encryption, retry, and failure semantics. | + +## Transport and Serialization + +| Transport | Typical serialization | Data movement | Appropriate use | +| --- | --- | --- | --- | +| Shared memory | Native in-memory vectors or custom binary layout | Zero-copy or low-copy local batches | Trusted local components that can share memory ownership and lifetime rules. | +| Unix-domain socket | Protobuf, FlatBuffers, Arrow IPC, or custom binary framing | Local bidirectional messages, inline batches, and optional FD-backed buffers | Isolated local workers where filesystem permissions can enforce endpoint access. | +| TCP stream with TLS | Protobuf, FlatBuffers, Arrow IPC, or custom binary framing | Persistent bidirectional messages and inline batches | Local-network or remote workers that need streaming, cancellation, and long-lived sessions. | +| Message middleware | Middleware-defined frames with binary or schema-based payloads | Asynchronous messages or request/reply | Decoupled local or distributed components where the middleware's delivery semantics are suitable. | +| Standard input/output pipes | Delimited text, JSON lines, or a binary stream format | Sequential local input and output | Trusted or tightly supervised executable UDFs and process pools. | +| HTTP(S) request/response | JSON, optionally compressed | Bounded request batches and responses | Interoperable remote scalar or batch calls with simple service integration. | +| RPC/gRPC | Protobuf by default; other encodings are possible | Unary RPC or streaming RPC | Typed remote interfaces that benefit from generated clients, streaming, and service contracts. | + +## Untrusted Code and Pipes + +Standard input/output pipes are a simple local process interface, not a security +boundary. When the executed UDF code is not trusted, it can emit logs, debug +output, malformed bytes, or partial writes that corrupt the protocol stream. It +can also stop reading input or draining output, filling pipe buffers and causing +both processes to block indefinitely. + +Pipes also leave cancellation, process exit, timeout, and partial-result +semantics to the supervising process. Text formats add parsing, escaping, and +type-conversion overhead for large data transfers. A pipe by itself provides no +authentication, encryption, peer identity, or message isolation. Reused +processes can retain state or sensitive input between invocations. + +For untrusted code, require all of the following in addition to the pipe +transport: + +- Send diagnostics through a separate channel so they cannot corrupt protocol output. +- Use strict framing, input and output validation, and explicit byte limits. +- Supervise process lifetime with timeouts, cancellation, and defined handling for partial output. +- Apply process or container sandboxing, resource limits, restricted filesystem access, and restricted network access. +- Reset worker state before reuse, or terminate the worker after each invocation. + +Use a framed socket or RPC protocol instead when execution requires authenticated +identity, multiplexing, structured cancellation, or remote deployment. + +## Serialization Formats + +| Format | Characteristics | Best fit | Main limitation | +| --- | --- | --- | --- | +| Native vectors | Engine-owned in-memory data structures. | In-process and shared-memory execution. | Couples UDF code to an engine ABI and memory-lifetime rules. | +| Custom binary | Application-defined message fields and typed values. | Controlled local protocols with narrow type requirements. | Requires explicit versioning, framing, and cross-language compatibility work. | +| Protobuf | Schema-defined binary messages with broad language support. | Control messages, metadata, and bounded row-oriented payloads. | Not a columnar table format; bulk data can require conversion and copying. | +| FlatBuffers | Schema-defined binary data designed for direct access. | Framed control messages and metadata where allocation and parsing overhead matter. | Requires careful schema evolution and buffer-lifetime handling. | +| Arrow IPC | Standardized columnar record batches and schemas. | High-throughput analytical table data. | Requires supported type mapping and explicit batch-size and memory-pressure policy. | +| JSON | Human-readable structured data. | Metadata, service integration, and small bounded requests. | Text encoding and row-oriented values are inefficient for large table transfers. | +| Delimited or text rows | Simple ordered fields separated by delimiters or line boundaries. | Small executable integrations and debugging-friendly workflows. | Escaping, type conversion, and parsing overhead limit throughput. | + +## Binding-Specific Buffer Handoff + +Inline byte transfer is portable across Unix-domain sockets and TCP/TLS. Unix-domain sockets can additionally pass +file descriptors for `memfd`-backed buffers and, where the platform permits it, GPU-memory handles. This requires +explicit descriptor correlation, ownership, lifetime, cleanup, integrity validation, and file-descriptor limits. + +TCP/TLS cannot use Unix file-descriptor passing. A remote binding must therefore use inline buffers unless a future +portable out-of-band mechanism is specified and negotiated. The protocol's call, stream, and flow-control semantics +should remain unchanged across these choices. + +## Data Movement and Flow Control + +| Model | Behavior | Use when | Design requirements | +| --- | --- | --- | --- | +| Scalar request/response | One invocation returns one result. | Inputs are small and independent. | Correlate failures and define timeout and retry behavior. | +| Bounded batch RPC | One request carries multiple rows and returns matching results. | Remote calls benefit from amortizing invocation overhead. | Preserve row order or identifiers; cap rows and bytes per request. | +| Unidirectional stream | A producer sends an ordered sequence of messages or batches. | Results or input data have one primary direction. | Define end-of-stream, cancellation, and receiver backpressure. | +| Bidirectional stream | Both peers send control and data while a call remains active. | The UDF can request more data, send results incrementally, or make callbacks. | Separate control from data, avoid request/reply deadlocks, and define ordering per direction. | +| Credit/window flow control | The receiver grants a byte or batch budget before more data is sent. | Data volume is large or receiver capacity varies. | State the credit unit, initial window, replenishment rule, and behavior on cancellation. | +| Multiplexed logical streams | Independent calls share one physical connection; each stream belongs to that connection only. | Many concurrent calls need one connection per worker or instance. | Scope stream identifiers to the connection, avoid ID reuse on it, isolate failures, and prevent one stream from starving others. | + +## Worker Lifecycle and Reliability + +- Reuse long-lived workers when startup or model initialization is expensive; define when state is retained, reset, or discarded. +- Bound memory, CPU, process count, queue depth, batch size, and time spent without making progress. +- Treat cancellation, peer disconnect, worker crash, and partial response as explicit protocol states. +- For remote transports, use authenticated endpoints, encrypted connections, peer identity validation, and idempotency rules before retrying an invocation. +- Keep physical transport independent from call semantics so the same call and data model can run over local IPC or a secured network transport. + +## Selection Guidance + +1. Prefer in-process vectors only when the UDF is trusted and isolation is not required. +2. Prefer Arrow IPC or another columnar format for high-throughput analytical batches. +3. Prefer protobuf or FlatBuffers for typed control messages and metadata. +4. Prefer HTTP(S)/JSON for simple interoperable remote calls, not sustained high-volume table transfer. +5. Prefer a persistent bidirectional stream with explicit credit and multiplexing when calls require callbacks, incremental results, or overlapping control and data traffic. +6. Prefer pipes only for trusted or tightly supervised local code; pipes do not isolate untrusted UDFs. + +## Terms + +| Term | Meaning | +| --- | --- | +| ABI | A binary calling and memory-layout contract between compiled components. | +| IPC | Communication between processes on the same machine. | +| Batch | A bounded collection of rows or columns transferred as one unit. | +| Framing | Delimiting messages in a byte stream so the receiver can recover message boundaries. | +| Multiplexing | Carrying multiple independent logical streams on one physical connection; stream identity is `(connection, stream_id)`. | +| Flow control | Limiting data in flight to the capacity explicitly granted by the receiver. | +| Serialization | Encoding typed data into bytes for transport or storage. | diff --git a/udf-runner-cpp/v2/udf_protocol.fbs b/udf-runner-cpp/v2/udf_protocol.fbs new file mode 100644 index 0000000..72c7fb7 --- /dev/null +++ b/udf-runner-cpp/v2/udf_protocol.fbs @@ -0,0 +1,227 @@ +// First draft of the wire-level flatbuffer schema for the new UDF protocol. + +namespace exasol.udf.protocol; + +enum BufferTransport : uint8 { + Inline = 0, // buffers follow immediately on the same socket, unframed + Memfd = 1, // buffers are backed by a memfd passed out of band (e.g. SCM_RIGHTS) + OutOfBand = 2, // some other out-of-band mechanism +} + +// A small, self-owned subset of Apache Arrow's Schema.fbs/Message.fbs shape, +// NOT Arrow's own flatbuffer types. We define these natively so producing them +// only requires Arrow's stable public C++ API (Array::length(), +// Array::null_count(), ArrayData::buffers, Field::type(), ...) -- Arrow's own +// bare (unframed) IPC message bytes are only reachable via Arrow-internal +// headers (arrow/ipc/metadata_internal.h, what Flight itself uses), which +// aren't part of the installed public SDK. + +enum Precision : uint8 { Half = 0, Single = 1, Double = 2 } +enum DateUnit : uint8 { Day = 0, Millisecond = 1 } +enum TimeUnit : uint8 { Second = 0, Millisecond = 1, Microsecond = 2, Nanosecond = 3 } + +table Int { bit_width: int32; is_signed: bool; } +table FloatingPoint { precision: Precision; } +table Utf8 {} +table Binary {} +table Bool {} +table Date { unit: DateUnit; } +table Timestamp { unit: TimeUnit; timezone: string; } +table Decimal { precision: int32; scale: int32; bit_width: int32 = 128; } +table List {} +// Trailing underscore mirrors Arrow's own Schema.fbs naming for this union member. +table Struct_ {} +// Run-End Encoded (REE): a compact encoding for runs of values (Arrow's own +// RunEndEncodedType). Has 0 buffers of its own -- no validity bitmap at this +// level, per the Arrow columnar spec -- and exactly 2 children in Field's +// children list, in fixed order: run_ends (an integer type), then values +// (any type). Reused as the storage type for extension types like a +// "range run" encoding (runs of s..s+n-1 instead of RLE's repeated value), +// where n_i is derived as run_ends[i] - run_ends[i-1] instead of being stored. +table RunEndEncoded {} + +union Type { + Int, FloatingPoint, Utf8, Binary, Bool, Date, Timestamp, Decimal, List, Struct_, + RunEndEncoded, +} + +// Mirrors Arrow's own Schema.fbs KeyValue: a single string/string metadata +// entry. +table KeyValue { + key: string; + value: string; +} + +table Field { + name: string (required); + nullable: bool; + type: Type (required); + children: [Field]; // List's single element type, Struct_'s member fields, + // or RunEndEncoded's [run_ends, values] pair + // Extension-type annotation, mirroring Arrow's own convention: a field + // representing an extension type is serialized using its storage type for + // `type` above, with the extension name/params carried here as + // ARROW:extension:name / ARROW:extension:metadata entries. A reader that + // doesn't recognize the name falls back to the plain storage type. + custom_metadata: [KeyValue]; +} + +table Schema { + fields: [Field]; +} + +struct FieldNode { + length: int64; + null_count: int64; +} + +struct Buffer { + offset: int64; + length: int64; +} + +table RecordBatchMetadata { + length: int64; + nodes: [FieldNode]; + buffers: [Buffer]; +} + +table Version { + major: uint32; + minor: uint32; +} + +table ServerCapabilities { + supported_version: Version; +} + +table KeepAlive {} + +// A JSON-encoded payload (the common case for call metadata, connection info, +// script content, etc.) or a raw binary payload (escape hatch for large/binary +// values that shouldn't be forced through JSON). +table StringPayload { + value: string; +} + +table BinaryPayload { + value: [ubyte]; +} + +union PayloadValue { + StringPayload, + BinaryPayload, +} + +// Opens a call. The enclosing connection and StreamMessage's stream_id identify +// this call for the connection's lifetime. The connection is implicit in the +// transport session, so it is not encoded in Frame. The same numeric stream_id +// on a different connection identifies a different logical stream. +// +// TODO: one call = one stream_id is a draft-level assumption to fill the gap +// between the prose model and an actual wire encoding, not a settled decision +// from the source docs. +// +// A call opened while another is active (e.g. a callback opened during Run) is +// just another independent call -- no parent/child link is carried on the wire. +// OpenCall carries no payload of its own: opening payloads travel as +// payloads set on the same StreamMessage instead of a duplicate field here. +// Additional payloads may also be sent later on the same connection-scoped +// stream_id while the call remains active. +table OpenCall { + call_name: string; +} + +// One named payload item sent by either side either on an active call's +// connection-scoped stream_id or on stream_id = 0 without any active call. `name` identifies the +// payload schema/meaning and therefore which parser to use for `payload` on the +// receiving side. Examples: +// - "virtual_schema_request" -> JSON with the virtual-schema request schema +// - "call_metadata" -> JSON with the call-metadata schema +// - "kill" -> name only, no payload body required +table Payload { + name: string; + payload: PayloadValue; +} + +// A StreamMessage may carry multiple payload items, including the opening +// payload(s) set alongside open_call on the same StreamMessage. Payloads +// may also be sent on a connection-scoped stream_id whose call is already active, or on +// stream_id = 0 without any active call. +table Payloads { + payloads: [Payload]; +} + +table Error { + code: string; + message: string; +} + +// Closes the call on this non-zero stream. When the enclosing StreamMessage +// also has error set, the close is abnormal; otherwise it is normal. +table CloseCall {} + +// Starts or acknowledges connection shutdown. This field is valid only on the +// control stream (stream_id = 0). The receiver replies with CloseConnection, +// then both peers close the underlying transport. +table CloseConnection {} + +// Grants transfer credit for one direction of a call's data stream. size is a +// byte budget for the permitted transfer window. row_id is the position to +// resume from when reset is true -- it is a seek position, not a row- +// correlation field between the two directions. +table Next { + byte_budget: uint32; + reset: bool = false; + row_id: uint64 = 0; +} + +// Sent once per direction of a call's data stream before, or in the same +// StreamMessage as, that direction's first DataRecordBatch. It contains this +// direction's column layout as our own native Schema table (see above -- not +// Arrow's own flatbuffer Schema). It is split from DataRecordBatch because the +// schema is only needed once, not repeated per batch. +table DataSchema { + schema: Schema; +} + +// Precedes the buffers for one RecordBatch in a direction of a call's data +// stream. metadata is our own native RecordBatchMetadata (row count, per- +// column FieldNode, per-buffer Buffer), populated from a real Arrow +// RecordBatch's public introspection API (Array::length(), null_count(), +// ArrayData::buffers) -- not Arrow's own flatbuffer RecordBatch message. The +// actual column buffers are transported separately per buffer_transport. +table DataRecordBatch { + buffer_transport: BufferTransport = Inline; + is_end: bool = false; + metadata: RecordBatchMetadata; +} + +// A StreamMessage is the typed composite payload carried inside a Frame. Each +// field is independently optional, so related parts such as CloseCall and +// Error can travel in one frame. Error without a close field is a non-terminal, +// stream-scoped diagnostic. stream_id = 0 is reserved for out-of-call control +// traffic such as ServerCapabilities, KeepAlive, and CloseConnection. +table StreamMessage { + server_capabilities: ServerCapabilities; + keep_alive: KeepAlive; + open_call: OpenCall; + payloads: Payloads; + close_call: CloseCall; + close_connection: CloseConnection; + next: Next; + data_schema: DataSchema; + data_record_batch: DataRecordBatch; + error: Error; +} + +// The single length-framed message type on the wire. The implicit transport +// connection and stream_id identify a logical stream. stream_id = 0 is the +// per-connection control stream used when client and server exchange messages +// independently of a call. A stream_id is not reused on its connection. +table Frame { + stream_id: uint64; + message: StreamMessage; +} + +root_type Frame; From 12e29d2d62ecf396b452591786156edabeda055a Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Fri, 14 Aug 2026 13:21:31 +0200 Subject: [PATCH 02/26] protocol: add run batch correlation --- .../v2/udf_protocol_high_level_calls.md | 36 +++++++++++++++++-- .../udf_protocol_high_level_run_sequence.mmd | 8 ++--- .../udf_protocol_high_level_run_sequence.svg | 2 +- udf-runner-cpp/v2/udf_protocol.fbs | 6 ++++ 4 files changed, 44 insertions(+), 8 deletions(-) diff --git a/doc/design/v2/udf_protocol_high_level_calls.md b/doc/design/v2/udf_protocol_high_level_calls.md index 67615a4..bc8d599 100644 --- a/doc/design/v2/udf_protocol_high_level_calls.md +++ b/doc/design/v2/udf_protocol_high_level_calls.md @@ -25,7 +25,7 @@ Related diagrams: | Call | Data stream | Notes | | --- | --- | --- | -| `Run` | Yes, bidirectional | Input and output belong to the same call. Scalar-return correlation is carried in the data itself. | +| `Run` | Yes, bidirectional | Each direction carries group and row correlation in the data itself. | | `Function` | No | Callback-driven work only. | | `Script` | Yes, bidirectional | One direction carries input parameters, the other direction carries table output. | @@ -35,7 +35,7 @@ Related diagrams: | --- | --- | --- | | `get_connection` | No | Returns connection information. | | `get_script` | No | Returns script content. | -| `execute_query` | Optional | Returns query status and may carry result rows. | +| `execute_query` | Optional | Returns query status and may carry row-correlated result rows. | These `UDFRunner`-opened calls are ordinary nested calls, not a separate callback transport. @@ -52,7 +52,35 @@ payload may be sent while the call remains active or together with `CloseCall` i - opened by `DB` - may carry opening payloads and the first input batch together - may stay active while nested calls such as `get_script` or `get_connection` execute -- output row correlation for scalar returns belongs in the data, not in `Next(...)` +- group and row correlation belong in the data, not in `Next(...)` + +#### Group and Row Correlation + +Each `Run` direction may combine multiple logical groups in one `DataRecordBatch`. Its `DataSchema` sets both +`has_group_id` and `has_row_id` to `true`, adding an ordered reserved prefix before user data columns: + +| Position | Column | Purpose | +| --- | --- | --- | +| `0` | Group ID | Identifies the logical input group. | +| `1` | Row ID | Identifies the input row to which an output row maps. | +| `2+` | User data | Input or output columns defined by the call. | + +Groups may span multiple rows. In particular, a `SET ... EMITS` UDF may receive multiple input rows in one group. +The group ID and row ID columns are correlation fields identified only by this prefix layout, not by field names. + +The preferred encodings are: + +| Column and direction | Preferred encoding | Compatible fallback | +| --- | --- | --- | +| Group ID, either direction | `RunEndEncoded` over unsigned 64-bit IDs when groups contain repeated rows. | Plain unsigned 64-bit. | +| Row ID, `DB` to `UDFRunner` | `exasol.udf.range_run` extension array. | Plain unsigned 64-bit. | +| Row ID, `UDFRunner` to `DB`, `RETURNS` UDF | `exasol.udf.range_run` extension array. | Plain unsigned 64-bit. | +| Row ID, `UDFRunner` to `DB`, `EMITS` UDF | `RunEndEncoded` over unsigned 64-bit IDs. | Plain unsigned 64-bit. | + +`exasol.udf.range_run` uses `RunEndEncoded` as its Arrow storage type. Its `run_ends` child is a signed 64-bit +integer array and its unsigned 64-bit `values` child stores the first row ID for each run; each following logical +value in that run increases by one. The field sets `ARROW:extension:name` to `exasol.udf.range_run`; no extension +metadata is required in version 1. ### `Function` @@ -83,6 +111,8 @@ payload may be sent while the call remains active or together with `CloseCall` i - opened by `UDFRunner` - returns `Payloads(result_set_status, ...)` - may optionally carry a result-row data stream +- a row-correlated result stream may set `has_group_id` to `false` and `has_row_id` to `true`; its row ID then + occupies prefix position `0` ## Representative Sequences diff --git a/doc/design/v2/udf_protocol_high_level_run_sequence.mmd b/doc/design/v2/udf_protocol_high_level_run_sequence.mmd index 9036794..57f05a5 100644 --- a/doc/design/v2/udf_protocol_high_level_run_sequence.mmd +++ b/doc/design/v2/udf_protocol_high_level_run_sequence.mmd @@ -4,10 +4,10 @@ sequenceDiagram Note over DB,UDFRunner: Run call opened by DB with flow-controlled bidirectional data exchange - DB->>UDFRunner: CallOpen(Run, call_metadata) + DataMetaData + first input RecordBatch + DB->>UDFRunner: CallOpen(Run, call_metadata) + DataSchema(group ID, row ID) + first input RecordBatch UDFRunner-->>DB: Next(byte_budget, reset=false, row_id=n) - DB-->>UDFRunner: RecordBatch(input rows starting at n) - UDFRunner-->>DB: DataMetaData + first output RecordBatch with row ids for scalar correlation + DB-->>UDFRunner: RecordBatch(group ID, row ID, input rows starting at n) + UDFRunner-->>DB: DataSchema(group ID, row ID) + first output RecordBatch DB->>UDFRunner: Next(byte_budget, reset=false, row_id=m) - UDFRunner-->>DB: RecordBatch(output rows starting at m) + UDFRunner-->>DB: RecordBatch(group ID, row ID, output rows starting at m) DB->>UDFRunner: CallClose(Run) diff --git a/doc/design/v2/udf_protocol_high_level_run_sequence.svg b/doc/design/v2/udf_protocol_high_level_run_sequence.svg index 742aafd..3f72525 100644 --- a/doc/design/v2/udf_protocol_high_level_run_sequence.svg +++ b/doc/design/v2/udf_protocol_high_level_run_sequence.svg @@ -1 +1 @@ -UDFRunnerDBUDFRunnerDBRun call opened by DB with flow-controlled bidirectional data exchangeCallOpen(Run, call_metadata) + DataMetaData + first input RecordBatchNext(byte_budget, reset=false, row_id=n)RecordBatch(input rows starting at n)DataMetaData + first output RecordBatch with row ids for scalar correlationNext(byte_budget, reset=false, row_id=m)RecordBatch(output rows starting at m)CallClose(Run) \ No newline at end of file +UDFRunnerDBUDFRunnerDBRun call opened by DB with flow-controlled bidirectional data exchangeCallOpen(Run, call_metadata) + DataSchema(group ID, row ID) + first input RecordBatchNext(byte_budget, reset=false, row_id=n)RecordBatch(group ID, row ID, input rows starting at n)DataSchema(group ID, row ID) + first output RecordBatchNext(byte_budget, reset=false, row_id=m)RecordBatch(group ID, row ID, output rows starting at m)CallClose(Run) \ No newline at end of file diff --git a/udf-runner-cpp/v2/udf_protocol.fbs b/udf-runner-cpp/v2/udf_protocol.fbs index 72c7fb7..dba42f6 100644 --- a/udf-runner-cpp/v2/udf_protocol.fbs +++ b/udf-runner-cpp/v2/udf_protocol.fbs @@ -181,8 +181,14 @@ table Next { // direction's column layout as our own native Schema table (see above -- not // Arrow's own flatbuffer Schema). It is split from DataRecordBatch because the // schema is only needed once, not repeated per batch. +// +// When present, correlation fields form an ordered prefix of schema.fields: +// group ID is field 0; row ID follows it, or is field 0 when no group ID is +// present. Call-specific rules decide which combinations are required. table DataSchema { schema: Schema; + has_group_id: bool = false; + has_row_id: bool = false; } // Precedes the buffers for one RecordBatch in a direction of a call's data From 695d2b18d3f45abb4f8633d259606810e21f9640 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Mon, 17 Aug 2026 16:53:36 +0200 Subject: [PATCH 03/26] protocol: mark completed run groups --- doc/design/v2/udf_protocol_high_level_calls.md | 11 +++++++++++ .../v2/udf_protocol_high_level_run_sequence.mmd | 4 ++-- .../v2/udf_protocol_high_level_run_sequence.svg | 2 +- udf-runner-cpp/v2/udf_protocol.fbs | 2 +- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/doc/design/v2/udf_protocol_high_level_calls.md b/doc/design/v2/udf_protocol_high_level_calls.md index bc8d599..b077be9 100644 --- a/doc/design/v2/udf_protocol_high_level_calls.md +++ b/doc/design/v2/udf_protocol_high_level_calls.md @@ -68,6 +68,17 @@ Each `Run` direction may combine multiple logical groups in one `DataRecordBatch Groups may span multiple rows. In particular, a `SET ... EMITS` UDF may receive multiple input rows in one group. The group ID and row ID columns are correlation fields identified only by this prefix layout, not by field names. +`DataRecordBatch.is_end_of_group` marks whether the group identified by the batch's final row is complete. It is +defined only for a `Run` direction whose schema sets `has_group_id` to `true`: + +- `true` means no later batch in that direction contains the trailing group. +- `false` means the trailing group continues in a later batch. +- a change in group ID still delimits each non-trailing group within the same batch. +- an empty batch does not complete a group. + +This is a group-boundary marker, not an end-of-stream marker. Generic stream-completion semantics remain a +low-level open question and are not encoded in `DataRecordBatch` metadata. + The preferred encodings are: | Column and direction | Preferred encoding | Compatible fallback | diff --git a/doc/design/v2/udf_protocol_high_level_run_sequence.mmd b/doc/design/v2/udf_protocol_high_level_run_sequence.mmd index 57f05a5..3101ec2 100644 --- a/doc/design/v2/udf_protocol_high_level_run_sequence.mmd +++ b/doc/design/v2/udf_protocol_high_level_run_sequence.mmd @@ -6,8 +6,8 @@ sequenceDiagram DB->>UDFRunner: CallOpen(Run, call_metadata) + DataSchema(group ID, row ID) + first input RecordBatch UDFRunner-->>DB: Next(byte_budget, reset=false, row_id=n) - DB-->>UDFRunner: RecordBatch(group ID, row ID, input rows starting at n) + DB-->>UDFRunner: RecordBatch(group ID, row ID, input rows starting at n, is_end_of_group=true) UDFRunner-->>DB: DataSchema(group ID, row ID) + first output RecordBatch DB->>UDFRunner: Next(byte_budget, reset=false, row_id=m) - UDFRunner-->>DB: RecordBatch(group ID, row ID, output rows starting at m) + UDFRunner-->>DB: RecordBatch(group ID, row ID, output rows starting at m, is_end_of_group=true) DB->>UDFRunner: CallClose(Run) diff --git a/doc/design/v2/udf_protocol_high_level_run_sequence.svg b/doc/design/v2/udf_protocol_high_level_run_sequence.svg index 3f72525..73fdc97 100644 --- a/doc/design/v2/udf_protocol_high_level_run_sequence.svg +++ b/doc/design/v2/udf_protocol_high_level_run_sequence.svg @@ -1 +1 @@ -UDFRunnerDBUDFRunnerDBRun call opened by DB with flow-controlled bidirectional data exchangeCallOpen(Run, call_metadata) + DataSchema(group ID, row ID) + first input RecordBatchNext(byte_budget, reset=false, row_id=n)RecordBatch(group ID, row ID, input rows starting at n)DataSchema(group ID, row ID) + first output RecordBatchNext(byte_budget, reset=false, row_id=m)RecordBatch(group ID, row ID, output rows starting at m)CallClose(Run) \ No newline at end of file +UDFRunnerDBUDFRunnerDBRun call opened by DB with flow-controlled bidirectional data exchangeCallOpen(Run, call_metadata) + DataSchema(group ID, row ID) + first input RecordBatchNext(byte_budget, reset=false, row_id=n)RecordBatch(group ID, row ID, input rows starting at n, is_end_of_group=true)DataSchema(group ID, row ID) + first output RecordBatchNext(byte_budget, reset=false, row_id=m)RecordBatch(group ID, row ID, output rows starting at m, is_end_of_group=true)CallClose(Run) \ No newline at end of file diff --git a/udf-runner-cpp/v2/udf_protocol.fbs b/udf-runner-cpp/v2/udf_protocol.fbs index dba42f6..0aa481d 100644 --- a/udf-runner-cpp/v2/udf_protocol.fbs +++ b/udf-runner-cpp/v2/udf_protocol.fbs @@ -199,7 +199,7 @@ table DataSchema { // actual column buffers are transported separately per buffer_transport. table DataRecordBatch { buffer_transport: BufferTransport = Inline; - is_end: bool = false; + is_end_of_group: bool = false; metadata: RecordBatchMetadata; } From 95841aa6c66c93bfa41d73817bcf5ab262c7dcfc Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Mon, 17 Aug 2026 17:47:03 +0200 Subject: [PATCH 04/26] protocol: define high-level payload contracts --- doc/design/v2/README.md | 6 +- doc/design/v2/udf_protocol_design_draft.md | 3 +- .../v2/udf_protocol_high_level_call_model.mmd | 7 +- .../v2/udf_protocol_high_level_call_model.svg | 2 +- .../v2/udf_protocol_high_level_calls.md | 49 ++++---- .../udf_protocol_high_level_nested_calls.mmd | 13 +- .../udf_protocol_high_level_nested_calls.svg | 2 +- .../v2/udf_protocol_high_level_payloads.md | 114 ++++++++++++++++++ .../udf_protocol_high_level_run_sequence.mmd | 2 +- .../udf_protocol_high_level_run_sequence.svg | 2 +- .../v2/json_schema/call_metadata.schema.json | 46 +++++++ .../connection_information.schema.json | 12 ++ .../export_specification.schema.json | 23 ++++ .../import_specification.schema.json | 34 ++++++ udf-runner-cpp/v2/udf_protocol.fbs | 6 +- 15 files changed, 268 insertions(+), 53 deletions(-) create mode 100644 doc/design/v2/udf_protocol_high_level_payloads.md create mode 100644 udf-runner-cpp/v2/json_schema/call_metadata.schema.json create mode 100644 udf-runner-cpp/v2/json_schema/connection_information.schema.json create mode 100644 udf-runner-cpp/v2/json_schema/export_specification.schema.json create mode 100644 udf-runner-cpp/v2/json_schema/import_specification.schema.json diff --git a/doc/design/v2/README.md b/doc/design/v2/README.md index aa05f0c..d6b541d 100644 --- a/doc/design/v2/README.md +++ b/doc/design/v2/README.md @@ -7,7 +7,9 @@ This directory contains the split protocol design for the new UDF protocol. - [udf_protocol_call_lifecycle.md](udf_protocol_call_lifecycle.md) describes the generic call abstraction. - [udf_protocol_data_stream.md](udf_protocol_data_stream.md) is the low-level companion for call-attached data-stream behavior. -- [udf_protocol_high_level_calls.md](udf_protocol_high_level_calls.md) describes high-level calls such as `Run`, - `Function`, `Script`, `get_connection`, `get_script`, and `execute_query`, plus DB/UDFRunner scheduling policy. +- [udf_protocol_high_level_calls.md](udf_protocol_high_level_calls.md) describes `Run`, Function operations, + `get_connection`, `get_script`, and DB/UDFRunner scheduling policy. +- [udf_protocol_high_level_payloads.md](udf_protocol_high_level_payloads.md) defines their named string and JSON + payload contracts. Mermaid sources and rendered SVGs use matching names and scopes so the textual and visual material stays aligned. diff --git a/doc/design/v2/udf_protocol_design_draft.md b/doc/design/v2/udf_protocol_design_draft.md index 5b009f6..a46f3ed 100644 --- a/doc/design/v2/udf_protocol_design_draft.md +++ b/doc/design/v2/udf_protocol_design_draft.md @@ -6,6 +6,7 @@ Detailed protocol references in this directory: - [udf_protocol_call_lifecycle.md](udf_protocol_call_lifecycle.md) - [udf_protocol_data_stream.md](udf_protocol_data_stream.md) - [udf_protocol_high_level_calls.md](udf_protocol_high_level_calls.md) +- [udf_protocol_high_level_payloads.md](udf_protocol_high_level_payloads.md) # General Information @@ -124,7 +125,7 @@ The new protocol changes a security-sensitive interface between the database and # How Other Databases Solve That Problem -TODO: Add comparative research for Oracle, PostgreSQL, MySQL, SQL Server, and other relevant systems once the transport direction is narrowed. +TODO: Link the technology overview, details and selection reasoning # Solution Approaches diff --git a/doc/design/v2/udf_protocol_high_level_call_model.mmd b/doc/design/v2/udf_protocol_high_level_call_model.mmd index 6482bb3..99ddbae 100644 --- a/doc/design/v2/udf_protocol_high_level_call_model.mmd +++ b/doc/design/v2/udf_protocol_high_level_call_model.mmd @@ -5,12 +5,9 @@ flowchart TD Call -->|optional| Nested["Nested Call(s)\n(started by nested CallOpen traffic)"] Run["Run (DB-opened)"] -.->|instance of| Call - Function["Function (DB-opened)"] -.->|instance of| Call - Script["Script (DB-opened)"] -.->|instance of| Call + Function["Function operation (DB-opened)"] -.->|instance of| Call GetConnection["get_connection (UDFRunner-opened)"] -.->|instance of| Call GetScript["get_script (UDFRunner-opened)"] -.->|instance of| Call - ExecuteQuery["execute_query / pquery (UDFRunner-opened)"] -.->|instance of| Call Run --> RunData["Input/Output Data Stream"] - Script --> ScriptData["Parameter Input / Table Output Data Stream"] - ExecuteQuery --> QueryData["Optional Result-Row Data Stream"] + Function --> FunctionKinds["default output columns / virtual schema / import SQL / export SQL"] diff --git a/doc/design/v2/udf_protocol_high_level_call_model.svg b/doc/design/v2/udf_protocol_high_level_call_model.svg index 775dddd..5d5ccc3 100644 --- a/doc/design/v2/udf_protocol_high_level_call_model.svg +++ b/doc/design/v2/udf_protocol_high_level_call_model.svg @@ -1 +1 @@ -

opened by DB or UDFRunner

optional, at most one

optional

instance of

instance of

instance of

instance of

instance of

instance of

Call
(CallOpen / first peer call-scoped message / CallClose)

Either side may open a call

Bidirectional Data Stream
(independent schema per direction)

Nested Call(s)
(started by nested CallOpen traffic)

Run (DB-opened)

Function (DB-opened)

Script (DB-opened)

get_connection (UDFRunner-opened)

get_script (UDFRunner-opened)

execute_query / pquery (UDFRunner-opened)

Input/Output Data Stream

Parameter Input / Table Output Data Stream

Optional Result-Row Data Stream

 \ No newline at end of file +

opened by DB or UDFRunner

optional, at most one

optional

instance of

instance of

instance of

instance of

Call
(CallOpen / first peer call-scoped message / CallClose)

Either side may open a call

Bidirectional Data Stream
(independent schema per direction)

Nested Call(s)
(started by nested CallOpen traffic)

Run (DB-opened)

Function operation (DB-opened)

get_connection (UDFRunner-opened)

get_script (UDFRunner-opened)

Input/Output Data Stream

default output columns / virtual schema / import SQL / export SQL

 \ No newline at end of file diff --git a/doc/design/v2/udf_protocol_high_level_calls.md b/doc/design/v2/udf_protocol_high_level_calls.md index b077be9..80e2441 100644 --- a/doc/design/v2/udf_protocol_high_level_calls.md +++ b/doc/design/v2/udf_protocol_high_level_calls.md @@ -6,8 +6,8 @@ This document describes the high-level protocol calls built on top of the generi This document covers: -- `Run`, `Function`, and `Script` -- `get_connection`, `get_script`, and `execute_query` +- `Run` and Function operations +- `get_connection` and `get_script` - which calls carry data streams - representative message sequences - DB/UDFRunner scheduling policy @@ -26,8 +26,7 @@ Related diagrams: | Call | Data stream | Notes | | --- | --- | --- | | `Run` | Yes, bidirectional | Each direction carries group and row correlation in the data itself. | -| `Function` | No | Callback-driven work only. | -| `Script` | Yes, bidirectional | One direction carries input parameters, the other direction carries table output. | +| Function operation | No | One of `default_output_columns`, `virtual_schema_adapter`, `generate_sql_for_import_spec`, or `generate_sql_for_export_spec`. | ### `UDFRunner`-opened calls @@ -35,7 +34,6 @@ Related diagrams: | --- | --- | --- | | `get_connection` | No | Returns connection information. | | `get_script` | No | Returns script content. | -| `execute_query` | Optional | Returns query status and may carry row-correlated result rows. | These `UDFRunner`-opened calls are ordinary nested calls, not a separate callback transport. @@ -50,7 +48,7 @@ payload may be sent while the call remains active or together with `CloseCall` i ### `Run` - opened by `DB` -- may carry opening payloads and the first input batch together +- carries `call_metadata` with the opening message and may carry the first input batch together - may stay active while nested calls such as `get_script` or `get_connection` execute - group and row correlation belong in the data, not in `Next(...)` @@ -93,45 +91,38 @@ integer array and its unsigned 64-bit `values` child stores the first row ID for value in that run increases by one. The field sets `ARROW:extension:name` to `exasol.udf.range_run`; no extension metadata is required in version 1. -### `Function` +### Function Operations -- opened by `DB` +- opened by `DB` with one of the Function operation names +- each call carries `call_metadata` with the opening message - has no attached data stream in the current model -- relies on nested calls for callback-style interaction - -### `Script` - -- opened by `DB` -- uses one bidirectional data stream -- input and output directions may use different schemas +- has the operation-specific request and result payloads defined in + [udf_protocol_high_level_payloads.md](udf_protocol_high_level_payloads.md) ### `get_connection` - opened by `UDFRunner` -- returns `Payloads(connection_information, ...)` +- returns `Payloads(connection_information)` - may still carry additional named payload traffic while active ### `get_script` - opened by `UDFRunner` -- returns `Payloads(script, ...)` +- returns `Payloads(script)` - may still carry additional named payload traffic while active -### `execute_query` +## Payload Contracts -- opened by `UDFRunner` -- returns `Payloads(result_set_status, ...)` -- may optionally carry a result-row data stream -- a row-correlated result stream may set `has_group_id` to `false` and `has_row_id` to `true`; its row ID then - occupies prefix position `0` +The complete call-metadata, script-metadata, Function, and nested-call payload contracts are defined in +[udf_protocol_high_level_payloads.md](udf_protocol_high_level_payloads.md). `StringPayload` is used directly for +scalar strings; JSON is used only where a payload has structured fields. ## Representative Sequences The current design keeps the high-level sequences intentionally simple: -- nested callback-style calls execute while a parent `Run`, `Function`, or `Script` call remains active -- `Run` may combine opening call metadata, input schema announcement, and the first input batch -- `execute_query` may or may not open a result-row data stream after its status result +- nested callback-style calls execute while a parent `Run` or Function call remains active +- `Run` combines `OpenCall`, `call_metadata`, input schema announcement, and the first input batch when practical See [udf_protocol_high_level_nested_calls.svg](udf_protocol_high_level_nested_calls.svg) and [udf_protocol_high_level_run_sequence.svg](udf_protocol_high_level_run_sequence.svg). @@ -157,8 +148,9 @@ call orchestration and do not alter the generic Client/Server stream rules in th See [udf_protocol_high_level_endpoint_scheduling.svg](udf_protocol_high_level_endpoint_scheduling.svg). ## Forward-Looking Ideas Still Open - -- whether `UDFRunner` may open its own pquery-style call to `DB` beyond the current `execute_query` shape +<> +- `Script` and `execute_query` call shapes and data streams +- whether `UDFRunner` may open its own pquery-style call to `DB` - whether table-prefetch-like declarations should be added for future call setup ## Relationship To Other Docs @@ -166,3 +158,4 @@ See [udf_protocol_high_level_endpoint_scheduling.svg](udf_protocol_high_level_en - low-level framing and generic stream rules live in [udf_protocol_low_level.md](udf_protocol_low_level.md) - generic open/close call behavior lives in [udf_protocol_call_lifecycle.md](udf_protocol_call_lifecycle.md) - generic data-stream behavior lives in [udf_protocol_data_stream.md](udf_protocol_data_stream.md) +- high-level payload contracts live in [udf_protocol_high_level_payloads.md](udf_protocol_high_level_payloads.md) diff --git a/doc/design/v2/udf_protocol_high_level_nested_calls.mmd b/doc/design/v2/udf_protocol_high_level_nested_calls.mmd index 76f2bf7..d9de5dc 100644 --- a/doc/design/v2/udf_protocol_high_level_nested_calls.mmd +++ b/doc/design/v2/udf_protocol_high_level_nested_calls.mmd @@ -4,14 +4,9 @@ sequenceDiagram Note over UDFRunner,DB: Nested calls opened by UDFRunner while a parent DB-opened call remains active - UDFRunner->>DB: CallOpen(get_connection, {connection_name}) - DB-->>UDFRunner: Payloads(connection_information, ...) + UDFRunner->>DB: CallOpen(get_connection) + Payloads(connection_name) + DB-->>UDFRunner: Payloads(connection_information) UDFRunner->>DB: CloseCall - UDFRunner->>DB: CallOpen(get_script, {script_name}) - DB-->>UDFRunner: Payloads(script, ...) + CloseCall - - UDFRunner->>DB: CallOpen(execute_query, {query}) - DB-->>UDFRunner: Payloads(result_set_status, ...) - DB-->>UDFRunner: DataMetaData + RecordBatch(...) on the call's data stream - UDFRunner->>DB: CloseCall + UDFRunner->>DB: CallOpen(get_script) + Payloads(script_name) + DB-->>UDFRunner: Payloads(script) + CloseCall diff --git a/doc/design/v2/udf_protocol_high_level_nested_calls.svg b/doc/design/v2/udf_protocol_high_level_nested_calls.svg index 57b0557..7a77e9c 100644 --- a/doc/design/v2/udf_protocol_high_level_nested_calls.svg +++ b/doc/design/v2/udf_protocol_high_level_nested_calls.svg @@ -1 +1 @@ -DBUDFRunnerDBUDFRunnerNested calls opened by UDFRunner while a parent DB-opened call remains activeCallOpen(get_connection, {connection_name})Payloads(connection_information, ...)CloseCallCallOpen(get_script, {script_name})Payloads(script, ...) + CloseCallCallOpen(execute_query, {query})Payloads(result_set_status, ...)DataMetaData + RecordBatch(...) on the call's data streamCloseCall \ No newline at end of file +DBUDFRunnerDBUDFRunnerNested calls opened by UDFRunner while a parent DB-opened call remains activeCallOpen(get_connection) + Payloads(connection_name)Payloads(connection_information)CloseCallCallOpen(get_script) + Payloads(script_name)Payloads(script) + CloseCall \ No newline at end of file diff --git a/doc/design/v2/udf_protocol_high_level_payloads.md b/doc/design/v2/udf_protocol_high_level_payloads.md new file mode 100644 index 0000000..72e83e6 --- /dev/null +++ b/doc/design/v2/udf_protocol_high_level_payloads.md @@ -0,0 +1,114 @@ +# UDF Protocol v2: High-Level Payload Contracts + +This document defines the named payloads used by the current high-level protocol. Every payload is carried in +`Payloads(...)`. A scalar value uses `StringPayload` directly. A payload marked JSON uses a `StringPayload` whose +value is UTF-8 JSON conforming to its linked schema. + +## Script Metadata + +After the `Server` sends `ServerCapabilities`, `DB` may set the connection's script metadata on control stream +`0`. It sends these `StringPayload` values together or in separate control messages: + +| Name | Value | Meaning | +| --- | --- | --- | +| `script_name` | string | Name of the script used by subsequent `Run` or Function calls. | +| `script_source` | string | Source code of that script. | + +Script metadata must be sent before a `Run` or Function call that uses it. It may be replaced only between such +calls; the last received values apply to subsequent calls. It is not carried on a `Run` or Function stream. + +## Call Metadata + +Every `Run` and Function call carries one `call_metadata` JSON payload in the same `StreamMessage` as `OpenCall`. +It conforms to [call_metadata.schema.json](../../../udf-runner-cpp/v2/json_schema/call_metadata.schema.json) and +supplies the per-invocation execution context and the input/output iterator and column definitions. The script name +and source are deliberately excluded because they are connection-scoped script metadata. + +```json +{ + "database_name": "EXASOL", + "database_version": "8.0", + "session_id": "42", + "statement_id": 1, + "node_count": 1, + "node_id": 0, + "vm_id": "7", + "maximal_memory_limit": "1073741824", + "script_schema": "SYS", + "input_iter_type": "EXACTLY_ONCE", + "output_iter_type": "EXACTLY_ONCE", + "input_columns": [], + "output_columns": [], + "single_call_mode": false +} +``` + +Unsigned 64-bit values are decimal strings so JSON implementations do not lose precision. + +## Function Calls + +`Function` is a family of DB-opened, non-streaming calls. Each operation uses its operation name as +`OpenCall.call_name`, carries `call_metadata`, and has the following operation-specific payload contract. + +| Call name | Request payload | Result payload | +| --- | --- | --- | +| `default_output_columns` | None | `default_output_columns_result`: `StringPayload` | +| `virtual_schema_adapter` | `virtual_schema_request`: `StringPayload` | `virtual_schema_result`: `StringPayload` | +| `generate_sql_for_import_spec` | `import_specification`: JSON | `import_specification_result`: `StringPayload` | +| `generate_sql_for_export_spec` | `export_specification`: JSON | `export_specification_result`: `StringPayload` | + +The virtual-schema request and all Function results retain their existing string representation. Their contents are +defined by the respective script API, not by this transport protocol. + +`import_specification` conforms to +[import_specification.schema.json](../../../udf-runner-cpp/v2/json_schema/import_specification.schema.json). +Example: + +```json +{ + "is_subselect": true, + "connection_name": "REMOTE_CONNECTION", + "subselect_column_specification": [{ "name": "ID", "type_name": "DECIMAL(18,0)" }], + "parameters": [{ "key": "encoding", "value": "UTF-8" }] +} +``` + +`export_specification` conforms to +[export_specification.schema.json](../../../udf-runner-cpp/v2/json_schema/export_specification.schema.json). +Example: + +```json +{ + "has_truncate": true, + "has_replace": false, + "source_column_names": ["ID", "NAME"], + "connection_name": "REMOTE_CONNECTION" +} +``` + +## Nested Calls + +`UDFRunner` may open these nested calls while a `Run` or Function call is active: + +| Call name | Request payload | Result payload | +| --- | --- | --- | +| `get_connection` | `connection_name`: `StringPayload` | `connection_information`: JSON | +| `get_script` | `script_name`: `StringPayload` | `script`: `StringPayload` | + +`connection_information` conforms to +[connection_information.schema.json](../../../udf-runner-cpp/v2/json_schema/connection_information.schema.json). +Example: + +```json +{ + "kind": "JDBC", + "address": "jdbc:example://host/database", + "user": "user", + "password": "secret" +} +``` + +## Deferred Calls + +`Script`, `execute_script`, and `execute_query` are not part of the current normative call set. They remain future +extensions and have no payload or data-stream contract in this version. diff --git a/doc/design/v2/udf_protocol_high_level_run_sequence.mmd b/doc/design/v2/udf_protocol_high_level_run_sequence.mmd index 3101ec2..2aafd72 100644 --- a/doc/design/v2/udf_protocol_high_level_run_sequence.mmd +++ b/doc/design/v2/udf_protocol_high_level_run_sequence.mmd @@ -4,7 +4,7 @@ sequenceDiagram Note over DB,UDFRunner: Run call opened by DB with flow-controlled bidirectional data exchange - DB->>UDFRunner: CallOpen(Run, call_metadata) + DataSchema(group ID, row ID) + first input RecordBatch + DB->>UDFRunner: CallOpen(Run) + Payloads(call_metadata) + DataSchema(group ID, row ID) + first input RecordBatch UDFRunner-->>DB: Next(byte_budget, reset=false, row_id=n) DB-->>UDFRunner: RecordBatch(group ID, row ID, input rows starting at n, is_end_of_group=true) UDFRunner-->>DB: DataSchema(group ID, row ID) + first output RecordBatch diff --git a/doc/design/v2/udf_protocol_high_level_run_sequence.svg b/doc/design/v2/udf_protocol_high_level_run_sequence.svg index 73fdc97..f3f50aa 100644 --- a/doc/design/v2/udf_protocol_high_level_run_sequence.svg +++ b/doc/design/v2/udf_protocol_high_level_run_sequence.svg @@ -1 +1 @@ -UDFRunnerDBUDFRunnerDBRun call opened by DB with flow-controlled bidirectional data exchangeCallOpen(Run, call_metadata) + DataSchema(group ID, row ID) + first input RecordBatchNext(byte_budget, reset=false, row_id=n)RecordBatch(group ID, row ID, input rows starting at n, is_end_of_group=true)DataSchema(group ID, row ID) + first output RecordBatchNext(byte_budget, reset=false, row_id=m)RecordBatch(group ID, row ID, output rows starting at m, is_end_of_group=true)CallClose(Run) \ No newline at end of file +UDFRunnerDBUDFRunnerDBRun call opened by DB with flow-controlled bidirectional data exchangeCallOpen(Run) + Payloads(call_metadata) + DataSchema(group ID, row ID) + first input RecordBatchNext(byte_budget, reset=false, row_id=n)RecordBatch(group ID, row ID, input rows starting at n, is_end_of_group=true)DataSchema(group ID, row ID) + first output RecordBatchNext(byte_budget, reset=false, row_id=m)RecordBatch(group ID, row ID, output rows starting at m, is_end_of_group=true)CallClose(Run) \ No newline at end of file diff --git a/udf-runner-cpp/v2/json_schema/call_metadata.schema.json b/udf-runner-cpp/v2/json_schema/call_metadata.schema.json new file mode 100644 index 0000000..aae1556 --- /dev/null +++ b/udf-runner-cpp/v2/json_schema/call_metadata.schema.json @@ -0,0 +1,46 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": true, + "required": [ + "database_name", "database_version", "session_id", "statement_id", + "node_count", "node_id", "vm_id", "maximal_memory_limit", "script_schema", + "input_iter_type", "output_iter_type", "input_columns", "output_columns", + "single_call_mode" + ], + "properties": { + "database_name": { "type": "string" }, + "database_version": { "type": "string" }, + "session_id": { "type": "string", "pattern": "^[0-9]+$" }, + "statement_id": { "type": "integer", "minimum": 0, "maximum": 4294967295 }, + "node_count": { "type": "integer", "minimum": 0, "maximum": 4294967295 }, + "node_id": { "type": "integer", "minimum": 0, "maximum": 4294967295 }, + "vm_id": { "type": "string", "pattern": "^[0-9]+$" }, + "maximal_memory_limit": { "type": "string", "pattern": "^[0-9]+$" }, + "meta_info": { "type": "string" }, + "script_schema": { "type": "string" }, + "current_user": { "type": "string" }, + "current_schema": { "type": "string" }, + "scope_user": { "type": "string" }, + "input_iter_type": { "enum": ["EXACTLY_ONCE", "MULTIPLE"] }, + "output_iter_type": { "enum": ["EXACTLY_ONCE", "MULTIPLE"] }, + "input_columns": { "type": "array", "items": { "$ref": "#/$defs/column" } }, + "output_columns": { "type": "array", "items": { "$ref": "#/$defs/column" } }, + "single_call_mode": { "type": "boolean" } + }, + "$defs": { + "column": { + "type": "object", + "additionalProperties": true, + "required": ["name", "type_name"], + "properties": { + "name": { "type": "string" }, + "type": { "enum": ["UNSUPPORTED", "DOUBLE", "INT32", "INT64", "NUMERIC", "TIMESTAMP", "DATE", "STRING", "BOOLEAN"] }, + "type_name": { "type": "string" }, + "size": { "type": "integer", "minimum": 0, "maximum": 4294967295 }, + "precision": { "type": "integer", "minimum": 0, "maximum": 4294967295 }, + "scale": { "type": "integer", "minimum": 0, "maximum": 4294967295 } + } + } + } +} diff --git a/udf-runner-cpp/v2/json_schema/connection_information.schema.json b/udf-runner-cpp/v2/json_schema/connection_information.schema.json new file mode 100644 index 0000000..4084a01 --- /dev/null +++ b/udf-runner-cpp/v2/json_schema/connection_information.schema.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": true, + "required": ["kind", "address", "user", "password"], + "properties": { + "kind": { "type": "string" }, + "address": { "type": "string" }, + "user": { "type": "string" }, + "password": { "type": "string" } + } +} diff --git a/udf-runner-cpp/v2/json_schema/export_specification.schema.json b/udf-runner-cpp/v2/json_schema/export_specification.schema.json new file mode 100644 index 0000000..0b489ea --- /dev/null +++ b/udf-runner-cpp/v2/json_schema/export_specification.schema.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": true, + "required": ["has_truncate", "has_replace"], + "properties": { + "has_truncate": { "type": "boolean" }, + "has_replace": { "type": "boolean" }, + "created_by": { "type": "string" }, + "source_column_names": { "type": "array", "items": { "type": "string" } }, + "connection_information": { "$ref": "connection_information.schema.json" }, + "connection_name": { "type": "string" }, + "parameters": { "type": "array", "items": { "$ref": "#/$defs/key_value" } } + }, + "$defs": { + "key_value": { + "type": "object", + "additionalProperties": true, + "required": ["key", "value"], + "properties": { "key": { "type": "string" }, "value": { "type": "string" } } + } + } +} diff --git a/udf-runner-cpp/v2/json_schema/import_specification.schema.json b/udf-runner-cpp/v2/json_schema/import_specification.schema.json new file mode 100644 index 0000000..db758db --- /dev/null +++ b/udf-runner-cpp/v2/json_schema/import_specification.schema.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": true, + "required": ["is_subselect"], + "properties": { + "is_subselect": { "type": "boolean" }, + "connection_information": { "$ref": "connection_information.schema.json" }, + "connection_name": { "type": "string" }, + "subselect_column_specification": { "type": "array", "items": { "$ref": "#/$defs/column" } }, + "parameters": { "type": "array", "items": { "$ref": "#/$defs/key_value" } } + }, + "$defs": { + "column": { + "type": "object", + "additionalProperties": true, + "required": ["name", "type_name"], + "properties": { + "name": { "type": "string" }, + "type": { "enum": ["UNSUPPORTED", "DOUBLE", "INT32", "INT64", "NUMERIC", "TIMESTAMP", "DATE", "STRING", "BOOLEAN"] }, + "type_name": { "type": "string" }, + "size": { "type": "integer", "minimum": 0, "maximum": 4294967295 }, + "precision": { "type": "integer", "minimum": 0, "maximum": 4294967295 }, + "scale": { "type": "integer", "minimum": 0, "maximum": 4294967295 } + } + }, + "key_value": { + "type": "object", + "additionalProperties": true, + "required": ["key", "value"], + "properties": { "key": { "type": "string" }, "value": { "type": "string" } } + } + } +} diff --git a/udf-runner-cpp/v2/udf_protocol.fbs b/udf-runner-cpp/v2/udf_protocol.fbs index 0aa481d..14d3f01 100644 --- a/udf-runner-cpp/v2/udf_protocol.fbs +++ b/udf-runner-cpp/v2/udf_protocol.fbs @@ -93,6 +93,8 @@ table Version { table ServerCapabilities { supported_version: Version; + // TODO: high level protocol + // TODO: number of worker } table KeepAlive {} @@ -118,10 +120,6 @@ union PayloadValue { // transport session, so it is not encoded in Frame. The same numeric stream_id // on a different connection identifies a different logical stream. // -// TODO: one call = one stream_id is a draft-level assumption to fill the gap -// between the prose model and an actual wire encoding, not a settled decision -// from the source docs. -// // A call opened while another is active (e.g. a callback opened during Run) is // just another independent call -- no parent/child link is carried on the wire. // OpenCall carries no payload of its own: opening payloads travel as From 486798b4a8b85c582a236e61962c7bd20d2dbf66 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Mon, 17 Aug 2026 17:55:35 +0200 Subject: [PATCH 05/26] protocol: remove legacy metadata fields --- doc/design/v2/udf_protocol_high_level_payloads.md | 4 ++-- udf-runner-cpp/v2/json_schema/call_metadata.schema.json | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/design/v2/udf_protocol_high_level_payloads.md b/doc/design/v2/udf_protocol_high_level_payloads.md index 72e83e6..4d0083a 100644 --- a/doc/design/v2/udf_protocol_high_level_payloads.md +++ b/doc/design/v2/udf_protocol_high_level_payloads.md @@ -110,5 +110,5 @@ Example: ## Deferred Calls -`Script`, `execute_script`, and `execute_query` are not part of the current normative call set. They remain future -extensions and have no payload or data-stream contract in this version. +`Script` and `execute_query` are not part of the current normative call set. They remain future extensions and +have no payload or data-stream contract in this version. diff --git a/udf-runner-cpp/v2/json_schema/call_metadata.schema.json b/udf-runner-cpp/v2/json_schema/call_metadata.schema.json index aae1556..036a8ac 100644 --- a/udf-runner-cpp/v2/json_schema/call_metadata.schema.json +++ b/udf-runner-cpp/v2/json_schema/call_metadata.schema.json @@ -17,7 +17,6 @@ "node_id": { "type": "integer", "minimum": 0, "maximum": 4294967295 }, "vm_id": { "type": "string", "pattern": "^[0-9]+$" }, "maximal_memory_limit": { "type": "string", "pattern": "^[0-9]+$" }, - "meta_info": { "type": "string" }, "script_schema": { "type": "string" }, "current_user": { "type": "string" }, "current_schema": { "type": "string" }, From 35c21be050fefaa74c14857337ef6a248b0b5cb6 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Tue, 18 Aug 2026 12:41:54 +0200 Subject: [PATCH 06/26] Use Draft-07 v2 JSON schemas --- udf-runner-cpp/v2/json_schema/call_metadata.schema.json | 8 ++++---- .../v2/json_schema/connection_information.schema.json | 2 +- .../v2/json_schema/export_specification.schema.json | 6 +++--- .../v2/json_schema/import_specification.schema.json | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/udf-runner-cpp/v2/json_schema/call_metadata.schema.json b/udf-runner-cpp/v2/json_schema/call_metadata.schema.json index 036a8ac..3f65e60 100644 --- a/udf-runner-cpp/v2/json_schema/call_metadata.schema.json +++ b/udf-runner-cpp/v2/json_schema/call_metadata.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft/2020-12/schema", + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "additionalProperties": true, "required": [ @@ -23,11 +23,11 @@ "scope_user": { "type": "string" }, "input_iter_type": { "enum": ["EXACTLY_ONCE", "MULTIPLE"] }, "output_iter_type": { "enum": ["EXACTLY_ONCE", "MULTIPLE"] }, - "input_columns": { "type": "array", "items": { "$ref": "#/$defs/column" } }, - "output_columns": { "type": "array", "items": { "$ref": "#/$defs/column" } }, + "input_columns": { "type": "array", "items": { "$ref": "#/definitions/column" } }, + "output_columns": { "type": "array", "items": { "$ref": "#/definitions/column" } }, "single_call_mode": { "type": "boolean" } }, - "$defs": { + "definitions": { "column": { "type": "object", "additionalProperties": true, diff --git a/udf-runner-cpp/v2/json_schema/connection_information.schema.json b/udf-runner-cpp/v2/json_schema/connection_information.schema.json index 4084a01..0f45f10 100644 --- a/udf-runner-cpp/v2/json_schema/connection_information.schema.json +++ b/udf-runner-cpp/v2/json_schema/connection_information.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft/2020-12/schema", + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "additionalProperties": true, "required": ["kind", "address", "user", "password"], diff --git a/udf-runner-cpp/v2/json_schema/export_specification.schema.json b/udf-runner-cpp/v2/json_schema/export_specification.schema.json index 0b489ea..ec993b9 100644 --- a/udf-runner-cpp/v2/json_schema/export_specification.schema.json +++ b/udf-runner-cpp/v2/json_schema/export_specification.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft/2020-12/schema", + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "additionalProperties": true, "required": ["has_truncate", "has_replace"], @@ -10,9 +10,9 @@ "source_column_names": { "type": "array", "items": { "type": "string" } }, "connection_information": { "$ref": "connection_information.schema.json" }, "connection_name": { "type": "string" }, - "parameters": { "type": "array", "items": { "$ref": "#/$defs/key_value" } } + "parameters": { "type": "array", "items": { "$ref": "#/definitions/key_value" } } }, - "$defs": { + "definitions": { "key_value": { "type": "object", "additionalProperties": true, diff --git a/udf-runner-cpp/v2/json_schema/import_specification.schema.json b/udf-runner-cpp/v2/json_schema/import_specification.schema.json index db758db..af4de2f 100644 --- a/udf-runner-cpp/v2/json_schema/import_specification.schema.json +++ b/udf-runner-cpp/v2/json_schema/import_specification.schema.json @@ -1,5 +1,5 @@ { - "$schema": "https://json-schema.org/draft/2020-12/schema", + "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "additionalProperties": true, "required": ["is_subselect"], @@ -7,10 +7,10 @@ "is_subselect": { "type": "boolean" }, "connection_information": { "$ref": "connection_information.schema.json" }, "connection_name": { "type": "string" }, - "subselect_column_specification": { "type": "array", "items": { "$ref": "#/$defs/column" } }, - "parameters": { "type": "array", "items": { "$ref": "#/$defs/key_value" } } + "subselect_column_specification": { "type": "array", "items": { "$ref": "#/definitions/column" } }, + "parameters": { "type": "array", "items": { "$ref": "#/definitions/key_value" } } }, - "$defs": { + "definitions": { "column": { "type": "object", "additionalProperties": true, From 90b51c4bce3cee5eb59efafcb64e844350fb30ff Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Tue, 18 Aug 2026 20:55:48 +0200 Subject: [PATCH 07/26] Reorganize v2 protocol design docs --- doc/design/v2/README.md | 12 ++++----- .../call_lifecycle.md} | 4 +-- .../call_lifecycle.mmd} | 0 .../call_lifecycle.svg} | 0 .../data_stream.md} | 2 +- .../data_stream_flow.mmd} | 0 .../data_stream_flow.svg} | 0 .../design_draft.md} | 10 +++---- .../high_level_call_model.mmd} | 0 .../high_level_call_model.svg} | 0 .../high_level_calls.md} | 26 +++++++++---------- .../high_level_endpoint_scheduling.mmd} | 0 .../high_level_endpoint_scheduling.svg} | 0 .../high_level_nested_calls.mmd} | 0 .../high_level_nested_calls.svg} | 0 .../high_level_payloads.md} | 8 +++--- .../high_level_run_sequence.mmd} | 0 .../high_level_run_sequence.svg} | 0 .../low_level.md} | 8 +++--- .../low_level_connection_lifecycle.mmd} | 0 .../low_level_connection_lifecycle.svg} | 0 .../transport_technology_overview.md} | 0 22 files changed, 35 insertions(+), 35 deletions(-) rename doc/design/v2/{udf_protocol_call_lifecycle.md => protocol/call_lifecycle.md} (96%) rename doc/design/v2/{udf_protocol_call_lifecycle.mmd => protocol/call_lifecycle.mmd} (100%) rename doc/design/v2/{udf_protocol_call_lifecycle.svg => protocol/call_lifecycle.svg} (100%) rename doc/design/v2/{udf_protocol_data_stream.md => protocol/data_stream.md} (98%) rename doc/design/v2/{udf_protocol_data_stream_flow.mmd => protocol/data_stream_flow.mmd} (100%) rename doc/design/v2/{udf_protocol_data_stream_flow.svg => protocol/data_stream_flow.svg} (100%) rename doc/design/v2/{udf_protocol_design_draft.md => protocol/design_draft.md} (97%) rename doc/design/v2/{udf_protocol_high_level_call_model.mmd => protocol/high_level_call_model.mmd} (100%) rename doc/design/v2/{udf_protocol_high_level_call_model.svg => protocol/high_level_call_model.svg} (100%) rename doc/design/v2/{udf_protocol_high_level_calls.md => protocol/high_level_calls.md} (82%) rename doc/design/v2/{udf_protocol_high_level_endpoint_scheduling.mmd => protocol/high_level_endpoint_scheduling.mmd} (100%) rename doc/design/v2/{udf_protocol_high_level_endpoint_scheduling.svg => protocol/high_level_endpoint_scheduling.svg} (100%) rename doc/design/v2/{udf_protocol_high_level_nested_calls.mmd => protocol/high_level_nested_calls.mmd} (100%) rename doc/design/v2/{udf_protocol_high_level_nested_calls.svg => protocol/high_level_nested_calls.svg} (100%) rename doc/design/v2/{udf_protocol_high_level_payloads.md => protocol/high_level_payloads.md} (89%) rename doc/design/v2/{udf_protocol_high_level_run_sequence.mmd => protocol/high_level_run_sequence.mmd} (100%) rename doc/design/v2/{udf_protocol_high_level_run_sequence.svg => protocol/high_level_run_sequence.svg} (100%) rename doc/design/v2/{udf_protocol_low_level.md => protocol/low_level.md} (94%) rename doc/design/v2/{udf_protocol_low_level_connection_lifecycle.mmd => protocol/low_level_connection_lifecycle.mmd} (100%) rename doc/design/v2/{udf_protocol_low_level_connection_lifecycle.svg => protocol/low_level_connection_lifecycle.svg} (100%) rename doc/design/v2/{udf_transport_technology_overview.md => protocol/transport_technology_overview.md} (100%) diff --git a/doc/design/v2/README.md b/doc/design/v2/README.md index d6b541d..eeff9ba 100644 --- a/doc/design/v2/README.md +++ b/doc/design/v2/README.md @@ -2,14 +2,14 @@ This directory contains the split protocol design for the new UDF protocol. -- [udf_protocol_design_draft.md](udf_protocol_design_draft.md) is the umbrella design draft. -- [udf_protocol_low_level.md](udf_protocol_low_level.md) describes the wire-level and control-stream rules. -- [udf_protocol_call_lifecycle.md](udf_protocol_call_lifecycle.md) describes the generic call abstraction. -- [udf_protocol_data_stream.md](udf_protocol_data_stream.md) is the low-level companion for call-attached +- [protocol/design_draft.md](protocol/design_draft.md) is the umbrella design draft. +- [protocol/low_level.md](protocol/low_level.md) describes the wire-level and control-stream rules. +- [protocol/call_lifecycle.md](protocol/call_lifecycle.md) describes the generic call abstraction. +- [protocol/data_stream.md](protocol/data_stream.md) is the low-level companion for call-attached data-stream behavior. -- [udf_protocol_high_level_calls.md](udf_protocol_high_level_calls.md) describes `Run`, Function operations, +- [protocol/high_level_calls.md](protocol/high_level_calls.md) describes `Run`, Function operations, `get_connection`, `get_script`, and DB/UDFRunner scheduling policy. -- [udf_protocol_high_level_payloads.md](udf_protocol_high_level_payloads.md) defines their named string and JSON +- [protocol/high_level_payloads.md](protocol/high_level_payloads.md) defines their named string and JSON payload contracts. Mermaid sources and rendered SVGs use matching names and scopes so the textual and visual material stays aligned. diff --git a/doc/design/v2/udf_protocol_call_lifecycle.md b/doc/design/v2/protocol/call_lifecycle.md similarity index 96% rename from doc/design/v2/udf_protocol_call_lifecycle.md rename to doc/design/v2/protocol/call_lifecycle.md index 0031764..cd33367 100644 --- a/doc/design/v2/udf_protocol_call_lifecycle.md +++ b/doc/design/v2/protocol/call_lifecycle.md @@ -14,7 +14,7 @@ This document covers: Related diagram: -- [udf_protocol_call_lifecycle.svg](udf_protocol_call_lifecycle.svg) +- [call_lifecycle.svg](call_lifecycle.svg) ## Call Abstraction @@ -88,7 +88,7 @@ Nested calls are a generic mechanism, not a special-case callback transport: ## Relationship To Data Streams This document only records that a call may carry at most one bidirectional data stream. The detailed data-stream -rules are defined in [udf_protocol_data_stream.md](udf_protocol_data_stream.md). +rules are defined in [data_stream.md](data_stream.md). ## Open Questions diff --git a/doc/design/v2/udf_protocol_call_lifecycle.mmd b/doc/design/v2/protocol/call_lifecycle.mmd similarity index 100% rename from doc/design/v2/udf_protocol_call_lifecycle.mmd rename to doc/design/v2/protocol/call_lifecycle.mmd diff --git a/doc/design/v2/udf_protocol_call_lifecycle.svg b/doc/design/v2/protocol/call_lifecycle.svg similarity index 100% rename from doc/design/v2/udf_protocol_call_lifecycle.svg rename to doc/design/v2/protocol/call_lifecycle.svg diff --git a/doc/design/v2/udf_protocol_data_stream.md b/doc/design/v2/protocol/data_stream.md similarity index 98% rename from doc/design/v2/udf_protocol_data_stream.md rename to doc/design/v2/protocol/data_stream.md index e2b1599..6df99fd 100644 --- a/doc/design/v2/udf_protocol_data_stream.md +++ b/doc/design/v2/protocol/data_stream.md @@ -15,7 +15,7 @@ This document covers: Related diagram: -- [udf_protocol_data_stream_flow.svg](udf_protocol_data_stream_flow.svg) +- [data_stream_flow.svg](data_stream_flow.svg) ## Model diff --git a/doc/design/v2/udf_protocol_data_stream_flow.mmd b/doc/design/v2/protocol/data_stream_flow.mmd similarity index 100% rename from doc/design/v2/udf_protocol_data_stream_flow.mmd rename to doc/design/v2/protocol/data_stream_flow.mmd diff --git a/doc/design/v2/udf_protocol_data_stream_flow.svg b/doc/design/v2/protocol/data_stream_flow.svg similarity index 100% rename from doc/design/v2/udf_protocol_data_stream_flow.svg rename to doc/design/v2/protocol/data_stream_flow.svg diff --git a/doc/design/v2/udf_protocol_design_draft.md b/doc/design/v2/protocol/design_draft.md similarity index 97% rename from doc/design/v2/udf_protocol_design_draft.md rename to doc/design/v2/protocol/design_draft.md index a46f3ed..857ab4a 100644 --- a/doc/design/v2/udf_protocol_design_draft.md +++ b/doc/design/v2/protocol/design_draft.md @@ -2,11 +2,11 @@ Detailed protocol references in this directory: -- [udf_protocol_low_level.md](udf_protocol_low_level.md) -- [udf_protocol_call_lifecycle.md](udf_protocol_call_lifecycle.md) -- [udf_protocol_data_stream.md](udf_protocol_data_stream.md) -- [udf_protocol_high_level_calls.md](udf_protocol_high_level_calls.md) -- [udf_protocol_high_level_payloads.md](udf_protocol_high_level_payloads.md) +- [low_level.md](low_level.md) +- [call_lifecycle.md](call_lifecycle.md) +- [data_stream.md](data_stream.md) +- [high_level_calls.md](high_level_calls.md) +- [high_level_payloads.md](high_level_payloads.md) # General Information diff --git a/doc/design/v2/udf_protocol_high_level_call_model.mmd b/doc/design/v2/protocol/high_level_call_model.mmd similarity index 100% rename from doc/design/v2/udf_protocol_high_level_call_model.mmd rename to doc/design/v2/protocol/high_level_call_model.mmd diff --git a/doc/design/v2/udf_protocol_high_level_call_model.svg b/doc/design/v2/protocol/high_level_call_model.svg similarity index 100% rename from doc/design/v2/udf_protocol_high_level_call_model.svg rename to doc/design/v2/protocol/high_level_call_model.svg diff --git a/doc/design/v2/udf_protocol_high_level_calls.md b/doc/design/v2/protocol/high_level_calls.md similarity index 82% rename from doc/design/v2/udf_protocol_high_level_calls.md rename to doc/design/v2/protocol/high_level_calls.md index 80e2441..c899f8d 100644 --- a/doc/design/v2/udf_protocol_high_level_calls.md +++ b/doc/design/v2/protocol/high_level_calls.md @@ -14,10 +14,10 @@ This document covers: Related diagrams: -- [udf_protocol_high_level_call_model.svg](udf_protocol_high_level_call_model.svg) -- [udf_protocol_high_level_nested_calls.svg](udf_protocol_high_level_nested_calls.svg) -- [udf_protocol_high_level_run_sequence.svg](udf_protocol_high_level_run_sequence.svg) -- [udf_protocol_high_level_endpoint_scheduling.svg](udf_protocol_high_level_endpoint_scheduling.svg) +- [high_level_call_model.svg](high_level_call_model.svg) +- [high_level_nested_calls.svg](high_level_nested_calls.svg) +- [high_level_run_sequence.svg](high_level_run_sequence.svg) +- [high_level_endpoint_scheduling.svg](high_level_endpoint_scheduling.svg) ## Call Families @@ -97,7 +97,7 @@ metadata is required in version 1. - each call carries `call_metadata` with the opening message - has no attached data stream in the current model - has the operation-specific request and result payloads defined in - [udf_protocol_high_level_payloads.md](udf_protocol_high_level_payloads.md) + [high_level_payloads.md](high_level_payloads.md) ### `get_connection` @@ -114,7 +114,7 @@ metadata is required in version 1. ## Payload Contracts The complete call-metadata, script-metadata, Function, and nested-call payload contracts are defined in -[udf_protocol_high_level_payloads.md](udf_protocol_high_level_payloads.md). `StringPayload` is used directly for +[high_level_payloads.md](high_level_payloads.md). `StringPayload` is used directly for scalar strings; JSON is used only where a payload has structured fields. ## Representative Sequences @@ -124,8 +124,8 @@ The current design keeps the high-level sequences intentionally simple: - nested callback-style calls execute while a parent `Run` or Function call remains active - `Run` combines `OpenCall`, `call_metadata`, input schema announcement, and the first input batch when practical -See [udf_protocol_high_level_nested_calls.svg](udf_protocol_high_level_nested_calls.svg) and -[udf_protocol_high_level_run_sequence.svg](udf_protocol_high_level_run_sequence.svg). +See [high_level_nested_calls.svg](high_level_nested_calls.svg) and +[high_level_run_sequence.svg](high_level_run_sequence.svg). ## Scheduling Policy @@ -145,7 +145,7 @@ call orchestration and do not alter the generic Client/Server stream rules in th 2. if nothing is ready to send, block waiting for new incoming messages 3. monitor peer liveness and terminate unhealthy sessions when needed -See [udf_protocol_high_level_endpoint_scheduling.svg](udf_protocol_high_level_endpoint_scheduling.svg). +See [high_level_endpoint_scheduling.svg](high_level_endpoint_scheduling.svg). ## Forward-Looking Ideas Still Open <> @@ -155,7 +155,7 @@ See [udf_protocol_high_level_endpoint_scheduling.svg](udf_protocol_high_level_en ## Relationship To Other Docs -- low-level framing and generic stream rules live in [udf_protocol_low_level.md](udf_protocol_low_level.md) -- generic open/close call behavior lives in [udf_protocol_call_lifecycle.md](udf_protocol_call_lifecycle.md) -- generic data-stream behavior lives in [udf_protocol_data_stream.md](udf_protocol_data_stream.md) -- high-level payload contracts live in [udf_protocol_high_level_payloads.md](udf_protocol_high_level_payloads.md) +- low-level framing and generic stream rules live in [low_level.md](low_level.md) +- generic open/close call behavior lives in [call_lifecycle.md](call_lifecycle.md) +- generic data-stream behavior lives in [data_stream.md](data_stream.md) +- high-level payload contracts live in [high_level_payloads.md](high_level_payloads.md) diff --git a/doc/design/v2/udf_protocol_high_level_endpoint_scheduling.mmd b/doc/design/v2/protocol/high_level_endpoint_scheduling.mmd similarity index 100% rename from doc/design/v2/udf_protocol_high_level_endpoint_scheduling.mmd rename to doc/design/v2/protocol/high_level_endpoint_scheduling.mmd diff --git a/doc/design/v2/udf_protocol_high_level_endpoint_scheduling.svg b/doc/design/v2/protocol/high_level_endpoint_scheduling.svg similarity index 100% rename from doc/design/v2/udf_protocol_high_level_endpoint_scheduling.svg rename to doc/design/v2/protocol/high_level_endpoint_scheduling.svg diff --git a/doc/design/v2/udf_protocol_high_level_nested_calls.mmd b/doc/design/v2/protocol/high_level_nested_calls.mmd similarity index 100% rename from doc/design/v2/udf_protocol_high_level_nested_calls.mmd rename to doc/design/v2/protocol/high_level_nested_calls.mmd diff --git a/doc/design/v2/udf_protocol_high_level_nested_calls.svg b/doc/design/v2/protocol/high_level_nested_calls.svg similarity index 100% rename from doc/design/v2/udf_protocol_high_level_nested_calls.svg rename to doc/design/v2/protocol/high_level_nested_calls.svg diff --git a/doc/design/v2/udf_protocol_high_level_payloads.md b/doc/design/v2/protocol/high_level_payloads.md similarity index 89% rename from doc/design/v2/udf_protocol_high_level_payloads.md rename to doc/design/v2/protocol/high_level_payloads.md index 4d0083a..0db0290 100644 --- a/doc/design/v2/udf_protocol_high_level_payloads.md +++ b/doc/design/v2/protocol/high_level_payloads.md @@ -20,7 +20,7 @@ calls; the last received values apply to subsequent calls. It is not carried on ## Call Metadata Every `Run` and Function call carries one `call_metadata` JSON payload in the same `StreamMessage` as `OpenCall`. -It conforms to [call_metadata.schema.json](../../../udf-runner-cpp/v2/json_schema/call_metadata.schema.json) and +It conforms to [call_metadata.schema.json](../../../../udf-runner-cpp/v2/json_schema/call_metadata.schema.json) and supplies the per-invocation execution context and the input/output iterator and column definitions. The script name and source are deliberately excluded because they are connection-scoped script metadata. @@ -61,7 +61,7 @@ The virtual-schema request and all Function results retain their existing string defined by the respective script API, not by this transport protocol. `import_specification` conforms to -[import_specification.schema.json](../../../udf-runner-cpp/v2/json_schema/import_specification.schema.json). +[import_specification.schema.json](../../../../udf-runner-cpp/v2/json_schema/import_specification.schema.json). Example: ```json @@ -74,7 +74,7 @@ Example: ``` `export_specification` conforms to -[export_specification.schema.json](../../../udf-runner-cpp/v2/json_schema/export_specification.schema.json). +[export_specification.schema.json](../../../../udf-runner-cpp/v2/json_schema/export_specification.schema.json). Example: ```json @@ -96,7 +96,7 @@ Example: | `get_script` | `script_name`: `StringPayload` | `script`: `StringPayload` | `connection_information` conforms to -[connection_information.schema.json](../../../udf-runner-cpp/v2/json_schema/connection_information.schema.json). +[connection_information.schema.json](../../../../udf-runner-cpp/v2/json_schema/connection_information.schema.json). Example: ```json diff --git a/doc/design/v2/udf_protocol_high_level_run_sequence.mmd b/doc/design/v2/protocol/high_level_run_sequence.mmd similarity index 100% rename from doc/design/v2/udf_protocol_high_level_run_sequence.mmd rename to doc/design/v2/protocol/high_level_run_sequence.mmd diff --git a/doc/design/v2/udf_protocol_high_level_run_sequence.svg b/doc/design/v2/protocol/high_level_run_sequence.svg similarity index 100% rename from doc/design/v2/udf_protocol_high_level_run_sequence.svg rename to doc/design/v2/protocol/high_level_run_sequence.svg diff --git a/doc/design/v2/udf_protocol_low_level.md b/doc/design/v2/protocol/low_level.md similarity index 94% rename from doc/design/v2/udf_protocol_low_level.md rename to doc/design/v2/protocol/low_level.md index 574d0f0..a13e128 100644 --- a/doc/design/v2/udf_protocol_low_level.md +++ b/doc/design/v2/protocol/low_level.md @@ -2,7 +2,7 @@ This document captures the low-level wire rules for the new UDF protocol. It intentionally excludes high-level call semantics and scheduling policy; those live in -[udf_protocol_high_level_calls.md](udf_protocol_high_level_calls.md). +[high_level_calls.md](high_level_calls.md). ## Scope @@ -15,12 +15,12 @@ This document covers: - transport binding and buffer handoff The generic data-stream rules are defined in the companion -[udf_protocol_data_stream.md](udf_protocol_data_stream.md). They apply below the high-level `DB`/`UDFRunner` +[data_stream.md](data_stream.md). They apply below the high-level `DB`/`UDFRunner` protocol layer. Related diagrams: -- [udf_protocol_low_level_connection_lifecycle.svg](udf_protocol_low_level_connection_lifecycle.svg) +- [low_level_connection_lifecycle.svg](low_level_connection_lifecycle.svg) ## Roles @@ -101,7 +101,7 @@ At the low level, a session progresses through: 3. normal call/control traffic 4. a two-way `CloseConnection` exchange on stream `0`, followed by transport close; or abort -See [udf_protocol_low_level_connection_lifecycle.svg](udf_protocol_low_level_connection_lifecycle.svg). +See [low_level_connection_lifecycle.svg](low_level_connection_lifecycle.svg). ## Close Semantics diff --git a/doc/design/v2/udf_protocol_low_level_connection_lifecycle.mmd b/doc/design/v2/protocol/low_level_connection_lifecycle.mmd similarity index 100% rename from doc/design/v2/udf_protocol_low_level_connection_lifecycle.mmd rename to doc/design/v2/protocol/low_level_connection_lifecycle.mmd diff --git a/doc/design/v2/udf_protocol_low_level_connection_lifecycle.svg b/doc/design/v2/protocol/low_level_connection_lifecycle.svg similarity index 100% rename from doc/design/v2/udf_protocol_low_level_connection_lifecycle.svg rename to doc/design/v2/protocol/low_level_connection_lifecycle.svg diff --git a/doc/design/v2/udf_transport_technology_overview.md b/doc/design/v2/protocol/transport_technology_overview.md similarity index 100% rename from doc/design/v2/udf_transport_technology_overview.md rename to doc/design/v2/protocol/transport_technology_overview.md From ab08878a0c9c20c0df6e32301f9a8a92940e6a52 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Mon, 24 Aug 2026 15:58:11 +0200 Subject: [PATCH 08/26] docs(v2): define high-level Exasol-to-Arrow type mapping --- doc/design/v2/README.md | 4 + doc/design/v2/protocol/high_level_payloads.md | 3 + .../v2/protocol/high_level_type_mapping.md | 288 ++++++++++++++++++ .../v2/protocol/high_level_type_mapping.mmd | 41 +++ .../v2/protocol/high_level_type_mapping.svg | 1 + doc/design/v2/protocol/low_level.md | 2 + .../protocol/transport_technology_overview.md | 4 + .../v2/json_schema/call_metadata.schema.json | 4 +- .../import_specification.schema.json | 4 +- udf-runner-cpp/v2/udf_protocol.fbs | 9 +- 10 files changed, 355 insertions(+), 5 deletions(-) create mode 100644 doc/design/v2/protocol/high_level_type_mapping.md create mode 100644 doc/design/v2/protocol/high_level_type_mapping.mmd create mode 100644 doc/design/v2/protocol/high_level_type_mapping.svg diff --git a/doc/design/v2/README.md b/doc/design/v2/README.md index eeff9ba..65b8107 100644 --- a/doc/design/v2/README.md +++ b/doc/design/v2/README.md @@ -11,5 +11,9 @@ This directory contains the split protocol design for the new UDF protocol. `get_connection`, `get_script`, and DB/UDFRunner scheduling policy. - [protocol/high_level_payloads.md](protocol/high_level_payloads.md) defines their named string and JSON payload contracts. +- [protocol/high_level_type_mapping.md](protocol/high_level_type_mapping.md) is the high-level Exasol-to-Arrow + column conversion contract, including physical types, parameters, and extension metadata. Mermaid sources and rendered SVGs use matching names and scopes so the textual and visual material stays aligned. +The low-level schema defines reusable Arrow-compatible physical type capabilities. Exasol type selection and +logical/extension metadata are defined by `high_level_type_mapping.md`. diff --git a/doc/design/v2/protocol/high_level_payloads.md b/doc/design/v2/protocol/high_level_payloads.md index 0db0290..1510482 100644 --- a/doc/design/v2/protocol/high_level_payloads.md +++ b/doc/design/v2/protocol/high_level_payloads.md @@ -23,6 +23,9 @@ Every `Run` and Function call carries one `call_metadata` JSON payload in the sa It conforms to [call_metadata.schema.json](../../../../udf-runner-cpp/v2/json_schema/call_metadata.schema.json) and supplies the per-invocation execution context and the input/output iterator and column definitions. The script name and source are deliberately excluded because they are connection-scoped script metadata. +Column `type` values use official Exasol type families, while `type_name` carries the complete parameterized Exasol +SQL declaration. Their Arrow physical representation and metadata rules are defined in +[high_level_type_mapping.md](high_level_type_mapping.md). ```json { diff --git a/doc/design/v2/protocol/high_level_type_mapping.md b/doc/design/v2/protocol/high_level_type_mapping.md new file mode 100644 index 0000000..ea592f9 --- /dev/null +++ b/doc/design/v2/protocol/high_level_type_mapping.md @@ -0,0 +1,288 @@ +# UDF Protocol v2: High-Level Exasol-to-Arrow Type Mapping + +This document is the normative high-level type-conversion contract for v2 data-stream column schemas. The protocol uses a +self-owned subset of Arrow's schema model in `udf_protocol.fbs`; it does not import Arrow's FlatBuffers schema. +`Field.custom_metadata` carries logical-type and extension annotations as string key/value pairs. + +The conversion is performed from the declared Exasol column type, never from values observed in a batch. A field's +`nullable` flag is preserved independently of its physical type. A conversion that is not defined here fails schema +conversion with a descriptive error. + +The [mapping diagram](high_level_type_mapping.svg) summarizes the decision paths below. + +## Official Exasol type mappings + +`type` in v2 payload metadata identifies one of Exasol's official type families. `type_name` is the canonical, +complete SQL declaration, including parameters. Arrow types in the table are physical protocol representations, not +alternate Exasol type names. + +| Exasol type family | Arrow-compatible representation | Contract | +| --- | --- | --- | +| `DOUBLE PRECISION` | `FloatingPoint(Double)` | Preserve nullable field semantics. | +| `DECIMAL` | signed `Int(32)`, signed `Int(64)`, or `Decimal(128)` | Apply declared precision and scale rules below; integer types are storage choices only. | +| `TIMESTAMP` | `Timestamp(unit, "")` | Select the smallest unit preserving declared precision. | +| `TIMESTAMP WITH LOCAL TIME ZONE` | `Timestamp(unit, "UTC")` | Normalize Exasol's UTC-normalized value for transport. | +| `DATE` | `Date(Day)` | Preserve calendar-day semantics. | +| `CHAR` | `Utf8` | Preserve fixed-length padding and character metadata as Exasol semantics. | +| `VARCHAR` | `Utf8` | Preserve declared character length and character metadata. | +| `BOOLEAN` | `Bool` | Preserve nullable values. | +| `HASHTYPE` | `FixedSizeBinary` | Preserve declared byte width and transport raw bytes. | +| `GEOMETRY` | `Binary` + `geoarrow.wkb` | Transport WKB and preserve SRID metadata. | +| `INTERVAL YEAR TO MONTH` | `Struct` + `exasol.interval.year_month` | Preserve the complete Exasol year/month range. | +| `INTERVAL DAY TO SECOND` | `Interval(MonthDayNano)` | Encode zero months, signed days, and nanoseconds. | + +The type family and source declaration metadata use the following field metadata keys: + +| Key | Value | +| --- | --- | +| `exasol:type_name` | Original Exasol type declaration or type name. | +| `exasol:type_family` | Official Exasol type family from the table above. | +| `exasol:precision` | Decimal or timestamp precision, when declared. | +| `exasol:scale` | Decimal scale, when declared. | +| `exasol:character_length` | Declared `CHAR`/`VARCHAR` character length, when applicable. | +| `exasol:character_set` | Declared `ASCII` or `UTF8` character set, when applicable. | +| `exasol:srid` | Decimal SRID text, when applicable. | +| `exasol:hash_byte_width` | Canonical fixed-width byte count, when applicable. | + +These keys are diagnostic and reconstruction metadata. They do not override the Arrow-compatible physical type. + +### Legacy v1 category replacement + +The v1 protocol used `DOUBLE`, `INT32`, `INT64`, `NUMERIC`, `TIMESTAMP`, and `STRING` as internal categories, and +`UNSUPPORTED` as an error sentinel. They are not Exasol SQL types and are not valid v2 `type` values. Their v2 +replacements are `DOUBLE PRECISION`, `DECIMAL`, `TIMESTAMP` or `TIMESTAMP WITH LOCAL TIME ZONE`, and `CHAR` or +`VARCHAR`. `INT32` and `INT64` may still appear only as Arrow physical storage choices when a declared +`DECIMAL(p,0)` range safely narrows; `UNSUPPORTED` is a conversion failure, never a type. + +## Concrete Exasol SQL mappings + +### Numeric values + +Exasol defines `DECIMAL(p,s)` with `1 <= p <= 36`, `0 <= s <= 36`, and `s <= p`; the defaults are `p = 18` and +`s = 0`. The declared range determines the representation: + +- `DOUBLE PRECISION` maps to `FloatingPoint(Double)`. +- `DECIMAL(p,s)` with `s > 0` maps to `Decimal(128)` with `precision = p`, `scale = s`, and `bit_width = 128`. +- Scale-zero `DECIMAL(p,0)` maps to signed `Int(32)` when `p <= 9`, signed `Int(64)` when `10 <= p <= 18`, and + `Decimal(128)` with scale `0` when `p > 18`. +- Integer narrowing is allowed only when the complete declared precision range fits the selected signed width. + Observed batch values cannot justify narrowing. +- Preserve the original type name, precision, and scale in field metadata. + +The Exasol aliases `INTEGER` and `BIGINT`, where exposed by metadata, are represented using their resolved +`DECIMAL` precision and scale rather than treated as undocumented independent types. + +For every decimal field, `0 <= scale <= precision` is required. Decimal conversion is parameter-preserving and +value-preserving within the declared range. Integer narrowing is value-preserving for the declared range but loses +the original decimal physical representation; the source metadata remains available for reconstruction. + +Example field metadata (the physical type is `Decimal(128, precision=12, scale=2)`): + +```json +{ + "name": "AMOUNT", + "nullable": true, + "arrow_storage_type": "Decimal(128)", + "metadata": { + "exasol:type_name": "DECIMAL(12,2)", + "exasol:type_family": "DECIMAL", + "exasol:precision": "12", + "exasol:scale": "2" + } +} +``` + +### Strings + +`CHAR(n)` and `VARCHAR(n)` map to `Utf8`. Exasol permits `1 <= n <= 2000` for `CHAR(n)`, with default `n = 1`, +and `1 <= n <= 2,000,000` for `VARCHAR(n)`. Preserve the source type name, declared character length, and +`ASCII`/`UTF8` character set. `CHAR` padding remains an Exasol logical concern; it is not a reason to use +`FixedSizeBinary`, and the mapping does not reinterpret character data as bytes. An empty Exasol string is `NULL` +and therefore follows the nullable-field semantics. + +### Date and time + +- `DATE` maps to `Date(Day)`. +- `TIMESTAMP(p)` maps to `Timestamp(unit, "")`. Exasol permits `0 <= p <= 9`, with default `p = 3`. +- Timestamp unit selection is: `p = 0` seconds; `1 <= p <= 3` milliseconds; `4 <= p <= 6` microseconds; and + `7 <= p <= 9` nanoseconds. +- `TIMESTAMP(p) WITH LOCAL TIME ZONE` is normalized to UTC using the session time zone and maps to + `Timestamp(unit, "UTC")`. Preserve the original type name and precision in metadata. + +UTC normalization is a semantic normalization, not lossless preservation of the original session-local +representation. Exasol internally stores these values normalized to UTC, while input and output are interpreted in +the session time zone. Timestamp precision is preserved by the unit selection. The Arrow timestamp unit is explicit +and its timezone is optional. See [Exasol data type details](https://docs.exasol.com/db/latest/sql_references/data_types/datatypedetails.htm) +and [the Arrow columnar format](https://arrow.apache.org/docs/format/Columnar.html). + +Example UTC timestamp field: + +```json +{ + "name": "CREATED_AT", + "nullable": false, + "arrow_storage_type": "Timestamp(Microsecond, UTC)", + "metadata": { + "exasol:type_name": "TIMESTAMP(6) WITH LOCAL TIME ZONE", + "exasol:type_family": "TIMESTAMP WITH LOCAL TIME ZONE", + "exasol:precision": "6" + } +} +``` + +### HASHTYPE + +`HASHTYPE(n BYTE)` maps to `FixedSizeBinary(byte_width = n)`. `HASHTYPE(m BIT)` maps to +`FixedSizeBinary(byte_width = m / 8)`. Exasol permits `1 <= n <= 1024` bytes, or `8 <= m <= 8192` bits, and `m` +must be a multiple of 8. The default is `HASHTYPE(16 BYTE)`. Invalid declarations are rejected. + +Exasol accepts hexadecimal, UUID, Base64, and Base64URL strings as SQL input syntax; UUID input is supported only for +`HASHTYPE(16 BYTE)`. Transport raw hash bytes rather than any of those textual forms. Preserve the source type name +and canonical byte width in metadata. The `HASHTYPE_FORMAT` display setting, including UUID display, does not change +the Arrow type. + +Example: + +```json +{ + "name": "DIGEST", + "nullable": true, + "arrow_storage_type": "FixedSizeBinary(32)", + "metadata": { + "exasol:type_name": "HASHTYPE(256 BIT)", + "exasol:type_family": "HASHTYPE", + "exasol:hash_byte_width": "32" + } +} +``` + +See [Exasol HASHTYPE documentation](https://docs.exasol.com/db/latest/sql_references/data_types/datatypedetails.htm?Highlight=hashtype). + +### Intervals + +`INTERVAL YEAR(p) TO MONTH` maps to an Arrow `Struct_` with two children, `years: Int(32, signed)` and +`months: Int(32, signed)`, annotated with `ARROW:extension:name = "exasol.interval.year_month"`. Encode the +normalized signed year and month components rather than a total-month integer. This preserves Exasol's complete +range, including `-999999999-11` through `999999999-11`. Preserve the declared leading-field precision in +`exasol:interval_leading_precision` and the complete SQL declaration in `exasol:type_name`. + +The standard Arrow `YEAR_MONTH` interval is not sufficient for this mapping because it stores one signed 32-bit +total-month value. Exasol permits `999999999` years and `11` months, which requires +`999999999 * 12 + 11 = 11999999999` months, exceeding Arrow's signed 32-bit maximum of `2147483647`. The struct +extension therefore stores the year and month components separately, preserving the complete Exasol range without +overflow or loss of calendar semantics. + +`INTERVAL DAY(lfp) TO SECOND(fsp)` maps to Arrow `Interval(MonthDayNano)`. Encode `months = 0`, the signed day count, +and the time-of-day component as signed nanoseconds. Preserve `lfp` in `exasol:interval_leading_precision`, `fsp` +in `exasol:interval_fractional_precision`, and the complete SQL declaration in `exasol:type_name`. + +Exasol currently documents effective millisecond accuracy; encode those milliseconds as nanoseconds by multiplying by +`1,000,000`. If Exasol enables true `fsp = 9` storage, place the nanosecond value directly in the same Arrow type. +This represents the complete day range and future nanosecond precision. Do not convert either interval to a timestamp, +fixed-duration value, text, or generic binary; calendar interval semantics must remain explicit. + +Example year-month interval field: + +```json +{ + "name": "AGE", + "nullable": true, + "arrow_storage_type": "Struct", + "metadata": { + "exasol:type_name": "INTERVAL YEAR(4) TO MONTH", + "exasol:type_family": "INTERVAL YEAR TO MONTH", + "exasol:interval_leading_precision": "4", + "ARROW:extension:name": "exasol.interval.year_month", + "ARROW:extension:metadata": "{\"layout\":\"years:int32,months:int32\"}" + } +} +``` + +Example day-time interval field: + +```json +{ + "name": "ELAPSED", + "nullable": true, + "arrow_storage_type": "Interval(MonthDayNano)", + "metadata": { + "exasol:type_name": "INTERVAL DAY(6) TO SECOND(3)", + "exasol:type_family": "INTERVAL DAY TO SECOND", + "exasol:interval_leading_precision": "6", + "exasol:interval_fractional_precision": "3" + } +} +``` + +### GEOMETRY + +`GEOMETRY(srid)` uses variable-size Arrow `Binary` storage with GeoArrow's WKB extension. Exasol's supported +geometry objects are `POINT`, `LINESTRING`, `POLYGON`, `MULTIPOINT`, `MULTILINESTRING`, `MULTIPOLYGON`, and +`GEOMETRYCOLLECTION`. `srid` is optional and defaults to `0`, meaning no coordinate system: + +- `ARROW:extension:name` is exactly `geoarrow.wkb`. +- `ARROW:extension:metadata` is UTF-8 JSON. +- For a present, nonzero SRID, the JSON object contains `"crs"` as the SRID text and `"crs_type"` as `"srid"`. +- For an absent or zero SRID, omit CRS metadata. Omit `edges` to select planar/linear edge semantics. +- Do not infer an EPSG authority from an Exasol SRID or force one geometry subtype: one column may contain the + supported geometry object types listed above. +- Preserve the source type name and SRID in `exasol:*` metadata when needed for diagnostics or reconstruction. +- WKT may be accepted at the SQL boundary, but WKB is the canonical Arrow transport representation. + +Example extension metadata: + +```json +{ + "ARROW:extension:name": "geoarrow.wkb", + "ARROW:extension:metadata": "{\"crs\":\"4326\",\"crs_type\":\"srid\"}", + "exasol:type_name": "GEOMETRY(4326)", + "exasol:type_family": "GEOMETRY", + "exasol:srid": "4326" +} +``` + +See [GeoArrow extension types](https://geoarrow.org/extension-types.html) and [Exasol geometry documentation](https://docs.exasol.com/db/latest/sql_references/data_types/datatypedetails.htm). + +## Official Exasol type surface + +The complete Exasol SQL type surface considered by this mapping is: + +`BOOLEAN`, `DECIMAL(p,s)`, `DOUBLE PRECISION`, `DATE`, `TIMESTAMP(p)`, +`TIMESTAMP(p) WITH LOCAL TIME ZONE`, `INTERVAL YEAR(p) TO MONTH`, +`INTERVAL DAY(lfp) TO SECOND(fsp)`, `GEOMETRY(srid)`, `HASHTYPE(n BYTE)`, +`HASHTYPE(m BIT)`, `CHAR(n)`, and `VARCHAR(n)`. + +`INTERVAL YEAR(p) TO MONTH` uses leading-field precision `1 <= p <= 9`, default `p = 2`. `INTERVAL DAY(lfp) TO +SECOND(fsp)` uses leading-field precision `1 <= lfp <= 9` and fractional-second precision `0 <= fsp <= 9`, with +defaults `lfp = 2` and `fsp = 3`. Both interval types now have mappings above; the year-month struct mapping +preserves the complete declared range. + +See the authoritative [Exasol data type overview](https://docs.exasol.com/db/latest/sql_references/data_types/datatypesoverview.htm) +and [data type details](https://docs.exasol.com/db/latest/sql_references/data_types/datatypedetails.htm). + +## Unsupported declarations and future extensions + +The following are unsupported in this version and must terminate schema conversion with a descriptive error: + +- any Exasol type not represented by the current v2 schema; +- geometry encodings other than the defined WKB representation; and +- hash declarations with invalid or non-byte-aligned widths. + +Unsupported declarations must not fall back to `Utf8` or generic `Binary` unless a future mapping explicitly defines the +required extension metadata. Future mappings may add Arrow interval types, native GeoArrow layouts such as +`geoarrow.point` or `geoarrow.polygon`, and additional Exasol-specific extension types. + +## Round-trip classification + +| Mapping | Classification | Reason | +| --- | --- | --- | +| `DOUBLE PRECISION`, `DATE`, `CHAR(n)`, `VARCHAR(n)`, `BOOLEAN` | Value-preserving | Source declaration metadata is retained. | +| Positive-scale `DECIMAL` | Parameter- and value-preserving | Decimal precision, scale, and bit width are explicit. | +| Scale-zero `DECIMAL` narrowed to Arrow `Int(32)` or `Int(64)` | Value-preserving, representation-normalized | The declared range fits; source parameters remain in metadata. | +| `TIMESTAMP(p)` | Precision- and value-preserving | The smallest sufficient Arrow unit is selected. | +| `TIMESTAMP ... WITH LOCAL TIME ZONE` | Value-preserving after UTC normalization | Original session-local representation is not preserved. | +| `HASHTYPE` | Byte-for-byte and width-preserving | Raw bytes and declared width are transported. | +| `GEOMETRY` | Geometry-value preserving, representation-normalized | WKT/engine representation becomes canonical WKB; SRID metadata is retained. | +| `INTERVAL YEAR(p) TO MONTH` | Range-preserving, extension-normalized | Struct fields preserve signed years and months across the complete Exasol range. | +| `INTERVAL DAY(lfp) TO SECOND(fsp)` | Range- and precision-preserving, representation-normalized | `MonthDayNano` stores zero months, days, and nanoseconds; current millisecond values are scaled to nanoseconds. | +| Unsupported type | Not representable | Schema conversion is rejected. | diff --git a/doc/design/v2/protocol/high_level_type_mapping.mmd b/doc/design/v2/protocol/high_level_type_mapping.mmd new file mode 100644 index 0000000..f6eaf44 --- /dev/null +++ b/doc/design/v2/protocol/high_level_type_mapping.mmd @@ -0,0 +1,41 @@ +flowchart TD + A[Exasol documented SQL types] --> B[official type family and declared parameters] + B --> N{Numeric} + N -->|DOUBLE PRECISION| ND[FloatingPoint Double] + N -->|DECIMAL p,s, s > 0| DEC[Decimal 128: precision p, scale s] + N -->|DECIMAL p,0 and p <= 9| I32[Int signed 32] + N -->|DECIMAL p,0 and 10 <= p <= 18| I64[Int signed 64] + N -->|DECIMAL p,0 and p > 18| D0[Decimal 128: scale 0] + B --> T{Timestamp} + T -->|p = 0| TS0[Timestamp seconds] + T -->|1 <= p <= 3| TS1[Timestamp milliseconds] + T -->|4 <= p <= 6| TS2[Timestamp microseconds] + T -->|7 <= p <= 9| TS3[Timestamp nanoseconds] + T -->|WITH LOCAL TIME ZONE| UTC[Normalize to UTC; timezone UTC] + B --> I{Intervals} + I -->|INTERVAL YEAR p TO MONTH| YM[Struct years:int32 + months:int32 + Exasol extension] + I -->|INTERVAL DAY lfp TO SECOND fsp| DT[Interval MonthDayNano: months=0 + days + nanoseconds] + B --> H[HASHTYPE] + H --> HW[FixedSizeBinary declared byte width] + B --> G[GEOMETRY] + G --> W[Binary + geoarrow.wkb; WKB and SRID metadata] + B --> C["CHAR(n) / VARCHAR(n) / DATE / BOOLEAN"] + C --> P[Utf8 / Date Day / Bool / Int] + B --> U{Unsupported or invalid} + U --> R[Reject schema conversion] + DT --> O + YM --> O + + ND --> O[Arrow-compatible protocol types] + DEC --> O + I32 --> O + I64 --> O + D0 --> O + TS0 --> O + TS1 --> O + TS2 --> O + TS3 --> O + UTC --> O + HW --> O + W --> O + P --> O diff --git a/doc/design/v2/protocol/high_level_type_mapping.svg b/doc/design/v2/protocol/high_level_type_mapping.svg new file mode 100644 index 0000000..7367446 --- /dev/null +++ b/doc/design/v2/protocol/high_level_type_mapping.svg @@ -0,0 +1 @@ +

DOUBLE PRECISION

DECIMAL p,s, s > 0

DECIMAL p,0 and p <= 9

DECIMAL p,0 and 10 <= p <= 18

DECIMAL p,0 and p > 18

p = 0

1 <= p <= 3

4 <= p <= 6

7 <= p <= 9

WITH LOCAL TIME ZONE

INTERVAL YEAR p TO MONTH

INTERVAL DAY lfp TO SECOND fsp

Exasol documented SQL types

official type family and declared parameters

Numeric

FloatingPoint Double

Decimal 128: precision p, scale s

Int signed 32

Int signed 64

Decimal 128: scale 0

Timestamp

Timestamp seconds

Timestamp milliseconds

Timestamp microseconds

Timestamp nanoseconds

Normalize to UTC; timezone UTC

Intervals

Struct years:int32 + months:int32 + Exasol extension

Interval MonthDayNano: months=0 + days + nanoseconds

HASHTYPE

FixedSizeBinary declared byte width

GEOMETRY

Binary + geoarrow.wkb; WKB and SRID metadata

CHAR(n) / VARCHAR(n) / DATE / BOOLEAN

Utf8 / Date Day / Bool / Int

Unsupported or invalid

Reject schema conversion

Arrow-compatible protocol types

 \ No newline at end of file diff --git a/doc/design/v2/protocol/low_level.md b/doc/design/v2/protocol/low_level.md index a13e128..18f512d 100644 --- a/doc/design/v2/protocol/low_level.md +++ b/doc/design/v2/protocol/low_level.md @@ -141,6 +141,8 @@ must use `Inline` buffers. - named metadata payload bodies such as connection objects or script content: typically JSON This document only records the mapping at a protocol level. It does not redefine the `.fbs` schema itself. +Data streams carry Arrow-compatible physical schemas and buffers; Exasol column selection and logical metadata are +defined by the high-level protocol. ## Open Questions diff --git a/doc/design/v2/protocol/transport_technology_overview.md b/doc/design/v2/protocol/transport_technology_overview.md index 694b3d3..7d09ab4 100644 --- a/doc/design/v2/protocol/transport_technology_overview.md +++ b/doc/design/v2/protocol/transport_technology_overview.md @@ -52,6 +52,10 @@ identity, multiplexing, structured cancellation, or remote deployment. ## Serialization Formats +For the v2 protocol's high-level Exasol column contract and its Arrow-compatible record-batch representation, see +[high_level_type_mapping.md](high_level_type_mapping.md), including the GeoArrow WKB representation for geometry +values. + | Format | Characteristics | Best fit | Main limitation | | --- | --- | --- | --- | | Native vectors | Engine-owned in-memory data structures. | In-process and shared-memory execution. | Couples UDF code to an engine ABI and memory-lifetime rules. | diff --git a/udf-runner-cpp/v2/json_schema/call_metadata.schema.json b/udf-runner-cpp/v2/json_schema/call_metadata.schema.json index 3f65e60..2a81c4e 100644 --- a/udf-runner-cpp/v2/json_schema/call_metadata.schema.json +++ b/udf-runner-cpp/v2/json_schema/call_metadata.schema.json @@ -34,8 +34,8 @@ "required": ["name", "type_name"], "properties": { "name": { "type": "string" }, - "type": { "enum": ["UNSUPPORTED", "DOUBLE", "INT32", "INT64", "NUMERIC", "TIMESTAMP", "DATE", "STRING", "BOOLEAN"] }, - "type_name": { "type": "string" }, + "type": { "enum": ["DOUBLE PRECISION", "DECIMAL", "DATE", "TIMESTAMP", "TIMESTAMP WITH LOCAL TIME ZONE", "CHAR", "VARCHAR", "BOOLEAN", "HASHTYPE", "GEOMETRY", "INTERVAL YEAR TO MONTH", "INTERVAL DAY TO SECOND"] }, + "type_name": { "type": "string", "description": "The complete Exasol SQL type declaration, including parameters where applicable." }, "size": { "type": "integer", "minimum": 0, "maximum": 4294967295 }, "precision": { "type": "integer", "minimum": 0, "maximum": 4294967295 }, "scale": { "type": "integer", "minimum": 0, "maximum": 4294967295 } diff --git a/udf-runner-cpp/v2/json_schema/import_specification.schema.json b/udf-runner-cpp/v2/json_schema/import_specification.schema.json index af4de2f..7af2c9e 100644 --- a/udf-runner-cpp/v2/json_schema/import_specification.schema.json +++ b/udf-runner-cpp/v2/json_schema/import_specification.schema.json @@ -17,8 +17,8 @@ "required": ["name", "type_name"], "properties": { "name": { "type": "string" }, - "type": { "enum": ["UNSUPPORTED", "DOUBLE", "INT32", "INT64", "NUMERIC", "TIMESTAMP", "DATE", "STRING", "BOOLEAN"] }, - "type_name": { "type": "string" }, + "type": { "enum": ["DOUBLE PRECISION", "DECIMAL", "DATE", "TIMESTAMP", "TIMESTAMP WITH LOCAL TIME ZONE", "CHAR", "VARCHAR", "BOOLEAN", "HASHTYPE", "GEOMETRY", "INTERVAL YEAR TO MONTH", "INTERVAL DAY TO SECOND"] }, + "type_name": { "type": "string", "description": "The complete Exasol SQL type declaration, including parameters where applicable." }, "size": { "type": "integer", "minimum": 0, "maximum": 4294967295 }, "precision": { "type": "integer", "minimum": 0, "maximum": 4294967295 }, "scale": { "type": "integer", "minimum": 0, "maximum": 4294967295 } diff --git a/udf-runner-cpp/v2/udf_protocol.fbs b/udf-runner-cpp/v2/udf_protocol.fbs index 14d3f01..1a0f2dd 100644 --- a/udf-runner-cpp/v2/udf_protocol.fbs +++ b/udf-runner-cpp/v2/udf_protocol.fbs @@ -19,14 +19,21 @@ enum BufferTransport : uint8 { enum Precision : uint8 { Half = 0, Single = 1, Double = 2 } enum DateUnit : uint8 { Day = 0, Millisecond = 1 } enum TimeUnit : uint8 { Second = 0, Millisecond = 1, Microsecond = 2, Nanosecond = 3 } +enum IntervalUnit : uint8 { YearMonth = 0, DayTime = 1, MonthDayNano = 2 } table Int { bit_width: int32; is_signed: bool; } table FloatingPoint { precision: Precision; } table Utf8 {} table Binary {} +table FixedSizeBinary { byte_width: int32; } table Bool {} table Date { unit: DateUnit; } table Timestamp { unit: TimeUnit; timezone: string; } +// Arrow calendar interval storage: +// YearMonth: signed int32 total months. +// DayTime: signed int32 days plus signed int32 milliseconds. +// MonthDayNano: signed int32 months, signed int32 days, and signed int64 nanoseconds. +table Interval { unit: IntervalUnit; } table Decimal { precision: int32; scale: int32; bit_width: int32 = 128; } table List {} // Trailing underscore mirrors Arrow's own Schema.fbs naming for this union member. @@ -41,7 +48,7 @@ table Struct_ {} table RunEndEncoded {} union Type { - Int, FloatingPoint, Utf8, Binary, Bool, Date, Timestamp, Decimal, List, Struct_, + Int, FloatingPoint, Utf8, Binary, FixedSizeBinary, Bool, Date, Timestamp, Interval, Decimal, List, Struct_, RunEndEncoded, } From a4822d6916195c2c26bdcf88895f6b4e4731e7ff Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Mon, 24 Aug 2026 16:57:59 +0200 Subject: [PATCH 09/26] docs(v2): expose Exasol column type parameters --- doc/design/v2/protocol/high_level_payloads.md | 57 ++++++++++++++++++- .../v2/protocol/high_level_type_mapping.md | 8 +++ .../v2/json_schema/call_metadata.schema.json | 12 ++-- .../import_specification.schema.json | 12 ++-- 4 files changed, 79 insertions(+), 10 deletions(-) diff --git a/doc/design/v2/protocol/high_level_payloads.md b/doc/design/v2/protocol/high_level_payloads.md index 1510482..9684c44 100644 --- a/doc/design/v2/protocol/high_level_payloads.md +++ b/doc/design/v2/protocol/high_level_payloads.md @@ -40,12 +40,65 @@ SQL declaration. Their Arrow physical representation and metadata rules are defi "script_schema": "SYS", "input_iter_type": "EXACTLY_ONCE", "output_iter_type": "EXACTLY_ONCE", - "input_columns": [], + "input_columns": [ + { + "name": "AMOUNT", + "type": "DECIMAL", + "type_name": "DECIMAL(12,2)", + "precision": 12, + "scale": 2 + } + ], "output_columns": [], "single_call_mode": false } ``` +In a column definition, `type` is the official Exasol type family, while `type_name` is the complete SQL +declaration. Parsed parameters such as `precision` and `scale` are included where applicable. The corresponding +Arrow physical storage type and field metadata are defined by the [high-level type mapping](high_level_type_mapping.md). + +The existing `size`, `precision`, and `scale` properties remain part of the column API. The following definitions show +how the additional mapped types expose their parameters without requiring the consumer to parse `type_name`: + +```json +[ + { + "name": "NAME", + "type": "VARCHAR", + "type_name": "VARCHAR(128)", + "size": 128, + "character_set": "UTF8" + }, + { + "name": "DIGEST", + "type": "HASHTYPE", + "type_name": "HASHTYPE(32 BYTE)", + "size": 32, + "size_unit": "BYTE" + }, + { + "name": "LOCATION", + "type": "GEOMETRY", + "type_name": "GEOMETRY(4326)", + "srid": 4326 + }, + { + "name": "AGE", + "type": "INTERVAL YEAR TO MONTH", + "type_name": "INTERVAL YEAR(4) TO MONTH", + "precision": 4 + }, + { + "name": "ELAPSED", + "type": "INTERVAL DAY TO SECOND", + "type_name": "INTERVAL DAY(6) TO SECOND(9)", + "precision": 6, + "fractional_second_precision": 9 + } +] +``` + Unsigned 64-bit values are decimal strings so JSON implementations do not lose precision. ## Function Calls @@ -71,7 +124,7 @@ Example: { "is_subselect": true, "connection_name": "REMOTE_CONNECTION", - "subselect_column_specification": [{ "name": "ID", "type_name": "DECIMAL(18,0)" }], + "subselect_column_specification": [{ "name": "ID", "type": "DECIMAL", "type_name": "DECIMAL(18,0)", "precision": 18, "scale": 0 }], "parameters": [{ "key": "encoding", "value": "UTF-8" }] } ``` diff --git a/doc/design/v2/protocol/high_level_type_mapping.md b/doc/design/v2/protocol/high_level_type_mapping.md index ea592f9..9442b61 100644 --- a/doc/design/v2/protocol/high_level_type_mapping.md +++ b/doc/design/v2/protocol/high_level_type_mapping.md @@ -46,6 +46,14 @@ The type family and source declaration metadata use the following field metadata These keys are diagnostic and reconstruction metadata. They do not override the Arrow-compatible physical type. +The high-level JSON column definition supplies conversion parameters without requiring an implementation to parse +`type_name`. Existing properties retain their API meaning: `size` is the character length for `CHAR`/`VARCHAR` or the +declared hash size for `HASHTYPE`; `precision` is decimal, timestamp, or interval leading-field precision; and +`scale` is reserved for decimal scale. `size_unit` distinguishes `BYTE` from `BIT` for `HASHTYPE`. +`character_set`, `srid`, and `fractional_second_precision` provide the additional parsed parameters needed by the +string, geometry, and day-to-second interval mappings. `type_name` remains the complete source declaration for +diagnostics and reconstruction. + ### Legacy v1 category replacement The v1 protocol used `DOUBLE`, `INT32`, `INT64`, `NUMERIC`, `TIMESTAMP`, and `STRING` as internal categories, and diff --git a/udf-runner-cpp/v2/json_schema/call_metadata.schema.json b/udf-runner-cpp/v2/json_schema/call_metadata.schema.json index 2a81c4e..84ce7e6 100644 --- a/udf-runner-cpp/v2/json_schema/call_metadata.schema.json +++ b/udf-runner-cpp/v2/json_schema/call_metadata.schema.json @@ -31,14 +31,18 @@ "column": { "type": "object", "additionalProperties": true, - "required": ["name", "type_name"], + "required": ["name", "type", "type_name"], "properties": { "name": { "type": "string" }, "type": { "enum": ["DOUBLE PRECISION", "DECIMAL", "DATE", "TIMESTAMP", "TIMESTAMP WITH LOCAL TIME ZONE", "CHAR", "VARCHAR", "BOOLEAN", "HASHTYPE", "GEOMETRY", "INTERVAL YEAR TO MONTH", "INTERVAL DAY TO SECOND"] }, "type_name": { "type": "string", "description": "The complete Exasol SQL type declaration, including parameters where applicable." }, - "size": { "type": "integer", "minimum": 0, "maximum": 4294967295 }, - "precision": { "type": "integer", "minimum": 0, "maximum": 4294967295 }, - "scale": { "type": "integer", "minimum": 0, "maximum": 4294967295 } + "size": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "Existing size property: character length for CHAR/VARCHAR or declared hash size for HASHTYPE." }, + "size_unit": { "enum": ["BYTE", "BIT"], "description": "HASHTYPE size unit; required for hash declarations." }, + "precision": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "DECIMAL or TIMESTAMP precision, or interval leading-field precision." }, + "scale": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "DECIMAL scale only." }, + "fractional_second_precision": { "type": "integer", "minimum": 0, "maximum": 9, "description": "INTERVAL DAY TO SECOND fractional-second precision." }, + "character_set": { "enum": ["ASCII", "UTF8"], "description": "CHAR/VARCHAR character set." }, + "srid": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "GEOMETRY spatial reference identifier; zero means no CRS." } } } } diff --git a/udf-runner-cpp/v2/json_schema/import_specification.schema.json b/udf-runner-cpp/v2/json_schema/import_specification.schema.json index 7af2c9e..7dd0bab 100644 --- a/udf-runner-cpp/v2/json_schema/import_specification.schema.json +++ b/udf-runner-cpp/v2/json_schema/import_specification.schema.json @@ -14,14 +14,18 @@ "column": { "type": "object", "additionalProperties": true, - "required": ["name", "type_name"], + "required": ["name", "type", "type_name"], "properties": { "name": { "type": "string" }, "type": { "enum": ["DOUBLE PRECISION", "DECIMAL", "DATE", "TIMESTAMP", "TIMESTAMP WITH LOCAL TIME ZONE", "CHAR", "VARCHAR", "BOOLEAN", "HASHTYPE", "GEOMETRY", "INTERVAL YEAR TO MONTH", "INTERVAL DAY TO SECOND"] }, "type_name": { "type": "string", "description": "The complete Exasol SQL type declaration, including parameters where applicable." }, - "size": { "type": "integer", "minimum": 0, "maximum": 4294967295 }, - "precision": { "type": "integer", "minimum": 0, "maximum": 4294967295 }, - "scale": { "type": "integer", "minimum": 0, "maximum": 4294967295 } + "size": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "Existing size property: character length for CHAR/VARCHAR or declared hash size for HASHTYPE." }, + "size_unit": { "enum": ["BYTE", "BIT"], "description": "HASHTYPE size unit; required for hash declarations." }, + "precision": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "DECIMAL or TIMESTAMP precision, or interval leading-field precision." }, + "scale": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "DECIMAL scale only." }, + "fractional_second_precision": { "type": "integer", "minimum": 0, "maximum": 9, "description": "INTERVAL DAY TO SECOND fractional-second precision." }, + "character_set": { "enum": ["ASCII", "UTF8"], "description": "CHAR/VARCHAR character set." }, + "srid": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "GEOMETRY spatial reference identifier; zero means no CRS." } } }, "key_value": { From d615498a42358db13524b769a3e2f9bb47c4d146 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Wed, 26 Aug 2026 20:21:21 +0200 Subject: [PATCH 10/26] docs(v2): name Exasol type enum --- doc/design/v2/protocol/high_level_payloads.md | 2 +- udf-runner-cpp/v2/json_schema/call_metadata.schema.json | 6 +++++- .../v2/json_schema/import_specification.schema.json | 6 +++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/doc/design/v2/protocol/high_level_payloads.md b/doc/design/v2/protocol/high_level_payloads.md index 9684c44..46bdb4c 100644 --- a/doc/design/v2/protocol/high_level_payloads.md +++ b/doc/design/v2/protocol/high_level_payloads.md @@ -54,7 +54,7 @@ SQL declaration. Their Arrow physical representation and metadata rules are defi } ``` -In a column definition, `type` is the official Exasol type family, while `type_name` is the complete SQL +In a column definition, `type` is the official Exasol type family constrained by the schema's `exasol_type` enum, while `type_name` is the complete SQL declaration. Parsed parameters such as `precision` and `scale` are included where applicable. The corresponding Arrow physical storage type and field metadata are defined by the [high-level type mapping](high_level_type_mapping.md). diff --git a/udf-runner-cpp/v2/json_schema/call_metadata.schema.json b/udf-runner-cpp/v2/json_schema/call_metadata.schema.json index 84ce7e6..3bb1c1e 100644 --- a/udf-runner-cpp/v2/json_schema/call_metadata.schema.json +++ b/udf-runner-cpp/v2/json_schema/call_metadata.schema.json @@ -28,13 +28,17 @@ "single_call_mode": { "type": "boolean" } }, "definitions": { + "exasol_type": { + "type": "string", + "enum": ["DOUBLE PRECISION", "DECIMAL", "DATE", "TIMESTAMP", "TIMESTAMP WITH LOCAL TIME ZONE", "CHAR", "VARCHAR", "BOOLEAN", "HASHTYPE", "GEOMETRY", "INTERVAL YEAR TO MONTH", "INTERVAL DAY TO SECOND"] + }, "column": { "type": "object", "additionalProperties": true, "required": ["name", "type", "type_name"], "properties": { "name": { "type": "string" }, - "type": { "enum": ["DOUBLE PRECISION", "DECIMAL", "DATE", "TIMESTAMP", "TIMESTAMP WITH LOCAL TIME ZONE", "CHAR", "VARCHAR", "BOOLEAN", "HASHTYPE", "GEOMETRY", "INTERVAL YEAR TO MONTH", "INTERVAL DAY TO SECOND"] }, + "type": { "$ref": "#/definitions/exasol_type" }, "type_name": { "type": "string", "description": "The complete Exasol SQL type declaration, including parameters where applicable." }, "size": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "Existing size property: character length for CHAR/VARCHAR or declared hash size for HASHTYPE." }, "size_unit": { "enum": ["BYTE", "BIT"], "description": "HASHTYPE size unit; required for hash declarations." }, diff --git a/udf-runner-cpp/v2/json_schema/import_specification.schema.json b/udf-runner-cpp/v2/json_schema/import_specification.schema.json index 7dd0bab..e0561cb 100644 --- a/udf-runner-cpp/v2/json_schema/import_specification.schema.json +++ b/udf-runner-cpp/v2/json_schema/import_specification.schema.json @@ -11,13 +11,17 @@ "parameters": { "type": "array", "items": { "$ref": "#/definitions/key_value" } } }, "definitions": { + "exasol_type": { + "type": "string", + "enum": ["DOUBLE PRECISION", "DECIMAL", "DATE", "TIMESTAMP", "TIMESTAMP WITH LOCAL TIME ZONE", "CHAR", "VARCHAR", "BOOLEAN", "HASHTYPE", "GEOMETRY", "INTERVAL YEAR TO MONTH", "INTERVAL DAY TO SECOND"] + }, "column": { "type": "object", "additionalProperties": true, "required": ["name", "type", "type_name"], "properties": { "name": { "type": "string" }, - "type": { "enum": ["DOUBLE PRECISION", "DECIMAL", "DATE", "TIMESTAMP", "TIMESTAMP WITH LOCAL TIME ZONE", "CHAR", "VARCHAR", "BOOLEAN", "HASHTYPE", "GEOMETRY", "INTERVAL YEAR TO MONTH", "INTERVAL DAY TO SECOND"] }, + "type": { "$ref": "#/definitions/exasol_type" }, "type_name": { "type": "string", "description": "The complete Exasol SQL type declaration, including parameters where applicable." }, "size": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "Existing size property: character length for CHAR/VARCHAR or declared hash size for HASHTYPE." }, "size_unit": { "enum": ["BYTE", "BIT"], "description": "HASHTYPE size unit; required for hash declarations." }, From a53378c79c0a86e7fd5c511d778e57caf9c3b465 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Wed, 26 Aug 2026 20:45:34 +0200 Subject: [PATCH 11/26] docs(v2): centralize schemas and examples with CI validation --- .github/workflows/check_json_schemas.yml | 25 +++++ doc/design/v2/README.md | 2 + .../v2/protocol/examples/call_metadata.json | 24 +++++ .../protocol/examples/column_definitions.json | 35 ++++++ .../examples/connection_information.json | 6 ++ .../day_time_interval_field_metadata.json | 11 ++ .../examples/decimal_field_metadata.json | 11 ++ .../examples/export_specification.json | 6 ++ .../examples/geometry_extension_metadata.json | 7 ++ .../examples/hashtype_field_metadata.json | 10 ++ .../examples/import_specification.json | 14 +++ .../examples/timestamp_field_metadata.json | 10 ++ .../year_month_interval_field_metadata.json | 12 +++ doc/design/v2/protocol/high_level_payloads.md | 100 ++---------------- .../v2/protocol/high_level_type_mapping.md | 79 ++------------ noxfile.py | 10 ++ poetry.lock | 16 +-- pyproject.toml | 3 +- .../v2/json_schema/call_metadata.schema.json | 27 +---- .../v2/json_schema/column.schema.json | 25 +++++ .../import_specification.schema.json | 23 +--- .../v2/json_schema/validate_schemas.py | 97 +++++++++++++++++ 22 files changed, 331 insertions(+), 222 deletions(-) create mode 100644 .github/workflows/check_json_schemas.yml create mode 100644 doc/design/v2/protocol/examples/call_metadata.json create mode 100644 doc/design/v2/protocol/examples/column_definitions.json create mode 100644 doc/design/v2/protocol/examples/connection_information.json create mode 100644 doc/design/v2/protocol/examples/day_time_interval_field_metadata.json create mode 100644 doc/design/v2/protocol/examples/decimal_field_metadata.json create mode 100644 doc/design/v2/protocol/examples/export_specification.json create mode 100644 doc/design/v2/protocol/examples/geometry_extension_metadata.json create mode 100644 doc/design/v2/protocol/examples/hashtype_field_metadata.json create mode 100644 doc/design/v2/protocol/examples/import_specification.json create mode 100644 doc/design/v2/protocol/examples/timestamp_field_metadata.json create mode 100644 doc/design/v2/protocol/examples/year_month_interval_field_metadata.json create mode 100644 udf-runner-cpp/v2/json_schema/column.schema.json create mode 100644 udf-runner-cpp/v2/json_schema/validate_schemas.py diff --git a/.github/workflows/check_json_schemas.yml b/.github/workflows/check_json_schemas.yml new file mode 100644 index 0000000..13e7b59 --- /dev/null +++ b/.github/workflows/check_json_schemas.yml @@ -0,0 +1,25 @@ +name: Check v2 JSON schemas + +on: + push: + branches: + - master + pull_request: + +jobs: + validate: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + + - name: Setup Python & Poetry Environment + uses: exasol/python-toolbox/.github/actions/python-environment@v9 + with: + python-version: "3.10" + poetry-version: "2.3.0" + + - name: Install Poetry dependencies + run: poetry install --with dev + + - name: Validate v2 JSON schemas + run: poetry run nox -s validate-json-schemas diff --git a/doc/design/v2/README.md b/doc/design/v2/README.md index 65b8107..9d3846e 100644 --- a/doc/design/v2/README.md +++ b/doc/design/v2/README.md @@ -17,3 +17,5 @@ This directory contains the split protocol design for the new UDF protocol. Mermaid sources and rendered SVGs use matching names and scopes so the textual and visual material stays aligned. The low-level schema defines reusable Arrow-compatible physical type capabilities. Exasol type selection and logical/extension metadata are defined by `high_level_type_mapping.md`. + +The JSON schemas and external examples can be validated with `poetry run nox -s validate-json-schemas`. diff --git a/doc/design/v2/protocol/examples/call_metadata.json b/doc/design/v2/protocol/examples/call_metadata.json new file mode 100644 index 0000000..82fdf8b --- /dev/null +++ b/doc/design/v2/protocol/examples/call_metadata.json @@ -0,0 +1,24 @@ +{ + "database_name": "EXASOL", + "database_version": "8.0", + "session_id": "42", + "statement_id": 1, + "node_count": 1, + "node_id": 0, + "vm_id": "7", + "maximal_memory_limit": "1073741824", + "script_schema": "SYS", + "input_iter_type": "EXACTLY_ONCE", + "output_iter_type": "EXACTLY_ONCE", + "input_columns": [ + { + "name": "AMOUNT", + "type": "DECIMAL", + "type_name": "DECIMAL(12,2)", + "precision": 12, + "scale": 2 + } + ], + "output_columns": [], + "single_call_mode": false +} diff --git a/doc/design/v2/protocol/examples/column_definitions.json b/doc/design/v2/protocol/examples/column_definitions.json new file mode 100644 index 0000000..844b096 --- /dev/null +++ b/doc/design/v2/protocol/examples/column_definitions.json @@ -0,0 +1,35 @@ +[ + { + "name": "NAME", + "type": "VARCHAR", + "type_name": "VARCHAR(128)", + "size": 128, + "character_set": "UTF8" + }, + { + "name": "DIGEST", + "type": "HASHTYPE", + "type_name": "HASHTYPE(32 BYTE)", + "size": 32, + "size_unit": "BYTE" + }, + { + "name": "LOCATION", + "type": "GEOMETRY", + "type_name": "GEOMETRY(4326)", + "srid": 4326 + }, + { + "name": "AGE", + "type": "INTERVAL YEAR TO MONTH", + "type_name": "INTERVAL YEAR(4) TO MONTH", + "precision": 4 + }, + { + "name": "ELAPSED", + "type": "INTERVAL DAY TO SECOND", + "type_name": "INTERVAL DAY(6) TO SECOND(9)", + "precision": 6, + "fractional_second_precision": 9 + } +] diff --git a/doc/design/v2/protocol/examples/connection_information.json b/doc/design/v2/protocol/examples/connection_information.json new file mode 100644 index 0000000..41f2092 --- /dev/null +++ b/doc/design/v2/protocol/examples/connection_information.json @@ -0,0 +1,6 @@ +{ + "kind": "JDBC", + "address": "jdbc:example://host/database", + "user": "user", + "password": "secret" +} diff --git a/doc/design/v2/protocol/examples/day_time_interval_field_metadata.json b/doc/design/v2/protocol/examples/day_time_interval_field_metadata.json new file mode 100644 index 0000000..3a9cc3a --- /dev/null +++ b/doc/design/v2/protocol/examples/day_time_interval_field_metadata.json @@ -0,0 +1,11 @@ +{ + "name": "ELAPSED", + "nullable": true, + "arrow_storage_type": "Interval(MonthDayNano)", + "metadata": { + "exasol:type_name": "INTERVAL DAY(6) TO SECOND(3)", + "exasol:type_family": "INTERVAL DAY TO SECOND", + "exasol:interval_leading_precision": "6", + "exasol:interval_fractional_precision": "3" + } +} diff --git a/doc/design/v2/protocol/examples/decimal_field_metadata.json b/doc/design/v2/protocol/examples/decimal_field_metadata.json new file mode 100644 index 0000000..f528a07 --- /dev/null +++ b/doc/design/v2/protocol/examples/decimal_field_metadata.json @@ -0,0 +1,11 @@ +{ + "name": "AMOUNT", + "nullable": true, + "arrow_storage_type": "Decimal(128)", + "metadata": { + "exasol:type_name": "DECIMAL(12,2)", + "exasol:type_family": "DECIMAL", + "exasol:precision": "12", + "exasol:scale": "2" + } +} diff --git a/doc/design/v2/protocol/examples/export_specification.json b/doc/design/v2/protocol/examples/export_specification.json new file mode 100644 index 0000000..b138e78 --- /dev/null +++ b/doc/design/v2/protocol/examples/export_specification.json @@ -0,0 +1,6 @@ +{ + "has_truncate": true, + "has_replace": false, + "source_column_names": ["ID", "NAME"], + "connection_name": "REMOTE_CONNECTION" +} diff --git a/doc/design/v2/protocol/examples/geometry_extension_metadata.json b/doc/design/v2/protocol/examples/geometry_extension_metadata.json new file mode 100644 index 0000000..6c2cb93 --- /dev/null +++ b/doc/design/v2/protocol/examples/geometry_extension_metadata.json @@ -0,0 +1,7 @@ +{ + "ARROW:extension:name": "geoarrow.wkb", + "ARROW:extension:metadata": "{\"crs\":\"4326\",\"crs_type\":\"srid\"}", + "exasol:type_name": "GEOMETRY(4326)", + "exasol:type_family": "GEOMETRY", + "exasol:srid": "4326" +} diff --git a/doc/design/v2/protocol/examples/hashtype_field_metadata.json b/doc/design/v2/protocol/examples/hashtype_field_metadata.json new file mode 100644 index 0000000..d925dd0 --- /dev/null +++ b/doc/design/v2/protocol/examples/hashtype_field_metadata.json @@ -0,0 +1,10 @@ +{ + "name": "DIGEST", + "nullable": true, + "arrow_storage_type": "FixedSizeBinary(32)", + "metadata": { + "exasol:type_name": "HASHTYPE(256 BIT)", + "exasol:type_family": "HASHTYPE", + "exasol:hash_byte_width": "32" + } +} diff --git a/doc/design/v2/protocol/examples/import_specification.json b/doc/design/v2/protocol/examples/import_specification.json new file mode 100644 index 0000000..a98fe49 --- /dev/null +++ b/doc/design/v2/protocol/examples/import_specification.json @@ -0,0 +1,14 @@ +{ + "is_subselect": true, + "connection_name": "REMOTE_CONNECTION", + "subselect_column_specification": [ + { + "name": "ID", + "type": "DECIMAL", + "type_name": "DECIMAL(18,0)", + "precision": 18, + "scale": 0 + } + ], + "parameters": [{ "key": "encoding", "value": "UTF-8" }] +} diff --git a/doc/design/v2/protocol/examples/timestamp_field_metadata.json b/doc/design/v2/protocol/examples/timestamp_field_metadata.json new file mode 100644 index 0000000..7f86572 --- /dev/null +++ b/doc/design/v2/protocol/examples/timestamp_field_metadata.json @@ -0,0 +1,10 @@ +{ + "name": "CREATED_AT", + "nullable": false, + "arrow_storage_type": "Timestamp(Microsecond, UTC)", + "metadata": { + "exasol:type_name": "TIMESTAMP(6) WITH LOCAL TIME ZONE", + "exasol:type_family": "TIMESTAMP WITH LOCAL TIME ZONE", + "exasol:precision": "6" + } +} diff --git a/doc/design/v2/protocol/examples/year_month_interval_field_metadata.json b/doc/design/v2/protocol/examples/year_month_interval_field_metadata.json new file mode 100644 index 0000000..01129e4 --- /dev/null +++ b/doc/design/v2/protocol/examples/year_month_interval_field_metadata.json @@ -0,0 +1,12 @@ +{ + "name": "AGE", + "nullable": true, + "arrow_storage_type": "Struct", + "metadata": { + "exasol:type_name": "INTERVAL YEAR(4) TO MONTH", + "exasol:type_family": "INTERVAL YEAR TO MONTH", + "exasol:interval_leading_precision": "4", + "ARROW:extension:name": "exasol.interval.year_month", + "ARROW:extension:metadata": "{\"layout\":\"years:int32,months:int32\"}" + } +} diff --git a/doc/design/v2/protocol/high_level_payloads.md b/doc/design/v2/protocol/high_level_payloads.md index 46bdb4c..f1d1f6b 100644 --- a/doc/design/v2/protocol/high_level_payloads.md +++ b/doc/design/v2/protocol/high_level_payloads.md @@ -26,33 +26,10 @@ and source are deliberately excluded because they are connection-scoped script m Column `type` values use official Exasol type families, while `type_name` carries the complete parameterized Exasol SQL declaration. Their Arrow physical representation and metadata rules are defined in [high_level_type_mapping.md](high_level_type_mapping.md). +The shared column-definition contract is defined in [column.schema.json](../../../../udf-runner-cpp/v2/json_schema/column.schema.json) +and is referenced by both call metadata and import specifications. -```json -{ - "database_name": "EXASOL", - "database_version": "8.0", - "session_id": "42", - "statement_id": 1, - "node_count": 1, - "node_id": 0, - "vm_id": "7", - "maximal_memory_limit": "1073741824", - "script_schema": "SYS", - "input_iter_type": "EXACTLY_ONCE", - "output_iter_type": "EXACTLY_ONCE", - "input_columns": [ - { - "name": "AMOUNT", - "type": "DECIMAL", - "type_name": "DECIMAL(12,2)", - "precision": 12, - "scale": 2 - } - ], - "output_columns": [], - "single_call_mode": false -} -``` +See the [call metadata example](examples/call_metadata.json). In a column definition, `type` is the official Exasol type family constrained by the schema's `exasol_type` enum, while `type_name` is the complete SQL declaration. Parsed parameters such as `precision` and `scale` are included where applicable. The corresponding @@ -61,43 +38,7 @@ Arrow physical storage type and field metadata are defined by the [high-level ty The existing `size`, `precision`, and `scale` properties remain part of the column API. The following definitions show how the additional mapped types expose their parameters without requiring the consumer to parse `type_name`: -```json -[ - { - "name": "NAME", - "type": "VARCHAR", - "type_name": "VARCHAR(128)", - "size": 128, - "character_set": "UTF8" - }, - { - "name": "DIGEST", - "type": "HASHTYPE", - "type_name": "HASHTYPE(32 BYTE)", - "size": 32, - "size_unit": "BYTE" - }, - { - "name": "LOCATION", - "type": "GEOMETRY", - "type_name": "GEOMETRY(4326)", - "srid": 4326 - }, - { - "name": "AGE", - "type": "INTERVAL YEAR TO MONTH", - "type_name": "INTERVAL YEAR(4) TO MONTH", - "precision": 4 - }, - { - "name": "ELAPSED", - "type": "INTERVAL DAY TO SECOND", - "type_name": "INTERVAL DAY(6) TO SECOND(9)", - "precision": 6, - "fractional_second_precision": 9 - } -] -``` +See the [column definitions example](examples/column_definitions.json). Unsigned 64-bit values are decimal strings so JSON implementations do not lose precision. @@ -118,29 +59,11 @@ defined by the respective script API, not by this transport protocol. `import_specification` conforms to [import_specification.schema.json](../../../../udf-runner-cpp/v2/json_schema/import_specification.schema.json). -Example: - -```json -{ - "is_subselect": true, - "connection_name": "REMOTE_CONNECTION", - "subselect_column_specification": [{ "name": "ID", "type": "DECIMAL", "type_name": "DECIMAL(18,0)", "precision": 18, "scale": 0 }], - "parameters": [{ "key": "encoding", "value": "UTF-8" }] -} -``` +See the [import specification example](examples/import_specification.json). `export_specification` conforms to [export_specification.schema.json](../../../../udf-runner-cpp/v2/json_schema/export_specification.schema.json). -Example: - -```json -{ - "has_truncate": true, - "has_replace": false, - "source_column_names": ["ID", "NAME"], - "connection_name": "REMOTE_CONNECTION" -} -``` +See the [export specification example](examples/export_specification.json). ## Nested Calls @@ -153,16 +76,7 @@ Example: `connection_information` conforms to [connection_information.schema.json](../../../../udf-runner-cpp/v2/json_schema/connection_information.schema.json). -Example: - -```json -{ - "kind": "JDBC", - "address": "jdbc:example://host/database", - "user": "user", - "password": "secret" -} -``` +See the [connection information example](examples/connection_information.json). ## Deferred Calls diff --git a/doc/design/v2/protocol/high_level_type_mapping.md b/doc/design/v2/protocol/high_level_type_mapping.md index 9442b61..1dad030 100644 --- a/doc/design/v2/protocol/high_level_type_mapping.md +++ b/doc/design/v2/protocol/high_level_type_mapping.md @@ -86,19 +86,7 @@ the original decimal physical representation; the source metadata remains availa Example field metadata (the physical type is `Decimal(128, precision=12, scale=2)`): -```json -{ - "name": "AMOUNT", - "nullable": true, - "arrow_storage_type": "Decimal(128)", - "metadata": { - "exasol:type_name": "DECIMAL(12,2)", - "exasol:type_family": "DECIMAL", - "exasol:precision": "12", - "exasol:scale": "2" - } -} -``` +See the [decimal field metadata example](examples/decimal_field_metadata.json). ### Strings @@ -125,18 +113,7 @@ and [the Arrow columnar format](https://arrow.apache.org/docs/format/Columnar.ht Example UTC timestamp field: -```json -{ - "name": "CREATED_AT", - "nullable": false, - "arrow_storage_type": "Timestamp(Microsecond, UTC)", - "metadata": { - "exasol:type_name": "TIMESTAMP(6) WITH LOCAL TIME ZONE", - "exasol:type_family": "TIMESTAMP WITH LOCAL TIME ZONE", - "exasol:precision": "6" - } -} -``` +See the [timestamp field metadata example](examples/timestamp_field_metadata.json). ### HASHTYPE @@ -151,18 +128,7 @@ the Arrow type. Example: -```json -{ - "name": "DIGEST", - "nullable": true, - "arrow_storage_type": "FixedSizeBinary(32)", - "metadata": { - "exasol:type_name": "HASHTYPE(256 BIT)", - "exasol:type_family": "HASHTYPE", - "exasol:hash_byte_width": "32" - } -} -``` +See the [HASHTYPE field metadata example](examples/hashtype_field_metadata.json). See [Exasol HASHTYPE documentation](https://docs.exasol.com/db/latest/sql_references/data_types/datatypedetails.htm?Highlight=hashtype). @@ -191,36 +157,11 @@ fixed-duration value, text, or generic binary; calendar interval semantics must Example year-month interval field: -```json -{ - "name": "AGE", - "nullable": true, - "arrow_storage_type": "Struct", - "metadata": { - "exasol:type_name": "INTERVAL YEAR(4) TO MONTH", - "exasol:type_family": "INTERVAL YEAR TO MONTH", - "exasol:interval_leading_precision": "4", - "ARROW:extension:name": "exasol.interval.year_month", - "ARROW:extension:metadata": "{\"layout\":\"years:int32,months:int32\"}" - } -} -``` +See the [year-month interval field metadata example](examples/year_month_interval_field_metadata.json). Example day-time interval field: -```json -{ - "name": "ELAPSED", - "nullable": true, - "arrow_storage_type": "Interval(MonthDayNano)", - "metadata": { - "exasol:type_name": "INTERVAL DAY(6) TO SECOND(3)", - "exasol:type_family": "INTERVAL DAY TO SECOND", - "exasol:interval_leading_precision": "6", - "exasol:interval_fractional_precision": "3" - } -} -``` +See the [day-time interval field metadata example](examples/day_time_interval_field_metadata.json). ### GEOMETRY @@ -239,15 +180,7 @@ geometry objects are `POINT`, `LINESTRING`, `POLYGON`, `MULTIPOINT`, `MULTILINES Example extension metadata: -```json -{ - "ARROW:extension:name": "geoarrow.wkb", - "ARROW:extension:metadata": "{\"crs\":\"4326\",\"crs_type\":\"srid\"}", - "exasol:type_name": "GEOMETRY(4326)", - "exasol:type_family": "GEOMETRY", - "exasol:srid": "4326" -} -``` +See the [GeoArrow extension metadata example](examples/geometry_extension_metadata.json). See [GeoArrow extension types](https://geoarrow.org/extension-types.html) and [Exasol geometry documentation](https://docs.exasol.com/db/latest/sql_references/data_types/datatypedetails.htm). diff --git a/noxfile.py b/noxfile.py index 4b19ec8..1ee6071 100644 --- a/noxfile.py +++ b/noxfile.py @@ -38,6 +38,16 @@ def run_oft_for_udf_client(session: nox.Session, *args) -> None: ) +@nox.session(name="validate-json-schemas", python=False) +def validate_json_schemas(session: nox.Session): + """Validate v2 JSON schemas, references, and external JSON examples.""" + # The udf-runner-cpp directory contains a hyphen and cannot be imported as a dotted Python module. + session.run( + "python", + str(ROOT / "udf-runner-cpp" / "v2" / "json_schema" / "validate_schemas.py"), + ) + + @nox.session(name="run-oft", python=False) def run_oft_udf_client_plaintext(session: nox.Session): """ diff --git a/poetry.lock b/poetry.lock index 109e3e9..ec7419b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.4.1 and should not be changed by hand. [[package]] name = "annotated-types" @@ -1043,7 +1043,7 @@ version = "4.26.0" description = "An implementation of JSON Schema validation for Python" optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["main", "dev"] files = [ {file = "jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce"}, {file = "jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326"}, @@ -1065,7 +1065,7 @@ version = "2025.9.1" description = "The JSON Schema meta-schemas and vocabularies, exposed as a Registry" optional = false python-versions = ">=3.9" -groups = ["main"] +groups = ["main", "dev"] files = [ {file = "jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe"}, {file = "jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d"}, @@ -2147,7 +2147,7 @@ version = "0.37.0" description = "JSON Referencing + Python" optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["main", "dev"] files = [ {file = "referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231"}, {file = "referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8"}, @@ -2186,7 +2186,7 @@ version = "0.30.0" description = "Python bindings to Rust's persistent data structures (rpds)" optional = false python-versions = ">=3.10" -groups = ["main"] +groups = ["main", "dev"] markers = "python_version == \"3.10\"" files = [ {file = "rpds_py-0.30.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:679ae98e00c0e8d68a7fda324e16b90fd5260945b45d3b824c892cec9eea3288"}, @@ -2312,7 +2312,7 @@ version = "2026.6.3" description = "Python bindings to Rust's persistent data structures (rpds)" optional = false python-versions = ">=3.11" -groups = ["main"] +groups = ["main", "dev"] markers = "python_version >= \"3.11\"" files = [ {file = "rpds_py-2026.6.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7b689145a1485c335569bd056464f3243a29af7ed3871c7be31ad624ba239bc7"}, @@ -2773,7 +2773,7 @@ files = [ {file = "typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"}, {file = "typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5"}, ] -markers = {dev = "python_version == \"3.10\""} +markers = {dev = "python_version < \"3.13\""} [[package]] name = "typing-inspection" @@ -2945,4 +2945,4 @@ dev = ["pytest", "setuptools"] [metadata] lock-version = "2.1" python-versions = ">=3.10, <3.14.0" -content-hash = "6d994147a64b24950c5fae9e09b42852b5c7b6ca80d59b957393e53efe3760e5" +content-hash = "30ff5c66a9481648b28a8052cca06d381c4dc18b3185638711791f65313efd02" diff --git a/pyproject.toml b/pyproject.toml index fb9f23c..f5f48c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,8 @@ homepage = "https://github.com/exasol/udf-runner-cpp" dev = [ "toml>=0.10.2", "gitpython>=3.1.0, <4.0.0", - "nox>=2026.4.10" + "nox>=2026.4.10", + "jsonschema>=4.22.0, <5.0.0" ] [build-system] diff --git a/udf-runner-cpp/v2/json_schema/call_metadata.schema.json b/udf-runner-cpp/v2/json_schema/call_metadata.schema.json index 3bb1c1e..5c1a6d4 100644 --- a/udf-runner-cpp/v2/json_schema/call_metadata.schema.json +++ b/udf-runner-cpp/v2/json_schema/call_metadata.schema.json @@ -23,31 +23,8 @@ "scope_user": { "type": "string" }, "input_iter_type": { "enum": ["EXACTLY_ONCE", "MULTIPLE"] }, "output_iter_type": { "enum": ["EXACTLY_ONCE", "MULTIPLE"] }, - "input_columns": { "type": "array", "items": { "$ref": "#/definitions/column" } }, - "output_columns": { "type": "array", "items": { "$ref": "#/definitions/column" } }, + "input_columns": { "type": "array", "items": { "$ref": "column.schema.json" } }, + "output_columns": { "type": "array", "items": { "$ref": "column.schema.json" } }, "single_call_mode": { "type": "boolean" } - }, - "definitions": { - "exasol_type": { - "type": "string", - "enum": ["DOUBLE PRECISION", "DECIMAL", "DATE", "TIMESTAMP", "TIMESTAMP WITH LOCAL TIME ZONE", "CHAR", "VARCHAR", "BOOLEAN", "HASHTYPE", "GEOMETRY", "INTERVAL YEAR TO MONTH", "INTERVAL DAY TO SECOND"] - }, - "column": { - "type": "object", - "additionalProperties": true, - "required": ["name", "type", "type_name"], - "properties": { - "name": { "type": "string" }, - "type": { "$ref": "#/definitions/exasol_type" }, - "type_name": { "type": "string", "description": "The complete Exasol SQL type declaration, including parameters where applicable." }, - "size": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "Existing size property: character length for CHAR/VARCHAR or declared hash size for HASHTYPE." }, - "size_unit": { "enum": ["BYTE", "BIT"], "description": "HASHTYPE size unit; required for hash declarations." }, - "precision": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "DECIMAL or TIMESTAMP precision, or interval leading-field precision." }, - "scale": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "DECIMAL scale only." }, - "fractional_second_precision": { "type": "integer", "minimum": 0, "maximum": 9, "description": "INTERVAL DAY TO SECOND fractional-second precision." }, - "character_set": { "enum": ["ASCII", "UTF8"], "description": "CHAR/VARCHAR character set." }, - "srid": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "GEOMETRY spatial reference identifier; zero means no CRS." } - } - } } } diff --git a/udf-runner-cpp/v2/json_schema/column.schema.json b/udf-runner-cpp/v2/json_schema/column.schema.json new file mode 100644 index 0000000..6f883b8 --- /dev/null +++ b/udf-runner-cpp/v2/json_schema/column.schema.json @@ -0,0 +1,25 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "UDF Protocol v2 column definition", + "type": "object", + "additionalProperties": true, + "required": ["name", "type", "type_name"], + "properties": { + "name": { "type": "string" }, + "type": { "$ref": "#/definitions/exasol_type" }, + "type_name": { "type": "string", "description": "The complete Exasol SQL type declaration, including parameters where applicable." }, + "size": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "Existing size property: character length for CHAR/VARCHAR or declared hash size for HASHTYPE." }, + "size_unit": { "enum": ["BYTE", "BIT"], "description": "HASHTYPE size unit; required for hash declarations." }, + "precision": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "DECIMAL or TIMESTAMP precision, or interval leading-field precision." }, + "scale": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "DECIMAL scale only." }, + "fractional_second_precision": { "type": "integer", "minimum": 0, "maximum": 9, "description": "INTERVAL DAY TO SECOND fractional-second precision." }, + "character_set": { "enum": ["ASCII", "UTF8"], "description": "CHAR/VARCHAR character set." }, + "srid": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "GEOMETRY spatial reference identifier; zero means no CRS." } + }, + "definitions": { + "exasol_type": { + "type": "string", + "enum": ["DOUBLE PRECISION", "DECIMAL", "DATE", "TIMESTAMP", "TIMESTAMP WITH LOCAL TIME ZONE", "CHAR", "VARCHAR", "BOOLEAN", "HASHTYPE", "GEOMETRY", "INTERVAL YEAR TO MONTH", "INTERVAL DAY TO SECOND"] + } + } +} diff --git a/udf-runner-cpp/v2/json_schema/import_specification.schema.json b/udf-runner-cpp/v2/json_schema/import_specification.schema.json index e0561cb..53d49b8 100644 --- a/udf-runner-cpp/v2/json_schema/import_specification.schema.json +++ b/udf-runner-cpp/v2/json_schema/import_specification.schema.json @@ -7,31 +7,10 @@ "is_subselect": { "type": "boolean" }, "connection_information": { "$ref": "connection_information.schema.json" }, "connection_name": { "type": "string" }, - "subselect_column_specification": { "type": "array", "items": { "$ref": "#/definitions/column" } }, + "subselect_column_specification": { "type": "array", "items": { "$ref": "column.schema.json" } }, "parameters": { "type": "array", "items": { "$ref": "#/definitions/key_value" } } }, "definitions": { - "exasol_type": { - "type": "string", - "enum": ["DOUBLE PRECISION", "DECIMAL", "DATE", "TIMESTAMP", "TIMESTAMP WITH LOCAL TIME ZONE", "CHAR", "VARCHAR", "BOOLEAN", "HASHTYPE", "GEOMETRY", "INTERVAL YEAR TO MONTH", "INTERVAL DAY TO SECOND"] - }, - "column": { - "type": "object", - "additionalProperties": true, - "required": ["name", "type", "type_name"], - "properties": { - "name": { "type": "string" }, - "type": { "$ref": "#/definitions/exasol_type" }, - "type_name": { "type": "string", "description": "The complete Exasol SQL type declaration, including parameters where applicable." }, - "size": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "Existing size property: character length for CHAR/VARCHAR or declared hash size for HASHTYPE." }, - "size_unit": { "enum": ["BYTE", "BIT"], "description": "HASHTYPE size unit; required for hash declarations." }, - "precision": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "DECIMAL or TIMESTAMP precision, or interval leading-field precision." }, - "scale": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "DECIMAL scale only." }, - "fractional_second_precision": { "type": "integer", "minimum": 0, "maximum": 9, "description": "INTERVAL DAY TO SECOND fractional-second precision." }, - "character_set": { "enum": ["ASCII", "UTF8"], "description": "CHAR/VARCHAR character set." }, - "srid": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "GEOMETRY spatial reference identifier; zero means no CRS." } - } - }, "key_value": { "type": "object", "additionalProperties": true, diff --git a/udf-runner-cpp/v2/json_schema/validate_schemas.py b/udf-runner-cpp/v2/json_schema/validate_schemas.py new file mode 100644 index 0000000..cc7a921 --- /dev/null +++ b/udf-runner-cpp/v2/json_schema/validate_schemas.py @@ -0,0 +1,97 @@ +import copy +import json +from pathlib import Path + +from jsonschema import Draft7Validator, ValidationError +from referencing import Registry, Resource +from referencing.jsonschema import DRAFT7 + + +ROOT = Path(__file__).resolve().parent +EXAMPLES = ROOT.parents[2] / "doc" / "design" / "v2" / "protocol" / "examples" + + +def load_schemas() -> dict[str, dict]: + schemas = { + path.name: json.loads(path.read_text()) + for path in ROOT.glob("*.schema.json") + } + for schema in schemas.values(): + Draft7Validator.check_schema(schema) + return schemas + + +def validate(schemas: dict[str, dict], schema_name: str, instance: object) -> None: + schema_path = ROOT / schema_name + resources = [] + for name, schema in schemas.items(): + resource = Resource.from_contents(schema, default_specification=DRAFT7) + resources.extend(((name, resource), ((ROOT / name).as_uri(), resource))) + registry = Registry().with_resources(resources) + + root_schema = copy.deepcopy(schemas[schema_name]) + root_schema["$id"] = schema_path.as_uri() + Draft7Validator(root_schema, registry=registry).validate(instance) + + +def read_example(name: str) -> object: + return json.loads((EXAMPLES / name).read_text()) + + +def validate_mapping_examples() -> None: + decimal = read_example("decimal_field_metadata.json") + assert decimal["metadata"]["exasol:type_family"] == "DECIMAL" + precision = int(decimal["metadata"]["exasol:precision"]) + scale = int(decimal["metadata"]["exasol:scale"]) + assert 0 <= scale <= precision + + timestamp = read_example("timestamp_field_metadata.json") + assert timestamp["metadata"]["exasol:type_family"] == "TIMESTAMP WITH LOCAL TIME ZONE" + + hashtype = read_example("hashtype_field_metadata.json") + assert 1 <= int(hashtype["metadata"]["exasol:hash_byte_width"]) <= 1024 + + year_month = read_example("year_month_interval_field_metadata.json") + assert year_month["metadata"]["ARROW:extension:name"] == "exasol.interval.year_month" + + day_time = read_example("day_time_interval_field_metadata.json") + assert day_time["metadata"]["exasol:type_family"] == "INTERVAL DAY TO SECOND" + + geometry = read_example("geometry_extension_metadata.json") + assert geometry["ARROW:extension:name"] == "geoarrow.wkb" + extension_metadata = json.loads(geometry["ARROW:extension:metadata"]) + assert extension_metadata["crs_type"] == "srid" + assert extension_metadata["crs"] == geometry["exasol:srid"] + + +def validate_schemas() -> None: + schemas = load_schemas() + call_metadata = read_example("call_metadata.json") + column_definitions = read_example("column_definitions.json") + import_specification = read_example("import_specification.json") + + validate(schemas, "call_metadata.schema.json", call_metadata) + validate( + schemas, + "import_specification.schema.json", + {"is_subselect": True, "subselect_column_specification": column_definitions}, + ) + validate(schemas, "import_specification.schema.json", import_specification) + validate(schemas, "export_specification.schema.json", read_example("export_specification.json")) + validate(schemas, "connection_information.schema.json", read_example("connection_information.json")) + validate_mapping_examples() + + invalid = copy.deepcopy(call_metadata) + invalid["input_columns"][0]["type"] = "STRING" + try: + validate(schemas, "call_metadata.schema.json", invalid) + except ValidationError: + pass + else: + raise AssertionError("legacy STRING type was accepted") + + print("v2 JSON schema validation passed") + + +if __name__ == "__main__": + validate_schemas() From cf85adb6f8a0c2ca3eb3d4cb9d5af911e9fa7dd5 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Wed, 26 Aug 2026 21:07:25 +0200 Subject: [PATCH 12/26] docs(v2): map decimals to sized Arrow types --- .../examples/decimal_field_metadata.json | 2 +- .../v2/protocol/high_level_type_mapping.md | 26 +++++++++---------- .../v2/protocol/high_level_type_mapping.mmd | 7 +++-- .../v2/protocol/high_level_type_mapping.svg | 2 +- .../v2/json_schema/validate_schemas.py | 10 +++++++ 5 files changed, 28 insertions(+), 19 deletions(-) diff --git a/doc/design/v2/protocol/examples/decimal_field_metadata.json b/doc/design/v2/protocol/examples/decimal_field_metadata.json index f528a07..48ea108 100644 --- a/doc/design/v2/protocol/examples/decimal_field_metadata.json +++ b/doc/design/v2/protocol/examples/decimal_field_metadata.json @@ -1,7 +1,7 @@ { "name": "AMOUNT", "nullable": true, - "arrow_storage_type": "Decimal(128)", + "arrow_storage_type": "Decimal(64)", "metadata": { "exasol:type_name": "DECIMAL(12,2)", "exasol:type_family": "DECIMAL", diff --git a/doc/design/v2/protocol/high_level_type_mapping.md b/doc/design/v2/protocol/high_level_type_mapping.md index 1dad030..87f48ca 100644 --- a/doc/design/v2/protocol/high_level_type_mapping.md +++ b/doc/design/v2/protocol/high_level_type_mapping.md @@ -19,7 +19,7 @@ alternate Exasol type names. | Exasol type family | Arrow-compatible representation | Contract | | --- | --- | --- | | `DOUBLE PRECISION` | `FloatingPoint(Double)` | Preserve nullable field semantics. | -| `DECIMAL` | signed `Int(32)`, signed `Int(64)`, or `Decimal(128)` | Apply declared precision and scale rules below; integer types are storage choices only. | +| `DECIMAL` | `Decimal(32)`, `Decimal(64)`, or `Decimal(128)` | Select the smallest decimal width that preserves the declared precision and scale. | | `TIMESTAMP` | `Timestamp(unit, "")` | Select the smallest unit preserving declared precision. | | `TIMESTAMP WITH LOCAL TIME ZONE` | `Timestamp(unit, "UTC")` | Normalize Exasol's UTC-normalized value for transport. | | `DATE` | `Date(Day)` | Preserve calendar-day semantics. | @@ -59,32 +59,32 @@ diagnostics and reconstruction. The v1 protocol used `DOUBLE`, `INT32`, `INT64`, `NUMERIC`, `TIMESTAMP`, and `STRING` as internal categories, and `UNSUPPORTED` as an error sentinel. They are not Exasol SQL types and are not valid v2 `type` values. Their v2 replacements are `DOUBLE PRECISION`, `DECIMAL`, `TIMESTAMP` or `TIMESTAMP WITH LOCAL TIME ZONE`, and `CHAR` or -`VARCHAR`. `INT32` and `INT64` may still appear only as Arrow physical storage choices when a declared -`DECIMAL(p,0)` range safely narrows; `UNSUPPORTED` is a conversion failure, never a type. +`VARCHAR`. `INT32` and `INT64` may still appear as Arrow physical storage choices for other mappings; they are not +the canonical representation of SQL `DECIMAL`. `UNSUPPORTED` is a conversion failure, never a type. ## Concrete Exasol SQL mappings ### Numeric values Exasol defines `DECIMAL(p,s)` with `1 <= p <= 36`, `0 <= s <= 36`, and `s <= p`; the defaults are `p = 18` and -`s = 0`. The declared range determines the representation: +`s = 0`. Select the smallest Arrow decimal width that supports the declared precision: - `DOUBLE PRECISION` maps to `FloatingPoint(Double)`. -- `DECIMAL(p,s)` with `s > 0` maps to `Decimal(128)` with `precision = p`, `scale = s`, and `bit_width = 128`. -- Scale-zero `DECIMAL(p,0)` maps to signed `Int(32)` when `p <= 9`, signed `Int(64)` when `10 <= p <= 18`, and - `Decimal(128)` with scale `0` when `p > 18`. -- Integer narrowing is allowed only when the complete declared precision range fits the selected signed width. - Observed batch values cannot justify narrowing. +- `1 <= p <= 9` maps to `Decimal(32)` with `bit_width = 32`. +- `10 <= p <= 18` maps to `Decimal(64)` with `bit_width = 64`. +- `19 <= p <= 36` maps to `Decimal(128)` with `bit_width = 128`. +- All three mappings preserve `precision = p` and `scale = s`, including scale-zero decimals. Consumers may cast a + decimal to an integer when appropriate, but observed values never change the protocol representation. - Preserve the original type name, precision, and scale in field metadata. The Exasol aliases `INTEGER` and `BIGINT`, where exposed by metadata, are represented using their resolved `DECIMAL` precision and scale rather than treated as undocumented independent types. For every decimal field, `0 <= scale <= precision` is required. Decimal conversion is parameter-preserving and -value-preserving within the declared range. Integer narrowing is value-preserving for the declared range but loses -the original decimal physical representation; the source metadata remains available for reconstruction. +value-preserving within the declared range. The selected decimal width is the smallest width supported by Arrow for +the declared precision. -Example field metadata (the physical type is `Decimal(128, precision=12, scale=2)`): +Example field metadata (the physical type is `Decimal(64, precision=12, scale=2)`): See the [decimal field metadata example](examples/decimal_field_metadata.json). @@ -219,7 +219,7 @@ required extension metadata. Future mappings may add Arrow interval types, nativ | --- | --- | --- | | `DOUBLE PRECISION`, `DATE`, `CHAR(n)`, `VARCHAR(n)`, `BOOLEAN` | Value-preserving | Source declaration metadata is retained. | | Positive-scale `DECIMAL` | Parameter- and value-preserving | Decimal precision, scale, and bit width are explicit. | -| Scale-zero `DECIMAL` narrowed to Arrow `Int(32)` or `Int(64)` | Value-preserving, representation-normalized | The declared range fits; source parameters remain in metadata. | +| `DECIMAL` | Parameter- and value-preserving | The smallest Decimal32/64/128 width is selected from declared precision; source parameters remain in metadata. | | `TIMESTAMP(p)` | Precision- and value-preserving | The smallest sufficient Arrow unit is selected. | | `TIMESTAMP ... WITH LOCAL TIME ZONE` | Value-preserving after UTC normalization | Original session-local representation is not preserved. | | `HASHTYPE` | Byte-for-byte and width-preserving | Raw bytes and declared width are transported. | diff --git a/doc/design/v2/protocol/high_level_type_mapping.mmd b/doc/design/v2/protocol/high_level_type_mapping.mmd index f6eaf44..d148a55 100644 --- a/doc/design/v2/protocol/high_level_type_mapping.mmd +++ b/doc/design/v2/protocol/high_level_type_mapping.mmd @@ -2,10 +2,9 @@ flowchart TD A[Exasol documented SQL types] --> B[official type family and declared parameters] B --> N{Numeric} N -->|DOUBLE PRECISION| ND[FloatingPoint Double] - N -->|DECIMAL p,s, s > 0| DEC[Decimal 128: precision p, scale s] - N -->|DECIMAL p,0 and p <= 9| I32[Int signed 32] - N -->|DECIMAL p,0 and 10 <= p <= 18| I64[Int signed 64] - N -->|DECIMAL p,0 and p > 18| D0[Decimal 128: scale 0] + N -->|1 <= p <= 9| D32[Decimal 32: precision p, scale s] + N -->|10 <= p <= 18| D64[Decimal 64: precision p, scale s] + N -->|19 <= p <= 36| D128[Decimal 128: precision p, scale s] B --> T{Timestamp} T -->|p = 0| TS0[Timestamp seconds] T -->|1 <= p <= 3| TS1[Timestamp milliseconds] diff --git a/doc/design/v2/protocol/high_level_type_mapping.svg b/doc/design/v2/protocol/high_level_type_mapping.svg index 7367446..562cc03 100644 --- a/doc/design/v2/protocol/high_level_type_mapping.svg +++ b/doc/design/v2/protocol/high_level_type_mapping.svg @@ -1 +1 @@ -

DOUBLE PRECISION

DECIMAL p,s, s > 0

DECIMAL p,0 and p <= 9

DECIMAL p,0 and 10 <= p <= 18

DECIMAL p,0 and p > 18

p = 0

1 <= p <= 3

4 <= p <= 6

7 <= p <= 9

WITH LOCAL TIME ZONE

INTERVAL YEAR p TO MONTH

INTERVAL DAY lfp TO SECOND fsp

Exasol documented SQL types

official type family and declared parameters

Numeric

FloatingPoint Double

Decimal 128: precision p, scale s

Int signed 32

Int signed 64

Decimal 128: scale 0

Timestamp

Timestamp seconds

Timestamp milliseconds

Timestamp microseconds

Timestamp nanoseconds

Normalize to UTC; timezone UTC

Intervals

Struct years:int32 + months:int32 + Exasol extension

Interval MonthDayNano: months=0 + days + nanoseconds

HASHTYPE

FixedSizeBinary declared byte width

GEOMETRY

Binary + geoarrow.wkb; WKB and SRID metadata

CHAR(n) / VARCHAR(n) / DATE / BOOLEAN

Utf8 / Date Day / Bool / Int

Unsupported or invalid

Reject schema conversion

Arrow-compatible protocol types

 \ No newline at end of file +

DOUBLE PRECISION

1 <= p <= 9

10 <= p <= 18

19 <= p <= 36

p = 0

1 <= p <= 3

4 <= p <= 6

7 <= p <= 9

WITH LOCAL TIME ZONE

INTERVAL YEAR p TO MONTH

INTERVAL DAY lfp TO SECOND fsp

Exasol documented SQL types

official type family and declared parameters

Numeric

FloatingPoint Double

Decimal 32: precision p, scale s

Decimal 64: precision p, scale s

Decimal 128: precision p, scale s

Timestamp

Timestamp seconds

Timestamp milliseconds

Timestamp microseconds

Timestamp nanoseconds

Normalize to UTC; timezone UTC

Intervals

Struct years:int32 + months:int32 + Exasol extension

Interval MonthDayNano: months=0 + days + nanoseconds

HASHTYPE

FixedSizeBinary declared byte width

GEOMETRY

Binary + geoarrow.wkb; WKB and SRID metadata

CHAR(n) / VARCHAR(n) / DATE / BOOLEAN

Utf8 / Date Day / Bool / Int

Unsupported or invalid

Reject schema conversion

Arrow-compatible protocol types

DEC

I32

I64

D0

 \ No newline at end of file diff --git a/udf-runner-cpp/v2/json_schema/validate_schemas.py b/udf-runner-cpp/v2/json_schema/validate_schemas.py index cc7a921..b0a131c 100644 --- a/udf-runner-cpp/v2/json_schema/validate_schemas.py +++ b/udf-runner-cpp/v2/json_schema/validate_schemas.py @@ -44,6 +44,16 @@ def validate_mapping_examples() -> None: precision = int(decimal["metadata"]["exasol:precision"]) scale = int(decimal["metadata"]["exasol:scale"]) assert 0 <= scale <= precision + assert decimal["arrow_storage_type"] == "Decimal(64)" + + for precision, expected_width in ((9, 32), (10, 64), (18, 64), (19, 128), (36, 128)): + if precision <= 9: + width = 32 + elif precision <= 18: + width = 64 + else: + width = 128 + assert width == expected_width timestamp = read_example("timestamp_field_metadata.json") assert timestamp["metadata"]["exasol:type_family"] == "TIMESTAMP WITH LOCAL TIME ZONE" From f458fb19881d7ff83c162c98ddb22e4490e057f9 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Wed, 26 Aug 2026 21:40:30 +0200 Subject: [PATCH 13/26] docs(v2): streamline type mapping metadata Make the high-level column definition the single source for Exasol type parameters and link to its Call Metadata contract from the Arrow mapping document. Remove repeated property ownership text and legacy exasol field metadata from the mapping examples. Keep only conversion-specific GeoArrow and interval extension metadata, and update validation assertions and rendered mapping documentation accordingly. --- .../day_time_interval_field_metadata.json | 8 +- .../examples/decimal_field_metadata.json | 8 +- .../examples/geometry_extension_metadata.json | 5 +- .../examples/hashtype_field_metadata.json | 7 +- .../examples/timestamp_field_metadata.json | 7 +- .../year_month_interval_field_metadata.json | 3 - .../v2/protocol/high_level_type_mapping.md | 85 +++++++------------ .../v2/protocol/high_level_type_mapping.mmd | 9 +- .../v2/protocol/high_level_type_mapping.svg | 2 +- .../v2/json_schema/validate_schemas.py | 21 +++-- 10 files changed, 53 insertions(+), 102 deletions(-) diff --git a/doc/design/v2/protocol/examples/day_time_interval_field_metadata.json b/doc/design/v2/protocol/examples/day_time_interval_field_metadata.json index 3a9cc3a..16e6cac 100644 --- a/doc/design/v2/protocol/examples/day_time_interval_field_metadata.json +++ b/doc/design/v2/protocol/examples/day_time_interval_field_metadata.json @@ -1,11 +1,5 @@ { "name": "ELAPSED", "nullable": true, - "arrow_storage_type": "Interval(MonthDayNano)", - "metadata": { - "exasol:type_name": "INTERVAL DAY(6) TO SECOND(3)", - "exasol:type_family": "INTERVAL DAY TO SECOND", - "exasol:interval_leading_precision": "6", - "exasol:interval_fractional_precision": "3" - } + "arrow_storage_type": "Interval(MonthDayNano)" } diff --git a/doc/design/v2/protocol/examples/decimal_field_metadata.json b/doc/design/v2/protocol/examples/decimal_field_metadata.json index 48ea108..4a6e39c 100644 --- a/doc/design/v2/protocol/examples/decimal_field_metadata.json +++ b/doc/design/v2/protocol/examples/decimal_field_metadata.json @@ -1,11 +1,5 @@ { "name": "AMOUNT", "nullable": true, - "arrow_storage_type": "Decimal(64)", - "metadata": { - "exasol:type_name": "DECIMAL(12,2)", - "exasol:type_family": "DECIMAL", - "exasol:precision": "12", - "exasol:scale": "2" - } + "arrow_storage_type": "Decimal(64)" } diff --git a/doc/design/v2/protocol/examples/geometry_extension_metadata.json b/doc/design/v2/protocol/examples/geometry_extension_metadata.json index 6c2cb93..f15d132 100644 --- a/doc/design/v2/protocol/examples/geometry_extension_metadata.json +++ b/doc/design/v2/protocol/examples/geometry_extension_metadata.json @@ -1,7 +1,4 @@ { "ARROW:extension:name": "geoarrow.wkb", - "ARROW:extension:metadata": "{\"crs\":\"4326\",\"crs_type\":\"srid\"}", - "exasol:type_name": "GEOMETRY(4326)", - "exasol:type_family": "GEOMETRY", - "exasol:srid": "4326" + "ARROW:extension:metadata": "{\"crs\":\"4326\",\"crs_type\":\"srid\"}" } diff --git a/doc/design/v2/protocol/examples/hashtype_field_metadata.json b/doc/design/v2/protocol/examples/hashtype_field_metadata.json index d925dd0..96271c5 100644 --- a/doc/design/v2/protocol/examples/hashtype_field_metadata.json +++ b/doc/design/v2/protocol/examples/hashtype_field_metadata.json @@ -1,10 +1,5 @@ { "name": "DIGEST", "nullable": true, - "arrow_storage_type": "FixedSizeBinary(32)", - "metadata": { - "exasol:type_name": "HASHTYPE(256 BIT)", - "exasol:type_family": "HASHTYPE", - "exasol:hash_byte_width": "32" - } + "arrow_storage_type": "FixedSizeBinary(32)" } diff --git a/doc/design/v2/protocol/examples/timestamp_field_metadata.json b/doc/design/v2/protocol/examples/timestamp_field_metadata.json index 7f86572..f628cb4 100644 --- a/doc/design/v2/protocol/examples/timestamp_field_metadata.json +++ b/doc/design/v2/protocol/examples/timestamp_field_metadata.json @@ -1,10 +1,5 @@ { "name": "CREATED_AT", "nullable": false, - "arrow_storage_type": "Timestamp(Microsecond, UTC)", - "metadata": { - "exasol:type_name": "TIMESTAMP(6) WITH LOCAL TIME ZONE", - "exasol:type_family": "TIMESTAMP WITH LOCAL TIME ZONE", - "exasol:precision": "6" - } + "arrow_storage_type": "Timestamp(Microsecond, UTC)" } diff --git a/doc/design/v2/protocol/examples/year_month_interval_field_metadata.json b/doc/design/v2/protocol/examples/year_month_interval_field_metadata.json index 01129e4..2c190db 100644 --- a/doc/design/v2/protocol/examples/year_month_interval_field_metadata.json +++ b/doc/design/v2/protocol/examples/year_month_interval_field_metadata.json @@ -3,9 +3,6 @@ "nullable": true, "arrow_storage_type": "Struct", "metadata": { - "exasol:type_name": "INTERVAL YEAR(4) TO MONTH", - "exasol:type_family": "INTERVAL YEAR TO MONTH", - "exasol:interval_leading_precision": "4", "ARROW:extension:name": "exasol.interval.year_month", "ARROW:extension:metadata": "{\"layout\":\"years:int32,months:int32\"}" } diff --git a/doc/design/v2/protocol/high_level_type_mapping.md b/doc/design/v2/protocol/high_level_type_mapping.md index 87f48ca..ea24610 100644 --- a/doc/design/v2/protocol/high_level_type_mapping.md +++ b/doc/design/v2/protocol/high_level_type_mapping.md @@ -2,7 +2,9 @@ This document is the normative high-level type-conversion contract for v2 data-stream column schemas. The protocol uses a self-owned subset of Arrow's schema model in `udf_protocol.fbs`; it does not import Arrow's FlatBuffers schema. -`Field.custom_metadata` carries logical-type and extension annotations as string key/value pairs. +`Field.custom_metadata` carries only conversion-specific extension annotations as string key/value pairs. +The high-level column definition in `call_metadata` supplies the Exasol type family and all declared type parameters; +this document defines the corresponding Arrow storage representation and conversion-specific metadata. The conversion is performed from the declared Exasol column type, never from values observed in a batch. A field's `nullable` flag is preserved independently of its physical type. A conversion that is not defined here fails schema @@ -31,28 +33,13 @@ alternate Exasol type names. | `INTERVAL YEAR TO MONTH` | `Struct` + `exasol.interval.year_month` | Preserve the complete Exasol year/month range. | | `INTERVAL DAY TO SECOND` | `Interval(MonthDayNano)` | Encode zero months, signed days, and nanoseconds. | -The type family and source declaration metadata use the following field metadata keys: - -| Key | Value | -| --- | --- | -| `exasol:type_name` | Original Exasol type declaration or type name. | -| `exasol:type_family` | Official Exasol type family from the table above. | -| `exasol:precision` | Decimal or timestamp precision, when declared. | -| `exasol:scale` | Decimal scale, when declared. | -| `exasol:character_length` | Declared `CHAR`/`VARCHAR` character length, when applicable. | -| `exasol:character_set` | Declared `ASCII` or `UTF8` character set, when applicable. | -| `exasol:srid` | Decimal SRID text, when applicable. | -| `exasol:hash_byte_width` | Canonical fixed-width byte count, when applicable. | - -These keys are diagnostic and reconstruction metadata. They do not override the Arrow-compatible physical type. - -The high-level JSON column definition supplies conversion parameters without requiring an implementation to parse -`type_name`. Existing properties retain their API meaning: `size` is the character length for `CHAR`/`VARCHAR` or the -declared hash size for `HASHTYPE`; `precision` is decimal, timestamp, or interval leading-field precision; and -`scale` is reserved for decimal scale. `size_unit` distinguishes `BYTE` from `BIT` for `HASHTYPE`. -`character_set`, `srid`, and `fractional_second_precision` provide the additional parsed parameters needed by the -string, geometry, and day-to-second interval mappings. `type_name` remains the complete source declaration for -diagnostics and reconstruction. +The column properties and their API meaning are defined by the [Call Metadata contract](high_level_payloads.md#call-metadata). + +The official Exasol documentation defines each type's SQL syntax, aliases, parameter limits, and default values. This +document defines only the base type-family to Arrow mapping and the conversion-critical parameter handling. See the +[Exasol data type overview](https://docs.exasol.com/db/latest/sql_references/data_types/datatypesoverview.htm) and +[Exasol data type details](https://docs.exasol.com/db/latest/sql_references/data_types/datatypedetails.htm) for the +complete SQL type definitions. ### Legacy v1 category replacement @@ -66,8 +53,7 @@ the canonical representation of SQL `DECIMAL`. `UNSUPPORTED` is a conversion fai ### Numeric values -Exasol defines `DECIMAL(p,s)` with `1 <= p <= 36`, `0 <= s <= 36`, and `s <= p`; the defaults are `p = 18` and -`s = 0`. Select the smallest Arrow decimal width that supports the declared precision: +For a declared `DECIMAL(p,s)`, select the smallest Arrow decimal width that supports the declared precision. - `DOUBLE PRECISION` maps to `FloatingPoint(Double)`. - `1 <= p <= 9` maps to `Decimal(32)` with `bit_width = 32`. @@ -75,11 +61,6 @@ Exasol defines `DECIMAL(p,s)` with `1 <= p <= 36`, `0 <= s <= 36`, and `s <= p`; - `19 <= p <= 36` maps to `Decimal(128)` with `bit_width = 128`. - All three mappings preserve `precision = p` and `scale = s`, including scale-zero decimals. Consumers may cast a decimal to an integer when appropriate, but observed values never change the protocol representation. -- Preserve the original type name, precision, and scale in field metadata. - -The Exasol aliases `INTEGER` and `BIGINT`, where exposed by metadata, are represented using their resolved -`DECIMAL` precision and scale rather than treated as undocumented independent types. - For every decimal field, `0 <= scale <= precision` is required. Decimal conversion is parameter-preserving and value-preserving within the declared range. The selected decimal width is the smallest width supported by Arrow for the declared precision. @@ -90,20 +71,20 @@ See the [decimal field metadata example](examples/decimal_field_metadata.json). ### Strings -`CHAR(n)` and `VARCHAR(n)` map to `Utf8`. Exasol permits `1 <= n <= 2000` for `CHAR(n)`, with default `n = 1`, -and `1 <= n <= 2,000,000` for `VARCHAR(n)`. Preserve the source type name, declared character length, and -`ASCII`/`UTF8` character set. `CHAR` padding remains an Exasol logical concern; it is not a reason to use +`CHAR(n)` and `VARCHAR(n)` map to `Utf8`. Preserve the declared character length and `ASCII`/`UTF8` character set. +The official Exasol documentation defines the valid lengths and character-set syntax. +`CHAR` padding remains an Exasol logical concern; it is not a reason to use `FixedSizeBinary`, and the mapping does not reinterpret character data as bytes. An empty Exasol string is `NULL` and therefore follows the nullable-field semantics. ### Date and time - `DATE` maps to `Date(Day)`. -- `TIMESTAMP(p)` maps to `Timestamp(unit, "")`. Exasol permits `0 <= p <= 9`, with default `p = 3`. +- `TIMESTAMP(p)` maps to `Timestamp(unit, "")`; the declared fractional precision selects the unit. - Timestamp unit selection is: `p = 0` seconds; `1 <= p <= 3` milliseconds; `4 <= p <= 6` microseconds; and `7 <= p <= 9` nanoseconds. - `TIMESTAMP(p) WITH LOCAL TIME ZONE` is normalized to UTC using the session time zone and maps to - `Timestamp(unit, "UTC")`. Preserve the original type name and precision in metadata. + `Timestamp(unit, "UTC")`. UTC normalization is a semantic normalization, not lossless preservation of the original session-local representation. Exasol internally stores these values normalized to UTC, while input and output are interpreted in @@ -118,13 +99,13 @@ See the [timestamp field metadata example](examples/timestamp_field_metadata.jso ### HASHTYPE `HASHTYPE(n BYTE)` maps to `FixedSizeBinary(byte_width = n)`. `HASHTYPE(m BIT)` maps to -`FixedSizeBinary(byte_width = m / 8)`. Exasol permits `1 <= n <= 1024` bytes, or `8 <= m <= 8192` bits, and `m` -must be a multiple of 8. The default is `HASHTYPE(16 BYTE)`. Invalid declarations are rejected. +`FixedSizeBinary(byte_width = m / 8)`. The declared unit is supplied through `size_unit`; bit declarations must be +byte-aligned and invalid declarations are rejected. The official Exasol documentation defines the valid sizes and +input/display formats. Exasol accepts hexadecimal, UUID, Base64, and Base64URL strings as SQL input syntax; UUID input is supported only for -`HASHTYPE(16 BYTE)`. Transport raw hash bytes rather than any of those textual forms. Preserve the source type name -and canonical byte width in metadata. The `HASHTYPE_FORMAT` display setting, including UUID display, does not change -the Arrow type. +`HASHTYPE(16 BYTE)`. Transport raw hash bytes rather than any of those textual forms. The `HASHTYPE_FORMAT` display +setting, including UUID display, does not change the Arrow type. Example: @@ -137,8 +118,7 @@ See [Exasol HASHTYPE documentation](https://docs.exasol.com/db/latest/sql_refere `INTERVAL YEAR(p) TO MONTH` maps to an Arrow `Struct_` with two children, `years: Int(32, signed)` and `months: Int(32, signed)`, annotated with `ARROW:extension:name = "exasol.interval.year_month"`. Encode the normalized signed year and month components rather than a total-month integer. This preserves Exasol's complete -range, including `-999999999-11` through `999999999-11`. Preserve the declared leading-field precision in -`exasol:interval_leading_precision` and the complete SQL declaration in `exasol:type_name`. +range. The extension metadata describes the struct layout. The standard Arrow `YEAR_MONTH` interval is not sufficient for this mapping because it stores one signed 32-bit total-month value. Exasol permits `999999999` years and `11` months, which requires @@ -147,11 +127,10 @@ extension therefore stores the year and month components separately, preserving overflow or loss of calendar semantics. `INTERVAL DAY(lfp) TO SECOND(fsp)` maps to Arrow `Interval(MonthDayNano)`. Encode `months = 0`, the signed day count, -and the time-of-day component as signed nanoseconds. Preserve `lfp` in `exasol:interval_leading_precision`, `fsp` -in `exasol:interval_fractional_precision`, and the complete SQL declaration in `exasol:type_name`. +and the time-of-day component as signed nanoseconds. -Exasol currently documents effective millisecond accuracy; encode those milliseconds as nanoseconds by multiplying by -`1,000,000`. If Exasol enables true `fsp = 9` storage, place the nanosecond value directly in the same Arrow type. +Encode the source fractional seconds as nanoseconds. If the source accuracy is milliseconds, multiply by +`1,000,000`; if nanosecond accuracy is available, place the nanosecond value directly in the same Arrow type. This represents the complete day range and future nanosecond precision. Do not convert either interval to a timestamp, fixed-duration value, text, or generic binary; calendar interval semantics must remain explicit. @@ -167,7 +146,7 @@ See the [day-time interval field metadata example](examples/day_time_interval_fi `GEOMETRY(srid)` uses variable-size Arrow `Binary` storage with GeoArrow's WKB extension. Exasol's supported geometry objects are `POINT`, `LINESTRING`, `POLYGON`, `MULTIPOINT`, `MULTILINESTRING`, `MULTIPOLYGON`, and -`GEOMETRYCOLLECTION`. `srid` is optional and defaults to `0`, meaning no coordinate system: +`GEOMETRYCOLLECTION`. - `ARROW:extension:name` is exactly `geoarrow.wkb`. - `ARROW:extension:metadata` is UTF-8 JSON. @@ -175,7 +154,6 @@ geometry objects are `POINT`, `LINESTRING`, `POLYGON`, `MULTIPOINT`, `MULTILINES - For an absent or zero SRID, omit CRS metadata. Omit `edges` to select planar/linear edge semantics. - Do not infer an EPSG authority from an Exasol SRID or force one geometry subtype: one column may contain the supported geometry object types listed above. -- Preserve the source type name and SRID in `exasol:*` metadata when needed for diagnostics or reconstruction. - WKT may be accepted at the SQL boundary, but WKB is the canonical Arrow transport representation. Example extension metadata: @@ -193,10 +171,8 @@ The complete Exasol SQL type surface considered by this mapping is: `INTERVAL DAY(lfp) TO SECOND(fsp)`, `GEOMETRY(srid)`, `HASHTYPE(n BYTE)`, `HASHTYPE(m BIT)`, `CHAR(n)`, and `VARCHAR(n)`. -`INTERVAL YEAR(p) TO MONTH` uses leading-field precision `1 <= p <= 9`, default `p = 2`. `INTERVAL DAY(lfp) TO -SECOND(fsp)` uses leading-field precision `1 <= lfp <= 9` and fractional-second precision `0 <= fsp <= 9`, with -defaults `lfp = 2` and `fsp = 3`. Both interval types now have mappings above; the year-month struct mapping -preserves the complete declared range. +Both interval mappings preserve the declared leading and fractional-second precision supplied by the dedicated column +properties. The official Exasol documentation defines the valid precision ranges and omitted-parameter behavior. See the authoritative [Exasol data type overview](https://docs.exasol.com/db/latest/sql_references/data_types/datatypesoverview.htm) and [data type details](https://docs.exasol.com/db/latest/sql_references/data_types/datatypedetails.htm). @@ -217,9 +193,8 @@ required extension metadata. Future mappings may add Arrow interval types, nativ | Mapping | Classification | Reason | | --- | --- | --- | -| `DOUBLE PRECISION`, `DATE`, `CHAR(n)`, `VARCHAR(n)`, `BOOLEAN` | Value-preserving | Source declaration metadata is retained. | -| Positive-scale `DECIMAL` | Parameter- and value-preserving | Decimal precision, scale, and bit width are explicit. | -| `DECIMAL` | Parameter- and value-preserving | The smallest Decimal32/64/128 width is selected from declared precision; source parameters remain in metadata. | +| `DOUBLE PRECISION`, `DATE`, `CHAR(n)`, `VARCHAR(n)`, `BOOLEAN` | Value-preserving | The Arrow representation preserves the values and declared semantics. | +| `DECIMAL` | Parameter- and value-preserving | The smallest Decimal32/64/128 width is selected from declared precision and scale. | | `TIMESTAMP(p)` | Precision- and value-preserving | The smallest sufficient Arrow unit is selected. | | `TIMESTAMP ... WITH LOCAL TIME ZONE` | Value-preserving after UTC normalization | Original session-local representation is not preserved. | | `HASHTYPE` | Byte-for-byte and width-preserving | Raw bytes and declared width are transported. | diff --git a/doc/design/v2/protocol/high_level_type_mapping.mmd b/doc/design/v2/protocol/high_level_type_mapping.mmd index d148a55..a376cd2 100644 --- a/doc/design/v2/protocol/high_level_type_mapping.mmd +++ b/doc/design/v2/protocol/high_level_type_mapping.mmd @@ -19,17 +19,16 @@ flowchart TD B --> G[GEOMETRY] G --> W[Binary + geoarrow.wkb; WKB and SRID metadata] B --> C["CHAR(n) / VARCHAR(n) / DATE / BOOLEAN"] - C --> P[Utf8 / Date Day / Bool / Int] + C --> P[Utf8 / Date Day / Bool] B --> U{Unsupported or invalid} U --> R[Reject schema conversion] DT --> O YM --> O ND --> O[Arrow-compatible protocol types] - DEC --> O - I32 --> O - I64 --> O - D0 --> O + D32 --> O + D64 --> O + D128 --> O TS0 --> O TS1 --> O TS2 --> O diff --git a/doc/design/v2/protocol/high_level_type_mapping.svg b/doc/design/v2/protocol/high_level_type_mapping.svg index 562cc03..ded0cb7 100644 --- a/doc/design/v2/protocol/high_level_type_mapping.svg +++ b/doc/design/v2/protocol/high_level_type_mapping.svg @@ -1 +1 @@ -

DOUBLE PRECISION

1 <= p <= 9

10 <= p <= 18

19 <= p <= 36

p = 0

1 <= p <= 3

4 <= p <= 6

7 <= p <= 9

WITH LOCAL TIME ZONE

INTERVAL YEAR p TO MONTH

INTERVAL DAY lfp TO SECOND fsp

Exasol documented SQL types

official type family and declared parameters

Numeric

FloatingPoint Double

Decimal 32: precision p, scale s

Decimal 64: precision p, scale s

Decimal 128: precision p, scale s

Timestamp

Timestamp seconds

Timestamp milliseconds

Timestamp microseconds

Timestamp nanoseconds

Normalize to UTC; timezone UTC

Intervals

Struct years:int32 + months:int32 + Exasol extension

Interval MonthDayNano: months=0 + days + nanoseconds

HASHTYPE

FixedSizeBinary declared byte width

GEOMETRY

Binary + geoarrow.wkb; WKB and SRID metadata

CHAR(n) / VARCHAR(n) / DATE / BOOLEAN

Utf8 / Date Day / Bool / Int

Unsupported or invalid

Reject schema conversion

Arrow-compatible protocol types

DEC

I32

I64

D0

 \ No newline at end of file +

DOUBLE PRECISION

1 <= p <= 9

10 <= p <= 18

19 <= p <= 36

p = 0

1 <= p <= 3

4 <= p <= 6

7 <= p <= 9

WITH LOCAL TIME ZONE

INTERVAL YEAR p TO MONTH

INTERVAL DAY lfp TO SECOND fsp

Exasol documented SQL types

official type family and declared parameters

Numeric

FloatingPoint Double

Decimal 32: precision p, scale s

Decimal 64: precision p, scale s

Decimal 128: precision p, scale s

Timestamp

Timestamp seconds

Timestamp milliseconds

Timestamp microseconds

Timestamp nanoseconds

Normalize to UTC; timezone UTC

Intervals

Struct years:int32 + months:int32 + Exasol extension

Interval MonthDayNano: months=0 + days + nanoseconds

HASHTYPE

FixedSizeBinary declared byte width

GEOMETRY

Binary + geoarrow.wkb; WKB and SRID metadata

CHAR(n) / VARCHAR(n) / DATE / BOOLEAN

Utf8 / Date Day / Bool

Unsupported or invalid

Reject schema conversion

Arrow-compatible protocol types

 \ No newline at end of file diff --git a/udf-runner-cpp/v2/json_schema/validate_schemas.py b/udf-runner-cpp/v2/json_schema/validate_schemas.py index b0a131c..6040eae 100644 --- a/udf-runner-cpp/v2/json_schema/validate_schemas.py +++ b/udf-runner-cpp/v2/json_schema/validate_schemas.py @@ -40,10 +40,7 @@ def read_example(name: str) -> object: def validate_mapping_examples() -> None: decimal = read_example("decimal_field_metadata.json") - assert decimal["metadata"]["exasol:type_family"] == "DECIMAL" - precision = int(decimal["metadata"]["exasol:precision"]) - scale = int(decimal["metadata"]["exasol:scale"]) - assert 0 <= scale <= precision + assert "metadata" not in decimal assert decimal["arrow_storage_type"] == "Decimal(64)" for precision, expected_width in ((9, 32), (10, 64), (18, 64), (19, 128), (36, 128)): @@ -56,22 +53,30 @@ def validate_mapping_examples() -> None: assert width == expected_width timestamp = read_example("timestamp_field_metadata.json") - assert timestamp["metadata"]["exasol:type_family"] == "TIMESTAMP WITH LOCAL TIME ZONE" + assert "metadata" not in timestamp hashtype = read_example("hashtype_field_metadata.json") - assert 1 <= int(hashtype["metadata"]["exasol:hash_byte_width"]) <= 1024 + assert "metadata" not in hashtype year_month = read_example("year_month_interval_field_metadata.json") + assert set(year_month["metadata"]) == { + "ARROW:extension:name", + "ARROW:extension:metadata", + } assert year_month["metadata"]["ARROW:extension:name"] == "exasol.interval.year_month" day_time = read_example("day_time_interval_field_metadata.json") - assert day_time["metadata"]["exasol:type_family"] == "INTERVAL DAY TO SECOND" + assert "metadata" not in day_time geometry = read_example("geometry_extension_metadata.json") + assert set(geometry) == { + "ARROW:extension:name", + "ARROW:extension:metadata", + } assert geometry["ARROW:extension:name"] == "geoarrow.wkb" extension_metadata = json.loads(geometry["ARROW:extension:metadata"]) assert extension_metadata["crs_type"] == "srid" - assert extension_metadata["crs"] == geometry["exasol:srid"] + assert extension_metadata["crs"] == "4326" def validate_schemas() -> None: From 834ff7f921a2b0445eee8817989351512aa8041c Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Thu, 27 Aug 2026 11:32:37 +0200 Subject: [PATCH 14/26] docs: consolidate v2 data stream references --- doc/design/v2/protocol/low_level.md | 35 +++++------------------------ 1 file changed, 5 insertions(+), 30 deletions(-) diff --git a/doc/design/v2/protocol/low_level.md b/doc/design/v2/protocol/low_level.md index 18f512d..eee78a9 100644 --- a/doc/design/v2/protocol/low_level.md +++ b/doc/design/v2/protocol/low_level.md @@ -11,12 +11,11 @@ This document covers: - framing and message layering - control-stream traffic on `stream_id = 0` - stream ownership rules -- generic call-attached data-stream rules -- transport binding and buffer handoff +- the relationship between calls and their attached data streams +- transport binding -The generic data-stream rules are defined in the companion -[data_stream.md](data_stream.md). They apply below the high-level `DB`/`UDFRunner` -protocol layer. +A call may have at most one bidirectional data stream. The complete data-stream +and buffer-transfer rules are defined in [data_stream.md](data_stream.md). Related diagrams: @@ -52,11 +51,7 @@ The initial protocol binding uses Unix-domain stream sockets. The transport carr stream unchanged; it does not alter logical stream ownership, call semantics, control traffic, or flow control. Future bindings may use TCP with the same framing. Any remote TCP deployment must use TLS together with endpoint -authentication and peer identity validation. TCP/TLS supports only inline record-batch buffers; it has no portable -file-descriptor transfer mechanism. - -Unix-domain sockets may later add descriptor passing for local buffer handoff. This is an optional local -optimization and is not required to implement the base protocol. +authentication and peer identity validation. ## Control Stream @@ -121,19 +116,6 @@ control traffic. Every active non-control stream is terminated when the underlyi `Error` without either close field is a non-terminal diagnostic for the enclosing stream. It does not by itself close a call or connection; peers may continue processing when the error is recoverable. -## Buffer Handoff By Binding - -`Inline` buffers are supported by every binding. Their bytes follow the `DataRecordBatch` metadata on the framed -connection according to the data-stream rules. - -`Memfd` and `OutOfBand` buffers are reserved for Unix-domain sockets using file-descriptor passing. `Memfd` -denotes file-descriptor-backed shared buffers. `OutOfBand` may later cover GPU-memory handles transferred through -the same Unix-only mechanism; it is not a portable GPU-memory transport. - -The initial binding does not define descriptor-to-batch correlation, supported GPU handle types, buffer ownership, -lifetime, cleanup, or capability negotiation. An implementation that cannot establish a compatible local handoff -must use `Inline` buffers. - ## Serialization Mapping - low-frequency call/control metadata: FlatBuffers @@ -141,10 +123,3 @@ must use `Inline` buffers. - named metadata payload bodies such as connection objects or script content: typically JSON This document only records the mapping at a protocol level. It does not redefine the `.fbs` schema itself. -Data streams carry Arrow-compatible physical schemas and buffers; Exasol column selection and logical metadata are -defined by the high-level protocol. - -## Open Questions - -- descriptor-to-batch correlation, ownership, lifetime, cleanup, and capability negotiation for Unix FD handoff -- GPU-memory handle types and validation requirements for any future `OutOfBand` binding From 5a087779a4993cb81a891dfa42b4a3db587a456b Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Thu, 27 Aug 2026 12:23:44 +0200 Subject: [PATCH 15/26] docs: place call lifecycle under low-level protocol --- doc/design/v2/README.md | 4 ++-- doc/design/v2/protocol/design_draft.md | 1 - doc/design/v2/protocol/high_level_calls.md | 2 +- doc/design/v2/protocol/low_level.md | 8 +++++--- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/doc/design/v2/README.md b/doc/design/v2/README.md index 9d3846e..0e0f434 100644 --- a/doc/design/v2/README.md +++ b/doc/design/v2/README.md @@ -3,8 +3,8 @@ This directory contains the split protocol design for the new UDF protocol. - [protocol/design_draft.md](protocol/design_draft.md) is the umbrella design draft. -- [protocol/low_level.md](protocol/low_level.md) describes the wire-level and control-stream rules. -- [protocol/call_lifecycle.md](protocol/call_lifecycle.md) describes the generic call abstraction. +- [protocol/low_level.md](protocol/low_level.md) describes the wire-level rules, generic call lifecycle, and control + stream. - [protocol/data_stream.md](protocol/data_stream.md) is the low-level companion for call-attached data-stream behavior. - [protocol/high_level_calls.md](protocol/high_level_calls.md) describes `Run`, Function operations, diff --git a/doc/design/v2/protocol/design_draft.md b/doc/design/v2/protocol/design_draft.md index 857ab4a..d1745a3 100644 --- a/doc/design/v2/protocol/design_draft.md +++ b/doc/design/v2/protocol/design_draft.md @@ -3,7 +3,6 @@ Detailed protocol references in this directory: - [low_level.md](low_level.md) -- [call_lifecycle.md](call_lifecycle.md) - [data_stream.md](data_stream.md) - [high_level_calls.md](high_level_calls.md) - [high_level_payloads.md](high_level_payloads.md) diff --git a/doc/design/v2/protocol/high_level_calls.md b/doc/design/v2/protocol/high_level_calls.md index c899f8d..c7446ec 100644 --- a/doc/design/v2/protocol/high_level_calls.md +++ b/doc/design/v2/protocol/high_level_calls.md @@ -156,6 +156,6 @@ See [high_level_endpoint_scheduling.svg](high_level_endpoint_scheduling.svg). ## Relationship To Other Docs - low-level framing and generic stream rules live in [low_level.md](low_level.md) -- generic open/close call behavior lives in [call_lifecycle.md](call_lifecycle.md) +- generic open/close call behavior is part of the low-level protocol - generic data-stream behavior lives in [data_stream.md](data_stream.md) - high-level payload contracts live in [high_level_payloads.md](high_level_payloads.md) diff --git a/doc/design/v2/protocol/low_level.md b/doc/design/v2/protocol/low_level.md index eee78a9..2736416 100644 --- a/doc/design/v2/protocol/low_level.md +++ b/doc/design/v2/protocol/low_level.md @@ -1,8 +1,7 @@ # UDF Protocol v2: Low-Level Protocol -This document captures the low-level wire rules for the new UDF protocol. It intentionally excludes -high-level call semantics and scheduling policy; those live in -[high_level_calls.md](high_level_calls.md). +This document captures the low-level wire rules and generic call lifecycle for the new UDF protocol. It intentionally +excludes high-level call semantics and scheduling policy; those live in [high_level_calls.md](high_level_calls.md). ## Scope @@ -11,9 +10,12 @@ This document covers: - framing and message layering - control-stream traffic on `stream_id = 0` - stream ownership rules +- generic call lifecycle - the relationship between calls and their attached data streams - transport binding +A call's generic open, close, and nested-call behavior is defined in +[call_lifecycle.md](call_lifecycle.md). A call may have at most one bidirectional data stream. The complete data-stream and buffer-transfer rules are defined in [data_stream.md](data_stream.md). From b83f88fc5e521a86809111e0b2b4b3df41f5b852 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Thu, 27 Aug 2026 12:28:17 +0200 Subject: [PATCH 16/26] docs: consolidate low-level data stream references --- doc/design/v2/README.md | 2 -- doc/design/v2/protocol/call_lifecycle.md | 4 ++-- doc/design/v2/protocol/design_draft.md | 1 - doc/design/v2/protocol/high_level_calls.md | 2 +- 4 files changed, 3 insertions(+), 6 deletions(-) diff --git a/doc/design/v2/README.md b/doc/design/v2/README.md index 0e0f434..193adf7 100644 --- a/doc/design/v2/README.md +++ b/doc/design/v2/README.md @@ -5,8 +5,6 @@ This directory contains the split protocol design for the new UDF protocol. - [protocol/design_draft.md](protocol/design_draft.md) is the umbrella design draft. - [protocol/low_level.md](protocol/low_level.md) describes the wire-level rules, generic call lifecycle, and control stream. -- [protocol/data_stream.md](protocol/data_stream.md) is the low-level companion for call-attached - data-stream behavior. - [protocol/high_level_calls.md](protocol/high_level_calls.md) describes `Run`, Function operations, `get_connection`, `get_script`, and DB/UDFRunner scheduling policy. - [protocol/high_level_payloads.md](protocol/high_level_payloads.md) defines their named string and JSON diff --git a/doc/design/v2/protocol/call_lifecycle.md b/doc/design/v2/protocol/call_lifecycle.md index cd33367..3abc264 100644 --- a/doc/design/v2/protocol/call_lifecycle.md +++ b/doc/design/v2/protocol/call_lifecycle.md @@ -87,8 +87,8 @@ Nested calls are a generic mechanism, not a special-case callback transport: ## Relationship To Data Streams -This document only records that a call may carry at most one bidirectional data stream. The detailed data-stream -rules are defined in [data_stream.md](data_stream.md). +This document only records that a call may carry at most one bidirectional data stream. Detailed data-stream rules are +part of the low-level protocol. ## Open Questions diff --git a/doc/design/v2/protocol/design_draft.md b/doc/design/v2/protocol/design_draft.md index d1745a3..498a38c 100644 --- a/doc/design/v2/protocol/design_draft.md +++ b/doc/design/v2/protocol/design_draft.md @@ -3,7 +3,6 @@ Detailed protocol references in this directory: - [low_level.md](low_level.md) -- [data_stream.md](data_stream.md) - [high_level_calls.md](high_level_calls.md) - [high_level_payloads.md](high_level_payloads.md) diff --git a/doc/design/v2/protocol/high_level_calls.md b/doc/design/v2/protocol/high_level_calls.md index c7446ec..233979a 100644 --- a/doc/design/v2/protocol/high_level_calls.md +++ b/doc/design/v2/protocol/high_level_calls.md @@ -157,5 +157,5 @@ See [high_level_endpoint_scheduling.svg](high_level_endpoint_scheduling.svg). - low-level framing and generic stream rules live in [low_level.md](low_level.md) - generic open/close call behavior is part of the low-level protocol -- generic data-stream behavior lives in [data_stream.md](data_stream.md) +- generic data-stream behavior is part of the low-level protocol - high-level payload contracts live in [high_level_payloads.md](high_level_payloads.md) From 274246129b78c4c571aeddaf6e0a919afe9f6384 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Thu, 27 Aug 2026 12:51:19 +0200 Subject: [PATCH 17/26] some cleanup --- doc/design/v2/protocol/call_lifecycle.md | 7 +------ doc/design/v2/protocol/data_stream.md | 4 +--- doc/design/v2/protocol/high_level_calls.md | 4 +--- doc/design/v2/protocol/low_level.md | 4 ++-- 4 files changed, 5 insertions(+), 14 deletions(-) diff --git a/doc/design/v2/protocol/call_lifecycle.md b/doc/design/v2/protocol/call_lifecycle.md index 3abc264..7f72e33 100644 --- a/doc/design/v2/protocol/call_lifecycle.md +++ b/doc/design/v2/protocol/call_lifecycle.md @@ -88,9 +88,4 @@ Nested calls are a generic mechanism, not a special-case callback transport: ## Relationship To Data Streams This document only records that a call may carry at most one bidirectional data stream. Detailed data-stream rules are -part of the low-level protocol. - -## Open Questions - -- precise command-selection fields carried in opening metadata -- stronger ordering guarantees between a call and the nested calls it spawns +described in [data_stream.md](data_stream.md) \ No newline at end of file diff --git a/doc/design/v2/protocol/data_stream.md b/doc/design/v2/protocol/data_stream.md index 6df99fd..d668a3f 100644 --- a/doc/design/v2/protocol/data_stream.md +++ b/doc/design/v2/protocol/data_stream.md @@ -125,6 +125,4 @@ The inbound model tracks data received by the local endpoint and `Next(...)` cre - exact end-of-stream marker - exact semantics of `reset` -- exact encoding of schema plus first-batch bundling -- descriptor-to-batch correlation, ownership, lifetime, and cleanup for Unix file-descriptor handoff -- GPU-memory handle types and validation rules for future `OutOfBand` transfer +- exact encoding of schema plus first-batch bundling \ No newline at end of file diff --git a/doc/design/v2/protocol/high_level_calls.md b/doc/design/v2/protocol/high_level_calls.md index 233979a..350abad 100644 --- a/doc/design/v2/protocol/high_level_calls.md +++ b/doc/design/v2/protocol/high_level_calls.md @@ -155,7 +155,5 @@ See [high_level_endpoint_scheduling.svg](high_level_endpoint_scheduling.svg). ## Relationship To Other Docs -- low-level framing and generic stream rules live in [low_level.md](low_level.md) -- generic open/close call behavior is part of the low-level protocol -- generic data-stream behavior is part of the low-level protocol +- low-level protocol live in [low_level.md](low_level.md) - high-level payload contracts live in [high_level_payloads.md](high_level_payloads.md) diff --git a/doc/design/v2/protocol/low_level.md b/doc/design/v2/protocol/low_level.md index 2736416..238024b 100644 --- a/doc/design/v2/protocol/low_level.md +++ b/doc/design/v2/protocol/low_level.md @@ -120,8 +120,8 @@ close a call or connection; peers may continue processing when the error is reco ## Serialization Mapping -- low-frequency call/control metadata: FlatBuffers +- high-frequency call/control metadata: FlatBuffers - record batches: Apache Arrow IPC-compatible payloads - named metadata payload bodies such as connection objects or script content: typically JSON -This document only records the mapping at a protocol level. It does not redefine the `.fbs` schema itself. +The exact flatbuffer definition can be found in [udf_protocol.fbs](../../../../udf-runner-cpp/v2/udf_protocol.fbs) \ No newline at end of file From 158f8c1b968d0e4d5fa66e6492cefed7dcab8848 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Thu, 27 Aug 2026 21:17:22 +0200 Subject: [PATCH 18/26] docs: use fixed-width year-month interval storage --- .../year_month_interval_field_metadata.json | 4 +-- .../v2/protocol/high_level_type_mapping.md | 26 +++++++++++-------- .../v2/protocol/high_level_type_mapping.mmd | 7 +++-- .../v2/protocol/high_level_type_mapping.svg | 2 +- .../v2/json_schema/validate_schemas.py | 21 +++++++++++++++ 5 files changed, 44 insertions(+), 16 deletions(-) diff --git a/doc/design/v2/protocol/examples/year_month_interval_field_metadata.json b/doc/design/v2/protocol/examples/year_month_interval_field_metadata.json index 2c190db..8b01bc4 100644 --- a/doc/design/v2/protocol/examples/year_month_interval_field_metadata.json +++ b/doc/design/v2/protocol/examples/year_month_interval_field_metadata.json @@ -1,9 +1,9 @@ { "name": "AGE", "nullable": true, - "arrow_storage_type": "Struct", + "arrow_storage_type": "Int64", "metadata": { "ARROW:extension:name": "exasol.interval.year_month", - "ARROW:extension:metadata": "{\"layout\":\"years:int32,months:int32\"}" + "ARROW:extension:metadata": "{\"layout\":\"signed_total_months\"}" } } diff --git a/doc/design/v2/protocol/high_level_type_mapping.md b/doc/design/v2/protocol/high_level_type_mapping.md index ea24610..5cdcaef 100644 --- a/doc/design/v2/protocol/high_level_type_mapping.md +++ b/doc/design/v2/protocol/high_level_type_mapping.md @@ -30,7 +30,7 @@ alternate Exasol type names. | `BOOLEAN` | `Bool` | Preserve nullable values. | | `HASHTYPE` | `FixedSizeBinary` | Preserve declared byte width and transport raw bytes. | | `GEOMETRY` | `Binary` + `geoarrow.wkb` | Transport WKB and preserve SRID metadata. | -| `INTERVAL YEAR TO MONTH` | `Struct` + `exasol.interval.year_month` | Preserve the complete Exasol year/month range. | +| `INTERVAL YEAR TO MONTH` | Signed `Int32` or `Int64` total-month count + `exasol.interval.year_month` | Use Int32 for precisions 1–8 and Int64 for precision 9. | | `INTERVAL DAY TO SECOND` | `Interval(MonthDayNano)` | Encode zero months, signed days, and nanoseconds. | The column properties and their API meaning are defined by the [Call Metadata contract](high_level_payloads.md#call-metadata). @@ -115,16 +115,20 @@ See [Exasol HASHTYPE documentation](https://docs.exasol.com/db/latest/sql_refere ### Intervals -`INTERVAL YEAR(p) TO MONTH` maps to an Arrow `Struct_` with two children, `years: Int(32, signed)` and -`months: Int(32, signed)`, annotated with `ARROW:extension:name = "exasol.interval.year_month"`. Encode the -normalized signed year and month components rather than a total-month integer. This preserves Exasol's complete -range. The extension metadata describes the struct layout. +`INTERVAL YEAR(p) TO MONTH` maps to a signed integer containing the normalized total number of months, annotated with +`ARROW:extension:name = "exasol.interval.year_month"`. Encode `years * 12 + months`; the integer remains signed so +negative intervals use negative month counts. Use signed `Int32` for declared year precisions 1 through 8 and signed +`Int64` for precision 9: -The standard Arrow `YEAR_MONTH` interval is not sufficient for this mapping because it stores one signed 32-bit -total-month value. Exasol permits `999999999` years and `11` months, which requires -`999999999 * 12 + 11 = 11999999999` months, exceeding Arrow's signed 32-bit maximum of `2147483647`. The struct -extension therefore stores the year and month components separately, preserving the complete Exasol range without -overflow or loss of calendar semantics. +| Year precision | Maximum absolute month count | Arrow storage | +| --- | ---: | --- | +| `p = 1–8` | `119` to `1,199,999,999` | `Int32` | +| `p = 9` | `11,999,999,999` | `Int64` | + +The maximum supported declaration, `999999999` years and `11` months, requires +`999999999 * 12 + 11 = 11999999999` months. It therefore requires signed `Int64`; precisions 1 through 8 use signed +`Int32` storage. The extension metadata describes the logical signed-total-month layout, while the Arrow field type +defines the physical width. `INTERVAL DAY(lfp) TO SECOND(fsp)` maps to Arrow `Interval(MonthDayNano)`. Encode `months = 0`, the signed day count, and the time-of-day component as signed nanoseconds. @@ -199,6 +203,6 @@ required extension metadata. Future mappings may add Arrow interval types, nativ | `TIMESTAMP ... WITH LOCAL TIME ZONE` | Value-preserving after UTC normalization | Original session-local representation is not preserved. | | `HASHTYPE` | Byte-for-byte and width-preserving | Raw bytes and declared width are transported. | | `GEOMETRY` | Geometry-value preserving, representation-normalized | WKT/engine representation becomes canonical WKB; SRID metadata is retained. | -| `INTERVAL YEAR(p) TO MONTH` | Range-preserving, extension-normalized | Struct fields preserve signed years and months across the complete Exasol range. | +| `INTERVAL YEAR(p) TO MONTH` | Range-preserving, extension-normalized | A precision-appropriate signed integer preserves the normalized total-month value across the complete Exasol range. | | `INTERVAL DAY(lfp) TO SECOND(fsp)` | Range- and precision-preserving, representation-normalized | `MonthDayNano` stores zero months, days, and nanoseconds; current millisecond values are scaled to nanoseconds. | | Unsupported type | Not representable | Schema conversion is rejected. | diff --git a/doc/design/v2/protocol/high_level_type_mapping.mmd b/doc/design/v2/protocol/high_level_type_mapping.mmd index a376cd2..6659676 100644 --- a/doc/design/v2/protocol/high_level_type_mapping.mmd +++ b/doc/design/v2/protocol/high_level_type_mapping.mmd @@ -12,7 +12,9 @@ flowchart TD T -->|7 <= p <= 9| TS3[Timestamp nanoseconds] T -->|WITH LOCAL TIME ZONE| UTC[Normalize to UTC; timezone UTC] B --> I{Intervals} - I -->|INTERVAL YEAR p TO MONTH| YM[Struct years:int32 + months:int32 + Exasol extension] + I -->|INTERVAL YEAR p TO MONTH| YP{Year precision p} + YP -->|p = 1..8| YM32[Signed Int32 total months + Exasol extension] + YP -->|p = 9| YM64[Signed Int64 total months + Exasol extension] I -->|INTERVAL DAY lfp TO SECOND fsp| DT[Interval MonthDayNano: months=0 + days + nanoseconds] B --> H[HASHTYPE] H --> HW[FixedSizeBinary declared byte width] @@ -23,7 +25,8 @@ flowchart TD B --> U{Unsupported or invalid} U --> R[Reject schema conversion] DT --> O - YM --> O + YM32 --> O + YM64 --> O ND --> O[Arrow-compatible protocol types] D32 --> O diff --git a/doc/design/v2/protocol/high_level_type_mapping.svg b/doc/design/v2/protocol/high_level_type_mapping.svg index ded0cb7..369653e 100644 --- a/doc/design/v2/protocol/high_level_type_mapping.svg +++ b/doc/design/v2/protocol/high_level_type_mapping.svg @@ -1 +1 @@ -

DOUBLE PRECISION

1 <= p <= 9

10 <= p <= 18

19 <= p <= 36

p = 0

1 <= p <= 3

4 <= p <= 6

7 <= p <= 9

WITH LOCAL TIME ZONE

INTERVAL YEAR p TO MONTH

INTERVAL DAY lfp TO SECOND fsp

Exasol documented SQL types

official type family and declared parameters

Numeric

FloatingPoint Double

Decimal 32: precision p, scale s

Decimal 64: precision p, scale s

Decimal 128: precision p, scale s

Timestamp

Timestamp seconds

Timestamp milliseconds

Timestamp microseconds

Timestamp nanoseconds

Normalize to UTC; timezone UTC

Intervals

Struct years:int32 + months:int32 + Exasol extension

Interval MonthDayNano: months=0 + days + nanoseconds

HASHTYPE

FixedSizeBinary declared byte width

GEOMETRY

Binary + geoarrow.wkb; WKB and SRID metadata

CHAR(n) / VARCHAR(n) / DATE / BOOLEAN

Utf8 / Date Day / Bool

Unsupported or invalid

Reject schema conversion

Arrow-compatible protocol types

 \ No newline at end of file +

DOUBLE PRECISION

1 <= p <= 9

10 <= p <= 18

19 <= p <= 36

p = 0

1 <= p <= 3

4 <= p <= 6

7 <= p <= 9

WITH LOCAL TIME ZONE

INTERVAL YEAR p TO MONTH

p = 1..8

p = 9

INTERVAL DAY lfp TO SECOND fsp

Exasol documented SQL types

official type family and declared parameters

Numeric

FloatingPoint Double

Decimal 32: precision p, scale s

Decimal 64: precision p, scale s

Decimal 128: precision p, scale s

Timestamp

Timestamp seconds

Timestamp milliseconds

Timestamp microseconds

Timestamp nanoseconds

Normalize to UTC; timezone UTC

Intervals

Year precision p

Signed Int32 total months + Exasol extension

Signed Int64 total months + Exasol extension

Interval MonthDayNano: months=0 + days + nanoseconds

HASHTYPE

FixedSizeBinary declared byte width

GEOMETRY

Binary + geoarrow.wkb; WKB and SRID metadata

CHAR(n) / VARCHAR(n) / DATE / BOOLEAN

Utf8 / Date Day / Bool

Unsupported or invalid

Reject schema conversion

Arrow-compatible protocol types

 \ No newline at end of file diff --git a/udf-runner-cpp/v2/json_schema/validate_schemas.py b/udf-runner-cpp/v2/json_schema/validate_schemas.py index 6040eae..e2d4e08 100644 --- a/udf-runner-cpp/v2/json_schema/validate_schemas.py +++ b/udf-runner-cpp/v2/json_schema/validate_schemas.py @@ -59,11 +59,32 @@ def validate_mapping_examples() -> None: assert "metadata" not in hashtype year_month = read_example("year_month_interval_field_metadata.json") + assert year_month["arrow_storage_type"] == "Int64" assert set(year_month["metadata"]) == { "ARROW:extension:name", "ARROW:extension:metadata", } assert year_month["metadata"]["ARROW:extension:name"] == "exasol.interval.year_month" + assert json.loads(year_month["metadata"]["ARROW:extension:metadata"])["layout"] == "signed_total_months" + + def year_month_width(year_precision: int) -> int: + max_months = (10**year_precision - 1) * 12 + 11 + for bit_width in (32, 64): + if max_months <= 2 ** (bit_width - 1) - 1: + return bit_width + raise AssertionError("year-month interval range does not fit in a signed integer") + + assert [(p, year_month_width(p)) for p in range(1, 10)] == [ + (1, 32), + (2, 32), + (3, 32), + (4, 32), + (5, 32), + (6, 32), + (7, 32), + (8, 32), + (9, 64), + ] day_time = read_example("day_time_interval_field_metadata.json") assert "metadata" not in day_time From a584dfad1216d2bceda09067736db020477b2013 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Thu, 27 Aug 2026 21:48:29 +0200 Subject: [PATCH 19/26] sort files into sub directories --- doc/design/v2/README.md | 10 ++++----- doc/design/v2/protocol/design_draft.md | 6 ++--- .../call_model.mmd} | 0 .../call_model.svg} | 0 .../calls.md} | 22 +++++++++---------- .../endpoint_scheduling.mmd} | 0 .../endpoint_scheduling.svg} | 0 .../examples/call_metadata.json | 0 .../examples/column_definitions.json | 0 .../examples/connection_information.json | 0 .../day_time_interval_field_metadata.json | 0 .../examples/decimal_field_metadata.json | 0 .../examples/export_specification.json | 0 .../examples/geometry_extension_metadata.json | 0 .../examples/hashtype_field_metadata.json | 0 .../examples/import_specification.json | 0 .../examples/timestamp_field_metadata.json | 0 .../year_month_interval_field_metadata.json | 0 .../nested_calls.mmd} | 0 .../nested_calls.svg} | 0 .../payloads.md} | 14 ++++++------ .../run_sequence.mmd} | 0 .../run_sequence.svg} | 0 .../type_mapping.md} | 4 ++-- .../type_mapping.mmd} | 0 .../type_mapping.svg} | 0 .../{ => low_level}/call_lifecycle.md | 2 +- .../{ => low_level}/call_lifecycle.mmd | 0 .../{ => low_level}/call_lifecycle.svg | 0 .../connection_lifecycle.mmd} | 0 .../connection_lifecycle.svg} | 0 .../protocol/{ => low_level}/data_stream.md | 0 .../{ => low_level}/data_stream_flow.mmd | 0 .../{ => low_level}/data_stream_flow.svg | 0 .../{low_level.md => low_level/protocol.md} | 8 +++---- .../transport.md} | 2 +- .../v2/json_schema/validate_schemas.py | 2 +- 37 files changed, 35 insertions(+), 35 deletions(-) rename doc/design/v2/protocol/{high_level_call_model.mmd => high_level/call_model.mmd} (100%) rename doc/design/v2/protocol/{high_level_call_model.svg => high_level/call_model.svg} (100%) rename doc/design/v2/protocol/{high_level_calls.md => high_level/calls.md} (88%) rename doc/design/v2/protocol/{high_level_endpoint_scheduling.mmd => high_level/endpoint_scheduling.mmd} (100%) rename doc/design/v2/protocol/{high_level_endpoint_scheduling.svg => high_level/endpoint_scheduling.svg} (100%) rename doc/design/v2/protocol/{ => high_level}/examples/call_metadata.json (100%) rename doc/design/v2/protocol/{ => high_level}/examples/column_definitions.json (100%) rename doc/design/v2/protocol/{ => high_level}/examples/connection_information.json (100%) rename doc/design/v2/protocol/{ => high_level}/examples/day_time_interval_field_metadata.json (100%) rename doc/design/v2/protocol/{ => high_level}/examples/decimal_field_metadata.json (100%) rename doc/design/v2/protocol/{ => high_level}/examples/export_specification.json (100%) rename doc/design/v2/protocol/{ => high_level}/examples/geometry_extension_metadata.json (100%) rename doc/design/v2/protocol/{ => high_level}/examples/hashtype_field_metadata.json (100%) rename doc/design/v2/protocol/{ => high_level}/examples/import_specification.json (100%) rename doc/design/v2/protocol/{ => high_level}/examples/timestamp_field_metadata.json (100%) rename doc/design/v2/protocol/{ => high_level}/examples/year_month_interval_field_metadata.json (100%) rename doc/design/v2/protocol/{high_level_nested_calls.mmd => high_level/nested_calls.mmd} (100%) rename doc/design/v2/protocol/{high_level_nested_calls.svg => high_level/nested_calls.svg} (100%) rename doc/design/v2/protocol/{high_level_payloads.md => high_level/payloads.md} (86%) rename doc/design/v2/protocol/{high_level_run_sequence.mmd => high_level/run_sequence.mmd} (100%) rename doc/design/v2/protocol/{high_level_run_sequence.svg => high_level/run_sequence.svg} (100%) rename doc/design/v2/protocol/{high_level_type_mapping.md => high_level/type_mapping.md} (98%) rename doc/design/v2/protocol/{high_level_type_mapping.mmd => high_level/type_mapping.mmd} (100%) rename doc/design/v2/protocol/{high_level_type_mapping.svg => high_level/type_mapping.svg} (100%) rename doc/design/v2/protocol/{ => low_level}/call_lifecycle.md (98%) rename doc/design/v2/protocol/{ => low_level}/call_lifecycle.mmd (100%) rename doc/design/v2/protocol/{ => low_level}/call_lifecycle.svg (100%) rename doc/design/v2/protocol/{low_level_connection_lifecycle.mmd => low_level/connection_lifecycle.mmd} (100%) rename doc/design/v2/protocol/{low_level_connection_lifecycle.svg => low_level/connection_lifecycle.svg} (100%) rename doc/design/v2/protocol/{ => low_level}/data_stream.md (100%) rename doc/design/v2/protocol/{ => low_level}/data_stream_flow.mmd (100%) rename doc/design/v2/protocol/{ => low_level}/data_stream_flow.svg (100%) rename doc/design/v2/protocol/{low_level.md => low_level/protocol.md} (95%) rename doc/design/v2/protocol/{transport_technology_overview.md => low_level/transport.md} (98%) diff --git a/doc/design/v2/README.md b/doc/design/v2/README.md index 193adf7..a86d4d3 100644 --- a/doc/design/v2/README.md +++ b/doc/design/v2/README.md @@ -3,17 +3,17 @@ This directory contains the split protocol design for the new UDF protocol. - [protocol/design_draft.md](protocol/design_draft.md) is the umbrella design draft. -- [protocol/low_level.md](protocol/low_level.md) describes the wire-level rules, generic call lifecycle, and control +- [protocol/low_level/protocol.md](protocol/low_level/protocol.md) describes the wire-level rules, generic call lifecycle, and control stream. -- [protocol/high_level_calls.md](protocol/high_level_calls.md) describes `Run`, Function operations, +- [protocol/high_level/calls.md](protocol/high_level/calls.md) describes `Run`, Function operations, `get_connection`, `get_script`, and DB/UDFRunner scheduling policy. -- [protocol/high_level_payloads.md](protocol/high_level_payloads.md) defines their named string and JSON +- [protocol/high_level/payloads.md](protocol/high_level/payloads.md) defines their named string and JSON payload contracts. -- [protocol/high_level_type_mapping.md](protocol/high_level_type_mapping.md) is the high-level Exasol-to-Arrow +- [protocol/high_level/type_mapping.md](protocol/high_level/type_mapping.md) is the high-level Exasol-to-Arrow column conversion contract, including physical types, parameters, and extension metadata. Mermaid sources and rendered SVGs use matching names and scopes so the textual and visual material stays aligned. The low-level schema defines reusable Arrow-compatible physical type capabilities. Exasol type selection and -logical/extension metadata are defined by `high_level_type_mapping.md`. +logical/extension metadata are defined by `high_level/type_mapping.md`. The JSON schemas and external examples can be validated with `poetry run nox -s validate-json-schemas`. diff --git a/doc/design/v2/protocol/design_draft.md b/doc/design/v2/protocol/design_draft.md index 498a38c..34a91ca 100644 --- a/doc/design/v2/protocol/design_draft.md +++ b/doc/design/v2/protocol/design_draft.md @@ -2,9 +2,9 @@ Detailed protocol references in this directory: -- [low_level.md](low_level.md) -- [high_level_calls.md](high_level_calls.md) -- [high_level_payloads.md](high_level_payloads.md) +- [low_level/protocol.md](low_level/protocol.md) +- [high_level/calls.md](high_level/calls.md) +- [high_level/payloads.md](high_level/payloads.md) # General Information diff --git a/doc/design/v2/protocol/high_level_call_model.mmd b/doc/design/v2/protocol/high_level/call_model.mmd similarity index 100% rename from doc/design/v2/protocol/high_level_call_model.mmd rename to doc/design/v2/protocol/high_level/call_model.mmd diff --git a/doc/design/v2/protocol/high_level_call_model.svg b/doc/design/v2/protocol/high_level/call_model.svg similarity index 100% rename from doc/design/v2/protocol/high_level_call_model.svg rename to doc/design/v2/protocol/high_level/call_model.svg diff --git a/doc/design/v2/protocol/high_level_calls.md b/doc/design/v2/protocol/high_level/calls.md similarity index 88% rename from doc/design/v2/protocol/high_level_calls.md rename to doc/design/v2/protocol/high_level/calls.md index 350abad..b5c7d73 100644 --- a/doc/design/v2/protocol/high_level_calls.md +++ b/doc/design/v2/protocol/high_level/calls.md @@ -14,10 +14,10 @@ This document covers: Related diagrams: -- [high_level_call_model.svg](high_level_call_model.svg) -- [high_level_nested_calls.svg](high_level_nested_calls.svg) -- [high_level_run_sequence.svg](high_level_run_sequence.svg) -- [high_level_endpoint_scheduling.svg](high_level_endpoint_scheduling.svg) +- [call_model.svg](call_model.svg) +- [nested_calls.svg](nested_calls.svg) +- [run_sequence.svg](run_sequence.svg) +- [endpoint_scheduling.svg](endpoint_scheduling.svg) ## Call Families @@ -97,7 +97,7 @@ metadata is required in version 1. - each call carries `call_metadata` with the opening message - has no attached data stream in the current model - has the operation-specific request and result payloads defined in - [high_level_payloads.md](high_level_payloads.md) + [payloads.md](payloads.md) ### `get_connection` @@ -114,7 +114,7 @@ metadata is required in version 1. ## Payload Contracts The complete call-metadata, script-metadata, Function, and nested-call payload contracts are defined in -[high_level_payloads.md](high_level_payloads.md). `StringPayload` is used directly for +[payloads.md](payloads.md). `StringPayload` is used directly for scalar strings; JSON is used only where a payload has structured fields. ## Representative Sequences @@ -124,8 +124,8 @@ The current design keeps the high-level sequences intentionally simple: - nested callback-style calls execute while a parent `Run` or Function call remains active - `Run` combines `OpenCall`, `call_metadata`, input schema announcement, and the first input batch when practical -See [high_level_nested_calls.svg](high_level_nested_calls.svg) and -[high_level_run_sequence.svg](high_level_run_sequence.svg). +See [nested_calls.svg](nested_calls.svg) and +[run_sequence.svg](run_sequence.svg). ## Scheduling Policy @@ -145,7 +145,7 @@ call orchestration and do not alter the generic Client/Server stream rules in th 2. if nothing is ready to send, block waiting for new incoming messages 3. monitor peer liveness and terminate unhealthy sessions when needed -See [high_level_endpoint_scheduling.svg](high_level_endpoint_scheduling.svg). +See [endpoint_scheduling.svg](endpoint_scheduling.svg). ## Forward-Looking Ideas Still Open <> @@ -155,5 +155,5 @@ See [high_level_endpoint_scheduling.svg](high_level_endpoint_scheduling.svg). ## Relationship To Other Docs -- low-level protocol live in [low_level.md](low_level.md) -- high-level payload contracts live in [high_level_payloads.md](high_level_payloads.md) +- low-level protocol lives in [../low_level/protocol.md](../low_level/protocol.md) +- high-level payload contracts live in [payloads.md](payloads.md) diff --git a/doc/design/v2/protocol/high_level_endpoint_scheduling.mmd b/doc/design/v2/protocol/high_level/endpoint_scheduling.mmd similarity index 100% rename from doc/design/v2/protocol/high_level_endpoint_scheduling.mmd rename to doc/design/v2/protocol/high_level/endpoint_scheduling.mmd diff --git a/doc/design/v2/protocol/high_level_endpoint_scheduling.svg b/doc/design/v2/protocol/high_level/endpoint_scheduling.svg similarity index 100% rename from doc/design/v2/protocol/high_level_endpoint_scheduling.svg rename to doc/design/v2/protocol/high_level/endpoint_scheduling.svg diff --git a/doc/design/v2/protocol/examples/call_metadata.json b/doc/design/v2/protocol/high_level/examples/call_metadata.json similarity index 100% rename from doc/design/v2/protocol/examples/call_metadata.json rename to doc/design/v2/protocol/high_level/examples/call_metadata.json diff --git a/doc/design/v2/protocol/examples/column_definitions.json b/doc/design/v2/protocol/high_level/examples/column_definitions.json similarity index 100% rename from doc/design/v2/protocol/examples/column_definitions.json rename to doc/design/v2/protocol/high_level/examples/column_definitions.json diff --git a/doc/design/v2/protocol/examples/connection_information.json b/doc/design/v2/protocol/high_level/examples/connection_information.json similarity index 100% rename from doc/design/v2/protocol/examples/connection_information.json rename to doc/design/v2/protocol/high_level/examples/connection_information.json diff --git a/doc/design/v2/protocol/examples/day_time_interval_field_metadata.json b/doc/design/v2/protocol/high_level/examples/day_time_interval_field_metadata.json similarity index 100% rename from doc/design/v2/protocol/examples/day_time_interval_field_metadata.json rename to doc/design/v2/protocol/high_level/examples/day_time_interval_field_metadata.json diff --git a/doc/design/v2/protocol/examples/decimal_field_metadata.json b/doc/design/v2/protocol/high_level/examples/decimal_field_metadata.json similarity index 100% rename from doc/design/v2/protocol/examples/decimal_field_metadata.json rename to doc/design/v2/protocol/high_level/examples/decimal_field_metadata.json diff --git a/doc/design/v2/protocol/examples/export_specification.json b/doc/design/v2/protocol/high_level/examples/export_specification.json similarity index 100% rename from doc/design/v2/protocol/examples/export_specification.json rename to doc/design/v2/protocol/high_level/examples/export_specification.json diff --git a/doc/design/v2/protocol/examples/geometry_extension_metadata.json b/doc/design/v2/protocol/high_level/examples/geometry_extension_metadata.json similarity index 100% rename from doc/design/v2/protocol/examples/geometry_extension_metadata.json rename to doc/design/v2/protocol/high_level/examples/geometry_extension_metadata.json diff --git a/doc/design/v2/protocol/examples/hashtype_field_metadata.json b/doc/design/v2/protocol/high_level/examples/hashtype_field_metadata.json similarity index 100% rename from doc/design/v2/protocol/examples/hashtype_field_metadata.json rename to doc/design/v2/protocol/high_level/examples/hashtype_field_metadata.json diff --git a/doc/design/v2/protocol/examples/import_specification.json b/doc/design/v2/protocol/high_level/examples/import_specification.json similarity index 100% rename from doc/design/v2/protocol/examples/import_specification.json rename to doc/design/v2/protocol/high_level/examples/import_specification.json diff --git a/doc/design/v2/protocol/examples/timestamp_field_metadata.json b/doc/design/v2/protocol/high_level/examples/timestamp_field_metadata.json similarity index 100% rename from doc/design/v2/protocol/examples/timestamp_field_metadata.json rename to doc/design/v2/protocol/high_level/examples/timestamp_field_metadata.json diff --git a/doc/design/v2/protocol/examples/year_month_interval_field_metadata.json b/doc/design/v2/protocol/high_level/examples/year_month_interval_field_metadata.json similarity index 100% rename from doc/design/v2/protocol/examples/year_month_interval_field_metadata.json rename to doc/design/v2/protocol/high_level/examples/year_month_interval_field_metadata.json diff --git a/doc/design/v2/protocol/high_level_nested_calls.mmd b/doc/design/v2/protocol/high_level/nested_calls.mmd similarity index 100% rename from doc/design/v2/protocol/high_level_nested_calls.mmd rename to doc/design/v2/protocol/high_level/nested_calls.mmd diff --git a/doc/design/v2/protocol/high_level_nested_calls.svg b/doc/design/v2/protocol/high_level/nested_calls.svg similarity index 100% rename from doc/design/v2/protocol/high_level_nested_calls.svg rename to doc/design/v2/protocol/high_level/nested_calls.svg diff --git a/doc/design/v2/protocol/high_level_payloads.md b/doc/design/v2/protocol/high_level/payloads.md similarity index 86% rename from doc/design/v2/protocol/high_level_payloads.md rename to doc/design/v2/protocol/high_level/payloads.md index f1d1f6b..e9be61a 100644 --- a/doc/design/v2/protocol/high_level_payloads.md +++ b/doc/design/v2/protocol/high_level/payloads.md @@ -20,20 +20,20 @@ calls; the last received values apply to subsequent calls. It is not carried on ## Call Metadata Every `Run` and Function call carries one `call_metadata` JSON payload in the same `StreamMessage` as `OpenCall`. -It conforms to [call_metadata.schema.json](../../../../udf-runner-cpp/v2/json_schema/call_metadata.schema.json) and +It conforms to [call_metadata.schema.json](../../../../../udf-runner-cpp/v2/json_schema/call_metadata.schema.json) and supplies the per-invocation execution context and the input/output iterator and column definitions. The script name and source are deliberately excluded because they are connection-scoped script metadata. Column `type` values use official Exasol type families, while `type_name` carries the complete parameterized Exasol SQL declaration. Their Arrow physical representation and metadata rules are defined in -[high_level_type_mapping.md](high_level_type_mapping.md). -The shared column-definition contract is defined in [column.schema.json](../../../../udf-runner-cpp/v2/json_schema/column.schema.json) +[type_mapping.md](type_mapping.md). +The shared column-definition contract is defined in [column.schema.json](../../../../../udf-runner-cpp/v2/json_schema/column.schema.json) and is referenced by both call metadata and import specifications. See the [call metadata example](examples/call_metadata.json). In a column definition, `type` is the official Exasol type family constrained by the schema's `exasol_type` enum, while `type_name` is the complete SQL declaration. Parsed parameters such as `precision` and `scale` are included where applicable. The corresponding -Arrow physical storage type and field metadata are defined by the [high-level type mapping](high_level_type_mapping.md). +Arrow physical storage type and field metadata are defined by the [high-level type mapping](type_mapping.md). The existing `size`, `precision`, and `scale` properties remain part of the column API. The following definitions show how the additional mapped types expose their parameters without requiring the consumer to parse `type_name`: @@ -58,11 +58,11 @@ The virtual-schema request and all Function results retain their existing string defined by the respective script API, not by this transport protocol. `import_specification` conforms to -[import_specification.schema.json](../../../../udf-runner-cpp/v2/json_schema/import_specification.schema.json). +[import_specification.schema.json](../../../../../udf-runner-cpp/v2/json_schema/import_specification.schema.json). See the [import specification example](examples/import_specification.json). `export_specification` conforms to -[export_specification.schema.json](../../../../udf-runner-cpp/v2/json_schema/export_specification.schema.json). +[export_specification.schema.json](../../../../../udf-runner-cpp/v2/json_schema/export_specification.schema.json). See the [export specification example](examples/export_specification.json). ## Nested Calls @@ -75,7 +75,7 @@ See the [export specification example](examples/export_specification.json). | `get_script` | `script_name`: `StringPayload` | `script`: `StringPayload` | `connection_information` conforms to -[connection_information.schema.json](../../../../udf-runner-cpp/v2/json_schema/connection_information.schema.json). +[connection_information.schema.json](../../../../../udf-runner-cpp/v2/json_schema/connection_information.schema.json). See the [connection information example](examples/connection_information.json). ## Deferred Calls diff --git a/doc/design/v2/protocol/high_level_run_sequence.mmd b/doc/design/v2/protocol/high_level/run_sequence.mmd similarity index 100% rename from doc/design/v2/protocol/high_level_run_sequence.mmd rename to doc/design/v2/protocol/high_level/run_sequence.mmd diff --git a/doc/design/v2/protocol/high_level_run_sequence.svg b/doc/design/v2/protocol/high_level/run_sequence.svg similarity index 100% rename from doc/design/v2/protocol/high_level_run_sequence.svg rename to doc/design/v2/protocol/high_level/run_sequence.svg diff --git a/doc/design/v2/protocol/high_level_type_mapping.md b/doc/design/v2/protocol/high_level/type_mapping.md similarity index 98% rename from doc/design/v2/protocol/high_level_type_mapping.md rename to doc/design/v2/protocol/high_level/type_mapping.md index 5cdcaef..7636974 100644 --- a/doc/design/v2/protocol/high_level_type_mapping.md +++ b/doc/design/v2/protocol/high_level/type_mapping.md @@ -10,7 +10,7 @@ The conversion is performed from the declared Exasol column type, never from val `nullable` flag is preserved independently of its physical type. A conversion that is not defined here fails schema conversion with a descriptive error. -The [mapping diagram](high_level_type_mapping.svg) summarizes the decision paths below. +The [mapping diagram](type_mapping.svg) summarizes the decision paths below. ## Official Exasol type mappings @@ -33,7 +33,7 @@ alternate Exasol type names. | `INTERVAL YEAR TO MONTH` | Signed `Int32` or `Int64` total-month count + `exasol.interval.year_month` | Use Int32 for precisions 1–8 and Int64 for precision 9. | | `INTERVAL DAY TO SECOND` | `Interval(MonthDayNano)` | Encode zero months, signed days, and nanoseconds. | -The column properties and their API meaning are defined by the [Call Metadata contract](high_level_payloads.md#call-metadata). +The column properties and their API meaning are defined by the [Call Metadata contract](payloads.md#call-metadata). The official Exasol documentation defines each type's SQL syntax, aliases, parameter limits, and default values. This document defines only the base type-family to Arrow mapping and the conversion-critical parameter handling. See the diff --git a/doc/design/v2/protocol/high_level_type_mapping.mmd b/doc/design/v2/protocol/high_level/type_mapping.mmd similarity index 100% rename from doc/design/v2/protocol/high_level_type_mapping.mmd rename to doc/design/v2/protocol/high_level/type_mapping.mmd diff --git a/doc/design/v2/protocol/high_level_type_mapping.svg b/doc/design/v2/protocol/high_level/type_mapping.svg similarity index 100% rename from doc/design/v2/protocol/high_level_type_mapping.svg rename to doc/design/v2/protocol/high_level/type_mapping.svg diff --git a/doc/design/v2/protocol/call_lifecycle.md b/doc/design/v2/protocol/low_level/call_lifecycle.md similarity index 98% rename from doc/design/v2/protocol/call_lifecycle.md rename to doc/design/v2/protocol/low_level/call_lifecycle.md index 7f72e33..d092029 100644 --- a/doc/design/v2/protocol/call_lifecycle.md +++ b/doc/design/v2/protocol/low_level/call_lifecycle.md @@ -88,4 +88,4 @@ Nested calls are a generic mechanism, not a special-case callback transport: ## Relationship To Data Streams This document only records that a call may carry at most one bidirectional data stream. Detailed data-stream rules are -described in [data_stream.md](data_stream.md) \ No newline at end of file +part of the low-level protocol. diff --git a/doc/design/v2/protocol/call_lifecycle.mmd b/doc/design/v2/protocol/low_level/call_lifecycle.mmd similarity index 100% rename from doc/design/v2/protocol/call_lifecycle.mmd rename to doc/design/v2/protocol/low_level/call_lifecycle.mmd diff --git a/doc/design/v2/protocol/call_lifecycle.svg b/doc/design/v2/protocol/low_level/call_lifecycle.svg similarity index 100% rename from doc/design/v2/protocol/call_lifecycle.svg rename to doc/design/v2/protocol/low_level/call_lifecycle.svg diff --git a/doc/design/v2/protocol/low_level_connection_lifecycle.mmd b/doc/design/v2/protocol/low_level/connection_lifecycle.mmd similarity index 100% rename from doc/design/v2/protocol/low_level_connection_lifecycle.mmd rename to doc/design/v2/protocol/low_level/connection_lifecycle.mmd diff --git a/doc/design/v2/protocol/low_level_connection_lifecycle.svg b/doc/design/v2/protocol/low_level/connection_lifecycle.svg similarity index 100% rename from doc/design/v2/protocol/low_level_connection_lifecycle.svg rename to doc/design/v2/protocol/low_level/connection_lifecycle.svg diff --git a/doc/design/v2/protocol/data_stream.md b/doc/design/v2/protocol/low_level/data_stream.md similarity index 100% rename from doc/design/v2/protocol/data_stream.md rename to doc/design/v2/protocol/low_level/data_stream.md diff --git a/doc/design/v2/protocol/data_stream_flow.mmd b/doc/design/v2/protocol/low_level/data_stream_flow.mmd similarity index 100% rename from doc/design/v2/protocol/data_stream_flow.mmd rename to doc/design/v2/protocol/low_level/data_stream_flow.mmd diff --git a/doc/design/v2/protocol/data_stream_flow.svg b/doc/design/v2/protocol/low_level/data_stream_flow.svg similarity index 100% rename from doc/design/v2/protocol/data_stream_flow.svg rename to doc/design/v2/protocol/low_level/data_stream_flow.svg diff --git a/doc/design/v2/protocol/low_level.md b/doc/design/v2/protocol/low_level/protocol.md similarity index 95% rename from doc/design/v2/protocol/low_level.md rename to doc/design/v2/protocol/low_level/protocol.md index 238024b..a37cd9f 100644 --- a/doc/design/v2/protocol/low_level.md +++ b/doc/design/v2/protocol/low_level/protocol.md @@ -1,7 +1,7 @@ # UDF Protocol v2: Low-Level Protocol This document captures the low-level wire rules and generic call lifecycle for the new UDF protocol. It intentionally -excludes high-level call semantics and scheduling policy; those live in [high_level_calls.md](high_level_calls.md). +excludes high-level call semantics and scheduling policy; those live in [../high_level/calls.md](../high_level/calls.md). ## Scope @@ -21,7 +21,7 @@ and buffer-transfer rules are defined in [data_stream.md](data_stream.md). Related diagrams: -- [low_level_connection_lifecycle.svg](low_level_connection_lifecycle.svg) +- [connection_lifecycle.svg](connection_lifecycle.svg) ## Roles @@ -98,7 +98,7 @@ At the low level, a session progresses through: 3. normal call/control traffic 4. a two-way `CloseConnection` exchange on stream `0`, followed by transport close; or abort -See [low_level_connection_lifecycle.svg](low_level_connection_lifecycle.svg). +See [connection_lifecycle.svg](connection_lifecycle.svg). ## Close Semantics @@ -124,4 +124,4 @@ close a call or connection; peers may continue processing when the error is reco - record batches: Apache Arrow IPC-compatible payloads - named metadata payload bodies such as connection objects or script content: typically JSON -The exact flatbuffer definition can be found in [udf_protocol.fbs](../../../../udf-runner-cpp/v2/udf_protocol.fbs) \ No newline at end of file +The exact flatbuffer definition can be found in [udf_protocol.fbs](../../../../../udf-runner-cpp/v2/udf_protocol.fbs) diff --git a/doc/design/v2/protocol/transport_technology_overview.md b/doc/design/v2/protocol/low_level/transport.md similarity index 98% rename from doc/design/v2/protocol/transport_technology_overview.md rename to doc/design/v2/protocol/low_level/transport.md index 7d09ab4..25e2ffa 100644 --- a/doc/design/v2/protocol/transport_technology_overview.md +++ b/doc/design/v2/protocol/low_level/transport.md @@ -53,7 +53,7 @@ identity, multiplexing, structured cancellation, or remote deployment. ## Serialization Formats For the v2 protocol's high-level Exasol column contract and its Arrow-compatible record-batch representation, see -[high_level_type_mapping.md](high_level_type_mapping.md), including the GeoArrow WKB representation for geometry +[../high_level/type_mapping.md](../high_level/type_mapping.md), including the GeoArrow WKB representation for geometry values. | Format | Characteristics | Best fit | Main limitation | diff --git a/udf-runner-cpp/v2/json_schema/validate_schemas.py b/udf-runner-cpp/v2/json_schema/validate_schemas.py index e2d4e08..6abe31e 100644 --- a/udf-runner-cpp/v2/json_schema/validate_schemas.py +++ b/udf-runner-cpp/v2/json_schema/validate_schemas.py @@ -8,7 +8,7 @@ ROOT = Path(__file__).resolve().parent -EXAMPLES = ROOT.parents[2] / "doc" / "design" / "v2" / "protocol" / "examples" +EXAMPLES = ROOT.parents[2] / "doc" / "design" / "v2" / "protocol" / "high_level" / "examples" def load_schemas() -> dict[str, dict]: From 9368b05e9c4fa417325b070fe6758a94a487ea25 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Fri, 28 Aug 2026 13:54:05 +0200 Subject: [PATCH 20/26] docs: address v2 protocol review comments --- .../v2/protocol/high_level/call_model.mmd | 2 +- .../v2/protocol/high_level/call_model.svg | 2 +- doc/design/v2/protocol/high_level/calls.md | 18 +++---- .../high_level/endpoint_scheduling.mmd | 6 +-- .../high_level/endpoint_scheduling.svg | 2 +- .../high_level/examples/call_metadata.json | 10 ---- .../high_level/examples/column_metadata.json | 12 +++++ doc/design/v2/protocol/high_level/payloads.md | 48 +++++++++++-------- .../v2/protocol/high_level/type_mapping.md | 3 ++ .../v2/json_schema/call_metadata.schema.json | 8 +++- .../json_schema/column_metadata.schema.json | 11 +++++ .../v2/json_schema/validate_schemas.py | 15 +++++- udf-runner-cpp/v2/udf_protocol.fbs | 5 +- 13 files changed, 90 insertions(+), 52 deletions(-) create mode 100644 doc/design/v2/protocol/high_level/examples/column_metadata.json create mode 100644 udf-runner-cpp/v2/json_schema/column_metadata.schema.json diff --git a/doc/design/v2/protocol/high_level/call_model.mmd b/doc/design/v2/protocol/high_level/call_model.mmd index 99ddbae..9d3cc97 100644 --- a/doc/design/v2/protocol/high_level/call_model.mmd +++ b/doc/design/v2/protocol/high_level/call_model.mmd @@ -1,6 +1,6 @@ flowchart TD Call["Call\n(CallOpen / first peer call-scoped message / CallClose)"] - Call -->|opened by DB or UDFRunner| Opener{{"Either side may open a call"}} + Call -->|opened by DB, or by UDFRunner only when nested| Opener{{"DB opens top-level calls"}} Call -->|optional, at most one| DataStream["Bidirectional Data Stream\n(independent schema per direction)"] Call -->|optional| Nested["Nested Call(s)\n(started by nested CallOpen traffic)"] diff --git a/doc/design/v2/protocol/high_level/call_model.svg b/doc/design/v2/protocol/high_level/call_model.svg index 5d5ccc3..e4552b1 100644 --- a/doc/design/v2/protocol/high_level/call_model.svg +++ b/doc/design/v2/protocol/high_level/call_model.svg @@ -1 +1 @@ -

opened by DB or UDFRunner

optional, at most one

optional

instance of

instance of

instance of

instance of

Call
(CallOpen / first peer call-scoped message / CallClose)

Either side may open a call

Bidirectional Data Stream
(independent schema per direction)

Nested Call(s)
(started by nested CallOpen traffic)

Run (DB-opened)

Function operation (DB-opened)

get_connection (UDFRunner-opened)

get_script (UDFRunner-opened)

Input/Output Data Stream

default output columns / virtual schema / import SQL / export SQL

 \ No newline at end of file +

opened by DB, or by UDFRunner only when nested

optional, at most one

optional

instance of

instance of

instance of

instance of

Call
(CallOpen / first peer call-scoped message / CallClose)

DB opens top-level calls

Bidirectional Data Stream
(independent schema per direction)

Nested Call(s)
(started by nested CallOpen traffic)

Run (DB-opened)

Function operation (DB-opened)

get_connection (UDFRunner-opened)

get_script (UDFRunner-opened)

Input/Output Data Stream

default output columns / virtual schema / import SQL / export SQL

 \ No newline at end of file diff --git a/doc/design/v2/protocol/high_level/calls.md b/doc/design/v2/protocol/high_level/calls.md index b5c7d73..9234f1b 100644 --- a/doc/design/v2/protocol/high_level/calls.md +++ b/doc/design/v2/protocol/high_level/calls.md @@ -28,14 +28,15 @@ Related diagrams: | `Run` | Yes, bidirectional | Each direction carries group and row correlation in the data itself. | | Function operation | No | One of `default_output_columns`, `virtual_schema_adapter`, `generate_sql_for_import_spec`, or `generate_sql_for_export_spec`. | -### `UDFRunner`-opened calls +### Nested `UDFRunner`-opened calls | Call | Data stream | Notes | | --- | --- | --- | | `get_connection` | No | Returns connection information. | | `get_script` | No | Returns script content. | -These `UDFRunner`-opened calls are ordinary nested calls, not a separate callback transport. +These `UDFRunner`-opened calls are ordinary nested calls, not a separate callback transport. `UDFRunner` does not open +top-level calls while idle; it can open them only while handling an active DB call. ## Call-Specific Result Payloads @@ -48,7 +49,8 @@ payload may be sent while the call remains active or together with `CloseCall` i ### `Run` - opened by `DB` -- carries `call_metadata` with the opening message and may carry the first input batch together +- uses `call_metadata` and `column_metadata`, sent before any call, between calls, or with the opening message +- may carry the first input batch together with the opening message - may stay active while nested calls such as `get_script` or `get_connection` execute - group and row correlation belong in the data, not in `Next(...)` @@ -94,7 +96,7 @@ metadata is required in version 1. ### Function Operations - opened by `DB` with one of the Function operation names -- each call carries `call_metadata` with the opening message +- each call uses `call_metadata` and `column_metadata`, sent before any call, between calls, or with the opening message - has no attached data stream in the current model - has the operation-specific request and result payloads defined in [payloads.md](payloads.md) @@ -134,9 +136,9 @@ call orchestration and do not alter the generic Client/Server stream rules in th ### `UDFRunner` -1. prioritize receiving over sending -2. only block during receive -3. do not receive more messages than were requested +1. run socket handling and user-code execution as independently wakeable activities +2. wait for either DB socket activity or user-code activity; do not block solely on socket receive +3. use `Next(...)` byte budgets to bound data in flight; do not impose a message-count limit 4. send regular `KeepAlive` messages so `DB` can continue housekeeping ### `DB` @@ -149,7 +151,7 @@ See [endpoint_scheduling.svg](endpoint_scheduling.svg). ## Forward-Looking Ideas Still Open <> -- `Script` and `execute_query` call shapes and data streams +- `ExecuteScript` and `execute_query` call shapes and data streams - whether `UDFRunner` may open its own pquery-style call to `DB` - whether table-prefetch-like declarations should be added for future call setup diff --git a/doc/design/v2/protocol/high_level/endpoint_scheduling.mmd b/doc/design/v2/protocol/high_level/endpoint_scheduling.mmd index faaea84..2105bb7 100644 --- a/doc/design/v2/protocol/high_level/endpoint_scheduling.mmd +++ b/doc/design/v2/protocol/high_level/endpoint_scheduling.mmd @@ -3,10 +3,10 @@ stateDiagram-v2 [*] --> UdfIdle UdfIdle --> UdfInitializing: Accept UdfInitializing --> UdfActive: Capabilities - UdfActive --> UdfWaitingForInboundData: Receive - UdfWaitingForInboundData --> UdfSendingData: Next credit + UdfActive --> UdfWaitingForActivity: Wait for DB or user code + UdfWaitingForActivity --> UdfSendingData: Next credit UdfSendingData --> UdfActive: Transfer done - UdfWaitingForInboundData --> UdfActive: Message + UdfWaitingForActivity --> UdfActive: Socket or user-code event } state "DB" as DB { diff --git a/doc/design/v2/protocol/high_level/endpoint_scheduling.svg b/doc/design/v2/protocol/high_level/endpoint_scheduling.svg index 398d351..008a7e6 100644 --- a/doc/design/v2/protocol/high_level/endpoint_scheduling.svg +++ b/doc/design/v2/protocol/high_level/endpoint_scheduling.svg @@ -1 +1 @@ -

UDFRunner

Accept

Capabilities

Receive

Next credit

Transfer done

Message

UdfIdle

UdfInitializing

UdfActive

UdfWaitingForInboundData

UdfSendingData

DB

Connect

Capabilities

Call

Reply

Wait

Message

Data transfer

Transfer done

DbIdle

DbInitializing

DbActive

DbHandlingNestedCall

DbWaitingForUdfMessage

DbSendingData

 \ No newline at end of file +

UDFRunner

Accept

Capabilities

Wait for DB or user code

Next credit

Transfer done

Socket or user-code event

UdfIdle

UdfInitializing

UdfActive

UdfWaitingForActivity

UdfSendingData

DB

Connect

Capabilities

Call

Reply

Wait

Message

Data transfer

Transfer done

DbIdle

DbInitializing

DbActive

DbHandlingNestedCall

DbWaitingForUdfMessage

DbSendingData

 \ No newline at end of file diff --git a/doc/design/v2/protocol/high_level/examples/call_metadata.json b/doc/design/v2/protocol/high_level/examples/call_metadata.json index 82fdf8b..b6dc87f 100644 --- a/doc/design/v2/protocol/high_level/examples/call_metadata.json +++ b/doc/design/v2/protocol/high_level/examples/call_metadata.json @@ -10,15 +10,5 @@ "script_schema": "SYS", "input_iter_type": "EXACTLY_ONCE", "output_iter_type": "EXACTLY_ONCE", - "input_columns": [ - { - "name": "AMOUNT", - "type": "DECIMAL", - "type_name": "DECIMAL(12,2)", - "precision": 12, - "scale": 2 - } - ], - "output_columns": [], "single_call_mode": false } diff --git a/doc/design/v2/protocol/high_level/examples/column_metadata.json b/doc/design/v2/protocol/high_level/examples/column_metadata.json new file mode 100644 index 0000000..3de2518 --- /dev/null +++ b/doc/design/v2/protocol/high_level/examples/column_metadata.json @@ -0,0 +1,12 @@ +{ + "input_columns": [ + { + "name": "AMOUNT", + "type": "DECIMAL", + "type_name": "DECIMAL(12,2)", + "precision": 12, + "scale": 2 + } + ], + "output_columns": [] +} diff --git a/doc/design/v2/protocol/high_level/payloads.md b/doc/design/v2/protocol/high_level/payloads.md index e9be61a..6c8544d 100644 --- a/doc/design/v2/protocol/high_level/payloads.md +++ b/doc/design/v2/protocol/high_level/payloads.md @@ -6,46 +6,52 @@ value is UTF-8 JSON conforming to its linked schema. ## Script Metadata -After the `Server` sends `ServerCapabilities`, `DB` may set the connection's script metadata on control stream -`0`. It sends these `StringPayload` values together or in separate control messages: +After the `Server` sends `ServerCapabilities`, `DB` may set the connection's script metadata on control stream `0` +before any call is sent, between calls, or in a call-opening message. It sends these `StringPayload` values together or +in separate messages: | Name | Value | Meaning | | --- | --- | --- | | `script_name` | string | Name of the script used by subsequent `Run` or Function calls. | | `script_source` | string | Source code of that script. | -Script metadata must be sent before a `Run` or Function call that uses it. It may be replaced only between such -calls; the last received values apply to subsequent calls. It is not carried on a `Run` or Function stream. +Script metadata must be received before a `Run` or Function call that uses it. Metadata sent before any call or between +calls applies to subsequent calls. Metadata sent with `OpenCall` applies to that call and subsequent calls. The latest +received value replaces the previous value; metadata is not sent during an active call. ## Call Metadata -Every `Run` and Function call carries one `call_metadata` JSON payload in the same `StreamMessage` as `OpenCall`. -It conforms to [call_metadata.schema.json](../../../../../udf-runner-cpp/v2/json_schema/call_metadata.schema.json) and -supplies the per-invocation execution context and the input/output iterator and column definitions. The script name -and source are deliberately excluded because they are connection-scoped script metadata. -Column `type` values use official Exasol type families, while `type_name` carries the complete parameterized Exasol -SQL declaration. Their Arrow physical representation and metadata rules are defined in -[type_mapping.md](type_mapping.md). -The shared column-definition contract is defined in [column.schema.json](../../../../../udf-runner-cpp/v2/json_schema/column.schema.json) -and is referenced by both call metadata and import specifications. +Every `Run` and Function call uses one `call_metadata` JSON payload. It may be sent on control stream `0` before any +call is sent or between calls, or in the same `StreamMessage` as `OpenCall`. It conforms to +[call_metadata.schema.json](../../../../../udf-runner-cpp/v2/json_schema/call_metadata.schema.json) and supplies the +per-invocation execution context and iterator settings. Metadata sent before any call or between calls applies to +subsequent calls. Metadata sent with `OpenCall` applies to that call and subsequent calls. The latest received value +replaces the previous value. It is not sent during an active call. + +Column definitions are carried separately in one `column_metadata` JSON payload. It conforms to +[column_metadata.schema.json](../../../../../udf-runner-cpp/v2/json_schema/column_metadata.schema.json) and may be +sent on control stream `0` before any call is sent or between calls, or in the same `StreamMessage` as `OpenCall`. +Metadata sent before any call or between calls applies to subsequent calls. Metadata sent with `OpenCall` applies to +that call and subsequent calls. The latest received value replaces the previous value. It is not sent during an active +call. Column `type` values use official Exasol type families, while `type_name` carries the +complete parameterized Exasol SQL declaration. Their Arrow physical representation and metadata rules are defined in +[type_mapping.md](type_mapping.md). The shared column-definition contract is defined in +[column.schema.json](../../../../../udf-runner-cpp/v2/json_schema/column.schema.json) and is referenced by both column +metadata and import specifications. See the [call metadata example](examples/call_metadata.json). -In a column definition, `type` is the official Exasol type family constrained by the schema's `exasol_type` enum, while `type_name` is the complete SQL -declaration. Parsed parameters such as `precision` and `scale` are included where applicable. The corresponding -Arrow physical storage type and field metadata are defined by the [high-level type mapping](type_mapping.md). - The existing `size`, `precision`, and `scale` properties remain part of the column API. The following definitions show how the additional mapped types expose their parameters without requiring the consumer to parse `type_name`: See the [column definitions example](examples/column_definitions.json). -Unsigned 64-bit values are decimal strings so JSON implementations do not lose precision. +Unsigned 64-bit values in JSON payload bodies are decimal strings so JSON implementations do not lose precision. ## Function Calls `Function` is a family of DB-opened, non-streaming calls. Each operation uses its operation name as -`OpenCall.call_name`, carries `call_metadata`, and has the following operation-specific payload contract. +`OpenCall.call_name`, uses the applicable `call_metadata` and `column_metadata`, and has the following operation-specific payload contract. | Call name | Request payload | Result payload | | --- | --- | --- | @@ -67,7 +73,7 @@ See the [export specification example](examples/export_specification.json). ## Nested Calls -`UDFRunner` may open these nested calls while a `Run` or Function call is active: +`UDFRunner` may open these nested calls only while a `Run` or Function call is active: | Call name | Request payload | Result payload | | --- | --- | --- | @@ -80,5 +86,5 @@ See the [connection information example](examples/connection_information.json). ## Deferred Calls -`Script` and `execute_query` are not part of the current normative call set. They remain future extensions and +`ExecuteScript` and `execute_query` are not part of the current normative call set. They remain future extensions and have no payload or data-stream contract in this version. diff --git a/doc/design/v2/protocol/high_level/type_mapping.md b/doc/design/v2/protocol/high_level/type_mapping.md index 7636974..2412e60 100644 --- a/doc/design/v2/protocol/high_level/type_mapping.md +++ b/doc/design/v2/protocol/high_level/type_mapping.md @@ -10,6 +10,9 @@ The conversion is performed from the declared Exasol column type, never from val `nullable` flag is preserved independently of its physical type. A conversion that is not defined here fails schema conversion with a descriptive error. +In the field metadata examples, `nullable` is the Arrow field nullability flag: `true` permits null values and `false` +states that the field must not contain nulls. It is independent of the selected physical storage type. + The [mapping diagram](type_mapping.svg) summarizes the decision paths below. ## Official Exasol type mappings diff --git a/udf-runner-cpp/v2/json_schema/call_metadata.schema.json b/udf-runner-cpp/v2/json_schema/call_metadata.schema.json index 5c1a6d4..2dd4a13 100644 --- a/udf-runner-cpp/v2/json_schema/call_metadata.schema.json +++ b/udf-runner-cpp/v2/json_schema/call_metadata.schema.json @@ -5,9 +5,15 @@ "required": [ "database_name", "database_version", "session_id", "statement_id", "node_count", "node_id", "vm_id", "maximal_memory_limit", "script_schema", - "input_iter_type", "output_iter_type", "input_columns", "output_columns", + "input_iter_type", "output_iter_type", "single_call_mode" ], + "not": { + "anyOf": [ + { "required": ["input_columns"] }, + { "required": ["output_columns"] } + ] + }, "properties": { "database_name": { "type": "string" }, "database_version": { "type": "string" }, diff --git a/udf-runner-cpp/v2/json_schema/column_metadata.schema.json b/udf-runner-cpp/v2/json_schema/column_metadata.schema.json new file mode 100644 index 0000000..a036ec4 --- /dev/null +++ b/udf-runner-cpp/v2/json_schema/column_metadata.schema.json @@ -0,0 +1,11 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "UDF Protocol v2 column metadata", + "type": "object", + "additionalProperties": true, + "required": ["input_columns", "output_columns"], + "properties": { + "input_columns": { "type": "array", "items": { "$ref": "column.schema.json" } }, + "output_columns": { "type": "array", "items": { "$ref": "column.schema.json" } } + } +} diff --git a/udf-runner-cpp/v2/json_schema/validate_schemas.py b/udf-runner-cpp/v2/json_schema/validate_schemas.py index 6abe31e..7a5daa5 100644 --- a/udf-runner-cpp/v2/json_schema/validate_schemas.py +++ b/udf-runner-cpp/v2/json_schema/validate_schemas.py @@ -103,10 +103,12 @@ def year_month_width(year_precision: int) -> int: def validate_schemas() -> None: schemas = load_schemas() call_metadata = read_example("call_metadata.json") + column_metadata = read_example("column_metadata.json") column_definitions = read_example("column_definitions.json") import_specification = read_example("import_specification.json") validate(schemas, "call_metadata.schema.json", call_metadata) + validate(schemas, "column_metadata.schema.json", column_metadata) validate( schemas, "import_specification.schema.json", @@ -117,15 +119,24 @@ def validate_schemas() -> None: validate(schemas, "connection_information.schema.json", read_example("connection_information.json")) validate_mapping_examples() - invalid = copy.deepcopy(call_metadata) + invalid = copy.deepcopy(column_metadata) invalid["input_columns"][0]["type"] = "STRING" try: - validate(schemas, "call_metadata.schema.json", invalid) + validate(schemas, "column_metadata.schema.json", invalid) except ValidationError: pass else: raise AssertionError("legacy STRING type was accepted") + invalid_call_metadata = copy.deepcopy(call_metadata) + invalid_call_metadata["input_columns"] = column_metadata["input_columns"] + try: + validate(schemas, "call_metadata.schema.json", invalid_call_metadata) + except ValidationError: + pass + else: + raise AssertionError("column metadata was accepted in call metadata") + print("v2 JSON schema validation passed") diff --git a/udf-runner-cpp/v2/udf_protocol.fbs b/udf-runner-cpp/v2/udf_protocol.fbs index 1a0f2dd..6f87c30 100644 --- a/udf-runner-cpp/v2/udf_protocol.fbs +++ b/udf-runner-cpp/v2/udf_protocol.fbs @@ -140,10 +140,7 @@ table OpenCall { // One named payload item sent by either side either on an active call's // connection-scoped stream_id or on stream_id = 0 without any active call. `name` identifies the // payload schema/meaning and therefore which parser to use for `payload` on the -// receiving side. Examples: -// - "virtual_schema_request" -> JSON with the virtual-schema request schema -// - "call_metadata" -> JSON with the call-metadata schema -// - "kill" -> name only, no payload body required +// receiving side. table Payload { name: string; payload: PayloadValue; From 8ee11b90d1cd33a056188d243788e7255ed822a8 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Tue, 1 Sep 2026 18:17:40 +0200 Subject: [PATCH 21/26] Add Arrow schema types and variadic buffers --- udf-runner-cpp/v2/udf_protocol.fbs | 41 +++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/udf-runner-cpp/v2/udf_protocol.fbs b/udf-runner-cpp/v2/udf_protocol.fbs index 6f87c30..2fab908 100644 --- a/udf-runner-cpp/v2/udf_protocol.fbs +++ b/udf-runner-cpp/v2/udf_protocol.fbs @@ -20,14 +20,24 @@ enum Precision : uint8 { Half = 0, Single = 1, Double = 2 } enum DateUnit : uint8 { Day = 0, Millisecond = 1 } enum TimeUnit : uint8 { Second = 0, Millisecond = 1, Microsecond = 2, Nanosecond = 3 } enum IntervalUnit : uint8 { YearMonth = 0, DayTime = 1, MonthDayNano = 2 } +enum UnionMode : uint8 { Sparse = 0, Dense = 1 } +table Null {} table Int { bit_width: int32; is_signed: bool; } table FloatingPoint { precision: Precision; } -table Utf8 {} table Binary {} +table Utf8 {} +table LargeBinary {} +table LargeUtf8 {} +table BinaryView {} +table Utf8View {} table FixedSizeBinary { byte_width: int32; } table Bool {} table Date { unit: DateUnit; } +table Time { + unit: TimeUnit = Millisecond; + bit_width: int32 = 32; +} table Timestamp { unit: TimeUnit; timezone: string; } // Arrow calendar interval storage: // YearMonth: signed int32 total months. @@ -36,6 +46,16 @@ table Timestamp { unit: TimeUnit; timezone: string; } table Interval { unit: IntervalUnit; } table Decimal { precision: int32; scale: int32; bit_width: int32 = 128; } table List {} +table LargeList {} +table ListView {} +table LargeListView {} +table FixedSizeList { list_size: int32; } +table Map { keys_sorted: bool; } +table Union { + mode: UnionMode; + type_ids: [int32]; +} +table Duration { unit: TimeUnit = Millisecond; } // Trailing underscore mirrors Arrow's own Schema.fbs naming for this union member. table Struct_ {} // Run-End Encoded (REE): a compact encoding for runs of values (Arrow's own @@ -48,8 +68,9 @@ table Struct_ {} table RunEndEncoded {} union Type { - Int, FloatingPoint, Utf8, Binary, FixedSizeBinary, Bool, Date, Timestamp, Interval, Decimal, List, Struct_, - RunEndEncoded, + Null, Int, FloatingPoint, Binary, Utf8, Bool, Decimal, Date, Time, Timestamp, Interval, List, Struct_, Union, + FixedSizeBinary, FixedSizeList, Map, Duration, LargeBinary, LargeUtf8, LargeList, RunEndEncoded, BinaryView, + Utf8View, ListView, LargeListView, } // Mirrors Arrow's own Schema.fbs KeyValue: a single string/string metadata @@ -63,7 +84,8 @@ table Field { name: string (required); nullable: bool; type: Type (required); - children: [Field]; // List's single element type, Struct_'s member fields, + children: [Field]; // List/view/fixed-size-list's element type, Struct_'s + // member fields, Map's entries field, Union's members, // or RunEndEncoded's [run_ends, values] pair // Extension-type annotation, mirroring Arrow's own convention: a field // representing an extension type is serialized using its storage type for @@ -91,6 +113,11 @@ table RecordBatchMetadata { length: int64; nodes: [FieldNode]; buffers: [Buffer]; + // For each variable-buffer field in the flattened schema, in preorder, + // records the number of variable buffers belonging to that field. + // Empty when the schema has no variable-buffer fields (for example, no + // Utf8View or BinaryView fields). + variadic_buffer_counts: [int64]; } table Version { @@ -195,10 +222,12 @@ table DataSchema { // Precedes the buffers for one RecordBatch in a direction of a call's data // stream. metadata is our own native RecordBatchMetadata (row count, per- -// column FieldNode, per-buffer Buffer), populated from a real Arrow +// flattened FieldNode, per-buffer Buffer), populated from a real Arrow // RecordBatch's public introspection API (Array::length(), null_count(), // ArrayData::buffers) -- not Arrow's own flatbuffer RecordBatch message. The -// actual column buffers are transported separately per buffer_transport. +// actual column buffers are transported separately per buffer_transport; view +// fields additionally use metadata.variadic_buffer_counts to delimit their +// variable buffer portions. table DataRecordBatch { buffer_transport: BufferTransport = Inline; is_end_of_group: bool = false; From ce2ddfaa3e59ea96434135e4ca3ace66e7e3049b Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Thu, 10 Sep 2026 16:34:22 +0200 Subject: [PATCH 22/26] Extract shared column schema definition --- .../json_schema/column_metadata.schema.json | 27 ++----------------- .../import_specification.schema.json | 23 +--------------- .../v2/json_schema_validation_test.cc | 23 ++++++++++++++-- 3 files changed, 24 insertions(+), 49 deletions(-) diff --git a/udf-runner-cpp/v2/json_schema/column_metadata.schema.json b/udf-runner-cpp/v2/json_schema/column_metadata.schema.json index fabe72a..a036ec4 100644 --- a/udf-runner-cpp/v2/json_schema/column_metadata.schema.json +++ b/udf-runner-cpp/v2/json_schema/column_metadata.schema.json @@ -5,30 +5,7 @@ "additionalProperties": true, "required": ["input_columns", "output_columns"], "properties": { - "input_columns": { "type": "array", "items": { "$ref": "#/definitions/column" } }, - "output_columns": { "type": "array", "items": { "$ref": "#/definitions/column" } } - }, - "definitions": { - "column": { - "type": "object", - "additionalProperties": true, - "required": ["name", "type", "type_name"], - "properties": { - "name": { "type": "string" }, - "type": { "$ref": "#/definitions/exasol_type" }, - "type_name": { "type": "string" }, - "size": { "type": "integer", "minimum": 0, "maximum": 4294967295 }, - "size_unit": { "enum": ["BYTE", "BIT"] }, - "precision": { "type": "integer", "minimum": 0, "maximum": 4294967295 }, - "scale": { "type": "integer", "minimum": 0, "maximum": 4294967295 }, - "fractional_second_precision": { "type": "integer", "minimum": 0, "maximum": 9 }, - "character_set": { "enum": ["ASCII", "UTF8"] }, - "srid": { "type": "integer", "minimum": 0, "maximum": 4294967295 } - } - }, - "exasol_type": { - "type": "string", - "enum": ["DOUBLE PRECISION", "DECIMAL", "DATE", "TIMESTAMP", "TIMESTAMP WITH LOCAL TIME ZONE", "CHAR", "VARCHAR", "BOOLEAN", "HASHTYPE", "GEOMETRY", "INTERVAL YEAR TO MONTH", "INTERVAL DAY TO SECOND"] - } + "input_columns": { "type": "array", "items": { "$ref": "column.schema.json" } }, + "output_columns": { "type": "array", "items": { "$ref": "column.schema.json" } } } } diff --git a/udf-runner-cpp/v2/json_schema/import_specification.schema.json b/udf-runner-cpp/v2/json_schema/import_specification.schema.json index dfb8ead..53d49b8 100644 --- a/udf-runner-cpp/v2/json_schema/import_specification.schema.json +++ b/udf-runner-cpp/v2/json_schema/import_specification.schema.json @@ -7,31 +7,10 @@ "is_subselect": { "type": "boolean" }, "connection_information": { "$ref": "connection_information.schema.json" }, "connection_name": { "type": "string" }, - "subselect_column_specification": { "type": "array", "items": { "$ref": "#/definitions/column" } }, + "subselect_column_specification": { "type": "array", "items": { "$ref": "column.schema.json" } }, "parameters": { "type": "array", "items": { "$ref": "#/definitions/key_value" } } }, "definitions": { - "column": { - "type": "object", - "additionalProperties": true, - "required": ["name", "type", "type_name"], - "properties": { - "name": { "type": "string" }, - "type": { "$ref": "#/definitions/exasol_type" }, - "type_name": { "type": "string" }, - "size": { "type": "integer", "minimum": 0, "maximum": 4294967295 }, - "size_unit": { "enum": ["BYTE", "BIT"] }, - "precision": { "type": "integer", "minimum": 0, "maximum": 4294967295 }, - "scale": { "type": "integer", "minimum": 0, "maximum": 4294967295 }, - "fractional_second_precision": { "type": "integer", "minimum": 0, "maximum": 9 }, - "character_set": { "enum": ["ASCII", "UTF8"] }, - "srid": { "type": "integer", "minimum": 0, "maximum": 4294967295 } - } - }, - "exasol_type": { - "type": "string", - "enum": ["DOUBLE PRECISION", "DECIMAL", "DATE", "TIMESTAMP", "TIMESTAMP WITH LOCAL TIME ZONE", "CHAR", "VARCHAR", "BOOLEAN", "HASHTYPE", "GEOMETRY", "INTERVAL YEAR TO MONTH", "INTERVAL DAY TO SECOND"] - }, "key_value": { "type": "object", "additionalProperties": true, diff --git a/udf-runner-cpp/v2/json_schema_validation_test.cc b/udf-runner-cpp/v2/json_schema_validation_test.cc index d79509a..e7d8f15 100644 --- a/udf-runner-cpp/v2/json_schema_validation_test.cc +++ b/udf-runner-cpp/v2/json_schema_validation_test.cc @@ -21,8 +21,15 @@ isolated_nlohmann::json read_json(const std::string& path) { int main() { const auto import_schema = read_json("json_schema/import_specification.schema.json"); isolated_nlohmann::json_schema::json_validator validator( - [](const isolated_nlohmann::json_uri&, isolated_nlohmann::json& schema) { - schema = read_json("json_schema/connection_information.schema.json"); + [](const isolated_nlohmann::json_uri& uri, isolated_nlohmann::json& schema) { + const auto path = uri.path(); + const auto filename = path.substr(path.find_last_of('/') + 1); + if (filename == "connection_information.schema.json" || + filename == "column.schema.json") { + schema = read_json("json_schema/" + filename); + return; + } + throw std::runtime_error("unsupported schema reference: " + uri.url()); }); validator.set_root_schema(import_schema); @@ -37,6 +44,18 @@ int main() { }; validator.validate(valid); + const isolated_nlohmann::json valid_with_columns = { + {"is_subselect", true}, + {"subselect_column_specification", {{{ + {"name", "ID"}, + {"type", "DECIMAL"}, + {"type_name", "DECIMAL(18,0)"}, + {"precision", 18}, + {"scale", 0}, + }}}}, + }; + validator.validate(valid_with_columns); + bool rejected = false; try { validator.validate(isolated_nlohmann::json::object()); From b7931578573803faca7e95d5183f264fc2ceb2d7 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Thu, 10 Sep 2026 16:49:18 +0200 Subject: [PATCH 23/26] Make extracted column schema compatible with validator --- .../v2/json_schema/column.schema.json | 8 +------ .../v2/json_schema_validation_test.cc | 23 ++----------------- 2 files changed, 3 insertions(+), 28 deletions(-) diff --git a/udf-runner-cpp/v2/json_schema/column.schema.json b/udf-runner-cpp/v2/json_schema/column.schema.json index 6f883b8..f7a7749 100644 --- a/udf-runner-cpp/v2/json_schema/column.schema.json +++ b/udf-runner-cpp/v2/json_schema/column.schema.json @@ -6,7 +6,7 @@ "required": ["name", "type", "type_name"], "properties": { "name": { "type": "string" }, - "type": { "$ref": "#/definitions/exasol_type" }, + "type": { "type": "string", "enum": ["DOUBLE PRECISION", "DECIMAL", "DATE", "TIMESTAMP", "TIMESTAMP WITH LOCAL TIME ZONE", "CHAR", "VARCHAR", "BOOLEAN", "HASHTYPE", "GEOMETRY", "INTERVAL YEAR TO MONTH", "INTERVAL DAY TO SECOND"] }, "type_name": { "type": "string", "description": "The complete Exasol SQL type declaration, including parameters where applicable." }, "size": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "Existing size property: character length for CHAR/VARCHAR or declared hash size for HASHTYPE." }, "size_unit": { "enum": ["BYTE", "BIT"], "description": "HASHTYPE size unit; required for hash declarations." }, @@ -15,11 +15,5 @@ "fractional_second_precision": { "type": "integer", "minimum": 0, "maximum": 9, "description": "INTERVAL DAY TO SECOND fractional-second precision." }, "character_set": { "enum": ["ASCII", "UTF8"], "description": "CHAR/VARCHAR character set." }, "srid": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "GEOMETRY spatial reference identifier; zero means no CRS." } - }, - "definitions": { - "exasol_type": { - "type": "string", - "enum": ["DOUBLE PRECISION", "DECIMAL", "DATE", "TIMESTAMP", "TIMESTAMP WITH LOCAL TIME ZONE", "CHAR", "VARCHAR", "BOOLEAN", "HASHTYPE", "GEOMETRY", "INTERVAL YEAR TO MONTH", "INTERVAL DAY TO SECOND"] - } } } diff --git a/udf-runner-cpp/v2/json_schema_validation_test.cc b/udf-runner-cpp/v2/json_schema_validation_test.cc index e7d8f15..d79509a 100644 --- a/udf-runner-cpp/v2/json_schema_validation_test.cc +++ b/udf-runner-cpp/v2/json_schema_validation_test.cc @@ -21,15 +21,8 @@ isolated_nlohmann::json read_json(const std::string& path) { int main() { const auto import_schema = read_json("json_schema/import_specification.schema.json"); isolated_nlohmann::json_schema::json_validator validator( - [](const isolated_nlohmann::json_uri& uri, isolated_nlohmann::json& schema) { - const auto path = uri.path(); - const auto filename = path.substr(path.find_last_of('/') + 1); - if (filename == "connection_information.schema.json" || - filename == "column.schema.json") { - schema = read_json("json_schema/" + filename); - return; - } - throw std::runtime_error("unsupported schema reference: " + uri.url()); + [](const isolated_nlohmann::json_uri&, isolated_nlohmann::json& schema) { + schema = read_json("json_schema/connection_information.schema.json"); }); validator.set_root_schema(import_schema); @@ -44,18 +37,6 @@ int main() { }; validator.validate(valid); - const isolated_nlohmann::json valid_with_columns = { - {"is_subselect", true}, - {"subselect_column_specification", {{{ - {"name", "ID"}, - {"type", "DECIMAL"}, - {"type_name", "DECIMAL(18,0)"}, - {"precision", 18}, - {"scale", 0}, - }}}}, - }; - validator.validate(valid_with_columns); - bool rejected = false; try { validator.validate(isolated_nlohmann::json::object()); From c7b68c71b4fe623f19bd8ef18eb4a5af7f755fec Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Thu, 10 Sep 2026 17:40:34 +0200 Subject: [PATCH 24/26] Add schema loader diagnostics --- .../v2/json_schema_validation_test.cc | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/udf-runner-cpp/v2/json_schema_validation_test.cc b/udf-runner-cpp/v2/json_schema_validation_test.cc index d79509a..e6810b3 100644 --- a/udf-runner-cpp/v2/json_schema_validation_test.cc +++ b/udf-runner-cpp/v2/json_schema_validation_test.cc @@ -1,6 +1,7 @@ #include #include #include +#include #include #include @@ -21,8 +22,25 @@ isolated_nlohmann::json read_json(const std::string& path) { int main() { const auto import_schema = read_json("json_schema/import_specification.schema.json"); isolated_nlohmann::json_schema::json_validator validator( - [](const isolated_nlohmann::json_uri&, isolated_nlohmann::json& schema) { + [](const isolated_nlohmann::json_uri& uri, isolated_nlohmann::json& schema) { + std::cerr << "schema loader request: url=" << uri.url() + << ", location=" << uri.location() + << ", path=" << uri.path() + << ", fragment=" << uri.fragment() << '\n'; + schema = read_json("json_schema/connection_information.schema.json"); + + std::cerr << "schema loader response: source=json_schema/connection_information.schema.json" + << ", type=" << schema.type_name() << ", keys=["; + bool first = true; + for (const auto& item : schema.items()) { + if (!first) { + std::cerr << ','; + } + std::cerr << item.key(); + first = false; + } + std::cerr << "]\n"; }); validator.set_root_schema(import_schema); From 2c0393e05b5a351eec547321b03a98c6f723841d Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Thu, 10 Sep 2026 17:43:50 +0200 Subject: [PATCH 25/26] Fix external schema loader dispatch --- .../v2/json_schema_validation_test.cc | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/udf-runner-cpp/v2/json_schema_validation_test.cc b/udf-runner-cpp/v2/json_schema_validation_test.cc index e6810b3..f1b2878 100644 --- a/udf-runner-cpp/v2/json_schema_validation_test.cc +++ b/udf-runner-cpp/v2/json_schema_validation_test.cc @@ -21,16 +21,23 @@ isolated_nlohmann::json read_json(const std::string& path) { int main() { const auto import_schema = read_json("json_schema/import_specification.schema.json"); - isolated_nlohmann::json_schema::json_validator validator( - [](const isolated_nlohmann::json_uri& uri, isolated_nlohmann::json& schema) { + const auto load_schema = [](const isolated_nlohmann::json_uri& uri, + isolated_nlohmann::json& schema) { std::cerr << "schema loader request: url=" << uri.url() << ", location=" << uri.location() << ", path=" << uri.path() << ", fragment=" << uri.fragment() << '\n'; - schema = read_json("json_schema/connection_information.schema.json"); + const auto path = uri.path(); + const auto filename = path.substr(path.find_last_of('/') + 1); + if (filename != "connection_information.schema.json" && + filename != "column.schema.json") { + throw std::runtime_error("unsupported schema reference: " + uri.url()); + } + const auto source = "json_schema/" + filename; + schema = read_json(source); - std::cerr << "schema loader response: source=json_schema/connection_information.schema.json" + std::cerr << "schema loader response: source=" << source << ", type=" << schema.type_name() << ", keys=["; bool first = true; for (const auto& item : schema.items()) { @@ -41,7 +48,9 @@ int main() { first = false; } std::cerr << "]\n"; - }); + }; + + isolated_nlohmann::json_schema::json_validator validator(load_schema); validator.set_root_schema(import_schema); const isolated_nlohmann::json valid = { From e272597a1349c0959bc6333d6288bcdde1658f28 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Fri, 11 Sep 2026 11:13:41 +0200 Subject: [PATCH 26/26] Inline column schemas and test supported types --- .../v2/json_schema/column.schema.json | 19 ---------- .../json_schema/column_metadata.schema.json | 36 +++++++++++++++++- .../import_specification.schema.json | 18 ++++++++- .../v2/json_schema_validation_test.cc | 37 +++++++++++++++++-- 4 files changed, 85 insertions(+), 25 deletions(-) delete mode 100644 udf-runner-cpp/v2/json_schema/column.schema.json diff --git a/udf-runner-cpp/v2/json_schema/column.schema.json b/udf-runner-cpp/v2/json_schema/column.schema.json deleted file mode 100644 index f7a7749..0000000 --- a/udf-runner-cpp/v2/json_schema/column.schema.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "UDF Protocol v2 column definition", - "type": "object", - "additionalProperties": true, - "required": ["name", "type", "type_name"], - "properties": { - "name": { "type": "string" }, - "type": { "type": "string", "enum": ["DOUBLE PRECISION", "DECIMAL", "DATE", "TIMESTAMP", "TIMESTAMP WITH LOCAL TIME ZONE", "CHAR", "VARCHAR", "BOOLEAN", "HASHTYPE", "GEOMETRY", "INTERVAL YEAR TO MONTH", "INTERVAL DAY TO SECOND"] }, - "type_name": { "type": "string", "description": "The complete Exasol SQL type declaration, including parameters where applicable." }, - "size": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "Existing size property: character length for CHAR/VARCHAR or declared hash size for HASHTYPE." }, - "size_unit": { "enum": ["BYTE", "BIT"], "description": "HASHTYPE size unit; required for hash declarations." }, - "precision": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "DECIMAL or TIMESTAMP precision, or interval leading-field precision." }, - "scale": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "DECIMAL scale only." }, - "fractional_second_precision": { "type": "integer", "minimum": 0, "maximum": 9, "description": "INTERVAL DAY TO SECOND fractional-second precision." }, - "character_set": { "enum": ["ASCII", "UTF8"], "description": "CHAR/VARCHAR character set." }, - "srid": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "GEOMETRY spatial reference identifier; zero means no CRS." } - } -} diff --git a/udf-runner-cpp/v2/json_schema/column_metadata.schema.json b/udf-runner-cpp/v2/json_schema/column_metadata.schema.json index a036ec4..6347f84 100644 --- a/udf-runner-cpp/v2/json_schema/column_metadata.schema.json +++ b/udf-runner-cpp/v2/json_schema/column_metadata.schema.json @@ -5,7 +5,39 @@ "additionalProperties": true, "required": ["input_columns", "output_columns"], "properties": { - "input_columns": { "type": "array", "items": { "$ref": "column.schema.json" } }, - "output_columns": { "type": "array", "items": { "$ref": "column.schema.json" } } + "input_columns": { "type": "array", "items": { + "type": "object", + "additionalProperties": true, + "required": ["name", "type", "type_name"], + "properties": { + "name": { "type": "string" }, + "type": { "type": "string", "enum": ["DOUBLE PRECISION", "DECIMAL", "DATE", "TIMESTAMP", "TIMESTAMP WITH LOCAL TIME ZONE", "CHAR", "VARCHAR", "BOOLEAN", "HASHTYPE", "GEOMETRY", "INTERVAL YEAR TO MONTH", "INTERVAL DAY TO SECOND"] }, + "type_name": { "type": "string", "description": "The complete Exasol SQL type declaration, including parameters where applicable." }, + "size": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "Existing size property: character length for CHAR/VARCHAR or declared hash size for HASHTYPE." }, + "size_unit": { "enum": ["BYTE", "BIT"], "description": "HASHTYPE size unit; required for hash declarations." }, + "precision": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "DECIMAL or TIMESTAMP precision, or interval leading-field precision." }, + "scale": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "DECIMAL scale only." }, + "fractional_second_precision": { "type": "integer", "minimum": 0, "maximum": 9, "description": "INTERVAL DAY TO SECOND fractional-second precision." }, + "character_set": { "enum": ["ASCII", "UTF8"], "description": "CHAR/VARCHAR character set." }, + "srid": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "GEOMETRY spatial reference identifier; zero means no CRS." } + } + } }, + "output_columns": { "type": "array", "items": { + "type": "object", + "additionalProperties": true, + "required": ["name", "type", "type_name"], + "properties": { + "name": { "type": "string" }, + "type": { "type": "string", "enum": ["DOUBLE PRECISION", "DECIMAL", "DATE", "TIMESTAMP", "TIMESTAMP WITH LOCAL TIME ZONE", "CHAR", "VARCHAR", "BOOLEAN", "HASHTYPE", "GEOMETRY", "INTERVAL YEAR TO MONTH", "INTERVAL DAY TO SECOND"] }, + "type_name": { "type": "string", "description": "The complete Exasol SQL type declaration, including parameters where applicable." }, + "size": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "Existing size property: character length for CHAR/VARCHAR or declared hash size for HASHTYPE." }, + "size_unit": { "enum": ["BYTE", "BIT"], "description": "HASHTYPE size unit; required for hash declarations." }, + "precision": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "DECIMAL or TIMESTAMP precision, or interval leading-field precision." }, + "scale": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "DECIMAL scale only." }, + "fractional_second_precision": { "type": "integer", "minimum": 0, "maximum": 9, "description": "INTERVAL DAY TO SECOND fractional-second precision." }, + "character_set": { "enum": ["ASCII", "UTF8"], "description": "CHAR/VARCHAR character set." }, + "srid": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "GEOMETRY spatial reference identifier; zero means no CRS." } + } + } } } } diff --git a/udf-runner-cpp/v2/json_schema/import_specification.schema.json b/udf-runner-cpp/v2/json_schema/import_specification.schema.json index 53d49b8..c657997 100644 --- a/udf-runner-cpp/v2/json_schema/import_specification.schema.json +++ b/udf-runner-cpp/v2/json_schema/import_specification.schema.json @@ -7,7 +7,23 @@ "is_subselect": { "type": "boolean" }, "connection_information": { "$ref": "connection_information.schema.json" }, "connection_name": { "type": "string" }, - "subselect_column_specification": { "type": "array", "items": { "$ref": "column.schema.json" } }, + "subselect_column_specification": { "type": "array", "items": { + "type": "object", + "additionalProperties": true, + "required": ["name", "type", "type_name"], + "properties": { + "name": { "type": "string" }, + "type": { "type": "string", "enum": ["DOUBLE PRECISION", "DECIMAL", "DATE", "TIMESTAMP", "TIMESTAMP WITH LOCAL TIME ZONE", "CHAR", "VARCHAR", "BOOLEAN", "HASHTYPE", "GEOMETRY", "INTERVAL YEAR TO MONTH", "INTERVAL DAY TO SECOND"] }, + "type_name": { "type": "string", "description": "The complete Exasol SQL type declaration, including parameters where applicable." }, + "size": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "Existing size property: character length for CHAR/VARCHAR or declared hash size for HASHTYPE." }, + "size_unit": { "enum": ["BYTE", "BIT"], "description": "HASHTYPE size unit; required for hash declarations." }, + "precision": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "DECIMAL or TIMESTAMP precision, or interval leading-field precision." }, + "scale": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "DECIMAL scale only." }, + "fractional_second_precision": { "type": "integer", "minimum": 0, "maximum": 9, "description": "INTERVAL DAY TO SECOND fractional-second precision." }, + "character_set": { "enum": ["ASCII", "UTF8"], "description": "CHAR/VARCHAR character set." }, + "srid": { "type": "integer", "minimum": 0, "maximum": 4294967295, "description": "GEOMETRY spatial reference identifier; zero means no CRS." } + } + } }, "parameters": { "type": "array", "items": { "$ref": "#/definitions/key_value" } } }, "definitions": { diff --git a/udf-runner-cpp/v2/json_schema_validation_test.cc b/udf-runner-cpp/v2/json_schema_validation_test.cc index f1b2878..6937042 100644 --- a/udf-runner-cpp/v2/json_schema_validation_test.cc +++ b/udf-runner-cpp/v2/json_schema_validation_test.cc @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -20,6 +21,30 @@ isolated_nlohmann::json read_json(const std::string& path) { } // namespace int main() { + const std::vector column_types = { + "DOUBLE PRECISION", "DECIMAL", "DATE", "TIMESTAMP", + "TIMESTAMP WITH LOCAL TIME ZONE", "CHAR", "VARCHAR", "BOOLEAN", + "HASHTYPE", "GEOMETRY", "INTERVAL YEAR TO MONTH", + "INTERVAL DAY TO SECOND", + }; + const auto make_column = [](const std::string& type) { + return isolated_nlohmann::json{ + {"name", "COLUMN_" + type}, + {"type", type}, + {"type_name", type}, + }; + }; + + const auto column_metadata_schema = read_json("json_schema/column_metadata.schema.json"); + isolated_nlohmann::json_schema::json_validator column_metadata_validator; + column_metadata_validator.set_root_schema(column_metadata_schema); + for (const auto& type : column_types) { + column_metadata_validator.validate({ + {"input_columns", {make_column(type)}}, + {"output_columns", isolated_nlohmann::json::array()}, + }); + } + const auto import_schema = read_json("json_schema/import_specification.schema.json"); const auto load_schema = [](const isolated_nlohmann::json_uri& uri, isolated_nlohmann::json& schema) { @@ -30,8 +55,7 @@ int main() { const auto path = uri.path(); const auto filename = path.substr(path.find_last_of('/') + 1); - if (filename != "connection_information.schema.json" && - filename != "column.schema.json") { + if (filename != "connection_information.schema.json") { throw std::runtime_error("unsupported schema reference: " + uri.url()); } const auto source = "json_schema/" + filename; @@ -48,11 +72,18 @@ int main() { first = false; } std::cerr << "]\n"; - }; + }; isolated_nlohmann::json_schema::json_validator validator(load_schema); validator.set_root_schema(import_schema); + for (const auto& type : column_types) { + validator.validate({ + {"is_subselect", true}, + {"subselect_column_specification", {make_column(type)}}, + }); + } + const isolated_nlohmann::json valid = { {"is_subselect", true}, {"connection_information", {