From 2c2b2b8f0c19b7f6b3c532f50e5cc6bd3ae2144a Mon Sep 17 00:00:00 2001
From: Phil Elwell <phil@raspberrypi.com>
Date: Mon, 13 Jul 2026 15:50:22 +0100
Subject: [PATCH 1/8] misc: rpi-pio: In-kernel API fixes

Fix a few issues discovered after testing the kernel API with a port of
the userspace apitest app.

Signed-off-by: Phil Elwell <phil@raspberrypi.com>
---
 drivers/misc/rp1-pio.c  | 23 ++++++++++++++++++++---
 include/linux/pio_rp1.h |  4 ++--
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/drivers/misc/rp1-pio.c b/drivers/misc/rp1-pio.c
index c26447000a3bb..74f7a3611dc65 100644
--- a/drivers/misc/rp1-pio.c
+++ b/drivers/misc/rp1-pio.c
@@ -67,6 +67,9 @@ struct dma_xfer_state {
 	struct dma_info *dma;
 	void (*callback)(void *param);
 	void *callback_param;
+	struct dma_buf_info *dbi;
+	void *data;
+	size_t data_bytes;
 };
 
 struct dma_buf_info {
@@ -225,7 +228,7 @@ int rp1_pio_can_add_program(struct rp1_pio_client *client, void *param)
 	offset = rp1_pio_find_program(pio, args);
 	mutex_unlock(&pio->instr_mutex);
 	if (offset >= 0)
-		return offset;
+		return 1;
 
 	/* Don't send the instructions, just the header */
 	return rp1_pio_message(pio, PIO_CAN_ADD_PROGRAM, args,
@@ -584,6 +587,9 @@ static void rp1_pio_sm_kernel_dma_callback(void *param)
 {
 	struct dma_xfer_state *dxs = param;
 
+	if (dxs->dbi)
+		memcpy(dxs->data, dxs->dbi->buf, dxs->data_bytes);
+
 	dxs->dma->tail_idx++;
 	up(&dxs->dma->buf_sem);
 
@@ -949,19 +955,25 @@ int rp1_pio_sm_xfer_data(struct rp1_pio_client *client, uint sm, uint dir,
 
 	if (!dma_addr) {
 		dxs = kmalloc(sizeof(*dxs), GFP_KERNEL);
+		if (!dxs)
+			return -ENOMEM;
 		dxs->dma = dma;
 		dxs->callback = callback;
 		dxs->callback_param = param;
+		dxs->dbi = NULL;
 		callback = rp1_pio_sm_kernel_dma_callback;
 		param = dxs;
 
-		if (!dma->buf_count || data_bytes > dma->buf_size)
+		if (!dma->buf_count || data_bytes > dma->buf_size) {
+			kfree(dxs);
 			return -EINVAL;
+		}
 
 		/* Grab a dma buffer */
 		if (dma->head_idx - dma->tail_idx == dma->buf_count) {
 			if (down_timeout(&dma->buf_sem, msecs_to_jiffies(1000))) {
 				dev_err(dev, "DMA wait timed out\n");
+				kfree(dxs);
 				return -ETIMEDOUT;
 			}
 		}
@@ -969,8 +981,13 @@ int rp1_pio_sm_xfer_data(struct rp1_pio_client *client, uint sm, uint dir,
 		dbi = &dma->bufs[dma->head_idx % dma->buf_count];
 		dma_addr = dbi->dma_addr;
 
-		if (dir == PIO_DIR_TO_SM)
+		if (dir == PIO_DIR_TO_SM) {
 			memcpy(dbi->buf, data, data_bytes);
+		} else {
+			dxs->dbi = dbi;
+			dxs->data = data;
+			dxs->data_bytes = data_bytes;
+		}
 	}
 
 	sg_init_table(&sg, 1);
diff --git a/include/linux/pio_rp1.h b/include/linux/pio_rp1.h
index 68083b83a2013..93b21c511e995 100644
--- a/include/linux/pio_rp1.h
+++ b/include/linux/pio_rp1.h
@@ -300,7 +300,7 @@ static inline bool pio_can_add_program_at_offset(struct rp1_pio_client *client,
 	args.num_instrs = program->length;
 
 	memcpy(args.instrs, program->instructions, args.num_instrs * sizeof(args.instrs[0]));
-	return !rp1_pio_can_add_program(client, &args);
+	return rp1_pio_can_add_program(client, &args);
 }
 
 static inline uint pio_add_program(struct rp1_pio_client *client, const pio_program_t *program)
@@ -517,7 +517,7 @@ static inline int pio_sm_set_pindirs_with_mask(struct rp1_pio_client *client, ui
 
 	if (bad_params_if(client, sm >= NUM_PIO_STATE_MACHINES ||
 			      (pin_dirs & GPIOS_MASK) != pin_dirs ||
-			      (pin_mask & pin_mask) != pin_mask))
+			      (pin_mask & GPIOS_MASK) != pin_mask))
 		return -EINVAL;
 	return rp1_pio_sm_set_pindirs(client, &args);
 }

From 2fa7139e763a161447daae8614ec73aeffbccc33 Mon Sep 17 00:00:00 2001
From: Phil Elwell <phil@raspberrypi.com>
Date: Tue, 16 Jun 2026 20:39:26 +0100
Subject: [PATCH 2/8] misc: rp1-pio: Add sm_get_dmactrl and sm_get_flags

pio_sm_get_dmactrl is the reverse of pio_sm_set_dmactrl.
pio_sm_get_flags allows the FIFO error flags - txoverflow, rxunderflow,
txstall and rxstall - to be polled, cleared, and optionally waited for.

Signed-off-by: Phil Elwell <phil@raspberrypi.com>
---
 drivers/misc/rp1-fw-pio.h      | 13 ++++++++++-
 drivers/misc/rp1-pio.c         | 40 ++++++++++++++++++++++++++++++++++
 include/linux/pio_rp1.h        | 29 ++++++++++++++++++++++++
 include/uapi/misc/rp1_pio_if.h | 20 ++++++++++++++++-
 4 files changed, 100 insertions(+), 2 deletions(-)

diff --git a/drivers/misc/rp1-fw-pio.h b/drivers/misc/rp1-fw-pio.h
index ba28cba38f84c..89338e04511c0 100644
--- a/drivers/misc/rp1-fw-pio.h
+++ b/drivers/misc/rp1-fw-pio.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 */
 /*
- *  Copyright (C) 2023 2023-2024 Raspberry Pi Ltd.
+ *  Copyright (C) 2023-2026 Raspberry Pi Ltd.
  */
 
 #ifndef __SOC_RP1_FIRMWARE_OPS_H__
@@ -49,8 +49,19 @@ enum rp1_pio_ops {
 
 	PIO_SM_FIFO_STATE,	// u16 sm, u8 tx -> u16 level, u8 empty, u8 full
 	PIO_SM_DRAIN_TX,	// u16 sm
+	PIO_SM_GET_FLAGS,	// u16 sm, u8 clear, u8 rsvd, u32 flags, u32 wait -> u32 level
+	PIO_SM_GET_DMACTRL,	// u16 sm, u16 is_tx -> u32 ctrl
 
 	PIO_COUNT
 };
 
+// Don't include this here to avoid multiple definitions
+//
+// enum pio_sm_flags {
+//     PIO_SM_FLAG_TXSTALL = 0x0001,
+//     PIO_SM_FLAG_RXSTALL = 0x0002,
+//     PIO_SM_FLAG_TXOVER  = 0x0004,
+//     PIO_SM_FLAG_RXUNDER = 0x0008,
+// };
+
 #endif
diff --git a/drivers/misc/rp1-pio.c b/drivers/misc/rp1-pio.c
index 74f7a3611dc65..fda1ce33d52c8 100644
--- a/drivers/misc/rp1-pio.c
+++ b/drivers/misc/rp1-pio.c
@@ -490,6 +490,30 @@ int rp1_pio_sm_set_dmactrl(struct rp1_pio_client *client, void *param)
 }
 EXPORT_SYMBOL_GPL(rp1_pio_sm_set_dmactrl);
 
+int rp1_pio_sm_get_dmactrl(struct rp1_pio_client *client, void *param)
+{
+	struct rp1_pio_device *pio = client->pio;
+	struct rp1_pio_sm_get_dmactrl_args *args = param;
+	int ret;
+
+	if (pio->fw_pio_count >= PIO_SM_GET_DMACTRL) {
+		ret = rp1_pio_message_resp(pio, PIO_SM_GET_DMACTRL, args, sizeof(*args),
+					   &args->ctrl, NULL, 4);
+	} else {
+		struct rp1_access_hw_args hwargs;
+
+		hwargs.addr = 0xf00000cc + args->sm * 0x20 + (args->is_tx ? 0x18 : 0x1c);
+		hwargs.len = 0x4;
+		hwargs.data = &args->ctrl;
+		ret = rp1_pio_message_resp(pio, READ_HW, &hwargs, 8,
+					   &args->ctrl, NULL, 4);
+	}
+	if (ret >= 0)
+		return offsetof(struct rp1_pio_sm_set_dmactrl_args, ctrl) + ret;
+	return ret;
+}
+EXPORT_SYMBOL_GPL(rp1_pio_sm_get_dmactrl);
+
 int rp1_pio_sm_fifo_state(struct rp1_pio_client *client, void *param)
 {
 	struct rp1_pio_sm_fifo_state_args *args = param;
@@ -512,6 +536,20 @@ int rp1_pio_sm_drain_tx(struct rp1_pio_client *client, void *param)
 }
 EXPORT_SYMBOL_GPL(rp1_pio_sm_drain_tx);
 
+int rp1_pio_sm_get_flags(struct rp1_pio_client *client, void *param)
+{
+	struct rp1_pio_sm_get_flags_args *args = param;
+	int ret;
+
+	ret = rp1_pio_message_resp(client->pio, PIO_SM_GET_FLAGS, args, sizeof(*args),
+				   &args->flags, NULL, 4);
+	if (ret >= 0)
+		return offsetof(struct rp1_pio_sm_get_flags_args, flags) + ret;
+	args->flags = ~0;
+	return ret;
+}
+EXPORT_SYMBOL_GPL(rp1_pio_sm_get_flags);
+
 int rp1_pio_gpio_init(struct rp1_pio_client *client, void *param)
 {
 	struct rp1_gpio_init_args *args = param;
@@ -1054,6 +1092,8 @@ struct handler_info {
 	HANDLER(SM_SET_DMACTRL, sm_set_dmactrl),
 	HANDLER(SM_FIFO_STATE, sm_fifo_state),
 	HANDLER(SM_DRAIN_TX, sm_drain_tx),
+	HANDLER(SM_GET_FLAGS, sm_get_flags),
+	HANDLER(SM_GET_DMACTRL, sm_get_dmactrl),
 
 	HANDLER(GPIO_INIT, gpio_init),
 	HANDLER(GPIO_SET_FUNCTION, gpio_set_function),
diff --git a/include/linux/pio_rp1.h b/include/linux/pio_rp1.h
index 93b21c511e995..bb9d398021c69 100644
--- a/include/linux/pio_rp1.h
+++ b/include/linux/pio_rp1.h
@@ -172,6 +172,13 @@ enum gpio_drive_strength {
 	GPIO_DRIVE_STRENGTH_12MA = 3
 };
 
+enum pio_sm_flags {
+	PIO_SM_FLAG_TXSTALL = 0x0001,
+	PIO_SM_FLAG_RXSTALL = 0x0002,
+	PIO_SM_FLAG_TXOVER  = 0x0004,
+	PIO_SM_FLAG_RXUNDER = 0x0008,
+};
+
 struct fp24_8 {
 	uint32_t val;
 };
@@ -213,8 +220,10 @@ int rp1_pio_sm_enable_sync(struct rp1_pio_client *client, void *param);
 int rp1_pio_sm_put(struct rp1_pio_client *client, void *param);
 int rp1_pio_sm_get(struct rp1_pio_client *client, void *param);
 int rp1_pio_sm_set_dmactrl(struct rp1_pio_client *client, void *param);
+int rp1_pio_sm_get_dmactrl(struct rp1_pio_client *client, void *param);
 int rp1_pio_sm_fifo_state(struct rp1_pio_client *client, void *param);
 int rp1_pio_sm_drain_tx(struct rp1_pio_client *client, void *param);
+int rp1_pio_sm_get_flags(struct rp1_pio_client *client, void *param);
 int rp1_pio_gpio_init(struct rp1_pio_client *client, void *param);
 int rp1_pio_gpio_set_function(struct rp1_pio_client *client, void *param);
 int rp1_pio_gpio_set_pulls(struct rp1_pio_client *client, void *param);
@@ -611,6 +620,15 @@ static inline int pio_sm_set_dmactrl(struct rp1_pio_client *client, uint sm, boo
 	return rp1_pio_sm_set_dmactrl(client, &args);
 };
 
+static inline uint32_t pio_sm_get_dmactrl(struct rp1_pio_client *client, uint sm, bool is_tx)
+{
+	struct rp1_pio_sm_get_dmactrl_args args = { .sm = sm, .is_tx = is_tx };
+
+	if (!bad_params_if(client, sm >= NUM_PIO_STATE_MACHINES))
+		rp1_pio_sm_get_dmactrl(client, &args);
+	return args.ctrl;
+};
+
 static inline int pio_sm_drain_tx_fifo(struct rp1_pio_client *client, uint sm)
 {
 	struct rp1_pio_sm_clear_fifos_args args = { .sm = sm };
@@ -620,6 +638,17 @@ static inline int pio_sm_drain_tx_fifo(struct rp1_pio_client *client, uint sm)
 	return rp1_pio_sm_drain_tx(client, &args);
 };
 
+static inline uint32_t pio_sm_get_flags(struct rp1_pio_client *client, uint sm, uint32_t flags,
+					bool clear, uint32_t timeout)
+{
+	struct rp1_pio_sm_get_flags_args args = { .sm = sm, .flags = flags, .clear = clear,
+						  .timeout = timeout };
+
+	if (!bad_params_if(client, sm >= NUM_PIO_STATE_MACHINES))
+		rp1_pio_sm_get_flags(client, &args);
+	return args.flags;
+};
+
 static inline int pio_sm_put(struct rp1_pio_client *client, uint sm, uint32_t data)
 {
 	struct rp1_pio_sm_put_args args = { .sm = (uint16_t)sm, .blocking = false, .data = data };
diff --git a/include/uapi/misc/rp1_pio_if.h b/include/uapi/misc/rp1_pio_if.h
index a9a54d3322ec6..4b527a2cc57a3 100644
--- a/include/uapi/misc/rp1_pio_if.h
+++ b/include/uapi/misc/rp1_pio_if.h
@@ -124,6 +124,13 @@ struct rp1_pio_sm_set_dmactrl_args {
 	uint32_t ctrl;
 };
 
+struct rp1_pio_sm_get_dmactrl_args {
+	uint16_t sm;
+	uint8_t is_tx;
+	uint8_t rsvd;
+	uint32_t ctrl; /* OUT */
+};
+
 struct rp1_pio_sm_fifo_state_args {
 	uint16_t sm;
 	uint8_t tx;
@@ -133,6 +140,14 @@ struct rp1_pio_sm_fifo_state_args {
 	uint8_t full; /* OUT */
 };
 
+struct rp1_pio_sm_get_flags_args {
+	uint16_t sm;
+	uint8_t clear;
+	uint8_t rsvd;
+	uint32_t flags; /* IN/OUT */
+	uint32_t timeout;
+};
+
 struct rp1_gpio_init_args {
 	uint16_t gpio;
 };
@@ -171,6 +186,7 @@ struct rp1_pio_sm_xfer_data_args {
 	uint16_t sm;
 	uint16_t dir;
 	uint16_t data_bytes;
+	uint16_t rsvd;
 	void *data;
 };
 
@@ -220,8 +236,10 @@ struct rp1_access_hw_args {
 #define PIO_IOC_SM_PUT _IOW(PIO_IOC_MAGIC, 41, struct rp1_pio_sm_put_args)
 #define PIO_IOC_SM_GET _IOWR(PIO_IOC_MAGIC, 42, struct rp1_pio_sm_get_args)
 #define PIO_IOC_SM_SET_DMACTRL _IOW(PIO_IOC_MAGIC, 43, struct rp1_pio_sm_set_dmactrl_args)
-#define PIO_IOC_SM_FIFO_STATE _IOW(PIO_IOC_MAGIC, 44, struct rp1_pio_sm_fifo_state_args)
+#define PIO_IOC_SM_FIFO_STATE _IOWR(PIO_IOC_MAGIC, 44, struct rp1_pio_sm_fifo_state_args)
 #define PIO_IOC_SM_DRAIN_TX _IOW(PIO_IOC_MAGIC, 45, struct rp1_pio_sm_clear_fifos_args)
+#define PIO_IOC_SM_GET_FLAGS _IOWR(PIO_IOC_MAGIC, 46, struct rp1_pio_sm_get_flags_args)
+#define PIO_IOC_SM_GET_DMACTRL _IOWR(PIO_IOC_MAGIC, 47, struct rp1_pio_sm_get_dmactrl_args)
 
 #define PIO_IOC_GPIO_INIT _IOW(PIO_IOC_MAGIC, 50, struct rp1_gpio_init_args)
 #define PIO_IOC_GPIO_SET_FUNCTION _IOW(PIO_IOC_MAGIC, 51, struct rp1_gpio_set_function_args)

From 1836b750943b417699b3d0cd3375a6fffa3d4284 Mon Sep 17 00:00:00 2001
From: Phil Elwell <phil@raspberrypi.com>
Date: Thu, 2 Jul 2026 17:15:56 +0100
Subject: [PATCH 3/8] misc: rp1-pio: Add support for PIO interrupts

The PIO block exposes 2 assignable interrupts. Make them available to
kernel and userspace PIO applications.

Signed-off-by: Phil Elwell <phil@raspberrypi.com>
---
 drivers/misc/rp1-fw-pio.h      |   5 +
 drivers/misc/rp1-pio.c         | 516 +++++++++++++++++++++++++++------
 include/linux/pio_rp1.h        | 171 ++++++++++-
 include/uapi/misc/rp1_pio_if.h |  60 +++-
 4 files changed, 657 insertions(+), 95 deletions(-)

diff --git a/drivers/misc/rp1-fw-pio.h b/drivers/misc/rp1-fw-pio.h
index 89338e04511c0..0a674709de6cf 100644
--- a/drivers/misc/rp1-fw-pio.h
+++ b/drivers/misc/rp1-fw-pio.h
@@ -52,6 +52,11 @@ enum rp1_pio_ops {
 	PIO_SM_GET_FLAGS,	// u16 sm, u8 clear, u8 rsvd, u32 flags, u32 wait -> u32 level
 	PIO_SM_GET_DMACTRL,	// u16 sm, u16 is_tx -> u32 ctrl
 
+	PIO_SET_IRQN_SOURCE_MASK_ENABLED,	// u16 irq_index, u8 enabled, u8 rsvd,
+						// uint32_t source_mask
+	PIO_INTERRUPT_GET,	// u16 pio_interrupt_num -> u8 active
+	PIO_INTERRUPT_CLEAR,	// u16 pio_interrupt_num
+
 	PIO_COUNT
 };
 
diff --git a/drivers/misc/rp1-pio.c b/drivers/misc/rp1-pio.c
index fda1ce33d52c8..b901e1f592b82 100644
--- a/drivers/misc/rp1-pio.c
+++ b/drivers/misc/rp1-pio.c
@@ -2,7 +2,7 @@
 /*
  * PIO driver for RP1
  *
- * Copyright (C) 2023-2024 Raspberry Pi Ltd.
+ * Copyright (C) 2023-2026 Raspberry Pi Ltd.
  *
  * Parts of this driver are based on:
  *  - vcio.c, by Noralf Trønnes
@@ -20,9 +20,11 @@
 #include <linux/dma-mapping.h>
 #include <linux/fs.h>
 #include <linux/init.h>
+#include <linux/interrupt.h>
 #include <linux/ioctl.h>
 #include <linux/module.h>
 #include <linux/of.h>
+#include <linux/of_irq.h>
 #include <linux/pio_rp1.h>
 #include <linux/platform_device.h>
 #include <linux/rp1-firmware.h>
@@ -88,6 +90,16 @@ struct dma_info {
 	struct dma_buf_info bufs[DMA_BOUNCE_BUFFER_COUNT];
 };
 
+struct rp1_pio_client;
+
+struct irq_info {
+	int irq;
+	bool enabled;
+	struct rp1_pio_client *owner;
+	pio_irq_handler_t handler;
+	void *context;
+};
+
 struct rp1_pio_device {
 	struct platform_device *pdev;
 	struct rp1_firmware *fw;
@@ -97,6 +109,8 @@ struct rp1_pio_device {
 	struct class *dev_class;
 	struct cdev cdev;
 	phys_addr_t phys_addr;
+	uint irq_count;
+	struct irq_info irqs[RP1_PIO_IRQ_COUNT];
 	uint32_t claimed_sms;
 	uint32_t claimed_dmas;
 	spinlock_t lock;
@@ -110,10 +124,14 @@ struct rp1_pio_device {
 
 struct rp1_pio_client {
 	struct rp1_pio_device *pio;
+	spinlock_t lock;
+	struct completion completion;
+	uint32_t wake_reason;
 	uint32_t claimed_sms;
 	uint32_t claimed_instrs;
 	uint32_t claimed_dmas;
 	int error;
+	int irqs[RP1_PIO_IRQ_COUNT];
 };
 
 static struct rp1_pio_device *g_pio;
@@ -300,7 +318,7 @@ int rp1_pio_remove_program(struct rp1_pio_client *client, void *param)
 {
 	struct rp1_pio_remove_program_args *args = param;
 	uint32_t used_mask;
-	int ret = -ENOENT;
+	int ret = -ENODEV;
 
 	if (args->num_instrs > RP1_PIO_INSTR_COUNT ||
 		args->origin >= RP1_PIO_INSTR_COUNT ||
@@ -542,7 +560,7 @@ int rp1_pio_sm_get_flags(struct rp1_pio_client *client, void *param)
 	int ret;
 
 	ret = rp1_pio_message_resp(client->pio, PIO_SM_GET_FLAGS, args, sizeof(*args),
-				   &args->flags, NULL, 4);
+				   &args->flags, NULL, sizeof(args->flags));
 	if (ret >= 0)
 		return offsetof(struct rp1_pio_sm_get_flags_args, flags) + ret;
 	args->flags = ~0;
@@ -614,6 +632,240 @@ int rp1_pio_gpio_set_drive_strength(struct rp1_pio_client *client, void *param)
 }
 EXPORT_SYMBOL_GPL(rp1_pio_gpio_set_drive_strength);
 
+static void rp1_pio_irq_handler_userspace(void *context)
+{
+	struct irq_info *irqi = context;
+	struct rp1_pio_client *client = irqi->owner;
+	struct rp1_pio_device *pio = client->pio;
+	uint32_t intmask = 1 << (irqi - pio->irqs);
+	unsigned long flags;
+
+	spin_lock_irqsave(&client->lock, flags);
+	client->wake_reason |= intmask;
+	spin_unlock_irqrestore(&client->lock, flags);
+
+	complete(&client->completion);
+}
+
+int rp1_pio_irq_claim(struct rp1_pio_client *client, void *param)
+{
+	struct rp1_pio_irq_claim_args *args = param;
+	struct rp1_pio_device *pio = client->pio;
+	struct irq_info *irqi;
+	int ret = -1;
+	int i;
+
+	// Try to claim one of the PIO interrupts
+
+	args->irq_index = -ENODEV;
+
+	spin_lock(&pio->lock);
+	for (i = 0; i < pio->irq_count; i++) {
+		irqi = &pio->irqs[i];
+		if (!irqi->owner) {
+			irqi->owner = client;
+			irqi->handler = rp1_pio_irq_handler_userspace;
+			irqi->context = irqi;
+			args->irq_index = i;
+			ret = sizeof(*args);
+			break;
+		}
+		args->irq_index = -EBUSY;
+	}
+	spin_unlock(&pio->lock);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(rp1_pio_irq_claim);
+
+int rp1_pio_irq_wait(struct rp1_pio_client *client, void *param)
+{
+	struct rp1_pio_irq_wait_args *args = param;
+	unsigned long timeout;
+	long rem;
+
+	if (args->timeout_ms)
+		timeout = msecs_to_jiffies(args->timeout_ms);
+	else
+		timeout = MAX_SCHEDULE_TIMEOUT;
+	rem = wait_for_completion_interruptible_timeout(&client->completion, timeout);
+	if (rem < 0)
+		return (int)rem;
+
+	spin_lock(&client->lock);
+	args->active_mask = rem ? client->wake_reason : 0;
+	client->wake_reason = 0;
+	spin_unlock(&client->lock);
+	return sizeof(*args);
+}
+EXPORT_SYMBOL_GPL(rp1_pio_irq_wait);
+
+int rp1_pio_set_irqn_source_mask_enabled(struct rp1_pio_client *client, void *param)
+{
+	struct rp1_pio_set_irqn_source_mask_enabled_args *args = param;
+
+	return rp1_pio_message(client->pio, PIO_SET_IRQN_SOURCE_MASK_ENABLED,
+			       args, sizeof(*args));
+}
+EXPORT_SYMBOL_GPL(rp1_pio_set_irqn_source_mask_enabled);
+
+int rp1_pio_interrupt_get(struct rp1_pio_client *client, void *param)
+{
+	struct rp1_pio_interrupt_get_args *args = param;
+	int ret;
+
+	ret = rp1_pio_message_resp(client->pio, PIO_INTERRUPT_GET, args, sizeof(*args),
+				   &args->active, NULL, sizeof(args->active));
+	if (ret >= 0)
+		return offsetof(struct rp1_pio_interrupt_get_args, active) + ret;
+	return ret;
+}
+EXPORT_SYMBOL_GPL(rp1_pio_interrupt_get);
+
+int rp1_pio_interrupt_clear(struct rp1_pio_client *client, void *param)
+{
+	struct rp1_pio_interrupt_clear_args *args = param;
+
+	return rp1_pio_message(client->pio, PIO_INTERRUPT_CLEAR, args, sizeof(*args));
+}
+EXPORT_SYMBOL_GPL(rp1_pio_interrupt_clear);
+
+int rp1_pio_irq_set_enabled(struct rp1_pio_client *client, void *param)
+{
+	struct rp1_pio_irq_set_enabled_args *args = param;
+	struct rp1_pio_device *pio = client->pio;
+	uint irq_index = args->irq_index;
+	struct irq_info *irqi;
+
+	if (irq_index >= ARRAY_SIZE(pio->irqs))
+		return -EINVAL;
+	if (irq_index >= pio->irq_count)
+		return -EINVAL;
+	irqi = &pio->irqs[irq_index];
+	if (irqi->owner != client)
+		return -EBUSY;
+
+	spin_lock(&pio->lock);
+	if (args->enabled && !irqi->enabled) {
+		enable_irq(irqi->irq);
+		irqi->enabled = true;
+	} else if (!args->enabled && irqi->enabled) {
+		disable_irq(irqi->irq);
+		irqi->enabled = false;
+	}
+	spin_unlock(&pio->lock);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(rp1_pio_irq_set_enabled);
+
+int rp1_pio_irq_is_enabled(struct rp1_pio_client *client, void *param)
+{
+	struct rp1_pio_irq_set_enabled_args *args = param;
+	struct rp1_pio_device *pio = client->pio;
+	uint irq_index = args->irq_index;
+	struct irq_info *irqi;
+
+	if (irq_index >= ARRAY_SIZE(pio->irqs))
+		return -EINVAL;
+	if (irq_index >= pio->irq_count)
+		return -EINVAL;
+	irqi = &pio->irqs[irq_index];
+	if (irqi->owner != client)
+		return -EBUSY;
+
+	args->enabled = irqi->enabled;
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(rp1_pio_irq_is_enabled);
+
+void rp1_pio_irq_add_handler(struct rp1_pio_client *client, uint num,
+			     pio_irq_handler_t handler, void *context)
+{
+	struct rp1_pio_device *pio = client->pio;
+	struct irq_info *irqi;
+
+	if (num >= pio->irq_count) {
+		dev_err(&pio->pdev->dev, "%s: bad irq index %u\n", __func__, num);
+		return;
+	}
+
+	irqi = &pio->irqs[num];
+
+	spin_lock(&pio->lock);
+	if (irqi->owner && irqi->owner != client) {
+		spin_unlock(&pio->lock);
+		dev_err(&pio->pdev->dev, "%s: irq %u already claimed\n", __func__, num);
+		return;
+	}
+
+	irqi->owner = client;
+	irqi->handler = handler;
+	irqi->context = context;
+	spin_unlock(&pio->lock);
+}
+EXPORT_SYMBOL_GPL(rp1_pio_irq_add_handler);
+
+void rp1_pio_irq_remove_handler(struct rp1_pio_client *client,
+			     uint num, pio_irq_handler_t handler)
+{
+	struct rp1_pio_device *pio = client->pio;
+	struct irq_info *irqi;
+
+	if (num >= pio->irq_count)
+		return;
+
+	irqi = &pio->irqs[num];
+
+	spin_lock(&pio->lock);
+	if (irqi->owner == client && irqi->handler == handler) {
+		irqi->handler = NULL;
+		irqi->context = NULL;
+		irqi->owner = NULL;
+	}
+	spin_unlock(&pio->lock);
+}
+EXPORT_SYMBOL_GPL(rp1_pio_irq_remove_handler);
+
+pio_irq_handler_t rp1_pio_irq_get_handler(struct rp1_pio_client *client, uint num)
+{
+	struct rp1_pio_device *pio = client->pio;
+	struct irq_info *irqi;
+	pio_irq_handler_t handler;
+
+	if (num >= pio->irq_count)
+		return false;
+
+	irqi = &pio->irqs[num];
+
+	spin_lock(&pio->lock);
+	handler = irqi->handler;
+	spin_unlock(&pio->lock);
+
+	return handler;
+}
+EXPORT_SYMBOL_GPL(rp1_pio_irq_get_handler);
+
+uint rp1_pio_irq_map(struct rp1_pio_client *client, uint irqn)
+{
+	int pirq;
+
+	spin_lock(&client->lock);
+	pirq = client->irqs[irqn];
+	if (pirq < 0) {
+		struct rp1_pio_irq_claim_args args = { .irq_index = -1 };
+
+		rp1_pio_irq_claim(client, &args);
+		pirq = args.irq_index;
+		if (pirq >= 0)
+			client->irqs[irqn] = pirq;
+	}
+	spin_unlock(&client->lock);
+	return (uint)pirq;
+}
+EXPORT_SYMBOL_GPL(rp1_pio_irq_map);
+
 static void rp1_pio_sm_dma_callback(void *param)
 {
 	struct dma_info *dma = param;
@@ -645,8 +897,8 @@ static void rp1_pio_sm_dma_free(struct device *dev, struct dma_info *dma)
 				  dma->bufs[dma->buf_count].buf,
 				  dma->bufs[dma->buf_count].dma_addr);
 	}
-
 	dma_release_channel(dma->chan);
+	dma->chan = NULL;
 }
 
 static int rp1_pio_sm_config_xfer_internal(struct rp1_pio_client *client, uint sm, uint dir,
@@ -722,6 +974,9 @@ static int rp1_pio_sm_config_xfer_internal(struct rp1_pio_client *client, uint s
 		sg_dma_address(&dbi->sgl) = dbi->dma_addr;
 	}
 
+	dma->head_idx = 0;
+	dma->tail_idx = 0;
+
 	fifo_addr = pio->phys_addr;
 	fifo_addr += sm * (RP1_PIO_FIFO_TX1 - RP1_PIO_FIFO_TX0);
 	fifo_addr += (dir == RP1_PIO_DIR_TO_SM) ? RP1_PIO_FIFO_TX0 : RP1_PIO_FIFO_RX0;
@@ -794,19 +1049,14 @@ static int rp1_pio_sm_tx_user(struct rp1_pio_device *pio, struct dma_info *dma,
 	struct device *dev = &pdev->dev;
 	int ret = 0;
 
-	/* Clean the slate - we're running synchronously */
-	dma->head_idx = 0;
-	dma->tail_idx = 0;
-
 	while (bytes > 0) {
 		size_t copy_bytes = min(bytes, dma->buf_size);
 		struct dma_buf_info *dbi;
 
 		/* grab the next free buffer, waiting if they're all full */
 		if (dma->head_idx - dma->tail_idx == dma->buf_count) {
-			if (down_timeout(&dma->buf_sem,
-				msecs_to_jiffies(1000))) {
-				dev_err(dev, "DMA bounce timed out\n");
+			if (down_interruptible(&dma->buf_sem)) {
+				dev_err(dev, "DMA bounce interrupted\n");
 				break;
 			}
 			dma->tail_idx++;
@@ -847,84 +1097,97 @@ static int rp1_pio_sm_tx_user(struct rp1_pio_device *pio, struct dma_info *dma,
 		bytes -= copy_bytes;
 	}
 
-	/* Block for completion */
-	while (dma->tail_idx != dma->head_idx) {
-		if (down_timeout(&dma->buf_sem, msecs_to_jiffies(1000))) {
-			dev_err(dev, "DMA wait timed out\n");
-			ret = -ETIMEDOUT;
-			break;
-		}
-		dma->tail_idx++;
-	}
-
 	return ret;
 }
 
-static int rp1_pio_sm_rx_user(struct rp1_pio_device *pio, struct dma_info *dma,
-				  void __user *userbuf, size_t bytes)
+static int rp1_pio_sm_rx_submit(struct rp1_pio_device *pio, struct dma_info *dma, size_t len)
 {
 	struct platform_device *pdev = pio->pdev;
 	struct dma_async_tx_descriptor *desc;
 	struct device *dev = &pdev->dev;
-	int ret = 0;
+	struct dma_buf_info *dbi;
+	int ret;
 
-	/* Clean the slate - we're running synchronously */
-	dma->head_idx = 0;
-	dma->tail_idx = 0;
+	if (len > dma->buf_size)
+		return -EINVAL;
 
-	while (bytes || dma->tail_idx != dma->head_idx) {
-		size_t copy_bytes = min(bytes, dma->buf_size);
-		struct dma_buf_info *dbi;
+	dbi = &dma->bufs[dma->head_idx % dma->buf_count];
+	sg_dma_len(&dbi->sgl) = len;
+	desc = dmaengine_prep_slave_sg(dma->chan, &dbi->sgl, 1,
+				       DMA_DEV_TO_MEM,
+				       DMA_PREP_INTERRUPT | DMA_CTRL_ACK |
+				       DMA_PREP_FENCE);
+	if (!desc) {
+		dev_err(dev, "DMA preparation failed\n");
+		return -EIO;
+	}
 
-		/*
-		 * wait for the next RX to complete if all the buffers are
-		 * outstanding or we're finishing up.
-		 */
-		if (!bytes || dma->head_idx - dma->tail_idx == dma->buf_count) {
-			if (down_timeout(&dma->buf_sem,
-				msecs_to_jiffies(1000))) {
-				dev_err(dev, "DMA wait timed out\n");
-				ret = -ETIMEDOUT;
-				break;
-			}
+	desc->callback = rp1_pio_sm_dma_callback;
+	desc->callback_param = dma;
 
-			dbi = &dma->bufs[dma->tail_idx++ % dma->buf_count];
-			ret = copy_to_user(userbuf, dbi->buf, sg_dma_len(&dbi->sgl));
-			if (ret < 0)
-				break;
-			userbuf += sg_dma_len(&dbi->sgl);
+	/* Submit the buffer - the callback will kick the semaphore */
+	ret = dmaengine_submit(desc);
+	if (ret < 0)
+		return ret;
 
-			if (!bytes)
-				continue;
-		}
+	dma->head_idx++;
+	dma_async_issue_pending(dma->chan);
 
-		dbi = &dma->bufs[dma->head_idx % dma->buf_count];
-		sg_dma_len(&dbi->sgl) = copy_bytes;
-		desc = dmaengine_prep_slave_sg(dma->chan, &dbi->sgl, 1,
-					       DMA_DEV_TO_MEM,
-					       DMA_PREP_INTERRUPT | DMA_CTRL_ACK |
-					       DMA_PREP_FENCE);
-		if (!desc) {
-			dev_err(dev, "DMA preparation failed\n");
-			ret = -EIO;
-			break;
-		}
+	return 0;
+}
 
-		desc->callback = rp1_pio_sm_dma_callback;
-		desc->callback_param = dma;
+/*
+ * A NULL userbuf queues a receive of exactly "bytes" and returns without
+ * waiting: it puts the app in control of its own read-ahead depth and
+ * chunk sizes, rather than us assuming every read is a full dma->buf_size
+ * chunk (reads can be any size up to buf_size, and need not match it).
+ */
+static int rp1_pio_sm_rx_user(struct rp1_pio_device *pio, struct dma_info *dma,
+				  void __user *userbuf, size_t bytes)
+{
+	struct device *dev = &pio->pdev->dev;
+	int ret;
 
-		/* Submit the buffer - the callback will kick the semaphore */
-		ret = dmaengine_submit(desc);
-		if (ret < 0)
-			break;
+	if (!bytes)
+		return -EINVAL;
 
-		dma->head_idx++;
-		dma_async_issue_pending(dma->chan);
+	if (!userbuf) {
+		if (dma->head_idx - dma->tail_idx == dma->buf_count)
+			return -EBUSY;
 
-		bytes -= copy_bytes;
+		return rp1_pio_sm_rx_submit(pio, dma, bytes);
 	}
 
-	return ret;
+	while (bytes) {
+		struct dma_buf_info *dbi;
+		size_t len;
+
+		if (dma->head_idx == dma->tail_idx) {
+			/* Nothing already queued: arm exactly what's needed */
+			len = min(bytes, dma->buf_size);
+			ret = rp1_pio_sm_rx_submit(pio, dma, len);
+			if (ret)
+				return ret;
+		}
+
+		dbi = &dma->bufs[dma->tail_idx % dma->buf_count];
+		len = sg_dma_len(&dbi->sgl);
+		if (len > bytes)
+			return -EINVAL;
+
+		if (down_interruptible(&dma->buf_sem)) {
+			dev_err(dev, "DMA wait interrupted\n");
+			return -ETIMEDOUT;
+		}
+		dma->tail_idx++;
+
+		if (copy_to_user(userbuf, dbi->buf, len))
+			return -EFAULT;
+		userbuf += len;
+		bytes -= len;
+	}
+
+	return 0;
 }
 
 static int rp1_pio_sm_xfer_data32_user(struct rp1_pio_client *client, void *param)
@@ -935,7 +1198,8 @@ static int rp1_pio_sm_xfer_data32_user(struct rp1_pio_client *client, void *para
 	uint32_t dma_mask;
 
 	if (args->sm >= RP1_PIO_SMS_COUNT || args->dir >= RP1_PIO_DIR_COUNT ||
-	    !args->data_bytes || !args->data)
+	    !args->data_bytes ||
+	    (!args->data && args->dir != RP1_PIO_DIR_FROM_SM))
 		return -EINVAL;
 
 	dma_mask = 1 << (args->sm * 2 + args->dir);
@@ -1007,13 +1271,13 @@ int rp1_pio_sm_xfer_data(struct rp1_pio_client *client, uint sm, uint dir,
 			return -EINVAL;
 		}
 
-		/* Grab a dma buffer */
+		/*
+		 * Never block: the caller manages its own outstanding count
+		 * and is told about freed slots via its callback.
+		 */
 		if (dma->head_idx - dma->tail_idx == dma->buf_count) {
-			if (down_timeout(&dma->buf_sem, msecs_to_jiffies(1000))) {
-				dev_err(dev, "DMA wait timed out\n");
-				kfree(dxs);
-				return -ETIMEDOUT;
-			}
+			kfree(dxs);
+			return -EBUSY;
 		}
 
 		dbi = &dma->bufs[dma->head_idx % dma->buf_count];
@@ -1106,11 +1370,20 @@ struct handler_info {
 
 	HANDLER(READ_HW, read_hw),
 	HANDLER(WRITE_HW, write_hw),
+
+	HANDLER(IRQ_CLAIM, irq_claim),
+	HANDLER(IRQ_WAIT, irq_wait),
+	HANDLER(IRQ_SET_ENABLED, irq_set_enabled),
+	HANDLER(IRQ_IS_ENABLED, irq_is_enabled),
+	HANDLER(SET_IRQN_SOURCE_MASK_ENABLED, set_irqn_source_mask_enabled),
+	HANDLER(INTERRUPT_GET, interrupt_get),
+	HANDLER(INTERRUPT_CLEAR, interrupt_clear),
 };
 
 struct rp1_pio_client *rp1_pio_open(void)
 {
 	struct rp1_pio_client *client;
+	int i;
 
 	if (!g_pio)
 		return ERR_PTR(-EPROBE_DEFER);
@@ -1121,7 +1394,12 @@ struct rp1_pio_client *rp1_pio_open(void)
 	if (!client)
 		return ERR_PTR(-ENOMEM);
 
+	for (i = 0; i < RP1_PIO_IRQ_COUNT; i++)
+		client->irqs[i] = -1;
+
 	client->pio = g_pio;
+	spin_lock_init(&client->lock);
+	init_completion(&client->completion);
 
 	return client;
 }
@@ -1130,18 +1408,41 @@ EXPORT_SYMBOL_GPL(rp1_pio_open);
 void rp1_pio_close(struct rp1_pio_client *client)
 {
 	struct rp1_pio_device *pio = client->pio;
-	uint claimed_dmas = client->claimed_dmas;
+	uint32_t claimed;
+	struct irq_info *irqi;
 	int i;
 
+	/* Wake any waiter */
+	spin_lock(&client->lock);
+	client->wake_reason = 0;
+	spin_unlock(&client->lock);
+	complete(&client->completion);
+
 	/* Free any allocated resources */
 
-	for (i = 0; claimed_dmas; i++) {
+	spin_lock(&pio->lock);
+	for (i = 0; i < pio->irq_count; i++) {
+		irqi = &pio->irqs[i];
+		if (irqi->owner == client) {
+			if (irqi->enabled)
+				irq_set_enabled(client, i, false);
+
+			irqi->owner = NULL;
+			irqi->handler = NULL;
+			irqi->context = NULL;
+		}
+	}
+	spin_unlock(&pio->lock);
+
+	claimed = client->claimed_dmas;
+
+	for (i = 0; claimed; i++) {
 		uint mask = (1 << i);
 
-		if (claimed_dmas & mask) {
+		if (claimed & mask) {
 			struct dma_info *dma = &pio->dma_configs[i >> 1][i & 1];
 
-			claimed_dmas &= ~mask;
+			claimed &= ~mask;
 			rp1_pio_sm_dma_free(&pio->pdev->dev, dma);
 		}
 	}
@@ -1333,6 +1634,30 @@ static long rp1_pio_compat_ioctl(struct file *filp, unsigned int ioctl_num,
 #define rp1_pio_compat_ioctl NULL
 #endif
 
+static irqreturn_t rp1_pio_irq_handler(int irq, void *dev_id)
+{
+	struct rp1_pio_device *pio = dev_id;
+	struct irq_info *irqi = NULL;
+	int i;
+
+	for (i = 0; i < pio->irq_count; i++) {
+		if (irq == pio->irqs[i].irq) {
+			irqi = &pio->irqs[i];
+			break;
+		}
+	}
+
+	if (!irqi || !irqi->handler) {
+		pr_err("disabling unwanted interrupt\n");
+		disable_irq_nosync(irq);
+		return IRQ_NONE;
+	}
+
+	irqi->handler(irqi->context);
+
+	return IRQ_HANDLED;
+}
+
 const struct file_operations rp1_pio_fops = {
 	.owner =	THIS_MODULE,
 	.open =		rp1_pio_file_open,
@@ -1352,6 +1677,7 @@ static int rp1_pio_probe(struct platform_device *pdev)
 	struct device *cdev;
 	char dev_name[16];
 	void *p;
+	int irq;
 	int ret;
 	int i;
 
@@ -1363,7 +1689,7 @@ static int rp1_pio_probe(struct platform_device *pdev)
 			return -EINVAL;
 	}
 
-	pdev->id = of_alias_get_id(pdev->dev.of_node, "pio");
+	pdev->id = of_alias_get_id(dev->of_node, "pio");
 	if (pdev->id < 0)
 		return dev_err_probe(dev, pdev->id, "alias is missing\n");
 
@@ -1379,7 +1705,7 @@ static int rp1_pio_probe(struct platform_device *pdev)
 	if (ret < 0)
 		goto out_err;
 
-	pio = devm_kzalloc(&pdev->dev, sizeof(*pio), GFP_KERNEL);
+	pio = devm_kzalloc(dev, sizeof(*pio), GFP_KERNEL);
 	if (!pio) {
 		ret = -ENOMEM;
 		goto out_err;
@@ -1399,6 +1725,25 @@ static int rp1_pio_probe(struct platform_device *pdev)
 		goto out_err;
 	}
 
+	if (pio->fw_pio_count > PIO_INTERRUPT_CLEAR) {
+		for (i = 0; i < ARRAY_SIZE(pio->irqs); i++) {
+			irq = of_irq_get(dev->of_node, i);
+			if (irq < 0)
+				break;
+			ret = devm_request_irq(dev, irq, rp1_pio_irq_handler,
+					0, pdev->name, pio);
+			if (ret < 0) {
+				dev_err(dev, "failed to request an irq (rc=%d)\n", ret);
+				goto out_err;
+			}
+
+			disable_irq(irq);
+			pio->irqs[i].irq = irq;
+		}
+
+		pio->irq_count = i;
+	}
+
 	pio->phys_addr = ioresource->start;
 
 	ret = alloc_chrdev_region(&pio->dev_num, 0, 1, DRIVER_NAME);
@@ -1431,7 +1776,8 @@ static int rp1_pio_probe(struct platform_device *pdev)
 
 	g_pio = pio;
 
-	dev_info(dev, "Created instance as %s\n", dev_name);
+	dev_info(dev, "Created instance as %s (op count %d, %d interrupts)\n",
+		 dev_name, pio->fw_pio_count, pio->irq_count);
 	return 0;
 
 out_cdev_del:
@@ -1475,7 +1821,7 @@ static struct platform_driver rp1_pio_driver = {
 		.of_match_table	= of_match_ptr(rp1_pio_ids),
 	},
 	.probe		= rp1_pio_probe,
-	.remove	= rp1_pio_remove,
+	.remove		= rp1_pio_remove,
 	.shutdown	= rp1_pio_remove,
 };
 
diff --git a/include/linux/pio_rp1.h b/include/linux/pio_rp1.h
index bb9d398021c69..825c15eedaf8b 100644
--- a/include/linux/pio_rp1.h
+++ b/include/linux/pio_rp1.h
@@ -46,6 +46,7 @@
 #define GPIOS_MASK			((1 << RP1_PIO_GPIO_COUNT) - 1)
 
 #define PICO_NO_HARDWARE		0
+#define PICO_PIO_VERSION		0
 
 #define pio0				pio_open_helper(0)
 
@@ -102,6 +103,18 @@
 #define PROC_PIO_SM0_EXECCTRL_STATUS_N_BITS	0x0000001f
 #define PROC_PIO_SM0_EXECCTRL_STATUS_N_LSB	0
 
+#define irq_add_shared_handler(num, handler, ...) \
+	irq_add_handler(pio, num, handler, NULL)
+
+#define irq_has_shared_handler(num, handler, ...) \
+	(irq_get_handler(pio, num, handler, NULL) != NULL)
+
+#define irq_set_exclusive_handler(num, handler) \
+	irq_add_handler(pio, num, handler, NULL)
+
+#define irq_get_exclusive_handler(num) \
+	((void *)irq_get_handler(pio, num))
+
 enum pio_fifo_join {
 	PIO_FIFO_JOIN_NONE = 0,
 	PIO_FIFO_JOIN_TX = 1,
@@ -179,14 +192,36 @@ enum pio_sm_flags {
 	PIO_SM_FLAG_RXUNDER = 0x0008,
 };
 
+
+enum pio_interrupt_source {
+	PIS_SM0_RX_FIFO_NOT_EMPTY,
+	PIS_SM1_RX_FIFO_NOT_EMPTY,
+	PIS_SM2_RX_FIFO_NOT_EMPTY,
+	PIS_SM3_RX_FIFO_NOT_EMPTY,
+	PIS_SM0_TX_FIFO_NOT_FULL,
+	PIS_SM1_TX_FIFO_NOT_FULL,
+	PIS_SM2_TX_FIFO_NOT_FULL,
+	PIS_SM3_TX_FIFO_NOT_FULL,
+	PIS_INTERRUPT0,
+	PIS_INTERRUPT1,
+	PIS_INTERRUPT2,
+	PIS_INTERRUPT3,
+
+	PIS_MAX
+};
+
 struct fp24_8 {
 	uint32_t val;
 };
 
 typedef rp1_pio_sm_config pio_sm_config;
 
+typedef void (*pio_irq_handler_t)(void *context);
+
 typedef struct rp1_pio_client *PIO;
 
+static struct rp1_pio_client *g_client;
+
 int rp1_pio_init(void);
 PIO rp1_pio_open(void);
 void rp1_pio_close(struct rp1_pio_client *client);
@@ -233,6 +268,22 @@ int rp1_pio_gpio_set_oeover(struct rp1_pio_client *client, void *param);
 int rp1_pio_gpio_set_input_enabled(struct rp1_pio_client *client, void *param);
 int rp1_pio_gpio_set_drive_strength(struct rp1_pio_client *client, void *param);
 
+int rp1_pio_irq_claim(struct rp1_pio_client *client, void *param);
+int rp1_pio_irq_wait(struct rp1_pio_client *client, void *param);
+int rp1_pio_set_irqn_source_mask_enabled(struct rp1_pio_client *client, void *param);
+int rp1_pio_interrupt_get(struct rp1_pio_client *client, void *param);
+int rp1_pio_interrupt_clear(struct rp1_pio_client *client, void *param);
+int rp1_pio_irq_set_enabled(struct rp1_pio_client *client, void *param);
+int rp1_pio_irq_is_enabled(struct rp1_pio_client *client, void *param);
+
+void rp1_pio_irq_add_handler(struct rp1_pio_client *client, uint num,
+			     pio_irq_handler_t handler, void *context);
+void rp1_pio_irq_remove_handler(struct rp1_pio_client *client,
+			     uint num, pio_irq_handler_t handler);
+pio_irq_handler_t rp1_pio_irq_get_handler(struct rp1_pio_client *client, uint num);
+uint rp1_pio_irq_map(struct rp1_pio_client *client, uint irqn);
+
+
 static inline int pio_init(void)
 {
 	return rp1_pio_init();
@@ -240,11 +291,14 @@ static inline int pio_init(void)
 
 static inline struct rp1_pio_client *pio_open(void)
 {
-	return rp1_pio_open();
+	g_client = rp1_pio_open();
+	return g_client;
 }
 
 static inline void pio_close(struct rp1_pio_client *client)
 {
+	if (g_client == client)
+		g_client = NULL;
 	rp1_pio_close(client);
 }
 
@@ -1046,4 +1100,119 @@ static inline int pio_gpio_disable_pulls(struct rp1_pio_client *client, uint gpi
 	return pio_gpio_set_pulls(client, gpio, false, false);
 }
 
+static inline int pio_get_irq_num(struct rp1_pio_client *client, uint irqn)
+{
+	return rp1_pio_irq_map(client, irqn);
+}
+
+static inline void pio_set_irqn_source_mask_enabled(struct rp1_pio_client *client, uint irq_index,
+						    uint32_t source_mask, bool enabled)
+{
+	struct rp1_pio_set_irqn_source_mask_enabled_args args = {
+		.irq_index = rp1_pio_irq_map(client, irq_index),
+		.source_mask = source_mask, .enabled = enabled,
+	};
+
+	rp1_pio_set_irqn_source_mask_enabled(client, &args);
+}
+
+static inline void pio_set_irq0_source_enabled(struct rp1_pio_client *client,
+					       enum pio_interrupt_source source,
+					       bool enabled)
+{
+	pio_set_irqn_source_mask_enabled(client, 0, 1 << source, enabled);
+}
+
+static inline void pio_set_irq1_source_enabled(struct rp1_pio_client *client,
+					       enum pio_interrupt_source source,
+					       bool enabled)
+{
+	pio_set_irqn_source_mask_enabled(client, 1, 1 << source, enabled);
+}
+
+static inline void pio_set_irq0_source_mask_enabled(struct rp1_pio_client *client,
+						    uint32_t source_mask,
+						    bool enabled)
+{
+	pio_set_irqn_source_mask_enabled(client, 0, source_mask, enabled);
+}
+
+static inline void pio_set_irq1_source_mask_enabled(struct rp1_pio_client *client,
+						    uint32_t source_mask,
+						    bool enabled)
+{
+	pio_set_irqn_source_mask_enabled(client, 1, source_mask, enabled);
+}
+
+static inline void pio_set_irqn_source_enabled(struct rp1_pio_client *client, uint irq_index,
+					       enum pio_interrupt_source source, bool enabled)
+{
+	pio_set_irqn_source_mask_enabled(client, irq_index, 1 << source, enabled);
+}
+
+static inline uint pio_interrupt_rel(uint sm, uint pio_interrupt_num)
+{
+	return (pio_interrupt_num & 0x1c) | ((pio_interrupt_num + sm) & 0x3);
+}
+
+static inline void irq_add_handler(struct rp1_pio_client *client, uint num,
+				   pio_irq_handler_t handler,
+				   void *context)
+{
+	return rp1_pio_irq_add_handler(client, num, handler, context);
+}
+
+static inline pio_irq_handler_t irq_get_handler(struct rp1_pio_client *client, uint num)
+{
+	return rp1_pio_irq_get_handler(client, num);
+}
+
+static inline bool irq_has_handler(struct rp1_pio_client *client, uint num)
+{
+	return rp1_pio_irq_get_handler(client, num) != NULL;
+}
+
+static inline void irq_set_enabled(struct rp1_pio_client *client, uint num, bool enabled)
+{
+	struct rp1_pio_irq_set_enabled_args args = { .irq_index = num, .enabled = enabled };
+
+	if (bad_params_if(client, num >= RP1_PIO_IRQ_COUNT))
+		return;
+
+	rp1_pio_irq_set_enabled(client, &args);
+};
+
+static inline bool irq_is_enabled(struct rp1_pio_client *client, uint num)
+{
+	struct rp1_pio_irq_set_enabled_args args = { .irq_index = num };
+
+	if (bad_params_if(client, num >= RP1_PIO_IRQ_COUNT))
+		return false;
+
+	(void)rp1_pio_irq_is_enabled(client, &args);
+
+	return !!args.enabled;
+}
+
+static inline bool pio_interrupt_get(struct rp1_pio_client *client, uint pio_interrupt_num)
+{
+	struct rp1_pio_interrupt_get_args args = { .pio_interrupt_num = pio_interrupt_num };
+
+	if (bad_params_if(client, pio_interrupt_num >= PIS_MAX))
+		return false;
+
+	(void)rp1_pio_interrupt_get(client, &args);
+
+	return !!args.active;
+}
+
+static inline void pio_interrupt_clear(struct rp1_pio_client *client, uint pio_interrupt_num)
+{
+	struct rp1_pio_interrupt_get_args args = { .pio_interrupt_num = pio_interrupt_num };
+
+	if (bad_params_if(client, pio_interrupt_num >= PIS_MAX))
+		return;
+	(void)rp1_pio_interrupt_clear(client, &args);
+}
+
 #endif
diff --git a/include/uapi/misc/rp1_pio_if.h b/include/uapi/misc/rp1_pio_if.h
index 4b527a2cc57a3..8827b60eeb34a 100644
--- a/include/uapi/misc/rp1_pio_if.h
+++ b/include/uapi/misc/rp1_pio_if.h
@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 + WITH Linux-syscall-note */
 /*
- * Copyright (c) 2023-24 Raspberry Pi Ltd.
+ * Copyright (c) 2023-26 Raspberry Pi Ltd.
  * All rights reserved.
  */
 #ifndef _PIO_RP1_IF_H
@@ -8,16 +8,17 @@
 
 #include <linux/ioctl.h>
 
-#define RP1_PIO_INSTRUCTION_COUNT   32
-#define RP1_PIO_SM_COUNT            4
-#define RP1_PIO_GPIO_COUNT          28
-#define RP1_GPIO_FUNC_PIO           7
+#define RP1_PIO_INSTRUCTION_COUNT	32
+#define RP1_PIO_SM_COUNT		4
+#define RP1_PIO_GPIO_COUNT		28
+#define RP1_GPIO_FUNC_PIO		7
+#define RP1_PIO_IRQ_COUNT		2
 
-#define RP1_PIO_ORIGIN_ANY          ((uint16_t)(~0))
+#define RP1_PIO_ORIGIN_ANY		((uint16_t)(~0))
 
-#define RP1_PIO_DIR_TO_SM           0
-#define RP1_PIO_DIR_FROM_SM         1
-#define RP1_PIO_DIR_COUNT           2
+#define RP1_PIO_DIR_TO_SM		0
+#define RP1_PIO_DIR_FROM_SM		1
+#define RP1_PIO_DIR_COUNT		2
 
 typedef struct {
 	uint32_t clkdiv;
@@ -203,6 +204,38 @@ struct rp1_access_hw_args {
 	void *data;
 };
 
+struct rp1_pio_irq_claim_args {
+	int irq_index; /* OUT */
+};
+
+struct rp1_pio_irq_wait_args {
+	uint32_t timeout_ms;
+	uint32_t active_mask; /* OUT */
+};
+
+struct rp1_pio_irq_set_enabled_args {
+	uint16_t irq_index;
+	uint8_t enabled;
+	uint8_t rsvd;
+};
+
+struct rp1_pio_set_irqn_source_mask_enabled_args {
+	uint16_t irq_index;
+	uint8_t enabled;
+	uint8_t rsvd;
+	uint32_t source_mask;
+};
+
+struct rp1_pio_interrupt_get_args {
+	uint16_t pio_interrupt_num;
+	uint16_t rsvd;
+	uint8_t active; /* OUT */
+};
+
+struct rp1_pio_interrupt_clear_args {
+	uint16_t pio_interrupt_num;
+};
+
 #define PIO_IOC_MAGIC 102
 
 #define PIO_IOC_SM_CONFIG_XFER _IOW(PIO_IOC_MAGIC, 0, struct rp1_pio_sm_config_xfer_args)
@@ -250,4 +283,13 @@ struct rp1_access_hw_args {
 #define PIO_IOC_GPIO_SET_INPUT_ENABLED _IOW(PIO_IOC_MAGIC, 56, struct rp1_gpio_set_args)
 #define PIO_IOC_GPIO_SET_DRIVE_STRENGTH _IOW(PIO_IOC_MAGIC, 57, struct rp1_gpio_set_args)
 
+#define PIO_IOC_IRQ_CLAIM _IOWR(PIO_IOC_MAGIC, 60, struct rp1_pio_irq_claim_args)
+#define PIO_IOC_IRQ_WAIT _IOWR(PIO_IOC_MAGIC, 61, struct rp1_pio_irq_wait_args)
+#define PIO_IOC_IRQ_SET_ENABLED _IOW(PIO_IOC_MAGIC, 62, struct rp1_pio_irq_set_enabled_args)
+#define PIO_IOC_IRQ_IS_ENABLED _IOWR(PIO_IOC_MAGIC, 63, struct rp1_pio_irq_set_enabled_args)
+#define PIO_IOC_SET_IRQN_SOURCE_MASK_ENABLED \
+	_IOW(PIO_IOC_MAGIC, 64, struct rp1_pio_set_irqn_source_mask_enabled_args)
+#define PIO_IOC_INTERRUPT_GET _IOWR(PIO_IOC_MAGIC, 65, struct rp1_pio_interrupt_get_args)
+#define PIO_IOC_INTERRUPT_CLEAR _IOW(PIO_IOC_MAGIC, 66, struct rp1_pio_interrupt_clear_args)
+
 #endif

From f24621cbfcb86cae38b06529e1be8594982a3519 Mon Sep 17 00:00:00 2001
From: Phil Elwell <phil@raspberrypi.com>
Date: Thu, 25 Jun 2026 12:06:11 +0100
Subject: [PATCH 4/8] arm64: dts: rp1: Declare the PIO interrupts

The PIO hardware drives 2 interrupt lines - declare them.

Signed-off-by: Phil Elwell <phil@raspberrypi.com>
---
 arch/arm64/boot/dts/broadcom/rp1.dtsi | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/arch/arm64/boot/dts/broadcom/rp1.dtsi b/arch/arm64/boot/dts/broadcom/rp1.dtsi
index 6a29960db5766..8847c32e1019f 100644
--- a/arch/arm64/boot/dts/broadcom/rp1.dtsi
+++ b/arch/arm64/boot/dts/broadcom/rp1.dtsi
@@ -1057,6 +1057,8 @@
 			reg = <0xc0 0x40178000  0x0 0x20>;
 			compatible = "raspberrypi,rp1-pio";
 			firmware = <&rp1_firmware>;
+			interrupts = <RP1_INT_PIO_0 IRQ_TYPE_EDGE_RISING>,
+				     <RP1_INT_PIO_1 IRQ_TYPE_EDGE_RISING>;
 			dmas = <&rp1_dma RP1_DMA_PIO_CH0_TX>, <&rp1_dma RP1_DMA_PIO_CH0_RX>,
 				   <&rp1_dma RP1_DMA_PIO_CH1_TX>, <&rp1_dma RP1_DMA_PIO_CH1_RX>,
 				   <&rp1_dma RP1_DMA_PIO_CH2_TX>, <&rp1_dma RP1_DMA_PIO_CH2_RX>,

From 4120e01b860ce0cb1b727a0cdee9873c2775a937 Mon Sep 17 00:00:00 2001
From: Phil Elwell <phil@raspberrypi.com>
Date: Tue, 21 Jul 2026 08:02:07 +0100
Subject: [PATCH 5/8] misc: rp1-pio: Non-blocking blocking operations

The piolib API has three blocking operations:
* pio_sm_get_blocking
* pio_sm_put_blocking
* pio_sm_exec_wait_blocking

Rather than blocking in the firmware, stalling the whole API, allow
the firmware to return EAGAIN to indicate that it has not completed.
With a small change on each side it then becomes possible to poll for
completion, allowing other operations to continue in the meantime.

Signed-off-by: Phil Elwell <phil@raspberrypi.com>
---
 drivers/misc/rp1-pio.c | 69 ++++++++++++++++++++++++++++++++++++++----
 1 file changed, 63 insertions(+), 6 deletions(-)

diff --git a/drivers/misc/rp1-pio.c b/drivers/misc/rp1-pio.c
index b901e1f592b82..cf701300bdb2d 100644
--- a/drivers/misc/rp1-pio.c
+++ b/drivers/misc/rp1-pio.c
@@ -15,6 +15,7 @@
 
 #include <linux/cdev.h>
 #include <linux/compat.h>
+#include <linux/delay.h>
 #include <linux/device.h>
 #include <linux/dmaengine.h>
 #include <linux/dma-mapping.h>
@@ -28,6 +29,7 @@
 #include <linux/pio_rp1.h>
 #include <linux/platform_device.h>
 #include <linux/rp1-firmware.h>
+#include <linux/sched/signal.h>
 #include <linux/semaphore.h>
 #include <linux/slab.h>
 #include <linux/spinlock.h>
@@ -56,6 +58,14 @@
 
 #define RP1_PIO_DMACTRL_DEFAULT	0x80000100
 
+#define POLL_INTERVAL_MIN	(100)
+#define POLL_INTERVAL_MAX	(100 * 1000)
+#define POLL_INTERVAL_NEXT(ival) \
+	do { \
+		if (ival < POLL_INTERVAL_MAX) \
+			ival = min(ival + (ival >> 1), POLL_INTERVAL_MAX); \
+	} while (0)
+
 #define HANDLER(_n, _f) \
 	[_IOC_NR(PIO_IOC_ ## _n)] = { #_n, rp1_pio_ ## _f, _IOC_SIZE(PIO_IOC_ ## _n) }
 
@@ -174,6 +184,10 @@ static int rp1_pio_message_resp(struct rp1_pio_device *pio,
 			memcpy(resp, &resp_buf[1], ret);
 		else if (copy_to_user(userbuf, &resp_buf[1], ret))
 			ret = -EFAULT;
+	} else if (ret == 4) {
+		ret = (int)resp_buf[0];
+		if (ret == -1)
+			ret = -EIO;
 	} else if (ret >= 0) {
 		ret = -EIO;
 	}
@@ -410,8 +424,23 @@ EXPORT_SYMBOL_GPL(rp1_pio_sm_set_config);
 int rp1_pio_sm_exec(struct rp1_pio_client *client, void *param)
 {
 	struct rp1_pio_sm_exec_args *args = param;
+	unsigned long interval = POLL_INTERVAL_MIN;
+	int ret;
+
+	if (args->blocking)
+		args->blocking = 2;
+
+	ret = rp1_pio_message(client->pio, PIO_SM_EXEC, args, sizeof(*args));
+	while (ret == -EAGAIN) {
+		if (signal_pending(current))
+			return -ERESTARTSYS;
+		fsleep(interval);
+		POLL_INTERVAL_NEXT(interval);
+		args->blocking = 3;
+		ret = rp1_pio_message(client->pio, PIO_SM_EXEC, args, sizeof(*args));
+	}
 
-	return rp1_pio_message(client->pio, PIO_SM_EXEC, args, sizeof(*args));
+	return ret;
 }
 EXPORT_SYMBOL_GPL(rp1_pio_sm_exec);
 
@@ -482,20 +511,48 @@ EXPORT_SYMBOL_GPL(rp1_pio_sm_enable_sync);
 int rp1_pio_sm_put(struct rp1_pio_client *client, void *param)
 {
 	struct rp1_pio_sm_put_args *args = param;
+	unsigned long interval = POLL_INTERVAL_MIN;
+	int ret;
 
-	return rp1_pio_message(client->pio, PIO_SM_PUT, args, sizeof(*args));
+	if (args->blocking)
+		args->blocking = 2;
+
+	while (1) {
+		ret = rp1_pio_message(client->pio, PIO_SM_PUT, args, sizeof(*args));
+		if (ret != -EAGAIN)
+			break;
+		if (signal_pending(current))
+			return -ERESTARTSYS;
+		fsleep(interval);
+		POLL_INTERVAL_NEXT(interval);
+	}
+
+	return ret;
 }
 EXPORT_SYMBOL_GPL(rp1_pio_sm_put);
 
 int rp1_pio_sm_get(struct rp1_pio_client *client, void *param)
 {
 	struct rp1_pio_sm_get_args *args = param;
+	unsigned long interval = POLL_INTERVAL_MIN;
 	int ret;
 
-	ret = rp1_pio_message_resp(client->pio, PIO_SM_GET, args, sizeof(*args),
-				   &args->data, NULL, sizeof(args->data));
-	if (ret >= 0)
-		return offsetof(struct rp1_pio_sm_get_args, data) + ret;
+	if (args->blocking)
+		args->blocking = 2;
+
+	while (1) {
+		ret = rp1_pio_message_resp(client->pio, PIO_SM_GET, args, sizeof(*args),
+					&args->data, NULL, sizeof(args->data));
+		if (ret >= 0)
+			return offsetof(struct rp1_pio_sm_get_args, data) + ret;
+		if (ret != -EAGAIN)
+			break;
+		if (signal_pending(current))
+			return -ERESTARTSYS;
+		fsleep(interval);
+		POLL_INTERVAL_NEXT(interval);
+	}
+
 	return ret;
 }
 EXPORT_SYMBOL_GPL(rp1_pio_sm_get);

From bd7ccdc1bb3e4b0c73e66cf98759def3595acb29 Mon Sep 17 00:00:00 2001
From: Phil Elwell <phil@raspberrypi.com>
Date: Wed, 15 Jul 2026 16:13:59 +0100
Subject: [PATCH 6/8] serial: Add rp1-pio-uart - a PIO-based UART

This is a tty driver that implements a UART using RP1's PIO block,
ported from pico-examples/pio/uart_dma. Like that example, it is
limited to 8N1, with no hardware flow control.

Signed-off-by: Phil Elwell <phil@raspberrypi.com>
---
 drivers/tty/serial/Kconfig        |  13 +
 drivers/tty/serial/Makefile       |   2 +
 drivers/tty/serial/rp1-pio-uart.c | 809 ++++++++++++++++++++++++++++++
 include/uapi/linux/serial_core.h  |   3 +
 4 files changed, 827 insertions(+)
 create mode 100644 drivers/tty/serial/rp1-pio-uart.c

diff --git a/drivers/tty/serial/Kconfig b/drivers/tty/serial/Kconfig
index b45e30a2c168d..451a981873cce 100644
--- a/drivers/tty/serial/Kconfig
+++ b/drivers/tty/serial/Kconfig
@@ -1630,6 +1630,19 @@ config SERIAL_RPI_FW
 
 	  If unsure, say N.
 
+config SERIAL_RP1_PIO_UART
+	tristate "Raspberry Pi RP1 PIO UART support"
+	depends on RP1_PIO || COMPILE_TEST
+	select SERIAL_CORE
+	help
+	  This selects a software UART implemented using the RP1 PIO block,
+	  with DMA moving data in and out of the PIO FIFOs and a PIO
+	  interrupt used for break detection. Only 8N1 with no hardware
+	  flow control is supported. Only useful on Raspberry Pi 5 and
+	  newer platforms that have an RP1 southbridge.
+
+	  If unsure, say N.
+
 endmenu
 
 config SERIAL_MCTRL_GPIO
diff --git a/drivers/tty/serial/Makefile b/drivers/tty/serial/Makefile
index f8a4c7491eb0f..8be48f9c5a3bb 100644
--- a/drivers/tty/serial/Makefile
+++ b/drivers/tty/serial/Makefile
@@ -11,6 +11,7 @@ obj-$(CONFIG_SERIAL_EARLYCON_SEMIHOST) += earlycon-semihost.o
 obj-$(CONFIG_SERIAL_EARLYCON_RISCV_SBI) += earlycon-riscv-sbi.o
 
 obj-$(CONFIG_SERIAL_RPI_FW) 		+= rpi-fw-uart.o
+obj-$(CONFIG_SERIAL_RP1_PIO_UART)	+= rp1-pio-uart.o
 # These Sparc drivers have to appear before others such as 8250
 # which share ttySx minor node space.  Otherwise console device
 # names change and other unplesantries.
@@ -73,6 +74,7 @@ obj-$(CONFIG_SERIAL_QE)			+= ucc_uart.o
 obj-$(CONFIG_SERIAL_RDA)		+= rda-uart.o
 obj-$(CONFIG_SERIAL_RP2)		+= rp2.o
 obj-$(CONFIG_SERIAL_RPI_FW) 		+= rpi-fw-uart.o
+obj-$(CONFIG_SERIAL_RP1_PIO_UART)	+= rp1-pio-uart.o
 obj-$(CONFIG_SERIAL_RSCI)		+= rsci.o
 obj-$(CONFIG_SERIAL_SA1100)		+= sa1100.o
 obj-$(CONFIG_SERIAL_SAMSUNG)		+= samsung_tty.o
diff --git a/drivers/tty/serial/rp1-pio-uart.c b/drivers/tty/serial/rp1-pio-uart.c
new file mode 100644
index 0000000000000..03a78afc2d54a
--- /dev/null
+++ b/drivers/tty/serial/rp1-pio-uart.c
@@ -0,0 +1,809 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * RP1 PIO-based UART driver
+ *
+ * Copyright (c) 2026, Raspberry Pi Ltd.  All rights reserved.
+ *
+ * Implements a UART entirely in the RP1 PIO block: one state machine
+ * bit-bangs TX, another bit-bangs RX, and DMA moves the data in and out of
+ * their FIFOs. Ported from the pico-examples "pio/uart_dma" demo (BSD
+ * 3-Clause licensed), turning its one-shot bare-metal transfer into a
+ * continuous tty stream.
+ *
+ * Every per-SM register access exposed by rp1-pio (put/get/fifo_state/
+ * interrupt_clear/...) is a synchronous RP1 firmware mailbox round trip, not
+ * a register read/write, so unlike the demo this driver can *only* move
+ * data via DMA: there is no affordable way to poll or push/pop the PIO
+ * FIFOs a byte at a time from the CPU. The one primitive that *is* cheap
+ * and hardirq-capable is the PIO interrupt itself, so that is reserved for
+ * something DMA cannot express: break detection.
+ *
+ * The PIO TX/RX FIFOs are 32 bits wide and rp1-pio's DMA path always moves
+ * data 32 bits at a time, but each of our PIO programs only cares about one
+ * byte per FIFO word (the TX program pulls a fresh word per start bit; the
+ * RX program pushes a fresh word per stop bit). So bytes are expanded to
+ * 32-bit words before transmission and picked back out of 32-bit words on
+ * reception -- see rp1_pio_uart_pack_tx()/rp1_pio_uart_unpack_rx().
+ */
+
+#include <linux/bits.h>
+#include <linux/gpio/consumer.h>
+#include <linux/jiffies.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/pio_rp1.h>
+#include <linux/platform_device.h>
+#include <linux/serial.h>
+#include <linux/serial_core.h>
+#include <linux/slab.h>
+#include <linux/timer.h>
+#include <linux/tty.h>
+#include <linux/tty_flip.h>
+#include <linux/workqueue.h>
+
+#define DRIVER_NAME "rp1-pio-uart"
+
+/* ---- PIO programs, translated from pico-examples/pio/uart_tx and uart_rx ---- */
+
+#define UART_TX_WRAP_TARGET	0
+#define UART_TX_WRAP		3
+
+static const u16 uart_tx_program_instructions[] = {
+	0x9fa0, //  0: pull   block           side 1 [7]
+	0xf727, //  1: set    x, 7            side 0 [7]
+	0x6001, //  2: out    pins, 1
+	0x0642, //  3: jmp    x--, 2                 [6]
+};
+
+/*
+ * The full uart_rx program, not the 4-instruction "mini" variant the demo
+ * uses for its DMA path: the extra instructions detect a break or framing
+ * error and raise a PIO interrupt instead of silently pushing a corrupt
+ * byte.
+ */
+#define UART_RX_WRAP_TARGET	0
+#define UART_RX_WRAP		8
+
+static const u16 uart_rx_program_instructions[] = {
+	0x2020, //  0: wait   0 pin, 0
+	0xea27, //  1: set    x, 7                   [10]
+	0x4001, //  2: in     pins, 1
+	0x0642, //  3: jmp    x--, 2                 [6]
+	0x00c8, //  4: jmp    pin, 8
+	0xc010, //  5: irq    nowait 0 rel
+	0x20a0, //  6: wait   1 pin, 0
+	0x0000, //  7: jmp    0
+	0x8020, //  8: push   block
+};
+
+static const struct pio_program uart_tx_program = {
+	.instructions = uart_tx_program_instructions,
+	.length = ARRAY_SIZE(uart_tx_program_instructions),
+	.origin = -1,
+};
+
+static const struct pio_program uart_rx_program = {
+	.instructions = uart_rx_program_instructions,
+	.length = ARRAY_SIZE(uart_rx_program_instructions),
+	.origin = -1,
+};
+
+/* Only 8N1 is implemented by the PIO programs above */
+#define RP1_PIO_UART_MIN_BAUD	400
+#define RP1_PIO_UART_MAX_BAUD	3000000
+#define RP1_PIO_UART_DEF_BAUD	115200
+
+/*
+ * TX: one 64-char chunk in flight at a time. RX: several small chunks kept
+ * pipelined so the PIO RX FIFO is always being drained. RX_CHUNK_CHARS
+ * bounds how long a burst of received bytes waits before being handed to
+ * the tty layer (a DMA chunk only "completes" once full) -- 8 chars is a
+ * throughput/latency compromise; lower it (down to 1) for lower latency at
+ * the cost of one DMA transaction per byte.
+ */
+#define RP1_PIO_UART_TX_CHUNK_CHARS	64
+#define RP1_PIO_UART_TX_CHUNK_BYTES	(RP1_PIO_UART_TX_CHUNK_CHARS * 4)
+
+#define RP1_PIO_UART_RX_CHUNK_CHARS	8
+#define RP1_PIO_UART_RX_CHUNK_BYTES	(RP1_PIO_UART_RX_CHUNK_CHARS * 4)
+#define RP1_PIO_UART_RX_SLOTS		4
+
+struct rp1_pio_uart;
+
+struct rp1_pio_uart_rx_slot {
+	struct rp1_pio_uart	*rfu;
+	u8			*buf; /* RP1_PIO_UART_RX_CHUNK_BYTES, x4-packed */
+};
+
+struct rp1_pio_uart {
+	struct uart_driver	driver;
+	struct uart_port	port;
+
+	struct gpio_desc	*tx_gpiod;
+	struct gpio_desc	*rx_gpiod;
+	unsigned int		tx_hwnum;
+	unsigned int		rx_hwnum;
+
+	PIO			pio;
+	unsigned int		sm_tx;
+	unsigned int		sm_rx;
+	unsigned int		offset_tx;
+	unsigned int		offset_rx;
+	int			irq_index;
+
+	bool			shutting_down;
+
+	/* TX: one chunk in flight at a time */
+	bool			tx_busy;
+	u8			*tx_buf; /* RP1_PIO_UART_TX_CHUNK_BYTES, x4-packed */
+	struct work_struct	tx_work;
+
+	/* RX: several chunks pipelined */
+	struct rp1_pio_uart_rx_slot rx_slots[RP1_PIO_UART_RX_SLOTS];
+	unsigned long		rx_rearm_pending;
+	bool			rx_stopped;
+	struct work_struct	rx_rearm_work;
+
+	/* RX: CPU fallback for characters stuck below the DMA threshold */
+	struct timer_list	rx_timeout;
+	struct work_struct	rx_timeout_work;
+
+	struct work_struct	break_work;
+};
+
+static inline struct rp1_pio_uart *port_to_rfu(struct uart_port *port)
+{
+	return container_of(port, struct rp1_pio_uart, port);
+}
+
+static void rp1_pio_uart_pack_tx(u8 *dst, const u8 *src, unsigned int count)
+{
+	unsigned int i;
+
+	for (i = 0; i < count; i++) {
+		dst[4 * i + 0] = src[i];
+		dst[4 * i + 1] = 0;
+		dst[4 * i + 2] = 0;
+		dst[4 * i + 3] = 0;
+	}
+}
+
+static void rp1_pio_uart_unpack_rx(const u8 *src, unsigned int count, u8 *dst)
+{
+	unsigned int i;
+
+	for (i = 0; i < count; i++)
+		dst[i] = src[4 * i + 3];
+}
+
+/* ---------------------------------- RX ---------------------------------- */
+
+static void rp1_pio_uart_rx_arm(struct rp1_pio_uart_rx_slot *slot);
+
+/*
+ * rp1_pio_sm_config_xfer_internal() (drivers/misc/rp1-pio.c) programs the RX
+ * SM's DREQ threshold to RP1_PIO_UART_RX_CHUNK_CHARS words, i.e. the joined,
+ * 8-deep RX FIFO is only drained by DMA once a full chunk has piled up in
+ * it. So if the sender goes quiet mid-chunk, the last few characters just
+ * sit in the FIFO forever -- DMA has nothing to trigger on and never comes
+ * back for them. rx_timeout notices the quiet spell and, since it runs in
+ * process context (see rp1_pio_uart_rx_timeout_work()), can afford to pop
+ * them out with pio_sm_get() instead.
+ */
+static unsigned long rp1_pio_uart_rx_timeout_jiffies(struct uart_port *port)
+{
+	u64 ns = (u64)READ_ONCE(port->frame_time) * RP1_PIO_UART_RX_CHUNK_CHARS;
+
+	/* Slop for scheduling/mailbox latency, and a floor for silly baud rates */
+	ns += 20 * NSEC_PER_MSEC;
+
+	return max(nsecs_to_jiffies(ns), 1UL);
+}
+
+/* DMA completion callback: context is whatever rp1-pio's dmaengine backend
+ * uses (tasklet/softirq in practice) -- do the context-safe part (deliver
+ * to the tty flip buffer) directly, defer the re-arm (which can allocate)
+ * to a workqueue.
+ */
+static void rp1_pio_uart_rx_complete(void *param)
+{
+	struct rp1_pio_uart_rx_slot *slot = param;
+	struct rp1_pio_uart *rfu = slot->rfu;
+	u8 chars[RP1_PIO_UART_RX_CHUNK_CHARS];
+	unsigned long flags;
+	bool stopped;
+
+	rp1_pio_uart_unpack_rx(slot->buf, RP1_PIO_UART_RX_CHUNK_CHARS, chars);
+	tty_insert_flip_string(&rfu->port.state->port, chars, RP1_PIO_UART_RX_CHUNK_CHARS);
+	tty_flip_buffer_push(&rfu->port.state->port);
+
+	spin_lock_irqsave(&rfu->port.lock, flags);
+	rfu->port.icount.rx += RP1_PIO_UART_RX_CHUNK_CHARS;
+	stopped = rfu->shutting_down || rfu->rx_stopped;
+	if (!stopped)
+		set_bit(slot - rfu->rx_slots, &rfu->rx_rearm_pending);
+	spin_unlock_irqrestore(&rfu->port.lock, flags);
+
+	if (!stopped) {
+		queue_work(system_highpri_wq, &rfu->rx_rearm_work);
+
+		/* A chunk just completed, so the FIFO isn't backed up: push the
+		 * deadline for the CPU fallback back out.
+		 */
+		mod_timer(&rfu->rx_timeout, jiffies + rp1_pio_uart_rx_timeout_jiffies(&rfu->port));
+	}
+}
+
+static void rp1_pio_uart_rx_arm(struct rp1_pio_uart_rx_slot *slot)
+{
+	struct rp1_pio_uart *rfu = slot->rfu;
+	int ret;
+
+	ret = pio_sm_xfer_data(rfu->pio, rfu->sm_rx, PIO_DIR_FROM_SM,
+			       RP1_PIO_UART_RX_CHUNK_BYTES, slot->buf, 0,
+			       rp1_pio_uart_rx_complete, slot);
+	if (ret)
+		dev_err_ratelimited(rfu->port.dev, "RX DMA submit failed (%d)\n", ret);
+}
+
+static void rp1_pio_uart_rx_rearm_work(struct work_struct *work)
+{
+	struct rp1_pio_uart *rfu = container_of(work, struct rp1_pio_uart, rx_rearm_work);
+	unsigned long flags;
+	int i;
+
+	for (i = 0; i < RP1_PIO_UART_RX_SLOTS; i++) {
+		bool go;
+
+		spin_lock_irqsave(&rfu->port.lock, flags);
+		go = test_and_clear_bit(i, &rfu->rx_rearm_pending) &&
+		     !rfu->shutting_down && !rfu->rx_stopped;
+		spin_unlock_irqrestore(&rfu->port.lock, flags);
+
+		if (go)
+			rp1_pio_uart_rx_arm(&rfu->rx_slots[i]);
+	}
+}
+
+/* Runs in process context so it can make the mailbox calls that pio_sm_get()
+ * and pio_sm_rx_fifo_level() require -- unlike the DMA completion path,
+ * there is no way to do this straight out of the timer callback.
+ */
+static void rp1_pio_uart_rx_timeout_work(struct work_struct *work)
+{
+	struct rp1_pio_uart *rfu = container_of(work, struct rp1_pio_uart, rx_timeout_work);
+	u8 chars[RP1_PIO_UART_RX_CHUNK_CHARS];
+	unsigned long flags;
+	bool stopped;
+	int level;
+	int i;
+
+	spin_lock_irqsave(&rfu->port.lock, flags);
+	stopped = rfu->shutting_down || rfu->rx_stopped;
+	spin_unlock_irqrestore(&rfu->port.lock, flags);
+	if (stopped)
+		return;
+
+	if (pio_sm_rx_fifo_level(rfu->pio, rfu->sm_rx) > 0) {
+		/*
+		 * Stop the RX SM so it cannot push any more words into the
+		 * FIFO while we drain it by hand: otherwise a fresh arrival
+		 * could push the level past the DMA threshold mid-drain and
+		 * race the DMA engine for the same words. A byte that is
+		 * mid-reception at this exact instant can be lost, but that
+		 * is the cost of doing this from a quiet-line timeout at all.
+		 */
+		pio_sm_set_enabled(rfu->pio, rfu->sm_rx, false);
+
+		level = pio_sm_rx_fifo_level(rfu->pio, rfu->sm_rx);
+		/* Every FIFO word carries one byte of data in its top 8 bits,
+		 * the rest is padding -- see rp1_pio_uart_unpack_rx().
+		 */
+		for (i = 0; i < level; i++)
+			chars[i] = pio_sm_get(rfu->pio, rfu->sm_rx) >> 24;
+
+		pio_sm_set_enabled(rfu->pio, rfu->sm_rx, true);
+
+		if (level > 0) {
+			tty_insert_flip_string(&rfu->port.state->port, chars, level);
+			tty_flip_buffer_push(&rfu->port.state->port);
+
+			spin_lock_irqsave(&rfu->port.lock, flags);
+			rfu->port.icount.rx += level;
+			spin_unlock_irqrestore(&rfu->port.lock, flags);
+		}
+	}
+
+	mod_timer(&rfu->rx_timeout, jiffies + rp1_pio_uart_rx_timeout_jiffies(&rfu->port));
+}
+
+/* Timer callback: softirq context, so just hand off to the workqueue */
+static void rp1_pio_uart_rx_timeout(struct timer_list *t)
+{
+	struct rp1_pio_uart *rfu = timer_container_of(rfu, t, rx_timeout);
+
+	queue_work(system_highpri_wq, &rfu->rx_timeout_work);
+}
+
+/* ---------------------------------- TX ---------------------------------- */
+
+static void rp1_pio_uart_tx_complete(void *param)
+{
+	struct rp1_pio_uart *rfu = param;
+	unsigned long flags;
+	bool more;
+
+	spin_lock_irqsave(&rfu->port.lock, flags);
+	rfu->tx_busy = false;
+	more = !rfu->shutting_down && !kfifo_is_empty(&rfu->port.state->port.xmit_fifo);
+	spin_unlock_irqrestore(&rfu->port.lock, flags);
+
+	if (more)
+		queue_work(system_highpri_wq, &rfu->tx_work);
+	else
+		uart_write_wakeup(&rfu->port);
+}
+
+static void rp1_pio_uart_tx_work(struct work_struct *work)
+{
+	struct rp1_pio_uart *rfu = container_of(work, struct rp1_pio_uart, tx_work);
+	struct uart_port *port = &rfu->port;
+	struct tty_port *tport = &port->state->port;
+	unsigned long flags;
+	unsigned char *tail;
+	unsigned int count;
+	int ret;
+
+	spin_lock_irqsave(&port->lock, flags);
+	if (rfu->tx_busy || rfu->shutting_down) {
+		spin_unlock_irqrestore(&port->lock, flags);
+		return;
+	}
+
+	count = kfifo_out_linear_ptr(&tport->xmit_fifo, &tail, RP1_PIO_UART_TX_CHUNK_CHARS);
+	if (!count) {
+		spin_unlock_irqrestore(&port->lock, flags);
+		return;
+	}
+
+	rp1_pio_uart_pack_tx(rfu->tx_buf, tail, count);
+	uart_xmit_advance(port, count);
+	rfu->tx_busy = true;
+	spin_unlock_irqrestore(&port->lock, flags);
+
+	ret = pio_sm_xfer_data(rfu->pio, rfu->sm_tx, PIO_DIR_TO_SM, count * 4, rfu->tx_buf, 0,
+			       rp1_pio_uart_tx_complete, rfu);
+	if (ret) {
+		dev_err_ratelimited(port->dev, "TX DMA submit failed (%d)\n", ret);
+		spin_lock_irqsave(&port->lock, flags);
+		rfu->tx_busy = false;
+		spin_unlock_irqrestore(&port->lock, flags);
+	}
+
+	uart_write_wakeup(port);
+}
+
+/* Called with port->lock held */
+static void rp1_pio_uart_start_tx(struct uart_port *port)
+{
+	struct rp1_pio_uart *rfu = port_to_rfu(port);
+
+	if (rfu->tx_busy || rfu->shutting_down)
+		return;
+
+	queue_work(system_highpri_wq, &rfu->tx_work);
+}
+
+static void rp1_pio_uart_stop_tx(struct uart_port *port)
+{
+	/*
+	 * Nothing exposed by pio_rp1.h can abort a DMA transfer already
+	 * submitted to the PIO FIFO -- any in-flight chunk simply finishes.
+	 */
+}
+
+static void rp1_pio_uart_stop_rx(struct uart_port *port)
+{
+	struct rp1_pio_uart *rfu = port_to_rfu(port);
+
+	rfu->rx_stopped = true;
+}
+
+/* -------------------------------- Break IRQ ------------------------------ */
+
+static void rp1_pio_uart_break_work(struct work_struct *work)
+{
+	struct rp1_pio_uart *rfu = container_of(work, struct rp1_pio_uart, break_work);
+	struct uart_port *port = &rfu->port;
+	unsigned long flags;
+	bool down;
+	uint intn;
+
+	intn = pio_interrupt_rel(rfu->sm_rx, 0);
+	pio_interrupt_clear(rfu->pio, intn);
+
+	spin_lock_irqsave(&port->lock, flags);
+	down = rfu->shutting_down;
+	if (!down)
+		port->icount.brk++;
+	spin_unlock_irqrestore(&port->lock, flags);
+
+	if (down)
+		return;
+
+	if (!uart_handle_break(port)) {
+		tty_insert_flip_char(&port->state->port, 0, TTY_BREAK);
+		tty_flip_buffer_push(&port->state->port);
+	}
+}
+
+/* Hardirq context: called directly from rp1-pio's own interrupt handler.
+ * pio_interrupt_clear() is a firmware mailbox call and must not run here.
+ */
+static void rp1_pio_uart_break_irq(void *context)
+{
+	struct rp1_pio_uart *rfu = context;
+
+	queue_work(system_highpri_wq, &rfu->break_work);
+}
+
+/* ------------------------------- uart_ops -------------------------------- */
+
+static unsigned int rp1_pio_uart_tx_empty(struct uart_port *port)
+{
+	struct rp1_pio_uart *rfu = port_to_rfu(port);
+	unsigned long flags;
+	bool empty;
+
+	spin_lock_irqsave(&port->lock, flags);
+	empty = !rfu->tx_busy && kfifo_is_empty(&port->state->port.xmit_fifo);
+	spin_unlock_irqrestore(&port->lock, flags);
+
+	return empty ? TIOCSER_TEMT : 0;
+}
+
+static void rp1_pio_uart_set_mctrl(struct uart_port *port, unsigned int mctrl)
+{
+	/* No hardware flow control */
+}
+
+static unsigned int rp1_pio_uart_get_mctrl(struct uart_port *port)
+{
+	return TIOCM_CTS;
+}
+
+static void rp1_pio_uart_break_ctl(struct uart_port *port, int ctl)
+{
+}
+
+static void rp1_pio_uart_set_baud(struct rp1_pio_uart *rfu, unsigned int baud)
+{
+	struct fp24_8 div = make_fp24_8(clock_get_hz(clk_sys), 8 * baud);
+
+	pio_sm_set_clkdiv(rfu->pio, rfu->sm_tx, div);
+	pio_sm_set_clkdiv(rfu->pio, rfu->sm_rx, div);
+}
+
+static void rp1_pio_uart_set_termios(struct uart_port *port, struct ktermios *new,
+				     const struct ktermios *old)
+{
+	struct rp1_pio_uart *rfu = port_to_rfu(port);
+	unsigned long flags;
+	unsigned int baud;
+
+	baud = uart_get_baud_rate(port, new, old, RP1_PIO_UART_MIN_BAUD, RP1_PIO_UART_MAX_BAUD);
+
+	/* Only 8N1, no flow control: that is all the PIO programs implement */
+	new->c_cflag &= ~(CSIZE | CSTOPB | PARENB | CRTSCTS);
+	new->c_cflag |= CS8;
+
+	spin_lock_irqsave(&port->lock, flags);
+	uart_update_timeout(port, new->c_cflag, baud);
+	spin_unlock_irqrestore(&port->lock, flags);
+
+	rp1_pio_uart_set_baud(rfu, baud);
+}
+
+static int rp1_pio_uart_startup(struct uart_port *port)
+{
+	struct rp1_pio_uart *rfu = port_to_rfu(port);
+	pio_sm_config cfg;
+	uint intn;
+	int ret, i;
+
+	rfu->pio = pio_open();
+	if (IS_ERR(rfu->pio)) {
+		ret = PTR_ERR(rfu->pio);
+		rfu->pio = NULL;
+		dev_err(port->dev, "Could not open PIO (%d)\n", ret);
+		return ret;
+	}
+
+	ret = pio_claim_unused_sm(rfu->pio, false);
+	if (ret < 0) {
+		ret = -EBUSY;
+		goto err_close;
+	}
+	rfu->sm_tx = ret;
+
+	ret = pio_claim_unused_sm(rfu->pio, false);
+	if (ret < 0) {
+		ret = -EBUSY;
+		goto err_close;
+	}
+	rfu->sm_rx = ret;
+
+	ret = pio_add_program(rfu->pio, &uart_tx_program);
+	if (ret == PIO_ORIGIN_INVALID) {
+		ret = -EBUSY;
+		goto err_close;
+	}
+	rfu->offset_tx = ret;
+
+	ret = pio_add_program(rfu->pio, &uart_rx_program);
+	if (ret == PIO_ORIGIN_INVALID) {
+		ret = -EBUSY;
+		goto err_close;
+	}
+	rfu->offset_rx = ret;
+
+	/* TX SM: side-set drives the start/stop bits, OUT drives data bits */
+	pio_sm_set_pins_with_mask(rfu->pio, rfu->sm_tx, BIT(rfu->tx_hwnum), BIT(rfu->tx_hwnum));
+	pio_sm_set_consecutive_pindirs(rfu->pio, rfu->sm_tx, rfu->tx_hwnum, 1, true);
+	pio_gpio_init(rfu->pio, rfu->tx_hwnum);
+
+	cfg = pio_get_default_sm_config();
+	sm_config_set_wrap(&cfg, rfu->offset_tx + UART_TX_WRAP_TARGET,
+			   rfu->offset_tx + UART_TX_WRAP);
+	sm_config_set_sideset(&cfg, 2, true, false);
+	sm_config_set_out_shift(&cfg, true, false, 32);
+	sm_config_set_out_pins(&cfg, rfu->tx_hwnum, 1);
+	sm_config_set_sideset_pins(&cfg, rfu->tx_hwnum);
+	sm_config_set_fifo_join(&cfg, PIO_FIFO_JOIN_TX);
+	pio_sm_init(rfu->pio, rfu->sm_tx, rfu->offset_tx, &cfg);
+
+	/* RX SM */
+	pio_sm_set_consecutive_pindirs(rfu->pio, rfu->sm_rx, rfu->rx_hwnum, 1, false);
+	pio_gpio_init(rfu->pio, rfu->rx_hwnum);
+	pio_gpio_pull_up(rfu->pio, rfu->rx_hwnum);
+
+	cfg = pio_get_default_sm_config();
+	sm_config_set_wrap(&cfg, rfu->offset_rx + UART_RX_WRAP_TARGET,
+			   rfu->offset_rx + UART_RX_WRAP);
+	sm_config_set_in_pins(&cfg, rfu->rx_hwnum);
+	sm_config_set_jmp_pin(&cfg, rfu->rx_hwnum);
+	sm_config_set_in_shift(&cfg, true, false, 32);
+	sm_config_set_fifo_join(&cfg, PIO_FIFO_JOIN_RX);
+	pio_sm_init(rfu->pio, rfu->sm_rx, rfu->offset_rx, &cfg);
+
+	rp1_pio_uart_set_baud(rfu, RP1_PIO_UART_DEF_BAUD);
+
+	/* Break detection: hooked to PIO IRQ 0, resolved relative to sm_rx */
+	ret = pio_get_irq_num(rfu->pio, 0);
+	if (ret < 0) {
+		if (ret == -ENODEV)
+			dev_err(port->dev, "No PIO interrupts - old EEPROM?\n");
+		else if (ret == -EBUSY)
+			dev_err(port->dev, "No free PIO interrupts\n");
+		goto err_close;
+	}
+	rfu->irq_index = ret;
+	irq_add_handler(rfu->pio, rfu->irq_index, rp1_pio_uart_break_irq, rfu);
+	intn = pio_interrupt_rel(rfu->sm_rx, 0);
+	pio_set_irqn_source_enabled(rfu->pio, 0, PIS_INTERRUPT0 + intn, true);
+	irq_set_enabled(rfu->pio, rfu->irq_index, true);
+
+	pio_interrupt_clear(rfu->pio, intn);
+
+	pio_sm_set_enabled(rfu->pio, rfu->sm_tx, true);
+	pio_sm_set_enabled(rfu->pio, rfu->sm_rx, true);
+
+	/* Request rp1-pio's internal DMA bounce buffers -- see the file
+	 * comment on why a client can't just dma_alloc_coherent its own.
+	 */
+	ret = pio_sm_config_xfer(rfu->pio, rfu->sm_tx, PIO_DIR_TO_SM,
+				 RP1_PIO_UART_TX_CHUNK_BYTES, 1);
+	if (ret)
+		goto err_close;
+
+	ret = pio_sm_config_xfer(rfu->pio, rfu->sm_rx, PIO_DIR_FROM_SM,
+				 RP1_PIO_UART_RX_CHUNK_BYTES, RP1_PIO_UART_RX_SLOTS);
+	if (ret)
+		goto err_close;
+
+	rfu->shutting_down = false;
+	rfu->tx_busy = false;
+	rfu->rx_stopped = false;
+	rfu->rx_rearm_pending = 0;
+
+	/* Keep every RX slot's DMA request outstanding at all times */
+	for (i = 0; i < RP1_PIO_UART_RX_SLOTS; i++)
+		rp1_pio_uart_rx_arm(&rfu->rx_slots[i]);
+
+	mod_timer(&rfu->rx_timeout, jiffies + rp1_pio_uart_rx_timeout_jiffies(&rfu->port));
+
+	return 0;
+
+err_close:
+	/* pio_close() unclaims any SMs/instructions/DMA this client claimed */
+	pio_close(rfu->pio);
+	rfu->pio = NULL;
+	return ret;
+}
+
+static void rp1_pio_uart_shutdown(struct uart_port *port)
+{
+	struct rp1_pio_uart *rfu = port_to_rfu(port);
+	unsigned long flags;
+
+	spin_lock_irqsave(&port->lock, flags);
+	rfu->shutting_down = true;
+	spin_unlock_irqrestore(&port->lock, flags);
+
+	if (rfu->irq_index >= 0) {
+		uint intn = pio_interrupt_rel(rfu->sm_rx, 0);
+
+		irq_set_enabled(rfu->pio, rfu->irq_index, false);
+		pio_set_irqn_source_enabled(rfu->pio, 0, PIS_INTERRUPT0 + intn, false);
+		pio_interrupt_clear(rfu->pio, intn);
+		rfu->irq_index = -1;
+	}
+
+	cancel_work_sync(&rfu->tx_work);
+	cancel_work_sync(&rfu->rx_rearm_work);
+	timer_delete_sync(&rfu->rx_timeout);
+	cancel_work_sync(&rfu->rx_timeout_work);
+	cancel_work_sync(&rfu->break_work);
+
+	pio_sm_set_enabled(rfu->pio, rfu->sm_tx, false);
+	pio_sm_set_enabled(rfu->pio, rfu->sm_rx, false);
+	pio_remove_program(rfu->pio, &uart_tx_program, rfu->offset_tx);
+	pio_remove_program(rfu->pio, &uart_rx_program, rfu->offset_rx);
+	pio_sm_unclaim(rfu->pio, rfu->sm_tx);
+	pio_sm_unclaim(rfu->pio, rfu->sm_rx);
+	pio_close(rfu->pio);
+	rfu->pio = NULL;
+}
+
+static const struct uart_ops rp1_pio_uart_ops = {
+	.tx_empty = rp1_pio_uart_tx_empty,
+	.set_mctrl = rp1_pio_uart_set_mctrl,
+	.get_mctrl = rp1_pio_uart_get_mctrl,
+	.stop_tx = rp1_pio_uart_stop_tx,
+	.start_tx = rp1_pio_uart_start_tx,
+	.stop_rx = rp1_pio_uart_stop_rx,
+	.break_ctl = rp1_pio_uart_break_ctl,
+	.startup = rp1_pio_uart_startup,
+	.shutdown = rp1_pio_uart_shutdown,
+	.set_termios = rp1_pio_uart_set_termios,
+};
+
+/* ------------------------------- Device setup ---------------------------- */
+
+static int rp1_pio_uart_get_gpio_hwnum(struct device *dev, const char *name)
+{
+	struct of_phandle_args of_args = { 0 };
+	char prop[16];
+	bool is_rp1_gpio;
+	int ret;
+
+	snprintf(prop, sizeof(prop), "%s-gpios", name);
+
+	ret = of_parse_phandle_with_args(dev->of_node, prop, "#gpio-cells", 0, &of_args);
+	if (ret)
+		return dev_err_probe(dev, ret, "can't parse %s\n", prop);
+
+	is_rp1_gpio = of_device_is_compatible(of_args.np, "raspberrypi,rp1-gpio");
+	of_node_put(of_args.np);
+	if (!is_rp1_gpio || of_args.args_count != 2)
+		return dev_err_probe(dev, -EINVAL, "%s is not an RP1 gpio\n", prop);
+
+	return of_args.args[0];
+}
+
+static int rp1_pio_uart_probe(struct platform_device *pdev)
+{
+	struct device *dev = &pdev->dev;
+	struct rp1_pio_uart *rfu;
+	int ret, i;
+
+	rfu = devm_kzalloc(dev, sizeof(*rfu), GFP_KERNEL);
+	if (!rfu)
+		return -ENOMEM;
+
+	rfu->tx_gpiod = devm_gpiod_get(dev, "tx", GPIOD_OUT_HIGH);
+	if (IS_ERR(rfu->tx_gpiod))
+		return PTR_ERR(rfu->tx_gpiod);
+
+	rfu->rx_gpiod = devm_gpiod_get(dev, "rx", GPIOD_IN);
+	if (IS_ERR(rfu->rx_gpiod))
+		return PTR_ERR(rfu->rx_gpiod);
+
+	ret = rp1_pio_uart_get_gpio_hwnum(dev, "tx");
+	if (ret < 0)
+		return ret;
+	rfu->tx_hwnum = ret;
+
+	ret = rp1_pio_uart_get_gpio_hwnum(dev, "rx");
+	if (ret < 0)
+		return ret;
+	rfu->rx_hwnum = ret;
+
+	rfu->tx_buf = devm_kzalloc(dev, RP1_PIO_UART_TX_CHUNK_BYTES, GFP_KERNEL);
+	if (!rfu->tx_buf)
+		return -ENOMEM;
+
+	for (i = 0; i < RP1_PIO_UART_RX_SLOTS; i++) {
+		rfu->rx_slots[i].rfu = rfu;
+		rfu->rx_slots[i].buf = devm_kzalloc(dev, RP1_PIO_UART_RX_CHUNK_BYTES, GFP_KERNEL);
+		if (!rfu->rx_slots[i].buf)
+			return -ENOMEM;
+	}
+
+	INIT_WORK(&rfu->tx_work, rp1_pio_uart_tx_work);
+	INIT_WORK(&rfu->rx_rearm_work, rp1_pio_uart_rx_rearm_work);
+	INIT_WORK(&rfu->rx_timeout_work, rp1_pio_uart_rx_timeout_work);
+	timer_setup(&rfu->rx_timeout, rp1_pio_uart_rx_timeout, 0);
+	INIT_WORK(&rfu->break_work, rp1_pio_uart_break_work);
+	rfu->irq_index = -1;
+
+	rfu->driver.owner = THIS_MODULE;
+	rfu->driver.driver_name = DRIVER_NAME;
+	rfu->driver.dev_name = "ttyPIO";
+	rfu->driver.nr = 1;
+
+	ret = uart_register_driver(&rfu->driver);
+	if (ret)
+		return dev_err_probe(dev, ret, "failed to register UART driver\n");
+
+	spin_lock_init(&rfu->port.lock);
+	rfu->port.dev = dev;
+	rfu->port.type = PORT_RP1_PIO;
+	rfu->port.ops = &rp1_pio_uart_ops;
+	rfu->port.fifosize = RP1_PIO_UART_TX_CHUNK_CHARS;
+	rfu->port.iotype = UPIO_MEM;
+	rfu->port.flags = UPF_BOOT_AUTOCONF;
+	rfu->port.private_data = rfu;
+
+	ret = uart_add_one_port(&rfu->driver, &rfu->port);
+	if (ret) {
+		dev_err(dev, "failed to add UART port: %d\n", ret);
+		goto err_unregister;
+	}
+
+	platform_set_drvdata(pdev, rfu);
+	dev_info(dev, "PIO UART on GPIOs tx %u rx %u\n", rfu->tx_hwnum, rfu->rx_hwnum);
+	return 0;
+
+err_unregister:
+	uart_unregister_driver(&rfu->driver);
+	return ret;
+}
+
+static void rp1_pio_uart_remove(struct platform_device *pdev)
+{
+	struct rp1_pio_uart *rfu = platform_get_drvdata(pdev);
+
+	uart_remove_one_port(&rfu->driver, &rfu->port);
+	uart_unregister_driver(&rfu->driver);
+}
+
+static const struct of_device_id rp1_pio_uart_match[] = {
+	{ .compatible = "raspberrypi,rp1-pio-uart" },
+	{ }
+};
+MODULE_DEVICE_TABLE(of, rp1_pio_uart_match);
+
+static struct platform_driver rp1_pio_uart_driver = {
+	.driver = {
+		.name = DRIVER_NAME,
+		.of_match_table = rp1_pio_uart_match,
+	},
+	.probe = rp1_pio_uart_probe,
+	.remove = rp1_pio_uart_remove,
+};
+module_platform_driver(rp1_pio_uart_driver);
+
+MODULE_AUTHOR("Raspberry Pi Ltd.");
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("RP1 PIO-based UART driver");
diff --git a/include/uapi/linux/serial_core.h b/include/uapi/linux/serial_core.h
index e0ca374c0db8b..5dc9e4910ccf6 100644
--- a/include/uapi/linux/serial_core.h
+++ b/include/uapi/linux/serial_core.h
@@ -234,6 +234,9 @@
 /* RPi firmware UART */
 #define PORT_RPI_FW	124
 
+/* RP1 PIO UART */
+#define PORT_RP1_PIO	125
+
 /* Generic type identifier for ports which type is not important to userspace. */
 #define PORT_GENERIC	(-1)
 

From 1be265f065a081f62fa1913f932bb7fe0ae87b2a Mon Sep 17 00:00:00 2001
From: Phil Elwell <phil@raspberrypi.com>
Date: Wed, 15 Jul 2026 16:25:47 +0100
Subject: [PATCH 7/8] configs: Add SERIAL_RP1_PIO_UART=m

Enable building the RP1 PIO UART for all kernels that support the Pi 5.

Signed-off-by: Phil Elwell <phil@raspberrypi.com>
---
 arch/arm64/configs/bcm2711_defconfig    | 1 +
 arch/arm64/configs/bcm2711_rt_defconfig | 1 +
 arch/arm64/configs/bcm2712_defconfig    | 1 +
 3 files changed, 3 insertions(+)

diff --git a/arch/arm64/configs/bcm2711_defconfig b/arch/arm64/configs/bcm2711_defconfig
index e9a14c80fe68c..6eb2e7d9edb95 100644
--- a/arch/arm64/configs/bcm2711_defconfig
+++ b/arch/arm64/configs/bcm2711_defconfig
@@ -746,6 +746,7 @@ CONFIG_SERIAL_AMBA_PL011=y
 CONFIG_SERIAL_AMBA_PL011_CONSOLE=y
 CONFIG_SERIAL_SC16IS7XX=m
 CONFIG_SERIAL_RPI_FW=m
+CONFIG_SERIAL_RP1_PIO_UART=m
 CONFIG_SERIAL_DEV_BUS=y
 CONFIG_TTY_PRINTK=y
 CONFIG_HW_RANDOM=y
diff --git a/arch/arm64/configs/bcm2711_rt_defconfig b/arch/arm64/configs/bcm2711_rt_defconfig
index 11232df846725..396c3c78e7a2e 100644
--- a/arch/arm64/configs/bcm2711_rt_defconfig
+++ b/arch/arm64/configs/bcm2711_rt_defconfig
@@ -745,6 +745,7 @@ CONFIG_SERIAL_AMBA_PL011=y
 CONFIG_SERIAL_AMBA_PL011_CONSOLE=y
 CONFIG_SERIAL_SC16IS7XX=m
 CONFIG_SERIAL_RPI_FW=m
+CONFIG_SERIAL_RP1_PIO_UART=m
 CONFIG_SERIAL_DEV_BUS=y
 CONFIG_TTY_PRINTK=y
 CONFIG_HW_RANDOM=y
diff --git a/arch/arm64/configs/bcm2712_defconfig b/arch/arm64/configs/bcm2712_defconfig
index c5573cf1bd11c..0ad2cc5fe8cfc 100644
--- a/arch/arm64/configs/bcm2712_defconfig
+++ b/arch/arm64/configs/bcm2712_defconfig
@@ -748,6 +748,7 @@ CONFIG_SERIAL_OF_PLATFORM=y
 CONFIG_SERIAL_AMBA_PL011=y
 CONFIG_SERIAL_AMBA_PL011_CONSOLE=y
 CONFIG_SERIAL_SC16IS7XX=m
+CONFIG_SERIAL_RP1_PIO_UART=m
 CONFIG_SERIAL_DEV_BUS=y
 CONFIG_TTY_PRINTK=y
 CONFIG_HW_RANDOM=y

From 898a4c57871af2b5bd01f5c4dcdd537f69393b0e Mon Sep 17 00:00:00 2001
From: Phil Elwell <phil@raspberrypi.com>
Date: Wed, 15 Jul 2026 16:16:46 +0100
Subject: [PATCH 8/8] overlays: Add the rp1-pio-uart overlay

An overlay to enable the PIO-based UART.

Signed-off-by: Phil Elwell <phil@raspberrypi.com>
---
 arch/arm/boot/dts/overlays/Makefile           |  1 +
 arch/arm/boot/dts/overlays/README             | 11 ++++++++
 .../dts/overlays/rp1-pio-uart-overlay.dts     | 25 +++++++++++++++++++
 3 files changed, 37 insertions(+)
 create mode 100644 arch/arm/boot/dts/overlays/rp1-pio-uart-overlay.dts

diff --git a/arch/arm/boot/dts/overlays/Makefile b/arch/arm/boot/dts/overlays/Makefile
index 85596878b6e1a..f1bf96aa11f91 100644
--- a/arch/arm/boot/dts/overlays/Makefile
+++ b/arch/arm/boot/dts/overlays/Makefile
@@ -247,6 +247,7 @@ dtbo-$(CONFIG_ARCH_BCM2835) += \
 	ramoops-pi4.dtbo \
 	rootmaster.dtbo \
 	rotary-encoder.dtbo \
+	rp1-pio-uart.dtbo \
 	rpi-backlight.dtbo \
 	rpi-codeczero.dtbo \
 	rpi-dacplus.dtbo \
diff --git a/arch/arm/boot/dts/overlays/README b/arch/arm/boot/dts/overlays/README
index 3ce83a441bfac..568c52adf5796 100644
--- a/arch/arm/boot/dts/overlays/README
+++ b/arch/arm/boot/dts/overlays/README
@@ -4527,6 +4527,17 @@ Params: pin_a                   GPIO connected to rotary encoder channel A
                                 common) and "binary".
 
 
+Name:   rp1-pio-uart
+Info:   Configures a software UART implemented using the RP1 PIO block, with
+        DMA moving data and a PIO interrupt used for break detection. Only
+        8N1 with no hardware flow control is supported. Only usable on
+        Raspberry Pi 5 and other platforms with an RP1 southbridge.
+Load:   dtoverlay=rp1-pio-uart,<param>=<val>
+Params: tx_pin                  GPIO used for TXD (any free - default 14)
+
+        rx_pin                  GPIO used for RXD (any free - default 15)
+
+
 Name:   rpi-backlight
 Info:   Raspberry Pi official display backlight driver
 Load:   dtoverlay=rpi-backlight
diff --git a/arch/arm/boot/dts/overlays/rp1-pio-uart-overlay.dts b/arch/arm/boot/dts/overlays/rp1-pio-uart-overlay.dts
new file mode 100644
index 0000000000000..7b1e980342b05
--- /dev/null
+++ b/arch/arm/boot/dts/overlays/rp1-pio-uart-overlay.dts
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: GPL-2.0
+// Device tree overlay for the RP1 PIO-based UART driver.
+/dts-v1/;
+/plugin/;
+
+/ {
+	compatible = "brcm,bcm2712";
+
+	fragment@0 {
+		target-path = "/";
+		__overlay__ {
+			rp1_pio_uart: rp1_pio_uart@e {
+				compatible = "raspberrypi,rp1-pio-uart";
+				tx-gpios = <&gpio 14 0>;
+				rx-gpios = <&gpio 15 0>;
+				status = "okay";
+			};
+		};
+	};
+
+	__overrides__ {
+		tx_pin = <&rp1_pio_uart>, "tx-gpios:4";
+		rx_pin = <&rp1_pio_uart>, "rx-gpios:4";
+	};
+};