wayland: fixes to frame insets (C2b), each with a suite assertion

include/SDL3/SDL_video.h

@@ -1362,11 +1362,20 @@ extern SDL_DECLSPEC SDL_Window * SDLCALL SDL_CreatePopupWindow(SDL_Window *paren
  *   the compositor is told the frame is the window
  *   (`xdg_surface.set_window_geometry`), so it places, snaps and tiles the
  *   frame; the surface grows by the margins around the size it configures;
- *   and only the frame and a band of up to 8 units around it take input, so
- *   the rest of the shadow passes clicks through. Maximized, tiled or
+ *   only the frame and a band around it take input (see
+ *   `SDL_PROP_WINDOW_CREATE_WAYLAND_FRAME_INPUT_MARGIN_NUMBER`), so the rest
+ *   of the shadow passes clicks through; and the opaque region of a window
+ *   that isn't `SDL_WINDOW_TRANSPARENT` is the frame. Maximized, tiled or
  *   fullscreen, the margins are zero (see
  *   `SDL_PROP_WINDOW_WAYLAND_FRAME_INSET_LEFT_NUMBER`). xdg-shell windows
- *   only, not libdecor's.
+ *   only, not libdecor's. The window keeps them if SDL has to make it again
+ *   (as for an OpenGL renderer on a window made without `SDL_WINDOW_OPENGL`).
+ * - `SDL_PROP_WINDOW_CREATE_WAYLAND_FRAME_INPUT_MARGIN_NUMBER` - the width, in
+ *   window coordinates, of the band around the frame that still takes input
+ *   while the window has frame insets, where the application's hit test (see
+ *   SDL_SetWindowHitTest()) can answer the edges it resizes from. Each side's
+ *   band is at most that side's inset. Defaults to 8; 0 leaves only the
+ *   frame taking input.
  *
  * These are additional supported properties on Windows:
  *
@@ -1460,6 +1469,7 @@ extern SDL_DECLSPEC SDL_Window * SDLCALL SDL_CreateWindowWithProperties(SDL_Prop
 #define SDL_PROP_WINDOW_CREATE_WAYLAND_FRAME_INSET_TOP_NUMBER      "SDL.window.create.wayland.frame_inset.top"
 #define SDL_PROP_WINDOW_CREATE_WAYLAND_FRAME_INSET_RIGHT_NUMBER    "SDL.window.create.wayland.frame_inset.right"
 #define SDL_PROP_WINDOW_CREATE_WAYLAND_FRAME_INSET_BOTTOM_NUMBER   "SDL.window.create.wayland.frame_inset.bottom"
+#define SDL_PROP_WINDOW_CREATE_WAYLAND_FRAME_INPUT_MARGIN_NUMBER   "SDL.window.create.wayland.frame_input_margin"
 #define SDL_PROP_WINDOW_CREATE_WIN32_HWND_POINTER                  "SDL.window.create.win32.hwnd"
 #define SDL_PROP_WINDOW_CREATE_WIN32_PIXEL_FORMAT_HWND_POINTER     "SDL.window.create.win32.pixel_format_hwnd"
 #define SDL_PROP_WINDOW_CREATE_X11_WINDOW_NUMBER                   "SDL.window.create.x11.window"
@@ -1624,8 +1634,10 @@ extern SDL_DECLSPEC SDL_Window * SDLCALL SDL_GetWindowParent(SDL_Window *window)
  *   `SDL_PROP_WINDOW_WAYLAND_FRAME_INSET_RIGHT_NUMBER` and
  *   `SDL_PROP_WINDOW_WAYLAND_FRAME_INSET_BOTTOM_NUMBER`: the frame insets in
  *   effect now, in window coordinates — those asked for at creation while the
- *   window floats, zero while it is maximized, tiled or fullscreen. They
- *   change with the window's size; read them on `SDL_EVENT_WINDOW_RESIZED`.
+ *   window floats, zero while it is maximized, tiled or fullscreen. Set from
+ *   creation on: until the window is first shown, the ones it will float
+ *   with. They change with the window's size; read them on
+ *   `SDL_EVENT_WINDOW_RESIZED`.
  *
  * On X11:
  *

src/video/wayland/SDL_waylandwindow.c

@@ -145,9 +145,10 @@ static void GetBufferSize(SDL_Window *window, int *width, int *height)
     }
 }
 
-/* The band around a frame with insets that still takes input, in window coordinates: where an
- * application's hit test answers the edges it resizes from, as a decorated window's border
- * would, while the rest of its shadow passes clicks through.
+/* The band around a frame with insets that still takes input by default, in window coordinates:
+ * where an application's hit test answers the edges it resizes from, as a decorated window's
+ * border would, while the rest of its shadow passes clicks through
+ * (SDL_PROP_WINDOW_CREATE_WAYLAND_FRAME_INPUT_MARGIN_NUMBER).
  */
 #define WAYLAND_FRAME_INPUT_MARGIN 8
 
@@ -169,24 +170,55 @@ static bool HasFrameInsets(SDL_WindowData *wind)
     return wind->frame_insets.left || wind->frame_insets.top || wind->frame_insets.right || wind->frame_insets.bottom;
 }
 
+/* The frame insets and input margin asked for at creation are kept in the window's properties
+ * as well, which outlive the window data: SDL_RecreateWindow makes the window again with no
+ * creation properties, and the window keeps its insets.
+ */
+#define WAYLAND_KEPT_FRAME_INSET_LEFT   "SDL.internal.wayland.frame_inset.left"
+#define WAYLAND_KEPT_FRAME_INSET_TOP    "SDL.internal.wayland.frame_inset.top"
+#define WAYLAND_KEPT_FRAME_INSET_RIGHT  "SDL.internal.wayland.frame_inset.right"
+#define WAYLAND_KEPT_FRAME_INSET_BOTTOM "SDL.internal.wayland.frame_inset.bottom"
+#define WAYLAND_KEPT_FRAME_INPUT_MARGIN "SDL.internal.wayland.frame_input_margin"
+
+static int GetKeptCreateNumber(SDL_PropertiesID create_props, SDL_PropertiesID window_props, const char *create_name, const char *kept_name, int default_value)
+{
+    const Sint64 kept = SDL_GetNumberProperty(window_props, kept_name, default_value);
+    const int value = SDL_max((int)SDL_GetNumberProperty(create_props, create_name, kept), 0);
+
+    SDL_SetNumberProperty(window_props, kept_name, value);
+    return value;
+}
+
+// Publishes the frame insets in effect (SDL_PROP_WINDOW_WAYLAND_FRAME_INSET_*).
+static void PublishFrameInsets(SDL_Window *window, int left, int top, int right, int bottom)
+{
+    const SDL_PropertiesID props = SDL_GetWindowProperties(window);
+    SDL_SetNumberProperty(props, SDL_PROP_WINDOW_WAYLAND_FRAME_INSET_LEFT_NUMBER, left);
+    SDL_SetNumberProperty(props, SDL_PROP_WINDOW_WAYLAND_FRAME_INSET_TOP_NUMBER, top);
+    SDL_SetNumberProperty(props, SDL_PROP_WINDOW_WAYLAND_FRAME_INSET_RIGHT_NUMBER, right);
+    SDL_SetNumberProperty(props, SDL_PROP_WINDOW_WAYLAND_FRAME_INSET_BOTTOM_NUMBER, bottom);
+}
+
 /* Tells the compositor the frame is the window, takes input on the frame and a band around it
- * only, and publishes the insets in effect. Called whenever the size or the insets change.
+ * only, and publishes the insets in effect. Called whenever the size or the insets change. A
+ * hidden window has no xdg_surface to frame; the configure that shows it applies them.
  */
 static void ApplyFrameInsets(SDL_Window *window, int left, int top, int right, int bottom)
 {
     SDL_WindowData *wind = window->internal;
     const int frame_w = SDL_max(wind->current.logical_width - left - right, 1);
     const int frame_h = SDL_max(wind->current.logical_height - top - bottom, 1);
 
-    if (wind->shell_surface.xdg.surface) {
-        xdg_surface_set_window_geometry(wind->shell_surface.xdg.surface, left, top, frame_w, frame_h);
+    if (!wind->shell_surface.xdg.surface) {
+        return;
     }
+    xdg_surface_set_window_geometry(wind->shell_surface.xdg.surface, left, top, frame_w, frame_h);
 
     if (left || top || right || bottom) {
-        const int ml = SDL_min(left, WAYLAND_FRAME_INPUT_MARGIN);
-        const int mt = SDL_min(top, WAYLAND_FRAME_INPUT_MARGIN);
-        const int mr = SDL_min(right, WAYLAND_FRAME_INPUT_MARGIN);
-        const int mb = SDL_min(bottom, WAYLAND_FRAME_INPUT_MARGIN);
+        const int ml = SDL_min(left, wind->frame_input_margin);
+        const int mt = SDL_min(top, wind->frame_input_margin);
+        const int mr = SDL_min(right, wind->frame_input_margin);
+        const int mb = SDL_min(bottom, wind->frame_input_margin);
         struct wl_region *region = wl_compositor_create_region(wind->waylandData->compositor);
         wl_region_add(region, left - ml, top - mt, frame_w + ml + mr, frame_h + mt + mb);
         wl_surface_set_input_region(wind->surface, region);
@@ -200,11 +232,7 @@ static void ApplyFrameInsets(SDL_Window *window, int left, int top, int right, i
     wind->applied_frame_insets.right = right;
     wind->applied_frame_insets.bottom = bottom;
 
-    const SDL_PropertiesID props = SDL_GetWindowProperties(window);
-    SDL_SetNumberProperty(props, SDL_PROP_WINDOW_WAYLAND_FRAME_INSET_LEFT_NUMBER, left);
-    SDL_SetNumberProperty(props, SDL_PROP_WINDOW_WAYLAND_FRAME_INSET_TOP_NUMBER, top);
-    SDL_SetNumberProperty(props, SDL_PROP_WINDOW_WAYLAND_FRAME_INSET_RIGHT_NUMBER, right);
-    SDL_SetNumberProperty(props, SDL_PROP_WINDOW_WAYLAND_FRAME_INSET_BOTTOM_NUMBER, bottom);
+    PublishFrameInsets(window, left, top, right, bottom);
 }
 
 static void SetMinMaxDimensions(SDL_Window *window)
@@ -382,9 +410,13 @@ static void SetSurfaceOpaqueRegion(SDL_WindowData *wind, bool is_opaque)
     SDL_VideoData *viddata = wind->waylandData;
 
     if (is_opaque) {
+        // Only the frame: any frame insets around it hold a shadow, which is never opaque.
+        const int left = wind->applied_frame_insets.left;
+        const int top = wind->applied_frame_insets.top;
+        const int width = SDL_max(wind->current.logical_width - left - wind->applied_frame_insets.right, 0);
+        const int height = SDL_max(wind->current.logical_height - top - wind->applied_frame_insets.bottom, 0);
         struct wl_region *region = wl_compositor_create_region(viddata->compositor);
-        wl_region_add(region, 0, 0,
-                      wind->current.logical_width, wind->current.logical_height);
+        wl_region_add(region, left, top, width, height);
         wl_surface_set_opaque_region(wind->surface, region);
         wl_region_destroy(region);
     } else {
@@ -970,11 +1002,15 @@ static void handle_xdg_toplevel_configure(void *data,
                     width = window->floating.w;
                     height = window->floating.h;
 
-                    // Clamp the window to the toplevel bounds, if any are set.
+                    /* Clamp the window to the toplevel bounds, if any are set. They bound the
+                     * frame, so the surface may be larger by its insets.
+                     */
                     if (wind->shell_surface_status == WAYLAND_SHELL_SURFACE_STATUS_WAITING_FOR_CONFIGURE &&
                         wind->toplevel_bounds.width && wind->toplevel_bounds.height) {
-                        width = SDL_min(wind->toplevel_bounds.width, width);
-                        height = SDL_min(wind->toplevel_bounds.height, height);
+                        int il, it, ir, ib;
+                        GetFrameInsets(wind, floating, &il, &it, &ir, &ib);
+                        width = SDL_min(wind->toplevel_bounds.width + il + ir, width);
+                        height = SDL_min(wind->toplevel_bounds.height + it + ib, height);
                     }
                 } else {
                     width = window->windowed.w;
@@ -2396,6 +2432,11 @@ void Wayland_HideWindow(SDL_VideoDevice *_this, SDL_Window *window)
     wl_surface_attach(wind->surface, NULL, 0, 0);
     wl_surface_commit(wind->surface);
 
+    /* The window geometry went with the xdg_surface: forget the insets applied, so the next
+     * configure frames the window again even at the same size.
+     */
+    SDL_zero(wind->applied_frame_insets);
+
     SDL_zero(wind->shell_surface);
     wind->show_hide_sync_required = true;
     struct wl_callback *cb = wl_display_sync(_this->internal->display);
@@ -2846,12 +2887,16 @@ bool Wayland_CreateWindow(SDL_VideoDevice *_this, SDL_Window *window, SDL_Proper
     // Default to all capabilities
     data->wm_caps = WAYLAND_WM_CAPS_ALL;
 
-    // Margins outside the frame for a shadow the application draws; toplevels only.
+    /* Margins outside the frame for a shadow the application draws; toplevels only. A window
+     * made again by SDL_RecreateWindow has no creation properties, and keeps those it had.
+     */
     if (!SDL_WINDOW_IS_POPUP(window) && !custom_surface_role) {
-        data->frame_insets.left = SDL_max((int)SDL_GetNumberProperty(create_props, SDL_PROP_WINDOW_CREATE_WAYLAND_FRAME_INSET_LEFT_NUMBER, 0), 0);
-        data->frame_insets.top = SDL_max((int)SDL_GetNumberProperty(create_props, SDL_PROP_WINDOW_CREATE_WAYLAND_FRAME_INSET_TOP_NUMBER, 0), 0);
-        data->frame_insets.right = SDL_max((int)SDL_GetNumberProperty(create_props, SDL_PROP_WINDOW_CREATE_WAYLAND_FRAME_INSET_RIGHT_NUMBER, 0), 0);
-        data->frame_insets.bottom = SDL_max((int)SDL_GetNumberProperty(create_props, SDL_PROP_WINDOW_CREATE_WAYLAND_FRAME_INSET_BOTTOM_NUMBER, 0), 0);
+        const SDL_PropertiesID window_props = SDL_GetWindowProperties(window);
+        data->frame_insets.left = GetKeptCreateNumber(create_props, window_props, SDL_PROP_WINDOW_CREATE_WAYLAND_FRAME_INSET_LEFT_NUMBER, WAYLAND_KEPT_FRAME_INSET_LEFT, 0);
+        data->frame_insets.top = GetKeptCreateNumber(create_props, window_props, SDL_PROP_WINDOW_CREATE_WAYLAND_FRAME_INSET_TOP_NUMBER, WAYLAND_KEPT_FRAME_INSET_TOP, 0);
+        data->frame_insets.right = GetKeptCreateNumber(create_props, window_props, SDL_PROP_WINDOW_CREATE_WAYLAND_FRAME_INSET_RIGHT_NUMBER, WAYLAND_KEPT_FRAME_INSET_RIGHT, 0);
+        data->frame_insets.bottom = GetKeptCreateNumber(create_props, window_props, SDL_PROP_WINDOW_CREATE_WAYLAND_FRAME_INSET_BOTTOM_NUMBER, WAYLAND_KEPT_FRAME_INSET_BOTTOM, 0);
+        data->frame_input_margin = GetKeptCreateNumber(create_props, window_props, SDL_PROP_WINDOW_CREATE_WAYLAND_FRAME_INPUT_MARGIN_NUMBER, WAYLAND_KEPT_FRAME_INPUT_MARGIN, WAYLAND_FRAME_INPUT_MARGIN);
     }
 
     data->scale_factor = 1.0;
@@ -3007,6 +3052,15 @@ bool Wayland_CreateWindow(SDL_VideoDevice *_this, SDL_Window *window, SDL_Proper
     SDL_SetPointerProperty(props, SDL_PROP_WINDOW_WAYLAND_VIEWPORT_POINTER, data->viewport);
     SDL_SetPointerProperty(props, SDL_PROP_WINDOW_WAYLAND_EGL_WINDOW_POINTER, data->egl_window);
 
+    /* Until the compositor first configures the window, the insets in effect are those it will
+     * float with, so an application can lay itself out before showing it.
+     */
+    if (HasFrameInsets(data)) {
+        int il, it, ir, ib;
+        GetFrameInsets(data, true, &il, &it, &ir, &ib);
+        PublishFrameInsets(window, il, it, ir, ib);
+    }
+
     data->hit_test_result = SDL_HITTEST_NORMAL;
 
     return true;

src/video/wayland/SDL_waylandwindow.h

@@ -209,6 +209,8 @@ struct SDL_WindowData
     {
         int left, top, right, bottom;
     } frame_insets, applied_frame_insets;
+    // The band round the frame that takes input (SDL_PROP_WINDOW_CREATE_WAYLAND_FRAME_INPUT_MARGIN_NUMBER).
+    int frame_input_margin;
 
     bool floating;
     bool suspended;