video: Remove more assumptions about window state in the video layer

Don't check the fullscreen flag when toggling resizable, bordered, always on top, minimum size and maximum size, as the flag doesn't reflect pending async changes that may be in progress.

These properties can be made to be safely toggled while the window is in fullscreen mode and applied when returning to windowed mode, which ensures that requested window settings aren't lost if calling these functions while async fullscreen changes are in flight.

src/video/SDL_video.c

@@ -2548,7 +2548,7 @@ int SDL_SetWindowBordered(SDL_Window *window, SDL_bool bordered)
 {
     CHECK_WINDOW_MAGIC(window, -1);
     CHECK_WINDOW_NOT_POPUP(window, -1);
-    if (!(window->flags & SDL_WINDOW_FULLSCREEN)) {
+
     const SDL_bool want = (bordered != SDL_FALSE); /* normalize the flag. */
     const SDL_bool have = !(window->flags & SDL_WINDOW_BORDERLESS);
     if ((want != have) && (_this->SetWindowBordered)) {
@@ -2559,7 +2559,7 @@ int SDL_SetWindowBordered(SDL_Window *window, SDL_bool bordered)
         }
         _this->SetWindowBordered(_this, window, want);
     }
-    }
+
     return 0;
 }
 
@@ -2567,7 +2567,7 @@ int SDL_SetWindowResizable(SDL_Window *window, SDL_bool resizable)
 {
     CHECK_WINDOW_MAGIC(window, -1);
     CHECK_WINDOW_NOT_POPUP(window, -1);
-    if (!(window->flags & SDL_WINDOW_FULLSCREEN)) {
+
     const SDL_bool want = (resizable != SDL_FALSE); /* normalize the flag. */
     const SDL_bool have = ((window->flags & SDL_WINDOW_RESIZABLE) != 0);
     if ((want != have) && (_this->SetWindowResizable)) {
@@ -2579,7 +2579,7 @@ int SDL_SetWindowResizable(SDL_Window *window, SDL_bool resizable)
         }
         _this->SetWindowResizable(_this, window, want);
     }
-    }
+
     return 0;
 }
 
@@ -2587,7 +2587,7 @@ int SDL_SetWindowAlwaysOnTop(SDL_Window *window, SDL_bool on_top)
 {
     CHECK_WINDOW_MAGIC(window, -1);
     CHECK_WINDOW_NOT_POPUP(window, -1);
-    if (!(window->flags & SDL_WINDOW_FULLSCREEN)) {
+
     const SDL_bool want = (on_top != SDL_FALSE); /* normalize the flag. */
     const SDL_bool have = ((window->flags & SDL_WINDOW_ALWAYS_ON_TOP) != 0);
     if ((want != have) && (_this->SetWindowAlwaysOnTop)) {
@@ -2598,7 +2598,7 @@ int SDL_SetWindowAlwaysOnTop(SDL_Window *window, SDL_bool on_top)
         }
         _this->SetWindowAlwaysOnTop(_this, window, want);
     }
-    }
+
     return 0;
 }
 
@@ -2716,6 +2716,8 @@ int SDL_GetWindowSizeInPixels(SDL_Window *window, int *w, int *h)
 
 int SDL_SetWindowMinimumSize(SDL_Window *window, int min_w, int min_h)
 {
+    int w, h;
+
     CHECK_WINDOW_MAGIC(window, -1);
     if (min_w < 0) {
         return SDL_InvalidParamError("min_w");
@@ -2732,9 +2734,6 @@ int SDL_SetWindowMinimumSize(SDL_Window *window, int min_w, int min_h)
     window->min_w = min_w;
     window->min_h = min_h;
 
-    if (!(window->flags & SDL_WINDOW_FULLSCREEN)) {
-        int w, h;
-
     if (_this->SetWindowMinimumSize) {
         _this->SetWindowMinimumSize(_this, window);
     }
@@ -2743,8 +2742,6 @@ int SDL_SetWindowMinimumSize(SDL_Window *window, int min_w, int min_h)
     w = window->min_w ? SDL_max(window->floating.w, window->min_w) : window->floating.w;
     h = window->min_h ? SDL_max(window->floating.h, window->min_h) : window->floating.h;
     return SDL_SetWindowSize(window, w, h);
-    }
-    return 0;
 }
 
 int SDL_GetWindowMinimumSize(SDL_Window *window, int *min_w, int *min_h)
@@ -2761,6 +2758,8 @@ int SDL_GetWindowMinimumSize(SDL_Window *window, int *min_w, int *min_h)
 
 int SDL_SetWindowMaximumSize(SDL_Window *window, int max_w, int max_h)
 {
+    int w, h;
+
     CHECK_WINDOW_MAGIC(window, -1);
     if (max_w < 0) {
         return SDL_InvalidParamError("max_w");
@@ -2776,9 +2775,6 @@ int SDL_SetWindowMaximumSize(SDL_Window *window, int max_w, int max_h)
     window->max_w = max_w;
     window->max_h = max_h;
 
-    if (!(window->flags & SDL_WINDOW_FULLSCREEN)) {
-        int w, h;
-
     if (_this->SetWindowMaximumSize) {
         _this->SetWindowMaximumSize(_this, window);
     }
@@ -2787,8 +2783,6 @@ int SDL_SetWindowMaximumSize(SDL_Window *window, int max_w, int max_h)
     w = window->max_w ? SDL_min(window->floating.w, window->max_w) : window->floating.w;
     h = window->max_h ? SDL_min(window->floating.h, window->max_h) : window->floating.h;
     return SDL_SetWindowSize(window, w, h);
-    }
-    return 0;
 }
 
 int SDL_GetWindowMaximumSize(SDL_Window *window, int *max_w, int *max_h)

src/video/cocoa/SDL_cocoawindow.m

@@ -2440,12 +2440,17 @@ void Cocoa_RestoreWindow(SDL_VideoDevice *_this, SDL_Window *window)
 void Cocoa_SetWindowBordered(SDL_VideoDevice *_this, SDL_Window *window, SDL_bool bordered)
 {
     @autoreleasepool {
+        SDL_CocoaWindowData *data = (__bridge SDL_CocoaWindowData *)window->driverdata;
+
+        /* If the window is in or transitioning to/from fullscreen, this will be set on leave. */
+        if (!(window->flags & SDL_WINDOW_FULLSCREEN) && ![data.listener isInFullscreenSpaceTransition]) {
             if (SetWindowStyle(window, GetWindowStyle(window))) {
                 if (bordered) {
                     Cocoa_SetWindowTitle(_this, window); /* this got blanked out. */
                 }
             }
         }
+    }
 }
 
 void Cocoa_SetWindowResizable(SDL_VideoDevice *_this, SDL_Window *window, SDL_bool resizable)
@@ -2476,13 +2481,18 @@ void Cocoa_SetWindowResizable(SDL_VideoDevice *_this, SDL_Window *window, SDL_bo
 void Cocoa_SetWindowAlwaysOnTop(SDL_VideoDevice *_this, SDL_Window *window, SDL_bool on_top)
 {
     @autoreleasepool {
-        NSWindow *nswindow = ((__bridge SDL_CocoaWindowData *)window->driverdata).nswindow;
+        SDL_CocoaWindowData *data = (__bridge SDL_CocoaWindowData *)window->driverdata;
+        NSWindow *nswindow = data.nswindow;
+
+        /* If the window is in or transitioning to/from fullscreen, this will be set on leave. */
+        if (!(window->flags & SDL_WINDOW_FULLSCREEN) && ![data.listener isInFullscreenSpaceTransition]) {
             if (on_top) {
                 [nswindow setLevel:NSFloatingWindowLevel];
             } else {
                 [nswindow setLevel:kCGNormalWindowLevel];
             }
         }
+    }
 }
 
 int Cocoa_SetWindowFullscreen(SDL_VideoDevice *_this, SDL_Window *window, SDL_VideoDisplay *display, SDL_bool fullscreen)

src/video/wayland/SDL_waylandwindow.c

@@ -741,9 +741,14 @@ static void handle_configure_xdg_toplevel(void *data,
             /* If we're a fixed-size window, we know our size for sure.
              * Always assume the configure is wrong.
              */
+            if (floating) {
+                width = window->floating.w;
+                height = window->floating.h;
+            } else {
                 width = window->windowed.w;
                 height = window->windowed.h;
             }
+        }
 
         /* The content limits are only a hint, which the compositor is free to ignore,
          * so apply them manually when appropriate.
@@ -1007,8 +1012,13 @@ static void decoration_frame_configure(struct libdecor_frame *frame,
         }
     } else {
         if (!(window->flags & SDL_WINDOW_RESIZABLE)) {
+            if (floating) {
+                width = window->floating.w;
+                height = window->floating.h;
+            } else {
                 width = window->windowed.w;
                 height = window->windowed.h;
+            }
 
             OverrideLibdecorLimits(window);
         } else {

src/video/x11/SDL_x11events.c

@@ -1650,7 +1650,10 @@ static void X11_DispatchEvent(SDL_VideoDevice *_this, XEvent *xevent)
                         }
                     } else {
                         SDL_SendWindowEvent(data->window, SDL_EVENT_WINDOW_LEAVE_FULLSCREEN, 0, 0);
-                        SDL_UpdateFullscreenMode(data->window, SDL_FALSE, SDL_TRUE);
+                        SDL_UpdateFullscreenMode(data->window, SDL_FALSE, SDL_FALSE);
+
+                        /* Need to restore or update any limits changed while the window was fullscreen. */
+                        X11_SetWindowMinMax(data->window, !!(flags & SDL_WINDOW_MAXIMIZED));
                     }
 
                     if ((flags & SDL_WINDOW_FULLSCREEN) &&
@@ -1675,6 +1678,11 @@ static void X11_DispatchEvent(SDL_VideoDevice *_this, XEvent *xevent)
                     } else {
                         data->disable_size_position_events = SDL_FALSE;
                         data->previous_borders_nonzero = SDL_FALSE;
+
+                        if (!(data->window->flags & SDL_WINDOW_FULLSCREEN) && data->toggle_borders) {
+                            data->toggle_borders = SDL_FALSE;
+                            X11_SetWindowBordered(_this, data->window, !(data->window->flags & SDL_WINDOW_BORDERLESS));
+                        }
                     }
                 }
                 if ((changed & SDL_WINDOW_MAXIMIZED) && ((flags & SDL_WINDOW_MAXIMIZED) && !(flags & SDL_WINDOW_MINIMIZED))) {
@@ -1737,6 +1745,11 @@ static void X11_DispatchEvent(SDL_VideoDevice *_this, XEvent *xevent)
                     X11_XMoveWindow(display, data->xwindow, data->window->floating.x - data->border_left, data->window->floating.y - data->border_top);
                     X11_XResizeWindow(display, data->xwindow, data->window->floating.w, data->window->floating.h);
                 }
+
+                if (!(data->window->flags & SDL_WINDOW_FULLSCREEN) && data->toggle_borders) {
+                    data->toggle_borders = SDL_FALSE;
+                    X11_SetWindowBordered(_this, data->window, !(data->window->flags & SDL_WINDOW_BORDERLESS));
+                }
             }
         }
     } break;

src/video/x11/SDL_x11window.c

@@ -1048,50 +1048,58 @@ static void X11_SetWMNormalHints(SDL_VideoDevice *_this, SDL_Window *window, XSi
     X11_XRaiseWindow(display, data->xwindow);
 }
 
-void X11_SetWindowMinimumSize(SDL_VideoDevice *_this, SDL_Window *window)
+void X11_SetWindowMinMax(SDL_Window *window, SDL_bool use_current)
 {
     SDL_WindowData *data = window->driverdata;
     Display *display = data->videodata->display;
-
-    if (window->flags & SDL_WINDOW_RESIZABLE) {
     XSizeHints *sizehints = X11_XAllocSizeHints();
-        long userhints;
+    long hint_flags = 0;
 
-        X11_XGetWMNormalHints(display, data->xwindow, sizehints, &userhints);
+    X11_XGetWMNormalHints(display, data->xwindow, sizehints, &hint_flags);
+    sizehints->flags &= ~(PMinSize | PMaxSize);
 
-        sizehints->min_width = window->min_w;
+    if (data->window->flags & SDL_WINDOW_RESIZABLE) {
-        sizehints->min_height = window->min_h;
+        if (data->window->min_w || data->window->min_h) {
             sizehints->flags |= PMinSize;
-
+            sizehints->min_width = data->window->min_w;
-        X11_SetWMNormalHints(_this, window, sizehints);
+            sizehints->min_height = data->window->min_h;
-
+        }
-        X11_XFree(sizehints);
+        if (data->window->max_w || data->window->max_h) {
+            sizehints->flags |= PMaxSize;
+            sizehints->max_width = data->window->max_w;
+            sizehints->max_height = data->window->max_h;
+        }
+    } else {
+        /* Set the min/max to the same values to make the window non-resizable */
+        sizehints->flags |= PMinSize | PMaxSize;
+        sizehints->min_width = sizehints->max_width = use_current ? data->window->floating.w : window->windowed.w;
+        sizehints->min_height = sizehints->max_height = use_current ? data->window->floating.h : window->windowed.h;
     }
 
-    X11_XFlush(display);
+    X11_XSetWMNormalHints(display, data->xwindow, sizehints);
+    X11_XFree(sizehints);
+}
+
+void X11_SetWindowMinimumSize(SDL_VideoDevice *_this, SDL_Window *window)
+{
+    if (window->driverdata->pending_operation & X11_PENDING_OP_FULLSCREEN) {
+        X11_SyncWindow(_this, window);
+    }
+
+    if (!(window->flags & SDL_WINDOW_FULLSCREEN)) {
+        X11_SetWindowMinMax(window, SDL_TRUE);
+    }
 }
 
 void X11_SetWindowMaximumSize(SDL_VideoDevice *_this, SDL_Window *window)
 {
-    SDL_WindowData *data = window->driverdata;
+    if (window->driverdata->pending_operation & X11_PENDING_OP_FULLSCREEN) {
-    Display *display = data->videodata->display;
+        X11_SyncWindow(_this, window);
-
-    if (window->flags & SDL_WINDOW_RESIZABLE) {
-        XSizeHints *sizehints = X11_XAllocSizeHints();
-        long userhints;
-
-        X11_XGetWMNormalHints(display, data->xwindow, sizehints, &userhints);
-
-        sizehints->max_width = window->max_w;
-        sizehints->max_height = window->max_h;
-        sizehints->flags |= PMaxSize;
-
-        X11_SetWMNormalHints(_this, window, sizehints);
-
-        X11_XFree(sizehints);
     }
 
-    X11_XFlush(display);
+    if (!(window->flags & SDL_WINDOW_FULLSCREEN)) {
+        X11_SetWindowMinMax(window, SDL_TRUE);
+    }
 }
 
 void X11_SetWindowSize(SDL_VideoDevice *_this, SDL_Window *window)
@@ -1198,6 +1206,12 @@ void X11_SetWindowBordered(SDL_VideoDevice *_this, SDL_Window *window, SDL_bool
     Display *display = data->videodata->display;
     XEvent event;
 
+    if (data->pending_operation & X11_PENDING_OP_FULLSCREEN) {
+        X11_SyncWindow(_this, window);
+    }
+
+    /* If the window is fullscreen, the resize capability will be set/cleared when it is returned to windowed mode. */
+    if (!(window->flags & SDL_WINDOW_FULLSCREEN)) {
         SetWindowBordered(display, displaydata->screen, data->xwindow, bordered);
         X11_XFlush(display);
 
@@ -1218,41 +1232,34 @@ void X11_SetWindowBordered(SDL_VideoDevice *_this, SDL_Window *window, SDL_bool
         X11_XCheckIfEvent(display, &event, &isUnmapNotify, (XPointer)&data->xwindow);
         X11_XCheckIfEvent(display, &event, &isMapNotify, (XPointer)&data->xwindow);
 
+        /* Turning the borders off doesn't send an extent event, so they must be cleared here. */
+        if (bordered) {
+            X11_GetBorderValues(data);
+        } else {
+            data->border_top = data->border_left = data->border_bottom = data->border_right = 0;
+        }
+
         /* Make sure the window manager didn't resize our window for the difference. */
         X11_XResizeWindow(display, data->xwindow, window->floating.w, window->floating.h);
         X11_XSync(display, False);
+    } else {
+        /* If fullscreen, set a flag to toggle the borders when returning to windowed mode. */
+        data->toggle_borders = SDL_TRUE;
+    }
 }
 
 void X11_SetWindowResizable(SDL_VideoDevice *_this, SDL_Window *window, SDL_bool resizable)
 {
     SDL_WindowData *data = window->driverdata;
-    Display *display = data->videodata->display;
 
-    XSizeHints *sizehints = X11_XAllocSizeHints();
+    if (data->pending_operation & X11_PENDING_OP_FULLSCREEN) {
-    long userhints;
+        X11_SyncWindow(_this, window);
-
-    X11_XGetWMNormalHints(display, data->xwindow, sizehints, &userhints);
-
-    if (resizable) {
-        /* FIXME: Is there a better way to get max window size from X? -flibit */
-        const int maxsize = 0x7FFFFFFF;
-        sizehints->min_width = window->min_w;
-        sizehints->min_height = window->min_h;
-        sizehints->max_width = (window->max_w == 0) ? maxsize : window->max_w;
-        sizehints->max_height = (window->max_h == 0) ? maxsize : window->max_h;
-    } else {
-        sizehints->min_width = window->w;
-        sizehints->min_height = window->h;
-        sizehints->max_width = window->w;
-        sizehints->max_height = window->h;
     }
-    sizehints->flags |= PMinSize | PMaxSize;
 
-    X11_SetWMNormalHints(_this, window, sizehints);
+    /* If the window is fullscreen, the resize capability will be set/cleared when it is returned to windowed mode. */
-
+    if (!(window->flags & SDL_WINDOW_FULLSCREEN)) {
-    X11_XFree(sizehints);
+        X11_SetWindowMinMax(window, SDL_TRUE);
-
+    }
-    X11_XFlush(display);
 }
 
 void X11_SetWindowAlwaysOnTop(SDL_VideoDevice *_this, SDL_Window *window, SDL_bool on_top)
@@ -1528,22 +1535,14 @@ static int X11_SetWindowFullscreenViaWM(SDL_VideoDevice *_this, SDL_Window *wind
             return 0;
         }
 
-        if (!(window->flags & SDL_WINDOW_RESIZABLE)) {
+        if (fullscreen && !(window->flags & SDL_WINDOW_RESIZABLE)) {
             /* Compiz refuses fullscreen toggle if we're not resizable, so update the hints so we
                can be resized to the fullscreen resolution (or reset so we're not resizable again) */
             XSizeHints *sizehints = X11_XAllocSizeHints();
             long flags = 0;
             X11_XGetWMNormalHints(display, data->xwindow, sizehints, &flags);
-            /* set the resize flags on */
-            if (fullscreen) {
             /* we are going fullscreen so turn the flags off */
             sizehints->flags &= ~(PMinSize | PMaxSize);
-            } else {
-                /* Reset the min/max width height to make the window non-resizable again */
-                sizehints->flags |= PMinSize | PMaxSize;
-                sizehints->min_width = sizehints->max_width = window->windowed.w;
-                sizehints->min_height = sizehints->max_height = window->windowed.h;
-            }
             X11_XSetWMNormalHints(display, data->xwindow, sizehints);
             X11_XFree(sizehints);
         }

src/video/x11/SDL_x11window.h

@@ -96,6 +96,7 @@ struct SDL_WindowData
     SDL_bool window_was_maximized;
     SDL_bool disable_size_position_events;
     SDL_bool previous_borders_nonzero;
+    SDL_bool toggle_borders;
     SDL_HitTestResult hit_test_result;
 };
 
@@ -137,5 +138,6 @@ extern int X11_SetWindowFocusable(SDL_VideoDevice *_this, SDL_Window *window, SD
 
 int SDL_X11_SetWindowTitle(Display *display, Window xwindow, char *title);
 void X11_UpdateWindowPosition(SDL_Window *window, SDL_bool use_current_position);
+void X11_SetWindowMinMax(SDL_Window *window, SDL_bool use_current);
 
 #endif /* SDL_x11window_h_ */