Fixed bug libsdl-org#6745 - Check for overflow in SDL_CalculateYUVSize (#6747)

Fixed bug #6745 - Check for overflow in SDL_CalculateYUVSize

src/render/SDL_yuv_sw.c

@@ -58,7 +58,10 @@ SDL_SW_CreateYUVTexture(Uint32 format, int w, int h)
     swdata->h = h;
     {
         size_t dst_size;
-        SDL_CalculateYUVSize(format, w, h, &dst_size, NULL);
+        if (SDL_CalculateYUVSize(format, w, h, &dst_size, NULL) < 0) {
+            SDL_OutOfMemory();
+            return NULL;
+        }
         swdata->pixels = (Uint8 *)SDL_SIMDAlloc(dst_size);
         if (!swdata->pixels) {
             SDL_SW_DestroyYUVTexture(swdata);

src/video/SDL_surface.c

@@ -39,13 +39,15 @@ SDL_COMPILE_TIME_ASSERT(can_indicate_overflow, SDL_SIZE_MAX > SDL_MAX_SINT32);
 /*
  * Calculate the pad-aligned scanline width of a surface.
  * Return SDL_SIZE_MAX on overflow.
+ *
+ * for FOURCC, use SDL_CalculateYUVSize()
  */
 static size_t
 SDL_CalculatePitch(Uint32 format, size_t width, SDL_bool minimal)
 {
     size_t pitch;
 
-    if (SDL_ISPIXELFORMAT_FOURCC(format) || SDL_BITSPERPIXEL(format) >= 8) {
+    if (SDL_BITSPERPIXEL(format) >= 8) {
         if (SDL_size_mul_overflow(width, SDL_BYTESPERPIXEL(format), &pitch)) {
             return SDL_SIZE_MAX;
         }
@@ -88,11 +90,21 @@ SDL_CreateSurface(int width, int height, Uint32 format)
         return NULL;
     }
 
-    pitch = SDL_CalculatePitch(format, width, SDL_FALSE);
+    if (SDL_ISPIXELFORMAT_FOURCC(format)) {
-    if (pitch > SDL_MAX_SINT32) {
+        int p;
-        /* Overflow... */
+        if (SDL_CalculateYUVSize(format, width, height, NULL, &p) < 0) {
-        SDL_OutOfMemory();
+            /* Overflow... */
-        return NULL;
+            SDL_OutOfMemory();
+            return NULL;
+        }
+        pitch = p;
+    } else {
+        pitch = SDL_CalculatePitch(format, width, SDL_FALSE);
+        if (pitch > SDL_MAX_SINT32) {
+            /* Overflow... */
+            SDL_OutOfMemory();
+            return NULL;
+        }
     }
 
     /* Allocate the surface */
@@ -134,20 +146,21 @@ SDL_CreateSurface(int width, int height, Uint32 format)
 
     /* Get the pixels */
     if (surface->w && surface->h) {
-        /* Assumptions checked in surface_size_assumptions assert above */
         size_t size;
-        if (SDL_size_mul_overflow(surface->h, surface->pitch, &size)) {
-            /* Overflow... */
-            SDL_FreeSurface(surface);
-            SDL_OutOfMemory();
-            return NULL;
-        }
-
         if (SDL_ISPIXELFORMAT_FOURCC(surface->format->format)) {
             /* Get correct size and pitch for YUV formats */
             if (SDL_CalculateYUVSize(surface->format->format, surface->w, surface->h, &size, &surface->pitch) < 0) {
-                SDL_OutOfMemory();
+                /* Overflow... */
                 SDL_FreeSurface(surface);
+                SDL_OutOfMemory();
+                return NULL;
+            }
+        } else {
+            /* Assumptions checked in surface_size_assumptions assert above */
+            if (SDL_size_mul_overflow(surface->h, surface->pitch, &size)) {
+                /* Overflow... */
+                SDL_FreeSurface(surface);
+                SDL_OutOfMemory();
                 return NULL;
             }
         }
@@ -202,7 +215,16 @@ SDL_CreateSurfaceFrom(void *pixels,
         return NULL;
     }
 
-    minimalPitch = SDL_CalculatePitch(format, width, SDL_TRUE);
+    if (SDL_ISPIXELFORMAT_FOURCC(format)) {
+        int p;
+        if (SDL_CalculateYUVSize(format, width, height, NULL, &p) < 0) {
+            SDL_InvalidParamError("pitch");
+            return NULL;
+        }
+        minimalPitch = p;
+    } else {
+        minimalPitch = SDL_CalculatePitch(format, width, SDL_TRUE);
+    }
 
     if (pitch < 0 || (pitch > 0 && ((size_t)pitch) < minimalPitch)) {
         SDL_InvalidParamError("pitch");

src/video/SDL_yuv.c

@@ -29,6 +29,10 @@
 
 static SDL_YUV_CONVERSION_MODE SDL_YUV_ConversionMode = SDL_YUV_CONVERSION_BT601;
 
+#if SDL_HAVE_YUV
+static SDL_bool IsPlanar2x2Format(Uint32 format);
+#endif
+
 void SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_MODE mode)
 {
     SDL_YUV_ConversionMode = mode;
@@ -53,51 +57,130 @@ SDL_YUV_CONVERSION_MODE SDL_GetYUVConversionModeForResolution(int width, int hei
 }
 
 /*
- * Calculate YUV size. 
+ * Calculate YUV size and pitch. Check for overflow.
  * Output 'pitch' that can be used with SDL_ConvertPixels()
  *
  * return 0 on success, -1 on error
  */
 int SDL_CalculateYUVSize(Uint32 format, int w, int h, size_t *size, int *pitch)
 {
-    const int sz_plane = w * h;
+#if SDL_HAVE_YUV
-    const int sz_plane_chroma = ((w + 1) / 2) * ((h + 1) / 2);
+    int sz_plane = 0, sz_plane_chroma = 0, sz_plane_packed = 0;
-    const int sz_plane_packed = ((w + 1) / 2) * h;
+
-    int dst_size = 0;
+    if (IsPlanar2x2Format(format) == SDL_TRUE) {
+        {
+            /* sz_plane == w * h; */
+            size_t s1;
+            if (SDL_size_mul_overflow(w, h, &s1) < 0) {
+                return -1;
+            }
+            sz_plane = (int) s1;
+        }
+
+        {
+            /* sz_plane_chroma == ((w + 1) / 2) * ((h + 1) / 2); */
+            size_t s1, s2, s3;
+            if (SDL_size_add_overflow(w, 1, &s1) < 0) {
+                return -1;
+            }
+            s1 = s1 / 2;
+            if (SDL_size_add_overflow(h, 1, &s2) < 0) {
+                return -1;
+            }
+            s2 = s2 / 2;
+            if (SDL_size_mul_overflow(s1, s2, &s3) < 0) {
+                return -1;
+            }
+            sz_plane_chroma = (int) s3;
+        }
+    } else {
+        /* sz_plane_packed == ((w + 1) / 2) * h; */
+        size_t s1, s2;
+        if (SDL_size_add_overflow(w, 1, &s1) < 0) {
+            return -1;
+        }
+        s1 = s1 / 2;
+        if (SDL_size_mul_overflow(s1, h, &s2) < 0) {
+            return -1;
+        }
+        sz_plane_packed = (int) s2;
+    }
+
     switch (format) {
     case SDL_PIXELFORMAT_YV12: /**< Planar mode: Y + V + U  (3 planes) */
     case SDL_PIXELFORMAT_IYUV: /**< Planar mode: Y + U + V  (3 planes) */
-        dst_size = sz_plane + sz_plane_chroma + sz_plane_chroma;
+
         if (pitch) {
             *pitch = w;
         }
+
+        if (size) {
+            /* dst_size == sz_plane + sz_plane_chroma + sz_plane_chroma; */
+            size_t s1, s2;
+            if (SDL_size_add_overflow(sz_plane, sz_plane_chroma, &s1) < 0) {
+                return -1;
+            }
+            if (SDL_size_add_overflow(s1, sz_plane_chroma, &s2) < 0) {
+                return -1;
+            }
+            *size = (int)s2;
+        }
         break;
 
     case SDL_PIXELFORMAT_YUY2: /**< Packed mode: Y0+U0+Y1+V0 (1 plane) */
     case SDL_PIXELFORMAT_UYVY: /**< Packed mode: U0+Y0+V0+Y1 (1 plane) */
     case SDL_PIXELFORMAT_YVYU: /**< Packed mode: Y0+V0+Y1+U0 (1 plane) */
-        dst_size = 4 * sz_plane_packed;
+
         if (pitch) {
-            *pitch = ((w + 1) / 2) * 4;
+            /* pitch == ((w + 1) / 2) * 4; */
+           size_t p1, p2;
+           if (SDL_size_add_overflow(w, 1, &p1) < 0) {
+               return -1;
+           }
+           p1 = p1 / 2;
+           if (SDL_size_mul_overflow(p1, 4, &p2) < 0) {
+               return -1;
+           }
+           *pitch = (int) p2;
+        }
+
+        if (size) {
+            /* dst_size == 4 * sz_plane_packed; */
+            size_t s1;
+            if (SDL_size_mul_overflow(sz_plane_packed, 4, &s1) < 0) {
+                return -1;
+            }
+            *size = (int) s1;
         }
         break;
 
     case SDL_PIXELFORMAT_NV12: /**< Planar mode: Y + U/V interleaved  (2 planes) */
     case SDL_PIXELFORMAT_NV21: /**< Planar mode: Y + V/U interleaved  (2 planes) */
-        dst_size = sz_plane + sz_plane_chroma + sz_plane_chroma;
         if (pitch) {
             *pitch = w;
         }
+
+        if (size) {
+            /* dst_size == sz_plane + sz_plane_chroma + sz_plane_chroma; */
+            size_t s1, s2;
+            if (SDL_size_add_overflow(sz_plane, sz_plane_chroma, &s1) < 0) {
+                return -1;
+            }
+            if (SDL_size_add_overflow(s1, sz_plane_chroma, &s2) < 0) {
+                return -1;
+            }
+            *size = (int) s2;
+        }
         break;
 
     default:
         return -1;
     }
 
-    if (size) {
-        *size = dst_size;
-    }
     return 0;
+#else
+    return -1;
+#endif
 }
 
 #if SDL_HAVE_YUV