Avoid integer overflow in gdk_rectangle_intersect

If e.g. the right edge of the leftmost rectangle is near MIN_INT, and
the left edge of the rightmost rectangle is large then subtracting these
can lead to an integer overflow, making the resultant "width" falsely
positive, thus returning a very wide result instead of the expected
no-intersection result.

We avoid the overflow by not doing the subtraction unless we know the
result will be positive. There are still risks for overflow if x + width
or y + width is larger than MAXINT, but we won't ever overflow for valid
rects now.

This may fix #607687

diff --git a/gdk/gdkrectangle.c b/gdk/gdkrectangle.c
index 17d7e0038dc2b5b7c0097a2ea37f74a9573cc598..1f06f7daf676823d2901889f8812de7a983069e0 100644 (file)
--- a/gdk/gdkrectangle.c
+++ b/gdk/gdkrectangle.c
@@ -79,7 +79,7 @@ gdk_rectangle_intersect (const GdkRectangle *src1,
                         GdkRectangle       *dest)
 {
   gint dest_x, dest_y;
-  gint dest_w, dest_h;
+  gint dest_x2, dest_y2;
   gint return_val;
 
   g_return_val_if_fail (src1 != NULL, FALSE);
@@ -89,17 +89,17 @@ gdk_rectangle_intersect (const GdkRectangle *src1,
 
   dest_x = MAX (src1->x, src2->x);
   dest_y = MAX (src1->y, src2->y);
-  dest_w = MIN (src1->x + src1->width, src2->x + src2->width) - dest_x;
-  dest_h = MIN (src1->y + src1->height, src2->y + src2->height) - dest_y;
+  dest_x2 = MIN (src1->x + src1->width, src2->x + src2->width);
+  dest_y2 = MIN (src1->y + src1->height, src2->y + src2->height);
 
-  if (dest_w > 0 && dest_h > 0)
+  if (dest_x2 > dest_x && dest_y2 > dest_y)
     {
       if (dest)
         {
           dest->x = dest_x;
           dest->y = dest_y;
-          dest->width = dest_w;
-          dest->height = dest_h;
+          dest->width = dest_x2 - dest_x;
+          dest->height = dest_y2 - dest_y;
         }
       return_val = TRUE;
     }