diff: fix integer overflow problem with --tabsize

Reported by Tobias Stoeckmann in: http://bugs.gnu.org/18857
* src/diff.c (main): Don't overflow if INTMAX_MAX / 2 < tabsize.
* tests/bignum: New file, to test for this bug.
* tests/Makefile.am (TESTS): Add it.

src/diff.c

@@ -595,7 +595,8 @@ main (int argc, char **argv)
 
 	case TABSIZE_OPTION:
 	  numval = strtoumax (optarg, &numend, 10);
-	  if (! (0 < numval && numval <= SIZE_MAX) || *numend)
+	  if (! (0 < numval && numval <= SIZE_MAX - GUTTER_WIDTH_MINIMUM)
+	      || *numend)
 	    try_help ("invalid tabsize '%s'", optarg);
 	  if (tabsize != numval)
 	    {
@@ -682,10 +683,14 @@ main (int argc, char **argv)
 		a half line plus a gutter is an integral number of tabs,
 		so that tabs in the right column line up.  */
 
-    intmax_t t = expand_tabs ? 1 : tabsize;
-    intmax_t w = width;
-    intmax_t off = (w + t + GUTTER_WIDTH_MINIMUM) / (2 * t)  *  t;
-    sdiff_half_width = MAX (0, MIN (off - GUTTER_WIDTH_MINIMUM, w - off)),
+    size_t t = expand_tabs ? 1 : tabsize;
+    size_t w = width;
+    size_t t_plus_g = t + GUTTER_WIDTH_MINIMUM;
+    size_t unaligned_off = (w >> 1) + (t_plus_g >> 1) + (w & t_plus_g & 1);
+    size_t off = unaligned_off - unaligned_off % t;
+    sdiff_half_width = (off <= GUTTER_WIDTH_MINIMUM || w <= off
+			? 0
+			: MIN (off - GUTTER_WIDTH_MINIMUM, w - off));
     sdiff_column2_offset = sdiff_half_width ? off : w;
   }
 

tests/Makefile.am

@@ -2,6 +2,7 @@
 
 TESTS = \
   basic \
+  bignum \
   binary \
   colliding-file-names \
   excess-slash \

tests/bignum

@@ -0,0 +1,14 @@
+#!/bin/sh
+# big numbers
+
+. "${srcdir=.}/init.sh"; path_prepend_ ../src
+
+fail=0
+
+for tabsize in 2147483648 9223372036854775808; do
+  diff --tabsize=$tabsize /dev/null /dev/null
+  status=$?
+  test $status -eq 0 || test $status -eq 2 || fail=1
+done
+
+Exit $fail