Fix in string.rep

The cast of n (number of repetitions) to size_t may truncate its value,
causing a buffer overflow later. Better to check the buffer size
using lua_Integer, as all string lengths must fit in a lua_Integer and
n already is a lua_Integer. If everything fits in MAX_SIZE, then we can
safely convert n to size_t and compute the buffer size as a size_t.

As a corner case, n can be larger than size_t if the strings being
repeated have length zero, but in this case it will be multiplied by
zero, so an overflow in the cast is irrelevant.

lstrlib.c

@@ -132,27 +132,31 @@ static int str_upper (lua_State *L) {
 }
 
 
+/*
+** MAX_SIZE is limited both by size_t and lua_Integer.
+** When x <= MAX_SIZE, x can be safely cast to size_t or lua_Integer.
+*/
 static int str_rep (lua_State *L) {
-  size_t l, lsep;
-  const char *s = luaL_checklstring(L, 1, &l);
+  size_t len, lsep;
+  const char *s = luaL_checklstring(L, 1, &len);
   lua_Integer n = luaL_checkinteger(L, 2);
   const char *sep = luaL_optlstring(L, 3, "", &lsep);
   if (n <= 0)
     lua_pushliteral(L, "");
-  else if (l_unlikely(l + lsep < l || l + lsep > MAX_SIZE / cast_sizet(n)))
+  else if (l_unlikely(len > MAX_SIZE - lsep ||
+               cast_st2S(len + lsep) > cast_st2S(MAX_SIZE) / n))
     return luaL_error(L, "resulting string too large");
   else {
-    size_t totallen = ((size_t)n * (l + lsep)) - lsep;
+    size_t totallen = (cast_sizet(n) * (len + lsep)) - lsep;
     luaL_Buffer b;
     char *p = luaL_buffinitsize(L, &b, totallen);
     while (n-- > 1) {  /* first n-1 copies (followed by separator) */
-      memcpy(p, s, l * sizeof(char)); p += l;
+      memcpy(p, s, len * sizeof(char)); p += len;
       if (lsep > 0) {  /* empty 'memcpy' is not that cheap */
-        memcpy(p, sep, lsep * sizeof(char));
-        p += lsep;
+        memcpy(p, sep, lsep * sizeof(char)); p += lsep;
       }
     }
-    memcpy(p, s, l * sizeof(char));  /* last copy (not followed by separator) */
+    memcpy(p, s, len * sizeof(char));  /* last copy without separator */
     luaL_pushresultsize(&b, totallen);
   }
   return 1;