GC checks stack space before running finalizer

If the stack does not have some minimum available space, the GC defers
calling a finalizer until the next cycle. That avoids errors while
running a finalizer that the programmer cannot control.

ldo.c

@@ -220,6 +220,17 @@ l_noret luaD_errerr (lua_State *L) {
 }
 
 
+/*
+** Check whether stack has enough space to run a simple function (such
+** as a finalizer): At least BASIC_STACK_SIZE in the Lua stack and
+** 2 slots in the C stack.
+*/
+int luaD_checkminstack (lua_State *L) {
+  return ((stacksize(L) < MAXSTACK - BASIC_STACK_SIZE) &&
+          (getCcalls(L) < LUAI_MAXCCALLS - 2));
+}
+
+
 /*
 ** In ISO C, any pointer use after the pointer has been deallocated is
 ** undefined behavior. So, before a stack reallocation, all pointers

ldo.h

@@ -89,6 +89,7 @@ LUAI_FUNC int luaD_reallocstack (lua_State *L, int newsize, int raiseerror);
 LUAI_FUNC int luaD_growstack (lua_State *L, int n, int raiseerror);
 LUAI_FUNC void luaD_shrinkstack (lua_State *L);
 LUAI_FUNC void luaD_inctop (lua_State *L);
+LUAI_FUNC int luaD_checkminstack (lua_State *L);
 
 LUAI_FUNC l_noret luaD_throw (lua_State *L, TStatus errcode);
 LUAI_FUNC l_noret luaD_throwbaselevel (lua_State *L, TStatus errcode);

lgc.c

@@ -1293,7 +1293,7 @@ static void finishgencycle (lua_State *L, global_State *g) {
   correctgraylists(g);
   checkSizes(L, g);
   g->gcstate = GCSpropagate;  /* skip restart */
-  if (!g->gcemergency)
+  if (!g->gcemergency && luaD_checkminstack(L))
     callallpendingfinalizers(L);
 }
 
@@ -1667,12 +1667,13 @@ static l_mem singlestep (lua_State *L, int fast) {
       break;
     }
     case GCScallfin: {  /* call finalizers */
-      if (g->tobefnz && !g->gcemergency) {
+      if (g->tobefnz && !g->gcemergency && luaD_checkminstack(L)) {
         g->gcstopem = 0;  /* ok collections during finalizers */
         GCTM(L);  /* call one finalizer */
         stepresult = CWUFIN;
       }
-      else {  /* emergency mode or no more finalizers */
+      else {  /* no more finalizers or emergency mode or no enough stack
+                 to run finalizers */
         g->gcstate = GCSpause;  /* finish collection */
         stepresult = step2pause;
       }