GCed dynamic libraries

classic Classic list List threaded Threaded
5 messages Options
Reply | Threaded
Open this post in threaded view
|

GCed dynamic libraries

Edgar Toernig
Hi,

Some days ago I mentioned a scheme to make dynamic libraries
garbage collected.  Here's a sample implementation.  As a
bonus, dynamic libraries get a private registry.

The concept:

  The only references to a dynamic library that Lua cares
  about are C-functions exported via lua_pushcclosure.
  All other possible references (through userdata or light
  userdata) are never accessed by the Lua core itself.

  After loading a library, there's only one C-function, the
  start- or init-function of the library.  This function in
  turn may export other functions, and so on.

  If the C-closure of the start-function gets a reference
  to the library and one makes sure that this reference is
  propagated through to future C-closures created by the
  library, the garbage collector would be able to detect
  when a library is no longer referenced.

  While working on an implementation I noticed, that a lot
  of libraries make use of the registry, either by storing
  metatables there or luaL_ref-erences or something else.
  This of course would prohibit any garbage collection -
  there would always be a reference from the registry to
  the dynamic library.

  As a consequence, I made the registry the commonly shared
  object of all exported C-closures of a particular library.

  All C-closures which are not created by a dynamic library 
  get the same "coreregistry".  Later on I noticed that some
  code needs access to this coreregistry to store systemwide
  data (luaL_getn/setn and the debug-hook code) so I added
  an additional special index (LUA_COREREGISTRYINDEX).

The dynamic loader:

  Changes are straight forward.  Create a new registry and
  a userdata with a handle for the library.  The userdata is
  set up to unload the library when it is collected.  Then the
  userdata is stored into the registry and the C-closure for
  the start-function with the new registry is created.

  [The last step is a little bit weird due to the fact that
   there's no direct way to change the registry of an arbitrary
   closure, only lua_replace(L, LUA_REGISTRYINDEX) which changes
   the registry of the currently active closure.]

On-exit handlers:

  There's no special code for an on-exit-function.  It can
  easily be implemented through the regular API.  Just create
  a userdata object, assign the on-exit function as the __gc
  function and stored it in the registry.  (luaL_onexit?)

Finally, the code:

diff -ru lua-5.0-orig/src/lobject.h lua-5.0-gcso/src/lobject.h
--- lua-5.0-orig/src/lobject.h	Tue Mar 18 13:50:04 2003
+++ lua-5.0-gcso/src/lobject.h	Tue Jun 22 00:18:53 2004
@@ -260,6 +260,7 @@
 typedef struct CClosure {
   ClosureHeader;
   lua_CFunction f;
+  TObject _registry;
   TObject upvalue[1];
 } CClosure;
 
diff -ru lua-5.0-orig/src/lstate.h lua-5.0-gcso/src/lstate.h
--- lua-5.0-orig/src/lstate.h	Thu Feb 27 12:52:30 2003
+++ lua-5.0-gcso/src/lstate.h	Wed Jun 23 15:58:39 2004
@@ -49,7 +49,8 @@
 #define gt(L)	(&L->_gt)
 
 /* registry */
-#define registry(L)	(&G(L)->_registry)
+#define registry(L)	(&clvalue((L)->base-1)->c._registry)
+#define coreregistry(L)	(&clvalue((L)->stack)->c._registry)
 
 
 /* extra stack space to handle TM calls and some other extras */
@@ -117,7 +118,6 @@
   lu_mem GCthreshold;
   lu_mem nblocks;  /* number of `bytes' currently allocated */
   lua_CFunction panic;  /* to be called in unprotected errors */
-  TObject _registry;
   TObject _defaultmeta;
   struct lua_State *mainthread;
   Node dummynode[1];  /* common node array for all empty tables */
diff -ru lua-5.0-orig/src/lstate.c lua-5.0-gcso/src/lstate.c
--- lua-5.0-orig/src/lstate.c	Thu Apr  3 15:35:34 2003
+++ lua-5.0-gcso/src/lstate.c	Thu Jun 24 00:41:16 2004
@@ -98,7 +98,6 @@
   g->strt.nuse = 0;
   g->strt.hash = NULL;
   setnilvalue(defaultmeta(L));
-  setnilvalue(registry(L));
   luaZ_initbuffer(L, &g->buff);
   g->panic = default_panic;
   g->rootgc = NULL;
@@ -114,7 +113,10 @@
   sethvalue(defaultmeta(L), luaH_new(L, 0, 0));
   hvalue(defaultmeta(L))->metatable = hvalue(defaultmeta(L));
   sethvalue(gt(L), luaH_new(L, 0, 4));  /* table of globals */
-  sethvalue(registry(L), luaH_new(L, 4, 4));  /* registry */
+  /* create dummy function with core registry */
+  setclvalue(L->stack, luaF_newCclosure(L, 0));
+  clvalue(L->stack)->c.f = NULL;
+  sethvalue(coreregistry(L), luaH_new(L, 4, 4));  /* registry */
   luaS_resize(L, MINSTRTABSIZE);  /* initial size of string table */
   luaT_init(L);
   luaX_init(L);
@@ -165,6 +167,7 @@
   preinit_state(L1);
   L1->l_G = L->l_G;
   stack_init(L1, L);  /* init stack */
+  setobj2n(L1->stack, L->stack);  /* share dummy func with core registry */
   setobj2n(gt(L1), gt(L));  /* share table of globals */
   return L1;
 }
diff -ru lua-5.0-orig/include/lua.h lua-5.0-gcso/include/lua.h
--- lua-5.0-orig/include/lua.h	Tue Mar 18 13:31:39 2003
+++ lua-5.0-gcso/include/lua.h	Wed Jun 23 15:56:30 2004
@@ -28,7 +28,8 @@
 ** pseudo-indices
 */
 #define LUA_REGISTRYINDEX	(-10000)
-#define LUA_GLOBALSINDEX	(-10001)
+#define LUA_COREREGISTRYINDEX	(-10001)
+#define LUA_GLOBALSINDEX	(-10002)
 #define lua_upvalueindex(i)	(LUA_GLOBALSINDEX-(i))
 
 
diff -ru lua-5.0-orig/src/lapi.c lua-5.0-gcso/src/lapi.c
--- lua-5.0-orig/src/lapi.c	Mon Apr  7 16:36:08 2003
+++ lua-5.0-gcso/src/lapi.c	Thu Jun 24 00:22:34 2004
@@ -52,6 +52,7 @@
   }
   else switch (idx) {  /* pseudo-indices */
     case LUA_REGISTRYINDEX: return registry(L);
+    case LUA_COREREGISTRYINDEX: return coreregistry(L);
     case LUA_GLOBALSINDEX: return gt(L);
     default: {
       TObject *func = (L->base - 1);
@@ -450,6 +451,7 @@
   api_checknelems(L, n);
   cl = luaF_newCclosure(L, n);
   cl->c.f = fn;
+  setobj2n(&cl->c._registry, registry(L)); /* share callers registry */
   L->top -= n;
   while (n--)
     setobj2n(&cl->c.upvalue[n], L->top+n);
@@ -702,6 +704,7 @@
   Closure *cl;
   cl = luaF_newCclosure(L, 0);
   cl->c.f = c->func;
+  setobj2n(&cl->c._registry, registry(L));  /* share callers registry */
   setclvalue(L->top, cl);  /* push function */
   incr_top(L);
   setpvalue(L->top, c->ud);  /* push only argument */
diff -ru lua-5.0-orig/src/lgc.c lua-5.0-gcso/src/lgc.c
--- lua-5.0-orig/src/lgc.c	Thu Apr  3 15:35:34 2003
+++ lua-5.0-gcso/src/lgc.c	Tue Jun 22 01:18:45 2004
@@ -205,6 +205,7 @@
 static void traverseclosure (GCState *st, Closure *cl) {
   if (cl->c.isC) {
     int i;
+    markobject(st, &cl->c._registry);
     for (i=0; i<cl->c.nupvalues; i++)  /* mark its upvalues */
       markobject(st, &cl->c.upvalue[i]);
   }
@@ -444,7 +445,6 @@
 static void markroot (GCState *st, lua_State *L) {
   global_State *g = st->g;
   markobject(st, defaultmeta(L));
-  markobject(st, registry(L));
   traversestack(st, g->mainthread);
   if (L != g->mainthread)  /* another thread is running? */
     markvalue(st, L);  /* cannot collect it */
diff -ru lua-5.0-orig/src/lib/lauxlib.c lua-5.0-gcso/src/lib/lauxlib.c
--- lua-5.0-orig/src/lib/lauxlib.c	Mon Apr  7 16:35:00 2003
+++ lua-5.0-gcso/src/lib/lauxlib.c	Thu Jun 24 00:16:16 2004
@@ -264,7 +264,7 @@
 
 
 static void getsizes (lua_State *L) {
-  lua_rawgeti(L, LUA_REGISTRYINDEX, ARRAYSIZE_REF);
+  lua_rawgeti(L, LUA_COREREGISTRYINDEX, ARRAYSIZE_REF);
   if (lua_isnil(L, -1)) {  /* no `size' table? */
     lua_pop(L, 1);  /* remove nil */
     lua_newtable(L);  /* create it */
@@ -274,7 +274,7 @@
     lua_pushliteral(L, "k");
     lua_rawset(L, -3);  /* metatable(N).__mode = "k" */
     lua_pushvalue(L, -1);
-    lua_rawseti(L, LUA_REGISTRYINDEX, ARRAYSIZE_REF);  /* store in register */
+    lua_rawseti(L, LUA_COREREGISTRYINDEX, ARRAYSIZE_REF);  /* store in register */
   }
 }
 
diff -ru lua-5.0-orig/src/lib/ldblib.c lua-5.0-gcso/src/lib/ldblib.c
--- lua-5.0-orig/src/lib/ldblib.c	Thu Apr  3 15:35:34 2003
+++ lua-5.0-gcso/src/lib/ldblib.c	Thu Jun 24 00:16:33 2004
@@ -140,7 +140,7 @@
   static const char *const hooknames[] =
     {"call", "return", "line", "count", "tail return"};
   lua_pushlightuserdata(L, (void *)&KEY_HOOK);
-  lua_rawget(L, LUA_REGISTRYINDEX);
+  lua_rawget(L, LUA_COREREGISTRYINDEX);
   if (lua_isfunction(L, -1)) {
     lua_pushstring(L, hooknames[(int)ar->event]);
     if (ar->currentline >= 0)
@@ -187,7 +187,7 @@
   }
   lua_pushlightuserdata(L, (void *)&KEY_HOOK);
   lua_pushvalue(L, 1);
-  lua_rawset(L, LUA_REGISTRYINDEX);  /* set new hook */
+  lua_rawset(L, LUA_COREREGISTRYINDEX);  /* set new hook */
   return 0;
 }
 
@@ -200,7 +200,7 @@
     lua_pushliteral(L, "external hook");
   else {
     lua_pushlightuserdata(L, (void *)&KEY_HOOK);
-    lua_rawget(L, LUA_REGISTRYINDEX);   /* get hook */
+    lua_rawget(L, LUA_COREREGISTRYINDEX);   /* get hook */
   }
   lua_pushstring(L, unmakemask(mask, buff));
   lua_pushnumber(L, (lua_Number)lua_gethookcount(L));
diff -ru lua-5.0-orig/src/lib/loadlib.c lua-5.0-gcso/src/lib/loadlib.c
--- lua-5.0-orig/src/lib/loadlib.c	Mon Apr  7 22:11:53 2003
+++ lua-5.0-gcso/src/lib/loadlib.c	Wed Jun 23 05:26:31 2004
@@ -45,6 +45,10 @@
 
 #include <dlfcn.h>
 
+static void pushinitfunc(lua_State *L, void *libhandle, lua_CFunction initf);
+static void *getunloadhandle(lua_State *L);
+
+
 static int loadlib(lua_State *L)
 {
  const char *path=luaL_checkstring(L,1);
@@ -55,8 +59,7 @@
   lua_CFunction f=(lua_CFunction) dlsym(lib,init);
   if (f!=NULL)
   {
-   lua_pushlightuserdata(L,lib);
-   lua_pushcclosure(L,f,1);
+   pushinitfunc(L, lib, f);
    return 1;
   }
  }
@@ -68,6 +71,14 @@
  return 3;
 }
 
+static int unloadlib(lua_State *L)
+{
+  void **lib = getunloadhandle(L);
+
+  if (lib)
+    dlclose(*lib);
+  return 0;
+}
 #endif
 
 
@@ -92,6 +103,10 @@
 
 #include <windows.h>
 
+static void pushinitfunc(lua_State *L, void *libhandle, lua_CFunction initf);
+static void getunloadhandle(lua_State *L);
+
+
 static void pusherror(lua_State *L)
 {
  int error=GetLastError();
@@ -113,8 +128,7 @@
   lua_CFunction f=(lua_CFunction) GetProcAddress(lib,init);
   if (f!=NULL)
   {
-   lua_pushlightuserdata(L,lib);
-   lua_pushcclosure(L,f,1);
+   pushinitfunc(L, lib, f);
    return 1;
   }
  }
@@ -125,11 +139,73 @@
  return 3;
 }
 
+static int unloadlib(lua_State *L)
+{
+  HINSTANCE *lib = getunloadhandle(L);
+
+  if (lib)
+    FreeLibrary(*lib);
+  return 0;
+}
+
 #endif
 
 
 
-#ifndef LOADLIB
+#ifdef LOADLIB
+
+/* common code */
+
+static int pushinitfunc2(lua_State *L)	/* initf,regtable */
+{
+  lua_replace(L, LUA_REGISTRYINDEX);
+  lua_pushcfunction(L, lua_touserdata(L, 1));
+  return 1;
+}
+
+static void pushinitfunc(lua_State *L, void *handle, lua_CFunction initf)
+{
+  /* ud = setmetatable(newuserdata(handle), <upval1>) */
+  /* pushinitfunc2(initf, { [ud]=true }) */
+  lua_pushcfunction(L, pushinitfunc2);
+  lua_pushlightuserdata(L, initf);
+  lua_newtable(L);
+  lua_boxpointer(L, handle);
+  lua_pushvalue(L, lua_upvalueindex(1));
+  lua_setmetatable(L, -2);
+  lua_pushboolean(L, 1);
+  lua_rawset(L, -3);
+  lua_call(L, 2, 1);
+}
+
+static void *getunloadhandle(lua_State *L)
+{
+  void *lib = NULL;
+  printf("*unload\n");
+
+  if (lua_getmetatable(L, 1) && lua_rawequal(L, -1, lua_upvalueindex(1)))
+  {
+    lib = lua_touserdata(L, 1);
+    lua_pushnil(L);
+    lua_setmetatable(L, 1);
+  }
+  return lib;
+}
+
+LUALIB_API int luaopen_loadlib (lua_State *L)
+{
+  lua_newtable(L);
+  lua_pushliteral(L, "__gc");
+  lua_pushvalue(L, -2);
+  lua_pushcclosure(L, unloadlib, 1);
+  lua_rawset(L, -3);
+  lua_pushcclosure(L, loadlib, 1);
+  lua_setglobal(L, "loadlib");
+  return 0;
+}
+
+
+#else
 /* Fallback for other systems */
 
 /*
@@ -170,13 +246,13 @@
  lua_pushliteral(L,"absent");
  return 3;
 }
-#endif
 
 LUALIB_API int luaopen_loadlib (lua_State *L)
 {
  lua_register(L,"loadlib",loadlib);
  return 0;
 }
+#endif
 
 /*
 * Here are some links to available implementations of dlfcn and

--------------

Todo: Debug hooks have no registry; only the coreregistry
      is available.  Workaround: store the registry in the
      coreregistry as long as the hook is registered and
      within the hook fetch it from there.

Ciao, ET.

Reply | Threaded
Open this post in threaded view
|

Re: GCed dynamic libraries

Roberto Ierusalimschy
If I understand it correctly, your idea of different registries for
C functions mimics the environment system of Lua functions. Each C
function has its own registry (in the same way that each Lua function
has its own environment).  New functions share the table of its
creator. The main function of a new library creates a new environment
(registry) to be shared by all functions of that library.

Is that understanding correct?

-- Roberto

Reply | Threaded
Open this post in threaded view
|

Re: GCed dynamic libraries

Edgar Toernig
Roberto Ierusalimschy wrote:
>
> If I understand it correctly, your idea of different registries for
> C functions mimics the environment system of Lua functions. Each C
> function has its own registry (in the same way that each Lua function
> has its own environment).  New functions share the table of its
> creator. The main function of a new library creates a new environment
> (registry) to be shared by all functions of that library.
> 
> Is that understanding correct?

Yes.  Except, it's not the main function of a library that creates
the new registry but the library loader.

Hmm... just in case you want to extend the "function environment" to
C-functions by replacing the private registry with private globals:
I still think, that the per Lua-function global table (the "environ-
ment") was a bad idea.  Globals are for communication "between"
modules, the local registries for private storage. The "function
environment" as found in Lua 5 combines these two concepts and the
result is difficult to use and has weird semantics.

So, please, don't do that.

Ciao, ET.

Reply | Threaded
Open this post in threaded view
|

Re: GCed dynamic libraries

Roberto Ierusalimschy
> Hmm... just in case you want to extend the "function environment" to
> C-functions by replacing the private registry with private globals:

Is there any difference? Each function has a pseudo-index for the
registry (the "real" one), a pseudo-index for the global table (the
"real" one), and a pseudo-index for its own private table. We can call
it a "private registry" or a "private environment" (or a "private global
table" :), but the mechanism is the same.

-- Roberto

Reply | Threaded
Open this post in threaded view
|

Re: GCed dynamic libraries

Edgar Toernig
Roberto Ierusalimschy wrote:
>
> > Hmm... just in case you want to extend the "function environment" to
> > C-functions by replacing the private registry with private globals:
> 
> Is there any difference? Each function has a pseudo-index for the
> registry (the "real" one), a pseudo-index for the global table (the
> "real" one), and a pseudo-index for its own private table.

As long as there are those three indexes I'm fine.

[I don't really care how the third index is named - my first name
 was PREGISTRY (private registry).  Only later I changed that to
 REGISTRY and made the old registry (the "real" one) the COREREG-
 ISTRY.  Required less changes to code - nearly all registry users
 need a private registry to make GCing work and there were only
 two places where the core registry was needed.]

What I feared was that you make the global table accessed via
LUA_GLOBALSINDEX the per library private table - that is, only
two special indexes, LUA_REGISTRYINDEX as before, and
LUA_GLOBALSINDEX for a private globals table - like the environ-
ment in Lua functions.

> We can call it a "private registry" or a "private environment" (or a
> "private global table" :), but the mechanism is the same.

The mechanism is the same, but the intend is different.  I don't want
to see lua_setglobal to put something in a private environment nor
would I like to see a script to change my private data because it has
access to or shares my environment.

Ciao, ET.