WIP: Modernize Lua integration with Sol2

src/LuaEngine/ALEIncludes.h

@@ -53,6 +53,8 @@
 #include "ArenaTeam.h"
 #include "WorldSessionMgr.h"
 
+#include <sol/sol.hpp>
+
 typedef Opcodes                 OpcodesList;
 
 /*

src/LuaEngine/ALETemplate.h

@@ -7,14 +7,8 @@
 #ifndef _ALE_TEMPLATE_H
 #define _ALE_TEMPLATE_H
 
-extern "C"
-{
-#include "lua.h"
-#include "lualib.h"
-#include "lauxlib.h"
-};
+#include <sol/sol.hpp>
 #include "LuaEngine.h"
-#include "ALECompat.h"
 #include "ALEUtility.h"
 #include "SharedDefines.h"
 
@@ -260,6 +254,46 @@ class ALETemplate
             return 1;
         }
 
+        // Check if this type is registered with Sol2
+        // Sol2 creates a global table for each usertype
+        lua_getglobal(L, tname);
+        bool isSol2Type = lua_istable(L, -1);
+        lua_pop(L, 1);
+
+        if (isSol2Type)
+        {
+            // Push Sol2 usertype manually to avoid template instantiation with incomplete types
+            // Create userdata that holds just the pointer (Sol2 format for pointer types)
+            T** userdata = static_cast<T**>(lua_newuserdata(L, sizeof(T*)));
+            *userdata = const_cast<T*>(obj);
+
+            // Get Sol2's metatable from the registry
+            // Sol2 stores metatables with prefix "sol." + typename
+            std::string meta_key = std::string("sol.") + tname;
+            lua_getfield(L, LUA_REGISTRYINDEX, meta_key.c_str());
+            if (lua_istable(L, -1))
+            {
+                lua_setmetatable(L, -2);
+                return 1;
+            }
+
+            // Try without prefix as fallback
+            lua_pop(L, 1);
+            lua_getfield(L, LUA_REGISTRYINDEX, tname);
+            if (lua_istable(L, -1))
+            {
+                lua_setmetatable(L, -2);
+                return 1;
+            }
+
+            // Metatable not found
+            ALE_LOG_ERROR("{} Sol2 metatable not found in registry", tname);
+            lua_pop(L, 2);
+            lua_pushnil(L);
+            return 1;
+        }
+
+        // Fall back to old metatable system (for non-migrated types)
         // Create new userdata
         ALEObject** ptrHold = static_cast<ALEObject**>(lua_newuserdata(L, sizeof(ALEObject*)));
         if (!ptrHold)
@@ -286,6 +320,59 @@ class ALETemplate
 
     static T* Check(lua_State* L, int narg, bool error = true)
     {
+        // Check if this type is registered with Sol2
+        lua_getglobal(L, tname);
+        bool isSol2Type = lua_istable(L, -1);
+        lua_pop(L, 1);
+
+        if (isSol2Type)
+        {
+            // Check Sol2 usertype manually to avoid template instantiation with incomplete types
+            // Check if argument is userdata
+            if (!lua_isuserdata(L, narg))
+            {
+                if (error)
+                {
+                    std::string err_msg = std::string(tname) + " expected, got " + lua_typename(L, lua_type(L, narg));
+                    luaL_argerror(L, narg, err_msg.c_str());
+                }
+                return nullptr;
+            }
+
+            // Get its metatable and verify it matches Sol2's metatable for this type
+            if (lua_getmetatable(L, narg))
+            {
+                // Get expected metatable
+                std::string meta_key = std::string("sol.") + tname;
+                lua_getfield(L, LUA_REGISTRYINDEX, meta_key.c_str());
+                if (lua_isnil(L, -1))
+                {
+                    // Try without prefix
+                    lua_pop(L, 1);
+                    lua_getfield(L, LUA_REGISTRYINDEX, tname);
+                }
+
+                // Compare metatables
+                if (lua_rawequal(L, -1, -2))
+                {
+                    lua_pop(L, 2);  // Pop both metatables
+                    // Extract pointer from userdata
+                    T** ptr = static_cast<T**>(lua_touserdata(L, narg));
+                    return ptr ? *ptr : nullptr;
+                }
+
+                lua_pop(L, 2);  // Pop both metatables
+            }
+
+            if (error)
+            {
+                std::string err_msg = std::string(tname) + " expected";
+                luaL_argerror(L, narg, err_msg.c_str());
+            }
+            return nullptr;
+        }
+
+        // Fall back to old system (for non-migrated types)
         ALEObject* ALEObj = ALE::CHECKTYPE(L, narg, tname, error);
         if (!ALEObj)
             return NULL;

src/LuaEngine/LuaEngine.cpp

@@ -8,7 +8,6 @@
 #include "LuaEngine.h"
 #include "BindingMap.h"
 #include "Chat.h"
-#include "ALECompat.h"
 #include "ALEEventMgr.h"
 #include "ALEIncludes.h"
 #include "ALETemplate.h"
@@ -31,15 +30,8 @@
 #include <sys/stat.h>
 #include <unordered_map>
 
-extern "C"
-{
-// Base lua libraries
-#include "lua.h"
-#include "lualib.h"
-#include "lauxlib.h"
+#include <sol/sol.hpp>
 
-// Additional lua libraries
-};
 
 ALE::ScriptList ALE::lua_scripts;
 ALE::ScriptList ALE::lua_extensions;
@@ -183,6 +175,7 @@ ALE::ALE() :
 event_level(0),
 push_counter(0),
 
+luaState(nullptr),
 L(NULL),
 eventMgr(NULL),
 httpManager(),
@@ -238,13 +231,17 @@ void ALE::CloseLua()
 
     DestroyBindStores();
 
-    // Must close lua state after deleting stores and mgr
-    if (L)
-        lua_close(L);
-    L = NULL;
+    if (luaState)
+    {
+        luaState->Shutdown();
+        luaState.reset(); // Destroy the LuaState instance
+    }
+    L = NULL; // Clear legacy pointer
 
     instanceDataRefs.clear();
     continentDataRefs.clear();
+
+    ALE_LOG_DEBUG("[ALE]: Lua state closed");
 }
 
 void ALE::OpenLua()
@@ -255,37 +252,25 @@ void ALE::OpenLua()
         return;
     }
 
-    L = luaL_newstate();
+    luaState = std::make_unique<LuaState>();
+    luaState->Initialize();
+
+    L = luaState->GetLuaState();
 
     lua_pushlightuserdata(L, this);
     lua_setfield(L, LUA_REGISTRYINDEX, ALE_STATE_PTR);
 
     CreateBindStores();
 
-    // open base lua libraries
-    luaL_openlibs(L);
-
-    // open additional lua libraries
-
-    // Register methods and functions
+    // Register ALE methods and functions (types, hooks, etc.)
     RegisterFunctions(this);
 
     // Set lua require folder paths (scripts folder structure)
-    lua_getglobal(L, "package");
-    lua_pushstring(L, GetRequirePath().c_str());
-    lua_setfield(L, -2, "path");
-    lua_pushstring(L, GetRequireCPath().c_str());
-    lua_setfield(L, -2, "cpath");
-
-    // Set package.loaders loader for precompiled scripts
-    lua_getfield(L, -1, "loaders");
-    if (lua_isnil(L, -1)) {
-        // Lua 5.2+ uses searchers instead of loaders
-        lua_pop(L, 1);
-        lua_getfield(L, -1, "searchers");
-    }
+    // Note: LuaState already opened standard libraries
+    luaState->SetRequirePath(GetRequirePath());
+    luaState->SetRequireCPath(GetRequireCPath());
 
-    lua_pop(L, 1);
+    ALE_LOG_DEBUG("[ALE]: Lua state opened with Sol2");
 }
 
 void ALE::CreateBindStores()
@@ -446,7 +431,7 @@ bool ALE::CompileScriptToGlobalCache(const std::string& filepath)
     BytecodeWriter writer;
     writer.buffer = &cacheEntry.bytecode;
 
-    int dumpResult = lua_dump(tempL, BytecodeWriter::writer, &writer);
+    int dumpResult = lua_dump(tempL, BytecodeWriter::writer, &writer, 0);
     if (dumpResult != LUA_OK || cacheEntry.bytecode.empty())
     {
         globalBytecodeCache.erase(filepath);
@@ -504,7 +489,7 @@ bool ALE::CompileMoonScriptToGlobalCache(const std::string& filepath)
     BytecodeWriter writer;
     writer.buffer = &cacheEntry.bytecode;
 
-    int dumpResult = lua_dump(tempL, BytecodeWriter::writer, &writer);
+    int dumpResult = lua_dump(tempL, BytecodeWriter::writer, &writer, 0);
     if (dumpResult != LUA_OK || cacheEntry.bytecode.empty())
     {
         globalBytecodeCache.erase(filepath);
@@ -924,7 +909,7 @@ void ALE::Push(lua_State* luastate, const int i)
 }
 void ALE::Push(lua_State* luastate, const unsigned int u)
 {
-    lua_pushunsigned(luastate, u);
+    lua_pushinteger(luastate, static_cast<lua_Integer>(u));
 }
 void ALE::Push(lua_State* luastate, const double d)
 {