Update hot reload finalization

The final libxposed API 102 hot reload contract makes the old generation responsible for retiring its own Java and native resources before returning true from onHotReloading. The framework also does not call UnregisterNatives, JNI_OnUnload, or dlclose during hot reload, so it must avoid keeping stale framework-owned references after the new generation is handed off.

This updates the libxposed submodule pointers to the finalized API 102 revisions and aligns Vector's implementation with that contract. onHotReloading exceptions now remain diagnostic failures instead of being collapsed into a false rejection, saved state is checked for nested old-classloader objects, old entries are detached after the new state is committed, and failed pre-commit reload attempts clean up only newly-created hooks while preserving the old hook handles.

The daemon-side service status mapping is also updated for the final service API: the removed PROP_RT_HOT_RELOAD bit is no longer advertised, HOT_RELOAD_SUCCEEDED is used instead of the draft HOT_RELOAD_SUCCESS name, unsupported targets report HOT_RELOAD_UNSUPPORTED, concurrent reloads report HOT_RELOAD_IN_PROGRESS, and dead target processes report HOT_RELOAD_PROCESS_DIED.

Documentation References:

libxposed XposedModuleInterface hot reload callbacks:

https://libxposed.github.io/api/io/github/libxposed/api/XposedModuleInterface.html#onHotReloaded(io.github.libxposed.api.XposedModuleInterface.HotReloadedParam)

Author: HSSkyBoy

daemon/src/main/kotlin/org/matrix/vector/daemon/ipc/ApplicationService.kt

@@ -27,6 +27,12 @@ const val DEX_TRANSACTION_CODE =
 const val OBFUSCATION_MAP_TRANSACTION_CODE =
     ('_'.code shl 24) or ('O'.code shl 16) or ('B'.code shl 8) or 'F'.code
 
+internal class HotReloadInProgressException(message: String) : IllegalStateException(message)
+
+internal class HotReloadProcessDiedException(message: String) : IllegalStateException(message)
+
+internal class HotReloadUnsupportedException(message: String) : IllegalStateException(message)
+
 object ApplicationService : ILSPApplicationService.Stub() {
 
   data class ProcessKey(val uid: Int, val pid: Int)
@@ -217,7 +223,7 @@ object ApplicationService : ILSPApplicationService.Stub() {
       throw SecurityException("Target $targetId does not belong to ${module.packageName}")
     }
     if (target.state == HookedProcess.TARGET_STATE_RELOADING) {
-      throw IllegalStateException("Target $targetId is already reloading")
+      throw HotReloadInProgressException("Target $targetId is already reloading")
     }
 
     target.state = HookedProcess.TARGET_STATE_RELOADING
@@ -227,9 +233,11 @@ object ApplicationService : ILSPApplicationService.Stub() {
           target.state = HookedProcess.TARGET_STATE_UP_TO_DATE
         }
         .onFailure {
-          target.state =
-              if (target.target.asBinder().isBinderAlive) HookedProcess.TARGET_STATE_FAILED
-              else HookedProcess.TARGET_STATE_FAILED
+          if (!target.target.asBinder().isBinderAlive) {
+            hotReloadTargets.remove(target.id, target)
+            throw HotReloadProcessDiedException("Target process died before hot reload completed")
+          }
+          target.state = HookedProcess.TARGET_STATE_FAILED
           throw it
         }
   }

daemon/src/main/kotlin/org/matrix/vector/daemon/ipc/ModuleService.kt

@@ -112,9 +112,6 @@ class ModuleService(private val loadedModule: Module) : IXposedService.Stub() {
   override fun getFrameworkProperties(): Long {
     ensureModule()
     var prop = IXposedService.PROP_CAP_SYSTEM or IXposedService.PROP_CAP_REMOTE
-    if (loadedModule.file.moduleClassNames.size == 1) {
-      prop = prop or IXposedService.PROP_RT_HOT_RELOAD
-    }
     if (ConfigCache.state.isDexObfuscateEnabled)
         prop = prop or IXposedService.PROP_RT_API_PROTECTION
     return prop
@@ -158,18 +155,26 @@ class ModuleService(private val loadedModule: Module) : IXposedService.Stub() {
     ensureModule()
     runCatching {
           if (loadedModule.file.moduleClassNames.size != 1) {
-            throw SecurityException("Hot reload requires exactly one Java entry class")
+            throw HotReloadUnsupportedException("Hot reload requires exactly one Java entry class")
           }
           val latest =
               ConfigCache.getModuleByPackage(loadedModule.packageName)
-                  ?: throw SecurityException("Module ${loadedModule.packageName} is not enabled")
+                  ?: throw HotReloadUnsupportedException(
+                      "Module ${loadedModule.packageName} is not enabled")
+          if (latest.file.moduleClassNames.size != 1) {
+            throw HotReloadUnsupportedException(
+                "Hot reload requires exactly one Java entry class")
+          }
           ApplicationService.hotReloadTarget(targetId, latest, data)
-          callback?.onHotReloadResult(IXposedService.HOT_RELOAD_SUCCESS, null)
+          callback?.onHotReloadResult(IXposedService.HOT_RELOAD_SUCCEEDED, null)
         }
         .onFailure { throwable ->
+          if (throwable is SecurityException) throw throwable
           val status =
               when (throwable) {
-                is IllegalStateException -> IXposedService.HOT_RELOAD_IN_PROGRESS
+                is HotReloadInProgressException -> IXposedService.HOT_RELOAD_IN_PROGRESS
+                is HotReloadProcessDiedException -> IXposedService.HOT_RELOAD_PROCESS_DIED
+                is HotReloadUnsupportedException -> IXposedService.HOT_RELOAD_UNSUPPORTED
                 else -> IXposedService.HOT_RELOAD_FAILED
               }
           callback?.onHotReloadResult(status, throwable.message)

services/libxposed

@@ -9,12 +9,16 @@ import io.github.libxposed.api.XposedModuleInterface.HotReloadedParam
 import io.github.libxposed.api.XposedModuleInterface.HotReloadingParam
 import io.github.libxposed.api.XposedModuleInterface.ModuleLoadedParam
 import java.io.File
+import java.lang.reflect.Array
+import java.util.Collections
+import java.util.IdentityHashMap
 import org.lsposed.lspd.models.Module
 import org.lsposed.lspd.util.Utils.Log
 import org.matrix.vector.impl.VectorContext
 import org.matrix.vector.impl.VectorLifecycleManager
 import org.matrix.vector.impl.hooks.freezeHooks
 import org.matrix.vector.impl.hooks.getActiveHookHandles
+import org.matrix.vector.impl.hooks.unhookAllModuleHooks
 import org.matrix.vector.impl.hooks.unfreezeHooks
 import org.matrix.vector.impl.utils.VectorModuleClassLoader
 import org.matrix.vector.nativebridge.NativeAPI
@@ -118,14 +122,15 @@ object VectorModuleManager {
             throw IllegalArgumentException("Hot reload requires exactly one Java entry class")
         }
 
+        val oldClassLoaders =
+            oldState.entries.mapNotNullTo(mutableSetOf<ClassLoader>()) { it.javaClass.classLoader }
         var savedInstanceState: Any? = null
         val reloadingParam =
             object : HotReloadingParam {
                 override fun getExtras(): Bundle? = extras
 
                 override fun setSavedInstanceState(outState: Any?) {
-                    val loader = outState?.javaClass?.classLoader
-                    if (loader != null && oldState.entries.any { it.javaClass.classLoader == loader }) {
+                    if (containsOldClassLoaderObject(outState, oldClassLoaders)) {
                         throw IllegalArgumentException(
                             "Saved state must not be created by the old module classloader"
                         )
@@ -134,21 +139,24 @@ object VectorModuleManager {
                 }
             }
 
-        val allowReload =
-            oldState.entries.fold(true) { allowed, entry ->
-                allowed && runCatching { entry.onHotReloading(reloadingParam) }
+        var allowReload = true
+        oldState.entries.forEach { entry ->
+            if (!allowReload) return@forEach
+            allowReload =
+                runCatching { entry.onHotReloading(reloadingParam) }
                     .onFailure { Log.e(TAG, "Error in onHotReloading for ${module.packageName}", it) }
-                    .getOrDefault(false)
-            }
+                    .getOrThrow()
+        }
         if (!allowReload) {
-            throw IllegalStateException("Module ${module.packageName} rejected hot reload")
+            Log.d(TAG, "Module ${module.packageName} rejected hot reload")
+            throw IllegalStateException()
         }
 
-        freezeHooks(module.packageName, oldState.entries.mapNotNull { it.javaClass.classLoader })
+        freezeHooks(module.packageName, oldClassLoaders)
         val oldHandles = getActiveHookHandles(module.packageName)
+        var newStateCommitted = false
+        var newEntries: List<XposedModule> = emptyList()
         try {
-            oldState.entries.forEach(VectorLifecycleManager::detach)
-
             val librarySearchPath = buildLibrarySearchPath(module)
             val moduleClassLoader =
                 VectorModuleClassLoader.loadApk(
@@ -163,7 +171,11 @@ object VectorModuleManager {
                     applicationInfo = module.applicationInfo,
                     service = module.service,
                 )
-            val newEntries = instantiateEntries(module, moduleClassLoader, vectorContext)
+            newEntries = instantiateEntries(module, moduleClassLoader, vectorContext)
+            if (newEntries.size != module.file.moduleClassNames.size) {
+                throw IllegalStateException("Failed to instantiate hot reload entry")
+            }
+
             val param =
                 object : HotReloadedParam {
                     override fun isSystemServer(): Boolean = oldState.isSystemServer
@@ -176,13 +188,24 @@ object VectorModuleManager {
 
                     override fun getOldHookHandles(): List<XposedInterface.HookHandle> = oldHandles
                 }
+            moduleStates[module.packageName] =
+                ModuleState(module, oldState.processName, oldState.isSystemServer, newEntries)
+            newStateCommitted = true
+            // Keep oldState strongly reachable until callbacks finish, but stop lifecycle dispatch.
+            oldState.entries.forEach(VectorLifecycleManager::detach)
             newEntries.forEach { entry ->
                 VectorLifecycleManager.activeModules.add(entry)
-                entry.onHotReloaded(param)
+                runCatching { entry.onHotReloaded(param) }
+                    .onFailure { Log.e(TAG, "Error in onHotReloaded for ${module.packageName}", it) }
+                    .getOrThrow()
             }
-            moduleStates[module.packageName] =
-                ModuleState(module, oldState.processName, oldState.isSystemServer, newEntries)
         } finally {
+            if (newStateCommitted) {
+                oldState.entries.forEach(VectorLifecycleManager::detach)
+            } else {
+                newEntries.forEach(VectorLifecycleManager::detach)
+                unhookAllModuleHooks(module.packageName, oldHandles.toSet())
+            }
             unfreezeHooks(module.packageName)
         }
     }
@@ -225,4 +248,39 @@ object VectorModuleManager {
         return entries
     }
 
+    @Suppress("DEPRECATION")
+    private fun containsOldClassLoaderObject(
+        value: Any?,
+        oldClassLoaders: Set<ClassLoader>,
+        seen: MutableSet<Any> = Collections.newSetFromMap(IdentityHashMap<Any, Boolean>()),
+    ): Boolean {
+        if (value == null || !seen.add(value)) return false
+        if (value is ClassLoader && value in oldClassLoaders) return true
+        if (value is Class<*> && value.classLoader in oldClassLoaders) return true
+        if (value.javaClass.classLoader in oldClassLoaders) return true
+        if (value is Bundle) {
+            return value.keySet().any { key ->
+                runCatching { containsOldClassLoaderObject(value.get(key), oldClassLoaders, seen) }
+                    .getOrDefault(true)
+            }
+        }
+        if (value is Map<*, *>) {
+            return value.entries.any {
+                containsOldClassLoaderObject(it.key, oldClassLoaders, seen) ||
+                    containsOldClassLoaderObject(it.value, oldClassLoaders, seen)
+            }
+        }
+        if (value is Iterable<*>) {
+            return value.any { containsOldClassLoaderObject(it, oldClassLoaders, seen) }
+        }
+        if (value.javaClass.isArray) {
+            for (index in 0 until Array.getLength(value)) {
+                if (containsOldClassLoaderObject(Array.get(value, index), oldClassLoaders, seen)) {
+                    return true
+                }
+            }
+        }
+        return false
+    }
+
 }

xposed/libxposed

@@ -130,8 +130,18 @@ internal fun unfreezeHooks(modulePackageName: String) {
     HookRegistry.unfreeze(modulePackageName)
 }
 
+internal fun unhookAllModuleHooks(
+    modulePackageName: String,
+    except: Set<HookHandle> = emptySet(),
+) {
+    val excludedRecords = except.mapNotNull { (it as? VectorHookHandle)?.record }.toSet()
+    HookRegistry.allRecords
+        .filter { it.modulePackageName == modulePackageName && it !in excludedRecords }
+        .forEach(::uninstallRecord)
+}
+
 private class VectorHookHandle(
-    private val record: VectorHookRecord,
+    val record: VectorHookRecord,
     private val hookKey: HookKey?,
 ) : HookHandle {
     override fun getExecutable(): Executable = record.executable
@@ -188,6 +198,9 @@ private fun installRecord(record: VectorHookRecord) {
 private fun uninstallRecord(record: VectorHookRecord): Boolean {
     if (!record.deactivate()) return false
     HookBridge.unhookMethod(true, record.executable, record)
+    record.id?.let { id ->
+        HookRegistry.records.remove(HookKey(record.modulePackageName, record.executable, id), record)
+    }
     HookRegistry.allRecords.remove(record)
     return true
 }

xposed/src/main/kotlin/org/matrix/vector/impl/core/VectorModuleManager.kt

@@ -7,5 +7,7 @@
 -keepclasseswithmembers class org.matrix.vector.service.BridgeService {
     public static boolean *(android.os.IBinder, int, long, long, int);
 }
+
+-dontwarn io.github.libxposed.annotation.**
 -repackageclasses
 -allowaccessmodification