From 7b38a5c7e3d26ec1d17387211451242a021c8cd4 Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Wed, 24 Jun 2026 18:50:21 +0300 Subject: [PATCH 1/6] Implement JVM HotSwap via Java Agent --- jjava-distro/pom.xml | 2 + .../dflib/jjava/distro/KernelExecutionIT.java | 19 +++++++++ .../jjava/kernel/execution/CodeEvaluator.java | 6 ++- .../jjava/kernel/execution/HotswapAgent.java | 32 ++++++++++++++ .../kernel/execution/JJavaLoaderDelegate.java | 42 ++++++++++++++++++- 5 files changed, 98 insertions(+), 3 deletions(-) create mode 100644 jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/HotswapAgent.java diff --git a/jjava-distro/pom.xml b/jjava-distro/pom.xml index 8ba7da4..419bfd1 100644 --- a/jjava-distro/pom.xml +++ b/jjava-distro/pom.xml @@ -132,6 +132,8 @@ org.dflib.jjava.distro.JJava + org.dflib.jjava.kernel.execution.HotswapAgent + true diff --git a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java index 2768ba5..8ea204a 100644 --- a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java +++ b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java @@ -22,4 +22,23 @@ public void variableSurvivesLaterImports() throws Exception { assertEquals(0, snippetResult.getExitCode(), snippetResult.getStdout()); assertThat(snippetResult.getStdout(), not(containsString("|"))); } + + /** + * @see #119 + */ + @Test + public void variableRedeclarationUsesLatestValue() throws Exception { + String snippet = String.join("\n", + "var v = \"first\";", + "v", + "var v = \"second\";", + "v" + ); + + Container.ExecResult snippetResult = executeInKernel(snippet); + + assertEquals(0, snippetResult.getExitCode(), snippetResult.getStdout()); + assertThat(snippetResult.getStdout(), containsString("Out[2]: first")); + assertThat(snippetResult.getStdout(), containsString("Out[4]: second")); + } } diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/CodeEvaluator.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/CodeEvaluator.java index 9b93b85..7e0b6fd 100644 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/CodeEvaluator.java +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/CodeEvaluator.java @@ -169,9 +169,11 @@ private void dropSnippet(JShell shell, Snippet snippet) { // snippet.classFullName() returns name of a wrapper class created for a snippet String className = snippetClassName(snippet); // check that this class is not used by other snippets - if (shell.snippets() + boolean isUnused = shell.snippets() + .filter(s -> shell.status(s).isActive()) .map(this::snippetClassName) - .noneMatch(className::equals)) { + .noneMatch(className::equals); + if (isUnused) { execControl.unloadClass(className); } } diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/HotswapAgent.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/HotswapAgent.java new file mode 100644 index 0000000..da9852c --- /dev/null +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/HotswapAgent.java @@ -0,0 +1,32 @@ +package org.dflib.jjava.kernel.execution; + +import java.lang.instrument.Instrumentation; + +/** + * Captures the JVM {@link Instrumentation} instance so that {@link JJavaLoaderDelegate} can perform + * in-place (HotSwap) redefinition of JShell snippet classes. + */ +public final class HotswapAgent { + + private static volatile Instrumentation instrumentation; + + private HotswapAgent() { + } + + public static void premain(String agentArgs, Instrumentation inst) { + instrumentation = inst; + } + + public static void agentmain(String agentArgs, Instrumentation inst) { + instrumentation = inst; + } + + public static Instrumentation instrumentation() { + return instrumentation; + } + + public static boolean canRedefine() { + Instrumentation inst = instrumentation; + return inst != null && inst.isRedefineClassesSupported(); + } +} diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaLoaderDelegate.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaLoaderDelegate.java index 8f7e39c..9d404fc 100644 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaLoaderDelegate.java +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaLoaderDelegate.java @@ -4,11 +4,16 @@ import jdk.jshell.spi.ExecutionControl; import org.dflib.jjava.jupyter.kernel.util.PathsHandler; +import java.lang.instrument.ClassDefinition; +import java.lang.instrument.Instrumentation; +import java.lang.instrument.UnmodifiableClassException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; import java.nio.file.Path; import java.security.CodeSource; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -53,8 +58,41 @@ public void load(ExecutionControl.ClassBytecodes[] cbcs) throws ExecutionControl @Override public void classesRedefined(ExecutionControl.ClassBytecodes[] cbcs) { + List redefinitions = new ArrayList<>(cbcs.length); for (ExecutionControl.ClassBytecodes cbc : cbcs) { declaredClasses.put(cbc.name(), cbc.bytecodes()); + + Class loaded = loadedClasses.get(cbc.name()); + if (loaded != null) { + redefinitions.add(new ClassDefinition(loaded, cbc.bytecodes())); + } + } + redefineLoadedClasses(redefinitions); + } + + /** + * Perform in-place (HotSwap) redefinition of the given classes. This mirrors how the {@code jshell} tool + * redefines classes in its remote JVM via JDI, preserving {@link Class} identity and the values of + * static fields (i.e. JShell variable values). + */ + private void redefineLoadedClasses(List redefinitions) { + if (redefinitions.isEmpty() || !HotswapAgent.canRedefine()) { + return; + } + + Instrumentation instrumentation = HotswapAgent.instrumentation(); + try { + // Redefine all classes at once to handle interdependent changes to more than one class quickly. + instrumentation.redefineClasses(redefinitions.toArray(new ClassDefinition[0])); + } catch (UnsupportedOperationException | UnmodifiableClassException | ClassNotFoundException e) { + // Schema-changing redefinition in some class: try to redefine each class separately. + for (ClassDefinition redefinition : redefinitions) { + try { + instrumentation.redefineClasses(redefinition); + } catch (UnsupportedOperationException | UnmodifiableClassException | ClassNotFoundException ee) { + // Schema-changing redefinition: keep the new bytecode to declaredClasses. + } + } } } @@ -120,7 +158,9 @@ protected Class findClass(String name) throws ClassNotFoundException { return super.findClass(name); } try { - return super.defineClass(name, data, 0, data.length, (CodeSource) null); + Class klass = super.defineClass(name, data, 0, data.length, (CodeSource) null); + loadedClasses.put(name, klass); + return klass; } catch (LinkageError er) { // rethrow as ClassNotFoundException to let the caller properly handle this case // this error could be thrown in some cases (like static method signature change) From a63d6dfe85df348f47bac067353ef3d4ebe1d4bf Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Fri, 3 Jul 2026 13:42:54 +0300 Subject: [PATCH 2/6] Revert "Implement JVM HotSwap via Java Agent" This reverts commit 7b38a5c7e3d26ec1d17387211451242a021c8cd4. --- jjava-distro/pom.xml | 2 - .../dflib/jjava/distro/KernelExecutionIT.java | 19 --------- .../jjava/kernel/execution/CodeEvaluator.java | 6 +-- .../jjava/kernel/execution/HotswapAgent.java | 32 -------------- .../kernel/execution/JJavaLoaderDelegate.java | 42 +------------------ 5 files changed, 3 insertions(+), 98 deletions(-) delete mode 100644 jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/HotswapAgent.java diff --git a/jjava-distro/pom.xml b/jjava-distro/pom.xml index 419bfd1..8ba7da4 100644 --- a/jjava-distro/pom.xml +++ b/jjava-distro/pom.xml @@ -132,8 +132,6 @@ org.dflib.jjava.distro.JJava - org.dflib.jjava.kernel.execution.HotswapAgent - true diff --git a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java index 8ea204a..2768ba5 100644 --- a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java +++ b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java @@ -22,23 +22,4 @@ public void variableSurvivesLaterImports() throws Exception { assertEquals(0, snippetResult.getExitCode(), snippetResult.getStdout()); assertThat(snippetResult.getStdout(), not(containsString("|"))); } - - /** - * @see #119 - */ - @Test - public void variableRedeclarationUsesLatestValue() throws Exception { - String snippet = String.join("\n", - "var v = \"first\";", - "v", - "var v = \"second\";", - "v" - ); - - Container.ExecResult snippetResult = executeInKernel(snippet); - - assertEquals(0, snippetResult.getExitCode(), snippetResult.getStdout()); - assertThat(snippetResult.getStdout(), containsString("Out[2]: first")); - assertThat(snippetResult.getStdout(), containsString("Out[4]: second")); - } } diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/CodeEvaluator.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/CodeEvaluator.java index 7e0b6fd..9b93b85 100644 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/CodeEvaluator.java +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/CodeEvaluator.java @@ -169,11 +169,9 @@ private void dropSnippet(JShell shell, Snippet snippet) { // snippet.classFullName() returns name of a wrapper class created for a snippet String className = snippetClassName(snippet); // check that this class is not used by other snippets - boolean isUnused = shell.snippets() - .filter(s -> shell.status(s).isActive()) + if (shell.snippets() .map(this::snippetClassName) - .noneMatch(className::equals); - if (isUnused) { + .noneMatch(className::equals)) { execControl.unloadClass(className); } } diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/HotswapAgent.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/HotswapAgent.java deleted file mode 100644 index da9852c..0000000 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/HotswapAgent.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.dflib.jjava.kernel.execution; - -import java.lang.instrument.Instrumentation; - -/** - * Captures the JVM {@link Instrumentation} instance so that {@link JJavaLoaderDelegate} can perform - * in-place (HotSwap) redefinition of JShell snippet classes. - */ -public final class HotswapAgent { - - private static volatile Instrumentation instrumentation; - - private HotswapAgent() { - } - - public static void premain(String agentArgs, Instrumentation inst) { - instrumentation = inst; - } - - public static void agentmain(String agentArgs, Instrumentation inst) { - instrumentation = inst; - } - - public static Instrumentation instrumentation() { - return instrumentation; - } - - public static boolean canRedefine() { - Instrumentation inst = instrumentation; - return inst != null && inst.isRedefineClassesSupported(); - } -} diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaLoaderDelegate.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaLoaderDelegate.java index 9d404fc..8f7e39c 100644 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaLoaderDelegate.java +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaLoaderDelegate.java @@ -4,16 +4,11 @@ import jdk.jshell.spi.ExecutionControl; import org.dflib.jjava.jupyter.kernel.util.PathsHandler; -import java.lang.instrument.ClassDefinition; -import java.lang.instrument.Instrumentation; -import java.lang.instrument.UnmodifiableClassException; import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; import java.nio.file.Path; import java.security.CodeSource; -import java.util.ArrayList; -import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -58,41 +53,8 @@ public void load(ExecutionControl.ClassBytecodes[] cbcs) throws ExecutionControl @Override public void classesRedefined(ExecutionControl.ClassBytecodes[] cbcs) { - List redefinitions = new ArrayList<>(cbcs.length); for (ExecutionControl.ClassBytecodes cbc : cbcs) { declaredClasses.put(cbc.name(), cbc.bytecodes()); - - Class loaded = loadedClasses.get(cbc.name()); - if (loaded != null) { - redefinitions.add(new ClassDefinition(loaded, cbc.bytecodes())); - } - } - redefineLoadedClasses(redefinitions); - } - - /** - * Perform in-place (HotSwap) redefinition of the given classes. This mirrors how the {@code jshell} tool - * redefines classes in its remote JVM via JDI, preserving {@link Class} identity and the values of - * static fields (i.e. JShell variable values). - */ - private void redefineLoadedClasses(List redefinitions) { - if (redefinitions.isEmpty() || !HotswapAgent.canRedefine()) { - return; - } - - Instrumentation instrumentation = HotswapAgent.instrumentation(); - try { - // Redefine all classes at once to handle interdependent changes to more than one class quickly. - instrumentation.redefineClasses(redefinitions.toArray(new ClassDefinition[0])); - } catch (UnsupportedOperationException | UnmodifiableClassException | ClassNotFoundException e) { - // Schema-changing redefinition in some class: try to redefine each class separately. - for (ClassDefinition redefinition : redefinitions) { - try { - instrumentation.redefineClasses(redefinition); - } catch (UnsupportedOperationException | UnmodifiableClassException | ClassNotFoundException ee) { - // Schema-changing redefinition: keep the new bytecode to declaredClasses. - } - } } } @@ -158,9 +120,7 @@ protected Class findClass(String name) throws ClassNotFoundException { return super.findClass(name); } try { - Class klass = super.defineClass(name, data, 0, data.length, (CodeSource) null); - loadedClasses.put(name, klass); - return klass; + return super.defineClass(name, data, 0, data.length, (CodeSource) null); } catch (LinkageError er) { // rethrow as ClassNotFoundException to let the caller properly handle this case // this error could be thrown in some cases (like static method signature change) From 3d88ae79b0593e6061ace072240f1bba4e81f145 Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Fri, 3 Jul 2026 13:43:07 +0300 Subject: [PATCH 3/6] Revert "Downgrade test library version" This reverts commit 7aa650d95633d3b03ca702dca30fdb7a066c23d0. --- jjava-distro/src/test/resources/nullifying_import.jshell | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jjava-distro/src/test/resources/nullifying_import.jshell b/jjava-distro/src/test/resources/nullifying_import.jshell index 56a2ac9..fe2a6fd 100644 --- a/jjava-distro/src/test/resources/nullifying_import.jshell +++ b/jjava-distro/src/test/resources/nullifying_import.jshell @@ -1,5 +1,5 @@ -%maven com.fasterxml.jackson.core:jackson-databind:2.21.2 -import com.fasterxml.jackson.databind.*; -var om = new ObjectMapper().findAndRegisterModules(); -import com.fasterxml.jackson.databind.node.*; import com.fasterxml.jackson.databind.type.*; +%maven tools.jackson.core:jackson-databind:3.1.2 +import tools.jackson.databind.*; import tools.jackson.databind.json.*; +var om = JsonMapper.builder().findAndAddModules().build(); +import tools.jackson.databind.node.*; import tools.jackson.databind.type.*; om.getClass().getName() From bd87a30e3009e7822c43a0ca0a105a55d767bfb1 Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Fri, 3 Jul 2026 13:43:12 +0300 Subject: [PATCH 4/6] Revert "Prevent variable nullification on later imports #119" This reverts commit ea06a1ce4e33c58b4286c799b5656205f4a812cd. --- .../dflib/jjava/distro/KernelExecutionIT.java | 25 ------------------- .../test/resources/nullifying_import.jshell | 5 ---- .../execution/JJavaExecutionControl.java | 5 ---- 3 files changed, 35 deletions(-) delete mode 100644 jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java delete mode 100644 jjava-distro/src/test/resources/nullifying_import.jshell diff --git a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java deleted file mode 100644 index 2768ba5..0000000 --- a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.dflib.jjava.distro; - -import org.junit.jupiter.api.Test; -import org.testcontainers.containers.Container; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.not; -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class KernelExecutionIT extends ContainerizedKernelCase { - - /** - * @see #119 - */ - @Test - public void variableSurvivesLaterImports() throws Exception { - String snippet = "%load " + CONTAINER_RESOURCES + "/nullifying_import.jshell"; - - Container.ExecResult snippetResult = executeInKernel(snippet); - - assertEquals(0, snippetResult.getExitCode(), snippetResult.getStdout()); - assertThat(snippetResult.getStdout(), not(containsString("|"))); - } -} diff --git a/jjava-distro/src/test/resources/nullifying_import.jshell b/jjava-distro/src/test/resources/nullifying_import.jshell deleted file mode 100644 index fe2a6fd..0000000 --- a/jjava-distro/src/test/resources/nullifying_import.jshell +++ /dev/null @@ -1,5 +0,0 @@ -%maven tools.jackson.core:jackson-databind:3.1.2 -import tools.jackson.databind.*; import tools.jackson.databind.json.*; -var om = JsonMapper.builder().findAndAddModules().build(); -import tools.jackson.databind.node.*; import tools.jackson.databind.type.*; -om.getClass().getName() diff --git a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaExecutionControl.java b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaExecutionControl.java index 1bbf21e..f85b735 100644 --- a/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaExecutionControl.java +++ b/jjava-kernel/src/main/java/org/dflib/jjava/kernel/execution/JJavaExecutionControl.java @@ -76,11 +76,6 @@ public void stop() { executor.shutdownNow(); } - @Override - public void redefine(ClassBytecodes[] cbcs) { - loaderDelegate.classesRedefined(cbcs); - } - public Object takeResult(String id) { Object result = this.results.remove(id); if (result == null) { From abcf640cc0197f647984af64753a1a09a61a801d Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Mon, 6 Jul 2026 15:50:50 +0300 Subject: [PATCH 5/6] Revert "Adding an import in a later cell nullifies variables defined in earlier cells #119" This reverts commit fa54ca42 --- RELEASE-NOTES.md | 1 - 1 file changed, 1 deletion(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 0b91bbd..49e6152 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -3,7 +3,6 @@ * #114 %load magic does not recognize notebooks * #115 "%load" magic should not be adding extensions to file names * #118 Exception stack trace truncated — cause chain not fully reported -* #119 Variable nullification on later imports ## 1.0-a7 From 88e3b90991488536d38716754697da7c4269999c Mon Sep 17 00:00:00 2001 From: Mikhail Dzianishchyts Date: Fri, 3 Jul 2026 13:54:21 +0300 Subject: [PATCH 6/6] Keep related test cases --- .../dflib/jjava/distro/KernelExecutionIT.java | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java diff --git a/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java new file mode 100644 index 0000000..d61bd0e --- /dev/null +++ b/jjava-distro/src/test/java/org/dflib/jjava/distro/KernelExecutionIT.java @@ -0,0 +1,51 @@ +package org.dflib.jjava.distro; + +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.Container; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.matchesRegex; +import static org.hamcrest.Matchers.not; +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class KernelExecutionIT extends ContainerizedKernelCase { + + /** + * @see #119 + */ + @Test + public void variableSurvivesLaterImports() throws Exception { + String snippet = String.join("\n", + "%maven com.fasterxml.jackson.core:jackson-databind:2.21.2", + "import com.fasterxml.jackson.databind.*;", + "var om = new ObjectMapper().findAndRegisterModules();", + "import com.fasterxml.jackson.databind.node.*; import com.fasterxml.jackson.databind.type.*;", + "om.getClass().getName()" + ); + Container.ExecResult snippetResult = executeInKernel(snippet); + + assertEquals(0, snippetResult.getExitCode(), snippetResult.getStdout()); + assertThat(snippetResult.getStdout(), containsString("com.fasterxml.jackson.databind.ObjectMapper")); + assertThat(snippetResult.getStdout(), not(matchesRegex(".*(at |\\|).*"))); + } + + /** + * @see #119 + */ + @Test + public void variableRedeclarationUsesLatestValue() throws Exception { + String snippet = String.join("\n", + "int v = 1;", + "v", + "float v = 2.4f;", + "v" + ); + + Container.ExecResult snippetResult = executeInKernel(snippet); + + assertEquals(0, snippetResult.getExitCode(), snippetResult.getStdout()); + assertThat(snippetResult.getStdout(), containsString("Out[2]: 1")); + assertThat(snippetResult.getStdout(), containsString("Out[4]: 2.4")); + } +}