Class retransformation doesn't work for dynamic agent on Java 11

It seems like dynamic retransformations of classes work for me only on Java 8 but not on Java 11. In the latter case, I get exceptions from javassist about different not found standard Java classes, for example, the ones directly referenced by me or even from the signature of the method-to-transform.

What should I do to fix that on Java 11? I want to transform classes dynamically here too.

For illustration purposes, I've created a repro file. Here I retransform two classes: one is my own, another is system. I've created both agentmain and premain to compare. Dynamic variant is executed when a main argument is passed to the app (I pass it as just "o"). After the retransformation, I call two methods (of my own class and of the system one). When the transformation is successful, I receive additional logging ("hi-" and "scaled!").

// MyMain.java
package mypackage;

import com.sun.tools.attach.VirtualMachine;
import javassist.*;
import javassist.bytecode.AccessFlag;
import sun.java2d.SunGraphics2D;
import sun.java2d.SurfaceData;

import javax.swing.*;
import java.awt.*;
import java.awt.geom.AffineTransform;
import java.awt.image.BufferedImage;
import java.io.IOException;
import java.lang.instrument.ClassFileTransformer;
import java.lang.instrument.IllegalClassFormatException;
import java.lang.instrument.Instrumentation;
import java.lang.instrument.UnmodifiableClassException;
import java.lang.management.ManagementFactory;
import java.security.ProtectionDomain;

public class MyMain {

    public static void premain(String args, Instrumentation inst) {
        System.out.println("premain start");
        inst.addTransformer(new MyFormer(), true);
        try {
            inst.retransformClasses(MyMain.class);
        } catch (UnmodifiableClassException e) {
            e.printStackTrace();
        }
        System.out.println("premain end");
    }

    public static void agentmain(String args, Instrumentation inst) {
        System.out.println("agentmain start");
        inst.addTransformer(new MyFormer(), true);
        try {
            inst.retransformClasses(MyMain.class);
        } catch (UnmodifiableClassException e) {
            e.printStackTrace();
        }
        System.out.println("agentmain end");
    }

    public static void main(String[] args) {
        if (args.length > 0) {
            attachToThisVm();
        }

        Frame f = new JFrame();
        f.setVisible(true);

        System.out.println(new MyMain().hi());

        SunGraphics2D system = new SunGraphics2D(SurfaceData.getPrimarySurfaceData(new BufferedImage(10, 10, BufferedImage.TYPE_INT_ARGB)), Color.BLACK, Color.WHITE, Font.getFont("System"));
        system.drawRenderedImage(null,new AffineTransform() {

            @Override
            public void setToScale(double sx, double sy) {
                super.setToScale(sx, sy);
                System.out.println("scaled!");
            }
        });
    }

    public static void attachToThisVm() {
        System.out.println("dynamically loading javaagent");
        String name = ManagementFactory.getRuntimeMXBean().getName();
        int p = name.indexOf('@');
        String pid = name.substring(0, p);

        try {
            VirtualMachine vm = VirtualMachine.attach(pid);
            vm.loadAgent("javaAgentTest-1.0-SNAPSHOT.jar", null);
            vm.detach();
        }
        catch (Exception e) {
            throw new RuntimeException(e);
        }
        System.out.println("dynamically loaded javaagent");
    }

    public int hi() {
        return 3;
    }

    public static class MyFormer implements ClassFileTransformer {

        @Override
        public byte[] transform(ClassLoader loader, String className, Class<?> classBeingRedefined, ProtectionDomain protectionDomain, byte[] classfileBuffer) throws IllegalClassFormatException {
            return transformClass(className, classfileBuffer);
        }

        private byte[] transformClass(String className, byte[] buffer) {
            if ("mypackage/MyMain".equals(className)) {
                System.out.println(className);

                ClassPool cp = ClassPool.getDefault();
                String name = className.replace("/", ".");
                cp.insertClassPath(new ByteArrayClassPath(name, buffer));
                try {
                    CtClass clazz = cp.get(name);
                    CtBehavior[] declaredBehaviors = clazz.getDeclaredBehaviors();
                    for (CtBehavior db : declaredBehaviors) {
                        if ("hi".equals(db.getName())) {
                            if ((db.getMethodInfo().getAccessFlags() & AccessFlag.STATIC) != 0) {
                                System.out.println("bad access flags, skipping...");
                                return buffer;
                            }

                            System.out.println("Forming hi...");
                            db.insertBefore("System.out.print(\"hi-\");");  // crashes on 11, direct usage case, even referencing to java.lang.Object will crash
                        }
                    }

                    return clazz.toBytecode();
                } catch (Throwable e) {
                    e.printStackTrace();
                    System.out.println("error");
                    return buffer;
                }
            }

            if ("sun/java2d/SunGraphics2D".equals(className)) {
                System.out.println(className);
                ClassPool cp = ClassPool.getDefault();
                String name = className.replace("/", ".");
                cp.insertClassPath(new ByteArrayClassPath(name, buffer));
                try {
                    CtClass clazz = cp.get(name);
                    CtBehavior[] declaredBehaviors = clazz.getDeclaredBehaviors();
                    for (CtBehavior db : declaredBehaviors) {
                        if ("sun.java2d.SunGraphics2D.drawRenderedImage(java.awt.image.RenderedImage,java.awt.geom.AffineTransform)".equals(db.getLongName())) {
                            if ((db.getMethodInfo().getAccessFlags() & AccessFlag.STATIC) != 0) {
                                System.out.println("bad access flags, skipping...");
                                return buffer;
                            }

                            System.out.println("Forming drawRenderedImage...");
                            db.insertBefore("$2.setToScale(2.0, 2.0);");  // crashes on 11, signature case
                        }
                    }
                    return clazz.toBytecode();
                } catch (NotFoundException | CannotCompileException | IOException e) {
                    e.printStackTrace();
                    return buffer;
                }
            }

            return buffer;
        }
    }
}

I build jar via Gradle:

// build.gradle, module name is javaAgentTest
plugins {
    id 'java'
}

group 'org.example'
version '1.0-SNAPSHOT'

repositories {
    mavenCentral()
}

sourceCompatibility = "1.8"
targetCompatibility = "1.8"

def inline = { deps -> deps.collect { it.isDirectory() ? it : zipTree(it) } }

jar {
    manifest {
        attributes(
                "Can-Redefine-Classes": true,
                "Can-Retransform-Classes": true,
                "Premain-Class": "mypackage.MyMain",
                "Agent-Class": "mypackage.MyMain",
        )
    }

    from {
        inline(configurations.runtimeClasspath)  // fat jar
    }
}

dependencies {
    implementation "org.javassist:javassist:3.27.0-GA"
}

On Java 8, both static and dynamic variants work:

$ java -version
openjdk version "1.8.0_265"
OpenJDK Runtime Environment (build 1.8.0_265-8u265-b01-0ubuntu2~20.04-b01)
OpenJDK 64-Bit Server VM (build 25.265-b01, mixed mode)
$ java -cp javaAgentTest-1.0-SNAPSHOT.jar -javaagent:javaAgentTest-1.0-SNAPSHOT.jar mypackage.MyMain
premain start
mypackage/MyMain
Forming hi...
premain end
sun/java2d/SunGraphics2D
Forming drawRenderedImage...
hi-3
scaled!
$ java -cp javaAgentTest-1.0-SNAPSHOT.jar:/usr/lib/jvm/java-8-openjdk-amd64/lib/tools.jar mypackage.MyMain o
dynamically loading javaagent
agentmain start
mypackage/MyMain
Forming hi...
agentmain end
dynamically loaded javaagent
sun/java2d/SunGraphics2D
Forming drawRenderedImage...
hi-3
scaled!

On Java 11, dynamic variant doesn't work (it will fork for hi method if there is no reference to System.out, for example, just db.insertBefore("return 22;");):

$ java -version
openjdk version "11.0.8" 2020-07-14
OpenJDK Runtime Environment (build 11.0.8+10-post-Ubuntu-0ubuntu120.04)
OpenJDK 64-Bit Server VM (build 11.0.8+10-post-Ubuntu-0ubuntu120.04, mixed mode, sharing)
$ java -cp javaAgentTest-1.0-SNAPSHOT.jar -javaagent:javaAgentTest-1.0-SNAPSHOT.jar mypackage.MyMain
premain start
mypackage/MyMain
Forming hi...
premain end
sun/java2d/SunGraphics2D
Forming drawRenderedImage...
hi-3
scaled!
$ java -cp javaAgentTest-1.0-SNAPSHOT.jar -Djdk.attach.allowAttachSelf=true mypackage.MyMain o
dynamically loading javaagent
agentmain start
mypackage/MyMain
Forming hi...
javassist.CannotCompileException: [source error] no such class: System.out
at javassist.CtBehavior.insertBefore(CtBehavior.java:806)
at javassist.CtBehavior.insertBefore(CtBehavior.java:766)
at mypackage.MyMain$MyFormer.transformClass(MyMain.java:112)
at mypackage.MyMain$MyFormer.transform(MyMain.java:91)
at java.instrument/java.lang.instrument.ClassFileTransformer.transform(ClassFileTransformer.java:246)
at java.instrument/sun.instrument.TransformerManager.transform(TransformerManager.java:188)
at java.instrument/sun.instrument.InstrumentationImpl.transform(InstrumentationImpl.java:563)
at java.instrument/sun.instrument.InstrumentationImpl.retransformClasses0(Native Method)
at java.instrument/sun.instrument.InstrumentationImpl.retransformClasses(InstrumentationImpl.java:167)
at mypackage.MyMain.agentmain(MyMain.java:38)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at java.instrument/sun.instrument.InstrumentationImpl.loadClassAndStartAgent(InstrumentationImpl.java:513)
at java.instrument/sun.instrument.InstrumentationImpl.loadClassAndCallAgentmain(InstrumentationImpl.java:535)
Caused by: compile error: no such class: System.out
at javassist.compiler.MemberResolver.searchImports(MemberResolver.java:479)
at javassist.compiler.MemberResolver.lookupClass(MemberResolver.java:422)
at javassist.compiler.MemberResolver.lookupClassByJvmName(MemberResolver.java:329)
at javassist.compiler.TypeChecker.atCallExpr(TypeChecker.java:711)
at javassist.compiler.JvstTypeChecker.atCallExpr(JvstTypeChecker.java:170)
at javassist.compiler.ast.CallExpr.accept(CallExpr.java:49)
at javassist.compiler.CodeGen.doTypeCheck(CodeGen.java:266)
at javassist.compiler.CodeGen.atStmnt(CodeGen.java:360)
at javassist.compiler.ast.Stmnt.accept(Stmnt.java:53)
at javassist.compiler.Javac.compileStmnt(Javac.java:578)
at javassist.CtBehavior.insertBefore(CtBehavior.java:786)
... 15 more
error
agentmain end
dynamically loaded javaagent
sun/java2d/SunGraphics2D
Forming drawRenderedImage...
javassist.CannotCompileException: cannot find java.awt.image.RenderedImage
at javassist.CtBehavior.insertBefore(CtBehavior.java:803)
at javassist.CtBehavior.insertBefore(CtBehavior.java:766)
at mypackage.MyMain$MyFormer.transformClass(MyMain.java:140)
at mypackage.MyMain$MyFormer.transform(MyMain.java:91)
at java.instrument/java.lang.instrument.ClassFileTransformer.transform(ClassFileTransformer.java:246)
at java.instrument/sun.instrument.TransformerManager.transform(TransformerManager.java:188)
at java.instrument/sun.instrument.InstrumentationImpl.transform(InstrumentationImpl.java:563)
at java.desktop/sun.java2d.loops.GraphicsPrimitiveMgr.<clinit>(GraphicsPrimitiveMgr.java:56)
at java.desktop/sun.java2d.loops.Blit.<clinit>(Blit.java:114)
at java.desktop/sun.java2d.xr.XRPMBlitLoops.register(XRPMBlitLoops.java:46)
at java.desktop/sun.java2d.xr.XRSurfaceData.initXRSurfaceData(XRSurfaceData.java:106)
at java.desktop/sun.awt.X11GraphicsEnvironment$1.run(X11GraphicsEnvironment.java:124)
at java.base/java.security.AccessController.doPrivileged(Native Method)
at java.desktop/sun.awt.X11GraphicsEnvironment.<clinit>(X11GraphicsEnvironment.java:61)
at java.base/java.lang.Class.forName0(Native Method)
at java.base/java.lang.Class.forName(Class.java:315)
at java.desktop/java.awt.GraphicsEnvironment$LocalGE.createGE(GraphicsEnvironment.java:101)
at java.desktop/java.awt.GraphicsEnvironment$LocalGE.<clinit>(GraphicsEnvironment.java:83)
at java.desktop/java.awt.GraphicsEnvironment.getLocalGraphicsEnvironment(GraphicsEnvironment.java:129)
at java.desktop/java.awt.Window.initGC(Window.java:487)
at java.desktop/java.awt.Window.init(Window.java:507)
at java.desktop/java.awt.Window.<init>(Window.java:549)
at java.desktop/java.awt.Frame.<init>(Frame.java:423)
at java.desktop/java.awt.Frame.<init>(Frame.java:388)
at java.desktop/javax.swing.JFrame.<init>(JFrame.java:180)
at mypackage.MyMain.main(Unknown Source)
Caused by: javassist.NotFoundException: java.awt.image.RenderedImage
at javassist.ClassPool.get(ClassPool.java:430)
at javassist.bytecode.Descriptor.toCtClass(Descriptor.java:571)
at javassist.bytecode.Descriptor.getParameterTypes(Descriptor.java:424)
at javassist.CtBehavior.getParameterTypes(CtBehavior.java:323)
at javassist.CtBehavior.insertBefore(CtBehavior.java:781)
... 25 more
3

I took some time to analyse your code and found several conceptual problems there, all under the overarching topic bootstrapping. Simply put, it is similar to the old question: What was there first, the chicken or the egg?

Your system has multiple components:

• Java agent
• class file transformer (using Javassist)
• main class doing agent hot-attachment, if necessary.
• transformation target classes

Instead of putting them into separate classes, you stuffed everything into a single class MyMain. Okay, the transformer is in a static inner class, but that does not change the situation in general. So what you are trying to do is to start an agent which transforms itself while it is already running, because it is its own transformation target. This is a bad idea.

If you just refactor your spaghetti code a little bit, the problems will go away. Sorry, I could not resist renaming a few things for better readability, but the two methods transforming the application class and the Java2D class still contain a lot of redundancy (duplicate code) which I did not clean up because I was running out of time. So I leave that up to you. The strange check for a static flag even though you exactly know that your target methods are non-static could also go away, unless your real code is more generic and transforms multiple methods. For simplicity's sake I removed it from my version of your sample code.

I would also recommend to put the Java agent + class file transformer into a separate agent JAR, even though my refactored version also works if everything is in one JAR.

Please also be careful to not manually call retransformClasses on classes which might have been transformed already, e.g. during class-loading.

Class file transformer

package mypackage;

import javassist.*;

import java.io.IOException;
import java.lang.instrument.ClassFileTransformer;
import java.lang.instrument.IllegalClassFormatException;
import java.security.ProtectionDomain;

public class MyTransformer implements ClassFileTransformer {
  @Override
  public byte[] transform(ClassLoader loader, String className, Class<?> classBeingRedefined, ProtectionDomain protectionDomain, byte[] classfileBuffer) throws IllegalClassFormatException {
    return transformClass(className, classfileBuffer);
  }

  private byte[] transformClass(String className, byte[] buffer) {
    switch (className) {
      case "mypackage/MyApplication":
        return transformMyApplication(className, buffer);
      case "sun/java2d/SunGraphics2D":
        return transformSunGraphics2D(className, buffer);
      default:
        return buffer;
    }
  }

  private byte[] transformMyApplication(String className, byte[] buffer) {
    System.out.println(className);

    ClassPool cp = ClassPool.getDefault();
    String name = className.replace("/", ".");
    cp.insertClassPath(new ByteArrayClassPath(name, buffer));
    try {
      CtClass clazz = cp.get(name);
      clazz.defrost();
      CtBehavior[] declaredBehaviors = clazz.getDeclaredBehaviors();
      for (CtBehavior db : declaredBehaviors) {
        if ("hi".equals(db.getName())) {
          System.out.println("Transforming hi...");
          db.insertBefore("System.out.println(\"Hi!\");");
        }
      }

      return clazz.toBytecode();
    }
    catch (Throwable e) {
      e.printStackTrace();
      System.out.println("error");
      return buffer;
    }
  }

  private byte[] transformSunGraphics2D(String className, byte[] buffer) {
    System.out.println(className);
    ClassPool cp = ClassPool.getDefault();
    String name = className.replace("/", ".");
    cp.insertClassPath(new ByteArrayClassPath(name, buffer));
    try {
      CtClass clazz = cp.get(name);
      CtBehavior[] declaredBehaviors = clazz.getDeclaredBehaviors();
      for (CtBehavior db : declaredBehaviors) {
        if ("sun.java2d.SunGraphics2D.drawRenderedImage(java.awt.image.RenderedImage,java.awt.geom.AffineTransform)".equals(db.getLongName())) {
          System.out.println("Transforming drawRenderedImage...");
          db.insertBefore("$2.setToScale(2.0, 2.0);");
        }
      }
      return clazz.toBytecode();
    }
    catch (NotFoundException | CannotCompileException | IOException e) {
      e.printStackTrace();
      return buffer;
    }
  }
}

Java agent:

Please note how MyAgent.instrumentation is used in order to make the information that the agent was attached publicly accessible. We will see this feature being used later.

package mypackage;

import java.lang.instrument.Instrumentation;

public class MyAgent {
  public static Instrumentation instrumentation;

  public static void premain(String args, Instrumentation inst) {
    System.out.println("premain - start");
    instrumentation = inst;
    inst.addTransformer(new MyTransformer(), true);
    System.out.println("premain - end");
  }

  public static void agentmain(String args, Instrumentation inst) {
    System.out.println("agentmain - start");
    premain(args, inst);
    System.out.println("agentmain - end");
  }
}

Main class doing on-demand hot-attachment:

As you can see, I use MyAgent.instrumentation in order to auto-detect if the agent was already attached or not. So there is no more need to use a command line parameter for it.

package mypackage;

import com.sun.tools.attach.VirtualMachine;

import java.lang.instrument.UnmodifiableClassException;
import java.lang.management.ManagementFactory;

public class MyMain {
  private static final String AGENT_PATH = "build/libs/SO_Javassist_SystemOutRecognisedAsClass_64340794-1.0-SNAPSHOT.jar";

  public static void main(String[] args) {
    if (MyAgent.instrumentation == null) {
      attachAgent();
      // This is only necessary if you want to transform an already loaded class,
      // which in this example is not the case
      // transform(MyApplication.class, SunGraphics2D.class);
    }
    MyApplication.main(args);
  }

  public static void attachAgent() {
    System.out.println("Dynamically attaching Java agent - start");
    String jvmName = ManagementFactory.getRuntimeMXBean().getName();
    String pid = jvmName.substring(0, jvmName.indexOf('@'));

    try {
      VirtualMachine vm = VirtualMachine.attach(pid);
      vm.loadAgent(AGENT_PATH, null);
      vm.detach();
    }
    catch (Exception e) {
      throw new RuntimeException(e);
    }
    finally {
      System.out.println("Dynamically attaching Java agent - end");
    }
  }

  public static void transform(Class<?>... targetClasses) {
    try {
      MyAgent.instrumentation.retransformClasses(targetClasses);
    }
    catch (UnmodifiableClassException e) {
      e.printStackTrace();
    }
  }
}

Sample target class:

This is just a sample class. In this case it contains the hi method you want to transform.

package mypackage;

import sun.java2d.SunGraphics2D;

import javax.swing.*;
import java.awt.*;
import java.awt.geom.AffineTransform;
import java.awt.image.BufferedImage;

import static javax.swing.WindowConstants.EXIT_ON_CLOSE;
import static sun.java2d.SurfaceData.getPrimarySurfaceData;

public class MyApplication {
  public static void main(String[] args) {
    System.out.println(new MyApplication().hi());

    JFrame jFrame = new JFrame();
    jFrame.setDefaultCloseOperation(EXIT_ON_CLOSE);
    SunGraphics2D graphics2D = new SunGraphics2D(
      getPrimarySurfaceData(new BufferedImage(10, 10, BufferedImage.TYPE_INT_ARGB)),
      Color.BLACK,
      Color.WHITE,
      Font.getFont("System")
    );
    graphics2D.drawRenderedImage(
      null,
      new AffineTransform() {
        @Override
        public void setToScale(double sx, double sy) {
          super.setToScale(sx, sy);
          System.out.println("scaled!");
        }
      }
    );
    jFrame.setVisible(true);
  }

  public int hi() {
    return 3;
  }
}

You also want to update the manifest file generator:

  manifest {
    attributes(
      "Can-Redefine-Classes": true,
      "Can-Retransform-Classes": true,
      "Premain-Class": "mypackage.MyAgent",
      "Agent-Class": "mypackage.MyAgent",
    )
  }

Now everything runs as expected also on Java 11+. There is no more system class path issue in Javassist because you no longer try to transform the agent itself while it is being hot-attached in the Attach Listener thread belonging to the system thread group.

XXX> java -cp build/libs/SO_Javassist_SystemOutRecognisedAsClass_64340794-1.0-SNAPSHOT.jar -javaagent:build/libs/SO_Javassist_SystemOutRecognisedAsClass_64340794-1.0-SNAPSHOT.jar mypackage.MyMain
premain - start
premain - end
mypackage/MyApplication
Transforming hi...
Hi!
3
sun/java2d/SunGraphics2D
Transforming drawRenderedImage...
scaled!
XXX> java -cp build/libs/SO_Javassist_SystemOutRecognisedAsClass_64340794-1.0-SNAPSHOT.jar -Djdk.attach.allowAttachSelf=true mypackage.MyMain
Dynamically attaching Java agent - start
agentmain - start
premain - start
premain - end
agentmain - end
Dynamically attaching Java agent - end
mypackage/MyApplication
Transforming hi...
Hi!
3
sun/java2d/SunGraphics2D
Transforming drawRenderedImage...
scaled!