chore: profiler and flamegraph support

Author: sureshg

.github/workflows/build.yml

@@ -46,13 +46,11 @@ jobs:
       fail-fast: true
       max-parallel: 3
       matrix:
-        os: [ ubuntu-latest, macos-latest, windows-latest ]
+        os: [ ubuntu-latest, macos-latest ]
         jdk: [ ea ]
         include:
           - os: macos-latest
             native_task: :macosArm64Test :macosX64Test
-          - os: windows-latest
-            native_task: :mingwX64Test
           - os: ubuntu-latest
             native_task: :linuxX64Test
 

backend/jvm/build.gradle.kts

@@ -86,6 +86,7 @@ dependencies {
   implementation(libs.ktor.server.cors)
   implementation(libs.ktor.server.host.common)
   implementation(libs.ktor.server.auto.head)
+  implementation(libs.ktor.server.partial.content)
   implementation(libs.ktor.server.resources)
   implementation(libs.ktor.server.auth)
   implementation(libs.ktor.server.auth.jwt)
@@ -109,13 +110,16 @@ dependencies {
   implementation(libs.ktor.cohort.core)
   implementation(libs.ktor.cohort.hikari)
   implementation(libs.micrometer.prometheus)
+  implementation(libs.ap.loader.all)
   // Logging
   implementation(libs.logback.classic)
+  implementation("io.ktor:ktor-server-partial-content-jvm:2.3.6")
   // Testing
   testImplementation(platform(libs.testcontainers.bom))
   testImplementation(libs.ktor.server.tests)
   testImplementation(libs.testcontainers.junit5)
   testImplementation(libs.testcontainers.postgresql)
+  testImplementation(libs.konsist)
 
   // Copy web app browserDist
   webapp(project(path = ":${projects.web.name}", configuration = webapp.name))

backend/jvm/src/main/java/one/converter/Arguments.java

@@ -0,0 +1,107 @@
+package one.converter;
+/*
+ * Copyright 2022 Andrei Pangin
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.util.Calendar;
+import java.util.GregorianCalendar;
+import java.util.StringTokenizer;
+import java.util.regex.Pattern;
+
+public class Arguments {
+    String title = "Flame Graph";
+    String highlight;
+    Pattern include;
+    Pattern exclude;
+    double minwidth;
+    int skip;
+    boolean reverse;
+    boolean cpu;
+    boolean alloc;
+    boolean live;
+    boolean lock;
+    boolean threads;
+    boolean total;
+    boolean lines;
+    boolean bci;
+    boolean simple;
+    boolean dot;
+    boolean collapsed;
+    long from;
+    long to;
+    String input;
+    String output;
+
+    public Arguments(String... args) {
+        for (int i = 0; i < args.length; i++) {
+            String arg = args[i];
+            if (arg.startsWith("--")) {
+                try {
+                    Field f = Arguments.class.getDeclaredField(arg.substring(2));
+                    if ((f.getModifiers() & (Modifier.PRIVATE | Modifier.STATIC | Modifier.FINAL)) != 0) {
+                        throw new IllegalStateException(arg);
+                    }
+
+                    Class<?> type = f.getType();
+                    if (type == String.class) {
+                        f.set(this, args[++i]);
+                    } else if (type == boolean.class) {
+                        f.setBoolean(this, true);
+                    } else if (type == int.class) {
+                        f.setInt(this, Integer.parseInt(args[++i]));
+                    } else if (type == double.class) {
+                        f.setDouble(this, Double.parseDouble(args[++i]));
+                    } else if (type == long.class) {
+                        f.setLong(this, parseTimestamp(args[++i]));
+                    } else if (type == Pattern.class) {
+                        f.set(this, Pattern.compile(args[++i]));
+                    }
+                } catch (NoSuchFieldException | IllegalAccessException e) {
+                    throw new IllegalArgumentException(arg);
+                }
+            } else if (!arg.isEmpty()) {
+                if (input == null) {
+                    input = arg;
+                } else {
+                    output = arg;
+                }
+            }
+        }
+    }
+
+    // Milliseconds or HH:mm:ss.S or yyyy-MM-dd'T'HH:mm:ss.S
+    private long parseTimestamp(String time) {
+        if (time.indexOf(':') < 0) {
+            return Long.parseLong(time);
+        }
+
+        GregorianCalendar cal = new GregorianCalendar();
+        StringTokenizer st = new StringTokenizer(time, "-:.T");
+
+        if (time.indexOf('T') > 0) {
+            cal.set(Calendar.YEAR, Integer.parseInt(st.nextToken()));
+            cal.set(Calendar.MONTH, Integer.parseInt(st.nextToken()) - 1);
+            cal.set(Calendar.DAY_OF_MONTH, Integer.parseInt(st.nextToken()));
+        }
+        cal.set(Calendar.HOUR_OF_DAY, st.hasMoreTokens() ? Integer.parseInt(st.nextToken()) : 0);
+        cal.set(Calendar.MINUTE, st.hasMoreTokens() ? Integer.parseInt(st.nextToken()) : 0);
+        cal.set(Calendar.SECOND, st.hasMoreTokens() ? Integer.parseInt(st.nextToken()) : 0);
+        cal.set(Calendar.MILLISECOND, st.hasMoreTokens() ? Integer.parseInt(st.nextToken()) : 0);
+
+        return cal.getTimeInMillis();
+    }
+}

backend/jvm/src/main/kotlin/dev/suresh/lang/FFM.kt

@@ -1,9 +1,6 @@
 package dev.suresh.lang
 
-import dev.suresh.LINKER
-import dev.suresh.SYMBOL_LOOKUP
-import dev.suresh.downcallHandle
-import dev.suresh.findOrNull
+import dev.suresh.*
 import io.github.oshai.kotlinlogging.KLogger
 import java.lang.foreign.*
 import java.lang.invoke.MethodHandle
@@ -14,7 +11,7 @@ import java.time.Instant
 object FFM {
 
   context(KLogger)
-  fun memoryLayout() {
+  suspend fun memoryLayout() = runOnVirtualThread {
     memoryAPIs()
     currTime()
     strlen("Hello Panama!")

backend/jvm/src/main/kotlin/dev/suresh/lang/VThread.kt

@@ -2,8 +2,6 @@ package dev.suresh.lang
 
 import dev.suresh.*
 import io.github.oshai.kotlinlogging.KLogger
-import java.lang.foreign.FunctionDescriptor
-import java.lang.foreign.ValueLayout
 import java.util.concurrent.StructuredTaskScope
 import kotlin.time.Duration.Companion.seconds
 import kotlinx.datetime.Clock
@@ -16,7 +14,7 @@ import stdlibFeatures
 object VThread {
 
   context(KLogger)
-  fun virtualThreads() {
+  suspend fun virtualThreads() = runOnVirtualThread {
     info { (Greeting().greeting()) }
     listOf("main", "jvm", "js").forEach {
       info { "common-$it --> ${ClassLoader.getSystemResource("common-$it-res.txt")?.readText()}" }
@@ -29,16 +27,13 @@ object VThread {
     structuredConcurrency()
     langFeatures()
     stdlibFeatures()
-
-    getPid()
     kotlinxMetaData()
     classFileApi()
   }
 }
 
 context(KLogger)
 fun structuredConcurrency() {
-
   info { "Structured concurrency..." }
   val taskList =
       StructuredTaskScope<String>().use { sts ->
@@ -52,13 +47,14 @@ fun structuredConcurrency() {
                   else -> {
                     while (!Thread.currentThread().isInterrupted) {
                       debug { "Task $it ..." }
+                      Thread.sleep(100)
                     }
                     "Task $it"
                   }
                 }
               }
             }
-        runCatching { sts.joinUntil(start.plus(2.seconds).toJavaInstant()) }
+        runCatching { sts.joinUntil(start.plus(1.seconds).toJavaInstant()) }
         tasks
       }
 
@@ -76,16 +72,6 @@ fun structuredConcurrency() {
   }
 }
 
-context(KLogger)
-fun getPid() {
-  val getpidAddr = SYMBOL_LOOKUP.findOrNull("getpid")
-  val getpidDesc = FunctionDescriptor.of(ValueLayout.JAVA_INT)
-  val getpid = LINKER.downcallHandle(getpidAddr, getpidDesc)
-  val pid = getpid.invokeExact() as Int
-  assert(pid.toLong() == ProcessHandle.current().pid())
-  info { "getpid() = $pid" }
-}
-
 context(KLogger)
 fun kotlinxMetaData() {
   val metadataAnnotation = LocalDateTime::class.java.getAnnotation(Metadata::class.java)

backend/jvm/src/main/kotlin/dev/suresh/plugins/Http.kt

@@ -10,13 +10,16 @@ import io.ktor.server.plugins.compression.*
 import io.ktor.server.plugins.cors.routing.*
 import io.ktor.server.plugins.defaultheaders.*
 import io.ktor.server.plugins.forwardedheaders.*
+import io.ktor.server.plugins.partialcontent.*
 import io.ktor.server.request.*
 import io.ktor.server.routing.*
 import org.slf4j.event.Level
 
 fun Application.configureHTTP() {
   install(IgnoreTrailingSlash)
 
+  install(PartialContent)
+
   install(AutoHeadResponse)
 
   install(ForwardedHeaders)

backend/jvm/src/main/kotlin/dev/suresh/routes/JvmFeature.kt

@@ -3,13 +3,40 @@ package dev.suresh.routes
 import dev.suresh.lang.FFM
 import dev.suresh.lang.VThread
 import dev.suresh.log.LoggerDelegate
+import dev.suresh.runOnVirtualThread
 import io.github.oshai.kotlinlogging.KotlinLogging
+import io.ktor.http.*
 import io.ktor.server.application.*
 import io.ktor.server.response.*
 import io.ktor.server.routing.*
+import java.io.PrintStream
+import jdk.jfr.Configuration
+import jdk.jfr.consumer.RecordingStream
+import kotlin.io.path.deleteIfExists
+import kotlin.io.path.name
+import kotlin.io.path.pathString
+import kotlin.time.Duration.Companion.milliseconds
+import kotlin.time.toJavaDuration
+import kotlinx.coroutines.sync.Mutex
+import kotlinx.coroutines.sync.withLock
+import one.converter.Arguments
+import one.converter.FlameGraph
+import one.converter.jfr2flame
+import one.jfr.JfrReader
+import one.profiler.AsyncProfiler
+import one.profiler.AsyncProfilerLoader
+import one.profiler.Events
 
 private val logger = KotlinLogging.logger {}
 
+val mutex = Mutex()
+
+val profiler: AsyncProfiler? by lazy {
+  val ap = AsyncProfilerLoader.loadOrNull()
+  ap.start(Events.CPU, 1000)
+  ap
+}
+
 fun Route.jvmFeatures() {
   get("/ffm") {
     call.respondTextWriter { with(LoggerDelegate(this, logger)) { FFM.memoryLayout() } }
@@ -18,4 +45,47 @@ fun Route.jvmFeatures() {
   get("/vthreads") {
     call.respondTextWriter { with(LoggerDelegate(this, logger)) { VThread.virtualThreads() } }
   }
+
+  get("/profile") {
+    // Run the blocking operation on virtual thread and make sure
+    // only one profile operation is running at a time.
+    when {
+      mutex.isLocked -> call.respondText("Profile operation is already running")
+      else ->
+          mutex.withLock {
+            runOnVirtualThread {
+              val jfrPath = kotlin.io.path.createTempFile("profile", ".jfr")
+              RecordingStream(Configuration.getConfiguration("profile")).use {
+                it.enable("jdk.CPULoad").withPeriod(100.milliseconds.toJavaDuration())
+                it.enable("jdk.JavaMonitorEnter").withStackTrace()
+                it.startAsync()
+                Thread.sleep(5_000)
+                it.dump(jfrPath)
+                println("JFR file written to ${jfrPath.toAbsolutePath()}")
+              }
+
+              when (call.request.queryParameters.contains("download")) {
+                true -> {
+                  call.response.header(
+                      HttpHeaders.ContentDisposition,
+                      ContentDisposition.Attachment.withParameter(
+                              ContentDisposition.Parameters.FileName, jfrPath.fileName.name)
+                          .toString())
+                  call.respondFile(jfrPath.toFile())
+                }
+                else -> {
+                  val jfr2flame = jfr2flame(JfrReader(jfrPath.pathString), Arguments())
+                  val flameGraph = FlameGraph()
+                  jfr2flame.convert(flameGraph)
+
+                  call.respondOutputStream(contentType = ContentType.Text.Html) {
+                    flameGraph.dump(PrintStream(this))
+                  }
+                  jfrPath.deleteIfExists()
+                }
+              }
+            }
+          }
+    }
+  }
 }

backend/jvm/src/test/kotlin/dev/suresh/Konsist.kt

@@ -0,0 +1,23 @@
+package dev.suresh
+
+import com.lemonappdev.konsist.api.Konsist
+import com.lemonappdev.konsist.api.ext.list.enumConstants
+import com.lemonappdev.konsist.api.ext.list.modifierprovider.withEnumModifier
+import com.lemonappdev.konsist.api.ext.list.withAnnotationOf
+import com.lemonappdev.konsist.api.verify.assertTrue
+import kotlin.test.Test
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+
+class AppTests {
+
+  @Test
+  fun `make sure the enum classes have serial annotation`() {
+    Konsist.scopeFromSourceSet("main")
+        .classes()
+        .withEnumModifier()
+        .withAnnotationOf(Serializable::class)
+        .enumConstants
+        .assertTrue { it.hasAnnotationOf(SerialName::class) }
+  }
+}