Add ability of collecting and viewing variables state

 - Introduce :vars command to print available variables in convenient form
 - Display variables in HTML table
 - Support private/protected/internal variables as well
 - Add possibility to get variables using Notebook API
 - Send variables using messaging protocol
 - Add smart updating and caching of variable values

Author: nikolay-egorov

docs/README.md

@@ -124,6 +124,7 @@ python -m kotlin_kernel add-kernel --name "JDK 15 Big 2 GPU" --jdk ~/.jdks/openj
 The following REPL commands are supported:
  - `:help` - display help
  - `:classpath` - show current classpath
+ - `:vars` - get visible variables values
 
 ### Dependencies resolving annotations
 

jupyter-lib/api/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/Notebook.kt

@@ -1,7 +1,5 @@
 package org.jetbrains.kotlinx.jupyter.api
 
-import kotlin.jvm.Throws
-
 /**
  * [Notebook] is a main entry point for Kotlin Jupyter API
  */
@@ -11,6 +9,19 @@ interface Notebook {
      */
     val cellsList: Collection<CodeCell>
 
+    /**
+     * Current state of visible variables
+     */
+    val variablesState: Map<String, VariableState>
+
+    /**
+     * Stores info about useful variables in a cell.
+     * Key: cellId;
+     * Value: set of variable names.
+     * Useful <==> declarations + modifying references
+     */
+    val cellVariables: Map<Int, Set<String>>
+
     /**
      * Mapping allowing to get cell by execution number
      */

jupyter-lib/api/src/main/kotlin/org/jetbrains/kotlinx/jupyter/api/VariableState.kt

@@ -0,0 +1,33 @@
+package org.jetbrains.kotlinx.jupyter.api
+
+import kotlin.reflect.KProperty
+import kotlin.reflect.KProperty1
+import kotlin.reflect.jvm.isAccessible
+
+interface VariableState {
+    val property: KProperty<*>
+    val scriptInstance: Any?
+    val stringValue: String?
+    val value: Any?
+}
+
+data class VariableStateImpl(
+    override val property: KProperty1<Any, *>,
+    override val scriptInstance: Any,
+) : VariableState {
+    private var cachedValue: Any? = null
+
+    fun update() {
+        val wasAccessible = property.isAccessible
+        property.isAccessible = true
+        val fieldValue = property.get(scriptInstance)
+        property.isAccessible = wasAccessible
+        cachedValue = fieldValue
+    }
+
+    override val stringValue: String?
+        get() = cachedValue?.toString()
+
+    override val value: Any?
+        get() = cachedValue
+}

jupyter-lib/shared-compiler/src/main/kotlin/org/jetbrains/kotlinx/jupyter/compiler/util/serializedCompiledScript.kt

@@ -25,6 +25,7 @@ class EvaluatedSnippetMetadata(
     val newClasspath: Classpath = emptyList(),
     val compiledData: SerializedCompiledScriptsData = SerializedCompiledScriptsData.EMPTY,
     val newImports: List<String> = emptyList(),
+    val evaluatedVariablesState: Map<String, String?> = mutableMapOf()
 ) {
     companion object {
         val EMPTY = EvaluatedSnippetMetadata()

kotlin-jupyter-plugin/common-dependencies/src/main/kotlin/org/jetbrains/kotlinx/jupyter/common/ReplCommand.kt

@@ -2,7 +2,8 @@ package org.jetbrains.kotlinx.jupyter.common
 
 enum class ReplCommand(val desc: String) {
     HELP("display help"),
-    CLASSPATH("show current classpath");
+    CLASSPATH("show current classpath"),
+    VARS("get visible variables values");
 
     val nameForUser = getNameForUser(name)
 

src/main/kotlin/org/jetbrains/kotlinx/jupyter/apiImpl.kt

@@ -9,7 +9,8 @@ import org.jetbrains.kotlinx.jupyter.api.JREInfoProvider
 import org.jetbrains.kotlinx.jupyter.api.KotlinKernelVersion
 import org.jetbrains.kotlinx.jupyter.api.Notebook
 import org.jetbrains.kotlinx.jupyter.api.RenderersProcessor
-import java.lang.IllegalStateException
+import org.jetbrains.kotlinx.jupyter.api.VariableState
+import org.jetbrains.kotlinx.jupyter.repl.InternalEvaluator
 
 class DisplayResultWrapper private constructor(
     val display: DisplayResult,
@@ -104,6 +105,11 @@ class NotebookImpl(
     override val cellsList: Collection<CodeCellImpl>
         get() = cells.values
 
+    override val variablesState = mutableMapOf<String, VariableState>()
+
+    override val cellVariables: Map<Int, Set<String>>
+        get() = currentCellVariables
+
     override fun getCell(id: Int): CodeCellImpl {
         return cells[id] ?: throw ArrayIndexOutOfBoundsException(
             "There is no cell with number '$id'"
@@ -114,6 +120,7 @@ class NotebookImpl(
         return getCell(id).result
     }
 
+    private var currentCellVariables = mapOf<Int, Set<String>>()
     private val history = arrayListOf<CodeCellImpl>()
     private var mainCellCreated = false
 
@@ -132,6 +139,32 @@ class NotebookImpl(
     override val jreInfo: JREInfoProvider
         get() = JavaRuntime
 
+    fun updateVariablesState(evaluator: InternalEvaluator) {
+        variablesState += evaluator.variablesHolder
+        currentCellVariables = evaluator.cellVariables
+    }
+
+    fun updateVariablesState(varsStateUpdate: Map<String, VariableState>) {
+        variablesState += varsStateUpdate
+    }
+
+    fun variablesReportAsHTML(): String {
+        return generateHTMLVarsReport(variablesState)
+    }
+
+    fun variablesReport(): String {
+        return if (variablesState.isEmpty()) ""
+        else {
+            buildString {
+                append("Visible vars: \n")
+                variablesState.forEach { (name, currentState) ->
+                    append("\t$name : ${currentState.stringValue}\n")
+                }
+                append('\n')
+            }
+        }
+    }
+
     fun addCell(
         internalId: Int,
         preprocessedCode: String,

src/main/kotlin/org/jetbrains/kotlinx/jupyter/commands.kt

@@ -1,5 +1,6 @@
 package org.jetbrains.kotlinx.jupyter
 
+import org.jetbrains.kotlinx.jupyter.api.htmlResult
 import org.jetbrains.kotlinx.jupyter.api.textResult
 import org.jetbrains.kotlinx.jupyter.common.ReplCommand
 import org.jetbrains.kotlinx.jupyter.common.ReplLineMagic
@@ -56,6 +57,9 @@ fun runCommand(code: String, repl: ReplForJupyter): Response {
             val cp = repl.currentClasspath
             OkResponseWithMessage(textResult("Current classpath (${cp.count()} paths):\n${cp.joinToString("\n")}"))
         }
+        ReplCommand.VARS -> {
+            OkResponseWithMessage(htmlResult(repl.notebook.variablesReportAsHTML()))
+        }
         ReplCommand.HELP -> {
             val commands = ReplCommand.values().asIterable().joinToStringIndented { ":${it.nameForUser} - ${it.desc}" }
             val magics = ReplLineMagic.values().asIterable().filter { it.visibleInHelp }.joinToStringIndented {

src/main/kotlin/org/jetbrains/kotlinx/jupyter/htmlUtil.kt

@@ -0,0 +1,64 @@
+package org.jetbrains.kotlinx.jupyter
+
+import org.jetbrains.kotlinx.jupyter.api.VariableState
+
+const val varsTableStyleClass = "variables_table"
+
+fun generateHTMLVarsReport(variablesState: Map<String, VariableState>): String {
+    return buildString {
+        append(generateStyleSection())
+        if (variablesState.isEmpty()) {
+            append("<h2 style=\"text-align:center;\">Variables State's Empty</h2>\n")
+            return toString()
+        }
+
+        append("<h2 style=\"text-align:center;\">Variables State</h2>\n")
+        append(generateVarsTable(variablesState))
+    }
+}
+
+fun generateStyleSection(borderPx: Int = 1, paddingPx: Int = 5): String {
+    //language=HTML
+    val styleSection = """
+    <style>
+    table.$varsTableStyleClass, .$varsTableStyleClass th, .$varsTableStyleClass td {
+      border: ${borderPx}px solid black;
+      border-collapse: collapse;
+      text-align:center;
+    }
+    .$varsTableStyleClass th, .$varsTableStyleClass td {
+      padding: ${paddingPx}px;
+    }
+    </style>
+    
+    """.trimIndent()
+    return styleSection
+}
+
+fun generateVarsTable(variablesState: Map<String, VariableState>): String {
+    return buildString {
+        append(
+            """
+            <table class="$varsTableStyleClass" style="width:80%;margin-left:auto;margin-right:auto;" align="center">
+              <tr>
+                <th>Variable</th>
+                <th>Value</th>
+              </tr>
+      
+            """.trimIndent()
+        )
+
+        variablesState.entries.forEach {
+            append(
+                """
+                <tr>
+                    <td>${it.key}</td>
+                    <td>${it.value.stringValue}</td>
+                </tr>
+                """.trimIndent()
+            )
+        }
+
+        append("\n</table>\n")
+    }
+}

src/main/kotlin/org/jetbrains/kotlinx/jupyter/repl.kt

@@ -17,8 +17,9 @@ import org.jetbrains.kotlinx.jupyter.codegen.FieldsProcessor
 import org.jetbrains.kotlinx.jupyter.codegen.FieldsProcessorImpl
 import org.jetbrains.kotlinx.jupyter.codegen.FileAnnotationsProcessor
 import org.jetbrains.kotlinx.jupyter.codegen.FileAnnotationsProcessorImpl
-import org.jetbrains.kotlinx.jupyter.codegen.ResultsRenderersProcessor
 import org.jetbrains.kotlinx.jupyter.codegen.RenderersProcessorImpl
+import org.jetbrains.kotlinx.jupyter.codegen.ResultsRenderersProcessor
+import org.jetbrains.kotlinx.jupyter.common.ReplCommand
 import org.jetbrains.kotlinx.jupyter.common.looksLikeReplCommand
 import org.jetbrains.kotlinx.jupyter.compiler.CompilerArgsConfigurator
 import org.jetbrains.kotlinx.jupyter.compiler.DefaultCompilerArgsConfigurator
@@ -360,6 +361,27 @@ class ReplForJupyterImpl(
         else context.compilationConfiguration.asSuccess()
     }
 
+    /**
+     * Used for debug purposes.
+     * @see ReplCommand
+     */
+    private fun printVariables(isHtmlFormat: Boolean = false) = log.debug(
+        if (isHtmlFormat) notebook.variablesReportAsHTML() else notebook.variablesReport()
+    )
+
+    private fun printUsagesInfo(cellId: Int, usedVariables: Set<String>?) {
+        log.debug(buildString {
+            if (usedVariables == null || usedVariables.isEmpty()) {
+                append("No usages for cell $cellId")
+                return@buildString
+            }
+            append("Usages for cell $cellId:\n")
+            usedVariables.forEach {
+                append(it + "\n")
+            }
+        })
+    }
+
     override fun eval(code: Code, displayHandler: DisplayHandler?, jupyterId: Int): EvalResult {
         return withEvalContext {
             rethrowAsLibraryException(LibraryProblemPart.BEFORE_CELL_CALLBACKS) {
@@ -371,14 +393,14 @@ class ReplForJupyterImpl(
             val compiledData: SerializedCompiledScriptsData
             val newImports: List<String>
             val result = try {
-                executor.execute(code, displayHandler) { internalId, codeToExecute ->
+                log.debug("Current cell id: $jupyterId")
+                executor.execute(code, displayHandler, currentCellId = jupyterId - 1) { internalId, codeToExecute ->
                     cell = notebook.addCell(internalId, codeToExecute, EvalData(jupyterId, code))
                 }
             } finally {
                 compiledData = internalEvaluator.popAddedCompiledScripts()
                 newImports = importsCollector.popAddedImports()
             }
-
             cell?.resultVal = result.result.value
 
             val rendered = result.result.let {
@@ -395,7 +417,13 @@ class ReplForJupyterImpl(
                 updateClasspath()
             } ?: emptyList()
 
-            EvalResult(rendered, EvaluatedSnippetMetadata(newClasspath, compiledData, newImports))
+            notebook.updateVariablesState(internalEvaluator)
+            // printVars()
+            // printUsagesInfo(jupyterId, cellVariables[jupyterId - 1])
+
+
+            val variablesStateUpdate = notebook.variablesState.mapValues { it.value.stringValue }
+            EvalResult(rendered, EvaluatedSnippetMetadata(newClasspath, compiledData, newImports, variablesStateUpdate))
         }
     }
 

src/main/kotlin/org/jetbrains/kotlinx/jupyter/repl/CellExecutor.kt

@@ -17,6 +17,7 @@ interface CellExecutor : ExecutionHost {
         processAnnotations: Boolean = true,
         processMagics: Boolean = true,
         invokeAfterCallbacks: Boolean = true,
+        currentCellId: Int = -1,
         callback: ExecutionStartedCallback? = null
     ): InternalEvalResult
 }