repl: add basic syntax highlighting

Signed-off-by: avivkeller <me@aviv.sh>

Author: avivkeller

lib/internal/readline/interface.js

@@ -156,13 +156,15 @@ const kPreviousCursorCols = Symbol('_previousCursorCols');
 const kMultilineMove = Symbol('_multilineMove');
 const kPreviousPrevRows = Symbol('_previousPrevRows');
 const kAddNewLineOnTTY = Symbol('_addNewLineOnTTY');
+const kColorize = Symbol('_colorize');
 
 function InterfaceConstructor(input, output, completer, terminal) {
   this[kSawReturnAt] = 0;
   // TODO(BridgeAR): Document this property. The name is not ideal, so we
   // might want to expose an alias and document that instead.
   this.isCompletionEnabled = true;
   this[kSawKeyPress] = false;
+  this[kColorize] = undefined;
   this[kPreviousKey] = null;
   this.escapeCodeTimeout = ESCAPE_CODE_TIMEOUT;
   this.tabSize = 8;
@@ -184,6 +186,7 @@ function InterfaceConstructor(input, output, completer, terminal) {
     const historySize = input.historySize;
     const history = input.history;
     const removeHistoryDuplicates = input.removeHistoryDuplicates;
+    this[kColorize] = input.colorize;
 
     if (input.tabSize !== undefined) {
       validateUint32(input.tabSize, 'tabSize', true);
@@ -514,15 +517,18 @@ class Interface extends InterfaceConstructor {
     if (this[kIsMultiline]) {
       const lines = StringPrototypeSplit(this.line, '\n');
       // Write first line with normal prompt
-      this[kWriteToOutput](this[kPrompt] + lines[0]);
+      this[kWriteToOutput](this[kPrompt] +
+                           (this[kColorize]?.(lines[0]) ?? lines[0]));
 
       // For continuation lines, add the "|" prefix
       for (let i = 1; i < lines.length; i++) {
-        this[kWriteToOutput](`\n${kMultilinePrompt.description}` + lines[i]);
+        this[kWriteToOutput](`\n${kMultilinePrompt.description}` +
+                             (this[kColorize]?.(lines[i]) ?? lines[i]));
       }
     } else {
       // Write the prompt and the current buffer content.
-      this[kWriteToOutput](line);
+      this[kWriteToOutput](this[kPrompt] +
+                           (this[kColorize]?.(this.line) ?? this.line));
     }
 
     // Force terminal to allocate a new line
@@ -671,7 +677,11 @@ class Interface extends InterfaceConstructor {
         this.line += c;
       }
       this.cursor += c.length;
-      this[kWriteToOutput](c);
+      if (this[kColorize] === undefined) {
+        this[kWriteToOutput](c);
+      } else {
+        this[kRefreshLine]();
+      }
       return;
     }
     if (this.cursor < this.line.length) {
@@ -690,7 +700,7 @@ class Interface extends InterfaceConstructor {
       this.cursor += c.length;
       const newPos = this.getCursorPos();
 
-      if (oldPos.rows < newPos.rows) {
+      if (oldPos.rows < newPos.rows || this[kColorize] !== undefined) {
         this[kRefreshLine]();
       } else {
         this[kWriteToOutput](c);

lib/internal/repl/highlight.js

@@ -0,0 +1,89 @@
+'use strict';
+
+const {
+  FunctionPrototypeBind,
+  StringPrototypeSlice,
+} = primordials;
+
+const { stylizeWithColor } = require('internal/util/inspect');
+const { Parser } = require('internal/deps/acorn/acorn/dist/acorn');
+
+const tokenizer = FunctionPrototypeBind(Parser.tokenizer, Parser);
+
+function tokenStyle(token) {
+  const { label, keyword } = token.type;
+
+  if (keyword !== undefined) {
+    if (label === 'true' || label === 'false') return 'boolean';
+    if (label === 'null') return 'null';
+    if (label === 'import' || label === 'export') return 'module';
+    return 'special';
+  }
+
+  switch (label) {
+    case 'name':
+      switch (token.value) {
+        case 'undefined':
+          return 'undefined';
+        case 'NaN':
+        case 'Infinity':
+          return 'number';
+        case 'Symbol':
+          return 'symbol';
+        case 'Date':
+          return 'date';
+        default:
+          return undefined;
+      }
+    case 'num':
+      return 'number';
+    case 'bigint':
+      return 'bigint';
+    case 'string':
+    case 'template':
+    case '`':
+      return 'string';
+    case 'regexp':
+      return 'regexp';
+    default:
+      return undefined;
+  }
+}
+
+function highlight(code) {
+  if (code.length === 0) return code;
+
+  let result = '';
+  let offset = 0;
+  function write(start, end, style) {
+    result += 
+      StringPrototypeSlice(code, offset, start) +
+      stylizeWithColor(StringPrototypeSlice(code, start, end), style);
+    offset = end;
+  }
+
+  try {
+    const iterator = tokenizer(code, {
+      __proto__: null,
+      allowHashBang: true,
+      ecmaVersion: 'latest',
+      onComment(_block, _text, start, end) {
+        write(start, end, 'undefined');
+      },
+    });
+
+    for (const token of iterator) {
+      const style = tokenStyle(token);
+      if (style !== undefined) {
+        write(token.start, token.end, style);
+      }
+    }
+  } catch {
+    // Acorn throws for unfinished strings, templates, and comments. Tokens and
+    // comments reported before that point are still useful while typing.
+  }
+
+  return result + StringPrototypeSlice(code, offset);
+}
+
+module.exports = { highlight };

lib/internal/util/inspect.js

@@ -3057,6 +3057,7 @@ module.exports = {
   identicalSequenceRange,
   inspect,
   inspectDefaultOptions,
+  stylizeWithColor,
   format,
   formatWithOptions,
   getStringWidth,

lib/repl.js

@@ -98,6 +98,7 @@ const {
 } = require('internal/readline/utils');
 const { Console } = require('console');
 const { shouldColorize } = require('internal/util/colors');
+const { highlight } = require('internal/repl/highlight');
 const CJSModule = require('internal/modules/cjs/loader').Module;
 const { AsyncLocalStorage } = require('async_hooks');
 let debug = require('internal/util/debuglog').debuglog('repl', (fn) => {
@@ -246,6 +247,7 @@ class REPLServer extends Interface {
       terminal: options.terminal,
       historySize: options.historySize,
       prompt,
+      colorize: options.useColors ? highlight : undefined,
     });
 
     ObjectDefineProperty(this, 'inputStream', {

test/parallel/test-repl-highlight.js

@@ -0,0 +1,59 @@
+// Flags: --expose-internals
+'use strict';
+
+require('../common');
+const assert = require('assert');
+const { inspect, stripVTControlCharacters } = require('util');
+const ArrayStream = require('../common/arraystream');
+const { highlight } = require('internal/repl/highlight');
+const { REPLServer } = require('repl');
+
+assert.strictEqual(stripVTControlCharacters(highlight(
+  'const /* comment */ answer = 42; // comment',
+)), 'const /* comment */ answer = 42; // comment');
+
+assert.strictEqual(
+  highlight('const value = "text"'),
+  `\u001b[${inspect.colors.cyan[0]}mconst\u001b[${inspect.colors.cyan[1]}m value = ` +
+  `\u001b[${inspect.colors.green[0]}m"text"\u001b[${inspect.colors.green[1]}m`,
+);
+
+for (const source of [
+  'undefined',
+  'NaN',
+  'Infinity',
+  'Symbol.iterator',
+  'Date.now()',
+  'true',
+  'null',
+  '1n',
+  '/regexp/u',
+  'import("node:fs")',
+]) {
+  assert.notStrictEqual(highlight(source), source);
+}
+
+// Incomplete source is expected while the user is typing.
+highlight('const value = "');
+
+{
+  const input = new ArrayStream();
+  const output = new ArrayStream();
+  let rendered = '';
+  output.write = (chunk) => {
+    rendered += chunk;
+  };
+
+  const repl = new REPLServer({
+    input,
+    output,
+    prompt: '',
+    terminal: true,
+    useColors: false,
+  });
+  rendered = '';
+  repl.write('const');
+
+  assert.strictEqual(rendered, 'const');
+  repl.close();
+}