Fix first character of non-code lines highlighted as a comment

apps/vscode/CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## 1.139.0 (Unreleased)
 
+- Fixed a bug where the first character of lines outside code cells (paragraphs, headings, YAML) could be highlighted as a comment (<https://github.com/quarto-dev/quarto/issues/985>).
+
 ## 1.138.0 (Release on 2026-09-11)
 
 - In Positron, running a Python cell in a knitr document now respects the `quarto.cells.useReticulate` setting, instead of always routing it through reticulate on the R console (<https://github.com/quarto-dev/quarto/pull/1116>).

apps/vscode/src/providers/semantic-tokens.ts

@@ -19,6 +19,7 @@ import { Token } from "quarto-core";
 import { MarkdownEngine } from "../markdown/engine";
 import { isQuartoDoc } from "../core/doc";
 import {
+  isBlockOfLanguage,
   unadjustedSemanticTokens,
   virtualDocForLanguage,
   withVirtualDocUri,
@@ -228,6 +229,16 @@ export function embeddedSemanticTokensProvider(engine: MarkdownEngine) {
     // Create virtual doc for all blocks of this language
     const vdoc = virtualDocForLanguage(document, tokens, language);
 
+    // Lines of the real document that are code of this language. The virtual
+    // doc fills all other lines with placeholder content (e.g. `#` comments),
+    // whose tokens must not be applied to the real document.
+    const codeLines = new Set<number>();
+    for (const block of tokens.filter(isBlockOfLanguage(language))) {
+      for (let line = block.range.start.line + 1; line < block.range.end.line; line++) {
+        codeLines.add(line);
+      }
+    }
+
     return await withVirtualDocUri(vdoc, document.uri, "semanticTokens", async (uri: Uri) => {
       try {
         // Get the legend from the embedded language provider
@@ -236,23 +247,24 @@ export function embeddedSemanticTokensProvider(engine: MarkdownEngine) {
           uri
         );
 
-        const tokens = await commands.executeCommand<SemanticTokens>(
+        const semanticTokens = await commands.executeCommand<SemanticTokens>(
           "vscode.provideDocumentSemanticTokens",
           uri
         );
 
-        if (!tokens || tokens.data.length === 0) {
-          return tokens;
+        if (!semanticTokens || semanticTokens.data.length === 0) {
+          return semanticTokens;
         }
 
         // Remap token indices from embedded provider's legend to our universal legend
-        let remappedTokens = tokens;
+        let remappedTokens = semanticTokens;
         if (legend) {
-          remappedTokens = remapTokenIndices(tokens, legend, QUARTO_SEMANTIC_TOKEN_LEGEND);
+          remappedTokens = remapTokenIndices(semanticTokens, legend, QUARTO_SEMANTIC_TOKEN_LEGEND);
         }
 
-        // Adjust token positions from virtual doc to real doc coordinates
-        return unadjustedSemanticTokens(vdoc.language, remappedTokens);
+        // Adjust token positions from virtual doc to real doc coordinates,
+        // keeping only tokens on code lines
+        return unadjustedSemanticTokens(vdoc.language, remappedTokens, codeLines);
       } catch (error) {
         return undefined;
       }

apps/vscode/src/test/semanticTokens.test.ts

@@ -1,6 +1,8 @@
 import * as vscode from "vscode";
 import * as assert from "assert";
 import { decodeSemanticTokens, encodeSemanticTokens, remapTokenIndices } from "../providers/semantic-tokens";
+import { unadjustedSemanticTokens } from "../vdoc/vdoc";
+import { embeddedLanguage } from "../vdoc/languages";
 
 suite("Semantic Tokens", function () {
 
@@ -164,4 +166,25 @@ suite("Semantic Tokens", function () {
     assert.strictEqual(decoded[0].tokenModifiers, expectedModifiers, "Unmapped modifiers should be filtered out");
   });
 
+  test("Unadjusting semantic tokens drops tokens outside code lines", function () {
+    // Python injects 2 lines at the top of the virtual doc
+    const python = embeddedLanguage("python")!;
+
+    // Virtual doc coordinates: lines 0-1 are injected, line 2 (real line 0) is a
+    // `#` filler line, and line 4 (real line 2) is code
+    const vdocTokens = encodeSemanticTokens([
+      { line: 0, startChar: 0, length: 14, tokenType: 0, tokenModifiers: 0 }, // injected
+      { line: 2, startChar: 0, length: 1, tokenType: 0, tokenModifiers: 0 },  // filler `#`
+      { line: 4, startChar: 0, length: 1, tokenType: 1, tokenModifiers: 0 },  // code
+      { line: 4, startChar: 4, length: 3, tokenType: 2, tokenModifiers: 0 },  // code
+    ]);
+
+    const decoded = decodeSemanticTokens(unadjustedSemanticTokens(python, vdocTokens, new Set([2])));
+
+    assert.deepStrictEqual(decoded, [
+      { line: 2, startChar: 0, length: 1, tokenType: 1, tokenModifiers: 0 },
+      { line: 2, startChar: 4, length: 3, tokenType: 2, tokenModifiers: 0 },
+    ]);
+  });
+
 });

apps/vscode/src/vdoc/vdoc.ts

@@ -287,30 +287,22 @@ export function unadjustedRange(language: EmbeddedLanguage, range: Range) {
 /**
  * Adjust semantic tokens from virtual document coordinates to real document coordinates
  *
- * This function decodes the tokens, adjusts each token's position using unadjustedRange,
- * and re-encodes them back to delta format.
+ * This function decodes the tokens, shifts each token's line using unadjustedLine,
+ * and re-encodes them back to delta format. When `lines` is given, only tokens on those
+ * real document lines are kept, dropping tokens on the virtual doc's filler and injected lines.
  */
 export function unadjustedSemanticTokens(
   language: EmbeddedLanguage,
-  tokens: SemanticTokens
+  tokens: SemanticTokens,
+  lines?: Set<number>
 ): SemanticTokens {
   // Decode tokens to absolute positions
   const decoded = decodeSemanticTokens(tokens);
 
-  // Adjust each token's position
-  const adjusted = decoded.map(t => {
-    const range = unadjustedRange(language, new Range(
-      new Position(t.line, t.startChar),
-      new Position(t.line, t.startChar + t.length)
-    ));
-    return {
-      line: range.start.line,
-      startChar: range.start.character,
-      length: range.end.character - range.start.character,
-      tokenType: t.tokenType,
-      tokenModifiers: t.tokenModifiers
-    };
-  });
+  // Adjust each token's line (tokens never span lines, so columns are unchanged)
+  const adjusted = decoded
+    .map(t => ({ ...t, line: unadjustedLine(language, t.line) }))
+    .filter(t => t.line >= 0 && (!lines || lines.has(t.line)));
 
   // Re-encode to delta format
   return encodeSemanticTokens(adjusted, tokens.resultId);