From 06d4ba970e9924d6d072e556a7c04d51bf30644e Mon Sep 17 00:00:00 2001 From: Johannes Piermeier Date: Sun, 27 Sep 2026 08:32:38 +0200 Subject: [PATCH] Update CHANGELOG --- apps/vscode/CHANGELOG.md | 2 ++ apps/vscode/src/providers/semantic-tokens.ts | 26 ++++++++++++++------ apps/vscode/src/test/semanticTokens.test.ts | 23 +++++++++++++++++ apps/vscode/src/vdoc/vdoc.ts | 26 +++++++------------- 4 files changed, 53 insertions(+), 24 deletions(-) diff --git a/apps/vscode/CHANGELOG.md b/apps/vscode/CHANGELOG.md index 0dd83247..26fbd799 100644 --- a/apps/vscode/CHANGELOG.md +++ b/apps/vscode/CHANGELOG.md @@ -2,6 +2,8 @@ ## 1.139.0 (Unreleased) +- Fixed a bug where the first character of lines outside code cells (paragraphs, headings, YAML) could be highlighted as a comment (). + ## 1.138.0 (Release on 2026-09-11) - In Positron, running a Python cell in a knitr document now respects the `quarto.cells.useReticulate` setting, instead of always routing it through reticulate on the R console (). diff --git a/apps/vscode/src/providers/semantic-tokens.ts b/apps/vscode/src/providers/semantic-tokens.ts index 20174a30..341db4c9 100644 --- a/apps/vscode/src/providers/semantic-tokens.ts +++ b/apps/vscode/src/providers/semantic-tokens.ts @@ -19,6 +19,7 @@ import { Token } from "quarto-core"; import { MarkdownEngine } from "../markdown/engine"; import { isQuartoDoc } from "../core/doc"; import { + isBlockOfLanguage, unadjustedSemanticTokens, virtualDocForLanguage, withVirtualDocUri, @@ -228,6 +229,16 @@ export function embeddedSemanticTokensProvider(engine: MarkdownEngine) { // Create virtual doc for all blocks of this language const vdoc = virtualDocForLanguage(document, tokens, language); + // Lines of the real document that are code of this language. The virtual + // doc fills all other lines with placeholder content (e.g. `#` comments), + // whose tokens must not be applied to the real document. + const codeLines = new Set(); + for (const block of tokens.filter(isBlockOfLanguage(language))) { + for (let line = block.range.start.line + 1; line < block.range.end.line; line++) { + codeLines.add(line); + } + } + return await withVirtualDocUri(vdoc, document.uri, "semanticTokens", async (uri: Uri) => { try { // Get the legend from the embedded language provider @@ -236,23 +247,24 @@ export function embeddedSemanticTokensProvider(engine: MarkdownEngine) { uri ); - const tokens = await commands.executeCommand( + const semanticTokens = await commands.executeCommand( "vscode.provideDocumentSemanticTokens", uri ); - if (!tokens || tokens.data.length === 0) { - return tokens; + if (!semanticTokens || semanticTokens.data.length === 0) { + return semanticTokens; } // Remap token indices from embedded provider's legend to our universal legend - let remappedTokens = tokens; + let remappedTokens = semanticTokens; if (legend) { - remappedTokens = remapTokenIndices(tokens, legend, QUARTO_SEMANTIC_TOKEN_LEGEND); + remappedTokens = remapTokenIndices(semanticTokens, legend, QUARTO_SEMANTIC_TOKEN_LEGEND); } - // Adjust token positions from virtual doc to real doc coordinates - return unadjustedSemanticTokens(vdoc.language, remappedTokens); + // Adjust token positions from virtual doc to real doc coordinates, + // keeping only tokens on code lines + return unadjustedSemanticTokens(vdoc.language, remappedTokens, codeLines); } catch (error) { return undefined; } diff --git a/apps/vscode/src/test/semanticTokens.test.ts b/apps/vscode/src/test/semanticTokens.test.ts index f2e0e8cb..d55feec2 100644 --- a/apps/vscode/src/test/semanticTokens.test.ts +++ b/apps/vscode/src/test/semanticTokens.test.ts @@ -1,6 +1,8 @@ import * as vscode from "vscode"; import * as assert from "assert"; import { decodeSemanticTokens, encodeSemanticTokens, remapTokenIndices } from "../providers/semantic-tokens"; +import { unadjustedSemanticTokens } from "../vdoc/vdoc"; +import { embeddedLanguage } from "../vdoc/languages"; suite("Semantic Tokens", function () { @@ -164,4 +166,25 @@ suite("Semantic Tokens", function () { assert.strictEqual(decoded[0].tokenModifiers, expectedModifiers, "Unmapped modifiers should be filtered out"); }); + test("Unadjusting semantic tokens drops tokens outside code lines", function () { + // Python injects 2 lines at the top of the virtual doc + const python = embeddedLanguage("python")!; + + // Virtual doc coordinates: lines 0-1 are injected, line 2 (real line 0) is a + // `#` filler line, and line 4 (real line 2) is code + const vdocTokens = encodeSemanticTokens([ + { line: 0, startChar: 0, length: 14, tokenType: 0, tokenModifiers: 0 }, // injected + { line: 2, startChar: 0, length: 1, tokenType: 0, tokenModifiers: 0 }, // filler `#` + { line: 4, startChar: 0, length: 1, tokenType: 1, tokenModifiers: 0 }, // code + { line: 4, startChar: 4, length: 3, tokenType: 2, tokenModifiers: 0 }, // code + ]); + + const decoded = decodeSemanticTokens(unadjustedSemanticTokens(python, vdocTokens, new Set([2]))); + + assert.deepStrictEqual(decoded, [ + { line: 2, startChar: 0, length: 1, tokenType: 1, tokenModifiers: 0 }, + { line: 2, startChar: 4, length: 3, tokenType: 2, tokenModifiers: 0 }, + ]); + }); + }); diff --git a/apps/vscode/src/vdoc/vdoc.ts b/apps/vscode/src/vdoc/vdoc.ts index 06f67b2a..1a693c61 100644 --- a/apps/vscode/src/vdoc/vdoc.ts +++ b/apps/vscode/src/vdoc/vdoc.ts @@ -287,30 +287,22 @@ export function unadjustedRange(language: EmbeddedLanguage, range: Range) { /** * Adjust semantic tokens from virtual document coordinates to real document coordinates * - * This function decodes the tokens, adjusts each token's position using unadjustedRange, - * and re-encodes them back to delta format. + * This function decodes the tokens, shifts each token's line using unadjustedLine, + * and re-encodes them back to delta format. When `lines` is given, only tokens on those + * real document lines are kept, dropping tokens on the virtual doc's filler and injected lines. */ export function unadjustedSemanticTokens( language: EmbeddedLanguage, - tokens: SemanticTokens + tokens: SemanticTokens, + lines?: Set ): SemanticTokens { // Decode tokens to absolute positions const decoded = decodeSemanticTokens(tokens); - // Adjust each token's position - const adjusted = decoded.map(t => { - const range = unadjustedRange(language, new Range( - new Position(t.line, t.startChar), - new Position(t.line, t.startChar + t.length) - )); - return { - line: range.start.line, - startChar: range.start.character, - length: range.end.character - range.start.character, - tokenType: t.tokenType, - tokenModifiers: t.tokenModifiers - }; - }); + // Adjust each token's line (tokens never span lines, so columns are unchanged) + const adjusted = decoded + .map(t => ({ ...t, line: unadjustedLine(language, t.line) })) + .filter(t => t.line >= 0 && (!lines || lines.has(t.line))); // Re-encode to delta format return encodeSemanticTokens(adjusted, tokens.resultId);