diff --git a/common/changes/@microsoft/tsdoc/fix-token-reader-subrange-bounds_2026-09-10.json b/common/changes/@microsoft/tsdoc/fix-token-reader-subrange-bounds_2026-09-10.json new file mode 100644 index 00000000..03fffd9f --- /dev/null +++ b/common/changes/@microsoft/tsdoc/fix-token-reader-subrange-bounds_2026-09-10.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/tsdoc", + "comment": "Enforce embedded TokenReader boundaries when peeking and backtracking.", + "type": "patch" + } + ], + "packageName": "@microsoft/tsdoc" +} diff --git a/tsdoc/src/parser/TokenReader.ts b/tsdoc/src/parser/TokenReader.ts index 770a06e7..fdf396bc 100644 --- a/tsdoc/src/parser/TokenReader.ts +++ b/tsdoc/src/parser/TokenReader.ts @@ -117,9 +117,13 @@ export class TokenReader { /** * Returns the next token that would be returned by _readToken(), without - * consuming anything. + * consuming anything. Throws if the reader has reached the end of an embedded range. */ public peekToken(): Token { + if (this._currentIndex >= this._readerEndIndex) { + // If this happens, it's a parser bug + throw new Error('Cannot peek past end of stream'); + } return this.tokens[this._currentIndex]; } @@ -180,7 +184,7 @@ export class TokenReader { * Returns the kind of the token immediately before the current token. */ public peekPreviousTokenKind(): TokenKind { - if (this._currentIndex === 0) { + if (this._currentIndex === this._readerStartIndex) { return TokenKind.EndOfInput; } return this.tokens[this._currentIndex - 1].kind; @@ -197,6 +201,10 @@ export class TokenReader { * Rewinds the stream pointer to a previous position in the stream. */ public backtrackToMarker(marker: number): void { + if (marker < this._readerStartIndex) { + // If this happens, it's a parser bug + throw new Error('The marker is outside the reader range'); + } if (marker > this._currentIndex) { // If this happens, it's a parser bug throw new Error('The marker has expired'); diff --git a/tsdoc/src/parser/__tests__/TokenReader.test.ts b/tsdoc/src/parser/__tests__/TokenReader.test.ts new file mode 100644 index 00000000..80d320bc --- /dev/null +++ b/tsdoc/src/parser/__tests__/TokenReader.test.ts @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { TSDocParser } from '../TSDocParser'; +import { TokenReader } from '../TokenReader'; +import { TokenSequence } from '../TokenSequence'; +import { TokenKind } from '../Token'; +import type { ParserContext } from '../ParserContext'; + +function createContext(): ParserContext { + return new TSDocParser().parseString('/** before inner after */'); +} + +function createEmbeddedReader(context: ParserContext, length: number = 1): TokenReader { + const startIndex: number = context.tokens.map((token) => token.toString()).indexOf('inner'); + expect(startIndex).toBeGreaterThan(0); + return new TokenReader( + context, + new TokenSequence({ parserContext: context, startIndex, endIndex: startIndex + length }) + ); +} + +describe('TokenReader subrange bounds', () => { + test('does not expose the token before an embedded range', () => { + const reader: TokenReader = createEmbeddedReader(createContext()); + expect(reader.peekPreviousTokenKind()).toBe(TokenKind.EndOfInput); + expect(reader.peekToken().toString()).toBe('inner'); + expect(reader.readToken().toString()).toBe('inner'); + expect(reader.peekPreviousTokenKind()).toBe(TokenKind.AsciiWord); + }); + + test('rejects peeking beyond an embedded range without consuming outer tokens', () => { + const reader: TokenReader = createEmbeddedReader(createContext()); + reader.readToken(); + const endMarker: number = reader.createMarker(); + expect(reader.peekTokenKind()).toBe(TokenKind.EndOfInput); + expect(reader.peekTokenAfterKind()).toBe(TokenKind.EndOfInput); + expect(reader.peekTokenAfterAfterKind()).toBe(TokenKind.EndOfInput); + expect(() => reader.peekToken()).toThrow('Cannot peek past end of stream'); + expect(() => reader.readToken()).toThrow('Cannot read past end of stream'); + expect(reader.createMarker()).toBe(endMarker); + }); + + test('honors an empty embedded range at a nonzero offset', () => { + const reader: TokenReader = createEmbeddedReader(createContext(), 0); + expect(reader.peekPreviousTokenKind()).toBe(TokenKind.EndOfInput); + expect(reader.peekTokenKind()).toBe(TokenKind.EndOfInput); + expect(() => reader.peekToken()).toThrow('Cannot peek past end of stream'); + expect(reader.isAccumulatedSequenceEmpty()).toBe(true); + }); + + test('honors an empty embedded range at zero', () => { + const context: ParserContext = createContext(); + const reader: TokenReader = new TokenReader(context, TokenSequence.createEmpty(context)); + expect(() => reader.peekToken()).toThrow('Cannot peek past end of stream'); + }); + + test('rejects backtracking before the embedded start without changing state', () => { + const reader: TokenReader = createEmbeddedReader(createContext()); + const startMarker: number = reader.createMarker(); + reader.readToken(); + const endMarker: number = reader.createMarker(); + expect(() => reader.backtrackToMarker(startMarker - 1)).toThrow('The marker is outside the reader range'); + expect(reader.createMarker()).toBe(endMarker); + expect(reader.extractAccumulatedSequence().toString()).toBe('inner'); + }); + + test('rejects negative markers for a full reader', () => { + const reader: TokenReader = new TokenReader(createContext()); + expect(() => reader.backtrackToMarker(-1)).toThrow('The marker is outside the reader range'); + expect(reader.createMarker()).toBe(0); + expect(reader.isAccumulatedSequenceEmpty()).toBe(true); + }); + + test('can rewind to the embedded start after extracting an accumulated sequence', () => { + const reader: TokenReader = createEmbeddedReader(createContext(), 3); + const startMarker: number = reader.createMarker(); + reader.readToken(); + reader.readToken(); + expect(reader.extractAccumulatedSequence().toString()).toBe('inner '); + reader.backtrackToMarker(startMarker); + expect(reader.peekPreviousTokenKind()).toBe(TokenKind.EndOfInput); + expect(reader.isAccumulatedSequenceEmpty()).toBe(true); + expect(reader.readToken().toString()).toBe('inner'); + expect(reader.extractAccumulatedSequence().toString()).toBe('inner'); + }); + + test('still rejects expired markers', () => { + const reader: TokenReader = createEmbeddedReader(createContext()); + const startMarker: number = reader.createMarker(); + reader.readToken(); + const endMarker: number = reader.createMarker(); + reader.backtrackToMarker(startMarker); + expect(() => reader.backtrackToMarker(endMarker)).toThrow('The marker has expired'); + expect(reader.createMarker()).toBe(startMarker); + }); + + test('preserves the full reader EndOfInput token', () => { + const reader: TokenReader = new TokenReader(createContext()); + while (reader.peekTokenKind() !== TokenKind.EndOfInput) { + expect(reader.peekToken()).toBe(reader.readToken()); + } + expect(reader.peekToken().kind).toBe(TokenKind.EndOfInput); + expect(() => reader.readToken()).toThrow('The EndOfInput token cannot be read'); + expect(reader.peekToken().kind).toBe(TokenKind.EndOfInput); + }); +});