Skip to content

Commit 2894e63

Browse files
bartvenemanbartveneman
authored
dedupe token restoring (#15)
1 parent b765620 commit 2894e63

File tree

6 files changed

+74
-99
lines changed

6 files changed

+74
-99
lines changed

‎src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ export { type ParserOptions } from './parse'
1313

1414
// Types
1515
export { CSSNode, type CSSNodeType } from './css-node'
16+
export type { LexerPosition } from './lexer'
1617

1718
export {
1819
ATTR_OPERATOR_NONE,

‎src/lexer.ts‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,17 @@ const CHAR_UPPERCASE_E = 0x45 // E
5757
const CHAR_CARRIAGE_RETURN = 0x0d // \r
5858
const CHAR_LINE_FEED = 0x0a // \n
5959

60+
export interface LexerPosition {
61+
pos: number
62+
line: number
63+
column: number
64+
token_type: TokenType
65+
token_start: number
66+
token_end: number
67+
token_line: number
68+
token_column: number
69+
}
70+
6071
export class Lexer {
6172
source: string
6273
pos: number
@@ -542,4 +553,36 @@ export class Lexer {
542553
column: this.token_column,
543554
}
544555
}
556+
557+
/**
558+
* Save complete lexer state for backtracking
559+
* @returns Object containing all lexer state
560+
*/
561+
save_position(): LexerPosition {
562+
return {
563+
pos: this.pos,
564+
line: this.line,
565+
column: this.column,
566+
token_type: this.token_type,
567+
token_start: this.token_start,
568+
token_end: this.token_end,
569+
token_line: this.token_line,
570+
token_column: this.token_column,
571+
}
572+
}
573+
574+
/**
575+
* Restore lexer state from saved position
576+
* @param saved The saved position to restore
577+
*/
578+
restore_position(saved: LexerPosition): void {
579+
this.pos = saved.pos
580+
this.line = saved.line
581+
this.column = saved.column
582+
this.token_type = saved.token_type
583+
this.token_start = saved.token_start
584+
this.token_end = saved.token_end
585+
this.token_line = saved.token_line
586+
this.token_column = saved.token_column
587+
}
545588
}

‎src/parse-anplusb.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@ export class ANplusBParser {
136136
// Handle +n pattern
137137
if (this.lexer.token_type === TOKEN_DELIM && this.source.charCodeAt(this.lexer.token_start) === CHAR_PLUS) {
138138
// Look ahead for 'n'
139-
const saved_pos = this.lexer.pos
139+
const saved = this.lexer.save_position()
140140
this.lexer.next_token_fast(true)
141141

142142
if ((this.lexer.token_type as TokenType) === TOKEN_IDENT) {
@@ -146,7 +146,7 @@ export class ANplusBParser {
146146
if (first_char === 0x6e /* n */) {
147147
// Store +n as authored (including the +)
148148
a = '+n'
149-
a_start = saved_pos - 1 // Position of the + delim
149+
a_start = saved.pos - 1 // Position of the + delim
150150
a_end = this.lexer.token_start + 1
151151

152152
// Check for attached n-digit pattern
@@ -171,7 +171,7 @@ export class ANplusBParser {
171171
}
172172
}
173173

174-
this.lexer.pos = saved_pos
174+
this.lexer.restore_position(saved)
175175
}
176176

177177
// Handle dimension tokens: 2n, 3n+1, -5n-2

‎src/parse-atrule-prelude.ts‎

Lines changed: 6 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -506,9 +506,7 @@ export class AtRulePreludeParser {
506506
// Parse import layer: layer or layer(name)
507507
private parse_import_layer(): number | null {
508508
// Peek at next token
509-
let saved_pos = this.lexer.pos
510-
let saved_line = this.lexer.line
511-
let saved_column = this.lexer.column
509+
const saved = this.lexer.save_position()
512510

513511
this.next_token()
514512

@@ -569,18 +567,14 @@ export class AtRulePreludeParser {
569567
}
570568

571569
// Not a layer, restore position
572-
this.lexer.pos = saved_pos
573-
this.lexer.line = saved_line
574-
this.lexer.column = saved_column
570+
this.lexer.restore_position(saved)
575571
return null
576572
}
577573

578574
// Parse import supports: supports(condition)
579575
private parse_import_supports(): number | null {
580576
// Peek at next token
581-
let saved_pos = this.lexer.pos
582-
let saved_line = this.lexer.line
583-
let saved_column = this.lexer.column
577+
const saved = this.lexer.save_position()
584578

585579
this.next_token()
586580

@@ -621,9 +615,7 @@ export class AtRulePreludeParser {
621615
}
622616

623617
// Not supports(), restore position
624-
this.lexer.pos = saved_pos
625-
this.lexer.line = saved_line
626-
this.lexer.column = saved_column
618+
this.lexer.restore_position(saved)
627619
return null
628620
}
629621

@@ -634,16 +626,12 @@ export class AtRulePreludeParser {
634626

635627
// Helper: Peek at next token type without consuming
636628
private peek_token_type(): number {
637-
let saved_pos = this.lexer.pos
638-
let saved_line = this.lexer.line
639-
let saved_column = this.lexer.column
629+
const saved = this.lexer.save_position()
640630

641631
this.next_token()
642632
let type = this.lexer.token_type
643633

644-
this.lexer.pos = saved_pos
645-
this.lexer.line = saved_line
646-
this.lexer.column = saved_column
634+
this.lexer.restore_position(saved)
647635

648636
return type
649637
}

‎src/parse-selector.ts‎

Lines changed: 18 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -174,9 +174,7 @@ export class SelectorParser {
174174

175175
// Check for leading combinator (relative selector) if allowed
176176
if (allow_relative && this.lexer.pos < this.selector_end) {
177-
let saved_pos = this.lexer.pos
178-
let saved_line = this.lexer.line
179-
let saved_column = this.lexer.column
177+
const saved = this.lexer.save_position()
180178

181179
this.lexer.next_token_fast(false)
182180
let token_type = this.lexer.token_type
@@ -192,15 +190,11 @@ export class SelectorParser {
192190
// Continue to parse the rest normally
193191
} else {
194192
// Not a combinator, restore position
195-
this.lexer.pos = saved_pos
196-
this.lexer.line = saved_line
197-
this.lexer.column = saved_column
193+
this.lexer.restore_position(saved)
198194
}
199195
} else {
200196
// Not a delimiter, restore position
201-
this.lexer.pos = saved_pos
202-
this.lexer.line = saved_line
203-
this.lexer.column = saved_column
197+
this.lexer.restore_position(saved)
204198
}
205199
}
206200

@@ -225,25 +219,19 @@ export class SelectorParser {
225219
}
226220

227221
// Peek ahead for comma or end
228-
let saved_pos = this.lexer.pos
229-
let saved_line = this.lexer.line
230-
let saved_column = this.lexer.column
222+
const saved = this.lexer.save_position()
231223
this.skip_whitespace()
232224
if (this.lexer.pos >= this.selector_end) break
233225

234226
this.lexer.next_token_fast(false)
235227
let token_type = this.lexer.token_type
236228
if (token_type === TOKEN_COMMA || this.lexer.pos >= this.selector_end) {
237229
// Reset position for comma handling
238-
this.lexer.pos = saved_pos
239-
this.lexer.line = saved_line
240-
this.lexer.column = saved_column
230+
this.lexer.restore_position(saved)
241231
break
242232
}
243233
// Reset for next iteration
244-
this.lexer.pos = saved_pos
245-
this.lexer.line = saved_line
246-
this.lexer.column = saved_column
234+
this.lexer.restore_position(saved)
247235
break
248236
}
249237

@@ -271,9 +259,7 @@ export class SelectorParser {
271259

272260
while (this.lexer.pos < this.selector_end) {
273261
// Save lexer state before getting token
274-
let saved_pos = this.lexer.pos
275-
let saved_line = this.lexer.line
276-
let saved_column = this.lexer.column
262+
const saved = this.lexer.save_position()
277263
this.lexer.next_token_fast(false)
278264

279265
if (this.lexer.token_start >= this.selector_end) break
@@ -286,9 +272,7 @@ export class SelectorParser {
286272
parts.push(part)
287273
} else {
288274
// Not a simple selector part, restore lexer state and break
289-
this.lexer.pos = saved_pos
290-
this.lexer.line = saved_line
291-
this.lexer.column = saved_column
275+
this.lexer.restore_position(saved)
292276
break
293277
}
294278
}
@@ -409,17 +393,13 @@ export class SelectorParser {
409393
// Parse class selector (.classname)
410394
private parse_class_selector(dot_pos: number): number | null {
411395
// Save lexer state for potential restoration
412-
let saved_pos = this.lexer.pos
413-
let saved_line = this.lexer.line
414-
let saved_column = this.lexer.column
396+
const saved = this.lexer.save_position()
415397

416398
// Next token should be identifier
417399
this.lexer.next_token_fast(false)
418400
if (this.lexer.token_type !== TOKEN_IDENT) {
419401
// Restore lexer state and return null
420-
this.lexer.pos = saved_pos
421-
this.lexer.line = saved_line
422-
this.lexer.column = saved_column
402+
this.lexer.restore_position(saved)
423403
return null
424404
}
425405

@@ -607,9 +587,7 @@ export class SelectorParser {
607587
// Parse pseudo-class or pseudo-element (:hover, ::before)
608588
private parse_pseudo(start: number): number | null {
609589
// Save lexer state for potential restoration
610-
let saved_pos = this.lexer.pos
611-
let saved_line = this.lexer.line
612-
let saved_column = this.lexer.column
590+
const saved = this.lexer.save_position()
613591

614592
// Check for double colon (::)
615593
let is_pseudo_element = false
@@ -643,9 +621,7 @@ export class SelectorParser {
643621
}
644622

645623
// Restore lexer state and return null
646-
this.lexer.pos = saved_pos
647-
this.lexer.line = saved_line
648-
this.lexer.column = saved_column
624+
this.lexer.restore_position(saved)
649625
return null
650626
}
651627

@@ -716,9 +692,7 @@ export class SelectorParser {
716692
// Parse as selector (for :is(), :where(), :has(), etc.)
717693
// Save current lexer state and selector_end
718694
let saved_selector_end = this.selector_end
719-
let saved_pos = this.lexer.pos
720-
let saved_line = this.lexer.line
721-
let saved_column = this.lexer.column
695+
const saved = this.lexer.save_position()
722696

723697
// Recursively parse the content as a selector
724698
// Only :has() accepts relative selectors (starting with combinator)
@@ -727,9 +701,7 @@ export class SelectorParser {
727701

728702
// Restore lexer state and selector_end
729703
this.selector_end = saved_selector_end
730-
this.lexer.pos = saved_pos
731-
this.lexer.line = saved_line
732-
this.lexer.column = saved_column
704+
this.lexer.restore_position(saved)
733705

734706
// Add as child if parsed successfully
735707
if (child_selector !== null) {
@@ -759,9 +731,7 @@ export class SelectorParser {
759731
private parse_lang_identifiers(start: number, end: number, parent_node: number): void {
760732
// Save current lexer state
761733
let saved_selector_end = this.selector_end
762-
let saved_pos = this.lexer.pos
763-
let saved_line = this.lexer.line
764-
let saved_column = this.lexer.column
734+
const saved = this.lexer.save_position()
765735

766736
// Set lexer to parse this range
767737
this.lexer.pos = start
@@ -822,9 +792,7 @@ export class SelectorParser {
822792

823793
// Restore lexer state
824794
this.selector_end = saved_selector_end
825-
this.lexer.pos = saved_pos
826-
this.lexer.line = saved_line
827-
this.lexer.column = saved_column
795+
this.lexer.restore_position(saved)
828796
}
829797

830798
// Parse An+B expression for nth-* pseudo-classes
@@ -846,9 +814,7 @@ export class SelectorParser {
846814

847815
// Save current state
848816
let saved_selector_end = this.selector_end
849-
let saved_pos = this.lexer.pos
850-
let saved_line = this.lexer.line
851-
let saved_column = this.lexer.column
817+
const saved = this.lexer.save_position()
852818

853819
// Parse selector list
854820
this.selector_end = end
@@ -857,9 +823,7 @@ export class SelectorParser {
857823

858824
// Restore state
859825
this.selector_end = saved_selector_end
860-
this.lexer.pos = saved_pos
861-
this.lexer.line = saved_line
862-
this.lexer.column = saved_column
826+
this.lexer.restore_position(saved)
863827

864828
// Create NTH_OF wrapper
865829
let of_node = this.arena.create_node()

‎src/parse.ts‎

Lines changed: 3 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -283,28 +283,14 @@ export class Parser {
283283
let decl_column = this.lexer.token_column
284284

285285
// Lookahead: save lexer state before consuming
286-
let saved_pos = this.lexer.pos
287-
let saved_line = this.lexer.line
288-
let saved_column = this.lexer.column
289-
let saved_token_type = this.lexer.token_type
290-
let saved_token_start = this.lexer.token_start
291-
let saved_token_end = this.lexer.token_end
292-
let saved_token_line = this.lexer.token_line
293-
let saved_token_column = this.lexer.token_column
286+
const saved = this.lexer.save_position()
294287

295288
this.next_token() // consume property name
296289

297290
// Expect ':'
298291
if (this.peek_type() !== TOKEN_COLON) {
299292
// Restore lexer state and return null
300-
this.lexer.pos = saved_pos
301-
this.lexer.line = saved_line
302-
this.lexer.column = saved_column
303-
this.lexer.token_type = saved_token_type
304-
this.lexer.token_start = saved_token_start
305-
this.lexer.token_end = saved_token_end
306-
this.lexer.token_line = saved_token_line
307-
this.lexer.token_column = saved_token_column
293+
this.lexer.restore_position(saved)
308294
return null
309295
}
310296
this.next_token() // consume ':'
@@ -340,14 +326,7 @@ export class Parser {
340326
// If we encounter '{', this is actually a style rule, not a declaration
341327
if (token_type === TOKEN_LEFT_BRACE) {
342328
// Restore lexer state and return null
343-
this.lexer.pos = saved_pos
344-
this.lexer.line = saved_line
345-
this.lexer.column = saved_column
346-
this.lexer.token_type = saved_token_type
347-
this.lexer.token_start = saved_token_start
348-
this.lexer.token_end = saved_token_end
349-
this.lexer.token_line = saved_token_line
350-
this.lexer.token_column = saved_token_column
329+
this.lexer.restore_position(saved)
351330
return null
352331
}
353332

0 commit comments

Comments
 (0)