nandi.uk
Fork of daniellemaywood.uk/gleam — Wasm codegen work
50 kB
1374 lines
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: 2020 The Gleam contributors
3
4use ecow::EcoString;
5
6use crate::ast::SrcSpan;
7use crate::parse::LiteralFloatValue;
8use crate::parse::error::{LexicalError, LexicalErrorType};
9use crate::parse::token::Token;
10use std::char;
11use std::ops::Neg;
12
13use super::error::InvalidUnicodeEscapeError;
14
15#[derive(Debug)]
16pub struct Lexer<T: Iterator<Item = (u32, char)>> {
17 chars: T,
18 pending: Vec<Spanned>,
19 chr0: Option<char>,
20 chr1: Option<char>,
21 loc0: u32,
22 loc1: u32,
23}
24pub type Spanned = (u32, Token, u32);
25pub type LexResult = Result<Spanned, LexicalError>;
26
27pub fn string_to_keyword(word: &str) -> Option<Token> {
28 // Alphabetical keywords:
29 match word {
30 "as" => Some(Token::As),
31 "assert" => Some(Token::Assert),
32 "auto" => Some(Token::Auto),
33 "case" => Some(Token::Case),
34 "const" => Some(Token::Const),
35 "delegate" => Some(Token::Delegate),
36 "derive" => Some(Token::Derive),
37 "echo" => Some(Token::Echo),
38 "else" => Some(Token::Else),
39 "fn" => Some(Token::Fn),
40 "if" => Some(Token::If),
41 "implement" => Some(Token::Implement),
42 "import" => Some(Token::Import),
43 "let" => Some(Token::Let),
44 "macro" => Some(Token::Macro),
45 "opaque" => Some(Token::Opaque),
46 "panic" => Some(Token::Panic),
47 "pub" => Some(Token::Pub),
48 "test" => Some(Token::Test),
49 "todo" => Some(Token::Todo),
50 "type" => Some(Token::Type),
51 "use" => Some(Token::Use),
52 _ => None,
53 }
54}
55
56pub fn make_tokenizer(source: &str) -> impl Iterator<Item = LexResult> + '_ {
57 let chars = source.char_indices().map(|(i, c)| (i as u32, c));
58 let nlh = NewlineHandler::new(chars);
59 Lexer::new(nlh)
60}
61
62// The newline handler is an iterator which collapses different newline
63// types into \n always.
64#[derive(Debug)]
65pub struct NewlineHandler<T: Iterator<Item = (u32, char)>> {
66 source: T,
67 chr0: Option<(u32, char)>,
68 chr1: Option<(u32, char)>,
69}
70
71impl<T> NewlineHandler<T>
72where
73 T: Iterator<Item = (u32, char)>,
74{
75 pub fn new(source: T) -> Self {
76 let mut nlh = NewlineHandler {
77 source,
78 chr0: None,
79 chr1: None,
80 };
81 let _ = nlh.shift();
82 let _ = nlh.shift();
83 nlh
84 }
85
86 fn shift(&mut self) -> Option<(u32, char)> {
87 let result = self.chr0;
88 self.chr0 = self.chr1;
89 self.chr1 = self.source.next();
90 result
91 }
92}
93
94impl<T> Iterator for NewlineHandler<T>
95where
96 T: Iterator<Item = (u32, char)>,
97{
98 type Item = (u32, char);
99
100 fn next(&mut self) -> Option<Self::Item> {
101 // Collapse \r\n into \n
102 if let Some((i, '\r')) = self.chr0 {
103 if let Some((_, '\n')) = self.chr1 {
104 // Transform windows EOL into \n
105 let _ = self.shift();
106 // using the position from the \r
107 self.chr0 = Some((i, '\n'));
108 } else {
109 // Transform MAC EOL into \n
110 self.chr0 = Some((i, '\n'));
111 }
112 }
113
114 self.shift()
115 }
116}
117
118impl<T> Lexer<T>
119where
120 T: Iterator<Item = (u32, char)>,
121{
122 pub fn new(input: T) -> Self {
123 let mut lxr = Lexer {
124 chars: input,
125 pending: Vec::new(),
126 chr0: None,
127 chr1: None,
128 loc0: 0,
129 loc1: 0,
130 };
131 let _ = lxr.next_char();
132 let _ = lxr.next_char();
133
134 // Check whether the first character is a UTF-8 byte order mark, and if so, consume it.
135 if lxr.chr0 == Some('\u{feff}') {
136 let _ = lxr.next_char();
137 }
138
139 lxr
140 }
141
142 // This is the main entry point. Call this function to retrieve the next token.
143 // This function is used by the iterator implementation.
144 fn inner_next(&mut self) -> LexResult {
145 // top loop, keep on processing, until we have something pending.
146 while self.pending.is_empty() {
147 self.consume_normal()?;
148 }
149
150 Ok(self.pending.remove(0))
151 }
152
153 // Take a look at the next character, if any, and decide upon the next steps.
154 fn consume_normal(&mut self) -> Result<(), LexicalError> {
155 // Check if we have some character:
156 if let Some(c) = self.chr0 {
157 let mut check_for_minus = false;
158 if self.is_upname_start(c) {
159 let name = self.lex_upname()?;
160 self.emit(name);
161 } else if self.is_name_start(c) {
162 check_for_minus = true;
163 let name = self.lex_name()?;
164 self.emit(name);
165 } else if self.is_number_start(c, self.chr1) {
166 check_for_minus = true;
167 let num = self.lex_number()?;
168 self.emit(num);
169 } else {
170 self.consume_character(c)?;
171 }
172 if check_for_minus {
173 // We want to lex `1-1` and `x-1` as `1 - 1` and `x - 1`
174 if Some('-') == self.chr0 && self.is_number_start('-', self.chr1) {
175 self.eat_single_char(Token::Minus);
176 }
177 }
178 } else {
179 // We reached end of file.
180 let tok_pos = self.get_pos();
181 self.emit((tok_pos, Token::EndOfFile, tok_pos));
182 }
183
184 Ok(())
185 }
186
187 fn consume_character(&mut self, c: char) -> Result<(), LexicalError> {
188 match c {
189 '@' => {
190 self.eat_single_char(Token::At);
191 }
192 '"' => {
193 let string = self.lex_string()?;
194 self.emit(string);
195 }
196 '=' => {
197 let tok_start = self.get_pos();
198 let _ = self.next_char();
199 match self.chr0 {
200 Some('=') => {
201 let _ = self.next_char();
202 let tok_end = self.get_pos();
203 if let Some('=') = self.chr0 {
204 return Err(LexicalError {
205 error: LexicalErrorType::InvalidTripleEqual,
206 location: SrcSpan {
207 start: tok_start,
208 end: tok_end + 1,
209 },
210 });
211 }
212 self.emit((tok_start, Token::EqualEqual, tok_end));
213 }
214 _ => {
215 let tok_end = self.get_pos();
216 self.emit((tok_start, Token::Equal, tok_end));
217 }
218 }
219 }
220 '+' => {
221 let tok_start = self.get_pos();
222 let _ = self.next_char();
223 if let Some('.') = self.chr0 {
224 let _ = self.next_char();
225 let tok_end = self.get_pos();
226 self.emit((tok_start, Token::PlusDot, tok_end));
227 } else {
228 let tok_end = self.get_pos();
229 self.emit((tok_start, Token::Plus, tok_end));
230 }
231 }
232 '*' => {
233 let tok_start = self.get_pos();
234 let _ = self.next_char();
235 match self.chr0 {
236 Some('.') => {
237 let _ = self.next_char();
238 let tok_end = self.get_pos();
239 self.emit((tok_start, Token::StarDot, tok_end));
240 }
241 _ => {
242 let tok_end = self.get_pos();
243 self.emit((tok_start, Token::Star, tok_end));
244 }
245 }
246 }
247 '/' => {
248 let tok_start = self.get_pos();
249 let _ = self.next_char();
250 match self.chr0 {
251 Some('.') => {
252 let _ = self.next_char();
253 let tok_end = self.get_pos();
254 self.emit((tok_start, Token::SlashDot, tok_end));
255 }
256 Some('/') => {
257 let _ = self.next_char();
258 let comment = self.lex_comment();
259 self.emit(comment);
260 }
261 _ => {
262 let tok_end = self.get_pos();
263 self.emit((tok_start, Token::Slash, tok_end));
264 }
265 }
266 }
267 '%' => {
268 self.eat_single_char(Token::Percent);
269 }
270 '|' => {
271 let tok_start = self.get_pos();
272 let _ = self.next_char();
273 if let Some('|') = self.chr0 {
274 let _ = self.next_char();
275 let tok_end = self.get_pos();
276 self.emit((tok_start, Token::VbarVbar, tok_end));
277 } else if let Some('>') = self.chr0 {
278 let _ = self.next_char();
279 let tok_end = self.get_pos();
280 self.emit((tok_start, Token::Pipe, tok_end));
281 } else {
282 let tok_end = self.get_pos();
283 self.emit((tok_start, Token::Vbar, tok_end));
284 }
285 }
286 '&' => {
287 let tok_start = self.get_pos();
288 let _ = self.next_char();
289 if let Some('&') = self.chr0 {
290 let _ = self.next_char();
291 let tok_end = self.get_pos();
292 self.emit((tok_start, Token::AmperAmper, tok_end));
293 } else {
294 return Err(LexicalError {
295 error: LexicalErrorType::UnrecognizedToken { tok: '&' },
296 location: SrcSpan {
297 start: tok_start,
298 end: tok_start,
299 },
300 });
301 }
302 }
303 '-' => {
304 let tok_start = self.get_pos();
305 let _ = self.next_char();
306 match self.chr0 {
307 Some('.') => {
308 let _ = self.next_char();
309 let tok_end = self.get_pos();
310 self.emit((tok_start, Token::MinusDot, tok_end));
311 }
312 Some('>') => {
313 let _ = self.next_char();
314 let tok_end = self.get_pos();
315 self.emit((tok_start, Token::RArrow, tok_end));
316 }
317 _ => {
318 let tok_end = self.get_pos();
319 self.emit((tok_start, Token::Minus, tok_end));
320 }
321 }
322 }
323 '!' => {
324 let tok_start = self.get_pos();
325 let _ = self.next_char();
326 if let Some('=') = self.chr0 {
327 let _ = self.next_char();
328 let tok_end = self.get_pos();
329 self.emit((tok_start, Token::NotEqual, tok_end));
330 } else {
331 let tok_end = self.get_pos();
332 self.emit((tok_start, Token::Bang, tok_end));
333 }
334 }
335 '(' => {
336 self.eat_single_char(Token::LeftParen);
337 }
338 ')' => {
339 self.eat_single_char(Token::RightParen);
340 }
341 '[' => {
342 self.eat_single_char(Token::LeftSquare);
343 }
344 ']' => {
345 self.eat_single_char(Token::RightSquare);
346 }
347 '{' => {
348 self.eat_single_char(Token::LeftBrace);
349 }
350 '}' => {
351 self.eat_single_char(Token::RightBrace);
352 }
353 ':' => {
354 self.eat_single_char(Token::Colon);
355 }
356 '<' => {
357 let tok_start = self.get_pos();
358 let _ = self.next_char();
359 match self.chr0 {
360 Some('>') => {
361 let _ = self.next_char();
362 let tok_end = self.get_pos();
363 self.emit((tok_start, Token::Concatenate, tok_end));
364 }
365 Some('<') => {
366 let _ = self.next_char();
367 let tok_end = self.get_pos();
368 self.emit((tok_start, Token::LtLt, tok_end));
369 }
370 Some('.') => {
371 let _ = self.next_char();
372 let tok_end = self.get_pos();
373 self.emit((tok_start, Token::LessDot, tok_end));
374 }
375 Some('-') => {
376 let _ = self.next_char();
377 let tok_end = self.get_pos();
378 self.emit((tok_start, Token::LArrow, tok_end));
379 }
380 Some('=') => {
381 let _ = self.next_char();
382 match self.chr0 {
383 Some('.') => {
384 let _ = self.next_char();
385 let tok_end = self.get_pos();
386 self.emit((tok_start, Token::LessEqualDot, tok_end));
387 }
388 _ => {
389 let tok_end = self.get_pos();
390 self.emit((tok_start, Token::LessEqual, tok_end));
391 }
392 }
393 }
394 _ => {
395 let tok_end = self.get_pos();
396 self.emit((tok_start, Token::Less, tok_end));
397 }
398 }
399 }
400 '>' => {
401 let tok_start = self.get_pos();
402 let _ = self.next_char();
403 match self.chr0 {
404 Some('>') => {
405 let _ = self.next_char();
406 let tok_end = self.get_pos();
407 self.emit((tok_start, Token::GtGt, tok_end));
408 }
409 Some('.') => {
410 let _ = self.next_char();
411 let tok_end = self.get_pos();
412 self.emit((tok_start, Token::GreaterDot, tok_end));
413 }
414 Some('=') => {
415 let _ = self.next_char();
416 match self.chr0 {
417 Some('.') => {
418 let _ = self.next_char();
419 let tok_end = self.get_pos();
420 self.emit((tok_start, Token::GreaterEqualDot, tok_end));
421 }
422 _ => {
423 let tok_end = self.get_pos();
424 self.emit((tok_start, Token::GreaterEqual, tok_end));
425 }
426 }
427 }
428 _ => {
429 let tok_end = self.get_pos();
430 self.emit((tok_start, Token::Greater, tok_end));
431 }
432 }
433 }
434 ',' => {
435 self.eat_single_char(Token::Comma);
436 }
437 '.' => {
438 let tok_start = self.get_pos();
439 let _ = self.next_char();
440 if let Some('.') = &self.chr0 {
441 let _ = self.next_char();
442 let tok_end = self.get_pos();
443 self.emit((tok_start, Token::DotDot, tok_end));
444 } else {
445 let tok_end = self.get_pos();
446 self.emit((tok_start, Token::Dot, tok_end));
447 self.maybe_lex_dot_access()?;
448 }
449 }
450 '#' => {
451 self.eat_single_char(Token::Hash);
452 }
453 '\n' | ' ' | '\t' | '\x0C' => {
454 let tok_start = self.get_pos();
455 let _ = self.next_char();
456 let tok_end = self.get_pos();
457 if c == '\n' {
458 self.emit((tok_start, Token::NewLine, tok_end));
459 }
460 }
461 '\u{201A}' => {
462 let location = self.get_pos();
463 return Err(LexicalError {
464 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
465 name: "low single comma quotation mark",
466 correct: "comma",
467 },
468 location: SrcSpan {
469 start: location,
470 end: location,
471 },
472 });
473 }
474 '\u{FF3B}' => {
475 let location = self.get_pos();
476 return Err(LexicalError {
477 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
478 name: "fullwidth left square bracket",
479 correct: "left square bracket",
480 },
481 location: SrcSpan {
482 start: location,
483 end: location,
484 },
485 });
486 }
487 '\u{FF3D}' => {
488 let location = self.get_pos();
489 return Err(LexicalError {
490 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
491 name: "fullwidth right square bracket",
492 correct: "right square bracket",
493 },
494 location: SrcSpan {
495 start: location,
496 end: location,
497 },
498 });
499 }
500 '\u{FF08}' => {
501 let location = self.get_pos();
502 return Err(LexicalError {
503 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
504 name: "fullwidth left parenthesis",
505 correct: "left parenthesis",
506 },
507 location: SrcSpan {
508 start: location,
509 end: location,
510 },
511 });
512 }
513 '\u{FF09}' => {
514 let location = self.get_pos();
515 return Err(LexicalError {
516 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
517 name: "fullwidth right parenthesis",
518 correct: "right parenthesis",
519 },
520 location: SrcSpan {
521 start: location,
522 end: location,
523 },
524 });
525 }
526 '\u{FF0E}' => {
527 let location = self.get_pos();
528 return Err(LexicalError {
529 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
530 name: "fullwidth full stop",
531 correct: "dot",
532 },
533 location: SrcSpan {
534 start: location,
535 end: location,
536 },
537 });
538 }
539 '\u{3002}' => {
540 let location = self.get_pos();
541 return Err(LexicalError {
542 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
543 name: "ideographic full stop",
544 correct: "dot",
545 },
546 location: SrcSpan {
547 start: location,
548 end: location,
549 },
550 });
551 }
552 '\u{FF1C}' => {
553 let location = self.get_pos();
554 return Err(LexicalError {
555 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
556 name: "fullwidth less-than sign",
557 correct: "less-than sign",
558 },
559 location: SrcSpan {
560 start: location,
561 end: location,
562 },
563 });
564 }
565 '\u{FF1E}' => {
566 let location = self.get_pos();
567 return Err(LexicalError {
568 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
569 name: "fullwidth greater-than sign",
570 correct: "greater-than sign",
571 },
572 location: SrcSpan {
573 start: location,
574 end: location,
575 },
576 });
577 }
578 '\u{FF5C}' => {
579 let location = self.get_pos();
580 return Err(LexicalError {
581 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
582 name: "fullwidth vertical line",
583 correct: "pipe",
584 },
585 location: SrcSpan {
586 start: location,
587 end: location,
588 },
589 });
590 }
591 '\u{FF20}' => {
592 let location = self.get_pos();
593 return Err(LexicalError {
594 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
595 name: "fullwidth commercial at",
596 correct: "at sign",
597 },
598 location: SrcSpan {
599 start: location,
600 end: location,
601 },
602 });
603 }
604 '\u{FF3E}' => {
605 let location = self.get_pos();
606 return Err(LexicalError {
607 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
608 name: "fullwidth circumflex accent",
609 correct: "caret",
610 },
611 location: SrcSpan {
612 start: location,
613 end: location,
614 },
615 });
616 }
617 '\u{FF1A}' => {
618 let location = self.get_pos();
619 return Err(LexicalError {
620 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
621 name: "fullwidth colon",
622 correct: "colon",
623 },
624 location: SrcSpan {
625 start: location,
626 end: location,
627 },
628 });
629 }
630 // Visually similar characters that are not valid Gleam source.
631 '\u{201C}' | '\u{201D}' => {
632 let location = self.get_pos();
633 return Err(LexicalError {
634 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
635 name: "double quotation mark",
636 correct: "double quote",
637 },
638 location: SrcSpan {
639 start: location,
640 end: location,
641 },
642 });
643 }
644 '\u{2018}' | '\u{2019}' => {
645 let location = self.get_pos();
646 return Err(LexicalError {
647 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
648 name: "single quotation mark",
649 correct: "single quote",
650 },
651 location: SrcSpan {
652 start: location,
653 end: location,
654 },
655 });
656 }
657 '\u{2013}' => {
658 let location = self.get_pos();
659 return Err(LexicalError {
660 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
661 name: "en dash",
662 correct: "minus sign",
663 },
664 location: SrcSpan {
665 start: location,
666 end: location,
667 },
668 });
669 }
670 '\u{2014}' => {
671 let location = self.get_pos();
672 return Err(LexicalError {
673 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
674 name: "em dash",
675 correct: "minus sign",
676 },
677 location: SrcSpan {
678 start: location,
679 end: location,
680 },
681 });
682 }
683 '\u{2217}' => {
684 let location = self.get_pos();
685 return Err(LexicalError {
686 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
687 name: "asterisk operator",
688 correct: "asterisk",
689 },
690 location: SrcSpan {
691 start: location,
692 end: location,
693 },
694 });
695 }
696 '\u{2215}' => {
697 let location = self.get_pos();
698 return Err(LexicalError {
699 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
700 name: "division slash",
701 correct: "forward slash",
702 },
703 location: SrcSpan {
704 start: location,
705 end: location,
706 },
707 });
708 }
709 '\u{00A0}' => {
710 let location = self.get_pos();
711 return Err(LexicalError {
712 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
713 name: "non-breaking space",
714 correct: "space",
715 },
716 location: SrcSpan {
717 start: location,
718 end: location,
719 },
720 });
721 }
722 '\u{200B}' => {
723 let location = self.get_pos();
724 return Err(LexicalError {
725 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
726 name: "zero-width space",
727 correct: "space",
728 },
729 location: SrcSpan {
730 start: location,
731 end: location,
732 },
733 });
734 }
735 '\u{0430}' => {
736 let location = self.get_pos();
737 return Err(LexicalError {
738 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
739 name: "Cyrillic letter а",
740 correct: "latin letter a",
741 },
742 location: SrcSpan {
743 start: location,
744 end: location,
745 },
746 });
747 }
748 '\u{0435}' => {
749 let location = self.get_pos();
750 return Err(LexicalError {
751 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
752 name: "Cyrillic letter е",
753 correct: "latin letter e",
754 },
755 location: SrcSpan {
756 start: location,
757 end: location,
758 },
759 });
760 }
761 '\u{043E}' => {
762 let location = self.get_pos();
763 return Err(LexicalError {
764 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
765 name: "Cyrillic letter о",
766 correct: "latin letter o",
767 },
768 location: SrcSpan {
769 start: location,
770 end: location,
771 },
772 });
773 }
774 '\u{0440}' => {
775 let location = self.get_pos();
776 return Err(LexicalError {
777 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
778 name: "Cyrillic letter р",
779 correct: "latin letter p",
780 },
781 location: SrcSpan {
782 start: location,
783 end: location,
784 },
785 });
786 }
787 '\u{1D35}' => {
788 let location = self.get_pos();
789 return Err(LexicalError {
790 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
791 name: "modifier letter capital I",
792 correct: "latin letter I",
793 },
794 location: SrcSpan {
795 start: location,
796 end: location,
797 },
798 });
799 }
800 '\u{FF0C}' => {
801 let location = self.get_pos();
802 return Err(LexicalError {
803 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
804 name: "fullwidth comma",
805 correct: "comma",
806 },
807 location: SrcSpan {
808 start: location,
809 end: location,
810 },
811 });
812 }
813 '\u{3001}' => {
814 let location = self.get_pos();
815 return Err(LexicalError {
816 error: LexicalErrorType::VisuallySimilarInvalidCharacter {
817 name: "ideographic comma",
818 correct: "comma",
819 },
820 location: SrcSpan {
821 start: location,
822 end: location,
823 },
824 });
825 }
826 c => {
827 let location = self.get_pos();
828 return Err(LexicalError {
829 error: LexicalErrorType::UnrecognizedToken { tok: c },
830 location: SrcSpan {
831 start: location,
832 end: location,
833 },
834 });
835 }
836 }
837
838 Ok(())
839 }
840
841 // Lexer helper functions:
842 // this can be either a reserved word, or a name
843 fn lex_name(&mut self) -> LexResult {
844 let mut name = String::new();
845 let start_pos = self.get_pos();
846
847 while self.is_name_continuation() {
848 name.push(self.next_char().expect("lex_name continue"));
849 }
850
851 let end_pos = self.get_pos();
852
853 match string_to_keyword(&name) {
854 Some(tok) => Ok((start_pos, tok, end_pos)),
855 _ => {
856 if name.starts_with('_') {
857 Ok((start_pos, Token::DiscardName { name: name.into() }, end_pos))
858 } else {
859 Ok((start_pos, Token::Name { name: name.into() }, end_pos))
860 }
861 }
862 }
863 }
864 // A type name or constructor
865 fn lex_upname(&mut self) -> LexResult {
866 let mut name = String::new();
867 let start_pos = self.get_pos();
868
869 while self.is_name_continuation() {
870 name.push(self.next_char().expect("lex_upname upname"));
871 }
872
873 let end_pos = self.get_pos();
874
875 match string_to_keyword(&name) {
876 Some(tok) => Ok((start_pos, tok, end_pos)),
877 _ => Ok((start_pos, Token::UpName { name: name.into() }, end_pos)),
878 }
879 }
880
881 fn lex_number(&mut self) -> LexResult {
882 let start_pos = self.get_pos();
883
884 // We call this function after making sure that what comes next starts
885 // with what seems to be a valid number. If we see that it starts with
886 // `-` we consume the token and record that the number is negative.
887 let is_negative = if self.chr0 == Some('-') {
888 let _ = self.next_char();
889 true
890 } else {
891 false
892 };
893
894 let num = if self.chr0 == Some('0') {
895 match self.chr1 {
896 Some('x' | 'X') => {
897 // Hex!
898 let _ = self.next_char();
899 let _ = self.next_char();
900 self.lex_number_radix(start_pos, 16, is_negative, "0x")?
901 }
902 Some('o' | 'O') => {
903 // Octal!
904 let _ = self.next_char();
905 let _ = self.next_char();
906 self.lex_number_radix(start_pos, 8, is_negative, "0o")?
907 }
908 Some('b' | 'B') => {
909 // Binary!
910 let _ = self.next_char();
911 let _ = self.next_char();
912 self.lex_number_radix(start_pos, 2, is_negative, "0b")?
913 }
914 _ => self.lex_decimal_number(start_pos, is_negative)?,
915 }
916 } else {
917 self.lex_decimal_number(start_pos, is_negative)?
918 };
919
920 if Some('_') == self.chr0 {
921 let location = self.get_pos();
922 Err(LexicalError {
923 error: LexicalErrorType::NumTrailingUnderscore,
924 location: SrcSpan {
925 start: location,
926 end: location,
927 },
928 })
929 } else {
930 Ok(num)
931 }
932 }
933
934 // Lex a hex/octal/decimal/binary number without a decimal point.
935 fn lex_number_radix(
936 &mut self,
937 start_pos: u32,
938 radix: u32,
939 is_negative: bool,
940 prefix: &str,
941 ) -> LexResult {
942 let num = self.radix_run(radix);
943 if num.is_empty() {
944 let location = self.get_pos() - 1;
945 Err(LexicalError {
946 error: LexicalErrorType::RadixIntNoValue,
947 location: SrcSpan {
948 start: location,
949 end: location,
950 },
951 })
952 } else if radix < 16 && Lexer::<T>::is_digit_of_radix(self.chr0, 16) {
953 let location = self.get_pos();
954 Err(LexicalError {
955 error: LexicalErrorType::DigitOutOfRadix,
956 location: SrcSpan {
957 start: location,
958 end: location,
959 },
960 })
961 } else {
962 let value = format!("{prefix}{num}");
963 let int_value = super::parse_int_value(&value).expect("int value to parse as bigint");
964 let end_pos = self.get_pos();
965
966 let (value, int_value) = if is_negative {
967 (format!("-{value}"), int_value.neg())
968 } else {
969 (value, int_value)
970 };
971
972 Ok((
973 start_pos,
974 Token::Int {
975 value: value.into(),
976 int_value,
977 },
978 end_pos,
979 ))
980 }
981 }
982
983 // Lex a normal number, that is, no octal, hex or binary number.
984 // This function cannot be reached without the head of the stream being either 0-9 or '-', 0-9
985 fn lex_decimal_number(&mut self, start_pos: u32, is_negative: bool) -> LexResult {
986 self.lex_decimal_or_int_number(start_pos, is_negative, true)
987 }
988
989 fn lex_int_number(&mut self, start_pos: u32, is_negative: bool) -> LexResult {
990 self.lex_decimal_or_int_number(start_pos, is_negative, false)
991 }
992
993 fn lex_decimal_or_int_number(
994 &mut self,
995 start_pos: u32,
996 is_negative: bool,
997 can_lex_decimal: bool,
998 ) -> LexResult {
999 let mut value = String::new();
1000 if is_negative {
1001 value.push('-')
1002 };
1003 // consume first run of digits
1004 value.push_str(&self.radix_run(10));
1005
1006 // If float:
1007 if can_lex_decimal && self.chr0 == Some('.') {
1008 value.push(self.next_char().expect("lex_normal_number float"));
1009 value.push_str(&self.radix_run(10));
1010
1011 // If scientific:
1012 if self.chr0 == Some('e') {
1013 value.push(self.next_char().expect("lex_normal_number scientific"));
1014 if self.chr0 == Some('-') {
1015 value.push(
1016 self.next_char()
1017 .expect("lex_normal_number scientific negative"),
1018 );
1019 }
1020 let exponent_run = self.radix_run(10);
1021 if exponent_run.is_empty() {
1022 return Err(LexicalError {
1023 error: LexicalErrorType::MissingExponent,
1024 location: SrcSpan::new(start_pos, self.get_pos()),
1025 });
1026 }
1027 value.push_str(&exponent_run);
1028 }
1029 let end_pos = self.get_pos();
1030 let float_value =
1031 LiteralFloatValue::parse(&value).expect("float value to parse as non-NaN f64");
1032 Ok((
1033 start_pos,
1034 Token::Float {
1035 value: value.into(),
1036 float_value,
1037 },
1038 end_pos,
1039 ))
1040 } else {
1041 let int_value = super::parse_int_value(&value).expect("int value to parse as bigint");
1042 let end_pos = self.get_pos();
1043 Ok((
1044 start_pos,
1045 Token::Int {
1046 value: value.into(),
1047 int_value,
1048 },
1049 end_pos,
1050 ))
1051 }
1052 }
1053
1054 // Maybe lex dot access that comes after name token.
1055 fn maybe_lex_dot_access(&mut self) -> Result<(), LexicalError> {
1056 // It can be nested like: `tuple.1.2.3.4`
1057 loop {
1058 if matches!(self.chr0, Some('0'..='9')) {
1059 let number = self.lex_int_number(self.get_pos(), false)?;
1060 self.emit(number);
1061 } else {
1062 break;
1063 }
1064 }
1065 Ok(())
1066 }
1067
1068 // Consume a sequence of numbers with the given radix,
1069 // the digits can be decorated with underscores
1070 // like this: '1_2_3_4' == '1234'
1071 fn radix_run(&mut self, radix: u32) -> String {
1072 let mut value_text = String::new();
1073
1074 loop {
1075 if let Some(c) = self.take_number(radix) {
1076 value_text.push(c);
1077 } else if self.chr0 == Some('_') && Lexer::<T>::is_digit_of_radix(self.chr1, radix) {
1078 value_text.push('_');
1079 let _ = self.next_char();
1080 } else {
1081 break;
1082 }
1083 }
1084 value_text
1085 }
1086
1087 // Consume a single character with the given radix.
1088 fn take_number(&mut self, radix: u32) -> Option<char> {
1089 let take_char = Lexer::<T>::is_digit_of_radix(self.chr0, radix);
1090
1091 if take_char {
1092 Some(self.next_char().expect("take_number next char"))
1093 } else {
1094 None
1095 }
1096 }
1097
1098 // Test if a digit is of a certain radix.
1099 fn is_digit_of_radix(c: Option<char>, radix: u32) -> bool {
1100 match radix {
1101 2 | 8 | 10 | 16 => c.filter(|c| c.is_digit(radix)).is_some(),
1102 other => panic!("Radix not implemented: {other}"),
1103 }
1104 }
1105
1106 // There are 3 kinds of comments
1107 // 2 slash, normal
1108 // 3 slash, document
1109 // 4 slash, module
1110 // this function is entered after 2 slashes
1111 fn lex_comment(&mut self) -> Spanned {
1112 enum Kind {
1113 Comment,
1114 Doc,
1115 ModuleDoc,
1116 }
1117 let kind = match (self.chr0, self.chr1) {
1118 (Some('/'), Some('/')) => {
1119 let _ = self.next_char();
1120 let _ = self.next_char();
1121 Kind::ModuleDoc
1122 }
1123 (Some('/'), _) => {
1124 let _ = self.next_char();
1125 Kind::Doc
1126 }
1127 _ => Kind::Comment,
1128 };
1129 let mut content = EcoString::new();
1130 let start_pos = self.get_pos();
1131 while Some('\n') != self.chr0 {
1132 match self.chr0 {
1133 Some(c) => content.push(c),
1134 None => break,
1135 }
1136 let _ = self.next_char();
1137 }
1138 let end_pos = self.get_pos();
1139 let token = match kind {
1140 Kind::Comment => Token::CommentNormal,
1141 Kind::Doc => Token::CommentDoc { content },
1142 Kind::ModuleDoc => Token::CommentModule,
1143 };
1144 (start_pos, token, end_pos)
1145 }
1146
1147 fn lex_string(&mut self) -> LexResult {
1148 let start_pos = self.get_pos();
1149 // advance past the first quote
1150 let _ = self.next_char();
1151 let mut string_content = String::new();
1152
1153 loop {
1154 match self.next_char() {
1155 Some('\\') => {
1156 let slash_pos = self.get_pos() - 1;
1157 if let Some(c) = self.chr0 {
1158 match c {
1159 'f' | 'n' | 'r' | 't' | '"' | '\\' => {
1160 let _ = self.next_char();
1161 string_content.push('\\');
1162 string_content.push(c);
1163 }
1164 'u' => {
1165 let _ = self.next_char();
1166
1167 if self.chr0 != Some('{') {
1168 return Err(LexicalError {
1169 error: LexicalErrorType::InvalidUnicodeEscape(
1170 InvalidUnicodeEscapeError::MissingOpeningBrace,
1171 ),
1172 location: SrcSpan {
1173 start: self.get_pos() - 1,
1174 end: self.get_pos(),
1175 },
1176 });
1177 }
1178
1179 // All digits inside \u{...}.
1180 let mut hex_digits = String::new();
1181
1182 loop {
1183 let _ = self.next_char();
1184
1185 let Some(chr) = self.chr0 else {
1186 break;
1187 };
1188
1189 // Don't break early when we've reached 6 digits to ensure a
1190 // useful error message
1191 if chr == '}' {
1192 break;
1193 }
1194
1195 hex_digits.push(chr);
1196
1197 if !chr.is_ascii_hexdigit() {
1198 return Err(LexicalError {
1199 error: LexicalErrorType::InvalidUnicodeEscape(
1200 InvalidUnicodeEscapeError::ExpectedHexDigitOrCloseBrace,
1201 ),
1202 location: SrcSpan {
1203 start: self.get_pos(),
1204 end: self.get_pos() + 1,
1205 },
1206 });
1207 }
1208 }
1209
1210 if self.chr0 != Some('}') {
1211 return Err(LexicalError {
1212 error: LexicalErrorType::InvalidUnicodeEscape(
1213 InvalidUnicodeEscapeError::ExpectedHexDigitOrCloseBrace,
1214 ),
1215 location: SrcSpan {
1216 start: self.get_pos() - 1,
1217 end: self.get_pos(),
1218 },
1219 });
1220 }
1221
1222 let _ = self.next_char();
1223
1224 if !(1..=6).contains(&hex_digits.len()) {
1225 return Err(LexicalError {
1226 error: LexicalErrorType::InvalidUnicodeEscape(
1227 InvalidUnicodeEscapeError::InvalidNumberOfHexDigits,
1228 ),
1229 location: SrcSpan {
1230 start: slash_pos,
1231 end: self.get_pos(),
1232 },
1233 });
1234 }
1235
1236 // Checks for i >= 0x110000 || (i >= 0xD800 && i < 0xE000),
1237 // where i is the unicode codepoint.
1238 if char::from_u32(u32::from_str_radix(&hex_digits, 16).expect(
1239 "Cannot parse codepoint number in Unicode escape sequence",
1240 ))
1241 .is_none()
1242 {
1243 return Err(LexicalError {
1244 error: LexicalErrorType::InvalidUnicodeEscape(
1245 InvalidUnicodeEscapeError::InvalidCodepoint,
1246 ),
1247 location: SrcSpan {
1248 start: slash_pos,
1249 end: self.get_pos(),
1250 },
1251 });
1252 }
1253
1254 string_content.push_str("\\u{");
1255 string_content.push_str(&hex_digits);
1256 string_content.push('}');
1257 }
1258 _ => {
1259 return Err(LexicalError {
1260 error: LexicalErrorType::BadStringEscape,
1261 location: SrcSpan {
1262 start: slash_pos,
1263 end: slash_pos + 1,
1264 },
1265 });
1266 }
1267 }
1268 } else {
1269 return Err(LexicalError {
1270 error: LexicalErrorType::BadStringEscape,
1271 location: SrcSpan {
1272 start: slash_pos,
1273 end: slash_pos,
1274 },
1275 });
1276 }
1277 }
1278 Some('"') => break,
1279 Some(c) => string_content.push(c),
1280 None => {
1281 return Err(LexicalError {
1282 error: LexicalErrorType::UnexpectedStringEnd,
1283 location: SrcSpan {
1284 start: start_pos,
1285 end: start_pos,
1286 },
1287 });
1288 }
1289 }
1290 }
1291 let end_pos = self.get_pos();
1292
1293 let tok = Token::String {
1294 value: string_content.into(),
1295 };
1296
1297 Ok((start_pos, tok, end_pos))
1298 }
1299
1300 fn is_name_start(&self, c: char) -> bool {
1301 matches!(c, '_' | 'a'..='z')
1302 }
1303 fn is_upname_start(&self, c: char) -> bool {
1304 c.is_ascii_uppercase()
1305 }
1306 fn is_number_start(&self, c: char, c1: Option<char>) -> bool {
1307 match c {
1308 '0'..='9' => true,
1309 '-' => matches!(c1, Some('0'..='9')),
1310 _ => false,
1311 }
1312 }
1313
1314 fn is_name_continuation(&self) -> bool {
1315 self.chr0
1316 .map(|c| matches!(c, '_' | '0'..='9' | 'a'..='z' | 'A'..='Z'))
1317 .unwrap_or(false)
1318 }
1319
1320 // advance the stream and emit a token
1321 fn eat_single_char(&mut self, ty: Token) {
1322 let tok_start = self.get_pos();
1323 let _ = self.next_char().expect("eat_single_char");
1324 let tok_end = self.get_pos();
1325 self.emit((tok_start, ty, tok_end));
1326 }
1327
1328 // Helper function to go to the next character coming up.
1329 fn next_char(&mut self) -> Option<char> {
1330 let c = self.chr0;
1331 let nxt = match self.chars.next() {
1332 Some((loc, c)) => {
1333 self.loc0 = self.loc1;
1334 self.loc1 = loc;
1335 Some(c)
1336 }
1337 None => {
1338 // EOF needs a single advance
1339 self.loc0 = self.loc1;
1340 self.loc1 += 1;
1341 None
1342 }
1343 };
1344 self.chr0 = self.chr1;
1345 self.chr1 = nxt;
1346 c
1347 }
1348
1349 // Helper function to retrieve the current position.
1350 fn get_pos(&self) -> u32 {
1351 self.loc0
1352 }
1353
1354 // Helper function to emit a lexed token to the queue of tokens.
1355 fn emit(&mut self, spanned: Spanned) {
1356 self.pending.push(spanned);
1357 }
1358}
1359
1360impl<T> Iterator for Lexer<T>
1361where
1362 T: Iterator<Item = (u32, char)>,
1363{
1364 type Item = LexResult;
1365
1366 fn next(&mut self) -> Option<Self::Item> {
1367 let token = self.inner_next();
1368
1369 match token {
1370 Ok((_, Token::EndOfFile, _)) => None,
1371 r => Some(r),
1372 }
1373 }
1374}