nandi.uk
Fork of daniellemaywood.uk/gleam — Wasm codegen work
1// SPDX-License-Identifier: Apache-2.0
2// SPDX-FileCopyrightText: 2026 The Gleam contributors
3
4//! Lexer for semver ranges.
5//!
6//! Breaks a string of input into an iterator of tokens that can be used with a parser.
7//!
8//! Based off https://github.com/steveklabnik/semver-parser/blob/bee9de80aaa9653c5eb46a83658606cb21151e65/src/lexer.rs
9//!
10use self::Error::*;
11use self::Token::*;
12use std::str;
13
14macro_rules! scan_while {
15 ($slf:expr, $start:expr, $first:pat_param $(| $rest:pat)*) => {{
16 let mut __end = $start;
17
18 loop {
19 if let Some((idx, c)) = $slf.one() {
20 __end = idx;
21
22 match c {
23 $first $(| $rest)* => $slf.step(),
24 _ => break,
25 }
26
27 continue;
28 } else {
29 __end = $slf.input.len();
30 }
31
32 break;
33 }
34
35 __end
36 }}
37}
38
39/// Semver tokens.
40#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
41pub enum Token<'input> {
42 /// `==`
43 Eq,
44 /// `!=`
45 NotEq,
46 /// `>`
47 Gt,
48 /// `<`
49 Lt,
50 /// `<=`
51 LtEq,
52 /// `>=`
53 GtEq,
54 /// '~>`
55 Pessimistic,
56 /// `.`
57 Dot,
58 /// `-`
59 Hyphen,
60 /// `+`
61 Plus,
62 /// 'or'
63 Or,
64 /// 'and'
65 And,
66 /// any number of whitespace (`\t\r\n `) and its span.
67 Whitespace(usize, usize),
68 /// Numeric component, like `0` or `42`.
69 Numeric(u32),
70 /// Alphanumeric component, like `alpha1` or `79deadbe`.
71 AlphaNumeric(&'input str),
72 /// An alphanumeric component with a leading zero, like `0alpha1` or `079deadbe`.
73 LeadingZero(&'input str),
74}
75
76#[cfg(test)]
77impl<'input> Token<'input> {
78 /// Check if the current token is a whitespace token.
79 pub fn is_whitespace(&self) -> bool {
80 match *self {
81 Whitespace(..) => true,
82 _ => false,
83 }
84 }
85}
86
87impl std::fmt::Display for Token<'_> {
88 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
89 match self {
90 Eq => write!(f, "=="),
91 NotEq => write!(f, "!="),
92 Gt => write!(f, ">"),
93 Lt => write!(f, "<"),
94 LtEq => write!(f, "<="),
95 GtEq => write!(f, ">="),
96 Pessimistic => write!(f, "~>"),
97 Dot => write!(f, "."),
98 Hyphen => write!(f, "-"),
99 Plus => write!(f, "+"),
100 Or => write!(f, "or"),
101 And => write!(f, "and"),
102 Whitespace(_, _) => write!(f, " "),
103 Numeric(i) => write!(f, "{i}"),
104 AlphaNumeric(a) => write!(f, "{a}"),
105 LeadingZero(z) => write!(f, "{z}"),
106 }
107 }
108}
109
110#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, thiserror::Error)]
111pub enum Error {
112 #[error("Unexpected character {0}")]
113 UnexpectedChar(char),
114}
115
116/// Lexer for semver tokens belonging to a range.
117#[derive(Debug)]
118pub struct Lexer<'input> {
119 input: &'input str,
120 chars: str::CharIndices<'input>,
121 // lookahead
122 c1: Option<(usize, char)>,
123 c2: Option<(usize, char)>,
124}
125
126impl<'input> Lexer<'input> {
127 /// Construct a new lexer for the given input.
128 pub fn new(input: &str) -> Lexer<'_> {
129 let mut chars = input.char_indices();
130 let c1 = chars.next();
131 let c2 = chars.next();
132
133 Lexer {
134 input,
135 chars,
136 c1,
137 c2,
138 }
139 }
140
141 /// Shift all lookahead storage by one.
142 fn step(&mut self) {
143 self.c1 = self.c2;
144 self.c2 = self.chars.next();
145 }
146
147 fn step_n(&mut self, n: usize) {
148 for _ in 0..n {
149 self.step();
150 }
151 }
152
153 /// Access the one character, or set it if it is not set.
154 fn one(&mut self) -> Option<(usize, char)> {
155 self.c1
156 }
157
158 /// Access two characters.
159 fn two(&mut self) -> Option<(usize, char, char)> {
160 self.c1
161 .and_then(|(start, c1)| self.c2.map(|(_, c2)| (start, c1, c2)))
162 }
163
164 /// Consume a component.
165 ///
166 /// A component can either be an alphanumeric or numeric.
167 /// Does not permit leading zeroes.
168 fn component(&mut self, start: usize) -> Result<Token<'input>, Error> {
169 let end = scan_while!(self, start, '0'..='9' | 'A'..='Z' | 'a'..='z');
170 let input = &self.input[start..end];
171
172 let mut it = input.chars();
173 let (a, b) = (it.next(), it.next());
174
175 // exactly zero
176 if a == Some('0') && b.is_none() {
177 return Ok(Numeric(0));
178 }
179
180 if let Ok(numeric) = input.parse::<u32>() {
181 // Only parse as a number if there is no leading zero
182 if a != Some('0') {
183 return Ok(Numeric(numeric));
184 } else {
185 return Ok(LeadingZero(input));
186 }
187 }
188
189 Ok(AlphaNumeric(input))
190 }
191
192 fn and(&mut self, start: usize) -> Result<Token<'input>, Error> {
193 match self.one() {
194 Some((_, 'd')) => {
195 self.step();
196 Ok(And)
197 }
198 _ => self.component(start),
199 }
200 }
201
202 /// Consume whitespace.
203 fn whitespace(&mut self, start: usize) -> Result<Token<'input>, Error> {
204 let end = scan_while!(self, start, ' ' | '\t' | '\n' | '\r');
205 Ok(Whitespace(start, end))
206 }
207}
208
209impl<'input> Iterator for Lexer<'input> {
210 type Item = Result<Token<'input>, Error>;
211
212 fn next(&mut self) -> Option<Self::Item> {
213 #[allow(clippy::never_loop)]
214 loop {
215 // two subsequent char tokens.
216 if let Some((start, a, b)) = self.two() {
217 let two = match (a, b) {
218 ('~', '>') => Some(Pessimistic),
219 ('!', '=') => Some(NotEq),
220 ('<', '=') => Some(LtEq),
221 ('>', '=') => Some(GtEq),
222 ('=', '=') => Some(Eq),
223 ('o', 'r') => Some(Or),
224 ('a', 'n') => {
225 self.step_n(2);
226 return Some(self.and(start));
227 }
228 _ => None,
229 };
230
231 if let Some(two) = two {
232 self.step_n(2);
233 return Some(Ok(two));
234 }
235 }
236
237 // single char and start of numeric tokens.
238 if let Some((start, c)) = self.one() {
239 let tok = match c {
240 ' ' | '\t' | '\n' | '\r' => {
241 self.step();
242 return Some(self.whitespace(start));
243 }
244 '>' => Gt,
245 '<' => Lt,
246 '.' => Dot,
247 '-' => Hyphen,
248 '+' => Plus,
249 '0'..='9' | 'a'..='z' | 'A'..='Z' => {
250 self.step();
251 return Some(self.component(start));
252 }
253 c => return Some(Err(UnexpectedChar(c))),
254 };
255
256 self.step();
257 return Some(Ok(tok));
258 };
259
260 return None;
261 }
262 }
263}
264
265#[cfg(test)]
266mod tests {
267 use super::*;
268
269 fn lex(input: &str) -> Vec<Token<'_>> {
270 Lexer::new(input).map(Result::unwrap).collect::<Vec<_>>()
271 }
272
273 #[test]
274 pub fn simple_tokens() {
275 assert_eq!(
276 lex("!===><<=>=~>.-+orand"),
277 vec![
278 NotEq,
279 Eq,
280 Gt,
281 Lt,
282 LtEq,
283 GtEq,
284 Pessimistic,
285 Dot,
286 Hyphen,
287 Plus,
288 Or,
289 And
290 ]
291 );
292 }
293
294 #[test]
295 pub fn whitespace() {
296 assert_eq!(
297 lex(" foo \t\n\rbar"),
298 vec![
299 Whitespace(0, 2),
300 AlphaNumeric("foo"),
301 Whitespace(5, 9),
302 AlphaNumeric("bar"),
303 ]
304 );
305 }
306
307 #[test]
308 pub fn components() {
309 assert_eq!(lex("42"), vec![Numeric(42)]);
310 assert_eq!(lex("0"), vec![Numeric(0)]);
311 assert_eq!(lex("5885644aa"), vec![AlphaNumeric("5885644aa")]);
312 assert_eq!(lex("beta2"), vec![AlphaNumeric("beta2")]);
313 assert_eq!(lex("beta.2"), vec![AlphaNumeric("beta"), Dot, Numeric(2)]);
314 }
315
316 #[test]
317 pub fn empty() {
318 assert_eq!(lex(""), vec![]);
319 }
320
321 #[test]
322 pub fn numeric_all_numbers() {
323 let expected: Vec<Token> = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
324 .into_iter()
325 .map(Numeric)
326 .collect::<Vec<_>>();
327
328 let actual: Vec<_> = lex("0 1 2 3 4 5 6 7 8 9")
329 .into_iter()
330 .filter(|t| !t.is_whitespace())
331 .collect();
332
333 assert_eq!(actual, expected);
334 }
335
336 #[test]
337 pub fn token_display() {
338 assert_eq!(Eq.to_string(), "==");
339 assert_eq!(NotEq.to_string(), "!=");
340 assert_eq!(Gt.to_string(), ">");
341 assert_eq!(Lt.to_string(), "<");
342 assert_eq!(LtEq.to_string(), "<=");
343 assert_eq!(GtEq.to_string(), ">=");
344 assert_eq!(Pessimistic.to_string(), "~>");
345 assert_eq!(Dot.to_string(), ".");
346 assert_eq!(Hyphen.to_string(), "-");
347 assert_eq!(Plus.to_string(), "+");
348 assert_eq!(Or.to_string(), "or");
349 assert_eq!(And.to_string(), "and");
350 assert_eq!(Whitespace(0, 1).to_string(), " ");
351 assert_eq!(Numeric(42).to_string(), "42");
352 assert_eq!(AlphaNumeric("abc").to_string(), "abc");
353 assert_eq!(LeadingZero("012").to_string(), "012");
354 }
355}