swan_core/token.rs
1
2//! Tokens produced by the [`Lexer`](crate::lexer::Lexer).
3//!
4//! The precedence numbers in the operator docs are the binding powers used by
5//! the [`Parser`](crate::parser::Parser): a higher number binds tighter.
6//! Function application, which has no token, binds at 28.
7
8/// All token kinds produced by the [`Lexer`](crate::lexer::Lexer).
9///
10/// The lifetime `'a` is tied to the source string: `Identifier`, `Text`,
11/// `Comment`, and `Undefined` variants borrow slices directly from it, so
12/// no heap allocation is needed for those payloads.
13#[derive(Debug, PartialEq, Clone)]
14pub enum Token<'a> {
15
16 // -------------------------------------------------------------------------
17 // Literals and identifiers
18 // -------------------------------------------------------------------------
19
20 /// A name — starts with a letter or `_`, continues with letters, digits,
21 /// or `_`. Examples: `foo`, `_bar`, `myVar1`.
22 Identifier(&'a str),
23
24 /// A floating-point numeric literal. Integer, decimal, and scientific
25 /// notation are all parsed into an `f64`. Examples: `42`, `3.14`,
26 /// `1.5e-3`.
27 Number(f64),
28
29 /// A double-quoted string literal. The slice contains only the content
30 /// between the quotes — the delimiters are not included.
31 /// Example: `"hello"` → `Text("hello")`.
32 ///
33 /// There are no escape sequences: a backslash is an ordinary character,
34 /// and a text literal cannot contain `"`. A literal may span several lines.
35 ///
36 /// > **Note**: if the closing `"` is absent the slice extends to the end
37 /// > of the source string (the lexer does not error on unterminated strings).
38 Text(&'a str),
39
40 // -------------------------------------------------------------------------
41 // Grouping
42 // -------------------------------------------------------------------------
43
44 /// `(`
45 LeftRoundBracket,
46 /// `)`
47 RightRoundBracket,
48
49 /// `[` — opens a list literal.
50 LeftSquareBracket,
51 /// `]` — closes a list literal.
52 RightSquareBracket,
53
54 /// `{` — opens a namespace literal.
55 LeftCurlyBracket,
56 /// `}` — closes a namespace literal.
57 RightCurlyBracket,
58
59 // -------------------------------------------------------------------------
60 // Binary and unary operators (listed low → high precedence)
61 // -------------------------------------------------------------------------
62
63 /// `,` (precedence 10) — pairing / tuple construction.
64 PairingOp,
65
66 /// `=` (precedence 12) — assignment.
67 AssignmentOp,
68 /// `:` (precedence 12) — labelling / named argument.
69 LabellingOp,
70
71 /// `=>` (precedence 13, right-associative) — mapping: applies the
72 /// right-hand function(s) to each item of the left-hand tuple.
73 MappingOp,
74
75 /// `~>` (precedence 13, left-associative) — then: chains a function to
76 /// a promise.
77 ThenOp,
78
79 /// `>>` (precedence 14) — reverse composition / pipe (left-to-right).
80 ReverseCompositionOp,
81 /// `<<` (precedence 14) — composition (right-to-left).
82 CompositionOp,
83
84 /// `->` (precedence 15) — function definition (right-associative).
85 FunctionDefinitionOp,
86
87 /// `;` (precedence 21) — alternative: evaluates the right-hand side only
88 /// when the left-hand side is `Nothing`.
89 AlternativeOp,
90
91 /// `?` (precedence 22) — conditional: evaluates the right-hand side only
92 /// when the left-hand side is truthy.
93 ConditionalOp,
94
95 /// `|` (precedence 23) — logical OR.
96 OrOp,
97 /// `&` (precedence 23) — logical AND.
98 AndOp,
99
100 /// `==` (precedence 24) — equality comparison.
101 EqualToOp,
102 /// `!=` (precedence 24) — inequality comparison.
103 NotEqualToOp,
104 /// `<` (precedence 24) — less-than comparison.
105 LessThanOp,
106 /// `<=` (precedence 24) — less-than-or-equal comparison.
107 LessThanOrEqualToOp,
108 /// `>` (precedence 24) — greater-than comparison.
109 GreaterThanOp,
110 /// `>=` (precedence 24) — greater-than-or-equal comparison.
111 GreaterThanOrEqualToOp,
112
113 /// `+` (precedence 25) — addition.
114 PlusOp,
115 /// `-` (precedence 25) — subtraction.
116 MinusOp,
117
118 /// `*` (precedence 26) — multiplication.
119 ProductOp,
120 /// `/` (precedence 26) — division.
121 DivisionOp,
122 /// `%` (precedence 26) — modulo.
123 ModuloOp,
124
125 /// `^` (precedence 27) — exponentiation.
126 PowerOp,
127
128 /// `.` (precedence 30) — sub-contexting: looks up the right-hand name
129 /// inside the namespace produced by the left-hand expression.
130 /// Example: `math.sin` retrieves `sin` from the `math` namespace.
131 SubcontextingOp,
132
133 /// `!` (prefix) — logical negation.
134 NotOp,
135
136 /// `$` (prefix) — lifting: looks a name up starting from the outermost
137 /// scope.
138 LiftingOp,
139
140 // -------------------------------------------------------------------------
141 // Special tokens
142 // -------------------------------------------------------------------------
143
144 /// A character sequence the lexer could not classify as any valid token.
145 /// The slice covers exactly the unrecognised source characters.
146 ///
147 /// Examples: `'`, `@`, a malformed number like `1e+` or `12abc`.
148 Undefined(&'a str),
149
150 /// A `#`-prefixed line comment. The slice contains the comment text
151 /// *after* the `#` and *before* the terminating `\n` (or end of source).
152 ///
153 /// The lexer yields comments as tokens so that a tool such as a formatter
154 /// can use them. The [`Parser`](crate::parser::Parser) skips them.
155 Comment(&'a str),
156
157 /// Signals that the source string has been fully consumed.
158 ///
159 /// `next_token` returns this sentinel instead of `Option::None` so that
160 /// call-sites do not need to handle two distinct "no more input" states.
161 /// The `Iterator` impl converts it to `None`.
162 EndOfExpression,
163}
164