flux_syntax/
token.rs

1use std::fmt;
2
3pub use TokenKind::*;
4use rustc_ast::token::InvisibleOrigin;
5pub use rustc_ast::token::{Delimiter, IdentIsRaw, Lit, LitKind};
6use rustc_span::{BytePos, Symbol};
7
8#[derive(Copy, Clone, Debug, PartialEq)]
9pub enum TokenKind {
10    Caret,
11    Or,
12    EqEq,
13    Eq,
14    AndAnd,
15    OrOr,
16    Plus,
17    Minus,
18    Slash,
19    Bang,
20    Star,
21    Colon,
22    Comma,
23    Semi,
24    RArrow,
25    Dot,
26    Le,
27    Ne,
28    GtFollowedByGt,
29    Gt,
30    LtFollowedByLt,
31    Lt,
32    Ge,
33    At,
34    Pound,
35    Iff,
36    FatArrow,
37    Literal(Lit),
38    Ident(Symbol, IdentIsRaw),
39    OpenParen,
40    CloseParen,
41    OpenBrace,
42    CloseBrace,
43    OpenBracket,
44    CloseBracket,
45    OpenInvisible(InvisibleOrigin),
46    CloseInvisible(InvisibleOrigin),
47    Invalid,
48    And,
49    Percent,
50    PathSep,
51    DotDot,
52    Eof,
53}
54
55#[derive(Clone, Copy)]
56pub struct Token {
57    pub kind: TokenKind,
58    pub lo: BytePos,
59    pub hi: BytePos,
60}
61
62impl Token {
63    pub fn new(kind: TokenKind, lo: BytePos, hi: BytePos) -> Self {
64        Self { kind, lo, hi }
65    }
66}
67
68impl TokenKind {
69    pub fn open_delim(delim: Delimiter) -> TokenKind {
70        match delim {
71            Delimiter::Parenthesis => TokenKind::OpenParen,
72            Delimiter::Bracket => TokenKind::OpenBracket,
73            Delimiter::Brace => TokenKind::OpenBrace,
74            Delimiter::Invisible(origin) => TokenKind::OpenInvisible(origin),
75        }
76    }
77
78    pub fn close_delim(delim: Delimiter) -> TokenKind {
79        match delim {
80            Delimiter::Parenthesis => TokenKind::CloseParen,
81            Delimiter::Bracket => TokenKind::CloseBracket,
82            Delimiter::Brace => TokenKind::CloseBrace,
83            Delimiter::Invisible(origin) => TokenKind::CloseInvisible(origin),
84        }
85    }
86
87    pub fn as_open_delim(&self) -> Option<Delimiter> {
88        match self {
89            Self::OpenParen => Some(Delimiter::Parenthesis),
90            Self::OpenBrace => Some(Delimiter::Brace),
91            Self::OpenBracket => Some(Delimiter::Bracket),
92            _ => None,
93        }
94    }
95
96    pub fn as_close_delim(&self) -> Option<Delimiter> {
97        match self {
98            Self::CloseParen => Some(Delimiter::Parenthesis),
99            Self::CloseBrace => Some(Delimiter::Brace),
100            Self::CloseBracket => Some(Delimiter::Bracket),
101            _ => None,
102        }
103    }
104
105    pub fn descr(&self) -> &'static str {
106        match self {
107            TokenKind::Caret => "^",
108            TokenKind::Or => "|",
109            TokenKind::EqEq => "==",
110            TokenKind::Eq => "=",
111            TokenKind::AndAnd => "&&",
112            TokenKind::OrOr => "||",
113            TokenKind::Plus => "+",
114            TokenKind::Minus => "-",
115            TokenKind::Slash => "/",
116            TokenKind::Bang => "!",
117            TokenKind::Star => "*",
118            TokenKind::Colon => ":",
119            TokenKind::Comma => ",",
120            TokenKind::Semi => ";",
121            TokenKind::RArrow => "->",
122            TokenKind::Dot => ".",
123            TokenKind::Le => "<=",
124            TokenKind::Ne => ">=",
125            TokenKind::GtFollowedByGt => ">",
126            TokenKind::Gt => ">",
127            TokenKind::LtFollowedByLt => "<",
128            TokenKind::Lt => "<",
129            TokenKind::Ge => ">=",
130            TokenKind::At => "@",
131            TokenKind::Pound => "#",
132            TokenKind::Iff => "<=>",
133            TokenKind::FatArrow => "=>",
134            TokenKind::Literal(_) => "literal",
135            TokenKind::Ident(..) => "identifier",
136            TokenKind::OpenParen => "(",
137            TokenKind::OpenBrace => "{",
138            TokenKind::OpenBracket => "[",
139            TokenKind::CloseParen => ")",
140            TokenKind::CloseBrace => "}",
141            TokenKind::CloseBracket => "]",
142            TokenKind::OpenInvisible(_) => "",
143            TokenKind::CloseInvisible(_) => "",
144            TokenKind::And => "&",
145            TokenKind::Percent => "%",
146            TokenKind::PathSep => "::",
147            TokenKind::DotDot => "..",
148            TokenKind::Eof => "<eof>",
149            TokenKind::Invalid => "<invalid>",
150        }
151    }
152
153    pub fn is_keyword(self, kw: Symbol) -> bool {
154        matches!(self, TokenKind::Ident(sym, IdentIsRaw::No) if sym == kw)
155    }
156
157    pub fn is_eof(self) -> bool {
158        matches!(self, TokenKind::Eof)
159    }
160}
161
162impl fmt::Display for TokenKind {
163    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
164        match self {
165            TokenKind::Literal(lit) => write!(f, "{lit}"),
166            TokenKind::Ident(sym, _) => write!(f, "{sym}"),
167            _ => write!(f, "{}", self.descr()),
168        }
169    }
170}
flux_syntax/token.rs

flux_syntax/
token.rs
