flux_syntax/
token.rs

1use std::fmt;
2
3pub use TokenKind::*;
4use rustc_ast::token::InvisibleOrigin;
5pub use rustc_ast::token::{Delimiter, IdentIsRaw, Lit, LitKind};
6use rustc_span::{BytePos, Symbol};
7
8#[derive(Copy, Clone, Debug, PartialEq)]
9pub enum TokenKind {
10    Caret,
11    Or,
12    EqEq,
13    Eq,
14    AndAnd,
15    OrOr,
16    Plus,
17    Minus,
18    Slash,
19    Bang,
20    Star,
21    Colon,
22    Comma,
23    Semi,
24    RArrow,
25    Dot,
26    Le,
27    Ne,
28    GtFollowedByGt,
29    Gt,
30    LtFollowedByLt,
31    Lt,
32    Ge,
33    At,
34    Pound,
35    Iff,
36    FatArrow,
37    Literal(Lit),
38    Ident(Symbol, IdentIsRaw),
39    OpenParen,
40    CloseParen,
41    OpenBrace,
42    CloseBrace,
43    OpenBracket,
44    CloseBracket,
45    OpenInvisible(InvisibleOrigin),
46    CloseInvisible(InvisibleOrigin),
47    Invalid,
48    And,
49    Percent,
50    PathSep,
51    DotDot,
52    Eof,
53}
54
55#[derive(Clone, Copy)]
56pub struct Token {
57    pub kind: TokenKind,
58    pub lo: BytePos,
59    pub hi: BytePos,
60}
61
62impl Token {
63    pub fn new(kind: TokenKind, lo: BytePos, hi: BytePos) -> Self {
64        Self { kind, lo, hi }
65    }
66}
67
68impl TokenKind {
69    pub fn open_delim(delim: Delimiter) -> TokenKind {
70        match delim {
71            Delimiter::Parenthesis => TokenKind::OpenParen,
72            Delimiter::Bracket => TokenKind::OpenBracket,
73            Delimiter::Brace => TokenKind::OpenBrace,
74            Delimiter::Invisible(origin) => TokenKind::OpenInvisible(origin),
75        }
76    }
77
78    pub fn close_delim(delim: Delimiter) -> TokenKind {
79        match delim {
80            Delimiter::Parenthesis => TokenKind::CloseParen,
81            Delimiter::Bracket => TokenKind::CloseBracket,
82            Delimiter::Brace => TokenKind::CloseBrace,
83            Delimiter::Invisible(origin) => TokenKind::CloseInvisible(origin),
84        }
85    }
86
87    pub fn descr(&self) -> &'static str {
88        match self {
89            TokenKind::Caret => "^",
90            TokenKind::Or => "|",
91            TokenKind::EqEq => "==",
92            TokenKind::Eq => "=",
93            TokenKind::AndAnd => "&&",
94            TokenKind::OrOr => "||",
95            TokenKind::Plus => "+",
96            TokenKind::Minus => "-",
97            TokenKind::Slash => "/",
98            TokenKind::Bang => "!",
99            TokenKind::Star => "*",
100            TokenKind::Colon => ":",
101            TokenKind::Comma => ",",
102            TokenKind::Semi => ";",
103            TokenKind::RArrow => "->",
104            TokenKind::Dot => ".",
105            TokenKind::Le => "<=",
106            TokenKind::Ne => ">=",
107            TokenKind::GtFollowedByGt => ">",
108            TokenKind::Gt => ">",
109            TokenKind::LtFollowedByLt => "<",
110            TokenKind::Lt => "<",
111            TokenKind::Ge => ">=",
112            TokenKind::At => "@",
113            TokenKind::Pound => "#",
114            TokenKind::Iff => "<=>",
115            TokenKind::FatArrow => "=>",
116            TokenKind::Literal(_) => "literal",
117            TokenKind::Ident(..) => "identifier",
118            TokenKind::OpenParen => "(",
119            TokenKind::OpenBrace => "{",
120            TokenKind::OpenBracket => "[",
121            TokenKind::CloseParen => ")",
122            TokenKind::CloseBrace => "}",
123            TokenKind::CloseBracket => "]",
124            TokenKind::OpenInvisible(_) => "",
125            TokenKind::CloseInvisible(_) => "",
126            TokenKind::And => "&",
127            TokenKind::Percent => "%",
128            TokenKind::PathSep => "::",
129            TokenKind::DotDot => "..",
130            TokenKind::Eof => "<eof>",
131            TokenKind::Invalid => "<invalid>",
132        }
133    }
134
135    pub fn is_keyword(self, kw: Symbol) -> bool {
136        matches!(self, TokenKind::Ident(sym, IdentIsRaw::No) if sym == kw)
137    }
138
139    pub fn is_eof(self) -> bool {
140        matches!(self, TokenKind::Eof)
141    }
142}
143
144impl fmt::Display for TokenKind {
145    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
146        match self {
147            TokenKind::Literal(lit) => write!(f, "{lit}"),
148            TokenKind::Ident(sym, _) => write!(f, "{sym}"),
149            _ => write!(f, "{}", self.descr()),
150        }
151    }
152}
flux_syntax/token.rs

flux_syntax/
token.rs
