|
|
@@ -1,1504 +0,0 @@
|
|
|
-//
|
|
|
-// Solidity custom lexer. Solidity needs a custom lexer for two reasons:
|
|
|
-// - comments and doc comments
|
|
|
-// - pragma value is [^;]+
|
|
|
-//
|
|
|
-use phf::phf_map;
|
|
|
-use std::fmt;
|
|
|
-use std::iter::Peekable;
|
|
|
-use std::str::CharIndices;
|
|
|
-use unicode_xid::UnicodeXID;
|
|
|
-
|
|
|
-use super::pt::Loc;
|
|
|
-
|
|
|
-pub type Spanned<Token, Loc, Error> = Result<(Loc, Token, Loc), Error>;
|
|
|
-
|
|
|
-#[derive(Copy, Clone, PartialEq, Debug)]
|
|
|
-pub enum CommentType {
|
|
|
- Line,
|
|
|
- Block,
|
|
|
-}
|
|
|
-
|
|
|
-#[derive(Copy, Clone, PartialEq, Debug)]
|
|
|
-pub enum Token<'input> {
|
|
|
- Identifier(&'input str),
|
|
|
- StringLiteral(&'input str),
|
|
|
- AddressLiteral(&'input str),
|
|
|
- HexLiteral(&'input str),
|
|
|
- Number(&'input str, &'input str),
|
|
|
- RationalNumber(&'input str, &'input str, &'input str),
|
|
|
- HexNumber(&'input str),
|
|
|
- DocComment(CommentType, &'input str),
|
|
|
- Divide,
|
|
|
- Contract,
|
|
|
- Library,
|
|
|
- Interface,
|
|
|
- Function,
|
|
|
- Pragma,
|
|
|
- Import,
|
|
|
-
|
|
|
- Struct,
|
|
|
- Event,
|
|
|
- Enum,
|
|
|
-
|
|
|
- Memory,
|
|
|
- Storage,
|
|
|
- Calldata,
|
|
|
-
|
|
|
- Public,
|
|
|
- Private,
|
|
|
- Internal,
|
|
|
- External,
|
|
|
-
|
|
|
- Constant,
|
|
|
-
|
|
|
- New,
|
|
|
- Delete,
|
|
|
-
|
|
|
- Pure,
|
|
|
- View,
|
|
|
- Payable,
|
|
|
-
|
|
|
- Do,
|
|
|
- Continue,
|
|
|
- Break,
|
|
|
-
|
|
|
- Throw,
|
|
|
- Emit,
|
|
|
- Return,
|
|
|
- Returns,
|
|
|
-
|
|
|
- Uint(u16),
|
|
|
- Int(u16),
|
|
|
- Bytes(u8),
|
|
|
- DynamicBytes,
|
|
|
- Bool,
|
|
|
- Address,
|
|
|
- String,
|
|
|
-
|
|
|
- Semicolon,
|
|
|
- Comma,
|
|
|
- OpenParenthesis,
|
|
|
- CloseParenthesis,
|
|
|
- OpenCurlyBrace,
|
|
|
- CloseCurlyBrace,
|
|
|
-
|
|
|
- BitwiseOr,
|
|
|
- BitwiseOrAssign,
|
|
|
- Or,
|
|
|
-
|
|
|
- BitwiseXor,
|
|
|
- BitwiseXorAssign,
|
|
|
-
|
|
|
- BitwiseAnd,
|
|
|
- BitwiseAndAssign,
|
|
|
- And,
|
|
|
-
|
|
|
- AddAssign,
|
|
|
- Increment,
|
|
|
- Add,
|
|
|
-
|
|
|
- SubtractAssign,
|
|
|
- Decrement,
|
|
|
- Subtract,
|
|
|
-
|
|
|
- MulAssign,
|
|
|
- Mul,
|
|
|
- Power,
|
|
|
- DivideAssign,
|
|
|
- ModuloAssign,
|
|
|
- Modulo,
|
|
|
-
|
|
|
- Equal,
|
|
|
- Assign,
|
|
|
- ColonAssign,
|
|
|
-
|
|
|
- NotEqual,
|
|
|
- Not,
|
|
|
-
|
|
|
- True,
|
|
|
- False,
|
|
|
- Else,
|
|
|
- Anonymous,
|
|
|
- For,
|
|
|
- While,
|
|
|
- If,
|
|
|
-
|
|
|
- ShiftRight,
|
|
|
- ShiftRightAssign,
|
|
|
- Less,
|
|
|
- LessEqual,
|
|
|
-
|
|
|
- ShiftLeft,
|
|
|
- ShiftLeftAssign,
|
|
|
- More,
|
|
|
- MoreEqual,
|
|
|
-
|
|
|
- Constructor,
|
|
|
- Indexed,
|
|
|
-
|
|
|
- Member,
|
|
|
- Colon,
|
|
|
- OpenBracket,
|
|
|
- CloseBracket,
|
|
|
- Complement,
|
|
|
- Question,
|
|
|
-
|
|
|
- Mapping,
|
|
|
- Arrow,
|
|
|
-
|
|
|
- Try,
|
|
|
- Catch,
|
|
|
-
|
|
|
- Receive,
|
|
|
- Fallback,
|
|
|
-
|
|
|
- Seconds,
|
|
|
- Minutes,
|
|
|
- Hours,
|
|
|
- Days,
|
|
|
- Weeks,
|
|
|
- Wei,
|
|
|
- Szabo,
|
|
|
- Finney,
|
|
|
- Ether,
|
|
|
-
|
|
|
- This,
|
|
|
- As,
|
|
|
- Is,
|
|
|
- Abstract,
|
|
|
- Virtual,
|
|
|
- Override,
|
|
|
- Using,
|
|
|
- Modifier,
|
|
|
- Immutable,
|
|
|
- Unchecked,
|
|
|
- Assembly,
|
|
|
- Let,
|
|
|
-}
|
|
|
-
|
|
|
-impl<'input> fmt::Display for Token<'input> {
|
|
|
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
|
- match self {
|
|
|
- Token::DocComment(CommentType::Line, s) => write!(f, "///{}", s),
|
|
|
- Token::DocComment(CommentType::Block, s) => write!(f, "/**{}\n*/", s),
|
|
|
- Token::Identifier(id) => write!(f, "{}", id),
|
|
|
- Token::StringLiteral(s) => write!(f, "\"{}\"", s),
|
|
|
- Token::HexLiteral(hex) => write!(f, "{}", hex),
|
|
|
- Token::AddressLiteral(address) => write!(f, "{}", address),
|
|
|
- Token::Number(base, exp) if exp.is_empty() => write!(f, "{}", base),
|
|
|
- Token::Number(base, exp) => write!(f, "{}e{}", base, exp),
|
|
|
- Token::RationalNumber(significand, mantissa, exp) if exp.is_empty() => {
|
|
|
- write!(f, "{}.{}", significand, mantissa)
|
|
|
- }
|
|
|
- Token::RationalNumber(significand, mantissa, exp) => {
|
|
|
- write!(f, "{}.{}e{}", significand, mantissa, exp)
|
|
|
- }
|
|
|
- Token::HexNumber(n) => write!(f, "{}", n),
|
|
|
- Token::Uint(w) => write!(f, "uint{}", w),
|
|
|
- Token::Int(w) => write!(f, "int{}", w),
|
|
|
- Token::Bytes(w) => write!(f, "bytes{}", w),
|
|
|
- Token::DynamicBytes => write!(f, "bytes"),
|
|
|
- Token::Semicolon => write!(f, ";"),
|
|
|
- Token::Comma => write!(f, ","),
|
|
|
- Token::OpenParenthesis => write!(f, "("),
|
|
|
- Token::CloseParenthesis => write!(f, ")"),
|
|
|
- Token::OpenCurlyBrace => write!(f, "{{"),
|
|
|
- Token::CloseCurlyBrace => write!(f, "}}"),
|
|
|
- Token::BitwiseOr => write!(f, "|"),
|
|
|
- Token::BitwiseOrAssign => write!(f, "|="),
|
|
|
- Token::Or => write!(f, "||"),
|
|
|
- Token::BitwiseXor => write!(f, "^"),
|
|
|
- Token::BitwiseXorAssign => write!(f, "^="),
|
|
|
- Token::BitwiseAnd => write!(f, "&"),
|
|
|
- Token::BitwiseAndAssign => write!(f, "&="),
|
|
|
- Token::And => write!(f, "&&"),
|
|
|
- Token::AddAssign => write!(f, "+="),
|
|
|
- Token::Increment => write!(f, "++"),
|
|
|
- Token::Add => write!(f, "+"),
|
|
|
- Token::SubtractAssign => write!(f, "-="),
|
|
|
- Token::Decrement => write!(f, "--"),
|
|
|
- Token::Subtract => write!(f, "-"),
|
|
|
- Token::MulAssign => write!(f, "*="),
|
|
|
- Token::Mul => write!(f, "*"),
|
|
|
- Token::Power => write!(f, "**"),
|
|
|
- Token::Divide => write!(f, "/"),
|
|
|
- Token::DivideAssign => write!(f, "/="),
|
|
|
- Token::ModuloAssign => write!(f, "%="),
|
|
|
- Token::Modulo => write!(f, "%"),
|
|
|
- Token::Equal => write!(f, "=="),
|
|
|
- Token::Assign => write!(f, "="),
|
|
|
- Token::ColonAssign => write!(f, ":="),
|
|
|
- Token::NotEqual => write!(f, "!="),
|
|
|
- Token::Not => write!(f, "!"),
|
|
|
- Token::ShiftLeft => write!(f, "<<"),
|
|
|
- Token::ShiftLeftAssign => write!(f, "<<="),
|
|
|
- Token::More => write!(f, ">"),
|
|
|
- Token::MoreEqual => write!(f, ">="),
|
|
|
- Token::Member => write!(f, "."),
|
|
|
- Token::Colon => write!(f, ":"),
|
|
|
- Token::OpenBracket => write!(f, "["),
|
|
|
- Token::CloseBracket => write!(f, "]"),
|
|
|
- Token::Complement => write!(f, "~"),
|
|
|
- Token::Question => write!(f, "?"),
|
|
|
- Token::ShiftRightAssign => write!(f, "<<="),
|
|
|
- Token::ShiftRight => write!(f, "<<"),
|
|
|
- Token::Less => write!(f, "<"),
|
|
|
- Token::LessEqual => write!(f, "<="),
|
|
|
- Token::Bool => write!(f, "bool"),
|
|
|
- Token::Address => write!(f, "address"),
|
|
|
- Token::String => write!(f, "string"),
|
|
|
- Token::Contract => write!(f, "contract"),
|
|
|
- Token::Library => write!(f, "library"),
|
|
|
- Token::Interface => write!(f, "interface"),
|
|
|
- Token::Function => write!(f, "function"),
|
|
|
- Token::Pragma => write!(f, "pragma"),
|
|
|
- Token::Import => write!(f, "import"),
|
|
|
- Token::Struct => write!(f, "struct"),
|
|
|
- Token::Event => write!(f, "event"),
|
|
|
- Token::Enum => write!(f, "enum"),
|
|
|
- Token::Memory => write!(f, "memory"),
|
|
|
- Token::Storage => write!(f, "storage"),
|
|
|
- Token::Calldata => write!(f, "calldata"),
|
|
|
- Token::Public => write!(f, "public"),
|
|
|
- Token::Private => write!(f, "private"),
|
|
|
- Token::Internal => write!(f, "internal"),
|
|
|
- Token::External => write!(f, "external"),
|
|
|
- Token::Constant => write!(f, "constant"),
|
|
|
- Token::New => write!(f, "new"),
|
|
|
- Token::Delete => write!(f, "delete"),
|
|
|
- Token::Pure => write!(f, "pure"),
|
|
|
- Token::View => write!(f, "view"),
|
|
|
- Token::Payable => write!(f, "payable"),
|
|
|
- Token::Do => write!(f, "do"),
|
|
|
- Token::Continue => write!(f, "continue"),
|
|
|
- Token::Break => write!(f, "break"),
|
|
|
- Token::Throw => write!(f, "throw"),
|
|
|
- Token::Emit => write!(f, "emit"),
|
|
|
- Token::Return => write!(f, "return"),
|
|
|
- Token::Returns => write!(f, "returns"),
|
|
|
- Token::True => write!(f, "true"),
|
|
|
- Token::False => write!(f, "false"),
|
|
|
- Token::Else => write!(f, "else"),
|
|
|
- Token::Anonymous => write!(f, "anonymous"),
|
|
|
- Token::For => write!(f, "for"),
|
|
|
- Token::While => write!(f, "while"),
|
|
|
- Token::If => write!(f, "if"),
|
|
|
- Token::Constructor => write!(f, "constructor"),
|
|
|
- Token::Indexed => write!(f, "indexed"),
|
|
|
- Token::Mapping => write!(f, "mapping"),
|
|
|
- Token::Arrow => write!(f, "=>"),
|
|
|
- Token::Try => write!(f, "try"),
|
|
|
- Token::Catch => write!(f, "catch"),
|
|
|
- Token::Receive => write!(f, "receive"),
|
|
|
- Token::Fallback => write!(f, "fallback"),
|
|
|
- Token::Seconds => write!(f, "seconds"),
|
|
|
- Token::Minutes => write!(f, "minutes"),
|
|
|
- Token::Hours => write!(f, "hours"),
|
|
|
- Token::Days => write!(f, "days"),
|
|
|
- Token::Weeks => write!(f, "weeks"),
|
|
|
- Token::Wei => write!(f, "wei"),
|
|
|
- Token::Szabo => write!(f, "szabo"),
|
|
|
- Token::Finney => write!(f, "finney"),
|
|
|
- Token::Ether => write!(f, "ether"),
|
|
|
- Token::This => write!(f, "this"),
|
|
|
- Token::As => write!(f, "as"),
|
|
|
- Token::Is => write!(f, "is"),
|
|
|
- Token::Abstract => write!(f, "abstract"),
|
|
|
- Token::Virtual => write!(f, "virtual"),
|
|
|
- Token::Override => write!(f, "override"),
|
|
|
- Token::Using => write!(f, "using"),
|
|
|
- Token::Modifier => write!(f, "modifier"),
|
|
|
- Token::Immutable => write!(f, "immutable"),
|
|
|
- Token::Unchecked => write!(f, "unchecked"),
|
|
|
- Token::Assembly => write!(f, "assembly"),
|
|
|
- Token::Let => write!(f, "let"),
|
|
|
- }
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-pub struct Lexer<'input> {
|
|
|
- input: &'input str,
|
|
|
- chars: Peekable<CharIndices<'input>>,
|
|
|
- last_tokens: [Option<Token<'input>>; 2],
|
|
|
-}
|
|
|
-
|
|
|
-#[derive(Debug, PartialEq)]
|
|
|
-pub enum LexicalError {
|
|
|
- EndOfFileInComment(usize, usize),
|
|
|
- EndOfFileInString(usize, usize),
|
|
|
- EndofFileInHex(usize, usize),
|
|
|
- MissingNumber(usize, usize),
|
|
|
- InvalidCharacterInHexLiteral(usize, char),
|
|
|
- UnrecognisedToken(usize, usize, String),
|
|
|
- MissingExponent(usize, usize),
|
|
|
- DoublePoints(usize, usize),
|
|
|
- UnrecognisedDecimal(usize, usize),
|
|
|
- ExpectedFrom(usize, usize, String),
|
|
|
-}
|
|
|
-
|
|
|
-impl fmt::Display for LexicalError {
|
|
|
- fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
|
- match self {
|
|
|
- LexicalError::EndOfFileInComment(_, _) => write!(f, "end of file found in comment"),
|
|
|
- LexicalError::EndOfFileInString(_, _) => {
|
|
|
- write!(f, "end of file found in string literal")
|
|
|
- }
|
|
|
- LexicalError::EndofFileInHex(_, _) => {
|
|
|
- write!(f, "end of file found in hex literal string")
|
|
|
- }
|
|
|
- LexicalError::MissingNumber(_, _) => write!(f, "missing number"),
|
|
|
- LexicalError::InvalidCharacterInHexLiteral(_, ch) => {
|
|
|
- write!(f, "invalid character ‘{}’ in hex literal string", ch)
|
|
|
- }
|
|
|
- LexicalError::UnrecognisedToken(_, _, t) => write!(f, "unrecognised token ‘{}’", t),
|
|
|
- LexicalError::ExpectedFrom(_, _, t) => write!(f, "‘{}’ found where ‘from’ expected", t),
|
|
|
- LexicalError::MissingExponent(_, _) => write!(f, "missing number"),
|
|
|
- LexicalError::DoublePoints(_, _) => write!(f, "found two dots in number"),
|
|
|
- LexicalError::UnrecognisedDecimal(_, _) => {
|
|
|
- write!(f, "expected number after decimal point")
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-impl LexicalError {
|
|
|
- pub fn loc(&self, file_no: usize) -> Loc {
|
|
|
- match self {
|
|
|
- LexicalError::EndOfFileInComment(start, end) => Loc(file_no, *start, *end),
|
|
|
- LexicalError::EndOfFileInString(start, end) => Loc(file_no, *start, *end),
|
|
|
- LexicalError::EndofFileInHex(start, end) => Loc(file_no, *start, *end),
|
|
|
- LexicalError::MissingNumber(start, end) => Loc(file_no, *start, *end),
|
|
|
- LexicalError::InvalidCharacterInHexLiteral(pos, _) => Loc(file_no, *pos, *pos),
|
|
|
- LexicalError::UnrecognisedToken(start, end, _) => Loc(file_no, *start, *end),
|
|
|
- LexicalError::ExpectedFrom(start, end, _) => Loc(file_no, *start, *end),
|
|
|
- LexicalError::MissingExponent(start, end) => Loc(file_no, *start, *end),
|
|
|
- LexicalError::DoublePoints(start, end) => Loc(file_no, *start, *end),
|
|
|
- LexicalError::UnrecognisedDecimal(start, end) => Loc(file_no, *start, *end),
|
|
|
- }
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-static KEYWORDS: phf::Map<&'static str, Token> = phf_map! {
|
|
|
- "address" => Token::Address,
|
|
|
- "anonymous" => Token::Anonymous,
|
|
|
- "bool" => Token::Bool,
|
|
|
- "break" => Token::Break,
|
|
|
- "bytes1" => Token::Bytes(1),
|
|
|
- "bytes2" => Token::Bytes(2),
|
|
|
- "bytes3" => Token::Bytes(3),
|
|
|
- "bytes4" => Token::Bytes(4),
|
|
|
- "bytes5" => Token::Bytes(5),
|
|
|
- "bytes6" => Token::Bytes(6),
|
|
|
- "bytes7" => Token::Bytes(7),
|
|
|
- "bytes8" => Token::Bytes(8),
|
|
|
- "bytes9" => Token::Bytes(9),
|
|
|
- "bytes10" => Token::Bytes(10),
|
|
|
- "bytes11" => Token::Bytes(11),
|
|
|
- "bytes12" => Token::Bytes(12),
|
|
|
- "bytes13" => Token::Bytes(13),
|
|
|
- "bytes14" => Token::Bytes(14),
|
|
|
- "bytes15" => Token::Bytes(15),
|
|
|
- "bytes16" => Token::Bytes(16),
|
|
|
- "bytes17" => Token::Bytes(17),
|
|
|
- "bytes18" => Token::Bytes(18),
|
|
|
- "bytes19" => Token::Bytes(19),
|
|
|
- "bytes20" => Token::Bytes(20),
|
|
|
- "bytes21" => Token::Bytes(21),
|
|
|
- "bytes22" => Token::Bytes(22),
|
|
|
- "bytes23" => Token::Bytes(23),
|
|
|
- "bytes24" => Token::Bytes(24),
|
|
|
- "bytes25" => Token::Bytes(25),
|
|
|
- "bytes26" => Token::Bytes(26),
|
|
|
- "bytes27" => Token::Bytes(27),
|
|
|
- "bytes28" => Token::Bytes(28),
|
|
|
- "bytes29" => Token::Bytes(29),
|
|
|
- "bytes30" => Token::Bytes(30),
|
|
|
- "bytes31" => Token::Bytes(31),
|
|
|
- "bytes32" => Token::Bytes(32),
|
|
|
- "bytes" => Token::DynamicBytes,
|
|
|
- "byte" => Token::Bytes(1),
|
|
|
- "calldata" => Token::Calldata,
|
|
|
- "constant" => Token::Constant,
|
|
|
- "constructor" => Token::Constructor,
|
|
|
- "continue" => Token::Continue,
|
|
|
- "contract" => Token::Contract,
|
|
|
- "delete" => Token::Delete,
|
|
|
- "do" => Token::Do,
|
|
|
- "else" => Token::Else,
|
|
|
- "emit" => Token::Emit,
|
|
|
- "enum" => Token::Enum,
|
|
|
- "event" => Token::Event,
|
|
|
- "external" => Token::External,
|
|
|
- "false" => Token::False,
|
|
|
- "for" => Token::For,
|
|
|
- "function" => Token::Function,
|
|
|
- "if" => Token::If,
|
|
|
- "import" => Token::Import,
|
|
|
- "indexed" => Token::Indexed,
|
|
|
- "int8" => Token::Int(8),
|
|
|
- "int16" => Token::Int(16),
|
|
|
- "int24" => Token::Int(24),
|
|
|
- "int32" => Token::Int(32),
|
|
|
- "int40" => Token::Int(40),
|
|
|
- "int48" => Token::Int(48),
|
|
|
- "int56" => Token::Int(56),
|
|
|
- "int64" => Token::Int(64),
|
|
|
- "int72" => Token::Int(72),
|
|
|
- "int80" => Token::Int(80),
|
|
|
- "int88" => Token::Int(88),
|
|
|
- "int96" => Token::Int(96),
|
|
|
- "int104" => Token::Int(104),
|
|
|
- "int112" => Token::Int(112),
|
|
|
- "int120" => Token::Int(120),
|
|
|
- "int128" => Token::Int(128),
|
|
|
- "int136" => Token::Int(136),
|
|
|
- "int144" => Token::Int(144),
|
|
|
- "int152" => Token::Int(152),
|
|
|
- "int160" => Token::Int(160),
|
|
|
- "int168" => Token::Int(168),
|
|
|
- "int176" => Token::Int(176),
|
|
|
- "int184" => Token::Int(184),
|
|
|
- "int192" => Token::Int(192),
|
|
|
- "int200" => Token::Int(200),
|
|
|
- "int208" => Token::Int(208),
|
|
|
- "int216" => Token::Int(216),
|
|
|
- "int224" => Token::Int(224),
|
|
|
- "int232" => Token::Int(232),
|
|
|
- "int240" => Token::Int(240),
|
|
|
- "int248" => Token::Int(248),
|
|
|
- "int256" => Token::Int(256),
|
|
|
- "interface" => Token::Interface,
|
|
|
- "internal" => Token::Internal,
|
|
|
- "int" => Token::Int(256),
|
|
|
- "library" => Token::Library,
|
|
|
- "mapping" => Token::Mapping,
|
|
|
- "memory" => Token::Memory,
|
|
|
- "new" => Token::New,
|
|
|
- "payable" => Token::Payable,
|
|
|
- "pragma" => Token::Pragma,
|
|
|
- "private" => Token::Private,
|
|
|
- "public" => Token::Public,
|
|
|
- "pure" => Token::Pure,
|
|
|
- "returns" => Token::Returns,
|
|
|
- "return" => Token::Return,
|
|
|
- "storage" => Token::Storage,
|
|
|
- "string" => Token::String,
|
|
|
- "struct" => Token::Struct,
|
|
|
- "throw" => Token::Throw,
|
|
|
- "true" => Token::True,
|
|
|
- "uint8" => Token::Uint(8),
|
|
|
- "uint16" => Token::Uint(16),
|
|
|
- "uint24" => Token::Uint(24),
|
|
|
- "uint32" => Token::Uint(32),
|
|
|
- "uint40" => Token::Uint(40),
|
|
|
- "uint48" => Token::Uint(48),
|
|
|
- "uint56" => Token::Uint(56),
|
|
|
- "uint64" => Token::Uint(64),
|
|
|
- "uint72" => Token::Uint(72),
|
|
|
- "uint80" => Token::Uint(80),
|
|
|
- "uint88" => Token::Uint(88),
|
|
|
- "uint96" => Token::Uint(96),
|
|
|
- "uint104" => Token::Uint(104),
|
|
|
- "uint112" => Token::Uint(112),
|
|
|
- "uint120" => Token::Uint(120),
|
|
|
- "uint128" => Token::Uint(128),
|
|
|
- "uint136" => Token::Uint(136),
|
|
|
- "uint144" => Token::Uint(144),
|
|
|
- "uint152" => Token::Uint(152),
|
|
|
- "uint160" => Token::Uint(160),
|
|
|
- "uint168" => Token::Uint(168),
|
|
|
- "uint176" => Token::Uint(176),
|
|
|
- "uint184" => Token::Uint(184),
|
|
|
- "uint192" => Token::Uint(192),
|
|
|
- "uint200" => Token::Uint(200),
|
|
|
- "uint208" => Token::Uint(208),
|
|
|
- "uint216" => Token::Uint(216),
|
|
|
- "uint224" => Token::Uint(224),
|
|
|
- "uint232" => Token::Uint(232),
|
|
|
- "uint240" => Token::Uint(240),
|
|
|
- "uint248" => Token::Uint(248),
|
|
|
- "uint256" => Token::Uint(256),
|
|
|
- "uint" => Token::Uint(256),
|
|
|
- "view" => Token::View,
|
|
|
- "while" => Token::While,
|
|
|
- "try" => Token::Try,
|
|
|
- "catch" => Token::Catch,
|
|
|
- "receive" => Token::Receive,
|
|
|
- "fallback" => Token::Fallback,
|
|
|
- "seconds" => Token::Seconds,
|
|
|
- "minutes" => Token::Minutes,
|
|
|
- "hours" => Token::Hours,
|
|
|
- "days" => Token::Days,
|
|
|
- "weeks" => Token::Weeks,
|
|
|
- "wei" => Token::Wei,
|
|
|
- "szabo" => Token::Szabo,
|
|
|
- "finney" => Token::Finney,
|
|
|
- "ether" => Token::Ether,
|
|
|
- "this" => Token::This,
|
|
|
- "as" => Token::As,
|
|
|
- "is" => Token::Is,
|
|
|
- "abstract" => Token::Abstract,
|
|
|
- "virtual" => Token::Virtual,
|
|
|
- "override" => Token::Override,
|
|
|
- "using" => Token::Using,
|
|
|
- "modifier" => Token::Modifier,
|
|
|
- "immutable" => Token::Immutable,
|
|
|
- "unchecked" => Token::Unchecked,
|
|
|
- "assembly" => Token::Assembly,
|
|
|
- "let" => Token::Let,
|
|
|
-};
|
|
|
-
|
|
|
-impl<'input> Lexer<'input> {
|
|
|
- pub fn new(input: &'input str) -> Self {
|
|
|
- Lexer {
|
|
|
- input,
|
|
|
- chars: input.char_indices().peekable(),
|
|
|
- last_tokens: [None, None],
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- fn parse_number(
|
|
|
- &mut self,
|
|
|
- start: usize,
|
|
|
- end: usize,
|
|
|
- ch: char,
|
|
|
- ) -> Result<(usize, Token<'input>, usize), LexicalError> {
|
|
|
- let mut is_rational = false;
|
|
|
- if ch == '0' {
|
|
|
- if let Some((_, 'x')) = self.chars.peek() {
|
|
|
- // hex number
|
|
|
- self.chars.next();
|
|
|
-
|
|
|
- let mut end = match self.chars.next() {
|
|
|
- Some((end, ch)) if ch.is_ascii_hexdigit() => end,
|
|
|
- Some((_, _)) => {
|
|
|
- return Err(LexicalError::MissingNumber(start, start + 1));
|
|
|
- }
|
|
|
- None => {
|
|
|
- return Err(LexicalError::EndofFileInHex(start, self.input.len()));
|
|
|
- }
|
|
|
- };
|
|
|
-
|
|
|
- while let Some((i, ch)) = self.chars.peek() {
|
|
|
- if !ch.is_ascii_hexdigit() && *ch != '_' {
|
|
|
- break;
|
|
|
- }
|
|
|
- end = *i;
|
|
|
- self.chars.next();
|
|
|
- }
|
|
|
-
|
|
|
- return Ok((start, Token::HexNumber(&self.input[start..=end]), end + 1));
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- let mut start = start;
|
|
|
- if ch == '.' {
|
|
|
- is_rational = true;
|
|
|
- start -= 1;
|
|
|
- }
|
|
|
-
|
|
|
- let mut end = end;
|
|
|
- while let Some((i, ch)) = self.chars.peek() {
|
|
|
- if !ch.is_ascii_digit() && *ch != '_' {
|
|
|
- break;
|
|
|
- }
|
|
|
- end = *i;
|
|
|
- self.chars.next();
|
|
|
- }
|
|
|
- let mut rational_end = end;
|
|
|
- let mut end_before_rational = end;
|
|
|
- let mut rational_start = end;
|
|
|
- if is_rational {
|
|
|
- end_before_rational = start - 1;
|
|
|
- rational_start = start + 1;
|
|
|
- }
|
|
|
-
|
|
|
- if let Some((i, '.')) = self.chars.peek() {
|
|
|
- if is_rational {
|
|
|
- return Err(LexicalError::DoublePoints(start, self.input.len()));
|
|
|
- }
|
|
|
- rational_start = *i + 1;
|
|
|
- rational_end = *i + 1;
|
|
|
- let mut has_number = false;
|
|
|
- is_rational = true;
|
|
|
- self.chars.next();
|
|
|
- while let Some((i, ch)) = self.chars.peek() {
|
|
|
- if *ch == '.' {
|
|
|
- return Err(LexicalError::DoublePoints(start, self.input.len()));
|
|
|
- }
|
|
|
- if !ch.is_ascii_digit() {
|
|
|
- break;
|
|
|
- }
|
|
|
- has_number = true;
|
|
|
- rational_end = *i;
|
|
|
- end = *i;
|
|
|
- self.chars.next();
|
|
|
- }
|
|
|
- if !has_number {
|
|
|
- return Err(LexicalError::UnrecognisedDecimal(start, self.input.len()));
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- let old_end = end;
|
|
|
- let mut exp_start = end + 1;
|
|
|
-
|
|
|
- if let Some((i, 'e')) = self.chars.peek() {
|
|
|
- exp_start = *i + 1;
|
|
|
- self.chars.next();
|
|
|
- while let Some((i, ch)) = self.chars.peek() {
|
|
|
- if !ch.is_ascii_digit() && *ch != '_' && *ch != '-' {
|
|
|
- break;
|
|
|
- }
|
|
|
- end = *i;
|
|
|
- self.chars.next();
|
|
|
- }
|
|
|
-
|
|
|
- if exp_start > end {
|
|
|
- return Err(LexicalError::MissingExponent(start, self.input.len()));
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- if is_rational {
|
|
|
- let significand = &self.input[start..=end_before_rational];
|
|
|
- let mantissa = &self.input[rational_start..=rational_end];
|
|
|
-
|
|
|
- if mantissa.is_empty() {
|
|
|
- return Err(LexicalError::UnrecognisedDecimal(start, self.input.len()));
|
|
|
- }
|
|
|
- let exp = &self.input[exp_start..=end];
|
|
|
- return Ok((
|
|
|
- start,
|
|
|
- Token::RationalNumber(significand, mantissa, exp),
|
|
|
- end + 1,
|
|
|
- ));
|
|
|
- }
|
|
|
-
|
|
|
- let base = &self.input[start..=old_end];
|
|
|
- let exp = &self.input[exp_start..=end];
|
|
|
-
|
|
|
- Ok((start, Token::Number(base, exp), end + 1))
|
|
|
- }
|
|
|
-
|
|
|
- fn string(
|
|
|
- &mut self,
|
|
|
- token_start: usize,
|
|
|
- string_start: usize,
|
|
|
- quote_char: char,
|
|
|
- ) -> Result<(usize, Token<'input>, usize), LexicalError> {
|
|
|
- let mut end;
|
|
|
-
|
|
|
- let mut last_was_escape = false;
|
|
|
-
|
|
|
- loop {
|
|
|
- if let Some((i, ch)) = self.chars.next() {
|
|
|
- end = i;
|
|
|
- if !last_was_escape {
|
|
|
- if ch == quote_char {
|
|
|
- break;
|
|
|
- }
|
|
|
- last_was_escape = ch == '\\';
|
|
|
- } else {
|
|
|
- last_was_escape = false;
|
|
|
- }
|
|
|
- } else {
|
|
|
- return Err(LexicalError::EndOfFileInString(
|
|
|
- token_start,
|
|
|
- self.input.len(),
|
|
|
- ));
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- Ok((
|
|
|
- token_start,
|
|
|
- Token::StringLiteral(&self.input[string_start..end]),
|
|
|
- end + 1,
|
|
|
- ))
|
|
|
- }
|
|
|
-
|
|
|
- fn next(&mut self) -> Option<Result<(usize, Token<'input>, usize), LexicalError>> {
|
|
|
- loop {
|
|
|
- match self.chars.next() {
|
|
|
- Some((start, ch)) if ch == '_' || ch == '$' || UnicodeXID::is_xid_start(ch) => {
|
|
|
- let end;
|
|
|
-
|
|
|
- loop {
|
|
|
- if let Some((i, ch)) = self.chars.peek() {
|
|
|
- if !UnicodeXID::is_xid_continue(*ch) && *ch != '$' {
|
|
|
- end = *i;
|
|
|
- break;
|
|
|
- }
|
|
|
- self.chars.next();
|
|
|
- } else {
|
|
|
- end = self.input.len();
|
|
|
- break;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- let id = &self.input[start..end];
|
|
|
-
|
|
|
- if id == "unicode" {
|
|
|
- match self.chars.peek() {
|
|
|
- Some((_, quote_char @ '"')) | Some((_, quote_char @ '\'')) => {
|
|
|
- let quote_char = *quote_char;
|
|
|
-
|
|
|
- self.chars.next();
|
|
|
-
|
|
|
- return Some(self.string(start, start + 8, quote_char));
|
|
|
- }
|
|
|
- _ => (),
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- if id == "hex" {
|
|
|
- match self.chars.peek() {
|
|
|
- Some((_, quote_char @ '"')) | Some((_, quote_char @ '\'')) => {
|
|
|
- let quote_char = *quote_char;
|
|
|
-
|
|
|
- self.chars.next();
|
|
|
-
|
|
|
- for (i, ch) in &mut self.chars {
|
|
|
- if ch == quote_char {
|
|
|
- return Some(Ok((
|
|
|
- start,
|
|
|
- Token::HexLiteral(&self.input[start..=i]),
|
|
|
- i + 1,
|
|
|
- )));
|
|
|
- }
|
|
|
-
|
|
|
- if !ch.is_ascii_hexdigit() && ch != '_' {
|
|
|
- // Eat up the remainer of the string
|
|
|
- for (_, ch) in &mut self.chars {
|
|
|
- if ch == quote_char {
|
|
|
- break;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- return Some(Err(
|
|
|
- LexicalError::InvalidCharacterInHexLiteral(i, ch),
|
|
|
- ));
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- return Some(Err(LexicalError::EndOfFileInString(
|
|
|
- start,
|
|
|
- self.input.len(),
|
|
|
- )));
|
|
|
- }
|
|
|
- _ => (),
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- if id == "address" {
|
|
|
- match self.chars.peek() {
|
|
|
- Some((_, quote_char @ '"')) | Some((_, quote_char @ '\'')) => {
|
|
|
- let quote_char = *quote_char;
|
|
|
-
|
|
|
- self.chars.next();
|
|
|
-
|
|
|
- for (i, ch) in &mut self.chars {
|
|
|
- if ch == quote_char {
|
|
|
- return Some(Ok((
|
|
|
- start,
|
|
|
- Token::AddressLiteral(&self.input[start..=i]),
|
|
|
- i + 1,
|
|
|
- )));
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- return Some(Err(LexicalError::EndOfFileInString(
|
|
|
- start,
|
|
|
- self.input.len(),
|
|
|
- )));
|
|
|
- }
|
|
|
- _ => (),
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- return if let Some(w) = KEYWORDS.get(id) {
|
|
|
- Some(Ok((start, *w, end)))
|
|
|
- } else {
|
|
|
- Some(Ok((start, Token::Identifier(id), end)))
|
|
|
- };
|
|
|
- }
|
|
|
- Some((start, quote_char @ '"')) | Some((start, quote_char @ '\'')) => {
|
|
|
- return Some(self.string(start, start + 1, quote_char));
|
|
|
- }
|
|
|
- Some((start, '/')) => {
|
|
|
- match self.chars.peek() {
|
|
|
- Some((_, '=')) => {
|
|
|
- self.chars.next();
|
|
|
- return Some(Ok((start, Token::DivideAssign, start + 2)));
|
|
|
- }
|
|
|
- Some((_, '/')) => {
|
|
|
- // line comment
|
|
|
- self.chars.next();
|
|
|
-
|
|
|
- let mut newline = false;
|
|
|
-
|
|
|
- let doc_comment_start = match self.chars.next() {
|
|
|
- Some((i, '/')) => match self.chars.peek() {
|
|
|
- // ///(/)+ is still a line comment
|
|
|
- Some((_, '/')) => None,
|
|
|
- _ => Some(i + 1),
|
|
|
- },
|
|
|
- Some((_, ch)) if ch == '\n' || ch == '\r' => {
|
|
|
- newline = true;
|
|
|
- None
|
|
|
- }
|
|
|
- _ => None,
|
|
|
- };
|
|
|
-
|
|
|
- let mut last = start + 3;
|
|
|
-
|
|
|
- if !newline {
|
|
|
- for (i, ch) in &mut self.chars {
|
|
|
- if ch == '\n' || ch == '\r' {
|
|
|
- break;
|
|
|
- }
|
|
|
- last = i;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- if let Some(doc_start) = doc_comment_start {
|
|
|
- if last > doc_start {
|
|
|
- return Some(Ok((
|
|
|
- start + 3,
|
|
|
- Token::DocComment(
|
|
|
- CommentType::Line,
|
|
|
- &self.input[doc_start..=last],
|
|
|
- ),
|
|
|
- last + 1,
|
|
|
- )));
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- Some((_, '*')) => {
|
|
|
- // multiline comment
|
|
|
- self.chars.next();
|
|
|
-
|
|
|
- let doc_comment_start = match self.chars.next() {
|
|
|
- Some((i, '*')) => match self.chars.peek() {
|
|
|
- Some((_, '*')) => None,
|
|
|
- _ => Some(i + 1),
|
|
|
- },
|
|
|
- _ => None,
|
|
|
- };
|
|
|
-
|
|
|
- let mut last = start + 3;
|
|
|
- let mut seen_star = false;
|
|
|
-
|
|
|
- loop {
|
|
|
- if let Some((i, ch)) = self.chars.next() {
|
|
|
- if seen_star && ch == '/' {
|
|
|
- break;
|
|
|
- }
|
|
|
- seen_star = ch == '*';
|
|
|
- last = i;
|
|
|
- } else {
|
|
|
- return Some(Err(LexicalError::EndOfFileInComment(
|
|
|
- start,
|
|
|
- self.input.len(),
|
|
|
- )));
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- if let Some(doc_start) = doc_comment_start {
|
|
|
- if last > doc_start {
|
|
|
- return Some(Ok((
|
|
|
- start + 3,
|
|
|
- Token::DocComment(
|
|
|
- CommentType::Block,
|
|
|
- &self.input[doc_start..last],
|
|
|
- ),
|
|
|
- last,
|
|
|
- )));
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- _ => {
|
|
|
- return Some(Ok((start, Token::Divide, start + 1)));
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- Some((start, ch)) if ch.is_ascii_digit() => {
|
|
|
- return Some(self.parse_number(start, start, ch))
|
|
|
- }
|
|
|
- Some((i, ';')) => return Some(Ok((i, Token::Semicolon, i + 1))),
|
|
|
- Some((i, ',')) => return Some(Ok((i, Token::Comma, i + 1))),
|
|
|
- Some((i, '(')) => return Some(Ok((i, Token::OpenParenthesis, i + 1))),
|
|
|
- Some((i, ')')) => return Some(Ok((i, Token::CloseParenthesis, i + 1))),
|
|
|
- Some((i, '{')) => return Some(Ok((i, Token::OpenCurlyBrace, i + 1))),
|
|
|
- Some((i, '}')) => return Some(Ok((i, Token::CloseCurlyBrace, i + 1))),
|
|
|
- Some((i, '~')) => return Some(Ok((i, Token::Complement, i + 1))),
|
|
|
- Some((i, '=')) => match self.chars.peek() {
|
|
|
- Some((_, '=')) => {
|
|
|
- self.chars.next();
|
|
|
- return Some(Ok((i, Token::Equal, i + 2)));
|
|
|
- }
|
|
|
- Some((_, '>')) => {
|
|
|
- self.chars.next();
|
|
|
- return Some(Ok((i, Token::Arrow, i + 2)));
|
|
|
- }
|
|
|
- _ => {
|
|
|
- return Some(Ok((i, Token::Assign, i + 1)));
|
|
|
- }
|
|
|
- },
|
|
|
- Some((i, '!')) => {
|
|
|
- if let Some((_, '=')) = self.chars.peek() {
|
|
|
- self.chars.next();
|
|
|
- return Some(Ok((i, Token::NotEqual, i + 2)));
|
|
|
- } else {
|
|
|
- return Some(Ok((i, Token::Not, i + 1)));
|
|
|
- }
|
|
|
- }
|
|
|
- Some((i, '|')) => {
|
|
|
- return match self.chars.peek() {
|
|
|
- Some((_, '=')) => {
|
|
|
- self.chars.next();
|
|
|
- Some(Ok((i, Token::BitwiseOrAssign, i + 2)))
|
|
|
- }
|
|
|
- Some((_, '|')) => {
|
|
|
- self.chars.next();
|
|
|
- Some(Ok((i, Token::Or, i + 2)))
|
|
|
- }
|
|
|
- _ => Some(Ok((i, Token::BitwiseOr, i + 1))),
|
|
|
- };
|
|
|
- }
|
|
|
- Some((i, '&')) => {
|
|
|
- return match self.chars.peek() {
|
|
|
- Some((_, '=')) => {
|
|
|
- self.chars.next();
|
|
|
- Some(Ok((i, Token::BitwiseAndAssign, i + 2)))
|
|
|
- }
|
|
|
- Some((_, '&')) => {
|
|
|
- self.chars.next();
|
|
|
- Some(Ok((i, Token::And, i + 2)))
|
|
|
- }
|
|
|
- _ => Some(Ok((i, Token::BitwiseAnd, i + 1))),
|
|
|
- };
|
|
|
- }
|
|
|
- Some((i, '^')) => {
|
|
|
- return match self.chars.peek() {
|
|
|
- Some((_, '=')) => {
|
|
|
- self.chars.next();
|
|
|
- Some(Ok((i, Token::BitwiseXorAssign, i + 2)))
|
|
|
- }
|
|
|
- _ => Some(Ok((i, Token::BitwiseXor, i + 1))),
|
|
|
- };
|
|
|
- }
|
|
|
- Some((i, '+')) => {
|
|
|
- return match self.chars.peek() {
|
|
|
- Some((_, '=')) => {
|
|
|
- self.chars.next();
|
|
|
- Some(Ok((i, Token::AddAssign, i + 2)))
|
|
|
- }
|
|
|
- Some((_, '+')) => {
|
|
|
- self.chars.next();
|
|
|
- Some(Ok((i, Token::Increment, i + 2)))
|
|
|
- }
|
|
|
- _ => Some(Ok((i, Token::Add, i + 1))),
|
|
|
- };
|
|
|
- }
|
|
|
- Some((i, '-')) => {
|
|
|
- return match self.chars.peek() {
|
|
|
- Some((_, '=')) => {
|
|
|
- self.chars.next();
|
|
|
- Some(Ok((i, Token::SubtractAssign, i + 2)))
|
|
|
- }
|
|
|
- Some((_, '-')) => {
|
|
|
- self.chars.next();
|
|
|
- Some(Ok((i, Token::Decrement, i + 2)))
|
|
|
- }
|
|
|
- _ => Some(Ok((i, Token::Subtract, i + 1))),
|
|
|
- };
|
|
|
- }
|
|
|
- Some((i, '*')) => {
|
|
|
- return match self.chars.peek() {
|
|
|
- Some((_, '=')) => {
|
|
|
- self.chars.next();
|
|
|
- Some(Ok((i, Token::MulAssign, i + 2)))
|
|
|
- }
|
|
|
- Some((_, '*')) => {
|
|
|
- self.chars.next();
|
|
|
- Some(Ok((i, Token::Power, i + 2)))
|
|
|
- }
|
|
|
- _ => Some(Ok((i, Token::Mul, i + 1))),
|
|
|
- };
|
|
|
- }
|
|
|
- Some((i, '%')) => {
|
|
|
- return match self.chars.peek() {
|
|
|
- Some((_, '=')) => {
|
|
|
- self.chars.next();
|
|
|
- Some(Ok((i, Token::ModuloAssign, i + 2)))
|
|
|
- }
|
|
|
- _ => Some(Ok((i, Token::Modulo, i + 1))),
|
|
|
- };
|
|
|
- }
|
|
|
- Some((i, '<')) => {
|
|
|
- return match self.chars.peek() {
|
|
|
- Some((_, '<')) => {
|
|
|
- self.chars.next();
|
|
|
- if let Some((_, '=')) = self.chars.peek() {
|
|
|
- self.chars.next();
|
|
|
- Some(Ok((i, Token::ShiftLeftAssign, i + 3)))
|
|
|
- } else {
|
|
|
- Some(Ok((i, Token::ShiftLeft, i + 2)))
|
|
|
- }
|
|
|
- }
|
|
|
- Some((_, '=')) => {
|
|
|
- self.chars.next();
|
|
|
- Some(Ok((i, Token::LessEqual, i + 2)))
|
|
|
- }
|
|
|
- _ => Some(Ok((i, Token::Less, i + 1))),
|
|
|
- };
|
|
|
- }
|
|
|
- Some((i, '>')) => {
|
|
|
- return match self.chars.peek() {
|
|
|
- Some((_, '>')) => {
|
|
|
- self.chars.next();
|
|
|
- if let Some((_, '=')) = self.chars.peek() {
|
|
|
- self.chars.next();
|
|
|
- Some(Ok((i, Token::ShiftRightAssign, i + 3)))
|
|
|
- } else {
|
|
|
- Some(Ok((i, Token::ShiftRight, i + 2)))
|
|
|
- }
|
|
|
- }
|
|
|
- Some((_, '=')) => {
|
|
|
- self.chars.next();
|
|
|
- Some(Ok((i, Token::MoreEqual, i + 2)))
|
|
|
- }
|
|
|
- _ => Some(Ok((i, Token::More, i + 1))),
|
|
|
- };
|
|
|
- }
|
|
|
- Some((i, '.')) => {
|
|
|
- if let Some((_, a)) = self.chars.peek() {
|
|
|
- if a.is_ascii_digit() {
|
|
|
- return Some(self.parse_number(i + 1, i + 1, '.'));
|
|
|
- }
|
|
|
- }
|
|
|
- return Some(Ok((i, Token::Member, i + 1)));
|
|
|
- }
|
|
|
- Some((i, '[')) => return Some(Ok((i, Token::OpenBracket, i + 1))),
|
|
|
- Some((i, ']')) => return Some(Ok((i, Token::CloseBracket, i + 1))),
|
|
|
- Some((i, ':')) => {
|
|
|
- return match self.chars.peek() {
|
|
|
- Some((_, '=')) => {
|
|
|
- self.chars.next();
|
|
|
- Some(Ok((i, Token::ColonAssign, i + 2)))
|
|
|
- }
|
|
|
- _ => Some(Ok((i, Token::Colon, i + 1))),
|
|
|
- };
|
|
|
- }
|
|
|
- Some((i, '?')) => return Some(Ok((i, Token::Question, i + 1))),
|
|
|
- Some((_, ch)) if ch.is_whitespace() => (),
|
|
|
- Some((start, _)) => {
|
|
|
- let mut end;
|
|
|
-
|
|
|
- loop {
|
|
|
- if let Some((i, ch)) = self.chars.next() {
|
|
|
- end = i;
|
|
|
-
|
|
|
- if ch.is_whitespace() {
|
|
|
- break;
|
|
|
- }
|
|
|
- } else {
|
|
|
- end = self.input.len();
|
|
|
- break;
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- return Some(Err(LexicalError::UnrecognisedToken(
|
|
|
- start,
|
|
|
- end,
|
|
|
- self.input[start..end].to_owned(),
|
|
|
- )));
|
|
|
- }
|
|
|
- None => return None, // End of file
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /// Next token is pragma value. Return it
|
|
|
- fn pragma_value(&mut self) -> Option<Result<(usize, Token<'input>, usize), LexicalError>> {
|
|
|
- // special parser for pragma solidity >=0.4.22 <0.7.0;
|
|
|
- let mut start = None;
|
|
|
- let mut end = 0;
|
|
|
-
|
|
|
- // solc will include anything upto the next semicolon, whitespace
|
|
|
- // trimmed on left and right
|
|
|
- loop {
|
|
|
- match self.chars.peek() {
|
|
|
- Some((_, ';')) | None => {
|
|
|
- return if let Some(start) = start {
|
|
|
- Some(Ok((
|
|
|
- start,
|
|
|
- Token::StringLiteral(&self.input[start..end]),
|
|
|
- end,
|
|
|
- )))
|
|
|
- } else {
|
|
|
- self.next()
|
|
|
- };
|
|
|
- }
|
|
|
- Some((_, ch)) if ch.is_whitespace() => {
|
|
|
- self.chars.next();
|
|
|
- }
|
|
|
- Some((i, _)) => {
|
|
|
- if start.is_none() {
|
|
|
- start = Some(*i);
|
|
|
- }
|
|
|
- self.chars.next();
|
|
|
-
|
|
|
- // end should point to the byte _after_ the character
|
|
|
- end = match self.chars.peek() {
|
|
|
- Some((i, _)) => *i,
|
|
|
- None => self.input.len(),
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-impl<'input> Iterator for Lexer<'input> {
|
|
|
- type Item = Spanned<Token<'input>, usize, LexicalError>;
|
|
|
-
|
|
|
- /// Return the next token
|
|
|
- fn next(&mut self) -> Option<Self::Item> {
|
|
|
- // Lexer should be aware of whether the last two tokens were
|
|
|
- // pragma followed by identifier. If this is true, then special parsing should be
|
|
|
- // done for the pragma value
|
|
|
- let token = if let [Some(Token::Pragma), Some(Token::Identifier(_))] = self.last_tokens {
|
|
|
- self.pragma_value()
|
|
|
- } else {
|
|
|
- self.next()
|
|
|
- };
|
|
|
-
|
|
|
- self.last_tokens = [
|
|
|
- self.last_tokens[1],
|
|
|
- match token {
|
|
|
- Some(Ok((_, n, _))) => Some(n),
|
|
|
- _ => None,
|
|
|
- },
|
|
|
- ];
|
|
|
-
|
|
|
- token
|
|
|
- }
|
|
|
-}
|
|
|
-
|
|
|
-#[test]
|
|
|
-fn lexertest() {
|
|
|
- let tokens = Lexer::new("bool").collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(tokens, vec!(Ok((0, Token::Bool, 4))));
|
|
|
-
|
|
|
- let tokens = Lexer::new("uint8").collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(tokens, vec!(Ok((0, Token::Uint(8), 5))));
|
|
|
-
|
|
|
- let tokens = Lexer::new("hex").collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(tokens, vec!(Ok((0, Token::Identifier("hex"), 3))));
|
|
|
-
|
|
|
- let tokens = Lexer::new("hex\"cafe_dead\" /* adad*** */")
|
|
|
- .collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(Ok((0, Token::HexLiteral("hex\"cafe_dead\""), 14)))
|
|
|
- );
|
|
|
-
|
|
|
- let tokens = Lexer::new("// foo bar\n0x00fead0_12 00090 0_0")
|
|
|
- .collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(
|
|
|
- Ok((11, Token::HexNumber("0x00fead0_12"), 23)),
|
|
|
- Ok((24, Token::Number("00090", ""), 29)),
|
|
|
- Ok((30, Token::Number("0_0", ""), 33))
|
|
|
- )
|
|
|
- );
|
|
|
-
|
|
|
- let tokens = Lexer::new("// foo bar\n0x00fead0_12 9.0008 0_0")
|
|
|
- .collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(
|
|
|
- Ok((11, Token::HexNumber("0x00fead0_12"), 23)),
|
|
|
- Ok((24, Token::RationalNumber("9", "0008", ""), 30)),
|
|
|
- Ok((31, Token::Number("0_0", ""), 34))
|
|
|
- )
|
|
|
- );
|
|
|
-
|
|
|
- let tokens = Lexer::new("// foo bar\n0x00fead0_12 .0008 0.9e2")
|
|
|
- .collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(
|
|
|
- Ok((11, Token::HexNumber("0x00fead0_12"), 23)),
|
|
|
- Ok((24, Token::RationalNumber("", "0008", ""), 29)),
|
|
|
- Ok((30, Token::RationalNumber("0", "9", "2"), 35))
|
|
|
- )
|
|
|
- );
|
|
|
-
|
|
|
- let tokens = Lexer::new("// foo bar\n0x00fead0_12 .0008 0.9e-2")
|
|
|
- .collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(
|
|
|
- Ok((11, Token::HexNumber("0x00fead0_12"), 23)),
|
|
|
- Ok((24, Token::RationalNumber("", "0008", ""), 29)),
|
|
|
- Ok((30, Token::RationalNumber("0", "9", "-2"), 36))
|
|
|
- )
|
|
|
- );
|
|
|
-
|
|
|
- let tokens =
|
|
|
- Lexer::new("\"foo\"").collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(tokens, vec!(Ok((0, Token::StringLiteral("foo"), 5)),));
|
|
|
-
|
|
|
- let tokens = Lexer::new("pragma solidity >=0.5.0 <0.7.0;")
|
|
|
- .collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(
|
|
|
- Ok((0, Token::Pragma, 6)),
|
|
|
- Ok((7, Token::Identifier("solidity"), 15)),
|
|
|
- Ok((16, Token::StringLiteral(">=0.5.0 <0.7.0"), 30)),
|
|
|
- Ok((30, Token::Semicolon, 31)),
|
|
|
- )
|
|
|
- );
|
|
|
-
|
|
|
- let tokens = Lexer::new("pragma solidity \t>=0.5.0 <0.7.0 \n ;")
|
|
|
- .collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(
|
|
|
- Ok((0, Token::Pragma, 6)),
|
|
|
- Ok((7, Token::Identifier("solidity"), 15)),
|
|
|
- Ok((17, Token::StringLiteral(">=0.5.0 <0.7.0"), 31)),
|
|
|
- Ok((34, Token::Semicolon, 35)),
|
|
|
- )
|
|
|
- );
|
|
|
-
|
|
|
- let tokens = Lexer::new("pragma solidity 赤;")
|
|
|
- .collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(
|
|
|
- Ok((0, Token::Pragma, 6)),
|
|
|
- Ok((7, Token::Identifier("solidity"), 15)),
|
|
|
- Ok((16, Token::StringLiteral("赤"), 19)),
|
|
|
- Ok((19, Token::Semicolon, 20))
|
|
|
- )
|
|
|
- );
|
|
|
-
|
|
|
- let tokens =
|
|
|
- Lexer::new(">>= >> >= >").collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(
|
|
|
- Ok((0, Token::ShiftRightAssign, 3)),
|
|
|
- Ok((4, Token::ShiftRight, 6)),
|
|
|
- Ok((7, Token::MoreEqual, 9)),
|
|
|
- Ok((10, Token::More, 11)),
|
|
|
- )
|
|
|
- );
|
|
|
-
|
|
|
- let tokens =
|
|
|
- Lexer::new("<<= << <= <").collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(
|
|
|
- Ok((0, Token::ShiftLeftAssign, 3)),
|
|
|
- Ok((4, Token::ShiftLeft, 6)),
|
|
|
- Ok((7, Token::LessEqual, 9)),
|
|
|
- Ok((10, Token::Less, 11)),
|
|
|
- )
|
|
|
- );
|
|
|
-
|
|
|
- let tokens =
|
|
|
- Lexer::new("-16 -- - -=").collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(
|
|
|
- Ok((0, Token::Subtract, 1)),
|
|
|
- Ok((1, Token::Number("16", ""), 3)),
|
|
|
- Ok((4, Token::Decrement, 6)),
|
|
|
- Ok((7, Token::Subtract, 8)),
|
|
|
- Ok((9, Token::SubtractAssign, 11)),
|
|
|
- )
|
|
|
- );
|
|
|
-
|
|
|
- let tokens = Lexer::new("-4 ").collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(
|
|
|
- Ok((0, Token::Subtract, 1)),
|
|
|
- Ok((1, Token::Number("4", ""), 2)),
|
|
|
- )
|
|
|
- );
|
|
|
-
|
|
|
- let tokens =
|
|
|
- Lexer::new(r#"hex"abcdefg""#).collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(Err(LexicalError::InvalidCharacterInHexLiteral(10, 'g')))
|
|
|
- );
|
|
|
-
|
|
|
- let tokens = Lexer::new(r#" € "#).collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(Err(LexicalError::UnrecognisedToken(1, 4, "€".to_owned())))
|
|
|
- );
|
|
|
-
|
|
|
- let tokens = Lexer::new(r#"€"#).collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(Err(LexicalError::UnrecognisedToken(0, 3, "€".to_owned())))
|
|
|
- );
|
|
|
-
|
|
|
- let tokens = Lexer::new(r#"pragma foo bar"#)
|
|
|
- .collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(
|
|
|
- Ok((0, Token::Pragma, 6)),
|
|
|
- Ok((7, Token::Identifier("foo"), 10)),
|
|
|
- Ok((11, Token::StringLiteral("bar"), 14)),
|
|
|
- )
|
|
|
- );
|
|
|
-
|
|
|
- let tokens =
|
|
|
- Lexer::new(r#"/// foo"#).collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(Ok((3, Token::DocComment(CommentType::Line, " foo"), 7)))
|
|
|
- );
|
|
|
-
|
|
|
- let tokens = Lexer::new("/// jadajadadjada\n// bar")
|
|
|
- .collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(Ok((
|
|
|
- 3,
|
|
|
- Token::DocComment(CommentType::Line, " jadajadadjada"),
|
|
|
- 17
|
|
|
- )))
|
|
|
- );
|
|
|
-
|
|
|
- let tokens =
|
|
|
- Lexer::new(r#"/** foo */"#).collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(Ok((3, Token::DocComment(CommentType::Block, " foo "), 8)))
|
|
|
- );
|
|
|
-
|
|
|
- let tokens = Lexer::new("/** jadajadadjada */\n/* bar */")
|
|
|
- .collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(Ok((
|
|
|
- 3,
|
|
|
- Token::DocComment(CommentType::Block, " jadajadadjada "),
|
|
|
- 18
|
|
|
- )))
|
|
|
- );
|
|
|
-
|
|
|
- let tokens = Lexer::new("/************/").next();
|
|
|
- assert_eq!(tokens, None);
|
|
|
-
|
|
|
- let tokens = Lexer::new("/**").next();
|
|
|
- assert_eq!(tokens, Some(Err(LexicalError::EndOfFileInComment(0, 3))));
|
|
|
-
|
|
|
- let tokens = Lexer::new("//////////////").next();
|
|
|
- assert_eq!(tokens, None);
|
|
|
-
|
|
|
- // some unicode tests
|
|
|
- let tokens = Lexer::new(">=\u{a0} . très\u{2028}αβγδεζηθικλμνξοπρστυφχψω\u{85}カラス")
|
|
|
- .collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(
|
|
|
- Ok((0, Token::MoreEqual, 2)),
|
|
|
- Ok((5, Token::Member, 6)),
|
|
|
- Ok((7, Token::Identifier("très"), 12)),
|
|
|
- Ok((15, Token::Identifier("αβγδεζηθικλμνξοπρστυφχψω"), 63)),
|
|
|
- Ok((65, Token::Identifier("カラス"), 74))
|
|
|
- )
|
|
|
- );
|
|
|
-
|
|
|
- let tokens =
|
|
|
- Lexer::new(r#"unicode"€""#).collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(tokens, vec!(Ok((0, Token::StringLiteral("€"), 12)),));
|
|
|
-
|
|
|
- let tokens =
|
|
|
- Lexer::new(r#"unicode "€""#).collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(
|
|
|
- Ok((0, Token::Identifier("unicode"), 7)),
|
|
|
- Ok((8, Token::StringLiteral("€"), 13)),
|
|
|
- )
|
|
|
- );
|
|
|
-
|
|
|
- // scientific notation
|
|
|
- let tokens =
|
|
|
- Lexer::new(r#" 1e0 "#).collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(tokens, vec!(Ok((1, Token::Number("1", "0"), 4)),));
|
|
|
-
|
|
|
- let tokens =
|
|
|
- Lexer::new(r#" -9e0123"#).collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(
|
|
|
- Ok((1, Token::Subtract, 2)),
|
|
|
- Ok((2, Token::Number("9", "0123"), 8)),
|
|
|
- )
|
|
|
- );
|
|
|
-
|
|
|
- let tokens =
|
|
|
- Lexer::new(r#" -9e"#).collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(
|
|
|
- Ok((1, Token::Subtract, 2)),
|
|
|
- Err(LexicalError::MissingExponent(2, 4))
|
|
|
- )
|
|
|
- );
|
|
|
-
|
|
|
- let tokens = Lexer::new(r#"9ea"#).collect::<Vec<Result<(usize, Token, usize), LexicalError>>>();
|
|
|
-
|
|
|
- assert_eq!(
|
|
|
- tokens,
|
|
|
- vec!(
|
|
|
- Err(LexicalError::MissingExponent(0, 3)),
|
|
|
- Ok((2, Token::Identifier("a"), 3))
|
|
|
- )
|
|
|
- );
|
|
|
-}
|
Matthias Seitz