Lexer.pudu

Pudu104 lines4.2 KB

Copy pathGitHub ↗

1/** @Log.Expressions.Lexer.Scanner — expression text into tokens */2module PuduLangLog.Expressions.Lexer34import Std.Char as Char5import Std.Set as Set67/** @Log.Expressions.Lexer.Kind — what a token is */8export type Kind = Identifier | BuiltIn | Number | Text | Keyword | Symbol | End910/** @Log.Expressions.Lexer.Token — a token and where it starts */11export type Token = { kind: Kind, text: Str, position: Int }1213/// Words with a meaning of their own, matched without regard to case.14const KEYWORDS: Set[Str] = setOf(["and", "or", "not", "like", "in", "is", "null", "true", "false", "if", "then", "else", "ci"])1516/// Two-character symbols, tried before single characters.17const PAIRS: Set[Str] = setOf(["<>", "<=", ">=", ".."])1819/// Single-character symbols.20const SINGLES: Str = "=<>+-*/%^()[]\{\},.:?"2122/// The tokens of the text followed by an end token, or the position and reason of the first23/// character that starts no token.24export fn tokens(text: Str) -> Result[Array[Token], Str] {25  let characters = text.chars()26  var found: Array[Token] = []27  var index = 028  while index < characters.length() {29    let character = characters[index]30    if Char.isWhitespace(character) {31      index = index + 132    } else if character == '\'' {33      let (literal, next) = textAt(&characters, index) ?34      found = found.push(Token{kind: Text, text: literal, position: index})35      index = next36    } else if Char.isDigit(character) {37      let next = numberEnd(&characters, index)38      found = found.push(Token{kind: Number, text: text.slice(index, next), position: index})39      index = next40    } else if character == '@' || isStart(character) {41      let next = wordEnd(&characters, index + 1)42      let word = text.slice(index, next)43      let kind = if character == '@' { BuiltIn } else if Set.contains(&KEYWORDS, word.toLower()) { Keyword } else { Identifier }44      found = found.push(Token{kind: kind, text: if kind == Keyword { word.toLower() } else { word }, position: index})45      index = next46    } else if Set.contains(&PAIRS, text.slice(index, index + 2)) {47      found = found.push(Token{kind: Symbol, text: text.slice(index, index + 2), position: index})48      index = index + 249    } else if SINGLES.contains(character.toText()) {50      found = found.push(Token{kind: Symbol, text: character.toText(), position: index})51      index = index + 152    } else {53      return Err("unexpected `" + character.toText() + "` at " + show(index))54    }55  }56  Ok(found.push(Token{kind: End, text: "", position: characters.length()}))57}5859/// Whether a character starts a name.60fn isStart(character: Char) -> Bool { Char.isLetter(character) || character == '_' || !Char.isAscii(character) }6162/// The index after the name characters from `start`.63fn wordEnd(characters: &Array[Char], start: Int) -> Int {64  var end = start65  while end < characters.length() && (isStart(characters[end]) || Char.isDigit(characters[end])) { end = end + 1 }66  end67}6869/// The index after a decimal number with an optional fraction, or a hexadecimal number after `0x`.70fn numberEnd(characters: &Array[Char], start: Int) -> Int {71  var end = start + 172  if characters[start] == '0' && end < characters.length() && characters[end] == 'x' {73    end = end + 174    while end < characters.length() && Char.isHexDigit(characters[end]) { end = end + 1 }75    return end76  }77  while end < characters.length() && Char.isDigit(characters[end]) { end = end + 1 }78  if end + 1 < characters.length() && characters[end] == '.' && Char.isDigit(characters[end + 1]) {79    end = end + 180    while end < characters.length() && Char.isDigit(characters[end]) { end = end + 1 }81  }82  end83}8485/// The text of a single-quoted literal starting at `start`, a doubled quote standing for one, and86/// the index after it.87fn textAt(characters: &Array[Char], start: Int) -> Result[(Str, Int), Str] {88  var pieces: Array[Str] = []89  var index = start + 190  while index < characters.length() {91    let character = characters[index]92    if character == '\'' {93      let doubled = index + 1 < characters.length() && characters[index + 1] == '\''94      if !doubled { return Ok((pieces.join(""), index + 1)) }95      pieces = pieces.push("'")96      index = index + 297    } else {98      pieces = pieces.push(character.toText())99      index = index + 1100    }101  }102  Err("unterminated text starting at " + show(start))103}104