Parser.pudu

Pudu371 lines16.0 KB

Copy pathGitHub ↗

1/** @Docgen.Api.Parser — module declarations and their documentation read from tokens */2module PuduLangDocgen.Api.Parser34import PuduLangDocgen.Api.Lexer as Lexer5import PuduLangDocgen.Api.Model as Model67/// Words that begin a declaration at module scope.8const STARTS: Array[Str] = ["export", "type", "trait", "impl", "const", "async", "module", "import"]910/// Longest constant value shown in its signature.11const VALUE_LIMIT: Int = 601213/// Symbols written without a space before them.14const TIGHT_BEFORE: Array[Str] = [")", "]", ",", ":", ".", "?"]1516/// Symbols written without a space after them.17const TIGHT_AFTER: Array[Str] = ["(", "[", ".", "&", "*", "!"]1819/// The module declared in a source file at a project-relative path.20export fn parse(path: Str, source: Str) -> Model.Unit {21  let tokens = Lexer.tokens(source)22  var unit = Model.Unit{uid: fallbackName(path), path: path, doc: "", line: 1, imports: [], members: [], implementations: []}23  var doc = ""24  var exported = false25  var index = 026  while index < tokens.length() {27    let token = tokens[index]28    let word = if token.kind == Lexer.Word { token.text } else { "" }29    if token.kind == Lexer.LineDoc || token.kind == Lexer.BlockDoc {30      let (text, next) = docAt(&tokens, index)31      doc = text32      index = next33    } else if word == "export" {34      exported = true35      index = index + 136    } else if word == "module" {37      let (name, next) = dotted(&tokens, index + 1)38      unit = Model.Unit{..unit, uid: name, doc: doc, line: token.line}39      doc = ""40      index = next41    } else if word == "import" {42      let (name, next) = dotted(&tokens, index + 1)43      var alias = name.split(".")[name.split(".").length() - 1]44      var after = next45      if after + 1 < tokens.length() && tokens[after].text == "as" {46        alias = tokens[after + 1].text47        after = after + 248      }49      unit = Model.Unit{..unit, imports: unit.imports.push((alias, name))}50      index = after51    } else if word == "fn" || word == "async" || word == "type" || word == "trait" || word == "const" {52      let (found, next) = declaration(&tokens, index, doc, exported, unit.uid)53      unit = Model.Unit{..unit, members: unit.members.push(found)}54      doc = ""55      exported = false56      index = next57    } else if word == "impl" {58      let (held, next) = implementation(&tokens, index)59      unit = Model.Unit{..unit, implementations: unit.implementations.push(held)}60      doc = ""61      exported = false62      index = next63    } else {64      doc = ""65      exported = false66      index = index + 167    }68  }69  unit70}7172/// Tokens joined the way declarations are conventionally written.73export fn render(tokens: &Array[Lexer.Token], from: Int, to: Int) -> Str {74  var pieces: Array[Str] = []75  var previous = ""76  for index in from..to {77    if tokens[index].kind == Lexer.LineDoc || tokens[index].kind == Lexer.BlockDoc { continue }78    let text = tokens[index].text79    let spaced = !pieces.isEmpty() && !TIGHT_BEFORE.contains(text) && !TIGHT_AFTER.contains(previous) && !((text == "(" || text == "[") && previous != "->" && previous != "=" && previous != "|" && previous != "," && previous != ":")80    if spaced { pieces = pieces.push(" ") }81    pieces = pieces.push(text)82    previous = text83  }84  pieces.join("")85}8687/// The module name implied by a source path when the file declares none.88fn fallbackName(path: Str) -> Str {89  let trimmed = if path.endsWith(".pudu") { path.take(path.length() - 5) } else { path }90  let parts = trimmed.split("/")91  parts[parts.length() - 1]92}9394/// Consecutive documentation comments as one text, and the index after them.95fn docAt(tokens: &Array[Lexer.Token], index: Int) -> (Str, Int) {96  if tokens[index].kind == Lexer.BlockDoc { return (anchorless(tokens[index].text), index + 1) }97  var lines: Array[Str] = []98  var cursor = index99  while cursor < tokens.length() && tokens[cursor].kind == Lexer.LineDoc {100    lines = lines.push(tokens[cursor].text)101    cursor = cursor + 1102  }103  (lines.join("\n"), cursor)104}105106/// Documentation of a namespace anchor `@Name.Role — text` reduced to its text.107fn anchorless(text: Str) -> Str {108  if !text.startsWith("@") { return text }109  for dash in [" — ", " – ", " - "] {110    let at = text.indexOf(dash)111    if at > 0 {112      let rest = text.drop(at + dash.length()).trim()113      return rest.take(1).toUpper() + rest.drop(1)114    }115  }116  text117}118119/// A dotted name and the index after it.120fn dotted(tokens: &Array[Lexer.Token], index: Int) -> (Str, Int) {121  var parts: Array[Str] = []122  var cursor = index123  while cursor < tokens.length() && tokens[cursor].kind == Lexer.Word {124    parts = parts.push(tokens[cursor].text)125    cursor = cursor + 1126    if cursor < tokens.length() && tokens[cursor].text == "." { cursor = cursor + 1 } else { break }127  }128  (parts.join("."), cursor)129}130131/// The index after the bracket matching the one at an index.132fn matching(tokens: &Array[Lexer.Token], index: Int) -> Int {133  let open = tokens[index].text134  let close = if open == "(" { ")" } else if open == "[" { "]" } else { "\}" }135  var depth = 0136  var cursor = index137  while cursor < tokens.length() {138    let text = tokens[cursor].text139    if tokens[cursor].kind == Lexer.Symbol {140      if text == open { depth = depth + 1 }141      if text == close {142        depth = depth - 1143        if depth == 0 { return cursor + 1 }144      }145    }146    cursor = cursor + 1147  }148  cursor149}150151/// Whether a token starts a new declaration or its documentation.152fn starts(tokens: &Array[Lexer.Token], index: Int) -> Bool {153  let token = tokens[index]154  if token.kind == Lexer.LineDoc || token.kind == Lexer.BlockDoc { return true }155  if token.kind != Lexer.Word { return false }156  if STARTS.contains(token.text) { return true }157  token.text == "fn" && index + 1 < tokens.length() && tokens[index + 1].kind == Lexer.Word158}159160/// The index of the next declaration start or closing brace outside brackets.161fn nextStart(tokens: &Array[Lexer.Token], index: Int) -> Int {162  var cursor = index163  var depth = 0164  while cursor < tokens.length() {165    let text = tokens[cursor].text166    let symbol = tokens[cursor].kind == Lexer.Symbol167    if depth == 0 && (starts(tokens, cursor) || (symbol && text == "\}")) { return cursor }168    if symbol && (text == "(" || text == "[" || text == "\{") { depth = depth + 1 }169    if symbol && (text == ")" || text == "]" || text == "\}") { depth = depth - 1 }170    cursor = cursor + 1171  }172  cursor173}174175/// Token ranges between commas at the top level of a bracketed list.176fn listed(tokens: &Array[Lexer.Token], from: Int, to: Int) -> Array[(Int, Int)] {177  var result: Array[(Int, Int)] = []178  var start = from179  var depth = 0180  for index in from..to {181    if tokens[index].kind == Lexer.LineDoc || tokens[index].kind == Lexer.BlockDoc { continue }182    let text = tokens[index].text183    if tokens[index].kind == Lexer.Symbol {184      if text == "(" || text == "[" || text == "\{" { depth = depth + 1 }185      if text == ")" || text == "]" || text == "\}" { depth = depth - 1 }186      if text == "," && depth == 0 {187        if index > start { result = result.push((start, index)) }188        start = index + 1189      }190    }191  }192  if to > start { result.push((start, to)) } else { result }193}194195/// A function, type, trait, or constant declaration at an index.196fn declaration(tokens: &Array[Lexer.Token], index: Int, doc: Str, exported: Bool, owner: Str) -> (Model.Member, Int) {197  let word = tokens[index].text198  if word == "type" { return typeAt(tokens, index, doc, exported, owner) }199  if word == "trait" { return traitAt(tokens, index, doc, exported, owner) }200  if word == "const" { return constAt(tokens, index, doc, exported, owner) }201  function(tokens, index, doc, exported, owner, "function")202}203204/// An empty member of a kind.205fn member(uid: Str, name: Str, kind: Str, doc: Str, line: Int, exported: Bool) -> Model.Member {206  Model.Member{uid: uid, name: name, kind: kind, signature: "", doc: doc, line: line, exported: exported, parameters: [], returns: "", members: []}207}208209/// A function or method; bodies, when present, are skipped.210fn function(tokens: &Array[Lexer.Token], index: Int, doc: Str, exported: Bool, owner: Str, kind: Str) -> (Model.Member, Int) {211  let start = index212  var cursor = if tokens[index].text == "async" { index + 1 } else { index }213  let name = if cursor + 1 < tokens.length() { tokens[cursor + 1].text } else { "" }214  cursor = cursor + 2215  if cursor < tokens.length() && tokens[cursor].text == "[" { cursor = matching(tokens, cursor) }216  var parameters: Array[Model.Parameter] = []217  if cursor < tokens.length() && tokens[cursor].text == "(" {218    let close = matching(tokens, cursor)219    for (from, to) in listed(tokens, cursor + 1, close - 1) {220      let colon = if from + 1 < to && tokens[from + 1].text == ":" { from + 2 } else { from + 1 }221      parameters = parameters.push(Model.Parameter{name: tokens[from].text, kind: render(tokens, colon, to)})222    }223    cursor = close224  }225  var returns = ""226  if cursor < tokens.length() && tokens[cursor].text == "->" {227    var end = cursor + 1228    var depth = 0229    while end < tokens.length() {230      let text = tokens[end].text231      if depth == 0 && (text == "\{" || text == "=" || text == "\}" || starts(tokens, end)) { break }232      if text == "(" || text == "[" { depth = depth + 1 }233      if text == ")" || text == "]" { depth = depth - 1 }234      end = end + 1235    }236    returns = render(tokens, cursor + 1, end)237    cursor = end238  }239  let signature = render(tokens, start, cursor)240  if cursor < tokens.length() && tokens[cursor].text == "\{" { cursor = matching(tokens, cursor) }241    else if cursor < tokens.length() && tokens[cursor].text == "=" { cursor = nextStart(tokens, cursor + 1) }242  let found = member(owner + "." + name, name, kind, doc, tokens[index].line, exported)243  (Model.Member{..found, signature: signature, parameters: parameters, returns: returns}, cursor)244}245246/// A record, union, alias, or opaque type.247fn typeAt(tokens: &Array[Lexer.Token], index: Int, doc: Str, exported: Bool, owner: Str) -> (Model.Member, Int) {248  let name = tokens[index + 1].text249  let uid = owner + "." + name250  var cursor = index + 2251  if cursor < tokens.length() && tokens[cursor].text == "[" { cursor = matching(tokens, cursor) }252  let head = render(tokens, index, cursor)253  let base = member(uid, name, "opaque", doc, tokens[index].line, exported)254  if cursor >= tokens.length() || tokens[cursor].text != "=" { return (Model.Member{..base, signature: head}, cursor) }255  cursor = cursor + 1256  if cursor < tokens.length() && tokens[cursor].text == "\{" {257    let close = matching(tokens, cursor)258    var fields: Array[Model.Member] = []259    var fieldDoc = ""260    for (start, end) in listed(tokens, cursor + 1, close - 1) {261      var first = start262      while first < end && (tokens[first].kind == Lexer.LineDoc || tokens[first].kind == Lexer.BlockDoc) {263        let (text, next) = docAt(tokens, first)264        fieldDoc = text265        first = next266      }267      if first + 2 <= end {268        let field = member(uid + "." + tokens[first].text, tokens[first].text, "field", fieldDoc, tokens[first].line, exported)269        fields = fields.push(Model.Member{..field, signature: render(tokens, first, end), returns: render(tokens, first + 2, end)})270      }271      fieldDoc = ""272    }273    let written = if fields.length() > 2 { head + " = \{\n" + fields.map(|field: Model.Member| "  " + field.signature).join(",\n") + "\n\}" } else { head + " = " + render(tokens, cursor, close) }274    return (Model.Member{..base, kind: "record", signature: written, members: fields}, close)275  }276  let end = nextStart(tokens, cursor)277  var variants: Array[Model.Member] = []278  var start = cursor279  var depth = 0280  for position in cursor..(end + 1) {281    let boundary = position == end || (depth == 0 && tokens[position].text == "|")282    if position < end {283      let text = tokens[position].text284      if text == "(" || text == "[" { depth = depth + 1 }285      if text == ")" || text == "]" { depth = depth - 1 }286    }287    if boundary && position > start {288      let variant = member(uid + "." + tokens[start].text, tokens[start].text, "variant", "", tokens[start].line, exported)289      variants = variants.push(Model.Member{..variant, signature: render(tokens, start, position), returns: render(tokens, start + 1, position)})290      start = position + 1291    } else if boundary { start = position + 1 }292  }293  let union = variants.length() > 1 || (variants.length() == 1 && cursor + 1 < end && tokens[cursor + 1].text == "(")294  let written = head + " = " + render(tokens, cursor, end)295  if union { (Model.Member{..base, kind: "union", signature: written, members: variants}, end) } else { (Model.Member{..base, kind: "alias", signature: written, returns: render(tokens, cursor, end)}, end) }296}297298/// A trait and the methods it requires.299fn traitAt(tokens: &Array[Lexer.Token], index: Int, doc: Str, exported: Bool, owner: Str) -> (Model.Member, Int) {300  let name = tokens[index + 1].text301  let uid = owner + "." + name302  var cursor = index + 2303  if cursor < tokens.length() && tokens[cursor].text == "[" { cursor = matching(tokens, cursor) }304  let signature = render(tokens, index, cursor)305  if cursor >= tokens.length() || tokens[cursor].text != "\{" { return (Model.Member{..member(uid, name, "trait", doc, tokens[index].line, exported), signature: signature}, cursor) }306  let close = matching(tokens, cursor)307  let methods = methodsIn(tokens, cursor + 1, close - 1, uid, exported)308  (Model.Member{..member(uid, name, "trait", doc, tokens[index].line, exported), signature: signature, members: methods}, close)309}310311/// Documented methods between two indexes.312fn methodsIn(tokens: &Array[Lexer.Token], from: Int, to: Int, owner: Str, exported: Bool) -> Array[Model.Member] {313  var methods: Array[Model.Member] = []314  var doc = ""315  var cursor = from316  while cursor < to {317    let token = tokens[cursor]318    if token.kind == Lexer.LineDoc || token.kind == Lexer.BlockDoc {319      let (text, next) = docAt(tokens, cursor)320      doc = text321      cursor = next322    } else if token.kind == Lexer.Word && (token.text == "fn" || token.text == "async") {323      let (method, next) = function(tokens, cursor, doc, exported, owner, "method")324      methods = methods.push(method)325      doc = ""326      cursor = if next > cursor { next } else { cursor + 1 }327    } else { cursor = cursor + 1 }328  }329  methods330}331332/// An implementation of a trait for a type.333fn implementation(tokens: &Array[Lexer.Token], index: Int) -> (Model.Implementation, Int) {334  var cursor = index + 1335  if cursor < tokens.length() && tokens[cursor].text == "[" { cursor = matching(tokens, cursor) }336  let start = cursor337  while cursor < tokens.length() && tokens[cursor].text != "for" && tokens[cursor].text != "\{" { cursor = cursor + 1 }338  var contract = ""339  var target = render(tokens, start, cursor)340  if cursor < tokens.length() && tokens[cursor].text == "for" {341    contract = target342    let after = cursor + 1343    cursor = after344    while cursor < tokens.length() && tokens[cursor].text != "\{" { cursor = cursor + 1 }345    target = render(tokens, after, cursor)346  }347  if cursor >= tokens.length() { return (Model.Implementation{contract: contract, target: target, line: tokens[index].line, methods: []}, cursor) }348  let close = matching(tokens, cursor)349  (Model.Implementation{contract: contract, target: target, line: tokens[index].line, methods: methodsIn(tokens, cursor + 1, close - 1, target, true)}, close)350}351352/// A constant with its type and, when short, its value.353fn constAt(tokens: &Array[Lexer.Token], index: Int, doc: Str, exported: Bool, owner: Str) -> (Model.Member, Int) {354  let name = tokens[index + 1].text355  var cursor = index + 2356  var kind = ""357  if cursor < tokens.length() && tokens[cursor].text == ":" {358    let from = cursor + 1359    while cursor < tokens.length() && tokens[cursor].text != "=" && !starts(tokens, cursor) { cursor = cursor + 1 }360    kind = render(tokens, from, cursor)361  }362  var signature = render(tokens, index, cursor)363  if cursor < tokens.length() && tokens[cursor].text == "=" {364    let end = nextStart(tokens, cursor + 1)365    let value = render(tokens, cursor + 1, end)366    if value.length() <= VALUE_LIMIT && !value.contains("\n") { signature = signature + " = " + value }367    cursor = end368  }369  (Model.Member{..member(owner + "." + name, name, "constant", doc, tokens[index].line, exported), signature: signature, returns: kind}, cursor)370}371