Search.pudu

Pudu91 lines3.3 KB

Copy pathGitHub ↗

1/** @Domain.Docs.Search.Module — ranks documentation sections for a query */2module PuduLangMcp.Domain.Docs.Search34import Std.Char as Char5import Std.List as List6import Std.Math as Math7import Std.Text as Text8import PuduLangMcp.Domain.Docs.Chapter as Chapter910/** @Domain.Docs.Search.Hit — a ranked documentation section */11export type Hit = { section: Chapter.Section, score: Int }1213/// Words too common to rank by.14const STOP_WORDS: Set[Str] = setOf([15    "a", "an", "and", "are", "as", "at", "be", "by", "can", "do", "does", "for", "from", "how", "i", "if",16    "in", "is", "it", "me", "my", "of", "on", "or", "pudu", "should", "the", "to", "use", "what", "when",17    "where", "which", "why", "with", "write"18  ])1920/// Points when the whole query appears in a heading.21const PHRASE_IN_HEADING: Int = 4022/// Points when the whole query appears in a section's text.23const PHRASE_IN_TEXT: Int = 1224/// Points per query term found in a heading.25const TERM_IN_HEADING: Int = 1026/// Points per occurrence of a term in a section's text.27const TERM_IN_TEXT: Int = 228/// Occurrences of one term counted before they stop adding points.29const MAX_COUNTED_OCCURRENCES: Int = 530/// Points when a section contains every term.31const EVERY_TERM: Int = 1532/// The shortest query scored as a phrase.33const MIN_PHRASE_LENGTH: Int = 33435/// The distinct search terms of a query, stop words removed.36export fn terms(query: Str) -> Array[Str] {37  var found: Array[Str] = []38  var current = ""39  for character in (query.toLower() + " ").chars() {40    if isTermCharacter(character) {41      current = current + Text.singleton(character)42    } else {43      let term = Text.dropAround(current, |c: Char| c == '.')44      if !term.isEmpty() && !STOP_WORDS.contains(term) && !found.contains(term) { found = found.push(term) }45      current = ""46    }47  }48  found49}5051/// How well one section answers a query; zero when no term matches.52export fn score(section: &Chapter.Section, query: Str, words: &Array[Str]) -> Int {53  if words.isEmpty() { return 0 }54  let heading = section.heading.toLower()55  let text = section.text.toLower()56  let phrase = query.trim().toLower()57  var total = 058  if phrase.length() >= MIN_PHRASE_LENGTH {59    if heading.contains(phrase) { total = total + PHRASE_IN_HEADING }60    if text.contains(phrase) { total = total + PHRASE_IN_TEXT }61  }62  var matched = 063  for word in *words {64    let inHeading = heading.contains(word)65    let occurrences = Text.countOccurrences(text, word)66    if inHeading { total = total + TERM_IN_HEADING }67    let counted = Math.min(occurrences, MAX_COUNTED_OCCURRENCES)68    total = total + counted * TERM_IN_TEXT69    if inHeading || occurrences > 0 { matched = matched + 1 }70  }71  if matched == words.length() { total = total + EVERY_TERM }72  if matched == 0 { 0 } else { total }73}7475/// The best-matching sections, highest score first.76export fn search(sections: &Array[Chapter.Section], query: Str, limit: Int) -> Array[Hit] {77  let words = terms(query)78  if words.isEmpty() { return [] }79  var ranked: Array[Hit] = []80  for section in *sections {81    let points = score(&section, query, &words)82    if points > 0 { ranked = ranked.push(Hit{section: section, score: points}) }83  }84  List.take(&List.sortOn(&ranked, |hit: Hit| 0 - hit.score), limit)85}8687/// Whether a character belongs to a search term.88fn isTermCharacter(character: Char) -> Bool {89  Char.isAlphanumeric(character) || character == '_' || character == '.'90}91