Search.pudu

Pudu52 lines2.0 KB

Copy pathGitHub ↗

1/** @Docgen.Site.Search — the searchable text of published pages */2module PuduLangDocgen.Site.Search34import Std.Json as Json5import PuduLangDocgen as Docgen6import PuduLangDocgen.Meta as Meta78/// Longest page text kept in the index.9const TEXT_LIMIT: Int = 80001011/// Elements whose content is never searchable text.12const HIDDEN: Array[Str] = ["script", "style", "button"]1314/// The search index as JSON: href, title, summary, and text of every indexable page.15/// Pages with `_noindex` and redirect pages are left out.16export fn index(pages: &Array[Docgen.Page]) -> Str {17  var entries: Array[Json.Json] = []18  for page in *pages {19    if page.kind == "redirect" || Meta.flag(&page.meta, "_noindex") { continue }20    let text = plainText(page.body)21    entries = entries.push(Json.object(&[("href", Json.Text(page.path)), ("title", Json.Text(page.title)), ("summary", Json.Text(page.summary)), ("text", Json.Text(text.take(TEXT_LIMIT)))]))22  }23  Json.encode(&Json.list(&entries))24}2526/// Visible text of HTML with white space collapsed and common references decoded.27export fn plainText(html: Str) -> Str {28  let cs = html.chars()29  var pieces: Array[Str] = []30  var cursor = 031  var hidden = ""32  while cursor < cs.length() {33    let character = cs[cursor]34    if character == '<' {35      var end = cursor + 136      while end < cs.length() && cs[end] != '>' { end = end + 1 }37      var tag: Array[Str] = []38      for position in (cursor + 1)..end { tag = tag.push(cs[position].toText()) }39      let name = tag.join("").split(" ")[0].toLower()40      if hidden.isEmpty() && HIDDEN.contains(name) { hidden = name }41        else if !hidden.isEmpty() && name == "/" + hidden { hidden = "" }42      pieces = pieces.push(" ")43      cursor = end + 144    } else {45      if hidden.isEmpty() { pieces = pieces.push(character.toText()) }46      cursor = cursor + 147    }48  }49  let text = pieces.join("").replace("&lt;", "<").replace("&gt;", ">").replace("&quot;", "\"").replace("&#39;", "'").replace("&amp;", "&")50  text.replace("\n", " ").replace("\t", " ").split(" ").filter(|word: Str| !word.isEmpty()).join(" ")51}52