Mutate.pudu

Pudu245 lines9.4 KB

Copy pathGitHub ↗

1/** @Tools.Mutate.Harness — mutation testing over the source tree */2module Mutate34import Std.Char as Char5import Std.Env as Env6import Std.Io as Io7import Std.List as List8import Std.Math as Math9import Std.Option as Option10import Std.Process as Process11import Std.Result as Result12import Std.Text as Text1314/** @Tools.Mutate.Mutant — one single-point change to one file */15type Mutant = { file: Str, line: Int, column: Int, from: Str, to: Str }1617/** @Tools.Mutate.Verdict — what the checks said about one mutant */18type Verdict = Killed | Survived | Invalid1920/// Operator replacements, tried at every code position in this order.21const OPERATORS: Array[(Str, Str)] = [22  ("==", "!="), ("!=", "=="), ("<=", "<"), (">=", ">"), ("<", "<="), (">", ">="),23  ("&&", "||"), ("||", "&&"), ("true", "false"), ("false", "true"), ("+ 1", "- 1"), ("- 1", "+ 1"), ("!", "")24]2526/// Longest a mutant's test run may take before it counts as killed.27const MUTANT_TIMEOUT_MS: Int = 600002829/// Folder holding the package's modules.30const SOURCE_ROOT: Str = "src/PuduLangDocgen/"3132/// Folder holding the package's suites.33const TEST_ROOT: Str = "test/PuduLangDocgen/"3435/// Source files outside the pure layer: seams to the disk, the network, the command line,36/// and generated reference tables, which `--domain` leaves alone.37const OUTSIDE_DOMAIN: Array[Str] = [38  "src/PuduLangDocgen/Command.pudu", "src/PuduLangDocgen/Docset.pudu", "src/PuduLangDocgen/Serve.pudu",39  "src/PuduLangDocgen/Docset/", "src/PuduLangDocgen/Markdown/Emoji", "src/PuduLangDocgen/Markdown/Entities.pudu",40  "src/PuduLangDocgen/Markdown/Languages/", "src/PuduLangDocgen/Theme/", "src/PuduLangDocgen/Constants/"41]4243/// Runs the harness and answers 0, or 1 when the score is below the threshold.44fn main() -> Int {45  let every = Option.unwrapOr(Option.andThen(Env.option("--every"), |given: Str| Text.countOf(given)), 1)46  let offset = Option.unwrapOr(Option.andThen(Env.option("--offset"), |given: Str| Text.countOf(given)), 0)47  let threshold = Option.unwrapOr(Option.andThen(Env.option("--threshold"), |given: Str| Text.countOf(given)), 0)48  let files = match Env.option("--file") {49    case Some(one) => [one]50    case None => if Env.hasFlag("--domain") { sources("src").filter(|file: Str| OUTSIDE_DOMAIN.filter(|outside: Str| file.startsWith(outside)).isEmpty()) } else { sources("src") }51  }52  var all: Array[Mutant] = []53  for file in files {54    all = all.concat(mutantsOf(file, Result.unwrapOr(Io.read(file), "")))55  }56  var chosen: Array[Mutant] = []57  var index = 058  for mutant in all {59    let step = Math.max(1, every)60    if index % step == offset % step { chosen = chosen.push(mutant) }61    index = index + 162  }63  if Env.hasFlag("--dry-run") {64    for mutant in chosen { let _listed = Io.writeLine(describe(&mutant)) }65    let _counted = Io.writeLine(show(chosen.length()) + " mutants")66    return 067  }68  let compiler = Env.variableOr("PUDU_BIN", "pudu")69  if !Process.isAvailable(compiler) {70    let _said = Io.writeErrorLine("Mutate: " + compiler + " was not found; set PUDU_BIN")71    return 172  }73  var killed = 074  var survived = 075  var invalid = 076  var number = 077  for mutant in chosen {78    number = number + 179    match judge(compiler, &mutant) {80      case Killed => { killed = killed + 1 }81      case Survived => {82        survived = survived + 183        let _reported = Io.writeLine("survived  " + describe(&mutant))84      }85      case Invalid => { invalid = invalid + 1 }86    }87    let _progress = Io.writeErrorLine("[" + show(number) + "/" + show(chosen.length()) + "] " + describe(&mutant))88  }89  let scored = killed + survived90  let score = if scored == 0 { 100 } else { killed * 100 / scored }91  let _summary = Io.writeLine("killed " + show(killed) + ", survived " + show(survived) + ", invalid " + show(invalid) + ", score " + show(score) + "%")92  if score < threshold { 1 } else { 0 }93}9495/// Applies one mutant, runs the checks, restores the file, and answers the verdict.96fn judge(compiler: Str, mutant: &Mutant) -> Verdict {97  let original = Result.unwrapOr(Io.read(mutant.file), "")98  let _mutated = Io.write(mutant.file, applied(original, mutant))99  let verdict = match Process.within(compiler, &["check", mutant.file], MUTANT_TIMEOUT_MS) {100    case Ok(Some(checked)) => {101      if checked.status != 0 || checked.output.contains("error[") {102        Invalid103      } else {104        match Option.map(suiteOf(mutant.file), |suite: Str| tested(compiler, suite)) {105          case Some(Killed) => Killed106          case _ => tested(compiler, "test")107        }108      }109    }110    case _ => Invalid111  }112  let _restored = Io.write(mutant.file, original)113  verdict114}115116/// The verdict of the checks under a test path: a failure or a run past the deadline kills.117fn tested(compiler: Str, path: Str) -> Verdict {118  match Process.within(compiler, &["test", path], MUTANT_TIMEOUT_MS) {119    case Ok(Some(finished)) => if finished.status == 0 { Survived } else { Killed }120    case Ok(None) => Killed121    case Err(_) => Invalid122  }123}124125/// The suite that owns a file, when one exists: `ApiExportTest` for `Api/Export`, else the126/// suite of its top folder, `ApiTest`. It runs first so most mutants die without the whole127/// test tree.128fn suiteOf(file: Str) -> Option[Str] {129  if !file.startsWith(SOURCE_ROOT) { return None }130  let parts = file.drop(SOURCE_ROOT.length()).replace(".pudu", "").split("/")131  let owners = if parts.length() > 1 { [parts[0] + parts[parts.length() - 1], parts[0]] } else { [parts[0]] }132  for owner in owners {133    let suite = TEST_ROOT + owner + "Test.pudu"134    if Result.isOk(&Io.read(suite)) { return Some(suite) }135  }136  None137}138139/// Every mutant of one file's text.140fn mutantsOf(file: Str, text: Str) -> Array[Mutant] {141  var found: Array[Mutant] = []142  var lineNumber = 0143  for line in text.split("\n") {144    lineNumber = lineNumber + 1145    let trimmed = line.trim()146    if !trimmed.startsWith("//") && !trimmed.startsWith("/*") && !trimmed.startsWith("import ") {147      let code = codeMask(line)148      for operator in OPERATORS {149        var at = 0150        while at < line.length() {151          if matchesAt(line, &code, at, operator[0]) {152            found = found.push(Mutant{file: file, line: lineNumber, column: at + 1, from: operator[0], to: operator[1]})153          }154          at = at + 1155        }156      }157    }158  }159  found160}161162/// Whether an operator occurs at a code position and is not part of a longer operator or word.163fn matchesAt(line: Str, code: &Array[Bool], at: Int, operator: Str) -> Bool {164  let end = at + operator.length()165  if end > line.length() || line.slice(at, end) != operator { return false }166  var position = at167  while position < end {168    if !code[position] { return false }169    position = position + 1170  }171  let before = if at > 0 { line.slice(at - 1, at) } else { "" }172  let after = if end < line.length() { line.slice(end, end + 1) } else { "" }173  if operator == "true" || operator == "false" { return !isWord(before) && !isWord(after) }174  if operator == "<" { return after != "=" && after != "-" && before != "<" && isSpaced(before, after) }175  if operator == ">" { return after != "=" && before != "-" && before != "=" && before != ">" && isSpaced(before, after) }176  if operator == "!" { return after != "=" && (isWord(after) || after == "(") }177  if operator == "- 1" || operator == "+ 1" { return !isWord(after) }178  true179}180181/// Whether a comparison has a space on both sides, which a type's brackets never do.182fn isSpaced(before: Str, after: Str) -> Bool { before == " " && after == " " }183184/// For each character of a line, whether it is code rather than a string literal or comment.185fn codeMask(line: Str) -> Array[Bool] {186  var mask: Array[Bool] = []187  var inString = false188  var escaped = false189  var commented = false190  var index = 0191  let characters = line.chars()192  for character in characters {193    if commented {194      mask = mask.push(false)195    } else if inString {196      mask = mask.push(false)197      if escaped { escaped = false } else if character == '\\' { escaped = true } else if character == '"' { inString = false }198    } else if character == '"' {199      inString = true200      mask = mask.push(false)201    } else if character == '/' && index + 1 < characters.length() && characters[index + 1] == '/' {202      commented = true203      mask = mask.push(false)204    } else if character == '\'' {205      mask = mask.push(false)206    } else {207      mask = mask.push(true)208    }209    index = index + 1210  }211  mask212}213214/// The text with one mutant applied.215fn applied(text: Str, mutant: &Mutant) -> Str {216  let lines = text.split("\n")217  let target = lines[mutant.line - 1]218  let start = mutant.column - 1219  let changed = target.slice(0, start) + mutant.to + target.drop(start + mutant.from.length())220  lines.slice(0, mutant.line - 1).concat([changed]).concat(lines.slice(mutant.line, lines.length())).join("\n")221}222223/// A mutant as `file:line:column  from -> to`.224fn describe(mutant: &Mutant) -> Str {225  mutant.file + ":" + show(mutant.line) + ":" + show(mutant.column) + "  " + mutant.from + " -> " + (if mutant.to.isEmpty() { "(removed)" } else { mutant.to })226}227228/// Whether text is one identifier character.229fn isWord(text: Str) -> Bool {230  !text.isEmpty() && Text.allChars(text, |c: Char| Char.isAlphanumeric(c) || c == '_')231}232233/// Every source file under a directory, sorted.234fn sources(directory: Str) -> Array[Str] {235  var found: Array[Str] = []236  for path in List.sortBy(&Result.unwrapOr(Io.listPaths(directory), []), fn(left: Str, right: Str) -> Bool { left < right }) {237    if path.endsWith(".pudu") {238      found = found.push(path)239    } else if !path.contains(".") {240      found = found.concat(sources(path))241    }242  }243  found244}245