iris/tests/shrink.rs

//! A fuzzer that reduces its own counterexample.
//!
//! A seed is not a lead anybody can read: the tree is hundreds of widgets,
//! and reconstructing the part that matters by hand has failed every time it
//! has been tried. This grows the trees `iris::random` describes, takes them
//! apart, and prints the smallest one that still fails as something to write
//! a fast test from.
//!
//!     cargo test --release --test shrink -- --ignored --nocapture
//!
//! `SHRINK_SEEDS` how many trees to try, `SHRINK_DEPTH` how deep to grow
//! them, `SHRINK_CASE` which scenario or `all` for every one. `SHRINK_SEED`
//! takes a single seed, which is how a failure `generated` printed is handed
//! straight here: the two run the same cases over the same trees, so a seed
//! that fails there fails here and is reduced.
//!
//! It is a fuzzer: run it once the ordinary tests pass, and turn what it
//! finds into a test of its own rather than leaving a seed as the record.

mod rig;
#[path = "scenario/mod.rs"]
mod scenario;

use iris::random::{Edits, Plan, plan};
use rig::env;
use scenario::{ALL, Case, diverges, over_seeds};

/// Takes the first simplification that still fails, until none does. The
/// simplifications come biggest first, so this walks down rather than
/// nibbling: a six-hundred-widget tree reaches single figures in a few
/// hundred builds.
fn shrink(mut node: Plan, case: Case, seed: u64) -> Plan {
    loop {
        let Some(next) = node
            .smaller()
            .into_iter()
            .find(|small| diverges(small, case, seed).is_some())
        else {
            return node;
        };
        node = next;
    }
}

fn cases() -> Vec<Case> {
    match env("SHRINK_CASE", String::from("all")).as_str() {
        "all" => ALL.to_vec(),
        name => match Case::named(name) {
            Some(case) => vec![case],
            None => panic!(
                "unknown SHRINK_CASE {name:?}; one of all, {}",
                ALL.map(Case::name).join(", ")
            ),
        },
    }
}

#[test]
#[ignore = "a fuzzer; run it once the ordinary tests pass"]
fn no_grown_tree_lays_out_differently_warm_than_cold() {
    let depth: usize = env("SHRINK_DEPTH", 5);
    let cases = cases();
    let seeds: Vec<u64> = match std::env::var("SHRINK_SEED")
        .ok()
        .and_then(|v| v.parse().ok())
    {
        Some(seed) => vec![seed],
        None => (1..=env("SHRINK_SEEDS", 400_u64)).collect(),
    };
    let count = seeds.len();

    over_seeds(seeds, |seed| {
        let grown = plan(seed, depth, &Edits::default());
        for &case in &cases {
            if diverges(&grown, case, seed).is_none() {
                continue;
            }
            let small = shrink(grown.clone(), case, seed);
            // Described from the shrunk tree: the grown tree's chain names
            // widgets that are no longer there, and the ancestry of the
            // failure is what a test is written from.
            let how = diverges(&small, case, seed).unwrap_or_default();
            println!(
                "seed {seed} case {}: {how}\ngrown {} widgets, shrank to {}\n{small:#?}",
                case.name(),
                grown.size(),
                small.size()
            );
            panic!(
                "seed {seed} lays out differently warm than cold after {}",
                case.name()
            );
        }
    });

    let sizes: Vec<usize> = (1..=count as u64)
        .map(|seed| plan(seed, depth, &Edits::default()).size())
        .collect();
    println!(
        "{count} trees at depth {depth} agree over {} case(s): {} widgets total, largest {}",
        cases.len(),
        sizes.iter().sum::<usize>(),
        sizes.iter().max().copied().unwrap_or(0)
    );
}