diff --git a/simulation/amaru-sim/src/simulator/generate.rs b/simulation/amaru-sim/src/simulator/generate.rs index 32dabe769..869ab0e8e 100644 --- a/simulation/amaru-sim/src/simulator/generate.rs +++ b/simulation/amaru-sim/src/simulator/generate.rs @@ -377,6 +377,19 @@ pub fn generate_u8(low: u8, high: u8) -> impl Fn(&mut StdRng) -> u8 { generate_u8_then(low, high, |x| x) } +pub fn generate_zip_with( + generator1: impl Fn(&mut StdRng) -> Vec, + generator2: impl Fn(&mut StdRng) -> Vec, + f: impl Fn(A, B) -> C, +) -> impl Fn(&mut StdRng) -> Vec { + move |rng| { + let xs = generator1(rng); + let ys = generator2(rng); + assert_eq!(xs.len(), ys.len()); + xs.into_iter().zip(ys).map(|(x, y)| f(x, y)).collect() + } +} + #[cfg(test)] mod test { use rand::rngs::StdRng; diff --git a/simulation/amaru-sim/src/simulator/mod.rs b/simulation/amaru-sim/src/simulator/mod.rs index 00364203e..5c954bf9c 100644 --- a/simulation/amaru-sim/src/simulator/mod.rs +++ b/simulation/amaru-sim/src/simulator/mod.rs @@ -53,6 +53,7 @@ pub use sync::*; mod bytes; pub mod generate; mod ledger; +pub mod shrink; pub mod simulate; mod sync; diff --git a/simulation/amaru-sim/src/simulator/shrink.rs b/simulation/amaru-sim/src/simulator/shrink.rs new file mode 100644 index 000000000..bb6ac6584 --- /dev/null +++ b/simulation/amaru-sim/src/simulator/shrink.rs @@ -0,0 +1,161 @@ +// Copyright 2025 PRAGMA +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fmt::Debug; + +// Andreas Zeller's delta debugging (`ddmin`) algorithm from the paper +// "Simplifying and Isolating Failure-Inducing Input" (2002). +// +// Basically tries to bisect the input (git's bisect algorithm uses the same technique). Will first +// try throwing away half of the input, but if that fails it will throw away smaller and smaller +// parts until it finds the smallest counter example. +pub fn shrink( + test: impl Fn(&[A]) -> B, + mut input: Vec, + error_predicate: impl Fn(&B) -> bool, +) -> (Vec, B, u32) { + let mut number_of_shrinks = 0; + let mut last_error: B; + let result = test(&input); + if error_predicate(&result) { + last_error = result; + } else { + panic!( + "shrink, error predicate doesn't hold for initial input: '{:?}'", + input + ) + } + let mut n = 2; + while input.len() >= 2 { + let mut start = 0; + let subset_length = input.len() / n; + let mut some_complement_is_failing = false; + while start < input.len() { + let mut complement: Vec = Vec::new(); + complement.extend_from_slice(&input[..start]); + if start + subset_length < input.len() { + complement.extend_from_slice(&input[start + subset_length..]); + } + // NOTE: that if we get a different error than the expected one, we treat it as a + // passing test. + let result = test(&complement); + if error_predicate(&result) { + number_of_shrinks += 1; + last_error = result; + input = complement; + n = n.max(2) - 1; + some_complement_is_failing = true; + break; + } + + start += subset_length; + } + + if !some_complement_is_failing { + if n == input.len() { + break; + } + n = (n * 2).min(input.len()) + } + } + (input, last_error, number_of_shrinks) +} + +#[cfg(test)] +mod test { + + use super::*; + + #[test] + fn test_shrink_failing() { + let failing_input = vec![1, 2, 3, 42, 5, 6]; + + let test = |input: &[u8]| { + // println!("input: {:?}", input); + // input: [1, 2, 3, 42, 5, 6] + // input: [42, 5, 6] + // input: [5, 6] + // input: [42, 6] + // input: [6] + // input: [42] + + if input.contains(&42) { + Err("Found 42".to_string()) + } else { + Ok(()) + } + }; + + assert_eq!( + shrink(test, failing_input, |err| *err + == Err("Found 42".to_string())), + (vec![42], Err("Found 42".to_string()), 3) + ); + } + + #[test] + fn test_shrink_unresolved() { + let failing_input = vec![1, 2, 3, 42, 5, 6]; + + let test = |input: &[u8]| { + // println!("input: {:?}", input); + // input: [1, 2, 3, 42, 5, 6] + // input: [42, 5, 6] <-- NOTE: This will return a different error message than the one + // we expect, which ddmin treats as a passing test. + // input: [1, 2, 3] + // input: [2, 3, 42, 5, 6] + // input: [3, 42, 5, 6] + // input: [5, 6] + // input: [3, 42] + // input: [42] + + if input.len() == 3 && input.contains(&5) { + assert_eq!(input, vec![42, 5, 6]); + return Err("Found 5".to_string()); + }; + if input.contains(&42) { + Err("Found 42".to_string()) + } else { + Ok(()) + } + }; + + assert_eq!( + shrink(test, failing_input, |err| *err + == Err("Found 42".to_string())), + (vec![42], Err("Found 42".to_string()), 4) + ) + } + + #[test] + #[should_panic( + expected = "shrink, error predicate doesn't hold for initial input: '[1, 2, 3]'" + )] + fn test_shrink_passing() { + let failing_input = vec![1, 2, 3]; + + let test = |input: &[u8]| { + if input.contains(&4) { + Err("Found 4".to_string()) + } else { + Ok(()) + } + }; + assert_eq!( + shrink(test, failing_input, |err| *err + == Err("Found 4".to_string())), + (vec![4], Err("Found 4".to_string()), 0) + ) + } +} diff --git a/simulation/amaru-sim/src/simulator/simulate.rs b/simulation/amaru-sim/src/simulator/simulate.rs index ad86af4d3..e0e2d159f 100644 --- a/simulation/amaru-sim/src/simulator/simulate.rs +++ b/simulation/amaru-sim/src/simulator/simulate.rs @@ -25,6 +25,7 @@ // Make assertions on the history to ensure the execution was correct, if not, shrink and present minimal history that breaks the assertion together with the seed that allows us to reproduce the execution. use crate::echo::{EchoMessage, Envelope}; +use crate::simulator::shrink::shrink; use anyhow::anyhow; use parking_lot::Mutex; use pure_stage::trace_buffer::TraceBuffer; @@ -249,6 +250,29 @@ impl Drop for World { } } +fn run_test NodeHandle>( + number_of_nodes: u8, + spawn: F, + property: impl Fn(History) -> Result<(), String>, +) -> impl Fn(&[Reverse>]) -> (History, Result<(), String>) { + move |entries| { + let node_handles: Vec<_> = (1..=number_of_nodes) + .map(|i| (format!("n{}", i), spawn())) + .collect(); + + let mut world = World::new(entries.to_vec(), node_handles); + + match world.run_world() { + Ok(history) => { + let history = History(history.to_vec()); + let result = property(history.clone()); + (history, result) + } + Err((reason, history)) => (History(history.to_vec()), Err(reason)), + } + } +} + pub fn simulate( config: SimulateConfig, spawn: F, @@ -262,41 +286,29 @@ pub fn simulate( { let mut rng = StdRng::seed_from_u64(config.seed); - for test_number in 0..config.number_of_tests { + for test_number in 1..=config.number_of_tests { let entries: Vec>> = generator(&mut rng); - let node_handles: Vec<_> = (1..=config.number_of_nodes) - .map(|i| (format!("n{}", i), spawn())) - .collect(); - - let mut world = World::new(entries.clone(), node_handles); - - match world.run_world() { - Err((reason, history)) => { + match run_test(config.number_of_nodes, &spawn, &property)(&entries) { + (_history, Err(reason)) => { + let (shrunk_entries, (shrunk_history, result), number_of_shrinks) = shrink( + run_test(config.number_of_nodes, &spawn, &property), + entries, + |result| result.1 == Err(reason.clone()), + ); + assert_eq!(Err(reason.clone()), result); display_failure( test_number, config.seed, - entries, - History(history.to_vec()), + shrunk_entries, + number_of_shrinks, + shrunk_history, trace_buffer.clone(), reason, ); break; } - Ok(history) => match property(History(history.to_vec())) { - Ok(()) => continue, - Err(reason) => { - display_failure( - test_number, - config.seed, - entries, - History(history.to_vec()), - trace_buffer.clone(), - reason, - ); - break; - } - }, + (_history, Ok(())) => continue, } } if persist_on_success { @@ -309,6 +321,7 @@ fn display_failure( test_number: u32, seed: u64, entries: Vec>>, + number_of_shrinks: u32, history: History, trace_buffer: Arc>, reason: String, @@ -331,8 +344,8 @@ fn display_failure( let panic_message = |mschedule_path| { format!( - "Failed after {test_number} tests\n\n \ - Found minimal failing case:\n\n{}\n \ + "\nFailed after {test_number} tests\n\n \ + Minimised input ({number_of_shrinks} shrinks):\n\n{}\n \ History:\n\n{}\n \ Error message:\n\n {}\n\n \ {} \ @@ -395,7 +408,9 @@ fn persist_schedule( mod tests { use std::fs; - use crate::simulator::generate::{generate_u8_then, generate_vec}; + use crate::simulator::generate::{ + generate_arrival_times, generate_u8, generate_u8_then, generate_vec, generate_zip_with, + }; use super::*; use pure_stage::{simulation::SimulationBuilder, StageGraph, Void}; @@ -441,7 +456,7 @@ mod tests { echo: echo_response, }, }; - println!(" ==> {:?}", reply); + // println!(" ==> {:?}", reply); eff.send(&state.1, reply).await; Ok(state) } else { @@ -457,21 +472,23 @@ mod tests { pure_stage_node_handle(rx, stage.without_state(), running).unwrap() }; let now = Instant::at_offset(Duration::from_secs(0)); - let generate_messages = generate_vec( - 10, - generate_u8_then(0, 128, |i| { + let size = 20; + let generator = generate_zip_with( + generate_vec(size, generate_u8(0, 128)), + |rng| generate_arrival_times(rng, now, 200.0, size), + |msg, arrival_time| { Reverse(Entry { - arrival_time: now, + arrival_time, envelope: Envelope { src: "c1".to_string(), dest: "n1".to_string(), body: EchoMessage::Echo { msg_id: 0, - echo: format!("Please echo {}", i), + echo: format!("Please echo {}", msg), }, }, }) - }), + }, ); simulate( @@ -481,7 +498,7 @@ mod tests { number_of_nodes, }, spawn, - generate_messages, + generator, ECHO_PROPERTY, TraceBuffer::new_shared(0, 0), false, @@ -505,15 +522,10 @@ mod tests { if in_reply_to == msg_id && resp_echo == echo) }); if response.is_none() { - let mut err = String::new(); - err += &format!( - "No matching response found for echo request:\n {:?}\n\nHistory:\n", + return Err(format!( + "No matching response found for echo request: {:?}", msg - ); - for envelope in history.0 { - err += &format!(" {envelope:?}\n"); - } - return Err(err); + )); } } }