tests: add JSON diff reports for sparse test snapshot failures
diff --git a/Cargo.lock b/Cargo.lock index c0671e2..f3e5bd3 100644 --- a/Cargo.lock +++ b/Cargo.lock
@@ -4090,6 +4090,8 @@ "image", "oxipng", "pollster", + "serde", + "serde_json", "skrifa 0.37.0", "smallvec", "vello_api",
diff --git a/Cargo.toml b/Cargo.toml index 3dad9c9..ac201d0 100644 --- a/Cargo.toml +++ b/Cargo.toml
@@ -154,3 +154,5 @@ proc-macro2 = "1.0.95" syn = { version = "2.0.101", features = ["full", "extra-traits"] } quote = "1.0.40" +serde = { version = "1.0", default-features = false } +serde_json = "1.0"
diff --git a/sparse_strips/vello_sparse_tests/Cargo.toml b/sparse_strips/vello_sparse_tests/Cargo.toml index aa6dbd5..aebfd59 100644 --- a/sparse_strips/vello_sparse_tests/Cargo.toml +++ b/sparse_strips/vello_sparse_tests/Cargo.toml
@@ -25,6 +25,8 @@ bytemuck = { workspace = true } oxipng = { workspace = true, features = ["freestanding", "parallel"] } image = { workspace = true, features = ["png"] } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } skrifa = { workspace = true } smallvec = { workspace = true }
diff --git a/sparse_strips/vello_sparse_tests/tests/util.rs b/sparse_strips/vello_sparse_tests/tests/util.rs index 33ec300..f3171e9 100644 --- a/sparse_strips/vello_sparse_tests/tests/util.rs +++ b/sparse_strips/vello_sparse_tests/tests/util.rs
@@ -21,6 +21,33 @@ #[cfg(not(target_arch = "wasm32"))] use std::path::PathBuf; +/// Aggregate diff report with statistics and individual pixel differences. +#[cfg(not(target_arch = "wasm32"))] +#[derive(Debug, serde::Serialize)] +pub(crate) struct DiffReport { + /// Total number of pixels that differ. + pub pixel_count: usize, + /// Maximum absolute difference per channel [R, G, B, A]. + pub max_difference: [i16; 4], + /// Individual pixel differences. + pub pixels: Vec<PixelDiff>, +} + +/// Represents a single pixel difference between reference and actual images. +#[derive(Debug, serde::Serialize)] +pub(crate) struct PixelDiff { + /// The x coordinate of the differing pixel. + pub x: u32, + /// The y coordinate of the differing pixel. + pub y: u32, + /// The RGBA values from the reference image. + pub reference: [u8; 4], + /// The RGBA values from the actual image. + pub actual: [u8; 4], + /// Per-channel difference (actual - reference) as signed values. + pub difference: [i16; 4], +} + #[cfg(not(target_arch = "wasm32"))] static REFS_PATH: std::sync::LazyLock<PathBuf> = std::sync::LazyLock::new(|| { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../vello_sparse_tests/snapshots") @@ -321,9 +348,9 @@ .into_rgba8(); let actual = load_from_memory(&encoded_image).unwrap().into_rgba8(); - let diff_image = get_diff(&ref_image, &actual, threshold, diff_pixels); + let diff_result = get_diff(&ref_image, &actual, threshold, diff_pixels); - if let Some(diff_image) = diff_image { + if let Some((diff_image, diff_data)) = diff_result { if should_replace() && is_reference { write_ref_image(); panic!("test was replaced"); @@ -338,6 +365,22 @@ .save_with_format(&diff_path, image::ImageFormat::Png) .unwrap(); + // Save diff data as JSON + let json_path = DIFFS_PATH.join(format!("{specific_name}.json")); + let max_difference: [i16; 4] = diff_data.iter().fold([0; 4], |mut max, p| { + for (m, d) in max.iter_mut().zip(&p.difference) { + *m = (*m).max(d.abs()); + } + max + }); + let report = DiffReport { + pixel_count: diff_data.len(), + max_difference, + pixels: diff_data, + }; + let json_data = serde_json::to_string_pretty(&report).unwrap(); + std::fs::write(&json_path, json_data).unwrap(); + panic!("test didnt match reference image"); } } @@ -365,7 +408,7 @@ let ref_image = load_from_memory(ref_data).unwrap().into_rgba8(); let diff_image = get_diff(&ref_image, &actual, threshold, diff_pixels); - if let Some(ref img) = diff_image { + if let Some((ref img, _)) = diff_image { append_diff_image_to_browser_document(specific_name, img); panic!("test didn't match reference image. Scroll to bottom of browser to view diff."); } @@ -445,11 +488,12 @@ actual_image: &RgbaImage, threshold: u8, diff_pixels: u16, -) -> Option<RgbaImage> { +) -> Option<(RgbaImage, Vec<PixelDiff>)> { let width = max(expected_image.width(), actual_image.width()); let height = max(expected_image.height(), actual_image.height()); let mut diff_image = RgbaImage::new(width * 3, height); + let mut diff_data = Vec::new(); let mut pixel_diff = 0; @@ -465,6 +509,18 @@ if is_pix_diff(expected, actual, threshold) { pixel_diff += 1; diff_image.put_pixel(x + width, y, Rgba([255, 0, 0, 255])); + diff_data.push(PixelDiff { + x, + y, + reference: expected.0, + actual: actual.0, + difference: [ + i16::from(actual.0[0]) - i16::from(expected.0[0]), + i16::from(actual.0[1]) - i16::from(expected.0[1]), + i16::from(actual.0[2]) - i16::from(expected.0[2]), + i16::from(actual.0[3]) - i16::from(expected.0[3]), + ], + }); } else { diff_image.put_pixel(x + width, y, Rgba([0, 0, 0, 255])); } @@ -473,23 +529,54 @@ pixel_diff += 1; diff_image.put_pixel(x + 2 * width, y, *actual); diff_image.put_pixel(x + width, y, Rgba([255, 0, 0, 255])); + diff_data.push(PixelDiff { + x, + y, + reference: [0, 0, 0, 0], + actual: actual.0, + difference: [ + i16::from(actual.0[0]), + i16::from(actual.0[1]), + i16::from(actual.0[2]), + i16::from(actual.0[3]), + ], + }); } (None, Some(expected)) => { pixel_diff += 1; diff_image.put_pixel(x, y, *expected); diff_image.put_pixel(x + width, y, Rgba([255, 0, 0, 255])); + diff_data.push(PixelDiff { + x, + y, + reference: expected.0, + actual: [0, 0, 0, 0], + difference: [ + -i16::from(expected.0[0]), + -i16::from(expected.0[1]), + -i16::from(expected.0[2]), + -i16::from(expected.0[3]), + ], + }); } _ => { pixel_diff += 1; diff_image.put_pixel(x, y, Rgba([255, 0, 0, 255])); diff_image.put_pixel(x + width, y, Rgba([255, 0, 0, 255])); + diff_data.push(PixelDiff { + x, + y, + reference: [0, 0, 0, 0], + actual: [0, 0, 0, 0], + difference: [0, 0, 0, 0], + }); } } } } if pixel_diff > diff_pixels { - Some(diff_image) + Some((diff_image, diff_data)) } else { None }