vello_cpu/src/fine/mod.rs

// Copyright 2025 the Vello Authors
// SPDX-License-Identifier: Apache-2.0 OR MIT

//! Fine rasterization stage of the rendering pipeline.
//!
//! This module implements the fine rasterization phase, which processes tiles at the pixel level.
//! It supports both high-precision (f32) and low-precision (u8) rendering paths, along with
//! various paint types including solid colors, gradients, images, and blurred rounded rectangles.

mod common;
mod highp;
mod lowp;

use crate::coarse::depth::DepthBuffer;
use crate::coarse::{CommandBucketer, RenderCmd, RowState};
use crate::filter::context::ScratchBuffer;
use crate::fine::common::gradient::GradientPainter;
pub(crate) use crate::fine::common::gradient::calculate_t_vals;
pub(crate) use crate::fine::common::gradient::linear::SimdLinearKind;
pub(crate) use crate::fine::common::gradient::radial::SimdRadialKind;
pub(crate) use crate::fine::common::gradient::sweep::SimdSweepKind;
use crate::fine::common::image::{FilteredImagePainter, NNImagePainter, PlainNNImagePainter};
use crate::fine::common::rounded_blurred_rect::BlurredRoundedRectFiller;
use crate::peniko::{BlendMode, ImageQuality};
use crate::region::Region;
use crate::util::EncodedImageExt;
use alloc::vec;
use alloc::vec::Vec;
use core::fmt::Debug;
use core::iter;
use vello_common::TargetInit;
use vello_common::color::AlphaColor;
use vello_common::encode::{
    EncodedBlurredRoundedRectangle, EncodedGradient, EncodedImage, EncodedKind, EncodedPaint,
};
use vello_common::fearless_simd::{
    Bytes, Simd, SimdBase, SimdFloat, SimdInt, SimdInto, f32x4, f32x8, f32x16, u8x16, u8x32, u32x4,
    u32x8,
};
use vello_common::filter_effects::Filter;
use vello_common::kurbo::Affine;
use vello_common::mask::Mask;
use vello_common::paint::{ImageResolver, ImageSource, Paint, PremulColor, Tint};
use vello_common::pixmap::Pixmap;
use vello_common::simd::Splat4thExt;
use vello_common::tile::Tile;
use vello_common::util::{VecPool, f32_to_u8};

#[doc(hidden)]
pub use crate::coarse::PaintFillAttrs;
#[doc(hidden)]
pub use crate::span::Span;
#[doc(hidden)]
pub use crate::span::TileAlignedSpan;
pub use highp::F32Kernel;
pub use lowp::U8Kernel;

/// Offset to shift from pixel corner to pixel center for sampling.
const PIXEL_CENTER_OFFSET: f64 = 0.5;

/// Number of color components per pixel (RGBA).
pub(crate) const COLOR_COMPONENTS: usize = 4;

/// Number of color components in a single column of a tile (height * components).
pub(crate) const TILE_HEIGHT_COMPONENTS: usize = Tile::HEIGHT as usize * COLOR_COMPONENTS;

/// Trait for numeric types used in fine rasterization.
///
/// This trait abstracts over `f32` and `u8` to allow the same rendering logic
/// to work with both high-precision (floating-point) and low-precision (integer)
/// representations. This enables performance optimizations while maintaining accuracy
/// where needed.
pub trait Numeric: Copy + Default + Clone + Debug + PartialEq + Send + Sync + 'static {
    /// The zero value for this numeric type (0.0 for f32, 0 for u8).
    const ZERO: Self;

    /// The maximum opacity value for this numeric type (1.0 for f32, 255 for u8).
    const ONE: Self;
}

impl Numeric for f32 {
    const ZERO: Self = 0.0;
    const ONE: Self = 1.0;
}

impl Numeric for u8 {
    const ZERO: Self = 0;
    const ONE: Self = 255;
}

/// Trait for SIMD vector types that can convert between f32 and u8 representations.
///
/// This trait enables efficient batch conversions between different numeric representations
/// during rendering operations, supporting both high-precision and low-precision rendering paths.
pub trait NumericVec<S: Simd>: Copy + Clone + Send + Sync {
    /// Convert from a SIMD vector of f32 values to this type.
    fn from_f32(simd: S, val: f32x16<S>) -> Self;

    /// Convert from a SIMD vector of u8 values to this type.
    fn from_u8(simd: S, val: u8x16<S>) -> Self;
}

impl<S: Simd> NumericVec<S> for f32x16<S> {
    #[inline(always)]
    fn from_f32(_: S, val: Self) -> Self {
        val
    }

    #[inline(always)]
    fn from_u8(simd: S, val: u8x16<S>) -> Self {
        let converted = u8_to_f32(val);
        converted * Self::splat(simd, 1.0 / 255.0)
    }
}

impl<S: Simd> NumericVec<S> for u8x16<S> {
    #[inline(always)]
    fn from_f32(simd: S, val: f32x16<S>) -> Self {
        let v1 = f32x16::splat(simd, 255.0);
        let v2 = f32x16::splat(simd, 0.5);
        let mulled = val.mul_add(v1, v2);

        f32_to_u8(mulled)
    }

    #[inline(always)]
    fn from_u8(_: S, val: Self) -> Self {
        val
    }
}

/// Convert a SIMD vector of u8 values to f32 values.
///
/// This function efficiently converts 16 u8 values to their f32 equivalents using SIMD operations,
/// preserving the values without normalization (i.e., 255 becomes 255.0, not 1.0).
#[inline(always)]
pub(crate) fn u8_to_f32<S: Simd>(val: u8x16<S>) -> f32x16<S> {
    let simd = val.simd;
    let zeroes = u8x16::splat(simd, 0);

    #[cfg(target_endian = "little")]
    let (p1, p2, p3, p4) = {
        let lo = simd.zip_low_u8x16(val, zeroes);
        let hi = simd.zip_high_u8x16(val, zeroes);

        (
            simd.zip_low_u8x16(lo, zeroes),
            simd.zip_high_u8x16(lo, zeroes),
            simd.zip_low_u8x16(hi, zeroes),
            simd.zip_high_u8x16(hi, zeroes),
        )
    };

    #[cfg(target_endian = "big")]
    let (p1, p2, p3, p4) = {
        let lo = simd.zip_low_u8x16(zeroes, val);
        let hi = simd.zip_high_u8x16(zeroes, val);

        (
            simd.zip_low_u8x16(zeroes, lo),
            simd.zip_high_u8x16(zeroes, lo),
            simd.zip_low_u8x16(zeroes, hi),
            simd.zip_high_u8x16(zeroes, hi),
        )
    };

    let p1 = p1.bitcast::<u32x4<S>>().to_float::<f32x4<S>>();
    let p2 = p2.bitcast::<u32x4<S>>().to_float::<f32x4<S>>();
    let p3 = p3.bitcast::<u32x4<S>>().to_float::<f32x4<S>>();
    let p4 = p4.bitcast::<u32x4<S>>().to_float::<f32x4<S>>();

    simd.combine_f32x8(simd.combine_f32x4(p1, p2), simd.combine_f32x4(p3, p4))
}

/// Trait for SIMD vector types used in compositing and blending operations.
///
/// This trait abstracts over different SIMD vector widths (f32x16 for high-precision,
/// u8x32 for low-precision) to enable efficient batch processing of pixel data during
/// blending and compositing.
pub trait CompositeType<N: Numeric, S: Simd>: Copy + Clone + Send + Sync {
    /// The number of numeric values this composite type can hold.
    const LENGTH: usize;

    /// Load values from a slice into this composite type.
    fn from_slice(simd: S, slice: &[N]) -> Self;

    /// Create a composite type by repeating a single RGBA color across all elements.
    fn from_color(simd: S, color: [N; 4]) -> Self;
}

impl<S: Simd> CompositeType<f32, S> for f32x16<S> {
    const LENGTH: usize = 16;

    #[inline(always)]
    fn from_slice(simd: S, slice: &[f32]) -> Self {
        <Self as SimdBase<_>>::from_slice(simd, slice)
    }

    #[inline(always)]
    fn from_color(simd: S, color: [f32; 4]) -> Self {
        Self::block_splat(f32x4::from_slice(simd, &color[..]))
    }
}

impl<S: Simd> CompositeType<u8, S> for u8x32<S> {
    const LENGTH: usize = 32;

    #[inline(always)]
    fn from_slice(simd: S, slice: &[u8]) -> Self {
        <Self as SimdBase<_>>::from_slice(simd, slice)
    }

    #[inline(always)]
    fn from_color(simd: S, color: [u8; 4]) -> Self {
        u32x8::block_splat(u32x4::splat(simd, u32::from_ne_bytes(color))).to_bytes()
    }
}

/// A kernel for performing fine rasterization.
///
/// This trait defines the interface for tile-level rendering operations, abstracting over
/// different numeric precisions (f32 vs u8). Implementations provide the low-level pixel
/// manipulation, blending, and painting operations needed to render tiles.
///
/// The two main implementations are:
/// - [`F32Kernel`]: High-precision rendering using 32-bit floating-point values
/// - [`U8Kernel`]: Low-precision rendering using 8-bit integer values
pub trait FineKernel<S: Simd>: Send + Sync + 'static {
    /// The basic underlying numerical type of the kernel (f32 or u8).
    type Numeric: Numeric;

    /// The SIMD composite type used for efficient batch blending and compositing operations.
    type Composite: CompositeType<Self::Numeric, S>;

    /// The SIMD vector type used for conversions between u8 and f32 representations.
    type NumericVec: NumericVec<S>;

    /// Extract and convert a premultiplied color to the kernel's numeric type.
    ///
    /// Converts RGBA components from the standard premultiplied color format to
    /// the kernel's internal representation (e.g., 0.0-1.0 for f32, 0-255 for u8).
    fn extract_color(color: PremulColor) -> [Self::Numeric; 4];

    /// Pack row scratch data into a row-major output buffer.
    fn pack(simd: S, scratch: &[Self::Numeric], width: usize, region: &mut Region<'_>);

    /// Unpack row-major input data into row scratch.
    fn unpack(simd: S, region: &mut Region<'_>, width: usize, scratch: &mut [Self::Numeric]);

    /// Apply a filter to a layer.
    ///
    /// This is used for applying filters to whole layers, which is necessary for
    /// spatial filters (like blur) that need to access neighboring pixels. The filter
    /// is applied in-place to the provided pixmap.
    ///
    /// The transform parameter is used to scale filter parameters based on the current
    /// transformation matrix (e.g., zoom level), ensuring filters look consistent
    /// regardless of scale.
    #[expect(
        private_interfaces,
        reason = "`FineKernel` is public but this specific method is not needed."
    )]
    fn filter_layer(
        pixmap: &mut Pixmap,
        filter: &Filter,
        filter_scratch: &mut ScratchBuffer,
        transform: Affine,
    );

    /// Fill the target buffer with a solid color.
    ///
    /// Efficiently replicates the given RGBA color across all pixels in the target buffer.
    fn copy_solid(simd: S, target: &mut [Self::Numeric], color: [Self::Numeric; 4]);
    /// Create a painter for rendering gradients.
    ///
    /// Returns a painter that can render linear, radial, or sweep gradients based on
    /// pre-computed t values (gradient interpolation parameters).
    fn gradient_painter<'a>(
        simd: S,
        gradient: &'a EncodedGradient,
        t_vals: &'a [f32],
    ) -> impl Painter + 'a {
        simd.vectorize(
            #[inline(always)]
            || GradientPainter::new(simd, gradient, t_vals),
        )
    }

    /// Create a painter for rendering gradients with undefined region support.
    ///
    /// Similar to `gradient_painter`, but with support for masking undefined locations
    /// (used for radial gradients that may have mathematically undefined regions).
    ///
    /// This is intentionally a duplicate of the default [`FineKernel::gradient_painter`]
    /// implementation--the `U8Kernel` overrides that method, but not this one.
    fn gradient_painter_with_undefined<'a>(
        simd: S,
        gradient: &'a EncodedGradient,
        t_vals: &'a [f32],
    ) -> impl Painter + 'a {
        simd.vectorize(
            #[inline(always)]
            || GradientPainter::new(simd, gradient, t_vals),
        )
    }
    /// Create a painter for rendering axis-aligned nearest-neighbor images.
    ///
    /// Optimized painter for images with `Low` quality and no skewing component in their
    /// transform. This is the fastest image rendering path.
    fn plain_nn_image_painter<'a>(
        simd: S,
        image: &'a EncodedImage,
        pixmap: &'a Pixmap,
        start_x: f64,
        start_y: f64,
    ) -> impl Painter + 'a {
        simd.vectorize(
            #[inline(always)]
            || PlainNNImagePainter::new(simd, image, pixmap, start_x, start_y),
        )
    }

    /// Create a painter for rendering nearest-neighbor images with transforms.
    ///
    /// Similar to `plain_nn_image_painter`, but supports arbitrary affine transforms
    /// including skewing and rotation.
    fn nn_image_painter<'a>(
        simd: S,
        image: &'a EncodedImage,
        pixmap: &'a Pixmap,
        start_x: f64,
        start_y: f64,
    ) -> impl Painter + 'a {
        simd.vectorize(
            #[inline(always)]
            || NNImagePainter::new(simd, image, pixmap, start_x, start_y),
        )
    }

    /// Create a painter for rendering images with `Medium` quality filtering.
    ///
    /// Uses bilinear filtering for smoother appearance than nearest-neighbor.
    fn medium_quality_image_painter<'a>(
        simd: S,
        image: &'a EncodedImage,
        pixmap: &'a Pixmap,
        start_x: f64,
        start_y: f64,
    ) -> impl Painter + 'a {
        simd.vectorize(
            #[inline(always)]
            || FilteredImagePainter::<S, 1>::new(simd, image, pixmap, start_x, start_y),
        )
    }

    /// Create a painter for rendering axis-aligned images with `Medium` quality filtering.
    ///
    /// Optimized painter for images with bilinear filtering and no skewing component.
    fn plain_medium_quality_image_painter<'a>(
        simd: S,
        image: &'a EncodedImage,
        pixmap: &'a Pixmap,
        start_x: f64,
        start_y: f64,
    ) -> impl Painter + 'a {
        simd.vectorize(
            #[inline(always)]
            || FilteredImagePainter::<S, 1>::new(simd, image, pixmap, start_x, start_y),
        )
    }

    /// Create a painter for rendering images with `High` quality filtering.
    ///
    /// Uses high-quality filtering for the best visual appearance.
    fn high_quality_image_painter<'a>(
        simd: S,
        image: &'a EncodedImage,
        pixmap: &'a Pixmap,
        start_x: f64,
        start_y: f64,
    ) -> impl Painter + 'a {
        simd.vectorize(
            #[inline(always)]
            || FilteredImagePainter::<S, 2>::new(simd, image, pixmap, start_x, start_y),
        )
    }

    /// Create a painter for rendering blurred rounded rectangles.
    ///
    /// Efficiently renders rounded rectangles with gaussian blur applied,
    /// computing the blur analytically rather than as a post-process.
    fn blurred_rounded_rectangle_painter(
        simd: S,
        rect: &EncodedBlurredRoundedRectangle,
        start_x: f64,
        start_y: f64,
    ) -> impl Painter {
        simd.vectorize(
            #[inline(always)]
            || BlurredRoundedRectFiller::new(simd, rect, start_x, start_y),
        )
    }
    /// Apply a mask to the destination buffer.
    ///
    /// Multiplies each pixel in the destination by the corresponding mask value,
    /// effectively masking out or reducing the opacity of pixels.
    fn apply_mask(simd: S, dest: &mut [Self::Numeric], src: impl Iterator<Item = Self::NumericVec>);

    /// Apply a painter to render content into the destination buffer.
    ///
    /// Invokes the painter to generate pixel values and writes them to the destination.
    fn apply_painter<'a>(simd: S, dest: &mut [Self::Numeric], painter: impl Painter + 'a);

    /// Apply an image tint to an already-painted buffer.
    ///
    /// This is called as a post-pass after `apply_painter`, only when a tint is
    /// present. Keeping tint application out of the per-pixel iterator avoids
    /// regressing the non-tinted fast path.
    fn apply_tint(simd: S, dest: &mut [Self::Numeric], tint: &Tint);

    /// Perform alpha compositing with a solid color over the target buffer.
    ///
    /// Blends a solid RGBA color over the existing contents using standard alpha compositing
    /// (Porter-Duff source-over). Optionally applies additional per-pixel alpha values.
    fn alpha_composite_solid(
        simd: S,
        target: &mut [Self::Numeric],
        src: [Self::Numeric; 4],
        alphas: Option<&[u8]>,
    );

    /// Perform alpha compositing with a source buffer over the destination buffer.
    ///
    /// Blends the source buffer contents over the destination using standard alpha compositing.
    /// Optionally applies additional per-pixel alpha values.
    fn alpha_composite_buffer(
        simd: S,
        dest: &mut [Self::Numeric],
        src: &[Self::Numeric],
        alphas: Option<&[u8]>,
    );

    /// Blend the source into the destination with a specified blend mode.
    ///
    /// Applies advanced blending operations (e.g., multiply, screen, overlay) as specified
    /// by the blend mode. Optionally applies additional per-pixel alpha values.
    fn blend(
        simd: S,
        dest: &mut [Self::Numeric],
        start_x: u16,
        start_y: u16,
        src: impl Iterator<Item = Self::Composite>,
        blend_mode: BlendMode,
        alphas: Option<&[u8]>,
        mask: Option<&Mask>,
    );

    /// Fill a row scratch span with a solid color, optionally modulated by per-pixel alphas.
    fn fill_solid(simd: S, dest: &mut [Self::Numeric], color: PremulColor, alphas: Option<&[u8]>) {
        let color = Self::extract_color(color);

        if color[3] == Self::Numeric::ONE && alphas.is_none() {
            Self::copy_solid(simd, dest, color);
        } else {
            Self::alpha_composite_solid(simd, dest, color, alphas);
        }
    }
}

pub(crate) fn rasterize_region<S: Simd, T: FineKernel<S>>(
    fine: &mut Fine<S, T>,
    depth: &mut DepthBuffer,
    region: &mut Region<'_>,
    bucketer: &CommandBucketer,
    resources: FineResources<'_>,
    mut target_init: TargetInit<PremulColor>,
    root_is_blend_target: bool,
) {
    let scene_y = region.row_idx as u16 * Tile::HEIGHT;
    let row = &bucketer.rows()[region.row_idx];
    let span = Span::new(0, region.width()).tile_aligned();

    fine.set_row_y(scene_y);
    depth.clear();

    let has_cmds = !row.depth_cmds.is_empty() || !row.render_cmds.is_empty();
    let isolate_bg = root_is_blend_target
        && has_cmds
        && match target_init {
            TargetInit::SrcOver => true,
            TargetInit::Clear(color) => !color.is_transparent(),
        };
    if isolate_bg {
        fine.init_uncovered_range(span, region, target_init, depth);
        fine.push_buf(None);

        target_init = TargetInit::Clear(PremulColor::from_alpha_color(AlphaColor::TRANSPARENT));
    }

    // Render depth-buffer commands front-to-back, with depth-buffer read and write.
    for &cmd in row.depth_cmds.iter().rev() {
        let attrs = &bucketer.paint_fill_attrs[cmd.attrs_idx as usize];

        depth.for_each_unset_run_and_write(cmd.bucket_range(), attrs.draw_id, |bucket_range| {
            let span = bucket_range.span();
            fine.paint_fill(span, attrs, resources, None);
        });
    }

    // Initialize any regions in the fine buffer that haven't been filled with an opaque fill.
    fine.init_uncovered_range(span, region, target_init, depth);

    // Render the main commands back-to-front, with depth-buffer read.
    for cmd in &row.render_cmds {
        fine.run_cmd(*cmd, bucketer, row, scene_y, resources, depth);
    }

    if isolate_bg {
        fine.layer_fill(scene_y, span, BlendMode::default(), 1.0, None, None);
        fine.pop_buf();
    }

    // Pack the composited result back into the pixmap.
    fine.pack(region);
}

/// Fine rasterizer for processing strip rows at the pixel level.
#[derive(Debug)]
#[doc(hidden)]
pub struct Fine<S: Simd, T: FineKernel<S>> {
    /// The SIMD context used for vectorized operations.
    simd: S,
    // TODO: If we make sure that strips never exceed the viewport, we can delete this.
    /// Pixel span covered by the blend buffers.
    buffer_span: TileAlignedSpan,
    /// Stack of blend buffers for managing layers and composition.
    ///
    /// Each layer pushes a new buffer onto this stack, and layers are composited
    /// by popping and blending with the buffer below.
    blend_buffers: Vec<Vec<T::Numeric>>,
    /// Pool for reusing layer buffer allocations.
    buffer_pool: VecPool<T::Numeric>,
    /// Intermediate buffer used by painters to store generated pixel data before compositing.
    paint_buf: Vec<T::Numeric>,
    /// Buffer for storing gradient interpolation parameters (t values).
    f32_buf: Vec<f32>,
    /// The current strip row y-coordinate in scene/filter coordinates.
    row_y: u16,
    /// The origin of the current target we are rendering into.
    origin: (u16, u16),
}

impl<S: Simd, T: FineKernel<S>> Fine<S, T> {
    /// Create a new fine rasterizer with the given SIMD context.
    ///
    /// Initializes all scratch buffers and sets up the initial blend buffer.
    #[doc(hidden)]
    pub fn new(simd: S, buffer_width: u16) -> Self {
        let scratch_len = usize::from(buffer_width) * TILE_HEIGHT_COMPONENTS;
        Self {
            simd,
            buffer_span: TileAlignedSpan::try_from(Span::new(0, buffer_width)).unwrap(),
            blend_buffers: vec![vec![T::Numeric::ZERO; scratch_len]],
            buffer_pool: VecPool::new(false),
            paint_buf: Vec::new(),
            f32_buf: Vec::new(),
            row_y: 0,
            origin: (0, 0),
        }
    }

    fn set_row_y(&mut self, row_y: u16) {
        self.row_y = row_y;
    }

    fn set_paint_offset(&mut self, paint_offset: (u16, u16)) {
        self.origin = paint_offset;
    }

    fn scratch_range(span: TileAlignedSpan) -> core::ops::Range<usize> {
        let start = usize::from(span.pixel_x()) * TILE_HEIGHT_COMPONENTS;
        let len = usize::from(span.pixel_width()) * TILE_HEIGHT_COMPONENTS;
        start..start + len
    }

    // The reason that we have this optimization is that, as was determined by profiling
    // just always clearing the whole fine buffer can be expensive, especially for larger
    // viewports. This is what a previous version of Vello CPU did.
    // With the current version, we can utilize the fact that we have a depth buffer where
    // a certain range of pixels is already filled with an opaque paint. Therefore, we only
    // need to clear (or unpack) the parts that are not covered by such a paint.
    /// Initialize every range in the buffer that has not been filled yet
    /// with a paint.
    fn init_uncovered_range(
        &mut self,
        scratch_span: TileAlignedSpan,
        region: &mut Region<'_>,
        target_init: TargetInit<PremulColor>,
        depth: &DepthBuffer,
    ) {
        match target_init {
            TargetInit::SrcOver => {
                depth.for_each_unset_run(
                    scratch_span,
                    #[inline]
                    |span| {
                        if let Some(span) = span
                            .as_span()
                            // The pixmap we load from does not necessarily have a width that is
                            // a multiple of the tile width! We therefore clamp and leave any
                            // tail in the blend buffer uninitialized.
                            .intersect(Span::new(0, region.width()))
                        {
                            let mut region = region.sub_span(span.pixel_x(), span.pixel_width());
                            self.unpack(span.pixel_x(), &mut region);
                        }
                    },
                );
            }
            // We can hint the compiler to lower this into a memory-zeroing operation,
            // hence why we handle this case explicitly.
            TargetInit::Clear(color) if color.is_transparent() => {
                depth.for_each_unset_run(
                    scratch_span,
                    #[inline]
                    |span| {
                        let range = Self::scratch_range(span);
                        self.blend_buffers.last_mut().unwrap()[range].fill(T::Numeric::ZERO);
                    },
                );
            }
            TargetInit::Clear(color) => {
                let color = T::extract_color(color);
                depth.for_each_unset_run(
                    scratch_span,
                    #[inline]
                    |span| {
                        let range = Self::scratch_range(span);
                        T::copy_solid(
                            self.simd,
                            &mut self.blend_buffers.last_mut().unwrap()[range],
                            color,
                        );
                    },
                );
            }
        }
    }

    fn push_buf(&mut self, span: Option<TileAlignedSpan>) {
        let mut buf = self.buffer_pool.take();
        // Reused vectors will retain their length, so in most cases this
        // will be a no-op.
        buf.resize(self.blend_buffers[0].len(), T::Numeric::ZERO);

        // Instead of always zeroing out the whole buffer, only zero the
        // row-local span that will be read when compositing this layer.
        if let Some(span) = span.and_then(|span| span.intersect(self.buffer_span)) {
            buf[Self::scratch_range(span)].fill(T::Numeric::ZERO);
        }
        self.blend_buffers.push(buf);
    }

    fn pop_buf(&mut self) {
        let popped = self.blend_buffers.pop().unwrap();
        self.buffer_pool.submit(popped);
    }

    /// Writes the current buffer contents to the output row.
    #[doc(hidden)]
    pub fn pack(&self, region: &mut Region<'_>) {
        let width = usize::from(region.width());
        let scratch = self.blend_buffers.last().unwrap();

        T::pack(self.simd, scratch, width, region);
    }

    /// Reads the pixels of the target back into the buffer.
    ///
    /// This does the opposite of [`Fine::pack`].
    #[doc(hidden)]
    pub fn unpack(&mut self, scratch_x_start: u16, region: &mut Region<'_>) {
        let scratch_x = usize::from(scratch_x_start);
        let width = usize::from(region.width());
        let scratch = self.blend_buffers.last_mut().unwrap();

        T::unpack(
            self.simd,
            region,
            width,
            &mut scratch[scratch_x * TILE_HEIGHT_COMPONENTS..],
        );
    }

    /// Execute a bucketed rendering command on the current strip row.
    ///
    /// This is the main dispatch method for fine rasterization. It processes paint fills,
    /// layer buffers, filter layer composites, masks, opacity, and layer blending.
    fn run_cmd(
        &mut self,
        cmd: RenderCmd,
        bucketer: &CommandBucketer,
        row: &RowState,
        row_y: u16,
        resources: FineResources<'_>,
        depth: &DepthBuffer,
    ) {
        match cmd {
            RenderCmd::PaintFill(cmd) => {
                let attrs = &bucketer.paint_fill_attrs[cmd.attrs_idx as usize];
                let alpha_buffer = resources.alpha_buffers[attrs.thread_idx as usize];

                let Some(span) = cmd.span.intersect(self.buffer_span) else {
                    return;
                };

                let paint_fill = |fine: &mut Self, span: TileAlignedSpan| {
                    let alphas = cmd.alpha_idx().map(|alpha_idx| {
                        let alpha_offset = alpha_idx as usize
                            + usize::from(span.pixel_x() - cmd.span.pixel_x())
                                * Tile::HEIGHT as usize;
                        &alpha_buffer[alpha_offset..]
                    });

                    fine.paint_fill(span, attrs, resources, alphas);
                };

                // Avoid using depth buffer if it's trivially skippable,
                // since it's generally cheaper to not use it at all than consult it just to be
                // returned the same span.
                if !row.can_skip_depth(span, attrs.draw_id) {
                    depth.for_each_visible_run(span, attrs.draw_id, |span| {
                        paint_fill(self, span);
                    });
                } else {
                    paint_fill(self, span);
                }
            }
            RenderCmd::PushBuf(span) => self.push_buf(span),
            RenderCmd::PopBuf => self.pop_buf(),
            RenderCmd::LayerFill(cmd) => {
                let Some(span) = cmd.span.intersect(self.buffer_span) else {
                    return;
                };

                let attrs = &bucketer.layer_fill_attrs[cmd.attrs_idx as usize];
                let alpha_buffer = resources.alpha_buffers[attrs.thread_idx as usize];

                let layer_fill = |fine: &mut Self, span: TileAlignedSpan| {
                    let alphas = cmd.alpha_idx().map(|alpha_idx| {
                        let alpha_offset = alpha_idx as usize
                            + usize::from(span.pixel_x() - cmd.span.pixel_x())
                                * Tile::HEIGHT as usize;
                        &alpha_buffer[alpha_offset..]
                    });

                    fine.layer_fill(
                        row_y,
                        span,
                        attrs.blend_mode,
                        attrs.opacity,
                        attrs.mask.as_ref(),
                        alphas,
                    );
                };

                // Same as for paint fills.
                if !row.can_skip_depth(span, attrs.draw_id) {
                    depth.for_each_visible_run(span, attrs.draw_id, |span| {
                        layer_fill(self, span);
                    });
                } else {
                    layer_fill(self, span);
                }
            }
        }
    }

    fn opacity(&mut self, span: TileAlignedSpan, opacity: f32) {
        let target = self.blend_buffers.last_mut().unwrap();
        let target = &mut target[Self::scratch_range(span)];

        T::apply_mask(
            self.simd,
            target,
            iter::repeat(T::NumericVec::from_f32(
                self.simd,
                f32x16::splat(self.simd, opacity),
            )),
        );
    }

    fn mask(&mut self, row_y: u16, span: TileAlignedSpan, mask: &Mask) {
        let x = span.pixel_x();
        let width = span.pixel_width();
        let target = self.blend_buffers.last_mut().unwrap();
        let target = &mut target[Self::scratch_range(span)];
        let y = u32::from(row_y) + u32x4::from_slice(self.simd, &[0, 1, 2, 3]);
        let iter = (x..x.saturating_add(width)).map(|x| {
            let x_in_range = x < mask.width();

            macro_rules! sample {
                ($idx:expr) => {
                    if x_in_range && (y[$idx] as u16) < mask.height() {
                        mask.sample(x, y[$idx] as u16)
                    } else {
                        0
                    }
                };
            }

            let s1 = sample!(0);
            let s2 = sample!(1);
            let s3 = sample!(2);
            let s4 = sample!(3);

            let samples = u8x16::from_slice(
                self.simd,
                &[
                    s1, s1, s1, s1, s2, s2, s2, s2, s3, s3, s3, s3, s4, s4, s4, s4,
                ],
            );
            T::NumericVec::from_u8(self.simd, samples)
        });

        T::apply_mask(self.simd, target, iter);
    }

    fn layer_fill(
        &mut self,
        row_y: u16,
        span: TileAlignedSpan,
        blend_mode: BlendMode,
        opacity: f32,
        mask: Option<&Mask>,
        alphas: Option<&[u8]>,
    ) {
        if opacity != 1.0 {
            self.opacity(span, opacity);
        }
        if let Some(mask) = mask {
            self.mask(row_y, span, mask);
        }

        let x = span.pixel_x();
        let (source, rest) = self.blend_buffers.split_last_mut().unwrap();
        let target = rest.last_mut().unwrap();
        let range = Self::scratch_range(span);
        let source = &mut source[range.clone()];
        let target = &mut target[range];

        if blend_mode == BlendMode::default() {
            T::alpha_composite_buffer(self.simd, target, source, alphas);
        } else {
            T::blend(
                self.simd,
                target,
                x,
                row_y,
                source
                    .chunks_exact(T::Composite::LENGTH)
                    .map(|s| T::Composite::from_slice(self.simd, s)),
                blend_mode,
                alphas,
                None,
            );
        }
    }

    #[doc(hidden)]
    pub fn paint_fill(
        &mut self,
        span: TileAlignedSpan,
        attrs: &PaintFillAttrs,
        resources: FineResources<'_>,
        alphas: Option<&[u8]>,
    ) {
        self.set_paint_offset(attrs.origin);
        match &attrs.paint {
            Paint::Solid(color) => {
                self.solid_fill(span, *color, attrs, alphas);
            }
            Paint::Indexed(index) => {
                self.indexed_fill(span, index.index(), attrs, resources, alphas);
            }
        }
    }

    fn solid_fill(
        &mut self,
        span: TileAlignedSpan,
        color: PremulColor,
        attrs: &PaintFillAttrs,
        alphas: Option<&[u8]>,
    ) {
        if attrs.blend_mode == BlendMode::default() && attrs.mask.is_none() {
            let scratch = self.blend_buffers.last_mut().unwrap();
            T::fill_solid(
                self.simd,
                &mut scratch[Self::scratch_range(span)],
                color,
                alphas,
            );
            return;
        }

        if span.pixel_width() == 0 {
            return;
        }

        let x = span.pixel_x();
        let color = T::extract_color(color);
        let simd = self.simd;
        let color = T::Composite::from_color(simd, color);
        let scratch = self.blend_buffers.last_mut().unwrap();
        T::blend(
            simd,
            &mut scratch[Self::scratch_range(span)],
            x,
            self.row_y,
            iter::repeat(color),
            attrs.blend_mode,
            alphas,
            attrs.mask.as_ref(),
        );
    }

    fn indexed_fill(
        &mut self,
        span: TileAlignedSpan,
        paint_index: usize,
        attrs: &PaintFillAttrs,
        resources: FineResources<'_>,
        alphas: Option<&[u8]>,
    ) {
        let x = span.pixel_x();
        let y = self.row_y;
        let sample_x = x.saturating_add(self.origin.0);
        let sample_y = y.saturating_add(self.origin.1);
        let width = span.pixel_width();
        let len = usize::from(width) * TILE_HEIGHT_COMPONENTS;
        if self.paint_buf.len() < len {
            self.paint_buf.resize(len, T::Numeric::ZERO);
        }

        let t_len = usize::from(width) * Tile::HEIGHT as usize;
        if self.f32_buf.len() < t_len {
            self.f32_buf.resize(t_len, 0.0);
        }

        let simd = self.simd;
        let width = usize::from(width);
        let start = usize::from(x) * TILE_HEIGHT_COMPONENTS;
        let dest = &mut self.blend_buffers.last_mut().unwrap()[start..start + len];
        let color_buf = &mut self.paint_buf[..len];
        let encoded_paint = resources
            .encoded_paints
            .get(paint_index)
            .unwrap_or_else(|| {
                &resources.filter_paints[paint_index - resources.encoded_paints.len()]
            });

        let sampler_x = f64::from(sample_x) + PIXEL_CENTER_OFFSET;
        let sampler_y = f64::from(sample_y) + PIXEL_CENTER_OFFSET;
        let default_blend = attrs.blend_mode == BlendMode::default();

        // We need to have this as a macro because closures cannot take generic arguments, and
        // we would have to repeatedly provide all arguments if we made it a function.
        macro_rules! fill_complex_paint {
            ($may_have_transparency:expr, $filler:expr) => {
                fill_complex_paint!($may_have_transparency, $filler, None::<&Tint>)
            };
            ($may_have_transparency:expr, $filler:expr, $tint:expr) => {
                if $may_have_transparency
                    || alphas.is_some()
                    || !default_blend
                    || attrs.mask.is_some()
                {
                    T::apply_painter(simd, color_buf, $filler);
                    if let Some(tint) = $tint {
                        T::apply_tint(simd, color_buf, tint);
                    }

                    if default_blend && attrs.mask.is_none() {
                        T::alpha_composite_buffer(simd, dest, color_buf, alphas);
                    } else {
                        T::blend(
                            simd,
                            dest,
                            x,
                            y,
                            color_buf
                                .chunks_exact(T::Composite::LENGTH)
                                .map(|s| T::Composite::from_slice(simd, s)),
                            attrs.blend_mode,
                            alphas,
                            attrs.mask.as_ref(),
                        );
                    }
                } else {
                    // Similarly to solid colors we can just override the previous values
                    // if all colors in the gradient are fully opaque.
                    T::apply_painter(simd, dest, $filler);
                    if let Some(tint) = $tint {
                        T::apply_tint(simd, dest, tint);
                    }
                }
            };
        }

        match encoded_paint {
            EncodedPaint::BlurredRoundedRect(rect) => {
                fill_complex_paint!(
                    true,
                    T::blurred_rounded_rectangle_painter(simd, rect, sampler_x, sampler_y)
                );
            }
            EncodedPaint::Gradient(gradient) => {
                // Note that we are calculating the t values first, store them in a separate
                // buffer and then pass that buffer to the iterator instead of calculating
                // the t values on the fly in the iterator. The latter would be faster, but
                // it would probably increase code size a lot, because the functions for
                // position calculation need to be inlined for good performance.
                let t_vals = &mut self.f32_buf[..width * Tile::HEIGHT as usize];

                match &gradient.kind {
                    EncodedKind::Linear(kind) => {
                        calculate_t_vals(
                            simd,
                            SimdLinearKind::new(simd, *kind),
                            t_vals,
                            gradient,
                            sampler_x,
                            sampler_y,
                        );
                        fill_complex_paint!(
                            gradient.may_have_transparency,
                            T::gradient_painter(simd, gradient, t_vals)
                        );
                    }
                    EncodedKind::Sweep(kind) => {
                        calculate_t_vals(
                            simd,
                            SimdSweepKind::new(simd, kind),
                            t_vals,
                            gradient,
                            sampler_x,
                            sampler_y,
                        );
                        fill_complex_paint!(
                            gradient.may_have_transparency,
                            T::gradient_painter(simd, gradient, t_vals)
                        );
                    }
                    EncodedKind::Radial(kind) => {
                        calculate_t_vals(
                            simd,
                            SimdRadialKind::new(simd, kind),
                            t_vals,
                            gradient,
                            sampler_x,
                            sampler_y,
                        );

                        if kind.has_undefined() {
                            fill_complex_paint!(
                                gradient.may_have_transparency,
                                T::gradient_painter_with_undefined(simd, gradient, t_vals)
                            );
                        } else {
                            fill_complex_paint!(
                                gradient.may_have_transparency,
                                T::gradient_painter(simd, gradient, t_vals)
                            );
                        }
                    }
                }
            }
            EncodedPaint::Image(image) => {
                let pixmap = match &image.source {
                    ImageSource::Pixmap(pixmap) => pixmap.clone(),
                    ImageSource::OpaqueId { id, .. } => resources
                        .image_resolver
                        .resolve(*id)
                        .unwrap_or_else(|| panic!("Image {:?} not found in registry", id)),
                    ImageSource::ExternalTexture { .. } => {
                        unimplemented!("External textures are not supported by `vello_cpu`")
                    }
                };
                let tint = image.tint.as_ref();

                match (image.has_skew(), image.nearest_neighbor()) {
                    (false, false) => {
                        // Axis-aligned with filtering - use optimized plain painters
                        if image.sampler.quality == ImageQuality::Medium {
                            fill_complex_paint!(
                                image.may_have_transparency,
                                T::plain_medium_quality_image_painter(
                                    simd, image, &pixmap, sampler_x, sampler_y
                                ),
                                tint
                            );
                        } else {
                            fill_complex_paint!(
                                image.may_have_transparency,
                                T::high_quality_image_painter(
                                    simd, image, &pixmap, sampler_x, sampler_y
                                ),
                                tint
                            );
                        }
                    }
                    (true, false) => {
                        // Skewed with filtering - use generic filtered painters
                        if image.sampler.quality == ImageQuality::Medium {
                            fill_complex_paint!(
                                image.may_have_transparency,
                                T::medium_quality_image_painter(
                                    simd, image, &pixmap, sampler_x, sampler_y
                                ),
                                tint
                            );
                        } else {
                            fill_complex_paint!(
                                image.may_have_transparency,
                                T::high_quality_image_painter(
                                    simd, image, &pixmap, sampler_x, sampler_y
                                ),
                                tint
                            );
                        }
                    }
                    (false, true) => {
                        fill_complex_paint!(
                            image.may_have_transparency,
                            T::plain_nn_image_painter(simd, image, &pixmap, sampler_x, sampler_y),
                            tint
                        );
                    }
                    (true, true) => {
                        fill_complex_paint!(
                            image.may_have_transparency,
                            T::nn_image_painter(simd, image, &pixmap, sampler_x, sampler_y),
                            tint
                        );
                    }
                }
            }
        }
    }
}

#[derive(Clone, Copy)]
#[doc(hidden)]
pub struct FineResources<'a> {
    pub alpha_buffers: &'a [&'a [u8]],
    pub encoded_paints: &'a [EncodedPaint],
    pub filter_paints: &'a [EncodedPaint],
    pub image_resolver: &'a dyn ImageResolver,
}

impl Debug for FineResources<'_> {
    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
        f.debug_struct("FineResources")
            .field("alpha_buffers", &self.alpha_buffers.len())
            .field("encoded_paints", &self.encoded_paints.len())
            .field("filter_paints", &self.filter_paints.len())
            .finish_non_exhaustive()
    }
}

/// Placement and compositing settings for fine rasterization into a target pixmap.
#[derive(Clone, Copy)]
pub(crate) struct FineRenderParams {
    /// Scene/filter dimensions before clipping to the destination pixmap.
    pub(crate) scene_size: (u16, u16),
    /// Destination offset in the target pixmap.
    pub(crate) target_offset: (u16, u16),
}

/// A trait for objects that can render pixel data into buffers.
///
/// Painters abstract over different content sources (gradients, images, etc.) and can
/// generate pixel data in either u8 or f32 format. Implementations should provide at least
/// one of these methods; the other can delegate through conversion.
///
/// Note: Some painters may only efficiently support one numeric type. The implementation
/// may convert between types as needed.
pub trait Painter: Sized {
    /// Paint pixel data into a u8 buffer (values in 0-255 range).
    fn paint_u8(self, buf: &mut [u8]);

    /// Paint pixel data into an f32 buffer (values in 0.0-1.0 range).
    fn paint_f32(self, buf: &mut [f32]);
}

/// Extension trait for creating position vectors for gradient and image sampling.
///
/// This trait provides a method to generate SIMD vectors of positions that advance
/// correctly across a tile. It's used by painters to compute per-pixel coordinates
/// for sampling operations.
pub trait PosExt<S: Simd> {
    /// Create a position vector that advances appropriately across a tile.
    ///
    /// Given a starting position and per-pixel advances in x and y directions,
    /// generates a SIMD vector with the correct position for each element.
    fn splat_pos(simd: S, pos: f32, x_advance: f32, y_advance: f32) -> Self;
}

impl<S: Simd> PosExt<S> for f32x4<S> {
    #[inline(always)]
    fn splat_pos(simd: S, pos: f32, _: f32, y_advance: f32) -> Self {
        let columns: [f32; Tile::HEIGHT as usize] = [0.0, 1.0, 2.0, 3.0];
        let column_mask: Self = columns.simd_into(simd);

        column_mask.mul_add(Self::splat(simd, y_advance), Self::splat(simd, pos))
    }
}

impl<S: Simd> PosExt<S> for f32x8<S> {
    #[inline(always)]
    fn splat_pos(simd: S, pos: f32, x_advance: f32, y_advance: f32) -> Self {
        simd.combine_f32x4(
            f32x4::splat_pos(simd, pos, x_advance, y_advance),
            f32x4::splat_pos(simd, pos + x_advance, x_advance, y_advance),
        )
    }
}

/// Intermediate shader result with color channels stored separately for efficient processing.
///
/// This structure holds 8 pixels worth of data in planar format (separate R, G, B, A vectors).
/// The planar layout is more efficient for certain SIMD operations before final interleaving.
pub(crate) struct ShaderResultF32<S: Simd> {
    /// Red channel values for 8 pixels.
    pub(crate) r: f32x8<S>,
    /// Green channel values for 8 pixels.
    pub(crate) g: f32x8<S>,
    /// Blue channel values for 8 pixels.
    pub(crate) b: f32x8<S>,
    /// Alpha channel values for 8 pixels.
    pub(crate) a: f32x8<S>,
}

impl<S: Simd> ShaderResultF32<S> {
    /// Convert from planar format to interleaved RGBA format.
    ///
    /// Returns two sets of four f32x4 vectors containing 8 pixels (4 RGBA components each),
    #[inline(always)]
    pub(crate) fn get(&self) -> [[f32x4<S>; 4]; 2] {
        let (r_1, r_2) = self.r.simd.split_f32x8(self.r);
        let (g_1, g_2) = self.g.simd.split_f32x8(self.g);
        let (b_1, b_2) = self.b.simd.split_f32x8(self.b);
        let (a_1, a_2) = self.a.simd.split_f32x8(self.a);

        [[r_1, g_1, b_1, a_1], [r_2, g_2, b_2, a_2]]
    }
}

mod macros {
    /// Implements the `Painter` trait for an iterator that produces f32x16 SIMD vectors.
    ///
    /// This macro generates both `paint_u8` and `paint_f32` methods, converting between
    /// formats as needed. Used for painters that work natively with high-precision f32 data.
    macro_rules! f32x16_painter {
        ($($type_path:tt)+) => {
            impl<S: Simd> crate::fine::Painter for $($type_path)+ {
                fn paint_u8(mut self, buf: &mut [u8]) {
                    use vello_common::fearless_simd::*;
                    use crate::fine::NumericVec;

                    self.simd.vectorize(#[inline(always)] || {
                        for chunk in buf.chunks_exact_mut(16) {
                            let next = self.next().unwrap();
                            let converted = u8x16::<S>::from_f32(next.simd, next);
                            converted.store_slice(chunk);
                        }
                    })
                }

                fn paint_f32(mut self, buf: &mut [f32]) {
                    self.simd.vectorize(#[inline(always)] || {
                        for chunk in buf.chunks_exact_mut(16) {
                            let next = self.next().unwrap();
                            next.store_slice(chunk);
                        }
                    })
                }
            }
        };
    }

    /// Implements the `Painter` trait for an iterator that produces u8x16 SIMD vectors.
    ///
    /// This macro generates both `paint_u8` and `paint_f32` methods, converting between
    /// formats as needed. Used for painters that work natively with low-precision u8 data.
    macro_rules! u8x16_painter {
        ($($type_path:tt)+) => {
            impl<S: Simd> crate::fine::Painter for $($type_path)+ {
                fn paint_u8(mut self, buf: &mut [u8]) {
                    self.simd.vectorize(#[inline(always)] || {
                        for chunk in buf.chunks_exact_mut(16) {
                            let next = self.next().unwrap();
                            next.store_slice(chunk);
                        }
                    })
                }

                fn paint_f32(mut self, buf: &mut [f32]) {
                    use vello_common::fearless_simd::*;
                    use crate::fine::NumericVec;

                    self.simd.vectorize(#[inline(always)] || {
                        for chunk in buf.chunks_exact_mut(16) {
                            let next = self.next().unwrap();
                            let converted = f32x16::<S>::from_u8(next.simd, next);
                            converted.store_slice(chunk);
                        }
                    })
                }
            }
        };
    }

    pub(crate) use f32x16_painter;
    pub(crate) use u8x16_painter;
}