Added optimisation for processing U8x2 images by MulDiv with helps of NEON SIMD instructions.

CHANGELOG.md

@@ -1,3 +1,10 @@
+## [Unreleased] - ReleaseDate
+
+### Crate
+
+- Added optimisation for processing `U8x2` images by `MulDiv` with
+  helps of `NEON SIMD` instructions.
+
 ## [2.1.0] - 2022-11-11
 
 ### Crate

benches/bench_alpha.rs

@@ -87,8 +87,8 @@ fn divides_alpha(bench: &mut Bench, pixel_type: PixelType, cpu_extensions: CpuEx
 
 fn bench_alpha(bench: &mut Bench) {
     let pixel_types = [
-        PixelType::U8x4,
         PixelType::U8x2,
+        PixelType::U8x4,
         PixelType::U16x2,
         PixelType::U16x4,
     ];

src/alpha/u8x2/mod.rs

@@ -7,6 +7,8 @@ use super::AlphaMulDiv;
 #[cfg(target_arch = "x86_64")]
 mod avx2;
 mod native;
+#[cfg(target_arch = "aarch64")]
+mod neon;
 #[cfg(target_arch = "x86_64")]
 mod sse4;
 
@@ -21,6 +23,8 @@ impl AlphaMulDiv for U8x2 {
             CpuExtensions::Avx2 => unsafe { avx2::multiply_alpha(src_image, dst_image) },
             #[cfg(target_arch = "x86_64")]
             CpuExtensions::Sse4_1 => unsafe { sse4::multiply_alpha(src_image, dst_image) },
+            #[cfg(target_arch = "aarch64")]
+            CpuExtensions::Neon => unsafe { neon::multiply_alpha(src_image, dst_image) },
             _ => native::multiply_alpha(src_image, dst_image),
         }
     }
@@ -31,6 +35,8 @@ impl AlphaMulDiv for U8x2 {
             CpuExtensions::Avx2 => unsafe { avx2::multiply_alpha_inplace(image) },
             #[cfg(target_arch = "x86_64")]
             CpuExtensions::Sse4_1 => unsafe { sse4::multiply_alpha_inplace(image) },
+            #[cfg(target_arch = "aarch64")]
+            CpuExtensions::Neon => unsafe { neon::multiply_alpha_inplace(image) },
             _ => native::multiply_alpha_inplace(image),
         }
     }
@@ -45,6 +51,8 @@ impl AlphaMulDiv for U8x2 {
             CpuExtensions::Avx2 => unsafe { avx2::divide_alpha(src_image, dst_image) },
             #[cfg(target_arch = "x86_64")]
             CpuExtensions::Sse4_1 => unsafe { sse4::divide_alpha(src_image, dst_image) },
+            #[cfg(target_arch = "aarch64")]
+            CpuExtensions::Neon => unsafe { neon::divide_alpha(src_image, dst_image) },
             _ => native::divide_alpha(src_image, dst_image),
         }
     }
@@ -55,6 +63,8 @@ impl AlphaMulDiv for U8x2 {
             CpuExtensions::Avx2 => unsafe { avx2::divide_alpha_inplace(image) },
             #[cfg(target_arch = "x86_64")]
             CpuExtensions::Sse4_1 => unsafe { sse4::divide_alpha_inplace(image) },
+            #[cfg(target_arch = "aarch64")]
+            CpuExtensions::Neon => unsafe { neon::divide_alpha_inplace(image) },
             _ => native::divide_alpha_inplace(image),
         }
     }

src/alpha/u8x2/neon.rs

@@ -0,0 +1,240 @@
+use std::arch::aarch64::*;
+
+use crate::neon_utils;
+use crate::pixels::U8x2;
+use crate::{ImageView, ImageViewMut};
+
+use super::native;
+
+#[target_feature(enable = "neon")]
+pub(crate) unsafe fn multiply_alpha(
+    src_image: &ImageView<U8x2>,
+    dst_image: &mut ImageViewMut<U8x2>,
+) {
+    let src_rows = src_image.iter_rows(0);
+    let dst_rows = dst_image.iter_rows_mut();
+
+    for (src_row, dst_row) in src_rows.zip(dst_rows) {
+        multiply_alpha_row(src_row, dst_row);
+    }
+}
+
+pub(crate) unsafe fn multiply_alpha_inplace(image: &mut ImageViewMut<U8x2>) {
+    for dst_row in image.iter_rows_mut() {
+        let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
+        multiply_alpha_row(src_row, dst_row);
+    }
+}
+
+#[inline]
+#[target_feature(enable = "neon")]
+unsafe fn multiply_alpha_row(src_row: &[U8x2], dst_row: &mut [U8x2]) {
+    let zero_u8x16 = vdupq_n_u8(0);
+    let zero_u8x8 = vdup_n_u8(0);
+
+    let src_chunks = src_row.chunks_exact(64);
+    let src_remainder = src_chunks.remainder();
+    let mut dst_chunks = dst_row.chunks_exact_mut(64);
+    for (src, dst) in src_chunks.zip(&mut dst_chunks) {
+        let mut pixels = neon_utils::load_deintrel_u8x16x4(src, 0);
+
+        let alpha_u16 = uint16x8x2_t(
+            vreinterpretq_u16_u8(vzip1q_u8(pixels.1, zero_u8x16)),
+            vreinterpretq_u16_u8(vzip2q_u8(pixels.1, zero_u8x16)),
+        );
+        pixels.0 = neon_utils::mul_color_to_alpha_u8x16(pixels.0, alpha_u16, zero_u8x16);
+
+        let alpha_u16 = uint16x8x2_t(
+            vreinterpretq_u16_u8(vzip1q_u8(pixels.3, zero_u8x16)),
+            vreinterpretq_u16_u8(vzip2q_u8(pixels.3, zero_u8x16)),
+        );
+        pixels.2 = neon_utils::mul_color_to_alpha_u8x16(pixels.2, alpha_u16, zero_u8x16);
+
+        let dst_ptr = dst.as_mut_ptr() as *mut u8;
+        vst4q_u8(dst_ptr, pixels);
+    }
+
+    let src_chunks = src_remainder.chunks_exact(32);
+    let src_remainder = src_chunks.remainder();
+    let dst_reminder = dst_chunks.into_remainder();
+    let mut dst_chunks = dst_reminder.chunks_exact_mut(32);
+    for (src, dst) in src_chunks.zip(&mut dst_chunks) {
+        let mut pixels = neon_utils::load_deintrel_u8x16x2(src, 0);
+
+        let alpha_u16 = uint16x8x2_t(
+            vreinterpretq_u16_u8(vzip1q_u8(pixels.1, zero_u8x16)),
+            vreinterpretq_u16_u8(vzip2q_u8(pixels.1, zero_u8x16)),
+        );
+        pixels.0 = neon_utils::mul_color_to_alpha_u8x16(pixels.0, alpha_u16, zero_u8x16);
+
+        let dst_ptr = dst.as_mut_ptr() as *mut u8;
+        vst2q_u8(dst_ptr, pixels);
+    }
+
+    let src_chunks = src_remainder.chunks_exact(16);
+    let src_remainder = src_chunks.remainder();
+    let dst_reminder = dst_chunks.into_remainder();
+    let mut dst_chunks = dst_reminder.chunks_exact_mut(16);
+    for (src, dst) in src_chunks.zip(&mut dst_chunks) {
+        let mut pixels = neon_utils::load_deintrel_u8x8x4(src, 0);
+
+        let alpha_u16_lo = vreinterpret_u16_u8(vzip1_u8(pixels.1, zero_u8x8));
+        let alpha_u16_hi = vreinterpret_u16_u8(vzip2_u8(pixels.1, zero_u8x8));
+        let alpha_u16 = vcombine_u16(alpha_u16_lo, alpha_u16_hi);
+        pixels.0 = neon_utils::mul_color_to_alpha_u8x8(pixels.0, alpha_u16, zero_u8x8);
+
+        let alpha_u16_lo = vreinterpret_u16_u8(vzip1_u8(pixels.3, zero_u8x8));
+        let alpha_u16_hi = vreinterpret_u16_u8(vzip2_u8(pixels.3, zero_u8x8));
+        let alpha_u16 = vcombine_u16(alpha_u16_lo, alpha_u16_hi);
+        pixels.2 = neon_utils::mul_color_to_alpha_u8x8(pixels.2, alpha_u16, zero_u8x8);
+
+        let dst_ptr = dst.as_mut_ptr() as *mut u8;
+        vst4_u8(dst_ptr, pixels);
+    }
+
+    let src_chunks = src_remainder.chunks_exact(8);
+    let src_remainder = src_chunks.remainder();
+    let dst_reminder = dst_chunks.into_remainder();
+    let mut dst_chunks = dst_reminder.chunks_exact_mut(8);
+    for (src, dst) in src_chunks.zip(&mut dst_chunks) {
+        let mut pixels = neon_utils::load_deintrel_u8x8x2(src, 0);
+
+        let alpha_u16_lo = vreinterpret_u16_u8(vzip1_u8(pixels.1, zero_u8x8));
+        let alpha_u16_hi = vreinterpret_u16_u8(vzip2_u8(pixels.1, zero_u8x8));
+        let alpha_u16 = vcombine_u16(alpha_u16_lo, alpha_u16_hi);
+        pixels.0 = neon_utils::mul_color_to_alpha_u8x8(pixels.0, alpha_u16, zero_u8x8);
+
+        let dst_ptr = dst.as_mut_ptr() as *mut u8;
+        vst2_u8(dst_ptr, pixels);
+    }
+
+    if !src_remainder.is_empty() {
+        let dst_reminder = dst_chunks.into_remainder();
+        native::multiply_alpha_row(src_remainder, dst_reminder);
+    }
+}
+
+// Divide
+
+#[target_feature(enable = "neon")]
+pub(crate) unsafe fn divide_alpha(src_image: &ImageView<U8x2>, dst_image: &mut ImageViewMut<U8x2>) {
+    let src_rows = src_image.iter_rows(0);
+    let dst_rows = dst_image.iter_rows_mut();
+
+    for (src_row, dst_row) in src_rows.zip(dst_rows) {
+        divide_alpha_row(src_row, dst_row);
+    }
+}
+
+#[target_feature(enable = "neon")]
+pub(crate) unsafe fn divide_alpha_inplace(image: &mut ImageViewMut<U8x2>) {
+    for dst_row in image.iter_rows_mut() {
+        let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
+        divide_alpha_row(src_row, dst_row);
+    }
+}
+
+#[inline]
+#[target_feature(enable = "neon")]
+unsafe fn divide_alpha_row(src_row: &[U8x2], dst_row: &mut [U8x2]) {
+    let src_chunks = src_row.chunks_exact(16);
+    let src_remainder = src_chunks.remainder();
+    let mut dst_chunks = dst_row.chunks_exact_mut(16);
+    for (src, dst) in src_chunks.zip(&mut dst_chunks) {
+        divide_alpha_16_pixels(src, dst);
+    }
+
+    let src_chunks = src_remainder.chunks_exact(8);
+    let src_remainder = src_chunks.remainder();
+    let dst_reminder = dst_chunks.into_remainder();
+    let mut dst_chunks = dst_reminder.chunks_exact_mut(8);
+    for (src, dst) in src_chunks.zip(&mut dst_chunks) {
+        divide_alpha_8_pixels(src, dst);
+    }
+
+    if !src_remainder.is_empty() {
+        let dst_reminder = dst_chunks.into_remainder();
+        let mut src_pixels = [U8x2::new(0); 8];
+        src_pixels
+            .iter_mut()
+            .zip(src_remainder)
+            .for_each(|(d, s)| *d = *s);
+
+        let mut dst_pixels = [U8x2::new(0); 8];
+        divide_alpha_8_pixels(src_pixels.as_slice(), dst_pixels.as_mut_slice());
+
+        dst_pixels
+            .iter()
+            .zip(dst_reminder)
+            .for_each(|(s, d)| *d = *s);
+    }
+}
+
+#[inline]
+#[target_feature(enable = "neon")]
+unsafe fn divide_alpha_16_pixels(src: &[U8x2], dst: &mut [U8x2]) {
+    let zero = vdupq_n_u8(0);
+    let alpha_scale = vdupq_n_f32(255.0 * 256.0);
+    let mut pixels = neon_utils::load_deintrel_u8x16x2(src, 0);
+    let nonzero_alpha_mask = vmvnq_u8(vceqzq_u8(pixels.1));
+
+    let alpha_u16_lo = vzip1q_u8(pixels.1, zero);
+    let alpha_u16_hi = vzip2q_u8(pixels.1, zero);
+
+    let alpha_f32_0 = vcvtq_f32_u32(vreinterpretq_u32_u8(vzip1q_u8(alpha_u16_lo, zero)));
+    let recip_alpha_f32_0 = vdivq_f32(alpha_scale, alpha_f32_0);
+    let recip_alpha_u16_0 = vmovn_u32(vcvtaq_u32_f32(recip_alpha_f32_0));
+
+    let alpha_f32_1 = vcvtq_f32_u32(vreinterpretq_u32_u8(vzip2q_u8(alpha_u16_lo, zero)));
+    let recip_alpha_f32_1 = vdivq_f32(alpha_scale, alpha_f32_1);
+    let recip_alpha_u16_1 = vmovn_u32(vcvtaq_u32_f32(recip_alpha_f32_1));
+
+    let alpha_f32_2 = vcvtq_f32_u32(vreinterpretq_u32_u8(vzip1q_u8(alpha_u16_hi, zero)));
+    let recip_alpha_f32_2 = vdivq_f32(alpha_scale, alpha_f32_2);
+    let recip_alpha_u16_2 = vmovn_u32(vcvtaq_u32_f32(recip_alpha_f32_2));
+
+    let alpha_f32_3 = vcvtq_f32_u32(vreinterpretq_u32_u8(vzip2q_u8(alpha_u16_hi, zero)));
+    let recip_alpha_f32_3 = vdivq_f32(alpha_scale, alpha_f32_3);
+    let recip_alpha_u16_3 = vmovn_u32(vcvtaq_u32_f32(recip_alpha_f32_3));
+
+    let recip_alpha = uint16x8x2_t(
+        vcombine_u16(recip_alpha_u16_0, recip_alpha_u16_1),
+        vcombine_u16(recip_alpha_u16_2, recip_alpha_u16_3),
+    );
+
+    pixels.0 = neon_utils::mul_color_recip_alpha_u8x16(pixels.0, recip_alpha, zero);
+    pixels.0 = vandq_u8(pixels.0, nonzero_alpha_mask);
+
+    let dst_ptr = dst.as_mut_ptr() as *mut u8;
+    vst2q_u8(dst_ptr, pixels);
+}
+
+#[inline]
+#[target_feature(enable = "neon")]
+unsafe fn divide_alpha_8_pixels(src: &[U8x2], dst: &mut [U8x2]) {
+    let zero_u8x8 = vdup_n_u8(0);
+    let zero_u8x16 = vdupq_n_u8(0);
+    let alpha_scale = vdupq_n_f32(255.0 * 256.0);
+    let mut pixels = neon_utils::load_deintrel_u8x8x2(src, 0);
+    let nonzero_alpha_mask = vmvn_u8(vceqz_u8(pixels.1));
+
+    let alpha_u16_lo = vzip1_u8(pixels.1, zero_u8x8);
+    let alpha_u16_hi = vzip2_u8(pixels.1, zero_u8x8);
+    let alpha_u16 = vcombine_u8(alpha_u16_lo, alpha_u16_hi);
+
+    let alpha_f32_0 = vcvtq_f32_u32(vreinterpretq_u32_u8(vzip1q_u8(alpha_u16, zero_u8x16)));
+    let recip_alpha_f32_0 = vdivq_f32(alpha_scale, alpha_f32_0);
+    let recip_alpha_u16_0 = vmovn_u32(vcvtaq_u32_f32(recip_alpha_f32_0));
+
+    let alpha_f32_1 = vcvtq_f32_u32(vreinterpretq_u32_u8(vzip2q_u8(alpha_u16, zero_u8x16)));
+    let recip_alpha_f32_1 = vdivq_f32(alpha_scale, alpha_f32_1);
+    let recip_alpha_u16_1 = vmovn_u32(vcvtaq_u32_f32(recip_alpha_f32_1));
+
+    let recip_alpha = vcombine_u16(recip_alpha_u16_0, recip_alpha_u16_1);
+
+    pixels.0 = neon_utils::mul_color_recip_alpha_u8x8(pixels.0, recip_alpha, zero_u8x8);
+    pixels.0 = vand_u8(pixels.0, nonzero_alpha_mask);
+
+    let dst_ptr = dst.as_mut_ptr() as *mut u8;
+    vst2_u8(dst_ptr, pixels);
+}

src/alpha/u8x4/neon.rs

@@ -40,13 +40,13 @@ unsafe fn multiply_alpha_row(src_row: &[U8x4], dst_row: &mut [U8x4]) {
     for (src, dst) in src_chunks.zip(&mut dst_chunks) {
         let mut pixels = neon_utils::load_deintrel_u8x16x4(src, 0);
 
-        let alpha_u8 = pixels.3;
-        let alpha_u16_lo = vreinterpretq_u16_u8(vzip1q_u8(alpha_u8, zero_u8x16));
-        let alpha_u16_hi = vreinterpretq_u16_u8(vzip2q_u8(alpha_u8, zero_u8x16));
-
-        pixels.0 = mul_color_to_alpha_u8x16(pixels.0, alpha_u16_lo, alpha_u16_hi, zero_u8x16);
-        pixels.1 = mul_color_to_alpha_u8x16(pixels.1, alpha_u16_lo, alpha_u16_hi, zero_u8x16);
-        pixels.2 = mul_color_to_alpha_u8x16(pixels.2, alpha_u16_lo, alpha_u16_hi, zero_u8x16);
+        let alpha_u16 = uint16x8x2_t(
+            vreinterpretq_u16_u8(vzip1q_u8(pixels.3, zero_u8x16)),
+            vreinterpretq_u16_u8(vzip2q_u8(pixels.3, zero_u8x16)),
+        );
+        pixels.0 = neon_utils::mul_color_to_alpha_u8x16(pixels.0, alpha_u16, zero_u8x16);
+        pixels.1 = neon_utils::mul_color_to_alpha_u8x16(pixels.1, alpha_u16, zero_u8x16);
+        pixels.2 = neon_utils::mul_color_to_alpha_u8x16(pixels.2, alpha_u16, zero_u8x16);
 
         let dst_ptr = dst.as_mut_ptr() as *mut u8;
         vst4q_u8(dst_ptr, pixels);
@@ -65,9 +65,9 @@ unsafe fn multiply_alpha_row(src_row: &[U8x4], dst_row: &mut [U8x4]) {
         let alpha_u16_hi = vreinterpret_u16_u8(vzip2_u8(alpha_u8, zero_u8x8));
         let alpha_u16 = vcombine_u16(alpha_u16_lo, alpha_u16_hi);
 
-        pixels.0 = mul_color_to_alpha_u8x8(pixels.0, alpha_u16, zero_u8x8);
-        pixels.1 = mul_color_to_alpha_u8x8(pixels.1, alpha_u16, zero_u8x8);
-        pixels.2 = mul_color_to_alpha_u8x8(pixels.2, alpha_u16, zero_u8x8);
+        pixels.0 = neon_utils::mul_color_to_alpha_u8x8(pixels.0, alpha_u16, zero_u8x8);
+        pixels.1 = neon_utils::mul_color_to_alpha_u8x8(pixels.1, alpha_u16, zero_u8x8);
+        pixels.2 = neon_utils::mul_color_to_alpha_u8x8(pixels.2, alpha_u16, zero_u8x8);
 
         let dst_ptr = dst.as_mut_ptr() as *mut u8;
         vst4_u8(dst_ptr, pixels);
@@ -79,43 +79,6 @@ unsafe fn multiply_alpha_row(src_row: &[U8x4], dst_row: &mut [U8x4]) {
     }
 }
 
-#[inline]
-#[target_feature(enable = "neon")]
-unsafe fn mul_color_to_alpha_u8x16(
-    color: uint8x16_t,
-    alpha_u16_lo: uint16x8_t,
-    alpha_u16_hi: uint16x8_t,
-    zero: uint8x16_t,
-) -> uint8x16_t {
-    let color_u16_lo = vreinterpretq_u16_u8(vzip1q_u8(color, zero));
-    let mut tmp_res = vmulq_u16(color_u16_lo, alpha_u16_lo);
-    tmp_res = vaddq_u16(tmp_res, vrshrq_n_u16::<8>(tmp_res));
-    let res_u16_lo = vrshrq_n_u16::<8>(tmp_res);
-
-    let color_u16_hi = vreinterpretq_u16_u8(vzip2q_u8(color, zero));
-    let mut tmp_res = vmulq_u16(color_u16_hi, alpha_u16_hi);
-    tmp_res = vaddq_u16(tmp_res, vrshrq_n_u16::<8>(tmp_res));
-    let res_u16_hi = vrshrq_n_u16::<8>(tmp_res);
-
-    vcombine_u8(vqmovn_u16(res_u16_lo), vqmovn_u16(res_u16_hi))
-}
-
-#[inline]
-#[target_feature(enable = "neon")]
-unsafe fn mul_color_to_alpha_u8x8(
-    color: uint8x8_t,
-    alpha_u16: uint16x8_t,
-    zero: uint8x8_t,
-) -> uint8x8_t {
-    let color_u16_lo = vreinterpret_u16_u8(vzip1_u8(color, zero));
-    let color_u16_hi = vreinterpret_u16_u8(vzip2_u8(color, zero));
-    let color_u16 = vcombine_u16(color_u16_lo, color_u16_hi);
-    let mut tmp_res = vmulq_u16(color_u16, alpha_u16);
-    tmp_res = vaddq_u16(tmp_res, vrshrq_n_u16::<8>(tmp_res));
-    let res_u16 = vrshrq_n_u16::<8>(tmp_res);
-    vqmovn_u16(res_u16)
-}
-
 // Divide
 
 #[target_feature(enable = "neon")]
@@ -204,32 +167,17 @@ unsafe fn divide_alpha_16_pixels(src: &[U8x4], dst: &mut [U8x4]) {
         vcombine_u16(recip_alpha_u16_2, recip_alpha_u16_3),
     );
 
-    pixels.0 = mul_color_recip_alpha_u8x16(pixels.0, recip_alpha, zero);
+    pixels.0 = neon_utils::mul_color_recip_alpha_u8x16(pixels.0, recip_alpha, zero);
     pixels.0 = vandq_u8(pixels.0, nonzero_alpha_mask);
-    pixels.1 = mul_color_recip_alpha_u8x16(pixels.1, recip_alpha, zero);
+    pixels.1 = neon_utils::mul_color_recip_alpha_u8x16(pixels.1, recip_alpha, zero);
     pixels.1 = vandq_u8(pixels.1, nonzero_alpha_mask);
-    pixels.2 = mul_color_recip_alpha_u8x16(pixels.2, recip_alpha, zero);
+    pixels.2 = neon_utils::mul_color_recip_alpha_u8x16(pixels.2, recip_alpha, zero);
     pixels.2 = vandq_u8(pixels.2, nonzero_alpha_mask);
 
     let dst_ptr = dst.as_mut_ptr() as *mut u8;
     vst4q_u8(dst_ptr, pixels);
 }
 
-#[inline]
-#[target_feature(enable = "neon")]
-unsafe fn mul_color_recip_alpha_u8x16(
-    color: uint8x16_t,
-    recip_alpha: uint16x8x2_t,
-    zero: uint8x16_t,
-) -> uint8x16_t {
-    let color_u16_lo = vreinterpretq_u16_u8(vzip1q_u8(zero, color));
-    let color_u16_hi = vreinterpretq_u16_u8(vzip2q_u8(zero, color));
-
-    let res_u16_lo = neon_utils::mulhi_u16x8(color_u16_lo, recip_alpha.0);
-    let res_u16_hi = neon_utils::mulhi_u16x8(color_u16_hi, recip_alpha.1);
-    vcombine_u8(vmovn_u16(res_u16_lo), vmovn_u16(res_u16_hi))
-}
-
 #[inline]
 #[target_feature(enable = "neon")]
 unsafe fn divide_alpha_8_pixels(src: &[U8x4], dst: &mut [U8x4]) {
@@ -253,28 +201,13 @@ unsafe fn divide_alpha_8_pixels(src: &[U8x4], dst: &mut [U8x4]) {
 
     let recip_alpha = vcombine_u16(recip_alpha_u16_0, recip_alpha_u16_1);
 
-    pixels.0 = mul_color_recip_alpha_u8x8(pixels.0, recip_alpha, zero_u8x8);
+    pixels.0 = neon_utils::mul_color_recip_alpha_u8x8(pixels.0, recip_alpha, zero_u8x8);
     pixels.0 = vand_u8(pixels.0, nonzero_alpha_mask);
-    pixels.1 = mul_color_recip_alpha_u8x8(pixels.1, recip_alpha, zero_u8x8);
+    pixels.1 = neon_utils::mul_color_recip_alpha_u8x8(pixels.1, recip_alpha, zero_u8x8);
     pixels.1 = vand_u8(pixels.1, nonzero_alpha_mask);
-    pixels.2 = mul_color_recip_alpha_u8x8(pixels.2, recip_alpha, zero_u8x8);
+    pixels.2 = neon_utils::mul_color_recip_alpha_u8x8(pixels.2, recip_alpha, zero_u8x8);
     pixels.2 = vand_u8(pixels.2, nonzero_alpha_mask);
 
     let dst_ptr = dst.as_mut_ptr() as *mut u8;
     vst4_u8(dst_ptr, pixels);
 }
-
-#[inline]
-#[target_feature(enable = "neon")]
-unsafe fn mul_color_recip_alpha_u8x8(
-    color: uint8x8_t,
-    recip_alpha: uint16x8_t,
-    zero: uint8x8_t,
-) -> uint8x8_t {
-    let color_u16_lo = vreinterpret_u16_u8(vzip1_u8(zero, color));
-    let color_u16_hi = vreinterpret_u16_u8(vzip2_u8(zero, color));
-    let color_u16 = vcombine_u16(color_u16_lo, color_u16_hi);
-
-    let res_u16 = neon_utils::mulhi_u16x8(color_u16, recip_alpha);
-    vmovn_u16(res_u16)
-}

src/neon_utils.rs

@@ -11,6 +11,11 @@ pub unsafe fn load_u8x8<T>(buf: &[T], index: usize) -> uint8x8_t {
     vld1_u8(buf.get_unchecked(index..).as_ptr() as *const u8)
 }
 
+#[inline(always)]
+pub unsafe fn load_deintrel_u8x8x2<T>(buf: &[T], index: usize) -> uint8x8x2_t {
+    vld2_u8(buf.get_unchecked(index..).as_ptr() as *const u8)
+}
+
 #[inline(always)]
 pub unsafe fn load_deintrel_u8x8x4<T>(buf: &[T], index: usize) -> uint8x8x4_t {
     vld4_u8(buf.get_unchecked(index..).as_ptr() as *const u8)
@@ -31,6 +36,11 @@ pub unsafe fn load_u8x16x4<T>(buf: &[T], index: usize) -> uint8x16x4_t {
     vld1q_u8_x4(buf.get_unchecked(index..).as_ptr() as *const u8)
 }
 
+#[inline(always)]
+pub unsafe fn load_deintrel_u8x16x2<T>(buf: &[T], index: usize) -> uint8x16x2_t {
+    vld2q_u8(buf.get_unchecked(index..).as_ptr() as *const u8)
+}
+
 #[inline(always)]
 pub unsafe fn load_deintrel_u8x16x4<T>(buf: &[T], index: usize) -> uint8x16x4_t {
     vld4q_u8(buf.get_unchecked(index..).as_ptr() as *const u8)
@@ -178,3 +188,69 @@ pub unsafe fn mulhi_u32x4(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t {
     let ab7654 = vmull_high_u32(a, b);
     vuzp2q_u32(vreinterpretq_u32_u64(ab3210), vreinterpretq_u32_u64(ab7654))
 }
+
+#[inline]
+#[target_feature(enable = "neon")]
+pub unsafe fn mul_color_to_alpha_u8x16(
+    color: uint8x16_t,
+    alpha_u16: uint16x8x2_t,
+    zero: uint8x16_t,
+) -> uint8x16_t {
+    let color_u16_lo = vreinterpretq_u16_u8(vzip1q_u8(color, zero));
+    let mut tmp_res = vmulq_u16(color_u16_lo, alpha_u16.0);
+    tmp_res = vaddq_u16(tmp_res, vrshrq_n_u16::<8>(tmp_res));
+    let res_u16_lo = vrshrq_n_u16::<8>(tmp_res);
+
+    let color_u16_hi = vreinterpretq_u16_u8(vzip2q_u8(color, zero));
+    let mut tmp_res = vmulq_u16(color_u16_hi, alpha_u16.1);
+    tmp_res = vaddq_u16(tmp_res, vrshrq_n_u16::<8>(tmp_res));
+    let res_u16_hi = vrshrq_n_u16::<8>(tmp_res);
+
+    vcombine_u8(vqmovn_u16(res_u16_lo), vqmovn_u16(res_u16_hi))
+}
+
+#[inline]
+#[target_feature(enable = "neon")]
+pub unsafe fn mul_color_to_alpha_u8x8(
+    color: uint8x8_t,
+    alpha_u16: uint16x8_t,
+    zero: uint8x8_t,
+) -> uint8x8_t {
+    let color_u16_lo = vreinterpret_u16_u8(vzip1_u8(color, zero));
+    let color_u16_hi = vreinterpret_u16_u8(vzip2_u8(color, zero));
+    let color_u16 = vcombine_u16(color_u16_lo, color_u16_hi);
+    let mut tmp_res = vmulq_u16(color_u16, alpha_u16);
+    tmp_res = vaddq_u16(tmp_res, vrshrq_n_u16::<8>(tmp_res));
+    let res_u16 = vrshrq_n_u16::<8>(tmp_res);
+    vqmovn_u16(res_u16)
+}
+
+#[inline]
+#[target_feature(enable = "neon")]
+pub unsafe fn mul_color_recip_alpha_u8x16(
+    color: uint8x16_t,
+    recip_alpha: uint16x8x2_t,
+    zero: uint8x16_t,
+) -> uint8x16_t {
+    let color_u16_lo = vreinterpretq_u16_u8(vzip1q_u8(zero, color));
+    let color_u16_hi = vreinterpretq_u16_u8(vzip2q_u8(zero, color));
+
+    let res_u16_lo = mulhi_u16x8(color_u16_lo, recip_alpha.0);
+    let res_u16_hi = mulhi_u16x8(color_u16_hi, recip_alpha.1);
+    vcombine_u8(vmovn_u16(res_u16_lo), vmovn_u16(res_u16_hi))
+}
+
+#[inline]
+#[target_feature(enable = "neon")]
+pub unsafe fn mul_color_recip_alpha_u8x8(
+    color: uint8x8_t,
+    recip_alpha: uint16x8_t,
+    zero: uint8x8_t,
+) -> uint8x8_t {
+    let color_u16_lo = vreinterpret_u16_u8(vzip1_u8(zero, color));
+    let color_u16_hi = vreinterpret_u16_u8(vzip2_u8(zero, color));
+    let color_u16 = vcombine_u16(color_u16_lo, color_u16_hi);
+
+    let res_u16 = mulhi_u16x8(color_u16, recip_alpha);
+    vmovn_u16(res_u16)
+}

tests/alpha_tests.rs

@@ -199,6 +199,12 @@ mod multiply_alpha_u8x2 {
         mul_div_alpha_test(Oper::Mul, SRC_PIXELS, RES_PIXELS, CpuExtensions::Sse4_1);
     }
 
+    #[cfg(target_arch = "aarch64")]
+    #[test]
+    fn neon_test() {
+        mul_div_alpha_test(OPER, SRC_PIXELS, RES_PIXELS, CpuExtensions::Neon);
+    }
+
     #[test]
     fn native_test() {
         mul_div_alpha_test(Oper::Mul, SRC_PIXELS, RES_PIXELS, CpuExtensions::None);
@@ -369,6 +375,12 @@ mod divide_alpha_u8x2 {
         mul_div_alpha_test(OPER, SRC_PIXELS, RES_PIXELS, CpuExtensions::Sse4_1);
     }
 
+    #[cfg(target_arch = "aarch64")]
+    #[test]
+    fn neon_test() {
+        mul_div_alpha_test(OPER, SRC_PIXELS, RES_PIXELS, CpuExtensions::Neon);
+    }
+
     #[test]
     fn native_test() {
         mul_div_alpha_test(OPER, SRC_PIXELS, RES_PIXELS, CpuExtensions::None);