Added support of optimisation with helps of NEON SIMD for convolution of U16x4 images.

CHANGELOG.md

@@ -5,6 +5,7 @@
 - Added method `CpuExtensions::is_supported(&self)`.
 - Internals of `PixelComponentMapper` changed to use heap to store its data.
 - Fixed link to documentation page in `README.md` file.
+- Added support of optimisation with helps of `NEON SIMD` for convolution of `U16x4` images.
 
 ## [2.0.0] - 2022-10-28
 

benches/bench_compare_l.rs

@@ -78,6 +78,10 @@ pub fn bench_downscale_l(bench: &mut Bench) {
         cpu_ext_and_name.push((CpuExtensions::Sse4_1, "sse4.1"));
         cpu_ext_and_name.push((CpuExtensions::Avx2, "avx2"));
     }
+    #[cfg(target_arch = "aarch64")]
+    {
+        cpu_ext_and_name.push((CpuExtensions::Neon, "neon"));
+    }
     for (cpu_ext, ext_name) in cpu_ext_and_name {
         for alg_name in alg_names {
             let src_image_data = U8::load_big_src_image();

benches/bench_compare_l16.rs

@@ -78,6 +78,10 @@ pub fn bench_downscale_l16(bench: &mut Bench) {
         cpu_ext_and_name.push((CpuExtensions::Sse4_1, "sse4.1"));
         cpu_ext_and_name.push((CpuExtensions::Avx2, "avx2"));
     }
+    #[cfg(target_arch = "aarch64")]
+    {
+        cpu_ext_and_name.push((CpuExtensions::Neon, "neon"));
+    }
     for (cpu_ext, ext_name) in cpu_ext_and_name {
         for alg_name in alg_names {
             let src_image_data = U16::load_big_src_image();

benches/bench_compare_la.rs

@@ -21,6 +21,10 @@ pub fn bench_downscale_la(bench: &mut Bench) {
         cpu_ext_and_name.push((CpuExtensions::Sse4_1, "sse4.1"));
         cpu_ext_and_name.push((CpuExtensions::Avx2, "avx2"));
     }
+    #[cfg(target_arch = "aarch64")]
+    {
+        cpu_ext_and_name.push((CpuExtensions::Neon, "neon"));
+    }
     for (cpu_ext, ext_name) in cpu_ext_and_name {
         for alg_name in alg_names {
             let resize_alg = match alg_name {

benches/bench_compare_la16.rs

@@ -21,6 +21,10 @@ pub fn bench_downscale_la16(bench: &mut Bench) {
         cpu_ext_and_name.push((CpuExtensions::Sse4_1, "sse4.1"));
         cpu_ext_and_name.push((CpuExtensions::Avx2, "avx2"));
     }
+    #[cfg(target_arch = "aarch64")]
+    {
+        cpu_ext_and_name.push((CpuExtensions::Neon, "neon"));
+    }
     for (cpu_ext, ext_name) in cpu_ext_and_name {
         for alg_name in alg_names {
             let resize_alg = match alg_name {

benches/bench_compare_rgb.rs

@@ -77,6 +77,10 @@ pub fn bench_downscale_rgb(bench: &mut Bench) {
         cpu_ext_and_name.push((CpuExtensions::Sse4_1, "sse4.1"));
         cpu_ext_and_name.push((CpuExtensions::Avx2, "avx2"));
     }
+    #[cfg(target_arch = "aarch64")]
+    {
+        cpu_ext_and_name.push((CpuExtensions::Neon, "neon"));
+    }
     for (cpu_ext, ext_name) in cpu_ext_and_name {
         for alg_name in alg_names {
             let src_image_data = U8x3::load_big_src_image();

benches/bench_compare_rgb16.rs

@@ -79,6 +79,10 @@ pub fn bench_downscale_rgb16(bench: &mut Bench) {
         cpu_ext_and_name.push((CpuExtensions::Sse4_1, "sse4.1"));
         cpu_ext_and_name.push((CpuExtensions::Avx2, "avx2"));
     }
+    #[cfg(target_arch = "aarch64")]
+    {
+        cpu_ext_and_name.push((CpuExtensions::Neon, "neon"));
+    }
     for (cpu_ext, ext_name) in cpu_ext_and_name {
         for alg_name in alg_names {
             let src_view = src_image_data.view();

benches/bench_compare_rgba16.rs

@@ -60,6 +60,10 @@ pub fn bench_downscale_rgba16(bench: &mut Bench) {
         cpu_ext_and_name.push((CpuExtensions::Sse4_1, "sse4.1"));
         cpu_ext_and_name.push((CpuExtensions::Avx2, "avx2"));
     }
+    #[cfg(target_arch = "aarch64")]
+    {
+        cpu_ext_and_name.push((CpuExtensions::Neon, "neon"));
+    }
     for (cpu_ext, ext_name) in cpu_ext_and_name {
         for alg_name in alg_names {
             let resize_alg = match alg_name {

src/convolution/macros.rs

@@ -37,3 +37,75 @@ macro_rules! constify_imm8 {
         }
     };
 }
+
+macro_rules! constify_64_imm8 {
+    ($imm8:expr, $expand:ident) => {
+        #[allow(overflowing_literals)]
+        match ($imm8) & 0b0111_1111 {
+            0 => {}
+            1 => $expand!(1),
+            2 => $expand!(2),
+            3 => $expand!(3),
+            4 => $expand!(4),
+            5 => $expand!(5),
+            6 => $expand!(6),
+            7 => $expand!(7),
+            8 => $expand!(8),
+            9 => $expand!(9),
+            10 => $expand!(10),
+            12 => $expand!(12),
+            13 => $expand!(13),
+            14 => $expand!(14),
+            15 => $expand!(15),
+            16 => $expand!(16),
+            17 => $expand!(17),
+            18 => $expand!(18),
+            19 => $expand!(19),
+            20 => $expand!(20),
+            21 => $expand!(21),
+            22 => $expand!(22),
+            23 => $expand!(23),
+            24 => $expand!(24),
+            25 => $expand!(25),
+            26 => $expand!(26),
+            27 => $expand!(27),
+            28 => $expand!(28),
+            29 => $expand!(29),
+            30 => $expand!(30),
+            31 => $expand!(31),
+            32 => $expand!(32),
+            33 => $expand!(33),
+            34 => $expand!(34),
+            35 => $expand!(35),
+            36 => $expand!(36),
+            37 => $expand!(37),
+            38 => $expand!(38),
+            39 => $expand!(39),
+            40 => $expand!(40),
+            41 => $expand!(41),
+            42 => $expand!(42),
+            43 => $expand!(43),
+            44 => $expand!(44),
+            45 => $expand!(45),
+            46 => $expand!(46),
+            47 => $expand!(47),
+            48 => $expand!(48),
+            49 => $expand!(49),
+            50 => $expand!(50),
+            51 => $expand!(51),
+            52 => $expand!(52),
+            53 => $expand!(53),
+            54 => $expand!(54),
+            55 => $expand!(55),
+            56 => $expand!(56),
+            57 => $expand!(57),
+            58 => $expand!(58),
+            59 => $expand!(59),
+            60 => $expand!(60),
+            61 => $expand!(61),
+            62 => $expand!(62),
+            63 => $expand!(63),
+            _ => unreachable!(),
+        }
+    };
+}

src/convolution/u16x4/mod.rs

@@ -8,6 +8,8 @@ use super::{Coefficients, Convolution};
 #[cfg(target_arch = "x86_64")]
 mod avx2;
 mod native;
+#[cfg(target_arch = "aarch64")]
+mod neon;
 #[cfg(target_arch = "x86_64")]
 mod sse4;
 
@@ -24,6 +26,8 @@ impl Convolution for U16x4 {
             CpuExtensions::Avx2 => avx2::horiz_convolution(src_image, dst_image, offset, coeffs),
             #[cfg(target_arch = "x86_64")]
             CpuExtensions::Sse4_1 => sse4::horiz_convolution(src_image, dst_image, offset, coeffs),
+            #[cfg(target_arch = "aarch64")]
+            CpuExtensions::Neon => neon::horiz_convolution(src_image, dst_image, offset, coeffs),
             _ => native::horiz_convolution(src_image, dst_image, offset, coeffs),
         }
     }

src/convolution/u16x4/neon.rs

@@ -0,0 +1,269 @@
+use std::arch::aarch64::*;
+
+use crate::convolution::{optimisations, Coefficients};
+use crate::image_view::{FourRows, FourRowsMut};
+use crate::neon_utils;
+use crate::pixels::U16x4;
+use crate::{ImageView, ImageViewMut};
+
+#[inline]
+pub(crate) fn horiz_convolution(
+    src_image: &ImageView<U16x4>,
+    dst_image: &mut ImageViewMut<U16x4>,
+    offset: u32,
+    coeffs: Coefficients,
+) {
+    let normalizer = optimisations::Normalizer32::new(coeffs);
+    let precision = normalizer.precision();
+    let coefficients_chunks = normalizer.normalized_chunks();
+    let dst_height = dst_image.height().get();
+
+    let src_iter = src_image.iter_4_rows(offset, dst_height + offset);
+    let dst_iter = dst_image.iter_4_rows_mut();
+    for (src_rows, dst_rows) in src_iter.zip(dst_iter) {
+        unsafe {
+            horiz_convolution_4_rows(src_rows, dst_rows, &coefficients_chunks, precision);
+        }
+    }
+
+    let mut yy = dst_height - dst_height % 4;
+    while yy < dst_height {
+        unsafe {
+            horiz_convolution_row(
+                src_image.get_row(yy + offset).unwrap(),
+                dst_image.get_row_mut(yy).unwrap(),
+                &coefficients_chunks,
+                precision,
+            );
+        }
+        yy += 1;
+    }
+}
+
+/// For safety, it is necessary to ensure the following conditions:
+/// - length of all rows in src_rows must be equal
+/// - length of all rows in dst_rows must be equal
+/// - coefficients_chunks.len() == dst_rows.0.len()
+/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.0.len()
+/// - precision <= MAX_COEFS_PRECISION
+#[target_feature(enable = "neon")]
+unsafe fn horiz_convolution_4_rows(
+    src_rows: FourRows<U16x4>,
+    dst_rows: FourRowsMut<U16x4>,
+    coefficients_chunks: &[optimisations::CoefficientsI32Chunk],
+    precision: u8,
+) {
+    let (s_row0, s_row1, s_row2, s_row3) = src_rows;
+    let s_rows = [s_row0, s_row1, s_row2, s_row3];
+    let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
+    let d_rows = [d_row0, d_row1, d_row2, d_row3];
+
+    let initial = vdupq_n_s64(1i64 << (precision - 1));
+    let zero_u16x8 = vdupq_n_u16(0);
+    let zero_u16x4 = vdup_n_u16(0);
+
+    for (dst_x, coeffs_chunk) in coefficients_chunks.iter().enumerate() {
+        let mut x: usize = coeffs_chunk.start as usize;
+
+        let mut sss_a = [int64x2x2_t(initial, initial); 4];
+
+        let mut coeffs = coeffs_chunk.values;
+
+        let coeffs_by_8 = coeffs.chunks_exact(4);
+        coeffs = coeffs_by_8.remainder();
+        for k in coeffs_by_8 {
+            let coeffs_i32x4 = neon_utils::load_i32x4(k, 0);
+            let coeff0 = vdup_laneq_s32::<0>(coeffs_i32x4);
+            let coeff1 = vdup_laneq_s32::<1>(coeffs_i32x4);
+            let coeff2 = vdup_laneq_s32::<2>(coeffs_i32x4);
+            let coeff3 = vdup_laneq_s32::<3>(coeffs_i32x4);
+
+            for i in 0..4 {
+                let mut sss = sss_a[i];
+                let source = neon_utils::load_u16x8(s_rows[i], x);
+
+                let pix_i32 = vreinterpretq_s32_u16(vzip1q_u16(source, zero_u16x8));
+                sss.0 = vmlal_s32(sss.0, vget_low_s32(pix_i32), coeff0);
+                sss.1 = vmlal_s32(sss.1, vget_high_s32(pix_i32), coeff0);
+
+                let pix_i32 = vreinterpretq_s32_u16(vzip2q_u16(source, zero_u16x8));
+                sss.0 = vmlal_s32(sss.0, vget_low_s32(pix_i32), coeff1);
+                sss.1 = vmlal_s32(sss.1, vget_high_s32(pix_i32), coeff1);
+
+                let source = neon_utils::load_u16x8(s_rows[i], x + 2);
+
+                let pix_i32 = vreinterpretq_s32_u16(vzip1q_u16(source, zero_u16x8));
+                sss.0 = vmlal_s32(sss.0, vget_low_s32(pix_i32), coeff2);
+                sss.1 = vmlal_s32(sss.1, vget_high_s32(pix_i32), coeff2);
+
+                let pix_i32 = vreinterpretq_s32_u16(vzip2q_u16(source, zero_u16x8));
+                sss.0 = vmlal_s32(sss.0, vget_low_s32(pix_i32), coeff3);
+                sss.1 = vmlal_s32(sss.1, vget_high_s32(pix_i32), coeff3);
+
+                sss_a[i] = sss;
+            }
+
+            x += 4;
+        }
+
+        let coeffs_by_4 = coeffs.chunks_exact(2);
+        coeffs = coeffs_by_4.remainder();
+
+        for k in coeffs_by_4 {
+            let coeffs_i32x2 = neon_utils::load_i32x2(k, 0);
+            let coeff0 = vdup_lane_s32::<0>(coeffs_i32x2);
+            let coeff1 = vdup_lane_s32::<1>(coeffs_i32x2);
+
+            for i in 0..4 {
+                let mut sss = sss_a[i];
+                let source = neon_utils::load_u16x8(s_rows[i], x);
+
+                let pix_i32 = vreinterpretq_s32_u16(vzip1q_u16(source, zero_u16x8));
+                sss.0 = vmlal_s32(sss.0, vget_low_s32(pix_i32), coeff0);
+                sss.1 = vmlal_s32(sss.1, vget_high_s32(pix_i32), coeff0);
+
+                let pix_i32 = vreinterpretq_s32_u16(vzip2q_u16(source, zero_u16x8));
+                sss.0 = vmlal_s32(sss.0, vget_low_s32(pix_i32), coeff1);
+                sss.1 = vmlal_s32(sss.1, vget_high_s32(pix_i32), coeff1);
+
+                sss_a[i] = sss;
+            }
+            x += 2;
+        }
+
+        if let Some(&k) = coeffs.first() {
+            let coeff = vdup_n_s32(k);
+
+            for i in 0..4 {
+                let mut sss = sss_a[i];
+                let source = vcombine_u16(neon_utils::load_u16x4(s_rows[i], x), zero_u16x4);
+
+                let pix_i32 = vreinterpretq_s32_u16(vzip1q_u16(source, zero_u16x8));
+                sss.0 = vmlal_s32(sss.0, vget_low_s32(pix_i32), coeff);
+                sss.1 = vmlal_s32(sss.1, vget_high_s32(pix_i32), coeff);
+
+                sss_a[i] = sss;
+            }
+        }
+
+        macro_rules! call {
+            ($imm8:expr) => {{
+                sss_a[0].0 = vshrq_n_s64::<$imm8>(sss_a[0].0);
+                sss_a[0].1 = vshrq_n_s64::<$imm8>(sss_a[0].1);
+                sss_a[1].0 = vshrq_n_s64::<$imm8>(sss_a[1].0);
+                sss_a[1].1 = vshrq_n_s64::<$imm8>(sss_a[1].1);
+                sss_a[2].0 = vshrq_n_s64::<$imm8>(sss_a[2].0);
+                sss_a[2].1 = vshrq_n_s64::<$imm8>(sss_a[2].1);
+                sss_a[3].0 = vshrq_n_s64::<$imm8>(sss_a[3].0);
+                sss_a[3].1 = vshrq_n_s64::<$imm8>(sss_a[3].1);
+            }};
+        }
+        constify_64_imm8!(precision as i64, call);
+
+        for i in 0..4 {
+            let sss = sss_a[i];
+            let sss_i32x4 = vcombine_s32(vqmovn_s64(sss.0), vqmovn_s64(sss.1));
+            let sss_u16x4 = vqmovun_s32(sss_i32x4);
+            let dst_pix = d_rows[i].get_unchecked_mut(dst_x);
+            let ptr = dst_pix as *mut U16x4 as *mut u16;
+            vst1_u16(ptr, sss_u16x4);
+        }
+    }
+}
+
+/// For safety, it is necessary to ensure the following conditions:
+/// - bounds.len() == dst_row.len()
+/// - coefficients_chunks.len() == dst_row.len()
+/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.len()
+/// - precision <= MAX_COEFS_PRECISION
+#[target_feature(enable = "neon")]
+unsafe fn horiz_convolution_row(
+    src_row: &[U16x4],
+    dst_row: &mut [U16x4],
+    coefficients_chunks: &[optimisations::CoefficientsI32Chunk],
+    precision: u8,
+) {
+    let initial = vdupq_n_s64(1i64 << (precision - 1));
+    let zero_u16x8 = vdupq_n_u16(0);
+    let zero_u16x4 = vdup_n_u16(0);
+
+    for (&coeffs_chunk, dst_pix) in coefficients_chunks.iter().zip(dst_row) {
+        let mut x: usize = coeffs_chunk.start as usize;
+        let mut sss = int64x2x2_t(initial, initial);
+        let mut coeffs = coeffs_chunk.values;
+
+        let coeffs_by_4 = coeffs.chunks_exact(4);
+        coeffs = coeffs_by_4.remainder();
+
+        for k in coeffs_by_4 {
+            let coeffs_i32x4 = neon_utils::load_i32x4(k, 0);
+            let source = neon_utils::load_u16x8(src_row, x);
+
+            let coeff = vdup_laneq_s32::<0>(coeffs_i32x4);
+            let pix_i32 = vreinterpretq_s32_u16(vzip1q_u16(source, zero_u16x8));
+            sss.0 = vmlal_s32(sss.0, vget_low_s32(pix_i32), coeff);
+            sss.1 = vmlal_s32(sss.1, vget_high_s32(pix_i32), coeff);
+
+            let coeff = vdup_laneq_s32::<1>(coeffs_i32x4);
+            let pix_i32 = vreinterpretq_s32_u16(vzip2q_u16(source, zero_u16x8));
+            sss.0 = vmlal_s32(sss.0, vget_low_s32(pix_i32), coeff);
+            sss.1 = vmlal_s32(sss.1, vget_high_s32(pix_i32), coeff);
+
+            let source = neon_utils::load_u16x8(src_row, x + 2);
+
+            let coeff = vdup_laneq_s32::<2>(coeffs_i32x4);
+            let pix_i32 = vreinterpretq_s32_u16(vzip1q_u16(source, zero_u16x8));
+            sss.0 = vmlal_s32(sss.0, vget_low_s32(pix_i32), coeff);
+            sss.1 = vmlal_s32(sss.1, vget_high_s32(pix_i32), coeff);
+
+            let coeff = vdup_laneq_s32::<3>(coeffs_i32x4);
+            let pix_i32 = vreinterpretq_s32_u16(vzip2q_u16(source, zero_u16x8));
+            sss.0 = vmlal_s32(sss.0, vget_low_s32(pix_i32), coeff);
+            sss.1 = vmlal_s32(sss.1, vget_high_s32(pix_i32), coeff);
+
+            x += 4;
+        }
+
+        let coeffs_by_2 = coeffs.chunks_exact(2);
+        coeffs = coeffs_by_2.remainder();
+
+        for k in coeffs_by_2 {
+            let coeffs_i32x2 = neon_utils::load_i32x2(k, 0);
+            let source = neon_utils::load_u16x8(src_row, x);
+
+            let coeff = vdup_lane_s32::<0>(coeffs_i32x2);
+            let pix_i32 = vreinterpretq_s32_u16(vzip1q_u16(source, zero_u16x8));
+            sss.0 = vmlal_s32(sss.0, vget_low_s32(pix_i32), coeff);
+            sss.1 = vmlal_s32(sss.1, vget_high_s32(pix_i32), coeff);
+
+            let coeff = vdup_lane_s32::<1>(coeffs_i32x2);
+            let pix_i32 = vreinterpretq_s32_u16(vzip2q_u16(source, zero_u16x8));
+            sss.0 = vmlal_s32(sss.0, vget_low_s32(pix_i32), coeff);
+            sss.1 = vmlal_s32(sss.1, vget_high_s32(pix_i32), coeff);
+
+            x += 2;
+        }
+
+        if let Some(&k) = coeffs.first() {
+            let source = vcombine_u16(neon_utils::load_u16x4(src_row, x), zero_u16x4);
+
+            let coeff = vdup_n_s32(k);
+            let pix_i32 = vreinterpretq_s32_u16(vzip1q_u16(source, zero_u16x8));
+            sss.0 = vmlal_s32(sss.0, vget_low_s32(pix_i32), coeff);
+            sss.1 = vmlal_s32(sss.1, vget_high_s32(pix_i32), coeff);
+        }
+
+        macro_rules! call {
+            ($imm8:expr) => {{
+                sss.0 = vshrq_n_s64::<$imm8>(sss.0);
+                sss.1 = vshrq_n_s64::<$imm8>(sss.1);
+            }};
+        }
+        constify_64_imm8!(precision as i64, call);
+
+        let sss_i32x4 = vcombine_s32(vqmovn_s64(sss.0), vqmovn_s64(sss.1));
+        let sss_u16x4 = vqmovun_s32(sss_i32x4);
+        let ptr = dst_pix as *mut U16x4 as *mut u16;
+        vst1_u16(ptr, sss_u16x4);
+    }
+}

src/convolution/vertical_u16/mod.rs

@@ -6,6 +6,8 @@ use crate::{ImageView, ImageViewMut};
 #[cfg(target_arch = "x86_64")]
 pub(crate) mod avx2;
 pub(crate) mod native;
+#[cfg(target_arch = "aarch64")]
+mod neon;
 #[cfg(target_arch = "x86_64")]
 pub(crate) mod sse4;
 
@@ -20,6 +22,8 @@ pub(crate) fn vert_convolution_u16<T: PixelExt<Component = u16>>(
         CpuExtensions::Avx2 => avx2::vert_convolution(src_image, dst_image, coeffs),
         #[cfg(target_arch = "x86_64")]
         CpuExtensions::Sse4_1 => sse4::vert_convolution(src_image, dst_image, coeffs),
+        #[cfg(target_arch = "aarch64")]
+        CpuExtensions::Neon => neon::vert_convolution(src_image, dst_image, coeffs),
         _ => native::vert_convolution(src_image, dst_image, coeffs),
     }
 }

src/convolution/vertical_u16/neon.rs

@@ -0,0 +1,176 @@
+use std::arch::aarch64::*;
+use std::mem::transmute;
+
+use crate::convolution::{optimisations, Coefficients};
+use crate::neon_utils;
+use crate::pixels::PixelExt;
+use crate::{ImageView, ImageViewMut};
+
+pub(crate) fn vert_convolution<T: PixelExt<Component = u16>>(
+    src_image: &ImageView<T>,
+    dst_image: &mut ImageViewMut<T>,
+    coeffs: Coefficients,
+) {
+    // Check safety conditions
+    debug_assert_eq!(src_image.width(), dst_image.width());
+    debug_assert_eq!(coeffs.bounds.len(), dst_image.height().get() as usize);
+
+    let normalizer = optimisations::Normalizer32::new(coeffs);
+    let coefficients_chunks = normalizer.normalized_chunks();
+    let precision = normalizer.precision();
+    let initial = 1i64 << (precision - 1);
+
+    let mut tmp_dst = vec![0i64; dst_image.width().get() as usize * T::count_of_components()];
+    let tmp_buf = tmp_dst.as_mut_slice();
+    let dst_rows = dst_image.iter_rows_mut();
+    for (dst_row, coeffs_chunk) in dst_rows.zip(coefficients_chunks) {
+        tmp_buf.fill(initial);
+        unsafe {
+            vert_convolution_into_one_row_i64(src_image, tmp_buf, coeffs_chunk);
+            let dst_comp = T::components_mut(dst_row);
+            macro_rules! call {
+                ($imm8:expr) => {{
+                    store_tmp_buf_into_dst_row::<$imm8>(tmp_buf, dst_comp, &normalizer);
+                }};
+            }
+            constify_64_imm8!(precision as i64, call);
+        }
+    }
+}
+
+#[target_feature(enable = "neon")]
+unsafe fn vert_convolution_into_one_row_i64<T: PixelExt<Component = u16>>(
+    src_img: &ImageView<T>,
+    dst_buf: &mut [i64],
+    coeffs_chunk: optimisations::CoefficientsI32Chunk,
+) {
+    let width = dst_buf.len();
+    let y_start = coeffs_chunk.start;
+    let coeffs = coeffs_chunk.values;
+
+    let zero_u16x8 = vdupq_n_u16(0);
+    let zero_u16x4 = vdup_n_u16(0);
+
+    for (s_row, &coeff) in src_img.iter_rows(y_start).zip(coeffs) {
+        let components = T::components(s_row);
+        let coeff_i32x2 = vdup_n_s32(coeff);
+
+        let mut x: usize = 0;
+        while x < width.saturating_sub(31) {
+            let source = neon_utils::load_u16x8x4(components, x);
+
+            for s in [source.0, source.1, source.2, source.3] {
+                let mut accum = neon_utils::load_i64x2x4(dst_buf, x);
+                let pix = vreinterpretq_s32_u16(vzip1q_u16(s, zero_u16x8));
+                accum.0 = vmlal_s32(accum.0, vget_low_s32(pix), coeff_i32x2);
+                accum.1 = vmlal_s32(accum.1, vget_high_s32(pix), coeff_i32x2);
+                let pix = vreinterpretq_s32_u16(vzip2q_u16(s, zero_u16x8));
+                accum.2 = vmlal_s32(accum.2, vget_low_s32(pix), coeff_i32x2);
+                accum.3 = vmlal_s32(accum.3, vget_high_s32(pix), coeff_i32x2);
+                neon_utils::store_i64x2x4(dst_buf, x, accum);
+                x += 8;
+            }
+        }
+
+        if x < width.saturating_sub(15) {
+            let source = neon_utils::load_u16x8x2(components, x);
+
+            for s in [source.0, source.1] {
+                let mut accum = neon_utils::load_i64x2x4(dst_buf, x);
+                let pix = vreinterpretq_s32_u16(vzip1q_u16(s, zero_u16x8));
+                accum.0 = vmlal_s32(accum.0, vget_low_s32(pix), coeff_i32x2);
+                accum.1 = vmlal_s32(accum.1, vget_high_s32(pix), coeff_i32x2);
+                let pix = vreinterpretq_s32_u16(vzip2q_u16(s, zero_u16x8));
+                accum.2 = vmlal_s32(accum.2, vget_low_s32(pix), coeff_i32x2);
+                accum.3 = vmlal_s32(accum.3, vget_high_s32(pix), coeff_i32x2);
+                neon_utils::store_i64x2x4(dst_buf, x, accum);
+                x += 8;
+            }
+        }
+
+        if x < width.saturating_sub(7) {
+            let s = neon_utils::load_u16x8(components, x);
+            let mut accum = neon_utils::load_i64x2x4(dst_buf, x);
+            let pix = vreinterpretq_s32_u16(vzip1q_u16(s, zero_u16x8));
+            accum.0 = vmlal_s32(accum.0, vget_low_s32(pix), coeff_i32x2);
+            accum.1 = vmlal_s32(accum.1, vget_high_s32(pix), coeff_i32x2);
+            let pix = vreinterpretq_s32_u16(vzip2q_u16(s, zero_u16x8));
+            accum.2 = vmlal_s32(accum.2, vget_low_s32(pix), coeff_i32x2);
+            accum.3 = vmlal_s32(accum.3, vget_high_s32(pix), coeff_i32x2);
+            neon_utils::store_i64x2x4(dst_buf, x, accum);
+            x += 8;
+        }
+
+        if x < width.saturating_sub(3) {
+            let s = vcombine_u16(neon_utils::load_u16x4(components, x), zero_u16x4);
+            let mut accum = neon_utils::load_i64x2x2(dst_buf, x);
+            let pix = vreinterpretq_s32_u16(vzip1q_u16(s, zero_u16x8));
+            accum.0 = vmlal_s32(accum.0, vget_low_s32(pix), coeff_i32x2);
+            accum.1 = vmlal_s32(accum.1, vget_high_s32(pix), coeff_i32x2);
+            neon_utils::store_i64x2x2(dst_buf, x, accum);
+            x += 4;
+        }
+
+        let coeff = coeff as i64;
+        let tmp_tail = dst_buf.iter_mut().skip(x);
+        let comp_tail = components.iter().skip(x);
+        for (accum, &comp) in tmp_tail.zip(comp_tail) {
+            *accum += coeff * comp as i64;
+        }
+    }
+}
+
+#[target_feature(enable = "neon")]
+unsafe fn store_tmp_buf_into_dst_row<const IMM: i32>(
+    mut src_buf: &[i64],
+    dst_buf: &mut [u16],
+    normalizer: &optimisations::Normalizer32,
+) {
+    let mut dst_chunks_8 = dst_buf.chunks_exact_mut(8);
+    let src_chunks_8 = src_buf.chunks_exact(8);
+    src_buf = src_chunks_8.remainder();
+    for (dst_chunk, src_chunk) in dst_chunks_8.by_ref().zip(src_chunks_8) {
+        let mut accum = neon_utils::load_i64x2x4(src_chunk, 0);
+        accum.0 = vshrq_n_s64::<IMM>(accum.0);
+        accum.1 = vshrq_n_s64::<IMM>(accum.1);
+        accum.2 = vshrq_n_s64::<IMM>(accum.2);
+        accum.3 = vshrq_n_s64::<IMM>(accum.3);
+        let sss0_i32 = vcombine_s32(vqmovn_s64(accum.0), vqmovn_s64(accum.1));
+        let sss1_i32 = vcombine_s32(vqmovn_s64(accum.2), vqmovn_s64(accum.3));
+        let sss_u16 = vcombine_u16(vqmovun_s32(sss0_i32), vqmovun_s32(sss1_i32));
+        let dst_ptr = dst_chunk.as_mut_ptr() as *mut u128;
+        vstrq_p128(dst_ptr, transmute(sss_u16));
+    }
+
+    let mut dst_chunks_4 = dst_chunks_8.into_remainder().chunks_exact_mut(4);
+    let src_chunks_4 = src_buf.chunks_exact(4);
+    src_buf = src_chunks_4.remainder();
+    for (dst_chunk, src_chunk) in dst_chunks_4.by_ref().zip(src_chunks_4) {
+        let mut accum = neon_utils::load_i64x2x2(src_chunk, 0);
+        accum.0 = vshrq_n_s64::<IMM>(accum.0);
+        accum.1 = vshrq_n_s64::<IMM>(accum.1);
+        let sss_i32 = vcombine_s32(vqmovn_s64(accum.0), vqmovn_s64(accum.1));
+        let sss_u16 = vcombine_u16(vqmovun_s32(sss_i32), vqmovun_s32(sss_i32));
+        let res = vdupd_laneq_u64::<0>(vreinterpretq_u64_u16(sss_u16));
+        let dst_ptr = dst_chunk.as_mut_ptr() as *mut u64;
+        *dst_ptr = res;
+    }
+
+    let mut dst_chunks_2 = dst_chunks_4.into_remainder().chunks_exact_mut(2);
+    let src_chunks_2 = src_buf.chunks_exact(2);
+    src_buf = src_chunks_2.remainder();
+    for (dst_chunk, src_chunk) in dst_chunks_2.by_ref().zip(src_chunks_2) {
+        let mut accum = neon_utils::load_i64x2(src_chunk, 0);
+        accum = vshrq_n_s64::<IMM>(accum);
+        let sss_i32 = vcombine_s32(vqmovn_s64(accum), vqmovn_s64(accum));
+        let sss_u16 = vcombine_u16(vqmovun_s32(sss_i32), vqmovun_s32(sss_i32));
+        let res = vdups_laneq_u32::<0>(vreinterpretq_u32_u16(sss_u16));
+        let dst_ptr = dst_chunk.as_mut_ptr() as *mut u32;
+        *dst_ptr = res;
+    }
+
+    let dst_chunk = dst_chunks_2.into_remainder();
+    for (dst, &src) in dst_chunk.iter_mut().zip(src_buf) {
+        *dst = normalizer.clip(src);
+    }
+}

src/neon_utils.rs

@@ -26,6 +26,31 @@ pub unsafe fn load_u8x16x4<T>(buf: &[T], index: usize) -> uint8x16x4_t {
     vld1q_u8_x4(buf.get_unchecked(index..).as_ptr() as *const u8)
 }
 
+#[inline(always)]
+pub unsafe fn load_u16x4<T>(buf: &[T], index: usize) -> uint16x4_t {
+    vld1_u16(buf.get_unchecked(index..).as_ptr() as *const u16)
+}
+
+#[inline(always)]
+pub unsafe fn load_u16x8<T>(buf: &[T], index: usize) -> uint16x8_t {
+    vld1q_u16(buf.get_unchecked(index..).as_ptr() as *const u16)
+}
+
+#[inline(always)]
+pub unsafe fn load_u16x8x2<T>(buf: &[T], index: usize) -> uint16x8x2_t {
+    vld1q_u16_x2(buf.get_unchecked(index..).as_ptr() as *const u16)
+}
+
+#[inline(always)]
+pub unsafe fn load_u16x8x4<T>(buf: &[T], index: usize) -> uint16x8x4_t {
+    vld1q_u16_x4(buf.get_unchecked(index..).as_ptr() as *const u16)
+}
+
+#[inline(always)]
+pub unsafe fn load_i32x2<T>(buf: &[T], index: usize) -> int32x2_t {
+    vld1_s32(buf.get_unchecked(index..).as_ptr() as *const i32)
+}
+
 #[inline(always)]
 pub unsafe fn load_i32x4<T>(buf: &[T], index: usize) -> int32x4_t {
     vld1q_s32(buf.get_unchecked(index..).as_ptr() as *const i32)
@@ -56,6 +81,31 @@ pub unsafe fn store_i32x4x4<T>(buf: &mut [T], index: usize, v: int32x4x4_t) {
     vst1q_s32_x4(buf.get_unchecked_mut(index..).as_mut_ptr() as *mut i32, v);
 }
 
+#[inline(always)]
+pub unsafe fn load_i64x2<T>(buf: &[T], index: usize) -> int64x2_t {
+    vld1q_s64(buf.get_unchecked(index..).as_ptr() as *const i64)
+}
+
+#[inline(always)]
+pub unsafe fn load_i64x2x2<T>(buf: &[T], index: usize) -> int64x2x2_t {
+    vld1q_s64_x2(buf.get_unchecked(index..).as_ptr() as *const i64)
+}
+
+#[inline(always)]
+pub unsafe fn load_i64x2x4<T>(buf: &[T], index: usize) -> int64x2x4_t {
+    vld1q_s64_x4(buf.get_unchecked(index..).as_ptr() as *const i64)
+}
+
+#[inline(always)]
+pub unsafe fn store_i64x2x2<T>(buf: &mut [T], index: usize, v: int64x2x2_t) {
+    vst1q_s64_x2(buf.get_unchecked_mut(index..).as_mut_ptr() as *mut i64, v);
+}
+
+#[inline(always)]
+pub unsafe fn store_i64x2x4<T>(buf: &mut [T], index: usize, v: int64x2x4_t) {
+    vst1q_s64_x4(buf.get_unchecked_mut(index..).as_mut_ptr() as *mut i64, v);
+}
+
 #[inline(always)]
 pub unsafe fn load_i16x4<T>(buf: &[T], index: usize) -> int16x4_t {
     vld1_s16(buf.get_unchecked(index..).as_ptr() as *const i16)

tests/resize_tests.rs

@@ -213,6 +213,10 @@ fn downscale_u8() {
         cpu_extensions_vec.push(CpuExtensions::Sse4_1);
         cpu_extensions_vec.push(CpuExtensions::Avx2);
     }
+    #[cfg(target_arch = "aarch64")]
+    {
+        cpu_extensions_vec.push(CpuExtensions::Neon);
+    }
     for cpu_extensions in cpu_extensions_vec {
         P::downscale_test(
             ResizeAlg::Convolution(FilterType::Lanczos3),
@@ -230,8 +234,13 @@ fn upscale_u8() {
     let mut cpu_extensions_vec = vec![CpuExtensions::None];
     #[cfg(target_arch = "x86_64")]
     {
+        cpu_extensions_vec.push(CpuExtensions::Sse4_1);
         cpu_extensions_vec.push(CpuExtensions::Avx2);
     }
+    #[cfg(target_arch = "aarch64")]
+    {
+        cpu_extensions_vec.push(CpuExtensions::Neon);
+    }
     for cpu_extensions in cpu_extensions_vec {
         P::upscale_test(
             ResizeAlg::Convolution(FilterType::Lanczos3),
@@ -252,6 +261,10 @@ fn downscale_u8x2() {
         cpu_extensions_vec.push(CpuExtensions::Sse4_1);
         cpu_extensions_vec.push(CpuExtensions::Avx2);
     }
+    #[cfg(target_arch = "aarch64")]
+    {
+        cpu_extensions_vec.push(CpuExtensions::Neon);
+    }
     for cpu_extensions in cpu_extensions_vec {
         P::downscale_test(
             ResizeAlg::Convolution(FilterType::Lanczos3),
@@ -276,6 +289,10 @@ fn upscale_u8x2() {
         cpu_extensions_vec.push(CpuExtensions::Sse4_1);
         cpu_extensions_vec.push(CpuExtensions::Avx2);
     }
+    #[cfg(target_arch = "aarch64")]
+    {
+        cpu_extensions_vec.push(CpuExtensions::Neon);
+    }
     for cpu_extensions in cpu_extensions_vec {
         P::upscale_test(
             ResizeAlg::Convolution(FilterType::Lanczos3),
@@ -300,6 +317,10 @@ fn downscale_u8x3() {
         cpu_extensions_vec.push(CpuExtensions::Sse4_1);
         cpu_extensions_vec.push(CpuExtensions::Avx2);
     }
+    #[cfg(target_arch = "aarch64")]
+    {
+        cpu_extensions_vec.push(CpuExtensions::Neon);
+    }
     for cpu_extensions in cpu_extensions_vec {
         P::downscale_test(
             ResizeAlg::Convolution(FilterType::Lanczos3),
@@ -324,6 +345,10 @@ fn upscale_u8x3() {
         cpu_extensions_vec.push(CpuExtensions::Sse4_1);
         cpu_extensions_vec.push(CpuExtensions::Avx2);
     }
+    #[cfg(target_arch = "aarch64")]
+    {
+        cpu_extensions_vec.push(CpuExtensions::Neon);
+    }
     for cpu_extensions in cpu_extensions_vec {
         P::upscale_test(
             ResizeAlg::Convolution(FilterType::Lanczos3),
@@ -382,6 +407,10 @@ fn upscale_u8x4() {
         cpu_extensions_vec.push(CpuExtensions::Sse4_1);
         cpu_extensions_vec.push(CpuExtensions::Avx2);
     }
+    #[cfg(target_arch = "aarch64")]
+    {
+        cpu_extensions_vec.push(CpuExtensions::Neon);
+    }
     for cpu_extensions in cpu_extensions_vec {
         P::upscale_test(
             ResizeAlg::Convolution(FilterType::Lanczos3),
@@ -402,6 +431,10 @@ fn downscale_u16() {
         cpu_extensions_vec.push(CpuExtensions::Sse4_1);
         cpu_extensions_vec.push(CpuExtensions::Avx2);
     }
+    #[cfg(target_arch = "aarch64")]
+    {
+        cpu_extensions_vec.push(CpuExtensions::Neon);
+    }
     for cpu_extensions in cpu_extensions_vec {
         P::downscale_test(
             ResizeAlg::Convolution(FilterType::Lanczos3),
@@ -422,6 +455,10 @@ fn upscale_u16() {
         cpu_extensions_vec.push(CpuExtensions::Sse4_1);
         cpu_extensions_vec.push(CpuExtensions::Avx2);
     }
+    #[cfg(target_arch = "aarch64")]
+    {
+        cpu_extensions_vec.push(CpuExtensions::Neon);
+    }
     for cpu_extensions in cpu_extensions_vec {
         P::upscale_test(
             ResizeAlg::Convolution(FilterType::Lanczos3),
@@ -446,6 +483,10 @@ fn downscale_u16x2() {
         cpu_extensions_vec.push(CpuExtensions::Sse4_1);
         cpu_extensions_vec.push(CpuExtensions::Avx2);
     }
+    #[cfg(target_arch = "aarch64")]
+    {
+        cpu_extensions_vec.push(CpuExtensions::Neon);
+    }
     for cpu_extensions in cpu_extensions_vec {
         P::downscale_test(
             ResizeAlg::Convolution(FilterType::Lanczos3),
@@ -470,6 +511,10 @@ fn upscale_u16x2() {
         cpu_extensions_vec.push(CpuExtensions::Sse4_1);
         cpu_extensions_vec.push(CpuExtensions::Avx2);
     }
+    #[cfg(target_arch = "aarch64")]
+    {
+        cpu_extensions_vec.push(CpuExtensions::Neon);
+    }
     for cpu_extensions in cpu_extensions_vec {
         P::upscale_test(
             ResizeAlg::Convolution(FilterType::Lanczos3),
@@ -494,6 +539,10 @@ fn downscale_u16x3() {
         cpu_extensions_vec.push(CpuExtensions::Sse4_1);
         cpu_extensions_vec.push(CpuExtensions::Avx2);
     }
+    #[cfg(target_arch = "aarch64")]
+    {
+        cpu_extensions_vec.push(CpuExtensions::Neon);
+    }
     for cpu_extensions in cpu_extensions_vec {
         P::downscale_test(
             ResizeAlg::Convolution(FilterType::Lanczos3),
@@ -518,6 +567,10 @@ fn upscale_u16x3() {
         cpu_extensions_vec.push(CpuExtensions::Sse4_1);
         cpu_extensions_vec.push(CpuExtensions::Avx2);
     }
+    #[cfg(target_arch = "aarch64")]
+    {
+        cpu_extensions_vec.push(CpuExtensions::Neon);
+    }
     for cpu_extensions in cpu_extensions_vec {
         P::upscale_test(
             ResizeAlg::Convolution(FilterType::Lanczos3),
@@ -542,6 +595,10 @@ fn downscale_u16x4() {
         cpu_extensions_vec.push(CpuExtensions::Sse4_1);
         cpu_extensions_vec.push(CpuExtensions::Avx2);
     }
+    #[cfg(target_arch = "aarch64")]
+    {
+        cpu_extensions_vec.push(CpuExtensions::Neon);
+    }
     for cpu_extensions in cpu_extensions_vec {
         P::downscale_test(
             ResizeAlg::Convolution(FilterType::Lanczos3),
@@ -566,6 +623,10 @@ fn upscale_u16x4() {
         cpu_extensions_vec.push(CpuExtensions::Sse4_1);
         cpu_extensions_vec.push(CpuExtensions::Avx2);
     }
+    #[cfg(target_arch = "aarch64")]
+    {
+        cpu_extensions_vec.push(CpuExtensions::Neon);
+    }
     for cpu_extensions in cpu_extensions_vec {
         P::upscale_test(
             ResizeAlg::Convolution(FilterType::Lanczos3),