Improved speed of native implementation for vertical convolution pass for pixel types based on u16 components.

CHANGELOG.md

@@ -4,6 +4,8 @@
 
 - Improved speed of `AVX2` implementation for vertical
   convolution pass for pixel types based on `u8` components.
+- Improved speed of `native` implementation for vertical
+  convolution pass for pixel types based on `u16` components.
 
 ### Fixed
 

src/convolution/vertical_u16/native.rs

@@ -1,6 +1,5 @@
 use crate::convolution::optimisations::Normalizer32;
 use crate::pixels::InnerPixel;
-use crate::utils::foreach_with_pre_reading;
 use crate::{ImageView, ImageViewMut};
 
 #[inline(always)]
@@ -25,23 +24,59 @@ pub(crate) fn vert_convolution<T>(
         let dst_components = T::components_mut(dst_row);
         let mut x_src = src_x_initial;
 
-        let (_, dst_chunks, tail) = unsafe { dst_components.align_to_mut::<[u16; 16]>() };
-        x_src = convolution_by_chunks(
-            src_view,
-            normalizer,
-            initial,
-            dst_chunks,
-            x_src,
-            first_y_src,
-            ks,
-        );
+        const CHUNK_SIZE: usize = 16;
+        let mut dst_chunks = dst_components.chunks_exact_mut(CHUNK_SIZE);
+        for dst_chunk in &mut dst_chunks {
+            x_src = convolution_one_chunk::<T, CHUNK_SIZE>(
+                src_view,
+                normalizer,
+                initial,
+                dst_chunk,
+                x_src,
+                first_y_src,
+                ks,
+            );
+        }
 
+        let tail = dst_chunks.into_remainder();
         if !tail.is_empty() {
             convolution_by_u16(src_view, normalizer, initial, tail, x_src, first_y_src, ks);
         }
     }
 }
 
+#[inline(always)]
+fn convolution_one_chunk<T, const CHUNK_SIZE: usize>(
+    src_view: &impl ImageView<Pixel = T>,
+    normalizer: &Normalizer32,
+    initial: i64,
+    dst_chunk: &mut [u16],
+    mut x_src: usize,
+    first_y_src: u32,
+    ks: &[i32],
+) -> usize
+where
+    T: InnerPixel<Component = u16>,
+{
+    let mut ss = [initial; CHUNK_SIZE];
+    let src_rows = src_view.iter_rows(first_y_src);
+    let x_end = x_src + CHUNK_SIZE;
+
+    for (k, src_row) in ks.iter().copied().zip(src_rows) {
+        let src_components = T::components(src_row);
+        let src_chunk: &[u16] = &src_components[x_src..x_end];
+        for (s, &c) in ss.iter_mut().zip(src_chunk) {
+            *s += c as i64 * (k as i64);
+        }
+    }
+
+    for (i, s) in ss.iter().copied().enumerate() {
+        dst_chunk[i] = normalizer.clip(s);
+    }
+    x_src += CHUNK_SIZE;
+    x_src
+}
+
 #[inline(always)]
 pub(crate) fn convolution_by_u16<T: InnerPixel<Component = u16>>(
     src_view: &impl ImageView<Pixel = T>,
@@ -56,10 +91,8 @@ pub(crate) fn convolution_by_u16<T: InnerPixel<Component = u16>>(
         let mut ss = initial;
         let src_rows = src_view.iter_rows(first_y_src);
         for (&k, src_row) in ks.iter().zip(src_rows) {
-            // SAFETY: Alignment of src_row is greater or equal than alignment u16
-            //         because one component of pixel type T is u16.
-            let src_ptr = src_row.as_ptr() as *const u16;
-            let src_component = unsafe { *src_ptr.add(x_src) };
+            let src_components = T::components(src_row);
+            let src_component = src_components[x_src];
             ss += src_component as i64 * (k as i64);
         }
         *dst_component = normalizer.clip(ss);
@@ -67,45 +100,3 @@ pub(crate) fn convolution_by_u16<T: InnerPixel<Component = u16>>(
     }
     x_src
 }
-
-#[inline(always)]
-fn convolution_by_chunks<T, const CHUNK_SIZE: usize>(
-    src_view: &impl ImageView<Pixel = T>,
-    normalizer: &Normalizer32,
-    initial: i64,
-    dst_chunks: &mut [[u16; CHUNK_SIZE]],
-    mut x_src: usize,
-    first_y_src: u32,
-    ks: &[i32],
-) -> usize
-where
-    T: InnerPixel<Component = u16>,
-{
-    for dst_chunk in dst_chunks {
-        let mut ss = [initial; CHUNK_SIZE];
-        let src_rows = src_view.iter_rows(first_y_src);
-
-        foreach_with_pre_reading(
-            ks.iter().zip(src_rows),
-            |(&k, src_row)| {
-                let src_ptr = src_row.as_ptr() as *const u16;
-                let src_chunk = unsafe {
-                    let ptr = src_ptr.add(x_src) as *const [u16; CHUNK_SIZE];
-                    ptr.read_unaligned()
-                };
-                (src_chunk, k)
-            },
-            |(src_chunk, k)| {
-                for (s, c) in ss.iter_mut().zip(src_chunk) {
-                    *s += c as i64 * (k as i64);
-                }
-            },
-        );
-
-        for (i, s) in ss.iter().copied().enumerate() {
-            dst_chunk[i] = normalizer.clip(s);
-        }
-        x_src += CHUNK_SIZE;
-    }
-    x_src
-}