Added optimisation of convolution grayscale images (U8) with helps of `AVX2` instructions.

CHANGELOG.md

@@ -1,3 +1,7 @@
+## [Unreleased] - ReleaseDate
+
+- Added optimisation of convolution grayscale images (U8) with helps of ``AVX2`` instructions.
+
 ## [0.4.0] - 2021-10-23
 
 - Added support of new type of pixels `U8` (without forced SIMD).

Cargo.toml

@@ -2,7 +2,7 @@
 name = "fast_image_resize"
 version = "0.4.1-alpha.0"
 authors = ["Kirill Kuzminykh <[email protected]>"]
-edition = "2018"
+edition = "2021"
 license = "MIT OR Apache-2.0"
 description = "Library for fast image resizing with using of SIMD instructions"
 readme = "README.md"

README.md

@@ -5,7 +5,7 @@ Rust library for fast image resizing with using of SIMD instructions.
 [CHANGELOG](https://github.com/Cykooz/fast_image_resize/blob/main/CHANGELOG.md)
 
 Supported pixel formats and available optimisations:
-- `U8x4` - four `u8` components per pixel:
+- `U8x4` - four `u8` components per pixel (RGB, RGBA, CMYK and other):
     - native Rust-code without forced SIMD
     - SSE4.1
     - AVX2
@@ -15,6 +15,7 @@ Supported pixel formats and available optimisations:
     - native Rust-code without forced SIMD
 - `U8` - one `u8` component per pixel:
     - native Rust-code without forced SIMD
+    - AVX2 
 
 ## Benchmarks
 
@@ -22,7 +23,7 @@ Environment:
 - CPU: Intel(R) Core(TM) i7-6700K CPU @ 4.00GHz
 - RAM: DDR4 3000 MHz
 - Ubuntu 20.04 (linux 5.11)
-- Rust 1.56
+- Rust 1.56.1
 - fast_image_resize = "0.4"
 - glassbench = "0.3.0"
 
@@ -47,11 +48,11 @@ Pipeline:
 
 |            | Nearest | Bilinear | CatmullRom | Lanczos3 |
 |------------|:-------:|:--------:|:----------:|:--------:|
-| image      | 107.950 | 198.726  |  288.085   | 380.573  |
-| resize     | 15.573  |  72.009  |  132.181   | 192.426  |
-| fir rust   |  0.473  |  56.476  |   86.983   | 120.115  |
-| fir sse4.1 |    -    |  11.856  |   17.748   |  25.288  |
-| fir avx2   |    -    |  9.052   |   12.027   |  17.477  |
+| image      | 106.320 | 199.150  |  288.609   | 380.830  |
+| resize     | 15.550  |  72.122  |  132.152   | 192.081  |
+| fir rust   |  0.476  |  56.451  |   86.984   | 119.357  |
+| fir sse4.1 |    -    |  11.798  |   17.768   |  25.296  |
+| fir avx2   |    -    |  8.995   |   13.533   |  19.525  |
 
 ### Resize RGBA image 4928x3279 => 852x567
 
@@ -64,13 +65,13 @@ Pipeline:
 
 |            | Nearest | Bilinear | CatmullRom | Lanczos3 |
 |------------|:-------:|:--------:|:----------:|:--------:|
-| image      | 107.165 | 191.338  |  281.272   | 372.183  |
-| resize     | 18.099  |  79.512  |  149.128   | 225.173  |
-| fir rust   | 13.265  |  69.358  |   99.794   | 132.545  |
-| fir sse4.1 | 11.739  |  23.080  |   29.013   |  36.556  |
-| fir avx2   |  6.958  |  15.590  |   18.610   |  24.219  |
+| image      | 107.186 | 191.834  |  281.246   | 372.796  |
+| resize     | 18.163  |  79.871  |  149.159   | 218.684  |
+| fir rust   | 13.630  |  69.949  |  100.425   | 133.011  |
+| fir sse4.1 | 12.034  |  23.566  |   29.809   |  37.232  |
+| fir avx2   |  6.890  |  15.015  |   18.394   |  23.827  |
 
-### Resize gray image (U8) 4928x3279 => 852x567
+### Resize grayscale image (U8) 4928x3279 => 852x567
 
 Pipeline:
 
@@ -80,11 +81,12 @@ Pipeline:
   has converted into grayscale image with one byte per pixel.
 - Numbers in table is mean duration of image resizing in milliseconds.
 
-|            | Nearest | Bilinear | CatmullRom | Lanczos3 |
-|------------|:-------:|:--------:|:----------:|:--------:|
-| image      | 96.792  | 143.195  |  188.121   | 240.504  |
-| resize     | 10.582  |  26.577  |   53.537   |  81.599  |
-| fir rust   |  0.203  |  24.832  |   30.962   |  46.958  |
+|          | Nearest | Bilinear | CatmullRom | Lanczos3 |
+|----------|:-------:|:--------:|:----------:|:--------:|
+| image    | 92.171  | 141.153  |  184.442   | 230.455  |
+| resize   |  9.890  |  26.205  |   53.054   |  81.181  |
+| fir rust |  0.197  |  26.333  |   29.450   |  43.266  |
+| fir avx2 |    -    |  14.766  |   12.567   |  16.641  |
 
 ## Examples
 

benches/bench_compare_u8.rs

@@ -65,6 +65,11 @@ pub fn bench_downscale_u8(bench: &mut Bench) {
 
     // fast_image_resize crate;
     let mut cpu_ext_and_name = vec![(CpuExtensions::None, "rust")];
+    #[cfg(target_arch = "x86_64")]
+    {
+        // cpu_ext_and_name.push((CpuExtensions::Sse4_1, "sse4.1"));
+        cpu_ext_and_name.push((CpuExtensions::Avx2, "avx2"));
+    }
     for (cpu_ext, ext_name) in cpu_ext_and_name {
         for alg_name in alg_names {
             let src_rgba_image = utils::get_big_luma8_image();

benches/bench_resize.rs

@@ -124,7 +124,7 @@ fn sse4_lanczos3_bench(bench: &mut Bench) {
     unsafe {
         resizer.set_cpu_extensions(CpuExtensions::Sse4_1);
     }
-    bench.task("sse4 lanczos3", |task| {
+    bench.task("lanczos3 sse4", |task| {
         task.iter(|| {
             resizer.resize(&src_image, &mut dst_image).unwrap();
         })
@@ -145,7 +145,7 @@ fn avx2_lanczos3_bench(bench: &mut Bench) {
     unsafe {
         resizer.set_cpu_extensions(CpuExtensions::Avx2);
     }
-    bench.task("avx2 lanczos3", |task| {
+    bench.task("lanczos3 avx2", |task| {
         task.iter(|| {
             resizer.resize(&src_image, &mut dst_image).unwrap();
         })
@@ -166,7 +166,7 @@ fn avx2_supersampling_lanczos3_bench(bench: &mut Bench) {
     unsafe {
         resizer.set_cpu_extensions(CpuExtensions::Avx2);
     }
-    bench.task("avx2 supersampling lanczos3", |task| {
+    bench.task("supersampling lanczos3 avx2", |task| {
         task.iter(|| {
             resizer.resize(&src_image, &mut dst_image).unwrap();
         })
@@ -187,7 +187,7 @@ fn avx2_lanczos3_upscale_bench(bench: &mut Bench) {
     unsafe {
         resizer.set_cpu_extensions(CpuExtensions::Avx2);
     }
-    bench.task("avx2 lanczos3 upscale", |task| {
+    bench.task("lanczos3 upscale avx2", |task| {
         task.iter(|| {
             resizer.resize(&src_image, &mut dst_image).unwrap();
         })
@@ -214,6 +214,26 @@ fn native_lanczos3_i32_bench(bench: &mut Bench) {
     });
 }
 
+fn avx2_lanczos3_u8_bench(bench: &mut Bench) {
+    let image = get_big_u8_image();
+    let mut res_image = Image::new(
+        NonZeroU32::new(NEW_WIDTH).unwrap(),
+        NonZeroU32::new(NEW_HEIGHT).unwrap(),
+        image.pixel_type(),
+    );
+    let src_image = image.view();
+    let mut dst_image = res_image.view_mut();
+    let mut resizer = Resizer::new(ResizeAlg::Convolution(FilterType::Lanczos3));
+    unsafe {
+        resizer.set_cpu_extensions(CpuExtensions::Avx2);
+    }
+    bench.task("u8 lanczos3 avx2", |task| {
+        task.iter(|| {
+            resizer.resize(&src_image, &mut dst_image).unwrap();
+        })
+    });
+}
+
 fn native_lanczos3_u8_bench(bench: &mut Bench) {
     let image = get_big_u8_image();
     let mut res_image = Image::new(
@@ -260,18 +280,19 @@ pub fn main() {
     let cmd = Command::read();
     if cmd.include_bench(name) {
         let mut bench = create_bench(name, "Resize", &cmd);
-        #[cfg(target_arch = "x86_64")]
-        {
-            avx2_lanczos3_bench(&mut bench);
-            avx2_supersampling_lanczos3_bench(&mut bench);
-            avx2_lanczos3_upscale_bench(&mut bench);
-            sse4_lanczos3_bench(&mut bench);
-        }
         native_nearest_bench(&mut bench);
         native_lanczos3_bench(&mut bench);
         native_lanczos3_i32_bench(&mut bench);
         native_lanczos3_u8_bench(&mut bench);
         native_nearest_u8_bench(&mut bench);
+        #[cfg(target_arch = "x86_64")]
+        {
+            sse4_lanczos3_bench(&mut bench);
+            avx2_supersampling_lanczos3_bench(&mut bench);
+            avx2_lanczos3_upscale_bench(&mut bench);
+            avx2_lanczos3_bench(&mut bench);
+            avx2_lanczos3_u8_bench(&mut bench);
+        }
         if let Err(e) = after_bench(&mut bench, &cmd) {
             eprintln!("{:?}", e);
         }

src/convolution/u8x1/avx2.rs

@@ -0,0 +1,408 @@
+use std::arch::x86_64::*;
+
+use crate::convolution::optimisations::CoefficientsI16Chunk;
+use crate::convolution::{optimisations, Bound, Coefficients};
+use crate::image_view::{FourRows, FourRowsMut, TypedImageView, TypedImageViewMut};
+use crate::pixels::U8;
+use crate::simd_utils;
+
+#[inline]
+pub(crate) fn horiz_convolution(
+    src_image: TypedImageView<U8>,
+    mut dst_image: TypedImageViewMut<U8>,
+    offset: u32,
+    coeffs: Coefficients,
+) {
+    let (values, window_size, bounds_per_pixel) =
+        (coeffs.values, coeffs.window_size, coeffs.bounds);
+
+    let normalizer_guard = optimisations::NormalizerGuard::new(values);
+    let precision = normalizer_guard.precision();
+    let coefficients_chunks =
+        normalizer_guard.normalized_i16_chunks(window_size, &bounds_per_pixel);
+    let dst_height = dst_image.height().get();
+
+    let src_iter = src_image.iter_4_rows(offset, dst_height + offset);
+    let dst_iter = dst_image.iter_4_rows_mut();
+    for (src_rows, dst_rows) in src_iter.zip(dst_iter) {
+        unsafe {
+            horiz_convolution_8u4x(src_rows, dst_rows, &coefficients_chunks, precision);
+        }
+    }
+
+    let mut yy = dst_height - dst_height % 4;
+    while yy < dst_height {
+        unsafe {
+            horiz_convolution_8u(
+                src_image.get_row(yy + offset).unwrap(),
+                dst_image.get_row_mut(yy).unwrap(),
+                &coefficients_chunks,
+                precision,
+            );
+        }
+        yy += 1;
+    }
+}
+
+#[inline]
+pub(crate) fn vert_convolution(
+    src_image: TypedImageView<U8>,
+    mut dst_image: TypedImageViewMut<U8>,
+    coeffs: Coefficients,
+) {
+    let (values, window_size, bounds) = (coeffs.values, coeffs.window_size, coeffs.bounds);
+
+    let normalizer_guard = optimisations::NormalizerGuard::new(values);
+    let precision = normalizer_guard.precision();
+    let coeffs_i16 = normalizer_guard.normalized_i16();
+    let coeffs_chunks = coeffs_i16.chunks(window_size);
+
+    let dst_rows = dst_image.iter_rows_mut();
+    for ((&bound, k), dst_row) in bounds.iter().zip(coeffs_chunks).zip(dst_rows) {
+        unsafe {
+            vert_convolution_8u(&src_image, dst_row, k, bound, precision);
+        }
+    }
+}
+
+/// For safety, it is necessary to ensure the following conditions:
+/// - length of all rows in src_rows must be equal
+/// - length of all rows in dst_rows must be equal
+/// - coefficients_chunks.len() == dst_rows.0.len()
+/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.0.len()
+/// - precision <= MAX_COEFS_PRECISION
+#[inline]
+#[target_feature(enable = "avx2")]
+unsafe fn horiz_convolution_8u4x(
+    src_rows: FourRows<u8>,
+    dst_rows: FourRowsMut<u8>,
+    coefficients_chunks: &[CoefficientsI16Chunk],
+    precision: u8,
+) {
+    let s_rows = [src_rows.0, src_rows.1, src_rows.2, src_rows.3];
+    let d_rows = [dst_rows.0, dst_rows.1, dst_rows.2, dst_rows.3];
+    let zero = _mm_setzero_si128();
+    let initial = _mm256_set1_epi32(1 << (precision - 1));
+
+    for (dst_x, coeffs_chunk) in coefficients_chunks.iter().enumerate() {
+        let coeffs = coeffs_chunk.values;
+        let mut x = coeffs_chunk.start as usize;
+        let mut result_i32x8 = [initial, initial, initial, initial];
+
+        let coeffs_by_16 = coeffs.chunks_exact(16);
+        let reminder16 = coeffs_by_16.remainder();
+        for k in coeffs_by_16 {
+            let coeffs_i16x16 = _mm256_loadu_si256(k.as_ptr() as *const __m256i);
+            for i in 0..4 {
+                let pixels_u8x16 = simd_utils::loadu_si128(s_rows[i], x);
+                let pixels_i16x16 = _mm256_cvtepu8_epi16(pixels_u8x16);
+                result_i32x8[i] = _mm256_add_epi32(
+                    result_i32x8[i],
+                    _mm256_madd_epi16(pixels_i16x16, coeffs_i16x16),
+                );
+            }
+            x += 16;
+        }
+
+        let mut coeffs_by_8 = reminder16.chunks_exact(8);
+        let reminder8 = coeffs_by_8.remainder();
+        if let Some(k) = coeffs_by_8.next() {
+            let coeffs_i16x8 = _mm_loadu_si128(k.as_ptr() as *const __m128i);
+            for i in 0..4 {
+                let pixels_u8x8 = simd_utils::loadl_epi64(s_rows[i], x);
+                let pixels_i16x8 = _mm_cvtepu8_epi16(pixels_u8x8);
+                result_i32x8[i] = _mm256_add_epi32(
+                    result_i32x8[i],
+                    _mm256_set_m128i(zero, _mm_madd_epi16(pixels_i16x8, coeffs_i16x8)),
+                );
+            }
+            x += 8;
+        }
+
+        let mut result_i32x4 = result_i32x8.map(|v| hsum_i32x8_avx2(v));
+
+        for &coeff in reminder8 {
+            let coeff_i32 = coeff as i32;
+            for i in 0..4 {
+                result_i32x4[i] += s_rows[i].get_unchecked(x).to_owned() as i32 * coeff_i32;
+            }
+            x += 1;
+        }
+
+        let result_u8x4 = result_i32x4.map(|v| optimisations::clip8(v, precision));
+        for i in 0..4 {
+            *d_rows[i].get_unchecked_mut(dst_x) = result_u8x4[i];
+        }
+    }
+}
+
+/// For safety, it is necessary to ensure the following conditions:
+/// - bounds.len() == dst_row.len()
+/// - coeffs.len() == dst_rows.0.len() * window_size
+/// - max(bound.start + bound.size for bound in bounds) <= src_row.len()
+/// - precision <= MAX_COEFS_PRECISION
+#[inline]
+#[target_feature(enable = "avx2")]
+unsafe fn horiz_convolution_8u(
+    src_row: &[u8],
+    dst_row: &mut [u8],
+    coefficients_chunks: &[CoefficientsI16Chunk],
+    precision: u8,
+) {
+    let zero = _mm_setzero_si128();
+    let initial = _mm256_set1_epi32(1 << (precision - 1));
+
+    for (dst_x, &coeffs_chunk) in coefficients_chunks.iter().enumerate() {
+        let coeffs = coeffs_chunk.values;
+        let mut x = coeffs_chunk.start as usize;
+        let mut result_i32x8 = initial;
+
+        let coeffs_by_16 = coeffs.chunks_exact(16);
+        let reminder16 = coeffs_by_16.remainder();
+        for k in coeffs_by_16 {
+            let coeffs_i16x16 = _mm256_loadu_si256(k.as_ptr() as *const __m256i);
+            let pixels_u8x16 = simd_utils::loadu_si128(src_row, x);
+            let pixels_i16x16 = _mm256_cvtepu8_epi16(pixels_u8x16);
+            result_i32x8 = _mm256_add_epi32(
+                result_i32x8,
+                _mm256_madd_epi16(pixels_i16x16, coeffs_i16x16),
+            );
+            x += 16;
+        }
+
+        let mut coeffs_by_8 = reminder16.chunks_exact(8);
+        let reminder8 = coeffs_by_8.remainder();
+        if let Some(k) = coeffs_by_8.next() {
+            let coeffs_i16x8 = _mm_loadu_si128(k.as_ptr() as *const __m128i);
+            let pixels_u8x8 = simd_utils::loadl_epi64(src_row, x);
+            let pixels_i16x8 = _mm_cvtepu8_epi16(pixels_u8x8);
+            result_i32x8 = _mm256_add_epi32(
+                result_i32x8,
+                _mm256_set_m128i(zero, _mm_madd_epi16(pixels_i16x8, coeffs_i16x8)),
+            );
+
+            x += 8;
+        }
+
+        let mut result_i32 = hsum_i32x8_avx2(result_i32x8);
+
+        for &coeff in reminder8 {
+            let coeff_i32 = coeff as i32;
+            result_i32 += *src_row.get_unchecked(x) as i32 * coeff_i32;
+            x += 1;
+        }
+
+        *dst_row.get_unchecked_mut(dst_x) = optimisations::clip8(result_i32, precision);
+    }
+}
+
+#[inline]
+#[target_feature(enable = "avx2")]
+unsafe fn vert_convolution_8u(
+    src_img: &TypedImageView<U8>,
+    dst_row: &mut [u8],
+    coeffs: &[i16],
+    bound: Bound,
+    precision: u8,
+) {
+    let src_width = src_img.width().get() as usize;
+    let y_start = bound.start;
+    let y_size = bound.size;
+
+    let initial = _mm_set1_epi32(1 << (precision - 1));
+    let initial_256 = _mm256_set1_epi32(1 << (precision - 1));
+    let zero_128 = _mm_setzero_si128();
+    let zero_256: __m256i = _mm256_setzero_si256();
+
+    let mut x: usize = 0;
+    while x < src_width.saturating_sub(31) {
+        let mut sss0 = initial_256;
+        let mut sss1 = initial_256;
+        let mut sss2 = initial_256;
+        let mut sss3 = initial_256;
+
+        let mut y: u32 = 0;
+
+        for (s_row1, s_row2) in src_img.iter_2_rows(y_start, y_start + y_size) {
+            // Load two coefficients at once
+            let two_coeffs = simd_utils::ptr_i16_to_256set1_epi32(coeffs, y as usize);
+            let row1 = simd_utils::loadu_si256(s_row1, x); // top line
+            let row2 = simd_utils::loadu_si256(s_row2, x); // bottom line
+
+            let lo_pixels = _mm256_unpacklo_epi8(row1, row2);
+            let lo_lo = _mm256_unpacklo_epi8(lo_pixels, zero_256);
+            sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(lo_lo, two_coeffs));
+            let hi_lo = _mm256_unpackhi_epi8(lo_pixels, zero_256);
+            sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(hi_lo, two_coeffs));
+
+            let hi_pixels = _mm256_unpackhi_epi8(row1, row2);
+            let lo_hi = _mm256_unpacklo_epi8(hi_pixels, zero_256);
+            sss2 = _mm256_add_epi32(sss2, _mm256_madd_epi16(lo_hi, two_coeffs));
+            let hi_hi = _mm256_unpackhi_epi8(hi_pixels, zero_256);
+            sss3 = _mm256_add_epi32(sss3, _mm256_madd_epi16(hi_hi, two_coeffs));
+
+            y += 2;
+        }
+
+        for s_row in src_img.iter_rows(y_start + y, y_start + y_size) {
+            let one_coeff = _mm256_set1_epi32(coeffs[y as usize] as i32);
+
+            let row1 = simd_utils::loadu_si256(s_row, x); // top line
+            let row2 = _mm256_setzero_si256(); // bottom line is empty
+
+            let lo_pixels = _mm256_unpacklo_epi8(row1, row2);
+            let lo_lo = _mm256_unpacklo_epi8(lo_pixels, zero_256);
+            sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(lo_lo, one_coeff));
+            let hi_lo = _mm256_unpackhi_epi8(lo_pixels, zero_256);
+            sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(hi_lo, one_coeff));
+
+            let hi_pixels = _mm256_unpackhi_epi8(row1, zero_256);
+            let lo_hi = _mm256_unpacklo_epi8(hi_pixels, zero_256);
+            sss2 = _mm256_add_epi32(sss2, _mm256_madd_epi16(lo_hi, one_coeff));
+            let hi_hi = _mm256_unpackhi_epi8(hi_pixels, zero_256);
+            sss3 = _mm256_add_epi32(sss3, _mm256_madd_epi16(hi_hi, one_coeff));
+
+            y += 1;
+        }
+
+        macro_rules! call {
+            ($imm8:expr) => {{
+                sss0 = _mm256_srai_epi32::<$imm8>(sss0);
+                sss1 = _mm256_srai_epi32::<$imm8>(sss1);
+                sss2 = _mm256_srai_epi32::<$imm8>(sss2);
+                sss3 = _mm256_srai_epi32::<$imm8>(sss3);
+            }};
+        }
+        constify_imm8!(precision, call);
+
+        sss0 = _mm256_packs_epi32(sss0, sss1);
+        sss2 = _mm256_packs_epi32(sss2, sss3);
+        sss0 = _mm256_packus_epi16(sss0, sss2);
+        let dst_ptr = dst_row.get_unchecked_mut(x..).as_mut_ptr() as *mut __m256i;
+        _mm256_storeu_si256(dst_ptr, sss0);
+
+        x += 32;
+    }
+
+    while x < src_width.saturating_sub(7) {
+        let mut sss0 = initial; // left row
+        let mut sss1 = initial; // right row
+        let mut y: u32 = 0;
+
+        for (s_row1, s_row2) in src_img.iter_2_rows(y_start, y_start + y_size) {
+            // Load two coefficients at once
+            let two_coeffs = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
+
+            let row1 = simd_utils::loadl_epi64(s_row1, x); // top line
+            let row2 = simd_utils::loadl_epi64(s_row2, x); // bottom line
+
+            let pixels = _mm_unpacklo_epi8(row1, row2);
+            let lo_pixels = _mm_unpacklo_epi8(pixels, zero_128);
+            sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(lo_pixels, two_coeffs));
+            let hi_pixels = _mm_unpackhi_epi8(pixels, zero_128);
+            sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(hi_pixels, two_coeffs));
+
+            y += 2;
+        }
+
+        for s_row in src_img.iter_rows(y_start + y, y_start + y_size) {
+            let one_coeff = _mm_set1_epi32(*coeffs.get_unchecked(y as usize) as i32);
+
+            let row1 = simd_utils::loadl_epi64(s_row, x); // top line
+            let row2 = _mm_setzero_si128(); // bottom line is empty
+
+            let pixels = _mm_unpacklo_epi8(row1, row2);
+            let lo_pixels = _mm_unpacklo_epi8(pixels, zero_128);
+            sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(lo_pixels, one_coeff));
+            let hi_pixels = _mm_unpackhi_epi8(pixels, zero_128);
+            sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(hi_pixels, one_coeff));
+
+            y += 1;
+        }
+
+        macro_rules! call {
+            ($imm8:expr) => {{
+                sss0 = _mm_srai_epi32::<$imm8>(sss0);
+                sss1 = _mm_srai_epi32::<$imm8>(sss1);
+            }};
+        }
+        constify_imm8!(precision, call);
+
+        sss0 = _mm_packs_epi32(sss0, sss1);
+        sss0 = _mm_packus_epi16(sss0, sss0);
+        let dst_ptr = dst_row.get_unchecked_mut(x..).as_mut_ptr() as *mut __m128i;
+        _mm_storel_epi64(dst_ptr, sss0);
+
+        x += 8;
+    }
+
+    while x < src_width.saturating_sub(3) {
+        let mut sss = initial;
+        let mut y: u32 = 0;
+        for (s_row1, s_row2) in src_img.iter_2_rows(y_start, y_start + y_size) {
+            // Load two coefficients at once
+            let two_coeffs = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
+
+            let row1 = simd_utils::mm_cvtsi32_si128_from_u8(s_row1, x); // top line
+            let row2 = simd_utils::mm_cvtsi32_si128_from_u8(s_row2, x); // bottom line
+
+            let pixels_u8 = _mm_unpacklo_epi8(row1, row2);
+            let pixels_i16 = _mm_unpacklo_epi8(pixels_u8, _mm_setzero_si128());
+            sss = _mm_add_epi32(sss, _mm_madd_epi16(pixels_i16, two_coeffs));
+
+            y += 2;
+        }
+
+        for s_row in src_img.iter_rows(y_start + y, y_start + y_size) {
+            let pix = simd_utils::mm_cvtepu8_epi32_from_u8(s_row, x);
+            let mmk = _mm_set1_epi32(*coeffs.get_unchecked(y as usize) as i32);
+            sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
+
+            y += 1;
+        }
+
+        macro_rules! call {
+            ($imm8:expr) => {{
+                sss = _mm_srai_epi32::<$imm8>(sss);
+            }};
+        }
+        constify_imm8!(precision, call);
+
+        sss = _mm_packs_epi32(sss, sss);
+        let u8x4: [u8; 4] = _mm_cvtsi128_si32(_mm_packus_epi16(sss, sss)).to_le_bytes();
+        *dst_row.get_unchecked_mut(x) = u8x4[0];
+        *dst_row.get_unchecked_mut(x + 1) = u8x4[1];
+        *dst_row.get_unchecked_mut(x + 2) = u8x4[2];
+        *dst_row.get_unchecked_mut(x + 3) = u8x4[3];
+        x += 4;
+    }
+
+    for dst_pixel in dst_row.iter_mut().skip(x) {
+        let mut ss0 = 1 << (precision - 1);
+        for (dy, &k) in coeffs.iter().take(y_size as usize).enumerate() {
+            let src_pixel = src_img.get_pixel(x as u32, y_start + dy as u32);
+            ss0 += src_pixel as i32 * (k as i32);
+        }
+        *dst_pixel = optimisations::clip8(ss0, precision);
+        x += 1;
+    }
+}
+
+// only needs AVX2
+#[inline]
+#[target_feature(enable = "avx2")]
+unsafe fn hsum_i32x8_avx2(v: __m256i) -> i32 {
+    let sum128 = _mm_add_epi32(_mm256_castsi256_si128(v), _mm256_extracti128_si256::<1>(v));
+    hsum_epi32_avx(sum128)
+}
+
+#[inline(always)]
+unsafe fn hsum_epi32_avx(x: __m128i) -> i32 {
+    // 3-operand non-destructive AVX lets us save a byte without needing a movdqa
+    let hi64 = _mm_unpackhi_epi64(x, x);
+    let sum64 = _mm_add_epi32(hi64, x);
+    const I: i32 = ((2 << 6) | (3 << 4) | 1) as i32;
+    let hi32 = _mm_shuffle_epi32::<I>(sum64); // Swap the low two elements
+    let sum32 = _mm_add_epi32(sum64, hi32);
+    _mm_cvtsi128_si32(sum32) // movd
+}

src/convolution/u8x1/mod.rs

@@ -3,6 +3,7 @@ use crate::image_view::{TypedImageView, TypedImageViewMut};
 use crate::pixels::U8;
 use crate::CpuExtensions;
 
+mod avx2;
 mod native;
 
 impl Convolution for U8 {
@@ -11,17 +12,25 @@ impl Convolution for U8 {
         dst_image: TypedImageViewMut<Self>,
         offset: u32,
         coeffs: Coefficients,
-        _cpu_extensions: CpuExtensions,
+        cpu_extensions: CpuExtensions,
     ) {
-        native::horiz_convolution(src_image, dst_image, offset, coeffs);
+        match cpu_extensions {
+            #[cfg(target_arch = "x86_64")]
+            CpuExtensions::Avx2 => avx2::horiz_convolution(src_image, dst_image, offset, coeffs),
+            _ => native::horiz_convolution(src_image, dst_image, offset, coeffs),
+        }
     }
 
     fn vert_convolution(
         src_image: TypedImageView<Self>,
         dst_image: TypedImageViewMut<Self>,
         coeffs: Coefficients,
-        _cpu_extensions: CpuExtensions,
+        cpu_extensions: CpuExtensions,
     ) {
-        native::vert_convolution(src_image, dst_image, coeffs);
+        match cpu_extensions {
+            #[cfg(target_arch = "x86_64")]
+            CpuExtensions::Avx2 => avx2::vert_convolution(src_image, dst_image, coeffs),
+            _ => native::vert_convolution(src_image, dst_image, coeffs),
+        }
     }
 }

src/convolution/u8x4/avx2.rs

@@ -334,7 +334,7 @@ unsafe fn horiz_convolution_8u(
 
 #[inline]
 #[target_feature(enable = "avx2")]
-pub(crate) unsafe fn vert_convolution_8u(
+unsafe fn vert_convolution_8u(
     src_img: &TypedImageView<U8x4>,
     dst_row: &mut [u32],
     coeffs: &[i16],
@@ -478,8 +478,8 @@ pub(crate) unsafe fn vert_convolution_8u(
             // Load two coefficients at once
             let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
 
-            let source1 = simd_utils::mm_cvtsi32_si128(s_row1, x); // top line
-            let source2 = simd_utils::mm_cvtsi32_si128(s_row2, x); // bottom line
+            let source1 = simd_utils::mm_cvtsi32_si128_from_u32(s_row1, x); // top line
+            let source2 = simd_utils::mm_cvtsi32_si128_from_u32(s_row2, x); // bottom line
 
             let source = _mm_unpacklo_epi8(source1, source2);
             let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());

src/convolution/u8x4/sse4.rs

@@ -504,8 +504,8 @@ pub(crate) unsafe fn vert_convolution_8u(
             // Load two coefficients at once
             let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
 
-            let source1 = simd_utils::mm_cvtsi32_si128(s_row1, xx); // top line
-            let source2 = simd_utils::mm_cvtsi32_si128(s_row2, xx); // bottom line
+            let source1 = simd_utils::mm_cvtsi32_si128_from_u32(s_row1, xx); // top line
+            let source2 = simd_utils::mm_cvtsi32_si128_from_u32(s_row2, xx); // bottom line
 
             let source = _mm_unpacklo_epi8(source1, source2);
             let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());

src/simd_utils.rs

@@ -23,11 +23,23 @@ pub unsafe fn mm_cvtepu8_epi32(buf: &[u32], index: usize) -> __m128i {
 }
 
 #[inline(always)]
-pub unsafe fn mm_cvtsi32_si128(buf: &[u32], index: usize) -> __m128i {
+pub unsafe fn mm_cvtepu8_epi32_from_u8(buf: &[u8], index: usize) -> __m128i {
+    let ptr = buf.get_unchecked(index..).as_ptr() as *const i32;
+    _mm_cvtepu8_epi32(_mm_cvtsi32_si128(*ptr))
+}
+
+#[inline(always)]
+pub unsafe fn mm_cvtsi32_si128_from_u32(buf: &[u32], index: usize) -> __m128i {
     let v: i32 = transmute(*buf.get_unchecked(index));
     _mm_cvtsi32_si128(v)
 }
 
+#[inline(always)]
+pub unsafe fn mm_cvtsi32_si128_from_u8(buf: &[u8], index: usize) -> __m128i {
+    let ptr = buf.get_unchecked(index..).as_ptr() as *const i32;
+    _mm_cvtsi32_si128(*ptr)
+}
+
 #[inline(always)]
 pub unsafe fn ptr_i16_to_set1_epi32(buf: &[i16], index: usize) -> __m128i {
     _mm_set1_epi32(*(buf.get_unchecked(index..).as_ptr() as *const i32))

tests/resize_tests.rs

@@ -235,7 +235,7 @@ fn resize_nearest_u8x1() {
 }
 
 #[test]
-fn resize_lanczos3_u8x1() {
+fn resize_lanczos3_u8x1_native() {
     let image = get_source_image_u8x1();
     assert!(matches!(image.pixel_type(), PixelType::U8));
 
@@ -254,3 +254,24 @@ fn resize_lanczos3_u8x1() {
         .is_ok());
     save_result(&result, "u8x1-lanczos3-native");
 }
+
+#[test]
+fn resize_lanczos3_u8x1_avx2() {
+    let image = get_source_image_u8x1();
+    assert!(matches!(image.pixel_type(), PixelType::U8));
+
+    let mut resizer = Resizer::new(ResizeAlg::Convolution(FilterType::Lanczos3));
+    unsafe {
+        resizer.set_cpu_extensions(CpuExtensions::Avx2);
+    }
+    let new_height = get_new_height(&image.view(), NEW_WIDTH);
+    let mut result = Image::new(
+        NonZeroU32::new(NEW_WIDTH).unwrap(),
+        NonZeroU32::new(new_height).unwrap(),
+        image.pixel_type(),
+    );
+    assert!(resizer
+        .resize(&image.view(), &mut result.view_mut())
+        .is_ok());
+    save_result(&result, "u8x1-lanczos3-avx2");
+}