external/fast_image_resize
1
0
Fork 0
mirror of https://github.com/Cykooz/fast_image_resize.git synced 2026-10-07 17:01:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

- Updated benchmarks for arm64.

Browse Source 
- Improved speed of `AVX2` implementation for vertical
  convolution pass for pixel types based on `u8` components.
This commit is contained in:
Kirill Kuzminykh
2026-06-25 20:26:07 +03:00
parent 4717518e16
commit 5dee12510e
8 changed files with 317 additions and 350 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+6 -1
View File
@@ -1,5 +1,10 @@
## [Unreleased] - ReleaseDate
### Changed
- Improved speed of `AVX2` implementation for vertical
convolution pass for pixel types based on `u8` components.
### Fixed
- Fixed a deadlock in case of using `rayon` thread pool with only one thread to
@@ -28,7 +33,7 @@ a new feature `std` was added to the list of default features.
### Added
- Added support of some variants of `ImageBuffer` type
form the `image` crate ([#57](https://github.com/Cykooz/fast_image_resize/issues/57)).
from the `image` crate ([#57](https://github.com/Cykooz/fast_image_resize/issues/57)).
## [5.4.0] - 2025-11-28
README.md
+9 -9
View File
@@ -94,9 +94,9 @@ Pipeline:
| image | 28.92 | - | 78.41 | 127.38 | 173.98 |
| resize | 8.49 | 24.35 | 48.44 | 92.29 | 137.61 |
| libvips | 2.41 | 61.63 | 5.67 | 9.76 | 16.07 |
| fir rust | 0.28 | 10.72 | 15.67 | 26.08 | 37.15 |
| fir sse4.1 | 0.28 | 3.74 | 5.64 | 10.28 | 15.98 |
| fir avx2 | 0.28 | 2.77 | 4.16 | 7.37 | 14.35 |
| fir rust | 0.28 | 10.86 | 15.71 | 26.43 | 37.47 |
| fir sse4.1 | 0.28 | 3.94 | 5.80 | 10.40 | 15.67 |
| fir avx2 | 0.28 | 2.63 | 3.53 | 7.23 | 13.26 |
<!-- bench_compare_rgb end -->
@@ -117,9 +117,9 @@ Pipeline:
|------------|:-------:|:------:|:--------:|:-------:|:--------:|
| resize | 13.97 | 44.38 | 88.03 | 150.33 | 216.80 |
| libvips | 4.19 | 169.24 | 137.35 | 228.62 | 329.39 |
| fir rust | 0.20 | 20.57 | 25.80 | 37.26 | 50.53 |
| fir sse4.1 | 0.20 | 10.17 | 13.77 | 18.20 | 25.35 |
| fir avx2 | 0.20 | 7.41 | 9.02 | 13.46 | 25.45 |
| fir rust | 0.19 | 20.99 | 26.40 | 38.18 | 51.62 |
| fir sse4.1 | 0.19 | 11.28 | 13.44 | 19.69 | 26.88 |
| fir avx2 | 0.19 | 9.90 | 10.69 | 15.47 | 24.48 |
<!-- bench_compare_rgba end -->
@@ -140,9 +140,9 @@ Pipeline:
| image | 26.33 | - | 57.97 | 86.30 | 113.91 |
| resize | 6.59 | 11.63 | 21.04 | 45.28 | 68.73 |
| libvips | 2.66 | 24.92 | 6.83 | 9.81 | 12.72 |
| fir rust | 0.16 | 4.34 | 5.46 | 8.73 | 12.09 |
| fir sse4.1 | 0.16 | 1.66 | 2.25 | 3.67 | 5.89 |
| fir avx2 | 0.16 | 1.92 | 2.04 | 3.16 | 4.51 |
| fir rust | 0.16 | 4.23 | 5.48 | 8.74 | 12.13 |
| fir sse4.1 | 0.16 | 1.62 | 2.25 | 3.68 | 5.88 |
| fir avx2 | 0.16 | 1.78 | 1.81 | 2.87 | 4.08 |
<!-- bench_compare_l end -->
benches/utils/resize_functions.rs
+3 -3
View File
@@ -1,6 +1,6 @@
use std::hint::black_box;
use std::ops::Deref;
use criterion::black_box;
use fast_image_resize::images::Image;
use fast_image_resize::{CpuExtensions, FilterType, ResizeAlg, ResizeOptions, Resizer};
use image::{imageops, ImageBuffer};
@@ -12,7 +12,7 @@ const ALG_NAMES: [&str; 5] = ["Nearest", "Box", "Bilinear", "Bicubic", "Lanczos3
const NEW_WIDTH: u32 = 852;
const NEW_HEIGHT: u32 = 567;
/// Resize image with help of "image" crate (https://crates.io/crates/image)
/// Resize the image with the help of "image" crate (https://crates.io/crates/image)
pub fn image_resize<P, C>(bench_group: &mut BenchGroup, src_image: &ImageBuffer<P, C>)
where
P: image::Pixel + 'static,
@@ -183,7 +183,7 @@ mod vips {
}
}
/// Resize image with help of "fast_imager_resize" crate
/// Resize image with the help of "fast_imager_resize" crate
pub fn fir_resize<P: PixelTestingExt>(bench_group: &mut BenchGroup, use_alpha: bool) {
let src_image_data = P::load_big_src_image();
let mut dst_image = Image::new(NEW_WIDTH, NEW_HEIGHT, src_image_data.pixel_type());
benchmarks-arm64.md
+63 -63
View File
@@ -5,15 +5,15 @@
Environment:
- CPU: Neoverse-N1 2GHz (Oracle Cloud Compute, VM.Standard.A1.Flex)
- Ubuntu 24.04 (linux 6.11.0)
- Rust 1.87.0
- criterion = "0.5.1"
- fast_image_resize = "5.1.4"
- Ubuntu 24.04 (linux 6.17.0)
- Rust 1.96.0
- criterion = "0.7.0"
- fast_image_resize = "6.0.1"
Other libraries used to compare of resizing speed:
- image = "0.25.6" (<https://crates.io/crates/image>)
- resize = "0.8.8" (<https://crates.io/crates/resize>, single-threaded mode)
- resize = "0.8.9" (<https://crates.io/crates/resize>, single-threaded mode)
- libvips = "8.15.1" (single-threaded mode)
Resize algorithms:
@@ -35,15 +35,15 @@ Pipeline:
`src_image => resize => dst_image`
- Source image [nasa-4928x3279.png](https://github.com/Cykooz/fast_image_resize/blob/main/data/nasa-4928x3279.png)
- Numbers in the table mean a duration of image resizing in milliseconds.
- Numbers in the table mean the duration of the image resizing in milliseconds.
| | Nearest | Box | Bilinear | Bicubic | Lanczos3 |
|----------|:-------:|:------:|:--------:|:-------:|:--------:|
| image | 83.12 | - | 170.54 | 305.90 | 433.80 |
| resize | 30.38 | 59.92 | 101.62 | 183.87 | 273.82 |
| libvips | 9.59 | 137.52 | 26.91 | 66.06 | 88.65 |
| fir rust | 0.92 | 19.25 | 32.19 | 82.86 | 110.04 |
| fir neon | 0.92 | 16.46 | 23.75 | 42.43 | 62.16 |
| image | 85.12 | - | 162.16 | 280.32 | 395.20 |
| resize | 18.01 | 57.46 | 99.42 | 181.77 | 271.01 |
| libvips | 9.50 | 137.87 | 27.05 | 66.46 | 88.16 |
| fir rust | 0.93 | 21.62 | 33.19 | 84.79 | 112.42 |
| fir neon | 0.93 | 19.03 | 29.35 | 53.89 | 80.08 |
<!-- bench_compare_rgb end -->
@@ -57,15 +57,15 @@ Pipeline:
- Source image
[nasa-4928x3279-rgba.png](https://github.com/Cykooz/fast_image_resize/blob/main/data/nasa-4928x3279-rgba.png)
- Numbers in the table mean a duration of image resizing in milliseconds.
- Numbers in the table mean the duration of the image resizing in milliseconds.
- The `image` crate does not support multiplying and dividing by alpha channel.
| | Nearest | Box | Bilinear | Bicubic | Lanczos3 |
|----------|:-------:|:------:|:--------:|:-------:|:--------:|
| resize | 20.69 | 73.58 | 113.62 | 195.21 | 291.60 |
| libvips | 13.87 | 326.93 | 234.51 | 471.17 | 599.71 |
| fir rust | 0.92 | 44.76 | 58.95 | 126.45 | 165.50 |
| fir neon | 0.92 | 29.08 | 40.81 | 64.57 | 90.29 |
| resize | 23.25 | 83.97 | 118.46 | 190.13 | 280.65 |
| libvips | 12.97 | 326.75 | 230.34 | 462.90 | 601.18 |
| fir rust | 1.05 | 48.55 | 62.64 | 128.41 | 167.95 |
| fir neon | 1.05 | 32.06 | 45.74 | 75.15 | 105.96 |
<!-- bench_compare_rgba end -->
@@ -79,15 +79,15 @@ Pipeline:
- Source image [nasa-4928x3279.png](https://github.com/Cykooz/fast_image_resize/blob/main/data/nasa-4928x3279.png)
has converted into grayscale image with one byte per pixel.
- Numbers in the table mean a duration of image resizing in milliseconds.
- Numbers in the table mean the duration of the image resizing in milliseconds.
| | Nearest | Box | Bilinear | Bicubic | Lanczos3 |
|----------|:-------:|:-----:|:--------:|:-------:|:--------:|
| image | 78.35 | - | 119.04 | 186.56 | 258.32 |
| resize | 11.31 | 26.78 | 40.38 | 70.32 | 93.02 |
| libvips | 5.68 | 51.41 | 14.45 | 24.51 | 31.19 |
| fir rust | 0.48 | 9.30 | 10.73 | 18.47 | 25.09 |
| fir neon | 0.48 | 4.92 | 7.48 | 13.16 | 20.15 |
| image | 77.77 | - | 105.01 | 154.93 | 209.09 |
| resize | 10.61 | 25.22 | 38.49 | 67.99 | 89.72 |
| libvips | 5.61 | 51.23 | 14.39 | 24.45 | 31.17 |
| fir rust | 0.52 | 8.12 | 11.53 | 19.06 | 25.75 |
| fir neon | 0.52 | 5.73 | 9.02 | 16.23 | 24.83 |
<!-- bench_compare_l end -->
@@ -102,15 +102,15 @@ Pipeline:
- Source image
[nasa-4928x3279-rgba.png](https://github.com/Cykooz/fast_image_resize/blob/main/data/nasa-4928x3279-rgba.png)
has converted into grayscale image with an alpha channel (two bytes per pixel).
- Numbers in the table mean a duration of image resizing in milliseconds.
- Numbers in the table mean the duration of the image resizing in milliseconds.
- The `image` crate does not support multiplying and dividing by alpha channel.
- The `resize` crate does not support this pixel format.
| | Nearest | Box | Bilinear | Bicubic | Lanczos3 |
|----------|:-------:|:------:|:--------:|:-------:|:--------:|
| libvips | 8.81 | 188.05 | 134.45 | 231.87 | 290.01 |
| fir rust | 0.66 | 18.79 | 25.40 | 43.31 | 55.68 |
| fir neon | 0.66 | 16.88 | 21.07 | 32.55 | 44.99 |
| libvips | 8.70 | 187.63 | 133.41 | 231.63 | 291.51 |
| fir rust | 0.68 | 33.65 | 39.85 | 56.77 | 69.76 |
| fir neon | 0.68 | 19.35 | 24.54 | 38.86 | 53.59 |
<!-- bench_compare_la end -->
@@ -124,15 +124,15 @@ Pipeline:
- Source image [nasa-4928x3279.png](https://github.com/Cykooz/fast_image_resize/blob/main/data/nasa-4928x3279.png)
has converted into RGB16 image.
- Numbers in the table mean a duration of image resizing in milliseconds.
- Numbers in the table mean the duration of the image resizing in milliseconds.
| | Nearest | Box | Bilinear | Bicubic | Lanczos3 |
|----------|:-------:|:------:|:--------:|:-------:|:--------:|
| image | 82.91 | - | 173.38 | 338.57 | 493.64 |
| resize | 19.50 | 58.09 | 98.29 | 183.34 | 267.44 |
| libvips | 23.67 | 197.37 | 109.00 | 228.02 | 301.44 |
| fir rust | 1.39 | 48.64 | 76.07 | 135.93 | 191.26 |
| fir neon | 1.39 | 54.50 | 72.78 | 111.57 | 138.66 |
| image | 87.96 | - | 167.95 | 311.81 | 439.21 |
| resize | 19.54 | 56.71 | 97.12 | 179.53 | 264.88 |
| libvips | 23.09 | 197.68 | 108.47 | 227.87 | 301.61 |
| fir rust | 1.55 | 48.06 | 75.94 | 134.00 | 190.20 |
| fir neon | 1.55 | 49.03 | 107.60 | 182.35 | 275.17 |
<!-- bench_compare_rgb16 end -->
@@ -146,15 +146,15 @@ Pipeline:
- Source image
[nasa-4928x3279-rgba.png](https://github.com/Cykooz/fast_image_resize/blob/main/data/nasa-4928x3279-rgba.png)
- Numbers in the table mean a duration of image resizing in milliseconds.
- Numbers in the table mean the duration of the image resizing in milliseconds.
- The `image` crate does not support multiplying and dividing by alpha channel.
| | Nearest | Box | Bilinear | Bicubic | Lanczos3 |
|----------|:-------:|:------:|:--------:|:-------:|:--------:|
| resize | 23.64 | 77.94 | 115.55 | 206.91 | 303.21 |
| libvips | 33.79 | 329.22 | 236.24 | 466.96 | 591.03 |
| fir rust | 1.51 | 73.59 | 108.54 | 200.42 | 267.85 |
| fir neon | 1.51 | 45.25 | 60.21 | 90.64 | 120.82 |
| resize | 27.67 | 83.52 | 120.86 | 202.04 | 293.93 |
| libvips | 32.69 | 330.39 | 232.21 | 463.59 | 597.62 |
| fir rust | 1.69 | 108.32 | 164.23 | 278.84 | 389.66 |
| fir neon | 1.69 | 78.89 | 122.18 | 212.33 | 305.96 |
<!-- bench_compare_rgba16 end -->
@@ -168,15 +168,15 @@ Pipeline:
- Source image [nasa-4928x3279.png](https://github.com/Cykooz/fast_image_resize/blob/main/data/nasa-4928x3279.png)
has converted into grayscale image with two bytes per pixel.
- Numbers in the table mean a duration of image resizing in milliseconds.
- Numbers in the table mean the duration of the image resizing in milliseconds.
| | Nearest | Box | Bilinear | Bicubic | Lanczos3 |
|----------|:-------:|:-----:|:--------:|:-------:|:--------:|
| image | 78.13 | - | 123.28 | 195.56 | 272.13 |
| resize | 11.84 | 25.06 | 38.29 | 68.43 | 95.25 |
| libvips | 9.27 | 69.76 | 38.82 | 76.85 | 100.79 |
| fir rust | 0.64 | 23.31 | 35.76 | 58.21 | 85.60 |
| fir neon | 0.64 | 11.80 | 16.48 | 26.00 | 36.87 |
| image | 78.06 | - | 109.47 | 169.31 | 224.94 |
| resize | 11.13 | 24.01 | 36.36 | 65.23 | 92.03 |
| libvips | 9.40 | 70.21 | 38.73 | 77.86 | 102.19 |
| fir rust | 0.70 | 24.55 | 35.58 | 57.41 | 81.62 |
| fir neon | 0.70 | 13.96 | 19.89 | 32.82 | 46.68 |
<!-- bench_compare_l16 end -->
@@ -191,15 +191,15 @@ Pipeline:
- Source image
[nasa-4928x3279-rgba.png](https://github.com/Cykooz/fast_image_resize/blob/main/data/nasa-4928x3279-rgba.png)
has converted into grayscale image with an alpha channel (four bytes per pixel).
- Numbers in the table mean a duration of image resizing in milliseconds.
- Numbers in the table mean the duration of the image resizing in milliseconds.
- The `image` crate does not support multiplying and dividing by alpha channel.
- The `resize` crate does not support this pixel format.
| | Nearest | Box | Bilinear | Bicubic | Lanczos3 |
|----------|:-------:|:------:|:--------:|:-------:|:--------:|
| libvips | 18.43 | 202.03 | 145.95 | 242.56 | 302.87 |
| fir rust | 0.93 | 41.87 | 60.11 | 101.73 | 140.40 |
| fir neon | 0.93 | 24.30 | 33.30 | 52.33 | 71.84 |
| libvips | 17.67 | 201.92 | 145.05 | 244.12 | 305.65 |
| fir rust | 1.05 | 54.67 | 74.63 | 118.01 | 151.70 |
| fir neon | 1.05 | 28.36 | 40.41 | 65.05 | 90.69 |
<!-- bench_compare_la16 end -->
@@ -213,14 +213,14 @@ Pipeline:
- Source image [nasa-4928x3279.png](https://github.com/Cykooz/fast_image_resize/blob/main/data/nasa-4928x3279.png)
has converted into grayscale image with two bytes per pixel.
- Numbers in the table mean a duration of image resizing in milliseconds.
- Numbers in the table mean the duration of the image resizing in milliseconds.
| | Nearest | Box | Bilinear | Bicubic | Lanczos3 |
|----------|:-------:|:-----:|:--------:|:-------:|:--------:|
| image | 7.06 | - | 39.97 | 72.83 | 106.05 |
| resize | 12.14 | 23.30 | 32.60 | 55.84 | 83.37 |
| libvips | 8.20 | 67.17 | 40.19 | 90.36 | 118.79 |
| fir rust | 0.93 | 18.92 | 30.89 | 54.12 | 78.40 |
| image | 45.55 | - | 100.08 | 174.50 | 235.24 |
| resize | 11.83 | 24.02 | 31.62 | 55.41 | 82.32 |
| libvips | 8.26 | 68.04 | 39.61 | 92.16 | 120.59 |
| fir rust | 1.05 | 18.56 | 30.89 | 53.71 | 77.71 |
<!-- bench_compare_l32f end -->
@@ -240,14 +240,14 @@ Pipeline:
- Source image
[nasa-4928x3279-rgba.png](https://github.com/Cykooz/fast_image_resize/blob/main/data/nasa-4928x3279-rgba.png)
has converted into grayscale image with an alpha channel (two `f32` values per pixel).
- Numbers in the table mean a duration of image resizing in milliseconds.
- Numbers in the table mean the duration of the image resizing in milliseconds.
- The `image` crate does not support multiplying and dividing by alpha channel.
- The `resize` crate does not support this pixel format.
| | Nearest | Box | Bilinear | Bicubic | Lanczos3 |
|----------|:-------:|:------:|:--------:|:-------:|:--------:|
| libvips | 16.89 | 184.38 | 129.68 | 225.33 | 283.49 |
| fir rust | 1.52 | 38.62 | 61.88 | 116.00 | 162.75 |
| libvips | 16.58 | 184.87 | 128.10 | 225.74 | 287.46 |
| fir rust | 1.72 | 42.67 | 67.39 | 119.26 | 165.99 |
<!-- bench_compare_la32f end -->
@@ -261,14 +261,14 @@ Pipeline:
- Source image [nasa-4928x3279.png](https://github.com/Cykooz/fast_image_resize/blob/main/data/nasa-4928x3279.png)
has converted into RGB32F image.
- Numbers in the table mean a duration of image resizing in milliseconds.
- Numbers in the table mean the duration of the image resizing in milliseconds.
| | Nearest | Box | Bilinear | Bicubic | Lanczos3 |
|----------|:-------:|:------:|:--------:|:-------:|:--------:|
| image | 8.00 | - | 51.74 | 95.36 | 139.32 |
| resize | 19.96 | 45.73 | 69.25 | 133.08 | 190.53 |
| libvips | 19.48 | 197.65 | 114.33 | 274.50 | 354.63 |
| fir rust | 2.29 | 39.09 | 71.75 | 149.96 | 214.76 |
| image | 56.01 | - | 129.91 | 297.73 | 404.21 |
| resize | 22.44 | 43.80 | 67.94 | 128.55 | 187.76 |
| libvips | 19.51 | 199.19 | 111.67 | 274.90 | 359.87 |
| fir rust | 2.52 | 40.51 | 71.07 | 149.98 | 213.66 |
<!-- bench_compare_rgb32f end -->
@@ -283,14 +283,14 @@ Pipeline:
- Source image
[nasa-4928x3279-rgba.png](https://github.com/Cykooz/fast_image_resize/blob/main/data/nasa-4928x3279-rgba.png)
- Numbers in the table mean a duration of image resizing in milliseconds.
- Numbers in the table mean the duration of the image resizing in milliseconds.
- The `image` crate does not support multiplying and dividing by alpha channel.
- The `resize` crate does not support multiplying and dividing by alpha channel
for this pixel format.
| | Nearest | Box | Bilinear | Bicubic | Lanczos3 |
|----------|:-------:|:------:|:--------:|:-------:|:--------:|
| libvips | 32.13 | 323.66 | 230.94 | 456.97 | 587.84 |
| fir rust | 3.01 | 68.00 | 111.35 | 209.27 | 315.19 |
| libvips | 30.45 | 323.63 | 224.27 | 451.64 | 587.34 |
| fir rust | 3.42 | 72.10 | 111.44 | 211.38 | 314.30 |
<!-- bench_compare_rgba32f end -->
benchmarks-x86_64.md
+12 -12
View File
@@ -43,9 +43,9 @@ Pipeline:
| image | 28.92 | - | 78.41 | 127.38 | 173.98 |
| resize | 8.49 | 24.35 | 48.44 | 92.29 | 137.61 |
| libvips | 2.41 | 61.63 | 5.67 | 9.76 | 16.07 |
| fir rust | 0.28 | 10.72 | 15.67 | 26.08 | 37.15 |
| fir sse4.1 | 0.28 | 3.74 | 5.64 | 10.28 | 15.98 |
| fir avx2 | 0.28 | 2.77 | 4.16 | 7.37 | 14.35 |
| fir rust | 0.28 | 10.86 | 15.71 | 26.43 | 37.47 |
| fir sse4.1 | 0.28 | 3.94 | 5.80 | 10.40 | 15.67 |
| fir avx2 | 0.28 | 2.63 | 3.53 | 7.23 | 13.26 |
<!-- bench_compare_rgb end -->
@@ -66,9 +66,9 @@ Pipeline:
|------------|:-------:|:------:|:--------:|:-------:|:--------:|
| resize | 13.97 | 44.38 | 88.03 | 150.33 | 216.80 |
| libvips | 4.19 | 169.24 | 137.35 | 228.62 | 329.39 |
| fir rust | 0.20 | 20.57 | 25.80 | 37.26 | 50.53 |
| fir sse4.1 | 0.20 | 10.17 | 13.77 | 18.20 | 25.35 |
| fir avx2 | 0.20 | 7.41 | 9.02 | 13.46 | 25.45 |
| fir rust | 0.19 | 20.99 | 26.40 | 38.18 | 51.62 |
| fir sse4.1 | 0.19 | 11.28 | 13.44 | 19.69 | 26.88 |
| fir avx2 | 0.19 | 9.90 | 10.69 | 15.47 | 24.48 |
<!-- bench_compare_rgba end -->
@@ -89,9 +89,9 @@ Pipeline:
| image | 26.33 | - | 57.97 | 86.30 | 113.91 |
| resize | 6.59 | 11.63 | 21.04 | 45.28 | 68.73 |
| libvips | 2.66 | 24.92 | 6.83 | 9.81 | 12.72 |
| fir rust | 0.16 | 4.34 | 5.46 | 8.73 | 12.09 |
| fir sse4.1 | 0.16 | 1.66 | 2.25 | 3.67 | 5.89 |
| fir avx2 | 0.16 | 1.92 | 2.04 | 3.16 | 4.51 |
| fir rust | 0.16 | 4.23 | 5.48 | 8.74 | 12.13 |
| fir sse4.1 | 0.16 | 1.62 | 2.25 | 3.68 | 5.88 |
| fir avx2 | 0.16 | 1.78 | 1.81 | 2.87 | 4.08 |
<!-- bench_compare_l end -->
@@ -113,9 +113,9 @@ Pipeline:
| | Nearest | Box | Bilinear | Bicubic | Lanczos3 |
|------------|:-------:|:-----:|:--------:|:-------:|:--------:|
| libvips | 3.73 | 94.13 | 76.50 | 123.09 | 165.76 |
| fir rust | 0.18 | 17.96 | 20.33 | 26.63 | 32.50 |
| fir sse4.1 | 0.18 | 6.24 | 7.27 | 10.11 | 13.94 |
| fir avx2 | 0.18 | 4.23 | 5.10 | 7.01 | 9.67 |
| fir rust | 0.17 | 17.38 | 19.72 | 26.08 | 31.96 |
| fir sse4.1 | 0.17 | 6.42 | 7.52 | 10.35 | 14.26 |
| fir avx2 | 0.17 | 4.85 | 5.23 | 6.76 | 9.25 |
<!-- bench_compare_la end -->
src/convolution/vertical_u8/avx2.rs
+112 -153
View File
@@ -1,7 +1,7 @@
use core::arch::x86_64::*;
use super::sse4;
use crate::convolution::optimisations::{CoefficientsI16Chunk, Normalizer16};
use crate::convolution::vertical_u8::native;
use crate::image_view::ImageViewMut;
use crate::pixels::InnerPixel;
use crate::{simd_utils, ImageView};
@@ -66,19 +66,14 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
let coeffs = coeffs_chunk.values();
let max_rows = coeffs.len() as u32;
let y_last = (y_start + max_rows).max(1) - 1;
let initial = _mm_set1_epi32(1 << (PRECISION as u8 - 1));
let initial_256 = _mm256_set1_epi32(1 << (PRECISION as u8 - 1));
let mut dst_u8 = T::components_mut(dst_row);
// 32 components in one register
let mut dst_chunks_32 = dst_u8.chunks_exact_mut(32);
for dst_chunk in &mut dst_chunks_32 {
let mut sss0 = initial_256;
let mut sss1 = initial_256;
let mut sss2 = initial_256;
let mut sss3 = initial_256;
let initial_256 = _mm256_set1_epi32(1 << (PRECISION as u8 - 1));
let mut dst_chunks_64 = dst_u8.chunks_exact_mut(64);
for dst_chunk in &mut dst_chunks_64 {
let mut sss0 = [initial_256; 4];
let mut sss1 = [initial_256; 4];
let coeffs_chunks = coeffs.chunks_exact(2);
let coeffs_reminder = coeffs_chunks.remainder();
@@ -90,20 +85,8 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
// Load two coefficients at once
let mmk = simd_utils::mm256_load_and_clone_i16x2(two_coeffs);
let source1 = simd_utils::loadu_si256(components1, src_x); // top line
let source2 = simd_utils::loadu_si256(components2, src_x); // bottom line
let source = _mm256_unpacklo_epi8(source1, source2);
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
let source = _mm256_unpackhi_epi8(source1, source2);
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
sss2 = _mm256_add_epi32(sss2, _mm256_madd_epi16(pix, mmk));
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
sss3 = _mm256_add_epi32(sss3, _mm256_madd_epi16(pix, mmk));
conv_32_components_two_rows(components1, components2, src_x, &mut sss0, mmk);
conv_32_components_two_rows(components1, components2, src_x + 32, &mut sss1, mmk);
}
if let Some(&k) = coeffs_reminder.first() {
@@ -111,142 +94,118 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
let components = T::components(s_row);
let mmk = _mm256_set1_epi32(k as i32);
let source1 = simd_utils::loadu_si256(components, src_x); // top line
let source2 = _mm256_setzero_si256(); // bottom line is empty
let source = _mm256_unpacklo_epi8(source1, source2);
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
let source = _mm256_unpackhi_epi8(source1, _mm256_setzero_si256());
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
sss2 = _mm256_add_epi32(sss2, _mm256_madd_epi16(pix, mmk));
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
sss3 = _mm256_add_epi32(sss3, _mm256_madd_epi16(pix, mmk));
conv_32_components_one_row(components, src_x, &mut sss0, mmk);
conv_32_components_one_row(components, src_x + 32, &mut sss1, mmk);
}
}
sss0 = _mm256_srai_epi32::<PRECISION>(sss0);
sss1 = _mm256_srai_epi32::<PRECISION>(sss1);
sss2 = _mm256_srai_epi32::<PRECISION>(sss2);
sss3 = _mm256_srai_epi32::<PRECISION>(sss3);
let (dst0, dst1) = dst_chunk.split_at_mut(32);
for (mut sss, dst) in [(sss0, dst0), (sss1, dst1)] {
sss = sss.map(|v| _mm256_srai_epi32::<PRECISION>(v));
let half0 = _mm256_packs_epi32(sss[0], sss[1]);
let half1 = _mm256_packs_epi32(sss[2], sss[3]);
let components = _mm256_packus_epi16(half0, half1);
let dst_ptr = dst.as_mut_ptr() as *mut __m256i;
_mm256_storeu_si256(dst_ptr, components);
}
sss0 = _mm256_packs_epi32(sss0, sss1);
sss2 = _mm256_packs_epi32(sss2, sss3);
sss0 = _mm256_packus_epi16(sss0, sss2);
src_x += 64;
}
// 32 components in one register
dst_u8 = dst_chunks_64.into_remainder();
let mut dst_chunks_32 = dst_u8.chunks_exact_mut(32);
for dst_chunk in &mut dst_chunks_32 {
let mut sss = [initial_256; 4];
let coeffs_chunks = coeffs.chunks_exact(2);
let coeffs_reminder = coeffs_chunks.remainder();
for (src_rows, two_coeffs) in src_view.iter_2_rows(y_start, max_rows).zip(coeffs_chunks) {
let components1 = T::components(src_rows[0]); // top line
let components2 = T::components(src_rows[1]); // bottom line
// Load two coefficients at once
let mmk = simd_utils::mm256_load_and_clone_i16x2(two_coeffs);
conv_32_components_two_rows(components1, components2, src_x, &mut sss, mmk);
}
if let Some(&k) = coeffs_reminder.first() {
if let Some(s_row) = src_view.iter_rows(y_last).next() {
let components = T::components(s_row);
let mmk = _mm256_set1_epi32(k as i32);
conv_32_components_one_row(components, src_x, &mut sss, mmk);
}
}
let dst_ptr = dst_chunk.as_mut_ptr() as *mut __m256i;
_mm256_storeu_si256(dst_ptr, sss0);
sss = sss.map(|v| _mm256_srai_epi32::<PRECISION>(v));
let half0 = _mm256_packs_epi32(sss[0], sss[1]);
let half1 = _mm256_packs_epi32(sss[2], sss[3]);
let dst = _mm256_packus_epi16(half0, half1);
_mm256_storeu_si256(dst_ptr, dst);
src_x += 32;
}
// 8 components in half of SSE register
dst_u8 = dst_chunks_32.into_remainder();
let mut dst_chunks_8 = dst_u8.chunks_exact_mut(8);
for dst_chunk in &mut dst_chunks_8 {
let mut sss0 = initial; // left row
let mut sss1 = initial; // right row
let coeffs_chunks = coeffs.chunks_exact(2);
let coeffs_reminder = coeffs_chunks.remainder();
for (src_rows, two_coeffs) in src_view.iter_2_rows(y_start, max_rows).zip(coeffs_chunks) {
let components1 = T::components(src_rows[0]);
let components2 = T::components(src_rows[1]);
// Load two coefficients at once
let mmk = simd_utils::mm_load_and_clone_i16x2(two_coeffs);
let source1 = simd_utils::loadl_epi64(components1, src_x); // top line
let source2 = simd_utils::loadl_epi64(components2, src_x); // bottom line
let source = _mm_unpacklo_epi8(source1, source2);
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
}
if let Some(&k) = coeffs_reminder.first() {
if let Some(s_row) = src_view.iter_rows(y_last).next() {
let components = T::components(s_row);
let mmk = _mm_set1_epi32(k as i32);
let source1 = simd_utils::loadl_epi64(components, src_x); // top line
let source2 = _mm_setzero_si128(); // bottom line is empty
let source = _mm_unpacklo_epi8(source1, source2);
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
}
}
sss0 = _mm_srai_epi32::<PRECISION>(sss0);
sss1 = _mm_srai_epi32::<PRECISION>(sss1);
sss0 = _mm_packs_epi32(sss0, sss1);
sss0 = _mm_packus_epi16(sss0, sss0);
let dst_ptr = dst_chunk.as_mut_ptr() as *mut __m128i;
_mm_storel_epi64(dst_ptr, sss0);
src_x += 8;
}
dst_u8 = dst_chunks_8.into_remainder();
let mut dst_chunks_4 = dst_u8.chunks_exact_mut(4);
if let Some(dst_chunk) = dst_chunks_4.next() {
let mut sss = initial;
let coeffs_chunks = coeffs.chunks_exact(2);
let coeffs_reminder = coeffs_chunks.remainder();
for (src_rows, two_coeffs) in src_view.iter_2_rows(y_start, max_rows).zip(coeffs_chunks) {
let components1 = T::components(src_rows[0]);
let components2 = T::components(src_rows[1]);
// Load two coefficients at once
let two_coeffs = simd_utils::mm_load_and_clone_i16x2(two_coeffs);
let row1 = simd_utils::mm_cvtsi32_si128_from_u8(components1, src_x); // top line
let row2 = simd_utils::mm_cvtsi32_si128_from_u8(components2, src_x); // bottom line
let pixels_u8 = _mm_unpacklo_epi8(row1, row2);
let pixels_i16 = _mm_unpacklo_epi8(pixels_u8, _mm_setzero_si128());
sss = _mm_add_epi32(sss, _mm_madd_epi16(pixels_i16, two_coeffs));
}
if let Some(&k) = coeffs_reminder.first() {
if let Some(s_row) = src_view.iter_rows(y_last).next() {
let components = T::components(s_row);
let pix = simd_utils::mm_cvtepu8_epi32_from_u8(components, src_x);
let mmk = _mm_set1_epi32(k as i32);
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
}
}
sss = _mm_srai_epi32::<PRECISION>(sss);
sss = _mm_packs_epi32(sss, sss);
let dst_ptr = dst_chunk.as_mut_ptr() as *mut i32;
dst_ptr.write_unaligned(_mm_cvtsi128_si32(_mm_packus_epi16(sss, sss)));
src_x += 4;
}
dst_u8 = dst_chunks_4.into_remainder();
if !dst_u8.is_empty() {
native::convolution_by_u8(
src_view,
normalizer,
1 << (PRECISION as u8 - 1),
dst_u8,
src_x,
y_start,
coeffs,
);
}
sse4::conv_less_that_32_components::<T, PRECISION>(
src_view,
dst_u8,
src_x,
coeffs_chunk,
normalizer,
);
}
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn conv_32_components_two_rows(
components1: &[u8],
components2: &[u8],
src_x: usize,
sss: &mut [__m256i; 4],
mmk: __m256i,
) {
let zero_256 = _mm256_setzero_si256();
let source1 = simd_utils::loadu_si256(components1, src_x); // top line
let source2 = simd_utils::loadu_si256(components2, src_x); // bottom line
conv_loaded_32_components(source1, source2, sss, mmk, zero_256);
}
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn conv_32_components_one_row(
components1: &[u8],
src_x: usize,
sss: &mut [__m256i; 4],
mmk: __m256i,
) {
let zero_256 = _mm256_setzero_si256();
let source1 = simd_utils::loadu_si256(components1, src_x); // top line
conv_loaded_32_components(source1, zero_256, sss, mmk, zero_256);
}
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn conv_loaded_32_components(
source1: __m256i,
source2: __m256i,
sss: &mut [__m256i; 4],
mmk: __m256i,
zero_256: __m256i,
) {
let source = _mm256_unpacklo_epi8(source1, source2);
let pix = _mm256_unpacklo_epi8(source, zero_256);
sss[0] = _mm256_add_epi32(sss[0], _mm256_madd_epi16(pix, mmk));
let pix = _mm256_unpackhi_epi8(source, zero_256);
sss[1] = _mm256_add_epi32(sss[1], _mm256_madd_epi16(pix, mmk));
let source = _mm256_unpackhi_epi8(source1, source2);
let pix = _mm256_unpacklo_epi8(source, zero_256);
sss[2] = _mm256_add_epi32(sss[2], _mm256_madd_epi16(pix, mmk));
let pix = _mm256_unpackhi_epi8(source, zero_256);
sss[3] = _mm256_add_epi32(sss[3], _mm256_madd_epi16(pix, mmk));
}
src/convolution/vertical_u8/sse4.rs
+111 -108
View File
@@ -1,7 +1,7 @@
use core::arch::x86_64::*;
use super::native;
use crate::convolution::optimisations::{CoefficientsI16Chunk, Normalizer16};
use crate::convolution::vertical_u8::native;
use crate::pixels::InnerPixel;
use crate::{simd_utils, ImageView, ImageViewMut};
@@ -34,7 +34,6 @@ fn vert_convolution_p<T, const PRECISION: i32>(
{
let coefficients_chunks = normalizer.chunks();
let src_x = offset as usize * T::count_of_components();
let dst_rows = dst_view.iter_rows_mut(0);
let dst_row_and_coefs = dst_rows.zip(coefficients_chunks);
@@ -51,6 +50,7 @@ fn vert_convolution_p<T, const PRECISION: i32>(
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
src_view: &impl ImageView<Pixel = T>,
@@ -71,14 +71,8 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
let mut dst_chunks_32 = dst_u8.chunks_exact_mut(32);
for dst_chunk in &mut dst_chunks_32 {
let mut sss0 = initial;
let mut sss1 = initial;
let mut sss2 = initial;
let mut sss3 = initial;
let mut sss4 = initial;
let mut sss5 = initial;
let mut sss6 = initial;
let mut sss7 = initial;
let mut sss0 = [initial; 4];
let mut sss1 = [initial; 4];
let coeffs_chunks = coeffs.chunks_exact(2);
let coeffs_reminder = coeffs_chunks.remainder();
@@ -90,35 +84,8 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
// Load two coefficients at once
let mmk = simd_utils::mm_load_and_clone_i16x2(two_coeffs);
let source1 = simd_utils::loadu_si128(components1, src_x); // top line
let source2 = simd_utils::loadu_si128(components2, src_x); // bottom line
let source = _mm_unpacklo_epi8(source1, source2);
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
let source = _mm_unpackhi_epi8(source1, source2);
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
sss2 = _mm_add_epi32(sss2, _mm_madd_epi16(pix, mmk));
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
sss3 = _mm_add_epi32(sss3, _mm_madd_epi16(pix, mmk));
let source1 = simd_utils::loadu_si128(components1, src_x + 16); // top line
let source2 = simd_utils::loadu_si128(components2, src_x + 16); // bottom line
let source = _mm_unpacklo_epi8(source1, source2);
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
sss4 = _mm_add_epi32(sss4, _mm_madd_epi16(pix, mmk));
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
sss5 = _mm_add_epi32(sss5, _mm_madd_epi16(pix, mmk));
let source = _mm_unpackhi_epi8(source1, source2);
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
sss6 = _mm_add_epi32(sss6, _mm_madd_epi16(pix, mmk));
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
sss7 = _mm_add_epi32(sss7, _mm_madd_epi16(pix, mmk));
sss0 = conv_16_components_two_rows(components1, components2, src_x, sss0, mmk);
sss1 = conv_16_components_two_rows(components1, components2, src_x + 16, sss1, mmk);
}
if let Some(&k) = coeffs_reminder.first() {
@@ -126,64 +93,49 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
let components = T::components(s_row);
let mmk = _mm_set1_epi32(k as i32);
let source1 = simd_utils::loadu_si128(components, src_x); // top line
let source = _mm_unpacklo_epi8(source1, _mm_setzero_si128());
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
let source = _mm_unpackhi_epi8(source1, _mm_setzero_si128());
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
sss2 = _mm_add_epi32(sss2, _mm_madd_epi16(pix, mmk));
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
sss3 = _mm_add_epi32(sss3, _mm_madd_epi16(pix, mmk));
let source1 = simd_utils::loadu_si128(components, src_x + 16); // top line
let source = _mm_unpacklo_epi8(source1, _mm_setzero_si128());
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
sss4 = _mm_add_epi32(sss4, _mm_madd_epi16(pix, mmk));
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
sss5 = _mm_add_epi32(sss5, _mm_madd_epi16(pix, mmk));
let source = _mm_unpackhi_epi8(source1, _mm_setzero_si128());
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
sss6 = _mm_add_epi32(sss6, _mm_madd_epi16(pix, mmk));
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
sss7 = _mm_add_epi32(sss7, _mm_madd_epi16(pix, mmk));
sss0 = convolution_16_components_one_row(components, src_x, sss0, mmk);
sss1 = convolution_16_components_one_row(components, src_x + 16, sss1, mmk);
}
}
sss0 = _mm_srai_epi32::<PRECISION>(sss0);
sss1 = _mm_srai_epi32::<PRECISION>(sss1);
sss2 = _mm_srai_epi32::<PRECISION>(sss2);
sss3 = _mm_srai_epi32::<PRECISION>(sss3);
sss4 = _mm_srai_epi32::<PRECISION>(sss4);
sss5 = _mm_srai_epi32::<PRECISION>(sss5);
sss6 = _mm_srai_epi32::<PRECISION>(sss6);
sss7 = _mm_srai_epi32::<PRECISION>(sss7);
sss0 = _mm_packs_epi32(sss0, sss1);
sss2 = _mm_packs_epi32(sss2, sss3);
sss0 = _mm_packus_epi16(sss0, sss2);
let dst_ptr = dst_chunk.as_mut_ptr() as *mut __m128i;
_mm_storeu_si128(dst_ptr, sss0);
sss4 = _mm_packs_epi32(sss4, sss5);
sss6 = _mm_packs_epi32(sss6, sss7);
sss4 = _mm_packus_epi16(sss4, sss6);
let dst_ptr = dst_ptr.add(1);
_mm_storeu_si128(dst_ptr, sss4);
let (dst0, dst1) = dst_chunk.split_at_mut(16);
for (mut sss, dst) in [(sss0, dst0), (sss1, dst1)] {
sss = sss.map(|v| _mm_srai_epi32::<PRECISION>(v));
let half0 = _mm_packs_epi32(sss[0], sss[1]);
let half1 = _mm_packs_epi32(sss[2], sss[3]);
let components = _mm_packus_epi16(half0, half1);
let dst_ptr = dst.as_mut_ptr() as *mut __m128i;
_mm_storeu_si128(dst_ptr, components);
}
src_x += 32;
}
dst_u8 = dst_chunks_32.into_remainder();
conv_less_that_32_components::<T, PRECISION>(src_view, dst_u8, src_x, coeffs_chunk, normalizer);
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn conv_less_that_32_components<T, const PRECISION: i32>(
src_view: &impl ImageView<Pixel = T>,
mut dst_u8: &mut [u8],
mut src_x: usize,
coeffs_chunk: &CoefficientsI16Chunk,
normalizer: &Normalizer16,
) where
T: InnerPixel<Component = u8>,
{
let y_start = coeffs_chunk.start;
let coeffs = coeffs_chunk.values();
let max_rows = coeffs.len() as u32;
let y_last = (y_start + max_rows).max(1) - 1;
let initial = _mm_set1_epi32(1 << (PRECISION - 1));
let zero_128 = _mm_setzero_si128();
let mut dst_chunks_8 = dst_u8.chunks_exact_mut(8);
for dst_chunk in &mut dst_chunks_8 {
let mut sss0 = initial; // left row
let mut sss1 = initial; // right row
let mut sss = [initial; 2];
let coeffs_chunks = coeffs.chunks_exact(2);
let coeffs_reminder = coeffs_chunks.remainder();
@@ -196,12 +148,7 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
let source1 = simd_utils::loadl_epi64(components1, src_x); // top line
let source2 = simd_utils::loadl_epi64(components2, src_x); // bottom line
let source = _mm_unpacklo_epi8(source1, source2);
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
sss = conf_8_components_two_rows(source1, source2, zero_128, mmk, sss);
}
if let Some(&k) = coeffs_reminder.first() {
@@ -210,22 +157,15 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
let mmk = _mm_set1_epi32(k as i32);
let source1 = simd_utils::loadl_epi64(components, src_x); // top line
let source = _mm_unpacklo_epi8(source1, _mm_setzero_si128());
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
sss = conf_8_components_two_rows(source1, zero_128, zero_128, mmk, sss);
}
}
sss0 = _mm_srai_epi32::<PRECISION>(sss0);
sss1 = _mm_srai_epi32::<PRECISION>(sss1);
sss0 = _mm_packs_epi32(sss0, sss1);
sss0 = _mm_packus_epi16(sss0, sss0);
sss = sss.map(|v| _mm_srai_epi32::<PRECISION>(v));
let v_i16x8 = _mm_packs_epi32(sss[0], sss[1]);
let v_u8x16 = _mm_packus_epi16(v_i16x8, v_i16x8);
let dst_ptr = dst_chunk.as_mut_ptr() as *mut __m128i;
_mm_storel_epi64(dst_ptr, sss0);
_mm_storel_epi64(dst_ptr, v_u8x16);
src_x += 8;
}
@@ -248,7 +188,7 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
let source2 = simd_utils::mm_cvtsi32_si128_from_u8(components2, src_x); // bottom line
let source = _mm_unpacklo_epi8(source1, source2);
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
let pix = _mm_unpacklo_epi8(source, zero_128);
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
}
@@ -262,11 +202,9 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
}
sss = _mm_srai_epi32::<PRECISION>(sss);
sss = _mm_packs_epi32(sss, sss);
let dst_ptr = dst_chunk.as_mut_ptr() as *mut i32;
dst_ptr.write_unaligned(_mm_cvtsi128_si32(_mm_packus_epi16(sss, sss)));
src_x += 4;
}
@@ -283,3 +221,68 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
unsafe fn conv_16_components_two_rows(
components1: &[u8],
components2: &[u8],
src_x: usize,
sss: [__m128i; 4],
mmk: __m128i,
) -> [__m128i; 4] {
let zero_128 = _mm_setzero_si128();
let source1 = simd_utils::loadu_si128(components1, src_x); // top line
let source2 = simd_utils::loadu_si128(components2, src_x); // bottom line
conv_loaded_16_components(source1, source2, sss, mmk, zero_128)
}
#[inline]
#[target_feature(enable = "sse4.1")]
unsafe fn convolution_16_components_one_row(
components1: &[u8],
src_x: usize,
sss: [__m128i; 4],
mmk: __m128i,
) -> [__m128i; 4] {
let zero_128 = _mm_setzero_si128();
let source1 = simd_utils::loadu_si128(components1, src_x); // top line
conv_loaded_16_components(source1, zero_128, sss, mmk, zero_128)
}
#[inline]
#[target_feature(enable = "sse4.1")]
unsafe fn conv_loaded_16_components(
source1: __m128i,
source2: __m128i,
sss: [__m128i; 4],
mmk: __m128i,
zero: __m128i,
) -> [__m128i; 4] {
let r0 = conf_8_components_two_rows(source1, source2, zero, mmk, [sss[0], sss[1]]);
let source = _mm_unpackhi_epi8(source1, source2);
let pix = _mm_unpacklo_epi8(source, zero);
let r3 = _mm_add_epi32(sss[2], _mm_madd_epi16(pix, mmk));
let pix = _mm_unpackhi_epi8(source, zero);
let r4 = _mm_add_epi32(sss[3], _mm_madd_epi16(pix, mmk));
[r0[0], r0[1], r3, r4]
}
#[inline]
#[target_feature(enable = "sse4.1")]
unsafe fn conf_8_components_two_rows(
source1: __m128i,
source2: __m128i,
zero: __m128i,
mmk: __m128i,
mut sss: [__m128i; 2],
) -> [__m128i; 2] {
let source = _mm_unpacklo_epi8(source1, source2);
let pix = _mm_unpacklo_epi8(source, zero);
sss[0] = _mm_add_epi32(sss[0], _mm_madd_epi16(pix, mmk));
let pix = _mm_unpackhi_epi8(source, zero);
sss[1] = _mm_add_epi32(sss[1], _mm_madd_epi16(pix, mmk));
sss
}
src/resizer.rs
+1 -1
View File
@@ -305,7 +305,7 @@ impl Resizer {
/// # Safety
/// This is unsafe because this method allows you to set a CPU extension
/// that is not supported by your CPU.
/// not supported by your CPU.
pub unsafe fn set_cpu_extensions(&mut self, extensions: CpuExtensions) {
self.cpu_extensions = extensions;
self.mul_div.set_cpu_extensions(extensions);