mirror of
https://github.com/Cykooz/fast_image_resize.git
synced 2026-10-08 01:11:09 +00:00
Added optimisation for convolution of U8x2 images with helps of SSE4.1.
This commit is contained in:
Kirill Kuzminykh
+24
-20
@@ -1,23 +1,27 @@
|
||||
## [Unreleased] - ReleaseDate
|
||||
|
||||
- Added optimisation for convolution of `U8x2` images with helps of `SSE4.1`.
|
||||
|
||||
## [0.9.1] - 2022-05-12
|
||||
|
||||
- Added optimisation for processing ``U8x2`` images by ``MulDiv`` with
|
||||
helps of ``SSE4.1`` and ``AVX2`` instructions.
|
||||
- Added optimisation for convolution of ``U16x2`` images with helps of
|
||||
``AVX2`` instructions.
|
||||
- Added optimisation for processing `U8x2` images by `MulDiv` with
|
||||
helps of `SSE4.1` and `AVX2` instructions.
|
||||
- Added optimisation for convolution of `U16x2` images with helps of
|
||||
`AVX2` instructions.
|
||||
|
||||
## [0.9.0] - 2022-05-01
|
||||
|
||||
- Added support of new type of pixels ``PixelType::U8x2``.
|
||||
- Added into ``MulDiv`` support of images with pixel type ``U8x2``.
|
||||
- Added method ``Image::into_vec(self) -> Vec<u8>``
|
||||
- Added support of new type of pixels `PixelType::U8x2`.
|
||||
- Added into `MulDiv` support of images with pixel type `U8x2`.
|
||||
- Added method `Image::into_vec(self) -> Vec<u8>`
|
||||
([#7](https://github.com/Cykooz/fast_image_resize/pull/7)).
|
||||
|
||||
## [0.8.0] - 2022-03-23
|
||||
|
||||
- Added optimisation for convolution of U16x3 images with helps of ``SSE4.1``
|
||||
and ``AVX2`` instructions.
|
||||
- Added optimisation for convolution of U16x3 images with helps of `SSE4.1`
|
||||
and `AVX2` instructions.
|
||||
- Added partial optimisation for convolution of U8 images with helps of
|
||||
``SSE4.1`` instructions.
|
||||
`SSE4.1` instructions.
|
||||
- Allowed to create an instance of `Image`, `ImageVew` and `ImageViewMut`
|
||||
from a buffer larger than necessary
|
||||
([#5](https://github.com/Cykooz/fast_image_resize/issues/5)).
|
||||
@@ -34,17 +38,17 @@
|
||||
## [0.6.0] - 2022-01-12
|
||||
|
||||
- Added optimisation of multiplying and dividing image by alpha channel with helps
|
||||
of ``SSE4.1`` instructions.
|
||||
of `SSE4.1` instructions.
|
||||
- Improved performance of dividing image by alpha channel without forced
|
||||
SIMD instructions.
|
||||
- Breaking changes:
|
||||
- Deleted variant `SSE2` from enum ``CpuExtensions``.
|
||||
- Deleted variant `SSE2` from enum `CpuExtensions`.
|
||||
|
||||
## [0.5.3] - 2021-12-14
|
||||
|
||||
- Added optimisation of convolution U8x3 images with helps of ``AVX2`` instructions.
|
||||
- Fixed error in code for convolution U8x4 images with helps of ``SSE4.1`` instructions.
|
||||
- Fixed error in code for convolution U8 images with helps of ``AVX2`` instructions.
|
||||
- Added optimisation of convolution U8x3 images with helps of `AVX2` instructions.
|
||||
- Fixed error in code for convolution U8x4 images with helps of `SSE4.1` instructions.
|
||||
- Fixed error in code for convolution U8 images with helps of `AVX2` instructions.
|
||||
|
||||
## [0.5.2] - 2021-11-26
|
||||
|
||||
@@ -70,16 +74,16 @@
|
||||
|
||||
## [0.4.1] - 2021-11-13
|
||||
|
||||
- Added optimisation of convolution grayscale images (U8) with helps of ``AVX2`` instructions.
|
||||
- Added optimisation of convolution grayscale images (U8) with helps of `AVX2` instructions.
|
||||
|
||||
## [0.4.0] - 2021-10-23
|
||||
|
||||
- Added support of new type of pixels `PixelType::U8` (without forced SIMD).
|
||||
- Breaking changes:
|
||||
- ``ImageData`` renamed into ``Image``.
|
||||
- ``SrcImageView`` and ``DstImageView`` replaced by ``ImageView``
|
||||
and ``ImageViewMut``.
|
||||
- Method ``Resizer.resize()`` now returns ``Result<(), DifferentTypesOfPixelsError>``.
|
||||
- `ImageData` renamed into `Image`.
|
||||
- `SrcImageView` and `DstImageView` replaced by `ImageView`
|
||||
and `ImageViewMut`.
|
||||
- Method `Resizer.resize()` now returns `Result<(), DifferentTypesOfPixelsError>`.
|
||||
|
||||
## [0.3.1] - 2021-10-09
|
||||
|
||||
|
||||
Generated
+25
-25
@@ -621,9 +621,9 @@ checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.1"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
|
||||
checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d"
|
||||
|
||||
[[package]]
|
||||
name = "jobserver"
|
||||
@@ -666,9 +666,9 @@ checksum = "7efd1d698db0759e6ef11a7cd44407407399a910c774dd804c64c032da7826ff"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.125"
|
||||
version = "0.2.126"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b"
|
||||
checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836"
|
||||
|
||||
[[package]]
|
||||
name = "libgit2-sys"
|
||||
@@ -867,9 +867,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.10.0"
|
||||
version = "1.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9"
|
||||
checksum = "7b10983b38c53aebdf33f542c6275b0f58a238129d00c4ae0e6fb59738d783ca"
|
||||
|
||||
[[package]]
|
||||
name = "open"
|
||||
@@ -958,11 +958,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.38"
|
||||
version = "1.0.39"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9027b48e9d4c9175fa2218adf3557f91c1137021739951d4932f5f8268ac48aa"
|
||||
checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f"
|
||||
dependencies = [
|
||||
"unicode-xid",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -976,9 +976,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rayon"
|
||||
version = "1.5.2"
|
||||
version = "1.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd249e82c21598a9a426a4e00dd7adc1d640b22445ec8545feef801d1a74c221"
|
||||
checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"crossbeam-deque",
|
||||
@@ -988,9 +988,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rayon-core"
|
||||
version = "1.9.2"
|
||||
version = "1.9.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f51245e1e62e1f1629cbfec37b5793bbabcaeb90f30e94d2ba03564687353e4"
|
||||
checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f"
|
||||
dependencies = [
|
||||
"crossbeam-channel",
|
||||
"crossbeam-deque",
|
||||
@@ -1069,9 +1069,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ryu"
|
||||
version = "1.0.9"
|
||||
version = "1.0.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f"
|
||||
checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695"
|
||||
|
||||
[[package]]
|
||||
name = "scoped_threadpool"
|
||||
@@ -1111,7 +1111,7 @@ version = "1.0.81"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c"
|
||||
dependencies = [
|
||||
"itoa 1.0.1",
|
||||
"itoa 1.0.2",
|
||||
"ryu",
|
||||
"serde",
|
||||
]
|
||||
@@ -1159,13 +1159,13 @@ checksum = "3bdb25a4593d6656239319426f4025f7a658157e25e89f0e0319d7516d46042d"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.93"
|
||||
version = "1.0.95"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "04066589568b72ec65f42d65a1a52436e954b168773148893c020269563decf2"
|
||||
checksum = "fbaf6116ab8924f39d52792136fb74fd60a80194cf1b1c6ffa6453eef1c3f942"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-xid",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1267,6 +1267,12 @@ version = "0.3.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-normalization"
|
||||
version = "0.1.19"
|
||||
@@ -1288,12 +1294,6 @@ version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04"
|
||||
|
||||
[[package]]
|
||||
name = "url"
|
||||
version = "2.2.2"
|
||||
|
||||
@@ -18,7 +18,7 @@ Supported pixel formats and available optimisations:
|
||||
- AVX2
|
||||
- `U8x2` - two `u8` components per pixel (e.g. LA):
|
||||
- native Rust-code without forced SIMD
|
||||
- SSE4.1 (partial)
|
||||
- SSE4.1
|
||||
- AVX2
|
||||
- `U8x3` - three `u8` components per pixel (e.g. RGB):
|
||||
- native Rust-code without forced SIMD
|
||||
@@ -44,8 +44,8 @@ Environment:
|
||||
- CPU: AMD Ryzen 9 5950X
|
||||
- RAM: DDR4 3800 MHz
|
||||
- Ubuntu 22.04 (linux 5.15.0)
|
||||
- Rust 1.60.0
|
||||
- fast_image_resize = "0.9.1"
|
||||
- Rust 1.61.0
|
||||
- fast_image_resize = "0.9.2"
|
||||
- glassbench = "0.3.1"
|
||||
- `rustflags = ["-C", "llvm-args=-x86-branches-within-32B-boundaries"]`
|
||||
|
||||
@@ -72,11 +72,11 @@ Pipeline:
|
||||
|
||||
| | Nearest | Bilinear | CatmullRom | Lanczos3 |
|
||||
|------------|:-------:|:--------:|:----------:|:--------:|
|
||||
| image | 20.31 | 77.37 | 136.35 | 183.63 |
|
||||
| resize | - | 45.60 | 87.53 | 129.12 |
|
||||
| fir rust | 0.26 | 42.40 | 78.29 | 115.52 |
|
||||
| fir sse4.1 | 0.26 | 26.11 | 39.54 | 53.26 |
|
||||
| fir avx2 | 0.26 | 6.93 | 8.74 | 12.62 |
|
||||
| image | 18.25 | 79.66 | 138.48 | 189.00 |
|
||||
| resize | - | 50.41 | 97.00 | 142.89 |
|
||||
| fir rust | 0.26 | 37.79 | 64.21 | 93.78 |
|
||||
| fir sse4.1 | 0.26 | 26.07 | 39.62 | 53.22 |
|
||||
| fir avx2 | 0.26 | 6.95 | 8.87 | 12.68 |
|
||||
|
||||
### Resize RGBA8 image (U8x4) 4928x3279 => 852x567
|
||||
|
||||
@@ -90,11 +90,11 @@ Pipeline:
|
||||
|
||||
| | Nearest | Bilinear | CatmullRom | Lanczos3 |
|
||||
|------------|:-------:|:--------:|:----------:|:--------:|
|
||||
| image | 20.43 | 78.27 | 131.97 | 185.15 |
|
||||
| resize | - | 53.60 | 102.31 | 152.19 |
|
||||
| fir rust | 0.17 | 33.25 | 46.95 | 67.66 |
|
||||
| fir sse4.1 | 0.17 | 12.06 | 15.78 | 20.67 |
|
||||
| fir avx2 | 0.17 | 9.32 | 11.55 | 15.32 |
|
||||
| image | 18.32 | 75.46 | 128.47 | 181.34 |
|
||||
| resize | - | 49.26 | 94.38 | 138.48 |
|
||||
| fir rust | 0.17 | 33.67 | 47.63 | 67.37 |
|
||||
| fir sse4.1 | 0.17 | 12.21 | 15.89 | 20.61 |
|
||||
| fir avx2 | 0.17 | 9.18 | 11.46 | 15.26 |
|
||||
|
||||
### Resize grayscale image (U8) 4928x3279 => 852x567
|
||||
|
||||
@@ -108,11 +108,11 @@ Pipeline:
|
||||
|
||||
| | Nearest | Bilinear | CatmullRom | Lanczos3 |
|
||||
|------------|:-------:|:--------:|:----------:|:--------:|
|
||||
| image | 16.95 | 47.55 | 74.95 | 102.03 |
|
||||
| resize | - | 16.25 | 32.60 | 55.48 |
|
||||
| fir rust | 0.14 | 13.00 | 14.72 | 21.98 |
|
||||
| fir sse4.1 | 0.14 | 11.02 | 11.07 | 16.60 |
|
||||
| fir avx2 | 0.14 | 6.06 | 4.40 | 7.37 |
|
||||
| image | 15.05 | 44.65 | 70.26 | 95.61 |
|
||||
| resize | - | 16.88 | 32.98 | 56.31 |
|
||||
| fir rust | 0.14 | 13.21 | 14.98 | 21.86 |
|
||||
| fir sse4.1 | 0.14 | 11.22 | 11.23 | 16.46 |
|
||||
| fir avx2 | 0.14 | 6.03 | 4.41 | 7.37 |
|
||||
|
||||
### Resize grayscale image with alpha channel (U8x2) 4928x3279 => 852x567
|
||||
|
||||
@@ -128,10 +128,10 @@ Pipeline:
|
||||
|
||||
| | Nearest | Bilinear | CatmullRom | Lanczos3 |
|
||||
|------------|:-------:|:--------:|:----------:|:--------:|
|
||||
| image | 18.84 | 63.52 | 109.12 | 147.87 |
|
||||
| fir rust | 0.15 | 23.03 | 28.14 | 39.05 |
|
||||
| fir sse4.1 | 0.15 | 18.68 | 20.35 | 27.82 |
|
||||
| fir avx2 | 0.15 | 10.31 | 11.68 | 14.19 |
|
||||
| image | 16.41 | 61.23 | 111.31 | 149.96 |
|
||||
| fir rust | 0.15 | 23.67 | 27.94 | 38.91 |
|
||||
| fir sse4.1 | 0.15 | 11.66 | 13.33 | 16.44 |
|
||||
| fir avx2 | 0.15 | 10.33 | 11.43 | 14.12 |
|
||||
|
||||
### Resize RGB16 image (U16x3) 4928x3279 => 852x567
|
||||
|
||||
@@ -145,11 +145,11 @@ Pipeline:
|
||||
|
||||
| | Nearest | Bilinear | CatmullRom | Lanczos3 |
|
||||
|------------|:-------:|:--------:|:----------:|:--------:|
|
||||
| image | 20.11 | 73.28 | 124.70 | 173.92 |
|
||||
| resize | - | 47.19 | 89.87 | 132.28 |
|
||||
| fir rust | 0.31 | 39.41 | 69.07 | 96.50 |
|
||||
| fir sse4.1 | 0.31 | 22.21 | 35.84 | 50.60 |
|
||||
| fir avx2 | 0.31 | 19.19 | 28.58 | 34.11 |
|
||||
| image | 17.54 | 72.24 | 126.80 | 176.03 |
|
||||
| resize | - | 52.51 | 100.40 | 147.31 |
|
||||
| fir rust | 0.31 | 39.56 | 65.65 | 92.90 |
|
||||
| fir sse4.1 | 0.31 | 22.14 | 35.69 | 50.40 |
|
||||
| fir avx2 | 0.31 | 19.12 | 28.25 | 33.86 |
|
||||
|
||||
## Examples
|
||||
|
||||
|
||||
@@ -8,8 +8,8 @@ use super::{Coefficients, Convolution};
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
mod avx2;
|
||||
mod native;
|
||||
// #[cfg(target_arch = "x86_64")]
|
||||
// mod sse4;
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
mod sse4;
|
||||
|
||||
impl Convolution for U8x2 {
|
||||
fn horiz_convolution(
|
||||
@@ -22,10 +22,8 @@ impl Convolution for U8x2 {
|
||||
match cpu_extensions {
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Avx2 => avx2::horiz_convolution(src_image, dst_image, offset, coeffs),
|
||||
// #[cfg(target_arch = "x86_64")]
|
||||
// CpuExtensions::Sse4_1 => unsafe {
|
||||
// sse4::horiz_convolution(src_image, dst_image, offset, coeffs)
|
||||
// },
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Sse4_1 => sse4::horiz_convolution(src_image, dst_image, offset, coeffs),
|
||||
_ => native::horiz_convolution(src_image, dst_image, offset, coeffs),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,332 @@
|
||||
use std::arch::x86_64::*;
|
||||
|
||||
use crate::convolution::{optimisations, Coefficients};
|
||||
use crate::image_view::{FourRows, FourRowsMut, TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::U8x2;
|
||||
use crate::simd_utils;
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn horiz_convolution(
|
||||
src_image: TypedImageView<U8x2>,
|
||||
mut dst_image: TypedImageViewMut<U8x2>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
let (values, window_size, bounds_per_pixel) =
|
||||
(coeffs.values, coeffs.window_size, coeffs.bounds);
|
||||
|
||||
let normalizer_guard = optimisations::NormalizerGuard16::new(values);
|
||||
let coefficients_chunks = normalizer_guard.normalized_chunks(window_size, &bounds_per_pixel);
|
||||
let dst_height = dst_image.height().get();
|
||||
|
||||
let src_iter = src_image.iter_4_rows(offset, dst_height + offset);
|
||||
let dst_iter = dst_image.iter_4_rows_mut();
|
||||
for (src_rows, dst_rows) in src_iter.zip(dst_iter) {
|
||||
unsafe {
|
||||
horiz_convolution_four_rows(
|
||||
src_rows,
|
||||
dst_rows,
|
||||
&coefficients_chunks,
|
||||
&normalizer_guard,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let mut yy = dst_height - dst_height % 4;
|
||||
while yy < dst_height {
|
||||
unsafe {
|
||||
horiz_convolution_one_row(
|
||||
src_image.get_row(yy + offset).unwrap(),
|
||||
dst_image.get_row_mut(yy).unwrap(),
|
||||
&coefficients_chunks,
|
||||
&normalizer_guard,
|
||||
);
|
||||
}
|
||||
yy += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/// For safety, it is necessary to ensure the following conditions:
|
||||
/// - length of all rows in src_rows must be equal
|
||||
/// - length of all rows in dst_rows must be equal
|
||||
/// - coefficients_chunks.len() == dst_rows.0.len()
|
||||
/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.0.len()
|
||||
/// - precision <= MAX_COEFS_PRECISION
|
||||
#[inline]
|
||||
#[target_feature(enable = "sse4.1")]
|
||||
unsafe fn horiz_convolution_four_rows(
|
||||
src_rows: FourRows<U8x2>,
|
||||
dst_rows: FourRowsMut<U8x2>,
|
||||
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
|
||||
normalizer_guard: &optimisations::NormalizerGuard16,
|
||||
) {
|
||||
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
|
||||
let s_rows = [s_row0, s_row1, s_row2, s_row3];
|
||||
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
|
||||
let d_rows = [d_row0, d_row1, d_row2, d_row3];
|
||||
let precision = normalizer_guard.precision();
|
||||
let initial = _mm_set1_epi32(1 << (precision - 2));
|
||||
|
||||
/*
|
||||
|L A | |L A | |L A | |L A | |L A | |L A | |L A | |L A |
|
||||
|00 01| |02 03| |04 05| |06 07| |08 09| |10 11| |12 13| |14 15|
|
||||
|
||||
Shuffle components with converting from u8 into i16:
|
||||
|
||||
A: |-1 07| |-1 05| |-1 03| |-1 01|
|
||||
L: |-1 06| |-1 04| |-1 02| |-1 00|
|
||||
*/
|
||||
#[rustfmt::skip]
|
||||
let sh1 = _mm_set_epi8(
|
||||
-1, 7, -1, 5, -1, 3, -1, 1, -1, 6, -1, 4, -1, 2, -1, 0,
|
||||
);
|
||||
/*
|
||||
A: |-1 15| |-1 13| |-1 11| |-1 09|
|
||||
L: |-1 14| |-1 12| |-1 10| |-1 08|
|
||||
*/
|
||||
#[rustfmt::skip]
|
||||
let sh2 = _mm_set_epi8(
|
||||
-1, 15, -1, 13, -1, 11, -1, 9, -1, 14, -1, 12, -1, 10, -1, 8,
|
||||
);
|
||||
|
||||
for (dst_x, coeffs_chunk) in coefficients_chunks.iter().enumerate() {
|
||||
let mut x = coeffs_chunk.start as usize;
|
||||
|
||||
let mut sss: [__m128i; 4] = [initial; 4];
|
||||
let coeffs = coeffs_chunk.values;
|
||||
|
||||
let coeffs_by_8 = coeffs.chunks_exact(8);
|
||||
let reminder = coeffs_by_8.remainder();
|
||||
|
||||
for k in coeffs_by_8 {
|
||||
let mmk0 = simd_utils::ptr_i16_to_set1_epi64x(k, 0);
|
||||
let mmk1 = simd_utils::ptr_i16_to_set1_epi64x(k, 4);
|
||||
|
||||
for i in 0..4 {
|
||||
let source = simd_utils::loadu_si128(s_rows[i], x);
|
||||
let pix = _mm_shuffle_epi8(source, sh1);
|
||||
let tmp_sum = _mm_add_epi32(sss[i], _mm_madd_epi16(pix, mmk0));
|
||||
let pix = _mm_shuffle_epi8(source, sh2);
|
||||
sss[i] = _mm_add_epi32(tmp_sum, _mm_madd_epi16(pix, mmk1));
|
||||
}
|
||||
x += 8;
|
||||
}
|
||||
|
||||
let coeffs_by_4 = reminder.chunks_exact(4);
|
||||
let reminder = coeffs_by_4.remainder();
|
||||
|
||||
for k in coeffs_by_4 {
|
||||
let mmk = simd_utils::ptr_i16_to_set1_epi64x(k, 0);
|
||||
|
||||
for i in 0..4 {
|
||||
let source = simd_utils::loadl_epi64(s_rows[i], x);
|
||||
let pix = _mm_shuffle_epi8(source, sh1);
|
||||
sss[i] = _mm_add_epi32(sss[i], _mm_madd_epi16(pix, mmk));
|
||||
}
|
||||
x += 4;
|
||||
}
|
||||
|
||||
let coeffs_by_2 = reminder.chunks_exact(2);
|
||||
let reminder = coeffs_by_2.remainder();
|
||||
|
||||
for k in coeffs_by_2 {
|
||||
let mmk = simd_utils::ptr_i16_to_set1_epi32(k, 0);
|
||||
|
||||
for i in 0..4 {
|
||||
let source = simd_utils::loadl_epi32(s_rows[i], x);
|
||||
let pix = _mm_shuffle_epi8(source, sh1);
|
||||
sss[i] = _mm_add_epi32(sss[i], _mm_madd_epi16(pix, mmk));
|
||||
}
|
||||
x += 2;
|
||||
}
|
||||
|
||||
if let Some(&k) = reminder.get(0) {
|
||||
let mmk = _mm_set1_epi32(k as i32);
|
||||
|
||||
for i in 0..4 {
|
||||
let source = simd_utils::loadl_epi16(s_rows[i], x);
|
||||
let pix = _mm_shuffle_epi8(source, sh1);
|
||||
sss[i] = _mm_add_epi32(sss[i], _mm_madd_epi16(pix, mmk));
|
||||
}
|
||||
}
|
||||
|
||||
for i in 0..4 {
|
||||
set_dst_pixel(sss[i], d_rows[i], dst_x, normalizer_guard);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[target_feature(enable = "sse4.1")]
|
||||
unsafe fn set_dst_pixel(
|
||||
raw: __m128i,
|
||||
d_row: &mut &mut [U8x2],
|
||||
dst_x: usize,
|
||||
normalizer_guard: &optimisations::NormalizerGuard16,
|
||||
) {
|
||||
let l32x2 = _mm_extract_epi64::<0>(raw);
|
||||
let a32x2 = _mm_extract_epi64::<1>(raw);
|
||||
let l32 = ((l32x2 >> 32) as i32).saturating_add((l32x2 & 0xffffffff) as i32);
|
||||
let a32 = ((a32x2 >> 32) as i32).saturating_add((a32x2 & 0xffffffff) as i32);
|
||||
let l8 = normalizer_guard.clip(l32);
|
||||
let a8 = normalizer_guard.clip(a32);
|
||||
d_row.get_unchecked_mut(dst_x).0 = u16::from_le_bytes([l8, a8]);
|
||||
}
|
||||
|
||||
/// For safety, it is necessary to ensure the following conditions:
|
||||
/// - bounds.len() == dst_row.len()
|
||||
/// - coeffs.len() == dst_rows.0.len() * window_size
|
||||
/// - max(bound.start + bound.size for bound in bounds) <= src_row.len()
|
||||
/// - precision <= MAX_COEFS_PRECISION
|
||||
#[inline]
|
||||
#[target_feature(enable = "sse4.1")]
|
||||
unsafe fn horiz_convolution_one_row(
|
||||
src_row: &[U8x2],
|
||||
dst_row: &mut [U8x2],
|
||||
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
|
||||
normalizer_guard: &optimisations::NormalizerGuard16,
|
||||
) {
|
||||
let precision = normalizer_guard.precision();
|
||||
/*
|
||||
|L A | |L A | |L A | |L A | |L A | |L A | |L A | |L A |
|
||||
|00 01| |02 03| |04 05| |06 07| |08 09| |10 11| |12 13| |14 15|
|
||||
|
||||
Scale first four pixels into i16:
|
||||
|
||||
A: |-1 07| |-1 05|
|
||||
L: |-1 06| |-1 04|
|
||||
A: |-1 03| |-1 01|
|
||||
L: |-1 02| |-1 00|
|
||||
*/
|
||||
#[rustfmt::skip]
|
||||
let pix_sh1 = _mm_set_epi8(
|
||||
-1, 7, -1, 5, -1, 6, -1, 4, -1, 3, -1, 1, -1, 2, -1, 0,
|
||||
);
|
||||
/*
|
||||
|C0 | |C1 | |C2 | |C3 | |C4 | |C5 | |C6 | |C7 |
|
||||
|00 01| |02 03| |04 05| |06 07| |08 09| |10 11| |12 13| |14 15|
|
||||
|
||||
Duplicate first four coefficients for A and L components of pixels:
|
||||
|
||||
CA: |07 06| |05 04|
|
||||
CL: |07 06| |05 04|
|
||||
CA: |03 02| |01 00|
|
||||
CL: |03 02| |01 00|
|
||||
*/
|
||||
#[rustfmt::skip]
|
||||
let coeff_sh1 = _mm_set_epi8(
|
||||
7, 6, 5, 4, 7, 6, 5, 4, 3, 2, 1, 0, 3,2, 1, 0,
|
||||
);
|
||||
|
||||
/*
|
||||
|L A | |L A | |L A | |L A | |L A | |L A | |L A | |L A |
|
||||
|00 01| |02 03| |04 05| |06 07| |08 09| |10 11| |12 13| |14 15|
|
||||
|
||||
Scale second four pixels into i16:
|
||||
|
||||
A: |-1 15| |-1 13|
|
||||
L: |-1 14| |-1 12|
|
||||
A: |-1 11| |-1 09|
|
||||
L: |-1 10| |-1 08|
|
||||
*/
|
||||
#[rustfmt::skip]
|
||||
let pix_sh2 = _mm_set_epi8(
|
||||
-1, 15, -1, 13, -1, 14, -1, 12, -1, 11, -1, 9, -1, 10, -1, 8,
|
||||
);
|
||||
/*
|
||||
|C0 | |C1 | |C2 | |C3 | |C4 | |C5 | |C6 | |C7 |
|
||||
|00 01| |02 03| |04 05| |06 07| |08 09| |10 11| |12 13| |14 15|
|
||||
|
||||
Duplicate second four coefficients for A and L components of pixels:
|
||||
|
||||
CA: |15 14| |13 12|
|
||||
CL: |15 14| |13 12|
|
||||
CA: |11 10| |09 08|
|
||||
CL: |11 10| |09 08|
|
||||
*/
|
||||
#[rustfmt::skip]
|
||||
let coeff_sh2 = _mm_set_epi8(
|
||||
15, 14, 13, 12, 15, 14, 13, 12, 11, 10, 9, 8, 11, 10, 9, 8,
|
||||
);
|
||||
|
||||
/*
|
||||
|L A | |L A | |L A | |L A |
|
||||
|00 01| |02 03| |04 05| |06 07| |08 09| |10 11| |12 13| |14 15|
|
||||
|
||||
Scale four pixels into i16:
|
||||
|
||||
A: |-1 07| |-1 05|
|
||||
L: |-1 06| |-1 04|
|
||||
A: |-1 03| |-1 01|
|
||||
L: |-1 02| |-1 00|
|
||||
*/
|
||||
let pix_sh3 = _mm_set_epi8(-1, 7, -1, 5, -1, 6, -1, 4, -1, 3, -1, 1, -1, 2, -1, 0);
|
||||
|
||||
for (dst_x, &coeffs_chunk) in coefficients_chunks.iter().enumerate() {
|
||||
let mut x = coeffs_chunk.start as usize;
|
||||
let mut coeffs = coeffs_chunk.values;
|
||||
|
||||
// Lower part will be added to higher, use only half of the error
|
||||
let mut sss = _mm_set1_epi32(1 << (precision - 2));
|
||||
|
||||
let coeffs_by_8 = coeffs.chunks_exact(8);
|
||||
coeffs = coeffs_by_8.remainder();
|
||||
|
||||
for k in coeffs_by_8 {
|
||||
let ksource = simd_utils::loadu_si128(k, 0);
|
||||
let source = simd_utils::loadu_si128(src_row, x);
|
||||
|
||||
let pix = _mm_shuffle_epi8(source, pix_sh1);
|
||||
let mmk = _mm_shuffle_epi8(ksource, coeff_sh1);
|
||||
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
let pix = _mm_shuffle_epi8(source, pix_sh2);
|
||||
let mmk = _mm_shuffle_epi8(ksource, coeff_sh2);
|
||||
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
x += 8;
|
||||
}
|
||||
|
||||
let coeffs_by_4 = coeffs.chunks_exact(4);
|
||||
let reminder1 = coeffs_by_4.remainder();
|
||||
|
||||
for k in coeffs_by_4 {
|
||||
let mmk = _mm_set_epi16(k[3], k[2], k[3], k[2], k[1], k[0], k[1], k[0]);
|
||||
let source = simd_utils::loadl_epi64(src_row, x);
|
||||
let pix = _mm_shuffle_epi8(source, pix_sh3);
|
||||
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
x += 4
|
||||
}
|
||||
|
||||
if !reminder1.is_empty() {
|
||||
let mut pixels: [i16; 6] = [0; 6];
|
||||
let mut coeffs: [i16; 3] = [0; 3];
|
||||
for (i, &coeff) in reminder1.iter().enumerate() {
|
||||
coeffs[i] = coeff;
|
||||
let pixel: [u8; 2] = (*src_row.get_unchecked(x)).0.to_le_bytes();
|
||||
pixels[i * 2] = pixel[0] as i16;
|
||||
pixels[i * 2 + 1] = pixel[1] as i16;
|
||||
x += 1;
|
||||
}
|
||||
|
||||
let pix = _mm_set_epi16(
|
||||
0, pixels[5], 0, pixels[4], pixels[3], pixels[1], pixels[2], pixels[0],
|
||||
);
|
||||
let mmk = _mm_set_epi16(
|
||||
0, coeffs[2], 0, coeffs[2], coeffs[1], coeffs[0], coeffs[1], coeffs[0],
|
||||
);
|
||||
|
||||
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
|
||||
}
|
||||
|
||||
let lo = _mm_extract_epi64::<0>(sss);
|
||||
let hi = _mm_extract_epi64::<1>(sss);
|
||||
|
||||
let a32 = ((lo >> 32) as i32).saturating_add((hi >> 32) as i32);
|
||||
let l32 = ((lo & 0xffffffff) as i32).saturating_add((hi & 0xffffffff) as i32);
|
||||
let a8 = normalizer_guard.clip(a32);
|
||||
let l8 = normalizer_guard.clip(l32);
|
||||
dst_row.get_unchecked_mut(dst_x).0 = u16::from_le_bytes([l8, a8]);
|
||||
}
|
||||
}
|
||||
@@ -66,6 +66,11 @@ pub unsafe fn ptr_i16_to_256set1_epi32(buf: &[i16], index: usize) -> __m256i {
|
||||
_mm256_set1_epi32(*(buf.get_unchecked(index..).as_ptr() as *const i32))
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub unsafe fn ptr_i16_to_set1_epi64x(buf: &[i16], index: usize) -> __m128i {
|
||||
_mm_set1_epi64x(*(buf.get_unchecked(index..).as_ptr() as *const i64))
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub unsafe fn ptr_i16_to_256set1_epi64x(buf: &[i16], index: usize) -> __m256i {
|
||||
_mm256_set1_epi64x(*(buf.get_unchecked(index..).as_ptr() as *const i64))
|
||||
|
||||
@@ -164,7 +164,7 @@ fn downscale_u8x2() {
|
||||
let mut cpu_extensions_vec = vec![CpuExtensions::None];
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
{
|
||||
// cpu_extensions_vec.push(CpuExtensions::Sse4_1);
|
||||
cpu_extensions_vec.push(CpuExtensions::Sse4_1);
|
||||
cpu_extensions_vec.push(CpuExtensions::Avx2);
|
||||
}
|
||||
for cpu_extensions in cpu_extensions_vec {
|
||||
@@ -186,7 +186,7 @@ fn upscale_u8x2() {
|
||||
let mut cpu_extensions_vec = vec![CpuExtensions::None];
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
{
|
||||
// cpu_extensions_vec.push(CpuExtensions::Sse4_1);
|
||||
cpu_extensions_vec.push(CpuExtensions::Sse4_1);
|
||||
cpu_extensions_vec.push(CpuExtensions::Avx2);
|
||||
}
|
||||
for cpu_extensions in cpu_extensions_vec {
|
||||
|
||||
Reference in New Issue
Block a user