external/fast_image_resize
1
0
Fork 0
mirror of https://github.com/Cykooz/fast_image_resize.git synced 2026-10-08 01:11:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

- Added optimisation of convolution U8x3 images with helps of `AVX2` instructions.

Browse Source 
- Fixed error in code for convolution U8x4 images with helps of ``SSE4.1`` instructions.
- Fixed error in code for convolution U8 images with helps of ``AVX2`` instructions.
This commit is contained in:
Kirill Kuzminykh
2021-12-14 00:17:45 +03:00
parent df1652bad3
commit 044478620d
15 changed files with 987 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+6
View File
@@ -1,3 +1,9 @@
## [Unreleased] - ReleaseDate
- Added optimisation of convolution U8x3 images with helps of ``AVX2`` instructions.
- Fixed error in code for convolution U8x4 images with helps of ``SSE4.1`` instructions.
- Fixed error in code for convolution U8 images with helps of ``AVX2`` instructions.
## [0.5.1] - 2021-11-24
- Fixed compile errors on non-x86 architectures.
Cargo.lock
Generated
+45 -44
View File
@@ -33,15 +33,15 @@ dependencies = [
[[package]]
name = "anyhow"
version = "1.0.44"
version = "1.0.51"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61604a8f862e1d5c3229fdd78f8b02c68dcf73a4c4b05fd636d12240aaa242c1"
checksum = "8b26702f315f53b6071259e15dd9d64528213b44d61de1ec926eca7715d62203"
[[package]]
name = "argh"
version = "0.1.6"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f023c76cd7975f9969f8e29f0e461decbdc7f51048ce43427107a3d192f1c9bf"
checksum = "dbb41d85d92dfab96cb95ab023c265c5e4261bb956c0fb49ca06d90c570f1958"
dependencies = [
"argh_derive",
"argh_shared",
@@ -49,9 +49,9 @@ dependencies = [
[[package]]
name = "argh_derive"
version = "0.1.6"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48ad219abc0c06ca788aface2e3a1970587e3413ab70acd20e54b6ec524c1f8f"
checksum = "be69f70ef5497dd6ab331a50bd95c6ac6b8f7f17a7967838332743fbd58dc3b5"
dependencies = [
"argh_shared",
"heck",
@@ -62,9 +62,9 @@ dependencies = [
[[package]]
name = "argh_shared"
version = "0.1.6"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38de00daab4eac7d753e97697066238d67ce9d7e2d823ab4f72fe14af29f3f33"
checksum = "e6f8c380fa28aa1b36107cd97f0196474bb7241bb95a453c5c01a15ac74b2eac"
[[package]]
name = "autocfg"
@@ -110,9 +110,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]]
name = "cc"
version = "1.0.71"
version = "1.0.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79c2681d6594606957bbb8631c4b90a7fcaaa72cdb714743a437b156d6a7eedd"
checksum = "22a9137b95ea06864e018375b72adfb7db6e6f68cfc8df5a04d00288050485ee"
dependencies = [
"jobserver",
]
@@ -155,9 +155,9 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
[[package]]
name = "crc32fast"
version = "1.2.1"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a"
checksum = "738c290dfaea84fc1ca15ad9c168d083b05a714e1efddd8edaab678dc28d2836"
dependencies = [
"cfg-if",
]
@@ -279,14 +279,15 @@ dependencies = [
[[package]]
name = "csv2svg"
version = "0.1.5"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1b29b8c39dc8a008d498bed2fa5a8ba779eee1d3f2c25fa25380fa109912ca3"
checksum = "a8e2619b51a7edc3570447c19b821b4f8e17b4e0ddab688582f6ceb2a83acf07"
dependencies = [
"anyhow",
"argh",
"chrono",
"cli-log",
"crossterm",
"csv",
"directories-next",
"libc",
@@ -400,9 +401,9 @@ dependencies = [
[[package]]
name = "git2"
version = "0.13.23"
version = "0.13.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a8057932925d3a9d9e4434ea016570d37420ddb1ceed45a174d577f24ed6700"
checksum = "f29229cc1b24c0e6062f6e742aa3e256492a5323365e5ed3413599f8a5eff7d6"
dependencies = [
"bitflags",
"libc",
@@ -413,9 +414,9 @@ dependencies = [
[[package]]
name = "glassbench"
version = "0.3.0"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03f4871371c7f0624fc191afef849664776c4906614eee6a435a9abd89bd871f"
checksum = "18232f8d0a65b776d32b887f2aa3ab0230a761ea2c82e4739ae824ec5f2a27a9"
dependencies = [
"base64",
"chrono",
@@ -550,15 +551,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.104"
version = "0.2.112"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b2f96d100e1cf1929e7719b7edb3b90ab5298072638fccd77be9ce942ecdfce"
checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125"
[[package]]
name = "libgit2-sys"
version = "0.12.24+1.3.0"
version = "0.12.26+1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddbd6021eef06fb289a8f54b3c2acfdd85ff2a585dfbb24b8576325373d2152c"
checksum = "19e1c899248e606fbfe68dcb31d8b0176ebab833b103824af31bddf4b7457494"
dependencies = [
"cc",
"libc",
@@ -621,9 +622,9 @@ checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
[[package]]
name = "memoffset"
version = "0.6.4"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9"
checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
dependencies = [
"autocfg",
]
@@ -793,9 +794,9 @@ checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]]
name = "pkg-config"
version = "0.3.20"
version = "0.3.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c9b1041b4387893b91ee6746cddfc28516aff326a3519fb2adf820932c5e6cb"
checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe"
[[package]]
name = "png"
@@ -811,15 +812,15 @@ dependencies = [
[[package]]
name = "ppv-lite86"
version = "0.2.14"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3ca011bd0129ff4ae15cd04c4eef202cadf6c51c21e47aba319b4e0501db741"
checksum = "ed0cfbc8191465bed66e1718596ee0b0b35d5ee1f41c5df2189d0fe8bde535ba"
[[package]]
name = "proc-macro2"
version = "1.0.30"
version = "1.0.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edc3358ebc67bc8b7fa0c007f945b0b18226f78437d61bec735a9eb96b61ee70"
checksum = "fb37d2df5df740e582f28f8560cf425f52bb267d872fe58358eadb554909f07a"
dependencies = [
"unicode-xid",
]
@@ -944,9 +945,9 @@ dependencies = [
[[package]]
name = "rgb"
version = "0.8.27"
version = "0.8.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fddb3b23626145d1776addfc307e1a1851f60ef6ca64f376bcb889697144cf0"
checksum = "08a9852b34c4628f8ad76797a933577059163651ec5a7dace462adc365bee66c"
dependencies = [
"bytemuck",
]
@@ -968,9 +969,9 @@ dependencies = [
[[package]]
name = "ryu"
version = "1.0.5"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f"
[[package]]
name = "scoped_threadpool"
@@ -986,18 +987,18 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "serde"
version = "1.0.130"
version = "1.0.131"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f12d06de37cf59146fbdecab66aa99f9fe4f78722e3607577a5375d66bd0c913"
checksum = "b4ad69dfbd3e45369132cc64e6748c2d65cdfb001a2b1c232d128b4ad60561c1"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.130"
version = "1.0.131"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7bc1a1ab1961464eae040d96713baa5a724a8152c1222492465b54322ec508b"
checksum = "b710a83c4e0dff6a3d511946b95274ad9ca9e5d3ae497b63fda866ac955358d2"
dependencies = [
"proc-macro2",
"quote",
@@ -1006,9 +1007,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.68"
version = "1.0.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f690853975602e1bfe1ccbf50504d67174e3bcf340f23b5ea9992e0587a52d8"
checksum = "d0ffa0837f2dfa6fb90868c2b5468cad482e175f7dad97e7421951e663f2b527"
dependencies = [
"itoa",
"ryu",
@@ -1049,9 +1050,9 @@ checksum = "3bdb25a4593d6656239319426f4025f7a658157e25e89f0e0319d7516d46042d"
[[package]]
name = "syn"
version = "1.0.80"
version = "1.0.82"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d010a1623fbd906d51d650a9916aaefc05ffa0e4053ff7fe601167f3e715d194"
checksum = "8daf5dd0bb60cbd4137b1b587d2fc0ae729bc07cf01cd70b36a1ed5ade3b9d59"
dependencies = [
"proc-macro2",
"quote",
@@ -1129,9 +1130,9 @@ dependencies = [
[[package]]
name = "tinyvec"
version = "1.5.0"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f83b2a3d4d9091d0abd7eba4dc2710b1718583bd4d8992e2190720ea38f391f7"
checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2"
dependencies = [
"tinyvec_macros",
]
Cargo.toml
+2 -2
View File
@@ -19,10 +19,10 @@ thiserror = "1.0.30"
[dev-dependencies]
glassbench = "0.3.0"
glassbench = "0.3.1"
image = "0.23.14"
resize = "0.7.2"
rgb = "0.8.27"
rgb = "0.8.30"
[[bench]]
README.md
+18 -16
View File
@@ -11,6 +11,7 @@ Supported pixel formats and available optimisations:
- `U8x3` - three `u8` components per pixel (e.g. RGB):
- native Rust-code without forced SIMD
- SSE4.1 (auto-vectorization)
- AVX2
- `U8x4` - four `u8` components per pixel (RGBA, RGBx, CMYK and other):
- native Rust-code without forced SIMD
- SSE4.1
@@ -26,9 +27,9 @@ Environment:
- CPU: Intel(R) Core(TM) i7-6700K CPU @ 4.00GHz
- RAM: DDR4 3000 MHz
- Ubuntu 20.04 (linux 5.11)
- Rust 1.56.1
- fast_image_resize = "0.5"
- glassbench = "0.3.0"
- Rust 1.57.0
- fast_image_resize = "0.5.2"
- glassbench = "0.3.1"
- `rustflags = ["-C", "llvm-args=-x86-branches-within-32B-boundaries"]`
Other Rust libraries used to compare of resizing speed:
@@ -52,10 +53,11 @@ Pipeline:
| | Nearest | Bilinear | CatmullRom | Lanczos3 |
|------------|:-------:|:--------:|:----------:|:--------:|
| image | 108.064 | 196.203 | 279.562 | 363.843 |
| resize | 15.607 | 72.011 | 132.167 | 205.827 |
| fir rust | 0.481 | 53.753 | 86.047 | 117.852 |
| fir sse4.1 | - | 43.236 | 54.124 | 76.111 |
| image | 91.896 | 176.959 | 256.786 | 341.548 |
| resize | 15.453 | 71.340 | 131.232 | 191.469 |
| fir rust | 0.481 | 53.733 | 90.519 | 121.121 |
| fir sse4.1 | - | 43.113 | 53.484 | 75.127 |
| fir avx2 | - | 10.765 | 14.131 | 19.827 |
### Resize RGBA image (U8x4) 4928x3279 => 852x567
@@ -68,11 +70,11 @@ Pipeline:
| | Nearest | Bilinear | CatmullRom | Lanczos3 |
|------------|:-------:|:--------:|:----------:|:--------:|
| image | 110.485 | 191.373 | 267.640 | 348.590 |
| resize | 18.169 | 81.034 | 152.473 | 219.331 |
| fir rust | 13.236 | 63.711 | 88.811 | 117.468 |
| fir sse4.1 | 11.760 | 23.090 | 29.461 | 36.958 |
| fir avx2 | 6.952 | 15.563 | 18.769 | 24.088 |
| image | 98.113 | 177.039 | 254.666 | 337.147 |
| resize | 17.875 | 79.014 | 148.691 | 218.400 |
| fir rust | 13.188 | 63.942 | 89.681 | 119.664 |
| fir sse4.1 | 11.868 | 22.957 | 29.164 | 36.799 |
| fir avx2 | 6.949 | 14.854 | 18.399 | 23.772 |
### Resize grayscale image (U8) 4928x3279 => 852x567
@@ -86,10 +88,10 @@ Pipeline:
| | Nearest | Bilinear | CatmullRom | Lanczos3 |
|----------|:-------:|:--------:|:----------:|:--------:|
| image | 94.548 | 140.978 | 178.725 | 218.875 |
| resize | 9.884 | 26.831 | 54.274 | 82.708 |
| fir rust | 0.196 | 22.045 | 24.734 | 35.630 |
| fir avx2 | - | 9.623 | 7.869 | 11.832 |
| image | 76.981 | 126.595 | 166.765 | 209.593 |
| resize | 9.632 | 24.332 | 47.533 | 80.667 |
| fir rust | 0.197 | 21.773 | 24.476 | 34.909 |
| fir avx2 | - | 9.467 | 7.691 | 11.776 |
## Examples
benches/bench_compare_rgb.rs
+1 -1
View File
@@ -67,7 +67,7 @@ pub fn bench_downscale_rgb(bench: &mut Bench) {
#[cfg(target_arch = "x86_64")]
{
cpu_ext_and_name.push((CpuExtensions::Sse4_1, "sse4.1"));
// cpu_ext_and_name.push((CpuExtensions::Avx2, "avx2"));
cpu_ext_and_name.push((CpuExtensions::Avx2, "avx2"));
}
for (cpu_ext, ext_name) in cpu_ext_and_name {
for alg_name in alg_names {
src/alpha/native/div.rs
+2 -2
View File
@@ -21,8 +21,8 @@ pub(crate) fn divide_alpha_inplace_native(mut image: TypedImageViewMut<U8x4>) {
}
#[inline(always)]
pub(crate) fn div_and_clip(v: u8, rev_alpha: f32) -> u8 {
let res = v as f32 * rev_alpha;
pub(crate) fn div_and_clip(v: u8, recip_alpha: f32) -> u8 {
let res = v as f32 * recip_alpha;
res.min(255.) as u8
}
src/convolution/mod.rs
+1 -1
View File
@@ -125,7 +125,7 @@ pub fn precompute_coefficients(
if ww != 0.0 {
coeffs[cur_index..].iter_mut().for_each(|w| *w /= ww);
}
// Remaining values should stay empty if they are used despite of x_max.
// Remaining values should stay empty if they are used despite x_max.
coeffs.resize(cur_index + window_size, 0.);
bounds.push(Bound {
start: x_min,
src/convolution/optimisations.rs
+14 -2
View File
@@ -99,7 +99,7 @@ impl NormalizerGuard {
pub fn new(mut values: Vec<f64>) -> Self {
let max_weight = values
.iter()
.max_by(|&x, &y| x.partial_cmp(&y).unwrap())
.max_by(|&x, &y| x.partial_cmp(y).unwrap())
.unwrap_or(&0.0)
.to_owned();
@@ -107,11 +107,12 @@ impl NormalizerGuard {
for cur_precision in 0..PRECISION_BITS {
precision = cur_precision;
let next_value: i32 = (max_weight * (1 << (precision + 1)) as f64).round() as i32;
// The next value will be outside of the range, so just stop
// The next value will be outside the range, so just stop
if next_value >= (1 << MAX_COEFS_PRECISION) {
break;
}
}
debug_assert!(precision >= 4); // required for some SIMD optimisations
let len = values.len();
let ptr = values.as_mut_ptr();
@@ -159,3 +160,14 @@ impl NormalizerGuard {
self.precision
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_minimal_precision() {
assert!(NormalizerGuard::new(vec![0.0]).precision() >= 4);
assert!(NormalizerGuard::new(vec![2.0]).precision() >= 4);
}
}
src/convolution/u8x1/avx2.rs
+4 -2
View File
@@ -82,7 +82,8 @@ unsafe fn horiz_convolution_8u4x(
let s_rows = [src_rows.0, src_rows.1, src_rows.2, src_rows.3];
let d_rows = [dst_rows.0, dst_rows.1, dst_rows.2, dst_rows.3];
let zero = _mm_setzero_si128();
let initial = _mm256_set1_epi32(1 << (precision - 1));
// 8 components will be added, use only 1/8 of the error
let initial = _mm256_set1_epi32(1 << (precision - 4));
for (dst_x, coeffs_chunk) in coefficients_chunks.iter().enumerate() {
let coeffs = coeffs_chunk.values;
@@ -150,7 +151,8 @@ unsafe fn horiz_convolution_8u(
precision: u8,
) {
let zero = _mm_setzero_si128();
let initial = _mm256_set1_epi32(1 << (precision - 1));
// 8 components will be added, use only 1/8 of the error
let initial = _mm256_set1_epi32(1 << (precision - 4));
for (dst_x, &coeffs_chunk) in coefficients_chunks.iter().enumerate() {
let coeffs = coeffs_chunk.values;
src/convolution/u8x3/avx2.rs
+602
View File
@@ -0,0 +1,602 @@
use std::arch::x86_64::*;
use std::intrinsics::transmute;
use crate::convolution::optimisations::CoefficientsI16Chunk;
use crate::convolution::{optimisations, Bound, Coefficients};
use crate::image_view::{FourRows, FourRowsMut, TypedImageView, TypedImageViewMut};
use crate::pixels::{Pixel, U8x3};
use crate::simd_utils;
#[inline]
pub(crate) fn horiz_convolution(
src_image: TypedImageView<U8x3>,
mut dst_image: TypedImageViewMut<U8x3>,
offset: u32,
coeffs: Coefficients,
) {
let (values, window_size, bounds_per_pixel) =
(coeffs.values, coeffs.window_size, coeffs.bounds);
let normalizer_guard = optimisations::NormalizerGuard::new(values);
let precision = normalizer_guard.precision();
let coefficients_chunks =
normalizer_guard.normalized_i16_chunks(window_size, &bounds_per_pixel);
let dst_height = dst_image.height().get();
let src_iter = src_image.iter_4_rows(offset, dst_height + offset);
let dst_iter = dst_image.iter_4_rows_mut();
for (src_rows, dst_rows) in src_iter.zip(dst_iter) {
unsafe {
horiz_convolution_8u4x(src_rows, dst_rows, &coefficients_chunks, precision);
}
}
let mut yy = dst_height - dst_height % 4;
while yy < dst_height {
unsafe {
horiz_convolution_8u(
src_image.get_row(yy + offset).unwrap(),
dst_image.get_row_mut(yy).unwrap(),
&coefficients_chunks,
precision,
);
}
yy += 1;
}
}
#[inline]
pub(crate) fn vert_convolution(
src_image: TypedImageView<U8x3>,
mut dst_image: TypedImageViewMut<U8x3>,
coeffs: Coefficients,
) {
let (values, window_size, bounds) = (coeffs.values, coeffs.window_size, coeffs.bounds);
let normalizer_guard = optimisations::NormalizerGuard::new(values);
let precision = normalizer_guard.precision();
let coeffs_i16 = normalizer_guard.normalized_i16();
let coeffs_chunks = coeffs_i16.chunks(window_size);
let dst_rows = dst_image.iter_rows_mut();
for ((&bound, k), dst_row) in bounds.iter().zip(coeffs_chunks).zip(dst_rows) {
unsafe {
vert_convolution_8u(&src_image, dst_row, k, bound, precision);
}
}
}
/// For safety, it is necessary to ensure the following conditions:
/// - length of all rows in src_rows must be equal
/// - length of all rows in dst_rows must be equal
/// - coefficients_chunks.len() == dst_rows.0.len()
/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.0.len()
/// - precision <= MAX_COEFS_PRECISION
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn horiz_convolution_8u4x(
src_rows: FourRows<U8x3>,
dst_rows: FourRowsMut<U8x3>,
coefficients_chunks: &[CoefficientsI16Chunk],
precision: u8,
) {
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
let zero = _mm256_setzero_si256();
let initial = _mm256_set1_epi32(1 << (precision - 1));
let src_width = s_row0.len();
/*
|R G B | |R G B | |R G B | |R G B | |R G B | |R |
|00 01 02| |03 04 05| |06 07 08| |09 10 11| |12 13 14| |15|
Ignore 12-15 bytes in each half of 32-bytes register and
shuffle other components with converting from u8 into i16:
x: |-1 -1| |-1 -1|
B: |-1 05| |-1 02|
G: |-1 04| |-1 01|
R: |-1 03| |-1 00|
*/
#[rustfmt::skip]
let sh1 = _mm256_set_epi8(
-1, -1, -1, -1, -1, 5, -1, 2, -1, 4, -1, 1, -1, 3, -1, 0,
-1, -1, -1, -1, -1, 5, -1, 2, -1, 4, -1, 1, -1, 3, -1, 0,
);
/*
x: |-1 -1| |-1 -1|
B: |-1 11| |-1 08|
G: |-1 10| |-1 07|
R: |-1 09| |-1 06|
*/
#[rustfmt::skip]
let sh2 = _mm256_set_epi8(
-1, -1, -1, -1, -1, 11, -1, 8, -1, 10, -1, 7, -1, 9, -1, 6,
-1, -1, -1, -1, -1, 11, -1, 8, -1, 10, -1, 7, -1, 9, -1, 6,
);
for (dst_x, coeffs_chunk) in coefficients_chunks.iter().enumerate() {
let x_start = coeffs_chunk.start as usize;
let mut x = x_start;
let mut sss0 = initial;
let mut sss1 = initial;
let mut coeffs = coeffs_chunk.values;
// (16 bytes) / (3 bytes per pixel) = 5 whole pixels + 1 byte
let max_x = src_width.saturating_sub(5);
if x < max_x {
let coeffs_by_4 = coeffs.chunks_exact(4);
for k in coeffs_by_4 {
let mmk0 = simd_utils::ptr_i16_to_256set1_epi32(k, 0);
let mmk1 = simd_utils::ptr_i16_to_256set1_epi32(k, 2);
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadu_si128(s_row0, x)),
simd_utils::loadu_si128(s_row1, x),
);
let pix = _mm256_shuffle_epi8(source, sh1);
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk0));
let pix = _mm256_shuffle_epi8(source, sh2);
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk1));
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadu_si128(s_row2, x)),
simd_utils::loadu_si128(s_row3, x),
);
let pix = _mm256_shuffle_epi8(source, sh1);
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk0));
let pix = _mm256_shuffle_epi8(source, sh2);
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk1));
x += 4;
if x >= max_x {
break;
}
}
}
// (8 bytes) / (3 bytes per pixel) = 2 whole pixels + 2 bytes
let max_x = src_width.saturating_sub(2);
if x < max_x {
let coeffs_by_2 = coeffs[x - x_start..].chunks_exact(2);
for k in coeffs_by_2 {
let mmk = simd_utils::ptr_i16_to_256set1_epi32(k, 0);
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadl_epi64(s_row0, x)),
simd_utils::loadl_epi64(s_row1, x),
);
let pix = _mm256_shuffle_epi8(source, sh1);
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadl_epi64(s_row2, x)),
simd_utils::loadl_epi64(s_row3, x),
);
let pix = _mm256_shuffle_epi8(source, sh1);
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
x += 2;
if x >= max_x {
break;
}
}
}
coeffs = coeffs.split_at(x - x_start).1;
for &k in coeffs {
// [16] xx k0 xx k0 xx k0 xx k0 xx k0 xx k0 xx k0 xx k0
let mmk = _mm256_set1_epi32(k as i32);
// [16] xx a0 xx b0 xx g0 xx r0 xx a0 xx b0 xx g0 xx r0
let pix = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::mm_cvtepu8_epi32_u8x3(s_row0, x)),
simd_utils::mm_cvtepu8_epi32_u8x3(s_row1, x),
);
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
let pix = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::mm_cvtepu8_epi32_u8x3(s_row2, x)),
simd_utils::mm_cvtepu8_epi32_u8x3(s_row3, x),
);
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
x += 1;
}
macro_rules! call {
($imm8:expr) => {{
sss0 = _mm256_srai_epi32::<$imm8>(sss0);
sss1 = _mm256_srai_epi32::<$imm8>(sss1);
}};
}
constify_imm8!(precision, call);
sss0 = _mm256_packs_epi32(sss0, zero);
sss1 = _mm256_packs_epi32(sss1, zero);
sss0 = _mm256_packus_epi16(sss0, zero);
sss1 = _mm256_packus_epi16(sss1, zero);
let pixel: u32 = transmute(_mm_cvtsi128_si32(_mm256_extracti128_si256::<0>(sss0)));
let bytes = pixel.to_le_bytes();
d_row0.get_unchecked_mut(dst_x).0 = [bytes[0], bytes[1], bytes[2]];
let pixel: u32 = transmute(_mm_cvtsi128_si32(_mm256_extracti128_si256::<1>(sss0)));
let bytes = pixel.to_le_bytes();
d_row1.get_unchecked_mut(dst_x).0 = [bytes[0], bytes[1], bytes[2]];
let pixel: u32 = transmute(_mm_cvtsi128_si32(_mm256_extracti128_si256::<0>(sss1)));
let bytes = pixel.to_le_bytes();
d_row2.get_unchecked_mut(dst_x).0 = [bytes[0], bytes[1], bytes[2]];
let pixel: u32 = transmute(_mm_cvtsi128_si32(_mm256_extracti128_si256::<1>(sss1)));
let bytes = pixel.to_le_bytes();
d_row3.get_unchecked_mut(dst_x).0 = [bytes[0], bytes[1], bytes[2]];
}
}
/// For safety, it is necessary to ensure the following conditions:
/// - bounds.len() == dst_row.len()
/// - coeffs.len() == dst_rows.0.len() * window_size
/// - max(bound.start + bound.size for bound in bounds) <= src_row.len()
/// - precision <= MAX_COEFS_PRECISION
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn horiz_convolution_8u(
src_row: &[U8x3],
dst_row: &mut [U8x3],
coefficients_chunks: &[CoefficientsI16Chunk],
precision: u8,
) {
#[rustfmt::skip]
let sh1 = _mm256_set_epi8(
-1, -1, -1, -1, -1, 5, -1, 2, -1, 4, -1, 1, -1, 3, -1, 0,
-1, -1, -1, -1, -1, 5, -1, 2, -1, 4, -1, 1, -1, 3, -1, 0,
);
#[rustfmt::skip]
let sh2 = _mm256_set_epi8(
11, 10, 9, 8, 11, 10, 9, 8, 11, 10, 9, 8, 11, 10, 9, 8,
3, 2, 1, 0, 3, 2, 1, 0, 3, 2, 1, 0, 3, 2, 1, 0,
);
#[rustfmt::skip]
let sh3 = _mm256_set_epi8(
-1, -1, -1, -1, -1, 11, -1, 8, -1, 10, -1, 7, -1, 9, -1, 6,
-1, -1, -1, -1, -1, 11, -1, 8, -1, 10, -1, 7, -1, 9, -1, 6,
);
#[rustfmt::skip]
let sh4 = _mm256_set_epi8(
15, 14, 13, 12, 15, 14, 13, 12, 15, 14, 13, 12, 15, 14, 13, 12,
7, 6, 5, 4, 7, 6, 5, 4, 7, 6, 5, 4, 7, 6, 5, 4,
);
#[rustfmt::skip]
let sh5 = _mm256_set_epi8(
-1, -1, -1, -1, -1, 11, -1, 8, -1, 10, -1, 7, -1, 9, -1, 6,
-1, -1, -1, -1, -1, 5, -1, 2, -1, 4, -1, 1, -1, 3, -1, 0,
);
#[rustfmt::skip]
let sh6 = _mm256_set_epi8(
7, 6, 5, 4, 7, 6, 5, 4, 7, 6, 5, 4, 7, 6, 5, 4,
3, 2, 1, 0, 3, 2, 1, 0, 3, 2, 1, 0, 3, 2, 1, 0,
);
/*
Load 8 bytes from memory into low half of 16-bytes register:
|R G B | |R G B | |R G |
|00 01 02| |03 04 05| |06 07| 08 09 10 11 12 13 14 15
Ignore 06-16 bytes in 16-bytes register and
shuffle other components with converting from u8 into i16:
x: |-1 -1| |-1 -1|
B: |-1 05| |-1 02|
G: |-1 04| |-1 01|
R: |-1 03| |-1 00|
*/
let sh7 = _mm_set_epi8(-1, -1, -1, -1, -1, 5, -1, 2, -1, 4, -1, 1, -1, 3, -1, 0);
let src_width = src_row.len();
for (dst_x, &coeffs_chunk) in coefficients_chunks.iter().enumerate() {
let x_start = coeffs_chunk.start as usize;
let mut x = x_start;
let mut coeffs = coeffs_chunk.values;
// (16 bytes) / (3 bytes per pixel) = 5 whole pixels + 1 bytes
// 4 + 5 = 9
let max_x = src_width.saturating_sub(9);
// (32 bytes) / (3 bytes per pixel) = 10 whole pixels + 2 bytes
let mut sss = if coeffs.len() < 8 || x >= max_x {
_mm_set1_epi32(1 << (precision - 1))
} else {
// Lower part will be added to higher, use only half of the error
let mut sss256 = _mm256_set1_epi32(1 << (precision - 2));
let coeffs_by_8 = coeffs.chunks_exact(8);
for k in coeffs_by_8 {
let tmp = simd_utils::loadu_si128(k, 0);
let ksource = _mm256_insertf128_si256::<1>(_mm256_castsi128_si256(tmp), tmp);
let s_upper = simd_utils::loadu_si128(src_row, x);
let s_lower = simd_utils::loadu_si128(src_row, x + 4);
let source = _mm256_inserti128_si256::<1>(_mm256_castsi128_si256(s_upper), s_lower);
let pix = _mm256_shuffle_epi8(source, sh1);
let mmk = _mm256_shuffle_epi8(ksource, sh2);
sss256 = _mm256_add_epi32(sss256, _mm256_madd_epi16(pix, mmk));
let pix = _mm256_shuffle_epi8(source, sh3);
let mmk = _mm256_shuffle_epi8(ksource, sh4);
sss256 = _mm256_add_epi32(sss256, _mm256_madd_epi16(pix, mmk));
x += 8;
if x >= max_x {
break;
}
}
// (16 bytes) / (3 bytes per pixel) = 5 whole pixels + 1 bytes
let max_x = src_width.saturating_sub(5);
if x < max_x {
let coeffs_by_4 = coeffs[x - x_start..].chunks_exact(4);
for k in coeffs_by_4 {
let tmp = simd_utils::loadl_epi64(k, 0);
let ksource = _mm256_insertf128_si256::<1>(_mm256_castsi128_si256(tmp), tmp);
let tmp = simd_utils::loadu_si128(src_row, x);
let source = _mm256_insertf128_si256::<1>(_mm256_castsi128_si256(tmp), tmp);
let pix = _mm256_shuffle_epi8(source, sh5);
let mmk = _mm256_shuffle_epi8(ksource, sh6);
sss256 = _mm256_add_epi32(sss256, _mm256_madd_epi16(pix, mmk));
x += 4;
if x >= max_x {
break;
}
}
}
_mm_add_epi32(
_mm256_extracti128_si256::<0>(sss256),
_mm256_extracti128_si256::<1>(sss256),
)
};
// (8 bytes) / (3 bytes per pixel) = 2 whole pixels + 2 bytes
let max_x = src_width.saturating_sub(2);
if x < max_x {
let coeffs_by_2 = coeffs[x - x_start..].chunks_exact(2);
for k in coeffs_by_2 {
let mmk = simd_utils::ptr_i16_to_set1_epi32(k, 0);
let source = simd_utils::loadl_epi64(src_row, x);
let pix = _mm_shuffle_epi8(source, sh7);
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
x += 2;
if x >= max_x {
break;
}
}
}
coeffs = coeffs.split_at(x - x_start).1;
for &k in coeffs {
let pix = simd_utils::mm_cvtepu8_epi32_u8x3(src_row, x);
let mmk = _mm_set1_epi32(k as i32);
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
x += 1;
}
macro_rules! call {
($imm8:expr) => {{
sss = _mm_srai_epi32::<$imm8>(sss);
}};
}
constify_imm8!(precision, call);
sss = _mm_packs_epi32(sss, sss);
let pixel: u32 = transmute(_mm_cvtsi128_si32(_mm_packus_epi16(sss, sss)));
let bytes = pixel.to_le_bytes();
dst_row.get_unchecked_mut(dst_x).0 = [bytes[0], bytes[1], bytes[2]];
}
}
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn vert_convolution_8u(
src_img: &TypedImageView<U8x3>,
dst_row: &mut [U8x3],
coeffs: &[i16],
bound: Bound,
precision: u8,
) {
let src_width = src_img.width().get() as usize;
let y_start = bound.start;
let y_size = bound.size;
let initial = _mm_set1_epi32(1 << (precision - 1));
let initial_256 = _mm256_set1_epi32(1 << (precision - 1));
let mut x_in_bytes: usize = 0;
let width_in_bytes = src_width * U8x3::size();
let dst_ptr_u8 = dst_row.as_mut_ptr() as *mut u8;
while x_in_bytes < width_in_bytes.saturating_sub(31) {
let mut sss0 = initial_256;
let mut sss1 = initial_256;
let mut sss2 = initial_256;
let mut sss3 = initial_256;
let mut y: u32 = 0;
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, y_start + y_size) {
// Load two coefficients at once
let mmk = simd_utils::ptr_i16_to_256set1_epi32(coeffs, y as usize);
let source1 = simd_utils::loadu_si256_raw(s_row1, x_in_bytes); // top line
let source2 = simd_utils::loadu_si256_raw(s_row2, x_in_bytes); // bottom line
let source = _mm256_unpacklo_epi8(source1, source2);
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
let source = _mm256_unpackhi_epi8(source1, source2);
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
sss2 = _mm256_add_epi32(sss2, _mm256_madd_epi16(pix, mmk));
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
sss3 = _mm256_add_epi32(sss3, _mm256_madd_epi16(pix, mmk));
y += 2;
}
if let Some(s_row) = src_img.get_row(y_start + y) {
let mmk = _mm256_set1_epi32(coeffs[y as usize] as i32);
let source1 = simd_utils::loadu_si256_raw(s_row, x_in_bytes); // top line
let source2 = _mm256_setzero_si256(); // bottom line is empty
let source = _mm256_unpacklo_epi8(source1, source2);
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
let source = _mm256_unpackhi_epi8(source1, _mm256_setzero_si256());
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
sss2 = _mm256_add_epi32(sss2, _mm256_madd_epi16(pix, mmk));
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
sss3 = _mm256_add_epi32(sss3, _mm256_madd_epi16(pix, mmk));
}
macro_rules! call {
($imm8:expr) => {{
sss0 = _mm256_srai_epi32::<$imm8>(sss0);
sss1 = _mm256_srai_epi32::<$imm8>(sss1);
sss2 = _mm256_srai_epi32::<$imm8>(sss2);
sss3 = _mm256_srai_epi32::<$imm8>(sss3);
}};
}
constify_imm8!(precision, call);
sss0 = _mm256_packs_epi32(sss0, sss1);
sss2 = _mm256_packs_epi32(sss2, sss3);
sss0 = _mm256_packus_epi16(sss0, sss2);
let dst_ptr = dst_ptr_u8.add(x_in_bytes) as *mut __m256i;
_mm256_storeu_si256(dst_ptr, sss0);
x_in_bytes += 32;
}
while x_in_bytes < width_in_bytes.saturating_sub(7) {
let mut sss0 = initial; // left row
let mut sss1 = initial; // right row
let mut y: u32 = 0;
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, y_start + y_size) {
// Load two coefficients at once
let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
let source1 = simd_utils::loadl_epi64_raw(s_row1, x_in_bytes); // top line
let source2 = simd_utils::loadl_epi64_raw(s_row2, x_in_bytes); // bottom line
let source = _mm_unpacklo_epi8(source1, source2);
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
y += 2;
}
if let Some(s_row) = src_img.get_row(y_start + y) {
let mmk = _mm_set1_epi32(*coeffs.get_unchecked(y as usize) as i32);
let source1 = simd_utils::loadl_epi64_raw(s_row, x_in_bytes); // top line
let source2 = _mm_setzero_si128(); // bottom line is empty
let source = _mm_unpacklo_epi8(source1, source2);
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
}
macro_rules! call {
($imm8:expr) => {{
sss0 = _mm_srai_epi32::<$imm8>(sss0);
sss1 = _mm_srai_epi32::<$imm8>(sss1);
}};
}
constify_imm8!(precision, call);
sss0 = _mm_packs_epi32(sss0, sss1);
sss0 = _mm_packus_epi16(sss0, sss0);
let dst_ptr = dst_ptr_u8.add(x_in_bytes) as *mut __m128i;
_mm_storel_epi64(dst_ptr, sss0);
x_in_bytes += 8;
}
while x_in_bytes < width_in_bytes.saturating_sub(3) {
let mut sss = initial;
let mut y: u32 = 0;
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, y_start + y_size) {
// Load two coefficients at once
let two_coeffs = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
let row1 = simd_utils::mm_cvtsi32_si128_from_raw(s_row1, x_in_bytes); // top line
let row2 = simd_utils::mm_cvtsi32_si128_from_raw(s_row2, x_in_bytes); // bottom line
let pixels_u8 = _mm_unpacklo_epi8(row1, row2);
let pixels_i16 = _mm_unpacklo_epi8(pixels_u8, _mm_setzero_si128());
sss = _mm_add_epi32(sss, _mm_madd_epi16(pixels_i16, two_coeffs));
y += 2;
}
if let Some(s_row) = src_img.get_row(y_start + y) {
let pix = simd_utils::mm_cvtepu8_epi32_from_raw(s_row, x_in_bytes);
let mmk = _mm_set1_epi32(*coeffs.get_unchecked(y as usize) as i32);
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
}
macro_rules! call {
($imm8:expr) => {{
sss = _mm_srai_epi32::<$imm8>(sss);
}};
}
constify_imm8!(precision, call);
sss = _mm_packs_epi32(sss, sss);
let dst_ptr_i32 = dst_ptr_u8.add(x_in_bytes) as *mut i32;
*dst_ptr_i32 = _mm_cvtsi128_si32(_mm_packus_epi16(sss, sss));
x_in_bytes += 4;
}
if x_in_bytes < width_in_bytes {
let dst_u8 =
std::slice::from_raw_parts_mut(dst_ptr_u8.add(x_in_bytes), width_in_bytes - x_in_bytes);
for dst_pixel in dst_u8 {
let mut ss0 = 1 << (precision - 1);
for (dy, &k) in coeffs.iter().take(y_size as usize).enumerate() {
if let Some(src_row) = src_img.get_row(y_start + dy as u32) {
let src_ptr = src_row.as_ptr() as *const u8;
let src_component = *src_ptr.add(x_in_bytes);
ss0 += src_component as i32 * (k as i32);
}
}
*dst_pixel = optimisations::clip8(ss0, precision);
x_in_bytes += 1;
}
}
}
src/convolution/u8x3/mod.rs
+5 -2
View File
@@ -3,6 +3,7 @@ use crate::image_view::{TypedImageView, TypedImageViewMut};
use crate::pixels::U8x3;
use crate::CpuExtensions;
mod avx2;
mod native;
mod sse4;
@@ -16,7 +17,8 @@ impl Convolution for U8x3 {
) {
match cpu_extensions {
#[cfg(target_arch = "x86_64")]
CpuExtensions::Avx2 | CpuExtensions::Sse4_1 => unsafe {
CpuExtensions::Avx2 => avx2::horiz_convolution(src_image, dst_image, offset, coeffs),
CpuExtensions::Sse4_1 => unsafe {
sse4::horiz_convolution(src_image, dst_image, offset, coeffs)
},
_ => native::horiz_convolution(src_image, dst_image, offset, coeffs),
@@ -31,7 +33,8 @@ impl Convolution for U8x3 {
) {
match cpu_extensions {
#[cfg(target_arch = "x86_64")]
CpuExtensions::Avx2 | CpuExtensions::Sse4_1 => unsafe {
CpuExtensions::Avx2 => avx2::vert_convolution(src_image, dst_image, coeffs),
CpuExtensions::Sse4_1 => unsafe {
sse4::vert_convolution(src_image, dst_image, coeffs)
},
_ => native::vert_convolution(src_image, dst_image, coeffs),
src/convolution/u8x4/avx2.rs
+61 -66
View File
@@ -89,19 +89,18 @@ unsafe fn horiz_convolution_8u4x(
let initial = _mm256_set1_epi32(1 << (precision - 1));
#[rustfmt::skip]
let sh1 = _mm256_set_epi8(
let sh1 = _mm256_set_epi8(
-1, 7, -1, 3, -1, 6, -1, 2, -1, 5, -1, 1, -1, 4, -1, 0,
-1, 7, -1, 3, -1, 6, -1, 2, -1, 5, -1, 1, -1, 4, -1, 0,
);
#[rustfmt::skip]
let sh2 = _mm256_set_epi8(
let sh2 = _mm256_set_epi8(
-1, 15, -1, 11, -1, 14, -1, 10, -1, 13, -1, 9, -1, 12, -1, 8,
-1, 15, -1, 11, -1, 14, -1, 10, -1, 13, -1, 9, -1, 12, -1, 8,
);
for (dst_x, coeffs_chunk) in coefficients_chunks.iter().enumerate() {
let x_start = coeffs_chunk.start as usize;
let mut x: usize = 0;
let mut x = coeffs_chunk.start as usize;
let mut sss0 = initial;
let mut sss1 = initial;
@@ -114,22 +113,22 @@ unsafe fn horiz_convolution_8u4x(
let mmk0 = simd_utils::ptr_i16_to_256set1_epi32(k, 0);
let mmk1 = simd_utils::ptr_i16_to_256set1_epi32(k, 2);
let mut source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadu_si128(s_row0, x + x_start)),
simd_utils::loadu_si128(s_row1, x + x_start),
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadu_si128(s_row0, x)),
simd_utils::loadu_si128(s_row1, x),
);
let mut pix = _mm256_shuffle_epi8(source, sh1);
let pix = _mm256_shuffle_epi8(source, sh1);
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk0));
pix = _mm256_shuffle_epi8(source, sh2);
let pix = _mm256_shuffle_epi8(source, sh2);
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk1));
source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadu_si128(s_row2, x + x_start)),
simd_utils::loadu_si128(s_row3, x + x_start),
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadu_si128(s_row2, x)),
simd_utils::loadu_si128(s_row3, x),
);
pix = _mm256_shuffle_epi8(source, sh1);
let pix = _mm256_shuffle_epi8(source, sh1);
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk0));
pix = _mm256_shuffle_epi8(source, sh2);
let pix = _mm256_shuffle_epi8(source, sh2);
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk1));
x += 4;
@@ -141,40 +140,39 @@ unsafe fn horiz_convolution_8u4x(
for k in coeffs_by_2 {
let mmk = simd_utils::ptr_i16_to_256set1_epi32(k, 0);
let mut pix = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadl_epi64(s_row0, x + x_start)),
simd_utils::loadl_epi64(s_row1, x + x_start),
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadl_epi64(s_row0, x)),
simd_utils::loadl_epi64(s_row1, x),
);
pix = _mm256_shuffle_epi8(pix, sh1);
let pix = _mm256_shuffle_epi8(source, sh1);
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
pix = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadl_epi64(s_row2, x + x_start)),
simd_utils::loadl_epi64(s_row3, x + x_start),
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadl_epi64(s_row2, x)),
simd_utils::loadl_epi64(s_row3, x),
);
pix = _mm256_shuffle_epi8(pix, sh1);
let pix = _mm256_shuffle_epi8(source, sh1);
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
x += 2;
}
for &k in reminder2 {
if let Some(&k) = reminder2.get(0) {
// [16] xx k0 xx k0 xx k0 xx k0 xx k0 xx k0 xx k0 xx k0
let mmk = _mm256_set1_epi32(k as i32);
// [16] xx a0 xx b0 xx g0 xx r0 xx a0 xx b0 xx g0 xx r0
let mut pix = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::mm_cvtepu8_epi32(s_row0, x + x_start)),
simd_utils::mm_cvtepu8_epi32(s_row1, x + x_start),
let pix = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::mm_cvtepu8_epi32(s_row0, x)),
simd_utils::mm_cvtepu8_epi32(s_row1, x),
);
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
pix = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::mm_cvtepu8_epi32(s_row2, x + x_start)),
simd_utils::mm_cvtepu8_epi32(s_row3, x + x_start),
let pix = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::mm_cvtepu8_epi32(s_row2, x)),
simd_utils::mm_cvtepu8_epi32(s_row3, x),
);
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
x += 1;
}
macro_rules! call {
@@ -214,43 +212,42 @@ unsafe fn horiz_convolution_8u(
precision: u8,
) {
#[rustfmt::skip]
let sh1 = _mm256_set_epi8(
let sh1 = _mm256_set_epi8(
-1, 7, -1, 3, -1, 6, -1, 2, -1, 5, -1, 1, -1, 4, -1, 0,
-1, 7, -1, 3, -1, 6, -1, 2, -1, 5, -1, 1, -1, 4, -1, 0,
);
#[rustfmt::skip]
let sh2 = _mm256_set_epi8(
let sh2 = _mm256_set_epi8(
11, 10, 9, 8, 11, 10, 9, 8, 11, 10, 9, 8, 11, 10, 9, 8,
3, 2, 1, 0, 3, 2, 1, 0, 3, 2, 1, 0, 3, 2, 1, 0,
);
#[rustfmt::skip]
let sh3 = _mm256_set_epi8(
let sh3 = _mm256_set_epi8(
-1, 15, -1, 11, -1, 14, -1, 10, -1, 13, -1, 9, -1, 12, -1, 8,
-1, 15, -1, 11, -1, 14, -1, 10, -1, 13, -1, 9, -1, 12, -1, 8,
);
#[rustfmt::skip]
let sh4 = _mm256_set_epi8(
let sh4 = _mm256_set_epi8(
15, 14, 13, 12, 15, 14, 13, 12, 15, 14, 13, 12, 15, 14, 13, 12,
7, 6, 5, 4, 7, 6, 5, 4, 7, 6, 5, 4, 7, 6, 5, 4,
);
#[rustfmt::skip]
let sh5 = _mm256_set_epi8(
let sh5 = _mm256_set_epi8(
-1, 15, -1, 11, -1, 14, -1, 10, -1, 13, -1, 9, -1, 12, -1, 8,
-1, 7, -1, 3, -1, 6, -1, 2, -1, 5, -1, 1, -1, 4, -1, 0,
);
#[rustfmt::skip]
let sh6 = _mm256_set_epi8(
let sh6 = _mm256_set_epi8(
7, 6, 5, 4, 7, 6, 5, 4, 7, 6, 5, 4, 7, 6, 5, 4,
3, 2, 1, 0, 3, 2, 1, 0, 3, 2, 1, 0, 3, 2, 1, 0,
);
let sh7 = _mm_set_epi8(-1, 7, -1, 3, -1, 6, -1, 2, -1, 5, -1, 1, -1, 4, -1, 0);
for (dst_x, &coeffs_chunk) in coefficients_chunks.iter().enumerate() {
let x_start = coeffs_chunk.start as usize;
let mut x: usize = 0;
let mut x = coeffs_chunk.start as usize;
let mut coeffs = coeffs_chunk.values;
let mut sss: __m128i = if coeffs.len() < 8 {
let mut sss = if coeffs.len() < 8 {
_mm_set1_epi32(1 << (precision - 1))
} else {
// Lower part will be added to higher, use only half of the error
@@ -263,14 +260,14 @@ unsafe fn horiz_convolution_8u(
let tmp = simd_utils::loadu_si128(k, 0);
let ksource = _mm256_insertf128_si256::<1>(_mm256_castsi128_si256(tmp), tmp);
let source = simd_utils::loadu_si256(src_row, x + x_start);
let source = simd_utils::loadu_si256(src_row, x);
let mut pix = _mm256_shuffle_epi8(source, sh1);
let mut mmk = _mm256_shuffle_epi8(ksource, sh2);
let pix = _mm256_shuffle_epi8(source, sh1);
let mmk = _mm256_shuffle_epi8(ksource, sh2);
sss256 = _mm256_add_epi32(sss256, _mm256_madd_epi16(pix, mmk));
pix = _mm256_shuffle_epi8(source, sh3);
mmk = _mm256_shuffle_epi8(ksource, sh4);
let pix = _mm256_shuffle_epi8(source, sh3);
let mmk = _mm256_shuffle_epi8(ksource, sh4);
sss256 = _mm256_add_epi32(sss256, _mm256_madd_epi16(pix, mmk));
x += 8;
@@ -283,7 +280,7 @@ unsafe fn horiz_convolution_8u(
let tmp = simd_utils::loadl_epi64(k, 0);
let ksource = _mm256_insertf128_si256::<1>(_mm256_castsi128_si256(tmp), tmp);
let tmp = simd_utils::loadu_si128(src_row, x + x_start);
let tmp = simd_utils::loadu_si128(src_row, x);
let source = _mm256_insertf128_si256::<1>(_mm256_castsi128_si256(tmp), tmp);
let pix = _mm256_shuffle_epi8(source, sh5);
@@ -304,19 +301,17 @@ unsafe fn horiz_convolution_8u(
for k in coeffs_by_2 {
let mmk = simd_utils::ptr_i16_to_set1_epi32(k, 0);
let source = simd_utils::loadl_epi64(src_row, x + x_start);
let source = simd_utils::loadl_epi64(src_row, x);
let pix = _mm_shuffle_epi8(source, sh7);
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
x += 2
}
for &k in reminder1 {
let pix = simd_utils::mm_cvtepu8_epi32(src_row, x + x_start);
if let Some(&k) = reminder1.get(0) {
let pix = simd_utils::mm_cvtepu8_epi32(src_row, x);
let mmk = _mm_set1_epi32(k as i32);
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
x += 1;
}
macro_rules! call {
@@ -364,16 +359,16 @@ unsafe fn vert_convolution_8u(
let source1 = simd_utils::loadu_si256(s_row1, x); // top line
let source2 = simd_utils::loadu_si256(s_row2, x); // bottom line
let mut source = _mm256_unpacklo_epi8(source1, source2);
let mut pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
let source = _mm256_unpacklo_epi8(source1, source2);
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
source = _mm256_unpackhi_epi8(source1, source2);
pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
let source = _mm256_unpackhi_epi8(source1, source2);
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
sss2 = _mm256_add_epi32(sss2, _mm256_madd_epi16(pix, mmk));
pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
sss3 = _mm256_add_epi32(sss3, _mm256_madd_epi16(pix, mmk));
y += 2;
@@ -385,16 +380,16 @@ unsafe fn vert_convolution_8u(
let source1 = simd_utils::loadu_si256(s_row, x); // top line
let source2 = _mm256_setzero_si256(); // bottom line is empty
let mut source = _mm256_unpacklo_epi8(source1, source2);
let mut pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
let source = _mm256_unpacklo_epi8(source1, source2);
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
source = _mm256_unpackhi_epi8(source1, _mm256_setzero_si256());
pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
let source = _mm256_unpackhi_epi8(source1, _mm256_setzero_si256());
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
sss2 = _mm256_add_epi32(sss2, _mm256_madd_epi16(pix, mmk));
pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
sss3 = _mm256_add_epi32(sss3, _mm256_madd_epi16(pix, mmk));
}
@@ -430,9 +425,9 @@ unsafe fn vert_convolution_8u(
let source2 = simd_utils::loadl_epi64(s_row2, x); // bottom line
let source = _mm_unpacklo_epi8(source1, source2);
let mut pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
y += 2;
@@ -445,9 +440,9 @@ unsafe fn vert_convolution_8u(
let source2 = _mm_setzero_si128(); // bottom line is empty
let source = _mm_unpacklo_epi8(source1, source2);
let mut pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
}
src/convolution/u8x4/sse4.rs
+15 -17
View File
@@ -90,8 +90,7 @@ unsafe fn horiz_convolution_8u4x(
let mask = _mm_set_epi8(-1, 7, -1, 3, -1, 6, -1, 2, -1, 5, -1, 1, -1, 4, -1, 0);
for (dst_x, coeffs_chunk) in coefficients_chunks.iter().enumerate() {
let x_start = coeffs_chunk.start as usize;
let mut x: usize = 0;
let mut x: usize = coeffs_chunk.start as usize;
let mut sss0 = initial;
let mut sss1 = initial;
@@ -107,7 +106,7 @@ unsafe fn horiz_convolution_8u4x(
let mmk_hi = simd_utils::ptr_i16_to_set1_epi32(k, 2);
// [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0
let mut source = simd_utils::loadu_si128(s_row0, x + x_start);
let mut source = simd_utils::loadu_si128(s_row0, x);
// [16] a1 a0 b1 b0 g1 g0 r1 r0
let mut pix = _mm_shuffle_epi8(source, mask_lo);
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk_lo));
@@ -115,19 +114,19 @@ unsafe fn horiz_convolution_8u4x(
pix = _mm_shuffle_epi8(source, mask_hi);
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk_hi));
source = simd_utils::loadu_si128(s_row1, x + x_start);
source = simd_utils::loadu_si128(s_row1, x);
pix = _mm_shuffle_epi8(source, mask_lo);
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk_lo));
pix = _mm_shuffle_epi8(source, mask_hi);
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk_hi));
source = simd_utils::loadu_si128(s_row2, x + x_start);
source = simd_utils::loadu_si128(s_row2, x);
pix = _mm_shuffle_epi8(source, mask_lo);
sss2 = _mm_add_epi32(sss2, _mm_madd_epi16(pix, mmk_lo));
pix = _mm_shuffle_epi8(source, mask_hi);
sss2 = _mm_add_epi32(sss2, _mm_madd_epi16(pix, mmk_hi));
source = simd_utils::loadu_si128(s_row3, x + x_start);
source = simd_utils::loadu_si128(s_row3, x);
pix = _mm_shuffle_epi8(source, mask_lo);
sss3 = _mm_add_epi32(sss3, _mm_madd_epi16(pix, mmk_lo));
pix = _mm_shuffle_epi8(source, mask_hi);
@@ -143,20 +142,20 @@ unsafe fn horiz_convolution_8u4x(
let mmk = simd_utils::ptr_i16_to_set1_epi32(k, 0);
// [8] x x x x x x x x a1 b1 g1 r1 a0 b0 g0 r0
let mut pix = simd_utils::loadl_epi64(s_row0, x + x_start);
let mut pix = simd_utils::loadl_epi64(s_row0, x);
// [16] a1 a0 b1 b0 g1 g0 r1 r0
pix = _mm_shuffle_epi8(pix, mask);
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
pix = simd_utils::loadl_epi64(s_row1, x + x_start);
pix = simd_utils::loadl_epi64(s_row1, x);
pix = _mm_shuffle_epi8(pix, mask);
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
pix = simd_utils::loadl_epi64(s_row2, x + x_start);
pix = simd_utils::loadl_epi64(s_row2, x);
pix = _mm_shuffle_epi8(pix, mask);
sss2 = _mm_add_epi32(sss2, _mm_madd_epi16(pix, mmk));
pix = simd_utils::loadl_epi64(s_row3, x + x_start);
pix = simd_utils::loadl_epi64(s_row3, x);
pix = _mm_shuffle_epi8(pix, mask);
sss3 = _mm_add_epi32(sss3, _mm_madd_epi16(pix, mmk));
@@ -229,8 +228,7 @@ unsafe fn horiz_convolution_8u(
let sh7 = _mm_set_epi8(-1, 7, -1, 3, -1, 6, -1, 2, -1, 5, -1, 1, -1, 4, -1, 0);
for (dst_x, &coeffs_chunk) in coefficients_chunks.iter().enumerate() {
let x_start = coeffs_chunk.start as usize;
let mut x: usize = 0;
let mut x: usize = coeffs_chunk.start as usize;
let mut sss = initial;
let coeffs_by_8 = coeffs_chunk.values.chunks_exact(8);
@@ -239,7 +237,7 @@ unsafe fn horiz_convolution_8u(
for k in coeffs_by_8 {
let ksource = simd_utils::loadu_si128(k, 0);
let mut source = simd_utils::loadu_si128(src_row, x + x_start);
let mut source = simd_utils::loadu_si128(src_row, x);
let mut pix = _mm_shuffle_epi8(source, sh1);
let mut mmk = _mm_shuffle_epi8(ksource, sh2);
@@ -249,7 +247,7 @@ unsafe fn horiz_convolution_8u(
mmk = _mm_shuffle_epi8(ksource, sh4);
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
source = simd_utils::loadu_si128(src_row, x + 4 + x_start);
source = simd_utils::loadu_si128(src_row, x + 4);
pix = _mm_shuffle_epi8(source, sh1);
mmk = _mm_shuffle_epi8(ksource, sh5);
@@ -266,7 +264,7 @@ unsafe fn horiz_convolution_8u(
let reminder4 = coeffs_by_4.remainder();
for k in coeffs_by_4 {
let source = simd_utils::loadu_si128(src_row, x + x_start);
let source = simd_utils::loadu_si128(src_row, x);
let ksource = simd_utils::loadl_epi64(k, 0);
let mut pix = _mm_shuffle_epi8(source, sh1);
@@ -285,7 +283,7 @@ unsafe fn horiz_convolution_8u(
for k in coeffs_by_2 {
let mmk = simd_utils::ptr_i16_to_set1_epi32(k, 0);
let source = simd_utils::loadl_epi64(src_row, x + x_start);
let source = simd_utils::loadl_epi64(src_row, x);
let pix = _mm_shuffle_epi8(source, sh7);
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
@@ -293,7 +291,7 @@ unsafe fn horiz_convolution_8u(
}
if let Some(&k) = reminder2.get(0) {
let pix = simd_utils::mm_cvtepu8_epi32(src_row, x + x_start);
let pix = simd_utils::mm_cvtepu8_epi32(src_row, x);
let mmk = _mm_set1_epi32(k as i32);
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
}
src/simd_utils.rs
+34 -2
View File
@@ -1,7 +1,8 @@
use crate::pixels::{U8x4, U8};
use std::arch::x86_64::*;
use std::intrinsics::transmute;
use crate::pixels::{U8x3, U8x4, U8};
#[inline(always)]
pub unsafe fn loadu_si128<T>(buf: &[T], index: usize) -> __m128i {
_mm_loadu_si128(buf.get_unchecked(index..).as_ptr() as *const __m128i)
@@ -12,14 +13,33 @@ pub unsafe fn loadu_si256<T>(buf: &[T], index: usize) -> __m256i {
_mm256_loadu_si256(buf.get_unchecked(index..).as_ptr() as *const __m256i)
}
#[inline(always)]
pub unsafe fn loadu_si256_raw<T>(buf: &[T], offset: usize) -> __m256i {
let ptr = buf.as_ptr() as *const u8;
_mm256_loadu_si256(ptr.add(offset) as *const __m256i)
}
#[inline(always)]
pub unsafe fn loadl_epi64<T>(buf: &[T], index: usize) -> __m128i {
_mm_loadl_epi64(buf.get_unchecked(index..).as_ptr() as *const __m128i)
}
#[inline(always)]
pub unsafe fn loadl_epi64_raw<T>(buf: &[T], offset: usize) -> __m128i {
let ptr = buf.as_ptr() as *const u8;
_mm_loadl_epi64(ptr.add(offset) as *const __m128i)
}
#[inline(always)]
pub unsafe fn mm_cvtepu8_epi32(buf: &[U8x4], index: usize) -> __m128i {
let v: i32 = transmute(*buf.get_unchecked(index));
let v: i32 = transmute(buf.get_unchecked(index).0);
_mm_cvtepu8_epi32(_mm_cvtsi32_si128(v))
}
#[inline(always)]
pub unsafe fn mm_cvtepu8_epi32_u8x3(buf: &[U8x3], index: usize) -> __m128i {
let pixel = buf.get_unchecked(index).0;
let v: i32 = i32::from_le_bytes([pixel[0], pixel[1], pixel[2], 0]);
_mm_cvtepu8_epi32(_mm_cvtsi32_si128(v))
}
@@ -29,6 +49,12 @@ pub unsafe fn mm_cvtepu8_epi32_from_u8(buf: &[U8], index: usize) -> __m128i {
_mm_cvtepu8_epi32(_mm_cvtsi32_si128(*ptr))
}
#[inline(always)]
pub unsafe fn mm_cvtepu8_epi32_from_raw<T>(buf: &[T], offset: usize) -> __m128i {
let ptr = (buf.as_ptr() as *const u8).add(offset) as *const i32;
_mm_cvtepu8_epi32(_mm_cvtsi32_si128(*ptr))
}
#[inline(always)]
pub unsafe fn mm_cvtsi32_si128_from_u32(buf: &[U8x4], index: usize) -> __m128i {
let v: i32 = transmute(*buf.get_unchecked(index));
@@ -41,6 +67,12 @@ pub unsafe fn mm_cvtsi32_si128_from_u8(buf: &[U8], index: usize) -> __m128i {
_mm_cvtsi32_si128(*ptr)
}
#[inline(always)]
pub unsafe fn mm_cvtsi32_si128_from_raw<T>(buf: &[T], offset: usize) -> __m128i {
let ptr = (buf.as_ptr() as *const u8).add(offset) as *const i32;
_mm_cvtsi32_si128(*ptr)
}
#[inline(always)]
pub unsafe fn ptr_i16_to_set1_epi32(buf: &[i16], index: usize) -> __m128i {
_mm_set1_epi32(*(buf.get_unchecked(index..).as_ptr() as *const i32))
tests/resize_tests.rs
+177 -70
View File
@@ -11,38 +11,6 @@ use fast_image_resize::{
Resizer,
};
fn get_source_image_u8x4() -> Image<'static> {
let img = ImageReader::open("./data/nasa-4928x3279.png")
.unwrap()
.decode()
.unwrap();
let width = img.width();
let height = img.height();
Image::from_vec_u8(
NonZeroU32::new(width).unwrap(),
NonZeroU32::new(height).unwrap(),
img.to_rgba8().into_raw(),
PixelType::U8x4,
)
.unwrap()
}
fn get_small_source_image() -> Image<'static> {
let img = ImageReader::open("./data/nasa-852x567.png")
.unwrap()
.decode()
.unwrap();
let width = img.width();
let height = img.height();
Image::from_vec_u8(
NonZeroU32::new(width).unwrap(),
NonZeroU32::new(height).unwrap(),
img.to_rgba8().into_raw(),
PixelType::U8x4,
)
.unwrap()
}
fn get_new_height(src_image: &ImageView, new_width: u32) -> u32 {
let scale = new_width as f32 / src_image.width().get() as f32;
(src_image.height().get() as f32 * scale).round() as u32
@@ -52,6 +20,9 @@ const NEW_WIDTH: u32 = 255;
const NEW_BIG_WIDTH: u32 = 5016;
fn save_result(image: &Image, name: &str) {
if std::env::var("DONT_SAVE_RESULT").unwrap_or_else(|_| "".to_owned()) == "1" {
return;
}
std::fs::create_dir_all("./data/result").unwrap();
let mut file = File::create(format!("./data/result/{}.png", name)).unwrap();
let color_type = match image.pixel_type() {
@@ -70,28 +41,17 @@ fn save_result(image: &Image, name: &str) {
.unwrap();
}
#[test]
fn resize_avx2_lanczos3_upscale_test() {
let image = get_small_source_image();
let mut resizer = Resizer::new(ResizeAlg::Convolution(FilterType::Lanczos3));
unsafe {
resizer.set_cpu_extensions(CpuExtensions::Avx2);
fn image_checksum<const N: usize>(buffer: &[u8]) -> [u32; N] {
let mut res = [0u32; N];
for pixel in buffer.chunks_exact(N) {
res.iter_mut().zip(pixel).for_each(|(d, &s)| *d += s as u32);
}
let new_height = get_new_height(&image.view(), NEW_BIG_WIDTH);
let mut result = Image::new(
NonZeroU32::new(NEW_BIG_WIDTH).unwrap(),
NonZeroU32::new(new_height).unwrap(),
image.pixel_type(),
);
assert!(resizer
.resize(&image.view(), &mut result.view_mut())
.is_ok());
save_result(&result, "u8x4-lanczos3_upscale-avx2");
res
}
#[test]
fn try_resize_to_other_pixel_type() {
let src_image = get_source_image_u8x4();
let src_image = U8x4::load_big_src_image();
let mut resizer = Resizer::new(ResizeAlg::Convolution(FilterType::Lanczos3));
let mut dst_image = Image::new(
NonZeroU32::new(1024).unwrap(),
@@ -115,7 +75,7 @@ trait PixelExt: Pixel {
}
}
fn load_src_image() -> Image<'static> {
fn load_big_src_image() -> Image<'static> {
let img = ImageReader::open("./data/nasa-4928x3279.png")
.unwrap()
.decode()
@@ -129,6 +89,20 @@ trait PixelExt: Pixel {
.unwrap()
}
fn load_small_src_image() -> Image<'static> {
let img = ImageReader::open("./data/nasa-852x567.png")
.unwrap()
.decode()
.unwrap();
Image::from_vec_u8(
NonZeroU32::new(img.width()).unwrap(),
NonZeroU32::new(img.height()).unwrap(),
Self::img_into_bytes(img),
Self::pixel_type(),
)
.unwrap()
}
fn img_into_bytes(img: DynamicImage) -> Vec<u8>;
}
@@ -172,8 +146,8 @@ impl PixelExt for F32 {
}
}
fn resize_test<P: PixelExt>(resize_alg: ResizeAlg, cpu_extensions: CpuExtensions) {
let image = P::load_src_image();
fn downscale_test<P: PixelExt>(resize_alg: ResizeAlg, cpu_extensions: CpuExtensions) -> Vec<u8> {
let image = P::load_big_src_image();
assert_eq!(image.pixel_type(), P::pixel_type());
let mut resizer = Resizer::new(resize_alg);
@@ -212,50 +186,183 @@ fn resize_test<P: PixelExt>(resize_alg: ResizeAlg, cpu_extensions: CpuExtensions
CpuExtensions::Avx2 => "avx2",
};
let name = format!("{}-{}-{}", P::pixel_type_str(), alg_name, ext_name);
let name = format!(
"downscale-{}-{}-{}",
P::pixel_type_str(),
alg_name,
ext_name
);
save_result(&result, &name);
result.buffer().to_owned()
}
fn upscale_test<P: PixelExt>(resize_alg: ResizeAlg, cpu_extensions: CpuExtensions) -> Vec<u8> {
let image = P::load_small_src_image();
assert_eq!(image.pixel_type(), P::pixel_type());
let mut resizer = Resizer::new(resize_alg);
unsafe {
resizer.set_cpu_extensions(cpu_extensions);
}
let new_height = get_new_height(&image.view(), NEW_BIG_WIDTH);
let mut result = Image::new(
NonZeroU32::new(NEW_BIG_WIDTH).unwrap(),
NonZeroU32::new(new_height).unwrap(),
image.pixel_type(),
);
assert!(resizer
.resize(&image.view(), &mut result.view_mut())
.is_ok());
let alg_name = match resize_alg {
ResizeAlg::Nearest => "nearest",
ResizeAlg::Convolution(filter) => match filter {
FilterType::Box => "box",
FilterType::Bilinear => "bilinear",
FilterType::Hamming => "hamming",
FilterType::Mitchell => "mitchell",
FilterType::CatmullRom => "catmullrom",
FilterType::Lanczos3 => "lanczos3",
_ => "unknown",
},
ResizeAlg::SuperSampling(_, _) => "supersampling",
_ => "unknown",
};
let ext_name = match cpu_extensions {
CpuExtensions::None => "native",
CpuExtensions::Sse2 => "sse2",
CpuExtensions::Sse4_1 => "sse41",
CpuExtensions::Avx2 => "avx2",
};
let name = format!("upscale-{}-{}-{}", P::pixel_type_str(), alg_name, ext_name);
save_result(&result, &name);
result.buffer().to_owned()
}
#[test]
fn resize_u8() {
fn downscale_u8() {
type P = U8;
resize_test::<P>(ResizeAlg::Nearest, CpuExtensions::None);
let buffer = downscale_test::<P>(ResizeAlg::Nearest, CpuExtensions::None);
assert_eq!(image_checksum::<1>(&buffer), [2920317]);
for cpu_extensions in [CpuExtensions::None, CpuExtensions::Avx2] {
resize_test::<P>(ResizeAlg::Convolution(FilterType::Lanczos3), cpu_extensions);
let buffer =
downscale_test::<P>(ResizeAlg::Convolution(FilterType::Lanczos3), cpu_extensions);
assert_eq!(image_checksum::<1>(&buffer), [2923520]);
}
}
#[test]
fn resize_u8x3() {
fn upscale_u8() {
type P = U8;
let buffer = upscale_test::<P>(ResizeAlg::Nearest, CpuExtensions::None);
assert_eq!(image_checksum::<1>(&buffer), [1148750539]);
for cpu_extensions in [CpuExtensions::None, CpuExtensions::Avx2] {
let buffer =
upscale_test::<P>(ResizeAlg::Convolution(FilterType::Lanczos3), cpu_extensions);
assert_eq!(image_checksum::<1>(&buffer), [1148808058]);
}
}
#[test]
fn downscale_u8x3() {
type P = U8x3;
resize_test::<P>(ResizeAlg::Nearest, CpuExtensions::None);
for cpu_extensions in [CpuExtensions::None] {
resize_test::<P>(ResizeAlg::Convolution(FilterType::Lanczos3), cpu_extensions);
}
}
let buffer = downscale_test::<P>(ResizeAlg::Nearest, CpuExtensions::None);
assert_eq!(image_checksum::<3>(&buffer), [2937940, 2945380, 2882679]);
#[test]
fn resize_u8x4() {
type P = U8x4;
resize_test::<P>(ResizeAlg::Nearest, CpuExtensions::None);
for cpu_extensions in [
CpuExtensions::None,
CpuExtensions::Sse4_1,
CpuExtensions::Avx2,
] {
resize_test::<P>(ResizeAlg::Convolution(FilterType::Lanczos3), cpu_extensions);
resize_test::<P>(
let buffer =
downscale_test::<P>(ResizeAlg::Convolution(FilterType::Lanczos3), cpu_extensions);
assert_eq!(image_checksum::<3>(&buffer), [2942479, 2947850, 2885072]);
}
}
#[test]
fn upscale_u8x3() {
type P = U8x3;
let buffer = upscale_test::<P>(ResizeAlg::Nearest, CpuExtensions::None);
assert_eq!(
image_checksum::<3>(&buffer),
[1156008260, 1158417906, 1135087540]
);
for cpu_extensions in [
CpuExtensions::None,
CpuExtensions::Sse4_1,
CpuExtensions::Avx2,
] {
let buffer =
upscale_test::<P>(ResizeAlg::Convolution(FilterType::Lanczos3), cpu_extensions);
assert_eq!(
image_checksum::<3>(&buffer),
[1156107005, 1158443335, 1135101759]
);
}
}
#[test]
fn downscale_u8x4() {
type P = U8x4;
let buffer = downscale_test::<P>(ResizeAlg::Nearest, CpuExtensions::None);
assert_eq!(
image_checksum::<4>(&buffer),
[2937940, 2945380, 2882679, 11054250]
);
for cpu_extensions in [
CpuExtensions::None,
CpuExtensions::Sse4_1,
CpuExtensions::Avx2,
] {
let buffer =
downscale_test::<P>(ResizeAlg::Convolution(FilterType::Lanczos3), cpu_extensions);
assert_eq!(
image_checksum::<4>(&buffer),
[2942479, 2947850, 2885072, 11054250]
);
downscale_test::<P>(
ResizeAlg::SuperSampling(FilterType::Lanczos3, 2),
cpu_extensions,
);
}
}
#[test]
fn upscale_u8x4() {
type P = U8x4;
let buffer = upscale_test::<P>(ResizeAlg::Nearest, CpuExtensions::None);
assert_eq!(
image_checksum::<4>(&buffer),
[1156008260, 1158417906, 1135087540, 4269569040]
);
for cpu_extensions in [
CpuExtensions::None,
CpuExtensions::Sse4_1,
CpuExtensions::Avx2,
] {
let buffer =
upscale_test::<P>(ResizeAlg::Convolution(FilterType::Lanczos3), cpu_extensions);
assert_eq!(
image_checksum::<4>(&buffer),
[1156107005, 1158443335, 1135101759, 4269569040]
);
}
}
// #[test]
fn _resize_i32() {
type P = I32;
resize_test::<P>(ResizeAlg::Nearest, CpuExtensions::None);
downscale_test::<P>(ResizeAlg::Nearest, CpuExtensions::None);
for cpu_extensions in [CpuExtensions::None] {
resize_test::<P>(ResizeAlg::Convolution(FilterType::Lanczos3), cpu_extensions);
downscale_test::<P>(ResizeAlg::Convolution(FilterType::Lanczos3), cpu_extensions);
}
}