mirror of
https://github.com/Cykooz/fast_image_resize.git
synced 2026-10-08 01:11:09 +00:00
- Added optimisation for processing `U8x2 images by MulDiv with helps of SSE4.1 and AVX2` instructions.
- Added optimisation for convolution of ``U16x2`` images with helps of ``AVX2`` instructions.
This commit is contained in:
Kirill Kuzminykh
@@ -1,3 +1,10 @@
|
||||
## [Unreleased] - ReleaseDate
|
||||
|
||||
- Added optimisation for processing ``U8x2`` images by ``MulDiv`` with
|
||||
helps of ``SSE4.1`` and ``AVX2`` instructions.
|
||||
- Added optimisation for convolution of ``U16x2`` images with helps of
|
||||
``AVX2`` instructions.
|
||||
|
||||
## [0.9.0] - 2022-05-01
|
||||
|
||||
- Added support of new type of pixels ``PixelType::U8x2``.
|
||||
|
||||
Generated
+14
-14
@@ -636,9 +636,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jpeg-decoder"
|
||||
version = "0.2.4"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "744c24117572563a98a7e9168a5ac1ee4a1ca7f702211258797bbe0ed0346c3c"
|
||||
checksum = "9478aa10f73e7528198d75109c8be5cd7d15fb530238040148d5f9a22d4c5b3b"
|
||||
dependencies = [
|
||||
"rayon",
|
||||
]
|
||||
@@ -717,9 +717,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.16"
|
||||
version = "0.4.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8"
|
||||
checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
@@ -848,9 +848,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.14"
|
||||
version = "0.2.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290"
|
||||
checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
@@ -958,9 +958,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.37"
|
||||
version = "1.0.38"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec757218438d5fda206afc041538b2f6d889286160d649a86a24d37e1235afd1"
|
||||
checksum = "9027b48e9d4c9175fa2218adf3557f91c1137021739951d4932f5f8268ac48aa"
|
||||
dependencies = [
|
||||
"unicode-xid",
|
||||
]
|
||||
@@ -1107,9 +1107,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.80"
|
||||
version = "1.0.81"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f972498cf015f7c0746cac89ebe1d6ef10c293b94175a243a2d9442c163d9944"
|
||||
checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c"
|
||||
dependencies = [
|
||||
"itoa 1.0.1",
|
||||
"ryu",
|
||||
@@ -1159,9 +1159,9 @@ checksum = "3bdb25a4593d6656239319426f4025f7a658157e25e89f0e0319d7516d46042d"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.92"
|
||||
version = "1.0.93"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ff7c592601f11445996a06f8ad0c27f094a58857c2f89e97974ab9235b92c52"
|
||||
checksum = "04066589568b72ec65f42d65a1a52436e954b168773148893c020269563decf2"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -1290,9 +1290,9 @@ checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
version = "0.2.2"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
|
||||
checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04"
|
||||
|
||||
[[package]]
|
||||
name = "url"
|
||||
|
||||
+2
-2
@@ -14,8 +14,8 @@ exclude = ["/data"]
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
num-traits = "0.2.14"
|
||||
thiserror = "1.0.30"
|
||||
num-traits = "0.2.15"
|
||||
thiserror = "1.0.31"
|
||||
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
@@ -19,7 +19,7 @@ Supported pixel formats and available optimisations:
|
||||
- `U8x2` - two `u8` components per pixel (e.g. LA):
|
||||
- native Rust-code without forced SIMD
|
||||
- SSE4.1 (partial)
|
||||
- AVX2 (partial)
|
||||
- AVX2
|
||||
- `U8x3` - three `u8` components per pixel (e.g. RGB):
|
||||
- native Rust-code without forced SIMD
|
||||
- SSE4.1 (partial)
|
||||
@@ -45,7 +45,7 @@ Environment:
|
||||
- RAM: DDR4 3800 MHz
|
||||
- Ubuntu 22.04 (linux 5.15.0)
|
||||
- Rust 1.60.0
|
||||
- fast_image_resize = "0.9.0"
|
||||
- fast_image_resize = "0.9.1"
|
||||
- glassbench = "0.3.1"
|
||||
- `rustflags = ["-C", "llvm-args=-x86-branches-within-32B-boundaries"]`
|
||||
|
||||
@@ -72,11 +72,11 @@ Pipeline:
|
||||
|
||||
| | Nearest | Bilinear | CatmullRom | Lanczos3 |
|
||||
|------------|:-------:|:--------:|:----------:|:--------:|
|
||||
| image | 20.53 | 77.41 | 135.87 | 184.40 |
|
||||
| resize | - | 45.60 | 87.53 | 129.11 |
|
||||
| fir rust | 0.26 | 37.42 | 63.68 | 92.93 |
|
||||
| fir sse4.1 | 0.26 | 25.76 | 39.52 | 52.82 |
|
||||
| fir avx2 | 0.26 | 6.89 | 8.69 | 12.64 |
|
||||
| image | 20.31 | 77.37 | 136.35 | 183.63 |
|
||||
| resize | - | 45.60 | 87.53 | 129.12 |
|
||||
| fir rust | 0.26 | 42.40 | 78.29 | 115.52 |
|
||||
| fir sse4.1 | 0.26 | 26.11 | 39.54 | 53.26 |
|
||||
| fir avx2 | 0.26 | 6.93 | 8.74 | 12.62 |
|
||||
|
||||
### Resize RGBA8 image (U8x4) 4928x3279 => 852x567
|
||||
|
||||
@@ -90,11 +90,11 @@ Pipeline:
|
||||
|
||||
| | Nearest | Bilinear | CatmullRom | Lanczos3 |
|
||||
|------------|:-------:|:--------:|:----------:|:--------:|
|
||||
| image | 20.71 | 77.57 | 131.22 | 183.61 |
|
||||
| resize | - | 53.38 | 101.44 | 150.50 |
|
||||
| fir rust | 0.17 | 33.06 | 47.36 | 68.15 |
|
||||
| fir sse4.1 | 0.17 | 12.00 | 15.64 | 20.46 |
|
||||
| fir avx2 | 0.17 | 9.13 | 11.38 | 15.12 |
|
||||
| image | 20.43 | 78.27 | 131.97 | 185.15 |
|
||||
| resize | - | 53.60 | 102.31 | 152.19 |
|
||||
| fir rust | 0.17 | 33.25 | 46.95 | 67.66 |
|
||||
| fir sse4.1 | 0.17 | 12.06 | 15.78 | 20.67 |
|
||||
| fir avx2 | 0.17 | 9.32 | 11.55 | 15.32 |
|
||||
|
||||
### Resize grayscale image (U8) 4928x3279 => 852x567
|
||||
|
||||
@@ -108,11 +108,11 @@ Pipeline:
|
||||
|
||||
| | Nearest | Bilinear | CatmullRom | Lanczos3 |
|
||||
|------------|:-------:|:--------:|:----------:|:--------:|
|
||||
| image | 17.09 | 47.71 | 75.01 | 102.30 |
|
||||
| resize | - | 16.23 | 32.66 | 55.36 |
|
||||
| fir rust | 0.14 | 13.00 | 14.74 | 22.12 |
|
||||
| fir sse4.1 | 0.14 | 11.06 | 11.03 | 16.75 |
|
||||
| fir avx2 | 0.13 | 5.90 | 4.39 | 7.12 |
|
||||
| image | 16.95 | 47.55 | 74.95 | 102.03 |
|
||||
| resize | - | 16.25 | 32.60 | 55.48 |
|
||||
| fir rust | 0.14 | 13.00 | 14.72 | 21.98 |
|
||||
| fir sse4.1 | 0.14 | 11.02 | 11.07 | 16.60 |
|
||||
| fir avx2 | 0.14 | 6.06 | 4.40 | 7.37 |
|
||||
|
||||
### Resize grayscale image with alpha channel (U8x2) 4928x3279 => 852x567
|
||||
|
||||
@@ -128,10 +128,10 @@ Pipeline:
|
||||
|
||||
| | Nearest | Bilinear | CatmullRom | Lanczos3 |
|
||||
|------------|:-------:|:--------:|:----------:|:--------:|
|
||||
| image | 18.96 | 62.93 | 112.37 | 149.47 |
|
||||
| fir rust | 0.17 | 23.41 | 27.85 | 39.30 |
|
||||
| fir sse4.1 | 0.17 | 19.56 | 20.62 | 28.77 |
|
||||
| fir avx2 | 0.17 | 19.41 | 20.55 | 28.24 |
|
||||
| image | 18.84 | 63.52 | 109.12 | 147.87 |
|
||||
| fir rust | 0.15 | 23.03 | 28.14 | 39.05 |
|
||||
| fir sse4.1 | 0.15 | 18.68 | 20.35 | 27.82 |
|
||||
| fir avx2 | 0.15 | 10.31 | 11.68 | 14.19 |
|
||||
|
||||
### Resize RGB16 image (U16x3) 4928x3279 => 852x567
|
||||
|
||||
@@ -145,11 +145,11 @@ Pipeline:
|
||||
|
||||
| | Nearest | Bilinear | CatmullRom | Lanczos3 |
|
||||
|------------|:-------:|:--------:|:----------:|:--------:|
|
||||
| image | 20.31 | 72.79 | 122.71 | 171.64 |
|
||||
| resize | - | 47.23 | 90.10 | 132.22 |
|
||||
| fir rust | 0.30 | 42.43 | 78.20 | 115.72 |
|
||||
| fir sse4.1 | 0.31 | 22.08 | 35.60 | 50.43 |
|
||||
| fir avx2 | 0.31 | 19.07 | 28.24 | 34.08 |
|
||||
| image | 20.11 | 73.28 | 124.70 | 173.92 |
|
||||
| resize | - | 47.19 | 89.87 | 132.28 |
|
||||
| fir rust | 0.31 | 39.41 | 69.07 | 96.50 |
|
||||
| fir sse4.1 | 0.31 | 22.21 | 35.84 | 50.60 |
|
||||
| fir avx2 | 0.31 | 19.19 | 28.58 | 34.11 |
|
||||
|
||||
## Examples
|
||||
|
||||
|
||||
+59
-123
@@ -6,148 +6,77 @@ use fast_image_resize::MulDiv;
|
||||
use fast_image_resize::PixelType;
|
||||
use fast_image_resize::{CpuExtensions, Image};
|
||||
|
||||
const fn p(r: u8, g: u8, b: u8, a: u8) -> u32 {
|
||||
u32::from_le_bytes([r, g, b, a])
|
||||
}
|
||||
|
||||
// Multiplies by alpha
|
||||
|
||||
fn get_src_image(width: NonZeroU32, height: NonZeroU32, pixel: u32) -> Image<'static> {
|
||||
fn get_src_image(
|
||||
width: NonZeroU32,
|
||||
height: NonZeroU32,
|
||||
pixel_type: PixelType,
|
||||
pixel: &[u8],
|
||||
) -> Image<'static> {
|
||||
let pixels_count = (width.get() * height.get()) as usize;
|
||||
let buffer = (0..pixels_count)
|
||||
.flat_map(|_| pixel.to_le_bytes())
|
||||
.flat_map(|_| pixel.iter().copied())
|
||||
.collect();
|
||||
Image::from_vec_u8(width, height, buffer, PixelType::U8x4).unwrap()
|
||||
Image::from_vec_u8(width, height, buffer, pixel_type).unwrap()
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
fn multiplies_alpha_avx2(bench: &mut Bench) {
|
||||
fn multiplies_alpha(bench: &mut Bench, pixel_type: PixelType, cpu_extensions: CpuExtensions) {
|
||||
let width = NonZeroU32::new(4096).unwrap();
|
||||
let height = NonZeroU32::new(2048).unwrap();
|
||||
let src_data = get_src_image(width, height, p(255, 128, 0, 128));
|
||||
let mut dst_data = Image::new(width, height, PixelType::U8x4);
|
||||
let pixel: &[u8] = match pixel_type {
|
||||
PixelType::U8x4 => &[255, 128, 0, 128],
|
||||
PixelType::U8x2 => &[255, 128],
|
||||
_ => unreachable!(),
|
||||
};
|
||||
let src_data = get_src_image(width, height, pixel_type, pixel);
|
||||
let mut dst_data = Image::new(width, height, pixel_type);
|
||||
let src_view = src_data.view();
|
||||
let mut dst_view = dst_data.view_mut();
|
||||
let mut alpha_mul_div: MulDiv = Default::default();
|
||||
unsafe {
|
||||
alpha_mul_div.set_cpu_extensions(CpuExtensions::Avx2);
|
||||
alpha_mul_div.set_cpu_extensions(cpu_extensions);
|
||||
}
|
||||
|
||||
bench.task("Multiplies alpha AVX2", |task| {
|
||||
task.iter(|| {
|
||||
alpha_mul_div
|
||||
.multiply_alpha(&src_view, &mut dst_view)
|
||||
.unwrap();
|
||||
})
|
||||
});
|
||||
bench.task(
|
||||
format!("Multiplies alpha {:?} {:?}", pixel_type, cpu_extensions),
|
||||
|task| {
|
||||
task.iter(|| {
|
||||
alpha_mul_div
|
||||
.multiply_alpha(&src_view, &mut dst_view)
|
||||
.unwrap();
|
||||
})
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
fn multiplies_alpha_sse4(bench: &mut Bench) {
|
||||
fn divides_alpha(bench: &mut Bench, pixel_type: PixelType, cpu_extensions: CpuExtensions) {
|
||||
let width = NonZeroU32::new(4096).unwrap();
|
||||
let height = NonZeroU32::new(2048).unwrap();
|
||||
let src_data = get_src_image(width, height, p(255, 128, 0, 128));
|
||||
let mut dst_data = Image::new(width, height, PixelType::U8x4);
|
||||
let pixel: &[u8] = match pixel_type {
|
||||
PixelType::U8x4 => &[128, 64, 0, 128],
|
||||
PixelType::U8x2 => &[128, 128],
|
||||
_ => unreachable!(),
|
||||
};
|
||||
let src_data = get_src_image(width, height, pixel_type, pixel);
|
||||
let mut dst_data = Image::new(width, height, pixel_type);
|
||||
let src_view = src_data.view();
|
||||
let mut dst_view = dst_data.view_mut();
|
||||
let mut alpha_mul_div: MulDiv = Default::default();
|
||||
unsafe {
|
||||
alpha_mul_div.set_cpu_extensions(CpuExtensions::Sse4_1);
|
||||
alpha_mul_div.set_cpu_extensions(cpu_extensions);
|
||||
}
|
||||
|
||||
bench.task("Multiplies alpha SSE4.1", |task| {
|
||||
task.iter(|| {
|
||||
alpha_mul_div
|
||||
.multiply_alpha(&src_view, &mut dst_view)
|
||||
.unwrap();
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
fn multiplies_alpha_native(bench: &mut Bench) {
|
||||
let width = NonZeroU32::new(4096).unwrap();
|
||||
let height = NonZeroU32::new(2048).unwrap();
|
||||
let src_data = get_src_image(width, height, p(255, 128, 0, 128));
|
||||
let mut dst_data = Image::new(width, height, PixelType::U8x4);
|
||||
let src_view = src_data.view();
|
||||
let mut dst_view = dst_data.view_mut();
|
||||
let mut alpha_mul_div: MulDiv = Default::default();
|
||||
unsafe {
|
||||
alpha_mul_div.set_cpu_extensions(CpuExtensions::None);
|
||||
}
|
||||
|
||||
bench.task("Multiplies alpha native", |task| {
|
||||
task.iter(|| {
|
||||
alpha_mul_div
|
||||
.multiply_alpha(&src_view, &mut dst_view)
|
||||
.unwrap();
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
fn divides_alpha_avx2(bench: &mut Bench) {
|
||||
let width = NonZeroU32::new(4096).unwrap();
|
||||
let height = NonZeroU32::new(2048).unwrap();
|
||||
let src_data = get_src_image(width, height, p(128, 64, 0, 128));
|
||||
let mut dst_data = Image::new(width, height, PixelType::U8x4);
|
||||
let src_view = src_data.view();
|
||||
let mut dst_view = dst_data.view_mut();
|
||||
let mut alpha_mul_div: MulDiv = Default::default();
|
||||
unsafe {
|
||||
alpha_mul_div.set_cpu_extensions(CpuExtensions::Avx2);
|
||||
}
|
||||
|
||||
bench.task("Divides alpha AVX2", |task| {
|
||||
task.iter(|| {
|
||||
alpha_mul_div
|
||||
.divide_alpha(&src_view, &mut dst_view)
|
||||
.unwrap();
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
fn divides_alpha_sse4(bench: &mut Bench) {
|
||||
let width = NonZeroU32::new(4096).unwrap();
|
||||
let height = NonZeroU32::new(2048).unwrap();
|
||||
let src_data = get_src_image(width, height, p(128, 64, 0, 128));
|
||||
let mut dst_data = Image::new(width, height, PixelType::U8x4);
|
||||
let src_view = src_data.view();
|
||||
let mut dst_view = dst_data.view_mut();
|
||||
let mut alpha_mul_div: MulDiv = Default::default();
|
||||
unsafe {
|
||||
alpha_mul_div.set_cpu_extensions(CpuExtensions::Sse4_1);
|
||||
}
|
||||
|
||||
bench.task("Divides alpha SSE4.1", |task| {
|
||||
task.iter(|| {
|
||||
alpha_mul_div
|
||||
.divide_alpha(&src_view, &mut dst_view)
|
||||
.unwrap();
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
fn divides_alpha_native(bench: &mut Bench) {
|
||||
let width = NonZeroU32::new(4096).unwrap();
|
||||
let height = NonZeroU32::new(2048).unwrap();
|
||||
let src_data = get_src_image(width, height, p(128, 64, 0, 128));
|
||||
let mut dst_data = Image::new(width, height, PixelType::U8x4);
|
||||
let src_view = src_data.view();
|
||||
let mut dst_view = dst_data.view_mut();
|
||||
let mut alpha_mul_div: MulDiv = Default::default();
|
||||
unsafe {
|
||||
alpha_mul_div.set_cpu_extensions(CpuExtensions::None);
|
||||
}
|
||||
|
||||
bench.task("Divides alpha native", |task| {
|
||||
task.iter(|| {
|
||||
alpha_mul_div
|
||||
.divide_alpha(&src_view, &mut dst_view)
|
||||
.unwrap();
|
||||
})
|
||||
});
|
||||
bench.task(
|
||||
format!("Divides alpha {:?} {:?}", pixel_type, cpu_extensions),
|
||||
|task| {
|
||||
task.iter(|| {
|
||||
alpha_mul_div
|
||||
.divide_alpha(&src_view, &mut dst_view)
|
||||
.unwrap();
|
||||
})
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
pub fn main() {
|
||||
@@ -163,16 +92,23 @@ pub fn main() {
|
||||
let mut bench = create_bench(name, "Alpha", &cmd);
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
{
|
||||
multiplies_alpha_avx2(&mut bench);
|
||||
multiplies_alpha_sse4(&mut bench);
|
||||
multiplies_alpha(&mut bench, PixelType::U8x4, CpuExtensions::Avx2);
|
||||
multiplies_alpha(&mut bench, PixelType::U8x4, CpuExtensions::Sse4_1);
|
||||
multiplies_alpha(&mut bench, PixelType::U8x2, CpuExtensions::Avx2);
|
||||
multiplies_alpha(&mut bench, PixelType::U8x2, CpuExtensions::Sse4_1);
|
||||
}
|
||||
multiplies_alpha_native(&mut bench);
|
||||
multiplies_alpha(&mut bench, PixelType::U8x4, CpuExtensions::None);
|
||||
multiplies_alpha(&mut bench, PixelType::U8x2, CpuExtensions::None);
|
||||
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
{
|
||||
divides_alpha_avx2(&mut bench);
|
||||
divides_alpha_sse4(&mut bench);
|
||||
divides_alpha(&mut bench, PixelType::U8x4, CpuExtensions::Avx2);
|
||||
divides_alpha(&mut bench, PixelType::U8x4, CpuExtensions::Sse4_1);
|
||||
divides_alpha(&mut bench, PixelType::U8x2, CpuExtensions::Avx2);
|
||||
divides_alpha(&mut bench, PixelType::U8x2, CpuExtensions::Sse4_1);
|
||||
}
|
||||
divides_alpha_native(&mut bench);
|
||||
divides_alpha(&mut bench, PixelType::U8x4, CpuExtensions::None);
|
||||
divides_alpha(&mut bench, PixelType::U8x2, CpuExtensions::None);
|
||||
if let Err(e) = after_bench(&mut bench, &cmd) {
|
||||
eprintln!("{:?}", e);
|
||||
}
|
||||
|
||||
@@ -26,6 +26,19 @@ fn get_big_source_image() -> Image<'static> {
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn get_big_u8x2_source_image() -> Image<'static> {
|
||||
let img = utils::get_big_luma_alpha8_image();
|
||||
let width = img.width();
|
||||
let height = img.height();
|
||||
Image::from_vec_u8(
|
||||
NonZeroU32::new(width).unwrap(),
|
||||
NonZeroU32::new(height).unwrap(),
|
||||
img.into_raw(),
|
||||
PixelType::U8x2,
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn get_big_u8x3_source_image() -> Image<'static> {
|
||||
let img = utils::get_big_rgb_image();
|
||||
let width = img.width();
|
||||
@@ -278,6 +291,26 @@ fn u16x3_lanczos3_bench(bench: &mut Bench, cpu_extensions: CpuExtensions, name:
|
||||
});
|
||||
}
|
||||
|
||||
fn u8x2_lanczos3_bench(bench: &mut Bench, cpu_extensions: CpuExtensions, name: &str) {
|
||||
let image = get_big_u8x2_source_image();
|
||||
let mut res_image = Image::new(
|
||||
NonZeroU32::new(NEW_WIDTH).unwrap(),
|
||||
NonZeroU32::new(NEW_HEIGHT).unwrap(),
|
||||
image.pixel_type(),
|
||||
);
|
||||
let src_image = image.view();
|
||||
let mut dst_image = res_image.view_mut();
|
||||
let mut resizer = Resizer::new(ResizeAlg::Convolution(FilterType::Lanczos3));
|
||||
unsafe {
|
||||
resizer.set_cpu_extensions(cpu_extensions);
|
||||
}
|
||||
bench.task(name, |task| {
|
||||
task.iter(|| {
|
||||
resizer.resize(&src_image, &mut dst_image).unwrap();
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
pub fn main() {
|
||||
// Pin process to #0 CPU core
|
||||
let mut cpu_set = nix::sched::CpuSet::new();
|
||||
@@ -302,6 +335,8 @@ pub fn main() {
|
||||
u8_lanczos3_bench(&mut bench, CpuExtensions::Sse4_1, "u8 lanczos3 sse4.1");
|
||||
u8_lanczos3_bench(&mut bench, CpuExtensions::Avx2, "u8 lanczos3 avx2");
|
||||
|
||||
u8x2_lanczos3_bench(&mut bench, CpuExtensions::Avx2, "u8x2 lanczos3 avx2");
|
||||
|
||||
u8x3_lanczos3_bench(&mut bench, CpuExtensions::Sse4_1, "u8x3 lanczos3 sse4.1");
|
||||
u8x3_lanczos3_bench(&mut bench, CpuExtensions::Avx2, "u8x3 lanczos3 avx2");
|
||||
u16x3_lanczos3_bench(&mut bench, CpuExtensions::Sse4_1, "u16x3 lanczos3 sse4.1");
|
||||
|
||||
@@ -34,6 +34,15 @@ pub fn get_big_rgba_image() -> RgbaImage {
|
||||
img.to_rgba8()
|
||||
}
|
||||
|
||||
pub fn get_big_luma_alpha8_image() -> GrayAlphaImage {
|
||||
let cur_dir = env::current_dir().unwrap();
|
||||
let img = Reader::open(cur_dir.join("data/nasa-4928x3279-rgba.png"))
|
||||
.unwrap()
|
||||
.decode()
|
||||
.unwrap();
|
||||
img.to_luma_alpha8()
|
||||
}
|
||||
|
||||
pub fn get_big_luma16_image() -> ImageBuffer<Luma<u16>, Vec<u16>> {
|
||||
let cur_dir = env::current_dir().unwrap();
|
||||
let img = Reader::open(cur_dir.join("data/nasa-4928x3279.png"))
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
use std::arch::x86_64::*;
|
||||
|
||||
use super::sse4;
|
||||
use crate::image_view::{TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::U8x2;
|
||||
use crate::simd_utils;
|
||||
|
||||
#[target_feature(enable = "avx2")]
|
||||
pub(crate) unsafe fn multiply_alpha(
|
||||
src_image: TypedImageView<U8x2>,
|
||||
mut dst_image: TypedImageViewMut<U8x2>,
|
||||
) {
|
||||
let width = src_image.width().get() as usize;
|
||||
let src_rows = src_image.iter_rows(0);
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
|
||||
for (src_row, dst_row) in src_rows.zip(dst_rows) {
|
||||
multiply_alpha_row(src_row, dst_row, width);
|
||||
}
|
||||
}
|
||||
|
||||
#[target_feature(enable = "avx2")]
|
||||
pub(crate) unsafe fn multiply_alpha_inplace(mut image: TypedImageViewMut<U8x2>) {
|
||||
let width = image.width().get() as usize;
|
||||
for dst_row in image.iter_rows_mut() {
|
||||
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
|
||||
multiply_alpha_row(src_row, dst_row, width);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[target_feature(enable = "avx2")]
|
||||
unsafe fn multiply_alpha_row(src_row: &[U8x2], dst_row: &mut [U8x2], width: usize) {
|
||||
let zero = _mm256_setzero_si256();
|
||||
let half = _mm256_set1_epi16(128);
|
||||
|
||||
const MAX_A: i16 = 0xff00u16 as i16;
|
||||
let max_alpha = _mm256_set1_epi16(MAX_A);
|
||||
/*
|
||||
|L A | |L A | |L A | |L A | |L A | |L A | |L A | |L A |
|
||||
|00 01| |02 03| |04 05| |06 07| |08 09| |10 11| |12 13| |14 15|
|
||||
*/
|
||||
#[rustfmt::skip]
|
||||
let factor_mask = _mm256_set_epi8(
|
||||
15, 15, 13, 13, 11, 11, 9, 9, 7, 7, 5, 5, 3, 3, 1, 1,
|
||||
15, 15, 13, 13, 11, 11, 9, 9, 7, 7, 5, 5, 3, 3, 1, 1
|
||||
);
|
||||
|
||||
let mut x: usize = 0;
|
||||
while x < width.saturating_sub(15) {
|
||||
let src_pixels = simd_utils::loadu_si256(src_row, x);
|
||||
|
||||
let factor_pixels = _mm256_shuffle_epi8(src_pixels, factor_mask);
|
||||
let factor_pixels = _mm256_or_si256(factor_pixels, max_alpha);
|
||||
|
||||
let src_i16_lo = _mm256_unpacklo_epi8(src_pixels, zero);
|
||||
let factors = _mm256_unpacklo_epi8(factor_pixels, zero);
|
||||
let src_i16_lo = _mm256_add_epi16(_mm256_mullo_epi16(src_i16_lo, factors), half);
|
||||
let dst_i16_lo = _mm256_add_epi16(src_i16_lo, _mm256_srli_epi16::<8>(src_i16_lo));
|
||||
let dst_i16_lo = _mm256_srli_epi16::<8>(dst_i16_lo);
|
||||
|
||||
let src_i16_hi = _mm256_unpackhi_epi8(src_pixels, zero);
|
||||
let factors = _mm256_unpackhi_epi8(factor_pixels, zero);
|
||||
let src_i16_hi = _mm256_add_epi16(_mm256_mullo_epi16(src_i16_hi, factors), half);
|
||||
let dst_i16_hi = _mm256_add_epi16(src_i16_hi, _mm256_srli_epi16::<8>(src_i16_hi));
|
||||
let dst_i16_hi = _mm256_srli_epi16::<8>(dst_i16_hi);
|
||||
|
||||
let dst_pixels = _mm256_packus_epi16(dst_i16_lo, dst_i16_hi);
|
||||
|
||||
let dst_ptr = dst_row.get_unchecked_mut(x..).as_mut_ptr() as *mut __m256i;
|
||||
_mm256_storeu_si256(dst_ptr, dst_pixels);
|
||||
|
||||
x += 16;
|
||||
}
|
||||
|
||||
let src_tail = &src_row[x..];
|
||||
let dst_tail = &mut dst_row[x..];
|
||||
sse4::multiply_alpha_row(src_tail, dst_tail);
|
||||
}
|
||||
|
||||
// Divide
|
||||
|
||||
#[target_feature(enable = "avx2")]
|
||||
pub(crate) unsafe fn divide_alpha(
|
||||
src_image: TypedImageView<U8x2>,
|
||||
mut dst_image: TypedImageViewMut<U8x2>,
|
||||
) {
|
||||
let src_rows = src_image.iter_rows(0);
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
|
||||
for (src_row, dst_row) in src_rows.zip(dst_rows) {
|
||||
divide_alpha_row(src_row, dst_row);
|
||||
}
|
||||
}
|
||||
|
||||
#[target_feature(enable = "avx2")]
|
||||
pub(crate) unsafe fn divide_alpha_inplace(mut image: TypedImageViewMut<U8x2>) {
|
||||
for dst_row in image.iter_rows_mut() {
|
||||
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
|
||||
divide_alpha_row(src_row, dst_row);
|
||||
}
|
||||
}
|
||||
|
||||
#[target_feature(enable = "avx2")]
|
||||
unsafe fn divide_alpha_row(src_row: &[U8x2], dst_row: &mut [U8x2]) {
|
||||
let src_chunks = src_row.chunks_exact(16);
|
||||
let src_remainder = src_chunks.remainder();
|
||||
let mut dst_chunks = dst_row.chunks_exact_mut(16);
|
||||
|
||||
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
|
||||
divide_alpha_sixteen_pixels(src.as_ptr(), dst.as_mut_ptr());
|
||||
}
|
||||
|
||||
if !src_remainder.is_empty() {
|
||||
let dst_reminder = dst_chunks.into_remainder();
|
||||
sse4::divide_alpha_row(src_remainder, dst_reminder);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[target_feature(enable = "avx2")]
|
||||
unsafe fn divide_alpha_sixteen_pixels(src: *const U8x2, dst: *mut U8x2) {
|
||||
let alpha_mask = _mm256_set1_epi16(0xff00u16 as i16);
|
||||
let luma_mask = _mm256_set1_epi16(0xff);
|
||||
#[rustfmt::skip]
|
||||
let alpha32_sh_lo = _mm256_set_epi8(
|
||||
-1, -1, -1, 7, -1, -1, -1, 5, -1, -1, -1, 3, -1, -1, -1, 1,
|
||||
-1, -1, -1, 7, -1, -1, -1, 5, -1, -1, -1, 3, -1, -1, -1, 1,
|
||||
);
|
||||
#[rustfmt::skip]
|
||||
let alpha32_sh_hi = _mm256_set_epi8(
|
||||
-1, -1, -1, 15, -1, -1, -1, 13, -1, -1, -1, 11, -1, -1, -1, 9,
|
||||
-1, -1, -1, 15, -1, -1, -1, 13, -1, -1, -1, 11, -1, -1, -1, 9,
|
||||
);
|
||||
let alpha_scale = _mm256_set1_ps(255.0 * 256.0);
|
||||
|
||||
let src_pixels = _mm256_loadu_si256(src as *const __m256i);
|
||||
|
||||
let alpha_lo_f32 = _mm256_cvtepi32_ps(_mm256_shuffle_epi8(src_pixels, alpha32_sh_lo));
|
||||
let scaled_alpha_lo_i32 = _mm256_cvtps_epi32(_mm256_div_ps(alpha_scale, alpha_lo_f32));
|
||||
let alpha_hi_f32 = _mm256_cvtepi32_ps(_mm256_shuffle_epi8(src_pixels, alpha32_sh_hi));
|
||||
let scaled_alpha_hi_i32 = _mm256_cvtps_epi32(_mm256_div_ps(alpha_scale, alpha_hi_f32));
|
||||
let scaled_alpha_i16 = _mm256_packus_epi32(scaled_alpha_lo_i32, scaled_alpha_hi_i32);
|
||||
|
||||
let luma_i16 = _mm256_and_si256(src_pixels, luma_mask);
|
||||
let scaled_luma_i16 = _mm256_mullo_epi16(luma_i16, scaled_alpha_i16);
|
||||
let scaled_luma_i16 = _mm256_srli_epi16::<8>(scaled_luma_i16);
|
||||
|
||||
let alpha = _mm256_and_si256(src_pixels, alpha_mask);
|
||||
let dst_pixels = _mm256_blendv_epi8(scaled_luma_i16, alpha, alpha_mask);
|
||||
_mm256_storeu_si256(dst as *mut __m256i, dst_pixels);
|
||||
}
|
||||
+20
-20
@@ -4,11 +4,11 @@ use crate::CpuExtensions;
|
||||
|
||||
use super::AlphaMulDiv;
|
||||
|
||||
// #[cfg(target_arch = "x86_64")]
|
||||
// mod avx2;
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
mod avx2;
|
||||
mod native;
|
||||
// #[cfg(target_arch = "x86_64")]
|
||||
// mod sse4;
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
mod sse4;
|
||||
|
||||
impl AlphaMulDiv for U8x2 {
|
||||
fn multiply_alpha(
|
||||
@@ -17,20 +17,20 @@ impl AlphaMulDiv for U8x2 {
|
||||
cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
match cpu_extensions {
|
||||
// #[cfg(target_arch = "x86_64")]
|
||||
// CpuExtensions::Avx2 => unsafe { avx2::multiply_alpha(src_image, dst_image) },
|
||||
// #[cfg(target_arch = "x86_64")]
|
||||
// CpuExtensions::Sse4_1 => unsafe { sse4::multiply_alpha(src_image, dst_image) },
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Avx2 => unsafe { avx2::multiply_alpha(src_image, dst_image) },
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Sse4_1 => unsafe { sse4::multiply_alpha(src_image, dst_image) },
|
||||
_ => native::multiply_alpha(src_image, dst_image),
|
||||
}
|
||||
}
|
||||
|
||||
fn multiply_alpha_inplace(image: TypedImageViewMut<Self>, cpu_extensions: CpuExtensions) {
|
||||
match cpu_extensions {
|
||||
// #[cfg(target_arch = "x86_64")]
|
||||
// CpuExtensions::Avx2 => unsafe { avx2::multiply_alpha_inplace(image) },
|
||||
// #[cfg(target_arch = "x86_64")]
|
||||
// CpuExtensions::Sse4_1 => unsafe { sse4::multiply_alpha_inplace(image) },
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Avx2 => unsafe { avx2::multiply_alpha_inplace(image) },
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Sse4_1 => unsafe { sse4::multiply_alpha_inplace(image) },
|
||||
_ => native::multiply_alpha_inplace(image),
|
||||
}
|
||||
}
|
||||
@@ -41,20 +41,20 @@ impl AlphaMulDiv for U8x2 {
|
||||
cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
match cpu_extensions {
|
||||
// #[cfg(target_arch = "x86_64")]
|
||||
// CpuExtensions::Avx2 => unsafe { avx2::divide_alpha(src_image, dst_image) },
|
||||
// #[cfg(target_arch = "x86_64")]
|
||||
// CpuExtensions::Sse4_1 => unsafe { sse4::divide_alpha(src_image, dst_image) },
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Avx2 => unsafe { avx2::divide_alpha(src_image, dst_image) },
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Sse4_1 => unsafe { sse4::divide_alpha(src_image, dst_image) },
|
||||
_ => native::divide_alpha(src_image, dst_image),
|
||||
}
|
||||
}
|
||||
|
||||
fn divide_alpha_inplace(image: TypedImageViewMut<Self>, cpu_extensions: CpuExtensions) {
|
||||
match cpu_extensions {
|
||||
// #[cfg(target_arch = "x86_64")]
|
||||
// CpuExtensions::Avx2 => unsafe { avx2::divide_alpha_inplace(image) },
|
||||
// #[cfg(target_arch = "x86_64")]
|
||||
// CpuExtensions::Sse4_1 => unsafe { sse4::divide_alpha_inplace(image) },
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Avx2 => unsafe { avx2::divide_alpha_inplace(image) },
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Sse4_1 => unsafe { sse4::divide_alpha_inplace(image) },
|
||||
_ => native::divide_alpha_inplace(image),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
use std::arch::x86_64::*;
|
||||
|
||||
use crate::image_view::{TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::U8x2;
|
||||
|
||||
use super::native;
|
||||
|
||||
#[target_feature(enable = "sse4.1")]
|
||||
pub(crate) unsafe fn multiply_alpha(
|
||||
src_image: TypedImageView<U8x2>,
|
||||
mut dst_image: TypedImageViewMut<U8x2>,
|
||||
) {
|
||||
let src_rows = src_image.iter_rows(0);
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
|
||||
for (src_row, dst_row) in src_rows.zip(dst_rows) {
|
||||
multiply_alpha_row(src_row, dst_row);
|
||||
}
|
||||
}
|
||||
|
||||
#[target_feature(enable = "sse4.1")]
|
||||
pub(crate) unsafe fn multiply_alpha_inplace(mut image: TypedImageViewMut<U8x2>) {
|
||||
for dst_row in image.iter_rows_mut() {
|
||||
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
|
||||
multiply_alpha_row(src_row, dst_row);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[target_feature(enable = "sse4.1")]
|
||||
pub(crate) unsafe fn multiply_alpha_row(src_row: &[U8x2], dst_row: &mut [U8x2]) {
|
||||
let zero = _mm_setzero_si128();
|
||||
let half = _mm_set1_epi16(128);
|
||||
|
||||
const MAX_A: i16 = 0xff00u16 as i16;
|
||||
let max_alpha = _mm_set1_epi16(MAX_A);
|
||||
/*
|
||||
|L A | |L A | |L A | |L A | |L A | |L A | |L A | |L A |
|
||||
|00 01| |02 03| |04 05| |06 07| |08 09| |10 11| |12 13| |14 15|
|
||||
*/
|
||||
let factor_mask = _mm_set_epi8(15, 15, 13, 13, 11, 11, 9, 9, 7, 7, 5, 5, 3, 3, 1, 1);
|
||||
|
||||
let src_chunks = src_row.chunks_exact(8);
|
||||
let src_remainder = src_chunks.remainder();
|
||||
let mut dst_chunks = dst_row.chunks_exact_mut(8);
|
||||
|
||||
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
|
||||
let src_pixels = _mm_loadu_si128(src.as_ptr() as *const __m128i);
|
||||
|
||||
let factor_pixels = _mm_shuffle_epi8(src_pixels, factor_mask);
|
||||
let factor_pixels = _mm_or_si128(factor_pixels, max_alpha);
|
||||
|
||||
let src_i16_lo = _mm_unpacklo_epi8(src_pixels, zero);
|
||||
let factors = _mm_unpacklo_epi8(factor_pixels, zero);
|
||||
let src_i16_lo = _mm_add_epi16(_mm_mullo_epi16(src_i16_lo, factors), half);
|
||||
let dst_i16_lo = _mm_add_epi16(src_i16_lo, _mm_srli_epi16::<8>(src_i16_lo));
|
||||
let dst_i16_lo = _mm_srli_epi16::<8>(dst_i16_lo);
|
||||
|
||||
let src_i16_hi = _mm_unpackhi_epi8(src_pixels, zero);
|
||||
let factors = _mm_unpackhi_epi8(factor_pixels, zero);
|
||||
let src_i16_hi = _mm_add_epi16(_mm_mullo_epi16(src_i16_hi, factors), half);
|
||||
let dst_i16_hi = _mm_add_epi16(src_i16_hi, _mm_srli_epi16::<8>(src_i16_hi));
|
||||
let dst_i16_hi = _mm_srli_epi16::<8>(dst_i16_hi);
|
||||
|
||||
let dst_pixels = _mm_packus_epi16(dst_i16_lo, dst_i16_hi);
|
||||
|
||||
_mm_storeu_si128(dst.as_mut_ptr() as *mut __m128i, dst_pixels);
|
||||
}
|
||||
|
||||
if !src_remainder.is_empty() {
|
||||
let dst_reminder = dst_chunks.into_remainder();
|
||||
native::multiply_alpha_row(src_remainder, dst_reminder);
|
||||
}
|
||||
}
|
||||
|
||||
// Divide
|
||||
|
||||
#[target_feature(enable = "sse4.1")]
|
||||
pub(crate) unsafe fn divide_alpha(
|
||||
src_image: TypedImageView<U8x2>,
|
||||
mut dst_image: TypedImageViewMut<U8x2>,
|
||||
) {
|
||||
let src_rows = src_image.iter_rows(0);
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
|
||||
for (src_row, dst_row) in src_rows.zip(dst_rows) {
|
||||
divide_alpha_row(src_row, dst_row);
|
||||
}
|
||||
}
|
||||
|
||||
#[target_feature(enable = "sse4.1")]
|
||||
pub(crate) unsafe fn divide_alpha_inplace(mut image: TypedImageViewMut<U8x2>) {
|
||||
for dst_row in image.iter_rows_mut() {
|
||||
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
|
||||
divide_alpha_row(src_row, dst_row);
|
||||
}
|
||||
}
|
||||
|
||||
#[target_feature(enable = "sse4.1")]
|
||||
pub(crate) unsafe fn divide_alpha_row(src_row: &[U8x2], dst_row: &mut [U8x2]) {
|
||||
let src_chunks = src_row.chunks_exact(8);
|
||||
let src_remainder = src_chunks.remainder();
|
||||
let mut dst_chunks = dst_row.chunks_exact_mut(8);
|
||||
|
||||
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
|
||||
divide_alpha_eight_pixels(src.as_ptr(), dst.as_mut_ptr());
|
||||
}
|
||||
|
||||
if !src_remainder.is_empty() {
|
||||
let dst_reminder = dst_chunks.into_remainder();
|
||||
let mut src_pixels = [U8x2(0); 8];
|
||||
src_pixels
|
||||
.iter_mut()
|
||||
.zip(src_remainder)
|
||||
.for_each(|(d, s)| *d = *s);
|
||||
|
||||
let mut dst_pixels = [U8x2(0); 8];
|
||||
divide_alpha_eight_pixels(src_pixels.as_ptr(), dst_pixels.as_mut_ptr());
|
||||
|
||||
dst_pixels
|
||||
.iter()
|
||||
.zip(dst_reminder)
|
||||
.for_each(|(s, d)| *d = *s);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[target_feature(enable = "sse4.1")]
|
||||
unsafe fn divide_alpha_eight_pixels(src: *const U8x2, dst: *mut U8x2) {
|
||||
let alpha_mask = _mm_set1_epi16(0xff00u16 as i16);
|
||||
let luma_mask = _mm_set1_epi16(0xff);
|
||||
let alpha32_sh_lo = _mm_set_epi8(-1, -1, -1, 7, -1, -1, -1, 5, -1, -1, -1, 3, -1, -1, -1, 1);
|
||||
let alpha32_sh_hi = _mm_set_epi8(
|
||||
-1, -1, -1, 15, -1, -1, -1, 13, -1, -1, -1, 11, -1, -1, -1, 9,
|
||||
);
|
||||
let alpha_scale = _mm_set1_ps(255.0 * 256.0);
|
||||
|
||||
let src_pixels = _mm_loadu_si128(src as *const __m128i);
|
||||
|
||||
let alpha_lo_f32 = _mm_cvtepi32_ps(_mm_shuffle_epi8(src_pixels, alpha32_sh_lo));
|
||||
let scaled_alpha_lo_i32 = _mm_cvtps_epi32(_mm_div_ps(alpha_scale, alpha_lo_f32));
|
||||
let alpha_hi_f32 = _mm_cvtepi32_ps(_mm_shuffle_epi8(src_pixels, alpha32_sh_hi));
|
||||
let scaled_alpha_hi_i32 = _mm_cvtps_epi32(_mm_div_ps(alpha_scale, alpha_hi_f32));
|
||||
let scaled_alpha_i16 = _mm_packus_epi32(scaled_alpha_lo_i32, scaled_alpha_hi_i32);
|
||||
|
||||
let luma_i16 = _mm_and_si128(src_pixels, luma_mask);
|
||||
let scaled_luma_i16 = _mm_mullo_epi16(luma_i16, scaled_alpha_i16);
|
||||
let scaled_luma_i16 = _mm_srli_epi16::<8>(scaled_luma_i16);
|
||||
|
||||
let alpha = _mm_and_si128(src_pixels, alpha_mask);
|
||||
let dst_pixels = _mm_blendv_epi8(scaled_luma_i16, alpha, alpha_mask);
|
||||
_mm_storeu_si128(dst as *mut __m128i, dst_pixels);
|
||||
}
|
||||
@@ -11,21 +11,17 @@ impl Convolution for F32 {
|
||||
dst_image: TypedImageViewMut<Self>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
cpu_extensions: CpuExtensions,
|
||||
_cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
match cpu_extensions {
|
||||
_ => native::horiz_convolution(src_image, dst_image, offset, coeffs),
|
||||
}
|
||||
native::horiz_convolution(src_image, dst_image, offset, coeffs);
|
||||
}
|
||||
|
||||
fn vert_convolution(
|
||||
src_image: TypedImageView<Self>,
|
||||
dst_image: TypedImageViewMut<Self>,
|
||||
coeffs: Coefficients,
|
||||
cpu_extensions: CpuExtensions,
|
||||
_cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
match cpu_extensions {
|
||||
_ => native::vert_convolution(src_image, dst_image, coeffs),
|
||||
}
|
||||
native::vert_convolution(src_image, dst_image, coeffs);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,21 +11,17 @@ impl Convolution for I32 {
|
||||
dst_image: TypedImageViewMut<Self>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
cpu_extensions: CpuExtensions,
|
||||
_cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
match cpu_extensions {
|
||||
_ => native::horiz_convolution(src_image, dst_image, offset, coeffs),
|
||||
}
|
||||
native::horiz_convolution(src_image, dst_image, offset, coeffs);
|
||||
}
|
||||
|
||||
fn vert_convolution(
|
||||
src_image: TypedImageView<Self>,
|
||||
dst_image: TypedImageViewMut<Self>,
|
||||
coeffs: Coefficients,
|
||||
cpu_extensions: CpuExtensions,
|
||||
_cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
match cpu_extensions {
|
||||
_ => native::vert_convolution(src_image, dst_image, coeffs),
|
||||
}
|
||||
native::vert_convolution(src_image, dst_image, coeffs);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,428 @@
|
||||
use std::arch::x86_64::*;
|
||||
|
||||
use crate::convolution::{optimisations, Coefficients};
|
||||
use crate::image_view::{FourRows, FourRowsMut, TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::U8x2;
|
||||
use crate::simd_utils;
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn horiz_convolution(
|
||||
src_image: TypedImageView<U8x2>,
|
||||
mut dst_image: TypedImageViewMut<U8x2>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
let (values, window_size, bounds_per_pixel) =
|
||||
(coeffs.values, coeffs.window_size, coeffs.bounds);
|
||||
|
||||
let normalizer_guard = optimisations::NormalizerGuard16::new(values);
|
||||
let coefficients_chunks = normalizer_guard.normalized_chunks(window_size, &bounds_per_pixel);
|
||||
let dst_height = dst_image.height().get();
|
||||
|
||||
let src_iter = src_image.iter_4_rows(offset, dst_height + offset);
|
||||
let dst_iter = dst_image.iter_4_rows_mut();
|
||||
for (src_rows, dst_rows) in src_iter.zip(dst_iter) {
|
||||
unsafe {
|
||||
horiz_convolution_four_rows(
|
||||
src_rows,
|
||||
dst_rows,
|
||||
&coefficients_chunks,
|
||||
&normalizer_guard,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let mut yy = dst_height - dst_height % 4;
|
||||
while yy < dst_height {
|
||||
unsafe {
|
||||
horiz_convolution_one_row(
|
||||
src_image.get_row(yy + offset).unwrap(),
|
||||
dst_image.get_row_mut(yy).unwrap(),
|
||||
&coefficients_chunks,
|
||||
&normalizer_guard,
|
||||
);
|
||||
}
|
||||
yy += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/// For safety, it is necessary to ensure the following conditions:
|
||||
/// - length of all rows in src_rows must be equal
|
||||
/// - length of all rows in dst_rows must be equal
|
||||
/// - coefficients_chunks.len() == dst_rows.0.len()
|
||||
/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.0.len()
|
||||
/// - precision <= MAX_COEFS_PRECISION
|
||||
#[inline]
|
||||
#[target_feature(enable = "avx2")]
|
||||
unsafe fn horiz_convolution_four_rows(
|
||||
src_rows: FourRows<U8x2>,
|
||||
dst_rows: FourRowsMut<U8x2>,
|
||||
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
|
||||
normalizer_guard: &optimisations::NormalizerGuard16,
|
||||
) {
|
||||
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
|
||||
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
|
||||
let precision = normalizer_guard.precision();
|
||||
let initial = _mm256_set1_epi32(1 << (precision - 2));
|
||||
|
||||
/*
|
||||
|L A | |L A | |L A | |L A | |L A | |L A | |L A | |L A |
|
||||
|00 01| |02 03| |04 05| |06 07| |08 09| |10 11| |12 13| |14 15|
|
||||
|
||||
Shuffle components with converting from u8 into i16:
|
||||
|
||||
A: |-1 07| |-1 05| |-1 03| |-1 01|
|
||||
L: |-1 06| |-1 04| |-1 02| |-1 00|
|
||||
*/
|
||||
#[rustfmt::skip]
|
||||
let sh1 = _mm256_set_epi8(
|
||||
-1, 7, -1, 5, -1, 3, -1, 1, -1, 6, -1, 4, -1, 2, -1, 0,
|
||||
-1, 7, -1, 5, -1, 3, -1, 1, -1, 6, -1, 4, -1, 2, -1, 0,
|
||||
);
|
||||
/*
|
||||
A: |-1 15| |-1 13| |-1 11| |-1 09|
|
||||
L: |-1 14| |-1 12| |-1 10| |-1 08|
|
||||
*/
|
||||
#[rustfmt::skip]
|
||||
let sh2 = _mm256_set_epi8(
|
||||
-1, 15, -1, 13, -1, 11, -1, 9, -1, 14, -1, 12, -1, 10, -1, 8,
|
||||
-1, 15, -1, 13, -1, 11, -1, 9, -1, 14, -1, 12, -1, 10, -1, 8,
|
||||
);
|
||||
|
||||
for (dst_x, coeffs_chunk) in coefficients_chunks.iter().enumerate() {
|
||||
let mut x = coeffs_chunk.start as usize;
|
||||
|
||||
let mut sss0 = initial;
|
||||
let mut sss1 = initial;
|
||||
let coeffs = coeffs_chunk.values;
|
||||
|
||||
let coeffs_by_8 = coeffs.chunks_exact(8);
|
||||
let reminder = coeffs_by_8.remainder();
|
||||
|
||||
for k in coeffs_by_8 {
|
||||
let mmk0 = simd_utils::ptr_i16_to_256set1_epi64x(k, 0);
|
||||
let mmk1 = simd_utils::ptr_i16_to_256set1_epi64x(k, 4);
|
||||
|
||||
let source = _mm256_inserti128_si256::<1>(
|
||||
_mm256_castsi128_si256(simd_utils::loadu_si128(s_row0, x)),
|
||||
simd_utils::loadu_si128(s_row1, x),
|
||||
);
|
||||
let pix = _mm256_shuffle_epi8(source, sh1);
|
||||
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk0));
|
||||
let pix = _mm256_shuffle_epi8(source, sh2);
|
||||
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk1));
|
||||
|
||||
let source = _mm256_inserti128_si256::<1>(
|
||||
_mm256_castsi128_si256(simd_utils::loadu_si128(s_row2, x)),
|
||||
simd_utils::loadu_si128(s_row3, x),
|
||||
);
|
||||
let pix = _mm256_shuffle_epi8(source, sh1);
|
||||
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk0));
|
||||
let pix = _mm256_shuffle_epi8(source, sh2);
|
||||
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk1));
|
||||
|
||||
x += 8;
|
||||
}
|
||||
|
||||
let coeffs_by_4 = reminder.chunks_exact(4);
|
||||
let reminder = coeffs_by_4.remainder();
|
||||
|
||||
for k in coeffs_by_4 {
|
||||
let mmk = simd_utils::ptr_i16_to_256set1_epi64x(k, 0);
|
||||
|
||||
let source = _mm256_inserti128_si256::<1>(
|
||||
_mm256_castsi128_si256(simd_utils::loadl_epi64(s_row0, x)),
|
||||
simd_utils::loadl_epi64(s_row1, x),
|
||||
);
|
||||
let pix = _mm256_shuffle_epi8(source, sh1);
|
||||
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
|
||||
|
||||
let source = _mm256_inserti128_si256::<1>(
|
||||
_mm256_castsi128_si256(simd_utils::loadl_epi64(s_row2, x)),
|
||||
simd_utils::loadl_epi64(s_row3, x),
|
||||
);
|
||||
let pix = _mm256_shuffle_epi8(source, sh1);
|
||||
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
|
||||
|
||||
x += 4;
|
||||
}
|
||||
|
||||
let coeffs_by_2 = reminder.chunks_exact(2);
|
||||
let reminder = coeffs_by_2.remainder();
|
||||
|
||||
for k in coeffs_by_2 {
|
||||
let mmk = simd_utils::ptr_i16_to_256set1_epi32(k, 0);
|
||||
|
||||
let source = _mm256_inserti128_si256::<1>(
|
||||
_mm256_castsi128_si256(simd_utils::loadl_epi32(s_row0, x)),
|
||||
simd_utils::loadl_epi32(s_row1, x),
|
||||
);
|
||||
let pix = _mm256_shuffle_epi8(source, sh1);
|
||||
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
|
||||
|
||||
let source = _mm256_inserti128_si256::<1>(
|
||||
_mm256_castsi128_si256(simd_utils::loadl_epi32(s_row2, x)),
|
||||
simd_utils::loadl_epi32(s_row3, x),
|
||||
);
|
||||
let pix = _mm256_shuffle_epi8(source, sh1);
|
||||
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
|
||||
|
||||
x += 2;
|
||||
}
|
||||
|
||||
if let Some(&k) = reminder.get(0) {
|
||||
// [16] xx k0 xx k0 xx k0 xx k0 xx k0 xx k0 xx k0 xx k0
|
||||
let mmk = _mm256_set1_epi32(k as i32);
|
||||
|
||||
// [16] xx a0 xx b0 xx g0 xx r0 xx a0 xx b0 xx g0 xx r0
|
||||
let source = _mm256_inserti128_si256::<1>(
|
||||
_mm256_castsi128_si256(simd_utils::loadl_epi16(s_row0, x)),
|
||||
simd_utils::loadl_epi16(s_row1, x),
|
||||
);
|
||||
let pix = _mm256_shuffle_epi8(source, sh1);
|
||||
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
|
||||
|
||||
let source = _mm256_inserti128_si256::<1>(
|
||||
_mm256_castsi128_si256(simd_utils::loadl_epi16(s_row2, x)),
|
||||
simd_utils::loadl_epi16(s_row3, x),
|
||||
);
|
||||
let pix = _mm256_shuffle_epi8(source, sh1);
|
||||
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
|
||||
}
|
||||
|
||||
let lo128 = _mm256_extracti128_si256::<0>(sss0);
|
||||
let hi128 = _mm256_extracti128_si256::<1>(sss0);
|
||||
set_dst_pixel(lo128, d_row0, dst_x, normalizer_guard);
|
||||
set_dst_pixel(hi128, d_row1, dst_x, normalizer_guard);
|
||||
|
||||
let lo128 = _mm256_extracti128_si256::<0>(sss1);
|
||||
let hi128 = _mm256_extracti128_si256::<1>(sss1);
|
||||
set_dst_pixel(lo128, d_row2, dst_x, normalizer_guard);
|
||||
set_dst_pixel(hi128, d_row3, dst_x, normalizer_guard);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[target_feature(enable = "avx2")]
|
||||
unsafe fn set_dst_pixel(
|
||||
raw: __m128i,
|
||||
d_row: &mut &mut [U8x2],
|
||||
dst_x: usize,
|
||||
normalizer_guard: &optimisations::NormalizerGuard16,
|
||||
) {
|
||||
let l32x2 = _mm_extract_epi64::<0>(raw);
|
||||
let a32x2 = _mm_extract_epi64::<1>(raw);
|
||||
let l32 = ((l32x2 >> 32) as i32).saturating_add((l32x2 & 0xffffffff) as i32);
|
||||
let a32 = ((a32x2 >> 32) as i32).saturating_add((a32x2 & 0xffffffff) as i32);
|
||||
let l8 = normalizer_guard.clip(l32);
|
||||
let a8 = normalizer_guard.clip(a32);
|
||||
d_row.get_unchecked_mut(dst_x).0 = u16::from_le_bytes([l8, a8]);
|
||||
}
|
||||
|
||||
/// For safety, it is necessary to ensure the following conditions:
|
||||
/// - bounds.len() == dst_row.len()
|
||||
/// - coeffs.len() == dst_rows.0.len() * window_size
|
||||
/// - max(bound.start + bound.size for bound in bounds) <= src_row.len()
|
||||
/// - precision <= MAX_COEFS_PRECISION
|
||||
#[inline]
|
||||
#[target_feature(enable = "avx2")]
|
||||
unsafe fn horiz_convolution_one_row(
|
||||
src_row: &[U8x2],
|
||||
dst_row: &mut [U8x2],
|
||||
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
|
||||
normalizer_guard: &optimisations::NormalizerGuard16,
|
||||
) {
|
||||
let precision = normalizer_guard.precision();
|
||||
/*
|
||||
|L A | |L A | |L A | |L A | |L A | |L A | |L A | |L A |
|
||||
|00 01| |02 03| |04 05| |06 07| |08 09| |10 11| |12 13| |14 15|
|
||||
|
||||
Scale first four pixels into i16:
|
||||
|
||||
A: |-1 07| |-1 05|
|
||||
L: |-1 06| |-1 04|
|
||||
A: |-1 03| |-1 01|
|
||||
L: |-1 02| |-1 00|
|
||||
*/
|
||||
#[rustfmt::skip]
|
||||
let pix_sh1 = _mm256_set_epi8(
|
||||
-1, 7, -1, 5, -1, 6, -1, 4, -1, 3, -1, 1, -1, 2, -1, 0,
|
||||
-1, 7, -1, 5, -1, 6, -1, 4, -1, 3, -1, 1, -1, 2, -1, 0,
|
||||
);
|
||||
/*
|
||||
|C0 | |C1 | |C2 | |C3 | |C4 | |C5 | |C6 | |C7 |
|
||||
|00 01| |02 03| |04 05| |06 07| |08 09| |10 11| |12 13| |14 15|
|
||||
|
||||
Duplicate first four coefficients for A and L components of pixels:
|
||||
|
||||
CA: |07 06| |05 04|
|
||||
CL: |07 06| |05 04|
|
||||
CA: |03 02| |01 00|
|
||||
CL: |03 02| |01 00|
|
||||
*/
|
||||
#[rustfmt::skip]
|
||||
let coeff_sh1 = _mm256_set_epi8(
|
||||
7, 6, 5, 4, 7, 6, 5, 4, 3, 2, 1, 0, 3,2, 1, 0,
|
||||
7, 6, 5, 4, 7, 6, 5, 4, 3, 2, 1, 0, 3,2, 1, 0,
|
||||
);
|
||||
|
||||
/*
|
||||
|L A | |L A | |L A | |L A | |L A | |L A | |L A | |L A |
|
||||
|00 01| |02 03| |04 05| |06 07| |08 09| |10 11| |12 13| |14 15|
|
||||
|
||||
Scale second four pixels into i16:
|
||||
|
||||
A: |-1 15| |-1 13|
|
||||
L: |-1 14| |-1 12|
|
||||
A: |-1 11| |-1 09|
|
||||
L: |-1 10| |-1 08|
|
||||
*/
|
||||
#[rustfmt::skip]
|
||||
let pix_sh2 = _mm256_set_epi8(
|
||||
-1, 15, -1, 13, -1, 14, -1, 12, -1, 11, -1, 9, -1, 10, -1, 8,
|
||||
-1, 15, -1, 13, -1, 14, -1, 12, -1, 11, -1, 9, -1, 10, -1, 8,
|
||||
);
|
||||
/*
|
||||
|C0 | |C1 | |C2 | |C3 | |C4 | |C5 | |C6 | |C7 |
|
||||
|00 01| |02 03| |04 05| |06 07| |08 09| |10 11| |12 13| |14 15|
|
||||
|
||||
Duplicate second four coefficients for A and L components of pixels:
|
||||
|
||||
CA: |15 14| |13 12|
|
||||
CL: |15 14| |13 12|
|
||||
CA: |11 10| |09 08|
|
||||
CL: |11 10| |09 08|
|
||||
*/
|
||||
#[rustfmt::skip]
|
||||
let coeff_sh2 = _mm256_set_epi8(
|
||||
15, 14, 13, 12, 15, 14, 13, 12, 11, 10, 9, 8, 11, 10, 9, 8,
|
||||
15, 14, 13, 12, 15, 14, 13, 12, 11, 10, 9, 8, 11, 10, 9, 8,
|
||||
);
|
||||
|
||||
/*
|
||||
Scale to i16 first four pixels in first half of register,
|
||||
and second four pixels in second half of register.
|
||||
*/
|
||||
#[rustfmt::skip]
|
||||
let pix_sh3 = _mm256_set_epi8(
|
||||
-1, 15, -1, 13, -1, 14, -1, 12, -1, 11, -1, 9, -1, 10, -1, 8,
|
||||
-1, 7, -1, 5, -1, 6, -1, 4, -1, 3, -1, 1, -1, 2, -1, 0,
|
||||
);
|
||||
/*
|
||||
Duplicate first four coefficients in first half of register,
|
||||
and second four coefficients in second half of register.
|
||||
*/
|
||||
#[rustfmt::skip]
|
||||
let coeff_sh3 = _mm256_set_epi8(
|
||||
15, 14, 13, 12, 15, 14, 13, 12, 11, 10, 9, 8, 11, 10, 9, 8,
|
||||
7, 6, 5, 4, 7, 6, 5, 4, 3, 2, 1, 0, 3,2, 1, 0,
|
||||
);
|
||||
|
||||
/*
|
||||
|L A | |L A | |L A | |L A |
|
||||
|00 01| |02 03| |04 05| |06 07| |08 09| |10 11| |12 13| |14 15|
|
||||
|
||||
Scale four pixels into i16:
|
||||
|
||||
A: |-1 07| |-1 05|
|
||||
L: |-1 06| |-1 04|
|
||||
A: |-1 03| |-1 01|
|
||||
L: |-1 02| |-1 00|
|
||||
*/
|
||||
let pix_sh4 = _mm_set_epi8(-1, 7, -1, 5, -1, 6, -1, 4, -1, 3, -1, 1, -1, 2, -1, 0);
|
||||
|
||||
for (dst_x, &coeffs_chunk) in coefficients_chunks.iter().enumerate() {
|
||||
let mut x = coeffs_chunk.start as usize;
|
||||
let mut coeffs = coeffs_chunk.values;
|
||||
|
||||
let mut sss = if coeffs.len() < 16 {
|
||||
// Lower part will be added to higher, use only half of the error
|
||||
_mm_set1_epi32(1 << (precision - 2))
|
||||
} else {
|
||||
// Lower part will be added to higher twice, use only quarter of the error
|
||||
let mut sss256 = _mm256_set1_epi32(1 << (precision - 3));
|
||||
|
||||
let coeffs_by_16 = coeffs.chunks_exact(16);
|
||||
let reminder = coeffs_by_16.remainder();
|
||||
|
||||
for k in coeffs_by_16 {
|
||||
let ksource = simd_utils::loadu_si256(k, 0);
|
||||
let source = simd_utils::loadu_si256(src_row, x);
|
||||
|
||||
let pix = _mm256_shuffle_epi8(source, pix_sh1);
|
||||
let mmk = _mm256_shuffle_epi8(ksource, coeff_sh1);
|
||||
sss256 = _mm256_add_epi32(sss256, _mm256_madd_epi16(pix, mmk));
|
||||
|
||||
let pix = _mm256_shuffle_epi8(source, pix_sh2);
|
||||
let mmk = _mm256_shuffle_epi8(ksource, coeff_sh2);
|
||||
sss256 = _mm256_add_epi32(sss256, _mm256_madd_epi16(pix, mmk));
|
||||
|
||||
x += 16;
|
||||
}
|
||||
|
||||
let coeffs_by_8 = reminder.chunks_exact(8);
|
||||
coeffs = coeffs_by_8.remainder();
|
||||
|
||||
for k in coeffs_by_8 {
|
||||
let tmp = simd_utils::loadu_si128(k, 0);
|
||||
let ksource = _mm256_insertf128_si256::<1>(_mm256_castsi128_si256(tmp), tmp);
|
||||
|
||||
let tmp = simd_utils::loadu_si128(src_row, x);
|
||||
let source = _mm256_insertf128_si256::<1>(_mm256_castsi128_si256(tmp), tmp);
|
||||
|
||||
let pix = _mm256_shuffle_epi8(source, pix_sh3);
|
||||
let mmk = _mm256_shuffle_epi8(ksource, coeff_sh3);
|
||||
sss256 = _mm256_add_epi32(sss256, _mm256_madd_epi16(pix, mmk));
|
||||
|
||||
x += 8;
|
||||
}
|
||||
|
||||
_mm_add_epi32(
|
||||
_mm256_extracti128_si256::<0>(sss256),
|
||||
_mm256_extracti128_si256::<1>(sss256),
|
||||
)
|
||||
};
|
||||
|
||||
let coeffs_by_4 = coeffs.chunks_exact(4);
|
||||
let reminder1 = coeffs_by_4.remainder();
|
||||
|
||||
for k in coeffs_by_4 {
|
||||
let mmk = _mm_set_epi16(k[3], k[2], k[3], k[2], k[1], k[0], k[1], k[0]);
|
||||
let source = simd_utils::loadl_epi64(src_row, x);
|
||||
let pix = _mm_shuffle_epi8(source, pix_sh4);
|
||||
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
x += 4
|
||||
}
|
||||
|
||||
if !reminder1.is_empty() {
|
||||
let mut pixels: [i16; 6] = [0; 6];
|
||||
let mut coeffs: [i16; 3] = [0; 3];
|
||||
for (i, &coeff) in reminder1.iter().enumerate() {
|
||||
coeffs[i] = coeff;
|
||||
let pixel: [u8; 2] = (*src_row.get_unchecked(x)).0.to_le_bytes();
|
||||
pixels[i * 2] = pixel[0] as i16;
|
||||
pixels[i * 2 + 1] = pixel[1] as i16;
|
||||
x += 1;
|
||||
}
|
||||
|
||||
let pix = _mm_set_epi16(
|
||||
0, pixels[5], 0, pixels[4], pixels[3], pixels[1], pixels[2], pixels[0],
|
||||
);
|
||||
let mmk = _mm_set_epi16(
|
||||
0, coeffs[2], 0, coeffs[2], coeffs[1], coeffs[0], coeffs[1], coeffs[0],
|
||||
);
|
||||
|
||||
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
|
||||
}
|
||||
|
||||
let lo = _mm_extract_epi64::<0>(sss);
|
||||
let hi = _mm_extract_epi64::<1>(sss);
|
||||
|
||||
let a32 = ((lo >> 32) as i32).saturating_add((hi >> 32) as i32);
|
||||
let l32 = ((lo & 0xffffffff) as i32).saturating_add((hi & 0xffffffff) as i32);
|
||||
let a8 = normalizer_guard.clip(a32);
|
||||
let l8 = normalizer_guard.clip(l32);
|
||||
dst_row.get_unchecked_mut(dst_x).0 = u16::from_le_bytes([l8, a8]);
|
||||
}
|
||||
}
|
||||
@@ -5,8 +5,8 @@ use crate::CpuExtensions;
|
||||
|
||||
use super::{Coefficients, Convolution};
|
||||
|
||||
// #[cfg(target_arch = "x86_64")]
|
||||
// mod avx2;
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
mod avx2;
|
||||
mod native;
|
||||
// #[cfg(target_arch = "x86_64")]
|
||||
// mod sse4;
|
||||
@@ -20,8 +20,8 @@ impl Convolution for U8x2 {
|
||||
cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
match cpu_extensions {
|
||||
// #[cfg(target_arch = "x86_64")]
|
||||
// CpuExtensions::Avx2 => avx2::horiz_convolution(src_image, dst_image, offset, coeffs),
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Avx2 => avx2::horiz_convolution(src_image, dst_image, offset, coeffs),
|
||||
// #[cfg(target_arch = "x86_64")]
|
||||
// CpuExtensions::Sse4_1 => unsafe {
|
||||
// sse4::horiz_convolution(src_image, dst_image, offset, coeffs)
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
use std::arch::x86_64::*;
|
||||
|
||||
use crate::convolution::optimisations::{CoefficientsI16Chunk, NormalizerGuard16};
|
||||
use crate::convolution::{optimisations, Coefficients};
|
||||
use crate::image_view::{TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::Pixel;
|
||||
@@ -33,8 +32,8 @@ pub(crate) fn vert_convolution<T>(
|
||||
unsafe fn vert_convolution_into_one_row_u8<T>(
|
||||
src_img: &TypedImageView<T>,
|
||||
dst_row: &mut [T],
|
||||
coeffs_chunk: CoefficientsI16Chunk,
|
||||
normalizer_guard: &NormalizerGuard16,
|
||||
coeffs_chunk: optimisations::CoefficientsI16Chunk,
|
||||
normalizer_guard: &optimisations::NormalizerGuard16,
|
||||
) where
|
||||
T: Pixel<Component = u8>,
|
||||
{
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
use crate::convolution::optimisations::NormalizerGuard16;
|
||||
use crate::convolution::{optimisations, Coefficients};
|
||||
use crate::image_view::{TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::Pixel;
|
||||
@@ -79,7 +78,7 @@ pub(crate) fn vert_convolution<T>(
|
||||
#[inline(always)]
|
||||
fn convolution_by_u8<T>(
|
||||
src_image: &TypedImageView<T>,
|
||||
normalizer_guard: &NormalizerGuard16,
|
||||
normalizer_guard: &optimisations::NormalizerGuard16,
|
||||
initial: i32,
|
||||
dst_components: &mut [u8],
|
||||
mut x_src: usize,
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
use std::arch::x86_64::*;
|
||||
|
||||
use crate::convolution::optimisations::{CoefficientsI16Chunk, NormalizerGuard16};
|
||||
use crate::convolution::{optimisations, Coefficients};
|
||||
use crate::image_view::{TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::Pixel;
|
||||
@@ -30,8 +29,8 @@ pub(crate) fn vert_convolution<T: Pixel<Component = u8>>(
|
||||
pub(crate) unsafe fn vert_convolution_into_one_row_u8<T: Pixel<Component = u8>>(
|
||||
src_img: &TypedImageView<T>,
|
||||
dst_row: &mut [T],
|
||||
coeffs_chunk: CoefficientsI16Chunk,
|
||||
normalizer_guard: &NormalizerGuard16,
|
||||
coeffs_chunk: optimisations::CoefficientsI16Chunk,
|
||||
normalizer_guard: &optimisations::NormalizerGuard16,
|
||||
) {
|
||||
let mut xx: usize = 0;
|
||||
let src_width = src_img.width().get() as usize * T::components_count();
|
||||
|
||||
+3
-3
@@ -484,7 +484,7 @@ where
|
||||
) -> impl Iterator<Item = FourRows<'b, P>> + 's {
|
||||
let start_y = start_y as usize;
|
||||
let max_y = max_y.min(self.height.get()) as usize;
|
||||
let rows = self.rows.get(start_y..max_y).unwrap_or_else(|| &[]);
|
||||
let rows = self.rows.get(start_y..max_y).unwrap_or(&[]);
|
||||
rows.chunks_exact(4).map(|rows| match *rows {
|
||||
[r0, r1, r2, r3] => (r0, r1, r2, r3),
|
||||
_ => unreachable!(),
|
||||
@@ -499,7 +499,7 @@ where
|
||||
) -> impl Iterator<Item = TwoRows<'b, P>> + 's {
|
||||
let start_y = start_y as usize;
|
||||
let max_y = max_y.min(self.height.get()) as usize;
|
||||
let rows = self.rows.get(start_y..max_y).unwrap_or_else(|| &[]);
|
||||
let rows = self.rows.get(start_y..max_y).unwrap_or(&[]);
|
||||
rows.chunks_exact(2).map(|rows| match *rows {
|
||||
[r0, r1] => (r0, r1),
|
||||
_ => unreachable!(),
|
||||
@@ -509,7 +509,7 @@ where
|
||||
#[inline(always)]
|
||||
pub(crate) fn iter_rows<'s>(&'s self, start_y: u32) -> impl Iterator<Item = &'b [P]> + 's {
|
||||
let start_y = start_y as usize;
|
||||
let rows = self.rows.get(start_y..).unwrap_or_else(|| &[]);
|
||||
let rows = self.rows.get(start_y..).unwrap_or(&[]);
|
||||
rows.iter().copied()
|
||||
}
|
||||
|
||||
|
||||
+2
-1
@@ -1,4 +1,5 @@
|
||||
//! Contains types of pixels.
|
||||
use std::fmt::Debug;
|
||||
use std::mem::size_of;
|
||||
use std::slice;
|
||||
|
||||
@@ -42,7 +43,7 @@ impl PixelType {
|
||||
/// Additional information about pixel type.
|
||||
pub trait Pixel
|
||||
where
|
||||
Self: Copy + Sized,
|
||||
Self: Copy + Sized + Debug,
|
||||
{
|
||||
/// Type of pixel components
|
||||
type Component;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use std::arch::x86_64::*;
|
||||
use std::intrinsics::transmute;
|
||||
use std::ptr;
|
||||
|
||||
use crate::pixels::{U8x3, U8x4};
|
||||
|
||||
@@ -13,6 +14,18 @@ pub unsafe fn loadu_si256<T>(buf: &[T], index: usize) -> __m256i {
|
||||
_mm256_loadu_si256(buf.get_unchecked(index..).as_ptr() as *const __m256i)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub unsafe fn loadl_epi16<T>(buf: &[T], index: usize) -> __m128i {
|
||||
let mem_addr = buf.get_unchecked(index..).as_ptr() as *const i16;
|
||||
_mm_set_epi16(0, 0, 0, 0, 0, 0, 0, ptr::read_unaligned(mem_addr))
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub unsafe fn loadl_epi32<T>(buf: &[T], index: usize) -> __m128i {
|
||||
let mem_addr = buf.get_unchecked(index..).as_ptr() as *const i32;
|
||||
_mm_set_epi32(0, 0, 0, ptr::read_unaligned(mem_addr))
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub unsafe fn loadl_epi64<T>(buf: &[T], index: usize) -> __m128i {
|
||||
_mm_loadl_epi64(buf.get_unchecked(index..).as_ptr() as *const __m128i)
|
||||
@@ -52,3 +65,8 @@ pub unsafe fn ptr_i16_to_set1_epi32(buf: &[i16], index: usize) -> __m128i {
|
||||
pub unsafe fn ptr_i16_to_256set1_epi32(buf: &[i16], index: usize) -> __m256i {
|
||||
_mm256_set1_epi32(*(buf.get_unchecked(index..).as_ptr() as *const i32))
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub unsafe fn ptr_i16_to_256set1_epi64x(buf: &[i16], index: usize) -> __m256i {
|
||||
_mm256_set1_epi64x(*(buf.get_unchecked(index..).as_ptr() as *const i64))
|
||||
}
|
||||
|
||||
+12
-2
@@ -98,7 +98,12 @@ where
|
||||
}
|
||||
|
||||
let dst_buffer = dst_image.buffer();
|
||||
assert!(dst_buffer.iter().zip(res_buffer).all(|(&d, &r)| d == r));
|
||||
assert!(
|
||||
dst_buffer.iter().zip(res_buffer).all(|(&d, &r)| d == r),
|
||||
"failed test: src={:?}, expected_result={:?}",
|
||||
src_pixel,
|
||||
result_pixel
|
||||
);
|
||||
|
||||
// Inplace
|
||||
let rows: Vec<&mut [P]> = src_pixels
|
||||
@@ -120,7 +125,12 @@ where
|
||||
}
|
||||
|
||||
let src_buffer = unsafe { src_pixels.align_to::<u8>().1 };
|
||||
assert!(src_buffer.iter().zip(res_buffer).all(|(&s, &r)| s == r));
|
||||
assert!(
|
||||
src_buffer.iter().zip(res_buffer).all(|(&s, &r)| s == r),
|
||||
"failed inplace test: src={:?}, expected_result={:?}",
|
||||
src_pixel,
|
||||
result_pixel
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
+10
-10
@@ -162,11 +162,11 @@ fn downscale_u8x2() {
|
||||
assert_eq!(utils::image_checksum::<2>(&buffer), [2920348, 11054250]);
|
||||
|
||||
let mut cpu_extensions_vec = vec![CpuExtensions::None];
|
||||
// #[cfg(target_arch = "x86_64")]
|
||||
// {
|
||||
// cpu_extensions_vec.push(CpuExtensions::Sse4_1);
|
||||
// cpu_extensions_vec.push(CpuExtensions::Avx2);
|
||||
// }
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
{
|
||||
// cpu_extensions_vec.push(CpuExtensions::Sse4_1);
|
||||
cpu_extensions_vec.push(CpuExtensions::Avx2);
|
||||
}
|
||||
for cpu_extensions in cpu_extensions_vec {
|
||||
let buffer =
|
||||
downscale_test::<P>(ResizeAlg::Convolution(FilterType::Lanczos3), cpu_extensions);
|
||||
@@ -184,11 +184,11 @@ fn upscale_u8x2() {
|
||||
);
|
||||
|
||||
let mut cpu_extensions_vec = vec![CpuExtensions::None];
|
||||
// #[cfg(target_arch = "x86_64")]
|
||||
// {
|
||||
// cpu_extensions_vec.push(CpuExtensions::Sse4_1);
|
||||
// cpu_extensions_vec.push(CpuExtensions::Avx2);
|
||||
// }
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
{
|
||||
// cpu_extensions_vec.push(CpuExtensions::Sse4_1);
|
||||
cpu_extensions_vec.push(CpuExtensions::Avx2);
|
||||
}
|
||||
for cpu_extensions in cpu_extensions_vec {
|
||||
let buffer =
|
||||
upscale_test::<P>(ResizeAlg::Convolution(FilterType::Lanczos3), cpu_extensions);
|
||||
|
||||
Reference in New Issue
Block a user