external/fast_image_resize
1
0
Fork 0
mirror of https://github.com/Cykooz/fast_image_resize.git synced 2026-10-08 01:11:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added support for optimization with help of SSE4.1 and AVX2 for the F32x3 pixel type (#30).

Browse Source
This commit is contained in:
Kirill Kuzminykh
2024-06-21 22:23:31 +03:00
parent 15fa98863b
commit f42b3b6edc
9 changed files with 396 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+3 -4
View File
@@ -4,10 +4,9 @@
- Added support for optimization with help of `SSE4.1` and `AVX2` for
the `F32` pixel type.
- Added support for the new pixel type `PixelType::F32x2` with
optimizations for SSE4.1 and AVX2 (#30).
- Added basic support for the new pixel type `PixelType::F32x3` (#30).
- Added basic support for the new pixel type `PixelType::F32x4` (#30).
- Added support for new pixel types `F32x2` and `F32x3` with
optimizations for `SSE4.1` and `AVX2` (#30).
- Added basic support for the new pixel type `F32x4` (#30).
## [4.0.0] - 2024-05-13
README.md
+1 -1
View File
@@ -23,7 +23,7 @@ Supported pixel formats and available optimizations:
| I32 | One `i32` component per pixel (e.g. L) | - | - | - | - |
| F32 | One `f32` component per pixel (e.g. L) | + | + | - | - |
| F32x2 | Two `f32` components per pixel (e.g. LA32F) | + | + | - | - |
| F32x3 | Three `f32` components per pixel (e.g. RGB32F) | - | - | - | - |
| F32x3 | Three `f32` components per pixel (e.g. RGB32F) | + | + | - | - |
| F32x4 | Four `f32` components per pixel (e.g. RGBA32F) | - | - | - | - |
## Colorspace
src/convolution/f32x2/avx2.rs
+5 -74
View File
@@ -18,7 +18,7 @@ pub(crate) fn horiz_convolution(
let dst_iter = dst_view.iter_4_rows_mut();
for (src_rows, dst_rows) in src_iter.zip(dst_iter) {
unsafe {
horiz_convolution_four_rows(src_rows, dst_rows, &coefficients_chunks);
horiz_convolution_rows(src_rows, dst_rows, &coefficients_chunks);
}
}
@@ -27,7 +27,7 @@ pub(crate) fn horiz_convolution(
let dst_rows = dst_view.iter_rows_mut(yy);
for (src_row, dst_row) in src_rows.zip(dst_rows) {
unsafe {
horiz_convolution_one_row(src_row, dst_row, &coefficients_chunks);
horiz_convolution_rows([src_row], [dst_row], &coefficients_chunks);
}
}
}
@@ -39,12 +39,11 @@ pub(crate) fn horiz_convolution(
/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.0.len()
/// - precision <= MAX_COEFS_PRECISION
#[target_feature(enable = "avx2")]
unsafe fn horiz_convolution_four_rows(
src_rows: [&[F32x2]; 4],
dst_rows: [&mut [F32x2]; 4],
unsafe fn horiz_convolution_rows<const ROWS_COUNT: usize>(
src_rows: [&[F32x2]; ROWS_COUNT],
dst_rows: [&mut [F32x2]; ROWS_COUNT],
coefficients_chunks: &[CoefficientsChunk],
) {
const ROWS_COUNT: usize = 4;
let mut ll_buf = [0f64; 2];
for (dst_x, coeffs_chunk) in coefficients_chunks.iter().enumerate() {
@@ -116,71 +115,3 @@ unsafe fn horiz_convolution_four_rows(
}
}
}
/// For safety, it is necessary to ensure the following conditions:
/// - bounds.len() == dst_row.len()
/// - coeffs.len() == dst_rows.0.len() * window_size
/// - max(bound.start + bound.size for bound in bounds) <= src_row.len()
/// - precision <= MAX_COEFS_PRECISION
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn horiz_convolution_one_row(
src_row: &[F32x2],
dst_row: &mut [F32x2],
coefficients_chunks: &[CoefficientsChunk],
) {
let mut ll_buf = [0f64; 2];
for (dst_x, coeffs_chunk) in coefficients_chunks.iter().enumerate() {
let mut x: usize = coeffs_chunk.start as usize;
let mut ll_sum = _mm256_set1_pd(0.);
let mut coeffs = coeffs_chunk.values;
let coeffs_by_4 = coeffs.chunks_exact(4);
coeffs = coeffs_by_4.remainder();
for k in coeffs_by_4 {
let coeff0_f64x4 = _mm256_set_pd(k[1], k[1], k[0], k[0]);
let coeff1_f64x4 = _mm256_set_pd(k[3], k[3], k[2], k[2]);
let pixels04_f32x8 = simd_utils::loadu_ps256(src_row, x);
let pixels01_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps::<0>(pixels04_f32x8));
ll_sum = _mm256_add_pd(ll_sum, _mm256_mul_pd(pixels01_f64x4, coeff0_f64x4));
let pixels23_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps::<1>(pixels04_f32x8));
ll_sum = _mm256_add_pd(ll_sum, _mm256_mul_pd(pixels23_f64x4, coeff1_f64x4));
x += 4;
}
let coeffs_by_2 = coeffs.chunks_exact(2);
coeffs = coeffs_by_2.remainder();
for k in coeffs_by_2 {
let coeff_f64x4 = _mm256_set_pd(k[1], k[1], k[0], k[0]);
let pixels01_f32x4 = simd_utils::loadu_ps(src_row, x);
let pixels01_f64x4 = _mm256_cvtps_pd(pixels01_f32x4);
ll_sum = _mm256_add_pd(ll_sum, _mm256_mul_pd(pixels01_f64x4, coeff_f64x4));
x += 2;
}
if let Some(&k) = coeffs.first() {
let coeff0_f64x4 = _mm256_set1_pd(k);
let pixel = src_row.get_unchecked(x);
let pixel0_f64x4 = _mm256_set_pd(0., 0., pixel.0[1] as f64, pixel.0[0] as f64);
ll_sum = _mm256_add_pd(ll_sum, _mm256_mul_pd(pixel0_f64x4, coeff0_f64x4));
}
let sum_f64x2 = _mm_add_pd(
_mm256_extractf128_pd::<0>(ll_sum),
_mm256_extractf128_pd::<1>(ll_sum),
);
_mm_storeu_pd(ll_buf.as_mut_ptr(), sum_f64x2);
let dst_pixel = dst_row.get_unchecked_mut(dst_x);
dst_pixel.0 = ll_buf.map(|v| v as f32);
}
}
src/convolution/f32x2/mod.rs
+4 -4
View File
@@ -28,10 +28,10 @@ impl Convolution for F32x2 {
CpuExtensions::Avx2 => avx2::horiz_convolution(src_view, dst_view, offset, coeffs),
#[cfg(target_arch = "x86_64")]
CpuExtensions::Sse4_1 => sse4::horiz_convolution(src_view, dst_view, offset, coeffs),
#[cfg(target_arch = "aarch64")]
CpuExtensions::Neon => neon::horiz_convolution(src_view, dst_view, offset, coeffs),
#[cfg(target_arch = "wasm32")]
CpuExtensions::Simd128 => wasm32::horiz_convolution(src_view, dst_view, offset, coeffs),
// #[cfg(target_arch = "aarch64")]
// CpuExtensions::Neon => neon::horiz_convolution(src_view, dst_view, offset, coeffs),
// #[cfg(target_arch = "wasm32")]
// CpuExtensions::Simd128 => wasm32::horiz_convolution(src_view, dst_view, offset, coeffs),
_ => native::horiz_convolution(src_view, dst_view, offset, coeffs),
}
}
src/convolution/f32x2/sse4.rs
+5 -59
View File
@@ -18,7 +18,7 @@ pub(crate) fn horiz_convolution(
let dst_iter = dst_view.iter_4_rows_mut();
for (src_rows, dst_rows) in src_iter.zip(dst_iter) {
unsafe {
horiz_convolution_four_rows(src_rows, dst_rows, &coefficients_chunks);
horiz_convolution_rows(src_rows, dst_rows, &coefficients_chunks);
}
}
@@ -27,7 +27,7 @@ pub(crate) fn horiz_convolution(
let dst_rows = dst_view.iter_rows_mut(yy);
for (src_row, dst_row) in src_rows.zip(dst_rows) {
unsafe {
horiz_convolution_one_row(src_row, dst_row, &coefficients_chunks);
horiz_convolution_rows([src_row], [dst_row], &coefficients_chunks);
}
}
}
@@ -39,12 +39,11 @@ pub(crate) fn horiz_convolution(
/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.0.len()
/// - precision <= MAX_COEFS_PRECISION
#[target_feature(enable = "sse4.1")]
unsafe fn horiz_convolution_four_rows(
src_rows: [&[F32x2]; 4],
dst_rows: [&mut [F32x2]; 4],
unsafe fn horiz_convolution_rows<const ROWS_COUNT: usize>(
src_rows: [&[F32x2]; ROWS_COUNT],
dst_rows: [&mut [F32x2]; ROWS_COUNT],
coefficients_chunks: &[CoefficientsChunk],
) {
const ROWS_COUNT: usize = 4;
let mut ll_buf = [0f64; 2];
for (dst_x, coeffs_chunk) in coefficients_chunks.iter().enumerate() {
@@ -97,56 +96,3 @@ unsafe fn horiz_convolution_four_rows(
}
}
}
/// For safety, it is necessary to ensure the following conditions:
/// - bounds.len() == dst_row.len()
/// - coeffs.len() == dst_rows.0.len() * window_size
/// - max(bound.start + bound.size for bound in bounds) <= src_row.len()
/// - precision <= MAX_COEFS_PRECISION
#[inline]
#[target_feature(enable = "sse4.1")]
unsafe fn horiz_convolution_one_row(
src_row: &[F32x2],
dst_row: &mut [F32x2],
coefficients_chunks: &[CoefficientsChunk],
) {
let mut ll_buf = [0f64; 2];
for (dst_x, coeffs_chunk) in coefficients_chunks.iter().enumerate() {
let mut x: usize = coeffs_chunk.start as usize;
let mut ll_sum = _mm_set1_pd(0.);
let mut coeffs = coeffs_chunk.values;
let coeffs_by_2 = coeffs.chunks_exact(2);
coeffs = coeffs_by_2.remainder();
for k in coeffs_by_2 {
let coeff0_f64x2 = _mm_set1_pd(k[0]);
let coeff1_f64x2 = _mm_set1_pd(k[1]);
let source = simd_utils::loadu_ps(src_row, x);
let pixel0_f64 = _mm_cvtps_pd(source);
ll_sum = _mm_add_pd(ll_sum, _mm_mul_pd(pixel0_f64, coeff0_f64x2));
let pixel1_f64 = _mm_cvtps_pd(_mm_movehl_ps(source, source));
ll_sum = _mm_add_pd(ll_sum, _mm_mul_pd(pixel1_f64, coeff1_f64x2));
x += 2;
}
if let Some(&k) = coeffs.first() {
let coeff0_f64x2 = _mm_set1_pd(k);
let pixel = src_row.get_unchecked(x);
let source = _mm_set_ps(0., 0., pixel.0[1], pixel.0[0]);
let pixel0_f64 = _mm_cvtps_pd(source);
ll_sum = _mm_add_pd(ll_sum, _mm_mul_pd(pixel0_f64, coeff0_f64x2));
}
_mm_storeu_pd(ll_buf.as_mut_ptr(), ll_sum);
let dst_pixel = dst_row.get_unchecked_mut(dst_x);
dst_pixel.0 = ll_buf.map(|v| v as f32);
}
}
src/convolution/f32x3/avx2.rs
+201
View File
@@ -0,0 +1,201 @@
use std::arch::x86_64::*;
use crate::convolution::{Coefficients, CoefficientsChunk};
use crate::pixels::{F32x3, InnerPixel};
use crate::{simd_utils, ImageView, ImageViewMut};
#[inline]
pub(crate) fn horiz_convolution(
src_view: &impl ImageView<Pixel = F32x3>,
dst_view: &mut impl ImageViewMut<Pixel = F32x3>,
offset: u32,
coeffs: Coefficients,
) {
let coefficients_chunks = coeffs.get_chunks();
let dst_height = dst_view.height();
let src_iter = src_view.iter_4_rows(offset, dst_height + offset);
let dst_iter = dst_view.iter_4_rows_mut();
for (src_rows, dst_rows) in src_iter.zip(dst_iter) {
unsafe {
horiz_convolution_rows(src_rows, dst_rows, &coefficients_chunks);
}
}
let yy = dst_height - dst_height % 4;
let src_rows = src_view.iter_rows(yy + offset);
let dst_rows = dst_view.iter_rows_mut(yy);
for (src_row, dst_row) in src_rows.zip(dst_rows) {
unsafe {
horiz_convolution_rows([src_row], [dst_row], &coefficients_chunks);
}
}
}
/// For safety, it is necessary to ensure the following conditions:
/// - length of all rows in src_rows must be equal
/// - length of all rows in dst_rows must be equal
/// - coefficients_chunks.len() == dst_rows.0.len()
/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.0.len()
/// - precision <= MAX_COEFS_PRECISION
#[target_feature(enable = "avx2")]
unsafe fn horiz_convolution_rows<const ROWS_COUNT: usize>(
src_rows: [&[F32x3]; ROWS_COUNT],
dst_rows: [&mut [F32x3]; ROWS_COUNT],
coefficients_chunks: &[CoefficientsChunk],
) {
/*
|R0 G0 B0| |R1 G1 B1| |R2 G2|
|00 01 02| |03 04 05| |06 07|
|B2| |R3 G3 B3| |R4 G4 B4| |R5|
|00| |01 02 03| |04 05 06| |07|
|G5 B5| |R6 G6 B6| |R7 G7 B7|
|00 01| |02 03 04| |05 06 07|
*/
let mut rg_buf = [0f64; 2];
let mut br_buf = [0f64; 2];
let mut gb_buf = [0f64; 2];
for (dst_x, coeffs_chunk) in coefficients_chunks.iter().enumerate() {
let mut x: usize = coeffs_chunk.start as usize;
let mut rgbr_sums = [_mm256_set1_pd(0.); ROWS_COUNT];
let mut gbrg_sums = [_mm256_set1_pd(0.); ROWS_COUNT];
let mut brgb_sums = [_mm256_set1_pd(0.); ROWS_COUNT];
let mut coeffs = coeffs_chunk.values;
let coeffs_by_8 = coeffs.chunks_exact(8);
coeffs = coeffs_by_8.remainder();
for k in coeffs_by_8 {
let coeff0001_f64x4 = _mm256_set_pd(k[1], k[0], k[0], k[0]);
let coeff1122_f64x4 = _mm256_set_pd(k[2], k[2], k[1], k[1]);
let coeff2333_f64x4 = _mm256_set_pd(k[3], k[3], k[3], k[2]);
let coeff4445_f64x4 = _mm256_set_pd(k[5], k[4], k[4], k[4]);
let coeff5566_f64x4 = _mm256_set_pd(k[6], k[6], k[5], k[5]);
let coeff6777_f64x4 = _mm256_set_pd(k[7], k[7], k[7], k[6]);
for i in 0..ROWS_COUNT {
let c = x * 3;
let components = F32x3::components(src_rows[i]);
let rgb0rgb1rg2 = simd_utils::loadu_ps256(components, c);
let rgb0r1_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps::<0>(rgb0rgb1rg2));
rgbr_sums[i] =
_mm256_add_pd(rgbr_sums[i], _mm256_mul_pd(rgb0r1_f64x4, coeff0001_f64x4));
let gb1rg2_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps::<1>(rgb0rgb1rg2));
gbrg_sums[i] =
_mm256_add_pd(gbrg_sums[i], _mm256_mul_pd(gb1rg2_f64x4, coeff1122_f64x4));
let b2rgb3rgb4r5 = simd_utils::loadu_ps256(components, c + 8);
let b2rgb3_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps::<0>(b2rgb3rgb4r5));
brgb_sums[i] =
_mm256_add_pd(brgb_sums[i], _mm256_mul_pd(b2rgb3_f64x4, coeff2333_f64x4));
let rgb4r5_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps::<1>(b2rgb3rgb4r5));
rgbr_sums[i] =
_mm256_add_pd(rgbr_sums[i], _mm256_mul_pd(rgb4r5_f64x4, coeff4445_f64x4));
let gb5rgb6rgb7 = simd_utils::loadu_ps256(components, c + 16);
let gb5rg6_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps::<0>(gb5rgb6rgb7));
gbrg_sums[i] =
_mm256_add_pd(gbrg_sums[i], _mm256_mul_pd(gb5rg6_f64x4, coeff5566_f64x4));
let b6rgb7_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps::<1>(gb5rgb6rgb7));
brgb_sums[i] =
_mm256_add_pd(brgb_sums[i], _mm256_mul_pd(b6rgb7_f64x4, coeff6777_f64x4));
}
x += 8;
}
let coeffs_by_4 = coeffs.chunks_exact(4);
coeffs = coeffs_by_4.remainder();
for k in coeffs_by_4 {
let coeff0001_f64x4 = _mm256_set_pd(k[1], k[0], k[0], k[0]);
let coeff1122_f64x4 = _mm256_set_pd(k[2], k[2], k[1], k[1]);
let coeff2333_f64x4 = _mm256_set_pd(k[3], k[3], k[3], k[2]);
for i in 0..ROWS_COUNT {
let c = x * 3;
let components = F32x3::components(src_rows[i]);
let rgb0rgb1rg2 = simd_utils::loadu_ps256(components, c);
let rgb0r1_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps::<0>(rgb0rgb1rg2));
rgbr_sums[i] =
_mm256_add_pd(rgbr_sums[i], _mm256_mul_pd(rgb0r1_f64x4, coeff0001_f64x4));
let gb1rg2_f64x4 = _mm256_cvtps_pd(_mm256_extractf128_ps::<1>(rgb0rgb1rg2));
gbrg_sums[i] =
_mm256_add_pd(gbrg_sums[i], _mm256_mul_pd(gb1rg2_f64x4, coeff1122_f64x4));
let b2rgb3 = simd_utils::loadu_ps(components, c + 8);
let b2rgb3_f64x4 = _mm256_cvtps_pd(b2rgb3);
brgb_sums[i] =
_mm256_add_pd(brgb_sums[i], _mm256_mul_pd(b2rgb3_f64x4, coeff2333_f64x4));
}
x += 4;
}
let coeffs_by_2 = coeffs.chunks_exact(2);
coeffs = coeffs_by_2.remainder();
for k in coeffs_by_2 {
let coeff0001_f64x4 = _mm256_set_pd(k[1], k[0], k[0], k[0]);
let coeff11xx_f64x4 = _mm256_set_pd(0., 0., k[1], k[1]);
for i in 0..ROWS_COUNT {
let c = x * 3;
let components = F32x3::components(src_rows[i]);
let rgb0r1 = simd_utils::loadu_ps(components, c);
let rgb0r1_f64x4 = _mm256_cvtps_pd(rgb0r1);
rgbr_sums[i] =
_mm256_add_pd(rgbr_sums[i], _mm256_mul_pd(rgb0r1_f64x4, coeff0001_f64x4));
let g1 = *components.get_unchecked(c + 4);
let b1 = *components.get_unchecked(c + 5);
let gb1xx = _mm_set_ps(0., 0., b1, g1);
let gb1xx_f64x4 = _mm256_cvtps_pd(gb1xx);
gbrg_sums[i] =
_mm256_add_pd(gbrg_sums[i], _mm256_mul_pd(gb1xx_f64x4, coeff11xx_f64x4));
}
x += 2;
}
for &k in coeffs {
let coeff0000_f64x2 = _mm256_set1_pd(k);
for i in 0..ROWS_COUNT {
let pixel = src_rows[i].get_unchecked(x);
let rgb0x = _mm_set_ps(0., pixel.0[2], pixel.0[1], pixel.0[0]);
let rgb0x_f64x4 = _mm256_cvtps_pd(rgb0x);
rgbr_sums[i] =
_mm256_add_pd(rgbr_sums[i], _mm256_mul_pd(rgb0x_f64x4, coeff0000_f64x2));
}
x += 1;
}
for i in 0..ROWS_COUNT {
let rg0_f64x2 = _mm256_extractf128_pd::<0>(rgbr_sums[i]);
let rg1_f64x2 = _mm256_extractf128_pd::<1>(gbrg_sums[i]);
let rg_f64x2 = _mm_add_pd(rg0_f64x2, rg1_f64x2);
_mm_storeu_pd(rg_buf.as_mut_ptr(), rg_f64x2);
let br0_f64x2 = _mm256_extractf128_pd::<1>(rgbr_sums[i]);
let br1_f64x2 = _mm256_extractf128_pd::<0>(brgb_sums[i]);
let br_f64x2 = _mm_add_pd(br0_f64x2, br1_f64x2);
_mm_storeu_pd(br_buf.as_mut_ptr(), br_f64x2);
let gb0_f64x2 = _mm256_extractf128_pd::<0>(gbrg_sums[i]);
let gb1_f64x2 = _mm256_extractf128_pd::<1>(brgb_sums[i]);
let gb_f64x2 = _mm_add_pd(gb0_f64x2, gb1_f64x2);
_mm_storeu_pd(gb_buf.as_mut_ptr(), gb_f64x2);
let dst_pixel = dst_rows[i].get_unchecked_mut(dst_x);
dst_pixel.0 = [
(rg_buf[0] + br_buf[1]) as f32,
(rg_buf[1] + gb_buf[0]) as f32,
(br_buf[0] + gb_buf[1]) as f32,
];
}
}
}
src/convolution/f32x3/mod.rs
+12 -12
View File
@@ -5,13 +5,13 @@ use crate::{ImageView, ImageViewMut};
use super::{Coefficients, Convolution};
// #[cfg(target_arch = "x86_64")]
// mod avx2;
#[cfg(target_arch = "x86_64")]
mod avx2;
mod native;
// #[cfg(target_arch = "aarch64")]
// mod neon;
// #[cfg(target_arch = "x86_64")]
// mod sse4;
#[cfg(target_arch = "x86_64")]
mod sse4;
// #[cfg(target_arch = "wasm32")]
// mod wasm32;
@@ -24,14 +24,14 @@ impl Convolution for F32x3 {
cpu_extensions: CpuExtensions,
) {
match cpu_extensions {
// #[cfg(target_arch = "x86_64")]
// CpuExtensions::Avx2 => avx2::horiz_convolution(src_view, dst_view, offset, coeffs),
// #[cfg(target_arch = "x86_64")]
// CpuExtensions::Sse4_1 => sse4::horiz_convolution(src_view, dst_view, offset, coeffs),
#[cfg(target_arch = "aarch64")]
CpuExtensions::Neon => neon::horiz_convolution(src_view, dst_view, offset, coeffs),
#[cfg(target_arch = "wasm32")]
CpuExtensions::Simd128 => wasm32::horiz_convolution(src_view, dst_view, offset, coeffs),
#[cfg(target_arch = "x86_64")]
CpuExtensions::Avx2 => avx2::horiz_convolution(src_view, dst_view, offset, coeffs),
#[cfg(target_arch = "x86_64")]
CpuExtensions::Sse4_1 => sse4::horiz_convolution(src_view, dst_view, offset, coeffs),
// #[cfg(target_arch = "aarch64")]
// CpuExtensions::Neon => neon::horiz_convolution(src_view, dst_view, offset, coeffs),
// #[cfg(target_arch = "wasm32")]
// CpuExtensions::Simd128 => wasm32::horiz_convolution(src_view, dst_view, offset, coeffs),
_ => native::horiz_convolution(src_view, dst_view, offset, coeffs),
}
}
src/convolution/f32x3/sse4.rs
+160
View File
@@ -0,0 +1,160 @@
use std::arch::x86_64::*;
use crate::convolution::{Coefficients, CoefficientsChunk};
use crate::pixels::{F32x3, InnerPixel};
use crate::{simd_utils, ImageView, ImageViewMut};
#[inline]
pub(crate) fn horiz_convolution(
src_view: &impl ImageView<Pixel = F32x3>,
dst_view: &mut impl ImageViewMut<Pixel = F32x3>,
offset: u32,
coeffs: Coefficients,
) {
let coefficients_chunks = coeffs.get_chunks();
let dst_height = dst_view.height();
let src_iter = src_view.iter_2_rows(offset, dst_height + offset);
let dst_iter = dst_view.iter_2_rows_mut();
for (src_rows, dst_rows) in src_iter.zip(dst_iter) {
unsafe {
horiz_convolution_rows(src_rows, dst_rows, &coefficients_chunks);
}
}
let yy = dst_height - dst_height % 2;
let src_rows = src_view.iter_rows(yy + offset);
let dst_rows = dst_view.iter_rows_mut(yy);
for (src_row, dst_row) in src_rows.zip(dst_rows) {
unsafe {
horiz_convolution_rows([src_row], [dst_row], &coefficients_chunks);
}
}
}
/// For safety, it is necessary to ensure the following conditions:
/// - length of all rows in src_rows must be equal
/// - length of all rows in dst_rows must be equal
/// - coefficients_chunks.len() == dst_rows.0.len()
/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.0.len()
/// - precision <= MAX_COEFS_PRECISION
#[target_feature(enable = "sse4.1")]
unsafe fn horiz_convolution_rows<const ROWS_COUNT: usize>(
src_rows: [&[F32x3]; ROWS_COUNT],
dst_rows: [&mut [F32x3]; ROWS_COUNT],
coefficients_chunks: &[CoefficientsChunk],
) {
/*
|R0 G0 B0| |R1|
|00 01 02| |03|
|G1 B1| |R2 G2|
|00 01| |02 03|
|B2| |R3 G3 B3|
|00| |01 02 03|
*/
let mut rg_buf = [0f64; 2];
let mut br_buf = [0f64; 2];
let mut gb_buf = [0f64; 2];
for (dst_x, coeffs_chunk) in coefficients_chunks.iter().enumerate() {
let mut x: usize = coeffs_chunk.start as usize;
let mut rg_sums = [_mm_set1_pd(0.); ROWS_COUNT];
let mut br_sums = [_mm_set1_pd(0.); ROWS_COUNT];
let mut gb_sums = [_mm_set1_pd(0.); ROWS_COUNT];
let mut coeffs = coeffs_chunk.values;
let coeffs_by_4 = coeffs.chunks_exact(4);
coeffs = coeffs_by_4.remainder();
for k in coeffs_by_4 {
let coeff00_f64x2 = _mm_set1_pd(k[0]);
let coeff01_f64x2 = _mm_set_pd(k[1], k[0]);
let coeff11_f64x2 = _mm_set1_pd(k[1]);
let coeff22_f64x2 = _mm_set1_pd(k[2]);
let coeff23_f64x2 = _mm_set_pd(k[3], k[2]);
let coeff33_f64x2 = _mm_set1_pd(k[3]);
for i in 0..ROWS_COUNT {
let c = x * 3;
let components = F32x3::components(src_rows[i]);
let rgb0r1 = simd_utils::loadu_ps(components, c);
let rg0_f64x2 = _mm_cvtps_pd(rgb0r1);
rg_sums[i] = _mm_add_pd(rg_sums[i], _mm_mul_pd(rg0_f64x2, coeff00_f64x2));
let b0r1_f64x2 = _mm_cvtps_pd(_mm_movehl_ps(rgb0r1, rgb0r1));
br_sums[i] = _mm_add_pd(br_sums[i], _mm_mul_pd(b0r1_f64x2, coeff01_f64x2));
let gb1rg2 = simd_utils::loadu_ps(components, c + 4);
let gb1_f64x2 = _mm_cvtps_pd(gb1rg2);
gb_sums[i] = _mm_add_pd(gb_sums[i], _mm_mul_pd(gb1_f64x2, coeff11_f64x2));
let rg2_f64x2 = _mm_cvtps_pd(_mm_movehl_ps(gb1rg2, gb1rg2));
rg_sums[i] = _mm_add_pd(rg_sums[i], _mm_mul_pd(rg2_f64x2, coeff22_f64x2));
let b2rgb3 = simd_utils::loadu_ps(components, c + 8);
let b2r3_f64x2 = _mm_cvtps_pd(b2rgb3);
br_sums[i] = _mm_add_pd(br_sums[i], _mm_mul_pd(b2r3_f64x2, coeff23_f64x2));
let gb3_f64x2 = _mm_cvtps_pd(_mm_movehl_ps(b2rgb3, b2rgb3));
gb_sums[i] = _mm_add_pd(gb_sums[i], _mm_mul_pd(gb3_f64x2, coeff33_f64x2));
}
x += 4;
}
let coeffs_by_2 = coeffs.chunks_exact(2);
coeffs = coeffs_by_2.remainder();
for k in coeffs_by_2 {
let coeff00_f64x2 = _mm_set1_pd(k[0]);
let coeff01_f64x2 = _mm_set_pd(k[1], k[0]);
let coeff11_f64x2 = _mm_set1_pd(k[1]);
for i in 0..ROWS_COUNT {
let c = x * 3;
let components = F32x3::components(src_rows[i]);
let rgb0r1 = simd_utils::loadu_ps(components, c);
let rg0_f64x2 = _mm_cvtps_pd(rgb0r1);
rg_sums[i] = _mm_add_pd(rg_sums[i], _mm_mul_pd(rg0_f64x2, coeff00_f64x2));
let b0r1_f64x2 = _mm_cvtps_pd(_mm_movehl_ps(rgb0r1, rgb0r1));
br_sums[i] = _mm_add_pd(br_sums[i], _mm_mul_pd(b0r1_f64x2, coeff01_f64x2));
let g1 = *components.get_unchecked(c + 4);
let b1 = *components.get_unchecked(c + 5);
let gb1_f64x2 = _mm_set_pd(b1 as f64, g1 as f64);
gb_sums[i] = _mm_add_pd(gb_sums[i], _mm_mul_pd(gb1_f64x2, coeff11_f64x2));
}
x += 2;
}
for &k in coeffs {
let coeff00_f64x2 = _mm_set1_pd(k);
let coeff0x_f64x2 = _mm_set_pd(0., k);
for i in 0..ROWS_COUNT {
let pixel = src_rows[i].get_unchecked(x);
let rgb0x = _mm_set_ps(0., pixel.0[2], pixel.0[1], pixel.0[0]);
let rg0_f64x2 = _mm_cvtps_pd(rgb0x);
rg_sums[i] = _mm_add_pd(rg_sums[i], _mm_mul_pd(rg0_f64x2, coeff00_f64x2));
let b0x_f64x2 = _mm_cvtps_pd(_mm_movehl_ps(rgb0x, rgb0x));
br_sums[i] = _mm_add_pd(br_sums[i], _mm_mul_pd(b0x_f64x2, coeff0x_f64x2));
}
x += 1;
}
for i in 0..ROWS_COUNT {
_mm_storeu_pd(rg_buf.as_mut_ptr(), rg_sums[i]);
_mm_storeu_pd(br_buf.as_mut_ptr(), br_sums[i]);
_mm_storeu_pd(gb_buf.as_mut_ptr(), gb_sums[i]);
let dst_pixel = dst_rows[i].get_unchecked_mut(dst_x);
dst_pixel.0 = [
(rg_buf[0] + br_buf[1]) as f32,
(rg_buf[1] + gb_buf[0]) as f32,
(br_buf[0] + gb_buf[1]) as f32,
];
}
}
}
src/image_view.rs
+5
View File
@@ -73,6 +73,11 @@ pub unsafe trait ImageViewMut: ImageView {
/// Returns iterator by mutable slices with image rows.
fn iter_rows_mut(&mut self, start_row: u32) -> impl Iterator<Item = &mut [Self::Pixel]>;
/// Returns iterator by arrays with two mutable image rows.
fn iter_2_rows_mut(&mut self) -> ArrayChunks<impl Iterator<Item = &mut [Self::Pixel]>, 2> {
ArrayChunks::new(self.iter_rows_mut(0))
}
/// Returns iterator by arrays with four mutable image rows.
fn iter_4_rows_mut(&mut self) -> ArrayChunks<impl Iterator<Item = &mut [Self::Pixel]>, 4> {
ArrayChunks::new(self.iter_rows_mut(0))