external/fast_image_resize
1
0
Fork 0
mirror of https://github.com/Cykooz/fast_image_resize.git synced 2026-10-08 01:11:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Slightly improved (about 3%) speed of AVX2 implementation of Convolution trait for U8x3 and U8x4 images.

Browse Source
This commit is contained in:
Kirill Kuzminykh
2023-12-06 01:03:17 +03:00
parent 63f6b517ab
commit bb6ffbedfc
22 changed files with 233 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+7
View File
@@ -1,3 +1,10 @@
## [Unreleased] - ReleaseDate
### Changed
- Slightly improved (about 3%) speed of `AVX2` implementation of `Convolution` trait
for `U8x3` and `U8x4` images.
## [2.7.3] - 2023-05-07
### Fixed
resizer/src/main.rs
+1 -1
View File
@@ -105,7 +105,7 @@ fn resize(cli: &Cli) -> Result<()> {
fn open_source_image(cli: &Cli) -> Result<(fr::Image<'static>, ColorType, fr::PixelType)> {
let source_path = &cli.source_path;
debug!("Opening the source image {:?}", source_path);
let image = ImageReader::open(&source_path)
let image = ImageReader::open(source_path)
.with_context(|| format!("Failed to read source file from {:?}", source_path))?
.decode()
.with_context(|| "Failed to decode source image")?;
src/alpha/common.rs
+2 -2
View File
@@ -16,7 +16,7 @@ const fn recip_alpha_array(precision: u32) -> [u32; 256] {
let scaled_max = 255 * scale;
let mut i: usize = 1;
while i < 256 {
res[i] = (((scaled_max / i as u32) + 1) >> 1) as u32;
res[i] = ((scaled_max / i as u32) + 1) >> 1;
i += 1;
}
res
@@ -28,7 +28,7 @@ const fn recip_alpha16_array(precision: u64) -> [u64; 65536] {
let scaled_max = 0xffff * scale;
let mut i: usize = 1;
while i < 65536 {
res[i] = (((scaled_max / i as u64) + 1) >> 1) as u64;
res[i] = ((scaled_max / i as u64) + 1) >> 1;
i += 1;
}
res
src/convolution/u16x1/avx2.rs
+2 -2
View File
@@ -188,7 +188,7 @@ unsafe fn horiz_convolution_four_rows(
// ll_sum.into_iter().enumerate() executes slowly than ll_sum.iter().enumerate()
for (i, &ll) in ll_sum.iter().enumerate() {
_mm256_storeu_si256((&mut ll_buf).as_mut_ptr() as *mut __m256i, ll);
_mm256_storeu_si256(ll_buf.as_mut_ptr() as *mut __m256i, ll);
let dst_pixel = dst_rows[i * 2].get_unchecked_mut(dst_x);
dst_pixel.0 = normalizer.clip(ll_buf[0] + ll_buf[1] + half_error);
@@ -340,7 +340,7 @@ unsafe fn horiz_convolution_one_row(
ll_sum = _mm256_add_epi64(ll_sum, _mm256_mul_epi32(source, coeff0_i64x4));
}
_mm256_storeu_si256((&mut ll_buf).as_mut_ptr() as *mut __m256i, ll_sum);
_mm256_storeu_si256(ll_buf.as_mut_ptr() as *mut __m256i, ll_sum);
let dst_pixel = dst_row.get_unchecked_mut(dst_x);
dst_pixel.0 = normalizer.clip(ll_buf.iter().sum::<i64>() + half_error);
}
src/convolution/u16x1/sse4.rs
+4 -4
View File
@@ -152,14 +152,14 @@ unsafe fn horiz_convolution_four_rows(
if let Some(&k) = coeffs.first() {
let coeff01_i64x2 = _mm_set_epi64x(0, k as i64);
for i in 0..4 {
let pixel = (*src_rows[i].get_unchecked(x)).0 as i64;
let pixel = src_rows[i].get_unchecked(x).0 as i64;
let source = _mm_set_epi64x(0, pixel);
ll_sum[i] = _mm_add_epi64(ll_sum[i], _mm_mul_epi32(source, coeff01_i64x2));
}
}
for i in 0..4 {
_mm_storeu_si128((&mut ll_buf).as_mut_ptr() as *mut __m128i, ll_sum[i]);
_mm_storeu_si128(ll_buf.as_mut_ptr() as *mut __m128i, ll_sum[i]);
let dst_pixel = dst_rows[i].get_unchecked_mut(dst_x);
dst_pixel.0 = normalizer.clip(ll_buf.iter().sum::<i64>() + half_error);
}
@@ -269,12 +269,12 @@ unsafe fn horiz_convolution_one_row(
if let Some(&k) = coeffs.first() {
let coeff01_i64x2 = _mm_set_epi64x(0, k as i64);
let pixel = (*src_row.get_unchecked(x)).0 as i64;
let pixel = src_row.get_unchecked(x).0 as i64;
let source = _mm_set_epi64x(0, pixel);
ll_sum = _mm_add_epi64(ll_sum, _mm_mul_epi32(source, coeff01_i64x2));
}
_mm_storeu_si128((&mut ll_buf).as_mut_ptr() as *mut __m128i, ll_sum);
_mm_storeu_si128(ll_buf.as_mut_ptr() as *mut __m128i, ll_sum);
let dst_pixel = dst_row.get_unchecked_mut(dst_x);
dst_pixel.0 = normalizer.clip(ll_buf[0] + ll_buf[1] + half_error);
}
src/convolution/u16x2/avx2.rs
+2 -2
View File
@@ -164,7 +164,7 @@ unsafe fn horiz_convolution_four_rows(
// ll_sum.into_iter().enumerate() executes slowly than ll_sum.iter().enumerate()
for (i, &ll) in ll_sum.iter().enumerate() {
_mm256_storeu_si256((&mut ll_buf).as_mut_ptr() as *mut __m256i, ll);
_mm256_storeu_si256(ll_buf.as_mut_ptr() as *mut __m256i, ll);
let dst_pixel = dst_rows[i * 2].get_unchecked_mut(dst_x);
dst_pixel.0 = [normalizer.clip(ll_buf[0]), normalizer.clip(ll_buf[1])];
@@ -308,7 +308,7 @@ unsafe fn horiz_convolution_one_row(
ll_sum = _mm256_add_epi64(ll_sum, _mm256_mul_epi32(p_i64x4, coeff0_i64x4));
}
_mm256_storeu_si256((&mut ll_buf).as_mut_ptr() as *mut __m256i, ll_sum);
_mm256_storeu_si256(ll_buf.as_mut_ptr() as *mut __m256i, ll_sum);
let dst_pixel = dst_row.get_unchecked_mut(dst_x);
dst_pixel.0 = [
normalizer.clip(ll_buf[0] + ll_buf[2] + half_error),
src/convolution/u16x2/sse4.rs
+2 -2
View File
@@ -147,7 +147,7 @@ unsafe fn horiz_convolution_four_rows(
}
for i in 0..4 {
_mm_storeu_si128((&mut ll_buf).as_mut_ptr() as *mut __m128i, ll_sum[i]);
_mm_storeu_si128(ll_buf.as_mut_ptr() as *mut __m128i, ll_sum[i]);
let dst_pixel = dst_rows[i].get_unchecked_mut(dst_x);
dst_pixel.0 = [normalizer.clip(ll_buf[0]), normalizer.clip(ll_buf[1])];
}
@@ -252,7 +252,7 @@ unsafe fn horiz_convolution_one_row(
ll_sum = _mm_add_epi64(ll_sum, _mm_mul_epi32(p_i64x2, coeff0_i64x2));
}
_mm_storeu_si128((&mut ll_buf).as_mut_ptr() as *mut __m128i, ll_sum);
_mm_storeu_si128(ll_buf.as_mut_ptr() as *mut __m128i, ll_sum);
let dst_pixel = dst_row.get_unchecked_mut(dst_x);
dst_pixel.0 = [normalizer.clip(ll_buf[0]), normalizer.clip(ll_buf[1])];
}
src/convolution/u16x3/avx2.rs
+6 -6
View File
@@ -161,9 +161,9 @@ unsafe fn horiz_convolution_four_rows(
}
for i in 0..4 {
_mm256_storeu_si256((&mut rg_buf).as_mut_ptr() as *mut __m256i, rg_sum[i]);
_mm256_storeu_si256((&mut rg_bb_buf).as_mut_ptr() as *mut __m256i, rg_bb_sum[i]);
_mm256_storeu_si256((&mut bbb_buf).as_mut_ptr() as *mut __m256i, bbb_sum[i]);
_mm256_storeu_si256(rg_buf.as_mut_ptr() as *mut __m256i, rg_sum[i]);
_mm256_storeu_si256(rg_bb_buf.as_mut_ptr() as *mut __m256i, rg_bb_sum[i]);
_mm256_storeu_si256(bbb_buf.as_mut_ptr() as *mut __m256i, bbb_sum[i]);
let dst_pixel = dst_rows[i].get_unchecked_mut(dst_x);
dst_pixel.0[0] = normalizer.clip(rg_buf[0] + rg_buf[2] + rg_bb_buf[0] + half_error);
dst_pixel.0[1] = normalizer.clip(rg_buf[1] + rg_buf[3] + rg_bb_buf[1] + half_error);
@@ -288,9 +288,9 @@ unsafe fn horiz_convolution_one_row(
x += 1;
}
_mm256_storeu_si256((&mut rg_buf).as_mut_ptr() as *mut __m256i, rg_sum);
_mm256_storeu_si256((&mut rg_bb_buf).as_mut_ptr() as *mut __m256i, rg_bb_sum);
_mm256_storeu_si256((&mut bbb_buf).as_mut_ptr() as *mut __m256i, bbb_sum);
_mm256_storeu_si256(rg_buf.as_mut_ptr() as *mut __m256i, rg_sum);
_mm256_storeu_si256(rg_bb_buf.as_mut_ptr() as *mut __m256i, rg_bb_sum);
_mm256_storeu_si256(bbb_buf.as_mut_ptr() as *mut __m256i, bbb_sum);
let dst_pixel = dst_row.get_unchecked_mut(dst_x);
dst_pixel.0[0] = normalizer.clip(rg_buf[0] + rg_buf[2] + rg_bb_buf[0] + half_error);
dst_pixel.0[1] = normalizer.clip(rg_buf[1] + rg_buf[3] + rg_bb_buf[1] + half_error);
src/convolution/u16x3/sse4.rs
+4 -4
View File
@@ -125,8 +125,8 @@ unsafe fn horiz_convolution_8u4x(
}
for i in 0..4 {
_mm_storeu_si128((&mut rg_buf).as_mut_ptr() as *mut __m128i, rg_sum[i]);
_mm_storeu_si128((&mut bb_buf).as_mut_ptr() as *mut __m128i, bb_sum[i]);
_mm_storeu_si128(rg_buf.as_mut_ptr() as *mut __m128i, rg_sum[i]);
_mm_storeu_si128(bb_buf.as_mut_ptr() as *mut __m128i, bb_sum[i]);
let dst_pixel = dst_rows[i].get_unchecked_mut(dst_x);
dst_pixel.0[0] = normalizer.clip(rg_buf[0] + half_error);
dst_pixel.0[1] = normalizer.clip(rg_buf[1] + half_error);
@@ -218,8 +218,8 @@ unsafe fn horiz_convolution_8u(
x += 1;
}
_mm_storeu_si128((&mut rg_buf).as_mut_ptr() as *mut __m128i, rg_sum);
_mm_storeu_si128((&mut bb_buf).as_mut_ptr() as *mut __m128i, bb_sum);
_mm_storeu_si128(rg_buf.as_mut_ptr() as *mut __m128i, rg_sum);
_mm_storeu_si128(bb_buf.as_mut_ptr() as *mut __m128i, bb_sum);
let dst_pixel = dst_row.get_unchecked_mut(dst_x);
dst_pixel.0[0] = normalizer.clip(rg_buf[0]);
dst_pixel.0[1] = normalizer.clip(rg_buf[1]);
src/convolution/u16x4/avx2.rs
+4 -4
View File
@@ -152,8 +152,8 @@ unsafe fn horiz_convolution_four_rows(
}
for i in 0..2 {
_mm256_storeu_si256((&mut rg_buf).as_mut_ptr() as *mut __m256i, rg_sum[i]);
_mm256_storeu_si256((&mut ba_buf).as_mut_ptr() as *mut __m256i, ba_sum[i]);
_mm256_storeu_si256(rg_buf.as_mut_ptr() as *mut __m256i, rg_sum[i]);
_mm256_storeu_si256(ba_buf.as_mut_ptr() as *mut __m256i, ba_sum[i]);
let dst_pixel = dst_rows[i * 2].get_unchecked_mut(dst_x);
dst_pixel.0 = [
@@ -288,8 +288,8 @@ unsafe fn horiz_convolution_one_row(
ba_sum = _mm256_add_epi64(ba_sum, _mm256_mul_epi32(ba_i64x4, coeff_i64x4));
}
_mm256_storeu_si256((&mut rg_buf).as_mut_ptr() as *mut __m256i, rg_sum);
_mm256_storeu_si256((&mut ba_buf).as_mut_ptr() as *mut __m256i, ba_sum);
_mm256_storeu_si256(rg_buf.as_mut_ptr() as *mut __m256i, rg_sum);
_mm256_storeu_si256(ba_buf.as_mut_ptr() as *mut __m256i, ba_sum);
let dst_pixel = dst_row.get_unchecked_mut(dst_x);
dst_pixel.0 = [
normalizer.clip(rg_buf[0] + rg_buf[2] + half_error),
src/convolution/u16x4/sse4.rs
+4 -4
View File
@@ -125,8 +125,8 @@ unsafe fn horiz_convolution_four_rows(
}
for i in 0..4 {
_mm_storeu_si128((&mut rg_buf).as_mut_ptr() as *mut __m128i, rg_sum[i]);
_mm_storeu_si128((&mut ba_buf).as_mut_ptr() as *mut __m128i, ba_sum[i]);
_mm_storeu_si128(rg_buf.as_mut_ptr() as *mut __m128i, rg_sum[i]);
_mm_storeu_si128(ba_buf.as_mut_ptr() as *mut __m128i, ba_sum[i]);
let dst_pixel = dst_rows[i].get_unchecked_mut(dst_x);
dst_pixel.0 = [
normalizer.clip(rg_buf[0]),
@@ -217,8 +217,8 @@ unsafe fn horiz_convolution_one_row(
ba_sum = _mm_add_epi64(ba_sum, _mm_mul_epi32(ba_i64x2, coeff0_i64x2));
}
_mm_storeu_si128((&mut rg_buf).as_mut_ptr() as *mut __m128i, rg_sum);
_mm_storeu_si128((&mut ba_buf).as_mut_ptr() as *mut __m128i, ba_sum);
_mm_storeu_si128(rg_buf.as_mut_ptr() as *mut __m128i, rg_sum);
_mm_storeu_si128(ba_buf.as_mut_ptr() as *mut __m128i, ba_sum);
let dst_pixel = dst_row.get_unchecked_mut(dst_x);
dst_pixel.0 = [
normalizer.clip(rg_buf[0]),
src/convolution/u8x1/avx2.rs
+5 -5
View File
@@ -20,14 +20,14 @@ pub(crate) fn horiz_convolution(
let dst_iter = dst_image.iter_4_rows_mut();
for (src_rows, dst_rows) in src_iter.zip(dst_iter) {
unsafe {
horiz_convolution_8u4x(src_rows, dst_rows, &coefficients_chunks, &normalizer);
horiz_convolution_four_rows(src_rows, dst_rows, &coefficients_chunks, &normalizer);
}
}
let mut yy = dst_height - dst_height % 4;
while yy < dst_height {
unsafe {
horiz_convolution_8u(
horiz_convolution_one_row(
src_image.get_row(yy + offset).unwrap(),
dst_image.get_row_mut(yy).unwrap(),
&coefficients_chunks,
@@ -46,7 +46,7 @@ pub(crate) fn horiz_convolution(
/// - precision <= MAX_COEFS_PRECISION
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn horiz_convolution_8u4x(
unsafe fn horiz_convolution_four_rows(
src_rows: [&[U8]; 4],
dst_rows: [&mut &mut [U8]; 4],
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
@@ -115,7 +115,7 @@ unsafe fn horiz_convolution_8u4x(
/// - precision <= MAX_COEFS_PRECISION
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn horiz_convolution_8u(
unsafe fn horiz_convolution_one_row(
src_row: &[U8],
dst_row: &mut [U8],
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
@@ -183,7 +183,7 @@ unsafe fn hsum_epi32_avx(x: __m128i) -> i32 {
// 3-operand non-destructive AVX lets us save a byte without needing a movdqa
let hi64 = _mm_unpackhi_epi64(x, x);
let sum64 = _mm_add_epi32(hi64, x);
const I: i32 = ((2 << 6) | (3 << 4) | 1) as i32;
const I: i32 = (2 << 6) | (3 << 4) | 1;
let hi32 = _mm_shuffle_epi32::<I>(sum64); // Swap the low two elements
let sum32 = _mm_add_epi32(sum64, hi32);
_mm_cvtsi128_si32(sum32) // movd
src/convolution/u8x1/sse4.rs
+2 -2
View File
@@ -27,7 +27,7 @@ pub(crate) fn horiz_convolution(
let mut yy = dst_height - dst_height % 4;
while yy < dst_height {
unsafe {
horiz_convolution_row(
horiz_convolution_one_row(
src_image.get_row(yy + offset).unwrap(),
dst_image.get_row_mut(yy).unwrap(),
&coefficients_chunks,
@@ -114,7 +114,7 @@ unsafe fn horiz_convolution_four_rows(
/// - precision <= MAX_COEFS_PRECISION
#[inline]
#[target_feature(enable = "sse4.1")]
unsafe fn horiz_convolution_row(
unsafe fn horiz_convolution_one_row(
src_row: &[U8],
dst_row: &mut [U8],
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
src/convolution/u8x2/avx2.rs
+1 -1
View File
@@ -395,7 +395,7 @@ unsafe fn horiz_convolution_one_row(
let mut coeffs: [i16; 3] = [0; 3];
for (i, &coeff) in reminder1.iter().enumerate() {
coeffs[i] = coeff;
let pixel: [u8; 2] = (*src_row.get_unchecked(x)).0.to_le_bytes();
let pixel: [u8; 2] = src_row.get_unchecked(x).0.to_le_bytes();
pixels[i * 2] = pixel[0] as i16;
pixels[i * 2 + 1] = pixel[1] as i16;
x += 1;
src/convolution/u8x2/sse4.rs
+1 -1
View File
@@ -292,7 +292,7 @@ unsafe fn horiz_convolution_one_row(
let mut coeffs: [i16; 3] = [0; 3];
for (i, &coeff) in reminder1.iter().enumerate() {
coeffs[i] = coeff;
let pixel: [u8; 2] = (*src_row.get_unchecked(x)).0.to_le_bytes();
let pixel: [u8; 2] = src_row.get_unchecked(x).0.to_le_bytes();
pixels[i * 2] = pixel[0] as i16;
pixels[i * 2 + 1] = pixel[1] as i16;
x += 1;
src/convolution/u8x3/avx2.rs
+25 -24
View File
@@ -15,6 +15,21 @@ pub(crate) fn horiz_convolution(
) {
let normalizer = optimisations::Normalizer16::new(coeffs);
let precision = normalizer.precision();
macro_rules! call {
($imm8:expr) => {{
horiz_convolution_p::<$imm8>(src_image, dst_image, offset, normalizer);
}};
}
constify_imm8!(precision, call);
}
fn horiz_convolution_p<const PRECISION: i32>(
src_image: &ImageView<U8x3>,
dst_image: &mut ImageViewMut<U8x3>,
offset: u32,
normalizer: optimisations::Normalizer16,
) {
let coefficients_chunks = normalizer.normalized_chunks();
let dst_height = dst_image.height().get();
@@ -22,18 +37,17 @@ pub(crate) fn horiz_convolution(
let dst_iter = dst_image.iter_4_rows_mut();
for (src_rows, dst_rows) in src_iter.zip(dst_iter) {
unsafe {
horiz_convolution_8u4x(src_rows, dst_rows, &coefficients_chunks, precision);
horiz_convolution_four_rows::<PRECISION>(src_rows, dst_rows, &coefficients_chunks);
}
}
let mut yy = dst_height - dst_height % 4;
while yy < dst_height {
unsafe {
horiz_convolution_8u(
horiz_convolution_one_row::<PRECISION>(
src_image.get_row(yy + offset).unwrap(),
dst_image.get_row_mut(yy).unwrap(),
&coefficients_chunks,
precision,
);
}
yy += 1;
@@ -48,14 +62,13 @@ pub(crate) fn horiz_convolution(
/// - precision <= MAX_COEFS_PRECISION
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn horiz_convolution_8u4x(
unsafe fn horiz_convolution_four_rows<const PRECISION: i32>(
src_rows: [&[U8x3]; 4],
dst_rows: [&mut &mut [U8x3]; 4],
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
precision: u8,
) {
let zero = _mm256_setzero_si256();
let initial = _mm256_set1_epi32(1 << (precision - 1));
let initial = _mm256_set1_epi32(1 << (PRECISION - 1));
let src_width = src_rows[0].len();
/*
@@ -179,13 +192,8 @@ unsafe fn horiz_convolution_8u4x(
x += 1;
}
macro_rules! call {
($imm8:expr) => {{
sss0 = _mm256_srai_epi32::<$imm8>(sss0);
sss1 = _mm256_srai_epi32::<$imm8>(sss1);
}};
}
constify_imm8!(precision, call);
sss0 = _mm256_srai_epi32::<PRECISION>(sss0);
sss1 = _mm256_srai_epi32::<PRECISION>(sss1);
sss0 = _mm256_packs_epi32(sss0, zero);
sss1 = _mm256_packs_epi32(sss1, zero);
@@ -217,11 +225,10 @@ unsafe fn horiz_convolution_8u4x(
/// - precision <= MAX_COEFS_PRECISION
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn horiz_convolution_8u(
unsafe fn horiz_convolution_one_row<const PRECISION: i32>(
src_row: &[U8x3],
dst_row: &mut [U8x3],
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
precision: u8,
) {
#[rustfmt::skip]
let sh1 = _mm256_set_epi8(
@@ -280,10 +287,10 @@ unsafe fn horiz_convolution_8u(
// (32 bytes) / (3 bytes per pixel) = 10 whole pixels + 2 bytes
let mut sss = if coeffs.len() < 8 || x >= max_x {
_mm_set1_epi32(1 << (precision - 1))
_mm_set1_epi32(1 << (PRECISION - 1))
} else {
// Lower part will be added to higher, use only half of the error
let mut sss256 = _mm256_set1_epi32(1 << (precision - 2));
let mut sss256 = _mm256_set1_epi32(1 << (PRECISION - 2));
let coeffs_by_8 = coeffs.chunks_exact(8);
for k in coeffs_by_8 {
@@ -363,13 +370,7 @@ unsafe fn horiz_convolution_8u(
x += 1;
}
macro_rules! call {
($imm8:expr) => {{
sss = _mm_srai_epi32::<$imm8>(sss);
}};
}
constify_imm8!(precision, call);
sss = _mm_srai_epi32::<PRECISION>(sss);
sss = _mm_packs_epi32(sss, sss);
let pixel: u32 = transmute(_mm_cvtsi128_si32(_mm_packus_epi16(sss, sss)));
let bytes = pixel.to_le_bytes();
src/convolution/u8x3/sse4.rs
+27 -25
View File
@@ -15,6 +15,21 @@ pub(crate) fn horiz_convolution(
) {
let normalizer = optimisations::Normalizer16::new(coeffs);
let precision = normalizer.precision();
macro_rules! call {
($imm8:expr) => {{
horiz_convolution_p::<$imm8>(src_image, dst_image, offset, normalizer);
}};
}
constify_imm8!(precision, call);
}
fn horiz_convolution_p<const PRECISION: i32>(
src_image: &ImageView<U8x3>,
dst_image: &mut ImageViewMut<U8x3>,
offset: u32,
normalizer: optimisations::Normalizer16,
) {
let coefficients_chunks = normalizer.normalized_chunks();
let dst_height = dst_image.height().get();
@@ -22,18 +37,17 @@ pub(crate) fn horiz_convolution(
let dst_iter = dst_image.iter_4_rows_mut();
for (src_rows, dst_rows) in src_iter.zip(dst_iter) {
unsafe {
horiz_convolution_8u4x(src_rows, dst_rows, &coefficients_chunks, precision);
horiz_convolution_four_rows::<PRECISION>(src_rows, dst_rows, &coefficients_chunks);
}
}
let mut yy = dst_height - dst_height % 4;
while yy < dst_height {
unsafe {
horiz_convolution_8u(
horiz_convolution_one_row::<PRECISION>(
src_image.get_row(yy + offset).unwrap(),
dst_image.get_row_mut(yy).unwrap(),
&coefficients_chunks,
precision,
);
}
yy += 1;
@@ -48,14 +62,13 @@ pub(crate) fn horiz_convolution(
/// - precision <= MAX_COEFS_PRECISION
#[inline]
#[target_feature(enable = "sse4.1")]
unsafe fn horiz_convolution_8u4x(
unsafe fn horiz_convolution_four_rows<const PRECISION: i32>(
src_rows: [&[U8x3]; 4],
dst_rows: [&mut &mut [U8x3]; 4],
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
precision: u8,
) {
let zero = _mm_setzero_si128();
let initial = _mm_set1_epi32(1 << (precision - 1));
let initial = _mm_set1_epi32(1 << (PRECISION - 1));
let src_width = src_rows[0].len();
/*
@@ -153,15 +166,11 @@ unsafe fn horiz_convolution_8u4x(
x += 1;
}
macro_rules! call {
($imm8:expr) => {{
sss_a[0] = _mm_srai_epi32::<$imm8>(sss_a[0]);
sss_a[1] = _mm_srai_epi32::<$imm8>(sss_a[1]);
sss_a[2] = _mm_srai_epi32::<$imm8>(sss_a[2]);
sss_a[3] = _mm_srai_epi32::<$imm8>(sss_a[3]);
}};
}
constify_imm8!(precision, call);
sss_a[0] = _mm_srai_epi32::<PRECISION>(sss_a[0]);
sss_a[1] = _mm_srai_epi32::<PRECISION>(sss_a[1]);
sss_a[2] = _mm_srai_epi32::<PRECISION>(sss_a[2]);
sss_a[3] = _mm_srai_epi32::<PRECISION>(sss_a[3]);
for i in 0..4 {
let sss = _mm_packs_epi32(sss_a[i], zero);
@@ -179,11 +188,10 @@ unsafe fn horiz_convolution_8u4x(
/// - precision <= MAX_COEFS_PRECISION
#[inline]
#[target_feature(enable = "sse4.1")]
unsafe fn horiz_convolution_8u(
unsafe fn horiz_convolution_one_row<const PRECISION: i32>(
src_row: &[U8x3],
dst_row: &mut [U8x3],
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
precision: u8,
) {
#[rustfmt::skip]
let pix_sh1 = _mm_set_epi8(
@@ -220,7 +228,7 @@ unsafe fn horiz_convolution_8u(
let x_start = coeffs_chunk.start as usize;
let mut x = x_start;
let mut coeffs = coeffs_chunk.values;
let mut sss = _mm_set1_epi32(1 << (precision - 1));
let mut sss = _mm_set1_epi32(1 << (PRECISION - 1));
// Next block of code will be load source pixels by 16 bytes per time.
// We must guarantee what this process will not go beyond
@@ -277,13 +285,7 @@ unsafe fn horiz_convolution_8u(
x += 1;
}
macro_rules! call {
($imm8:expr) => {{
sss = _mm_srai_epi32::<$imm8>(sss);
}};
}
constify_imm8!(precision, call);
sss = _mm_srai_epi32::<PRECISION>(sss);
sss = _mm_packs_epi32(sss, sss);
let pixel: u32 = transmute(_mm_cvtsi128_si32(_mm_packus_epi16(sss, sss)));
let bytes = pixel.to_le_bytes();
src/convolution/u8x4/avx2.rs
+25 -23
View File
@@ -18,6 +18,21 @@ pub(crate) fn horiz_convolution(
) {
let normalizer = optimisations::Normalizer16::new(coeffs);
let precision = normalizer.precision();
macro_rules! call {
($imm8:expr) => {{
horiz_convolution_p::<$imm8>(src_image, dst_image, offset, normalizer);
}};
}
constify_imm8!(precision, call);
}
fn horiz_convolution_p<const PRECISION: i32>(
src_image: &ImageView<U8x4>,
dst_image: &mut ImageViewMut<U8x4>,
offset: u32,
normalizer: optimisations::Normalizer16,
) {
let coefficients_chunks = normalizer.normalized_chunks();
let dst_height = dst_image.height().get();
@@ -25,18 +40,17 @@ pub(crate) fn horiz_convolution(
let dst_iter = dst_image.iter_4_rows_mut();
for (src_rows, dst_rows) in src_iter.zip(dst_iter) {
unsafe {
horiz_convolution_8u4x(src_rows, dst_rows, &coefficients_chunks, precision);
horiz_convolution_four_rows::<PRECISION>(src_rows, dst_rows, &coefficients_chunks);
}
}
let mut yy = dst_height - dst_height % 4;
while yy < dst_height {
unsafe {
horiz_convolution_8u(
horiz_convolution_one_row::<PRECISION>(
src_image.get_row(yy + offset).unwrap(),
dst_image.get_row_mut(yy).unwrap(),
&coefficients_chunks,
precision,
);
}
yy += 1;
@@ -51,14 +65,13 @@ pub(crate) fn horiz_convolution(
/// - precision <= MAX_COEFS_PRECISION
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn horiz_convolution_8u4x(
unsafe fn horiz_convolution_four_rows<const PRECISION: i32>(
src_rows: [&[U8x4]; 4],
dst_rows: [&mut &mut [U8x4]; 4],
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
precision: u8,
) {
let zero = _mm256_setzero_si256();
let initial = _mm256_set1_epi32(1 << (precision - 1));
let initial = _mm256_set1_epi32(1 << (PRECISION - 1));
#[rustfmt::skip]
let sh1 = _mm256_set_epi8(
@@ -147,13 +160,8 @@ unsafe fn horiz_convolution_8u4x(
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
}
macro_rules! call {
($imm8:expr) => {{
sss0 = _mm256_srai_epi32::<$imm8>(sss0);
sss1 = _mm256_srai_epi32::<$imm8>(sss1);
}};
}
constify_imm8!(precision, call);
sss0 = _mm256_srai_epi32::<PRECISION>(sss0);
sss1 = _mm256_srai_epi32::<PRECISION>(sss1);
sss0 = _mm256_packs_epi32(sss0, zero);
sss1 = _mm256_packs_epi32(sss1, zero);
@@ -177,11 +185,10 @@ unsafe fn horiz_convolution_8u4x(
/// - precision <= MAX_COEFS_PRECISION
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn horiz_convolution_8u(
unsafe fn horiz_convolution_one_row<const PRECISION: i32>(
src_row: &[U8x4],
dst_row: &mut [U8x4],
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
precision: u8,
) {
#[rustfmt::skip]
let sh1 = _mm256_set_epi8(
@@ -220,10 +227,10 @@ unsafe fn horiz_convolution_8u(
let mut coeffs = coeffs_chunk.values;
let mut sss = if coeffs.len() < 8 {
_mm_set1_epi32(1 << (precision - 1))
_mm_set1_epi32(1 << (PRECISION - 1))
} else {
// Lower part will be added to higher, use only half of the error
let mut sss256 = _mm256_set1_epi32(1 << (precision - 2));
let mut sss256 = _mm256_set1_epi32(1 << (PRECISION - 2));
let coeffs_by_8 = coeffs.chunks_exact(8);
let reminder1 = coeffs_by_8.remainder();
@@ -286,12 +293,7 @@ unsafe fn horiz_convolution_8u(
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
}
macro_rules! call {
($imm8:expr) => {{
sss = _mm_srai_epi32::<$imm8>(sss);
}};
}
constify_imm8!(precision, call);
sss = _mm_srai_epi32::<PRECISION>(sss);
sss = _mm_packs_epi32(sss, sss);
*dst_row.get_unchecked_mut(dst_x) =
src/convolution/u8x4/sse4.rs
+26 -25
View File
@@ -18,6 +18,21 @@ pub(crate) fn horiz_convolution(
) {
let normalizer = optimisations::Normalizer16::new(coeffs);
let precision = normalizer.precision();
macro_rules! call {
($imm8:expr) => {{
horiz_convolution_p::<$imm8>(src_image, dst_image, offset, normalizer);
}};
}
constify_imm8!(precision, call);
}
fn horiz_convolution_p<const PRECISION: i32>(
src_image: &ImageView<U8x4>,
dst_image: &mut ImageViewMut<U8x4>,
offset: u32,
normalizer: optimisations::Normalizer16,
) {
let coefficients_chunks = normalizer.normalized_chunks();
let dst_height = dst_image.height().get();
@@ -25,18 +40,17 @@ pub(crate) fn horiz_convolution(
let dst_iter = dst_image.iter_4_rows_mut();
for (src_rows, dst_rows) in src_iter.zip(dst_iter) {
unsafe {
horiz_convolution_8u4x(src_rows, dst_rows, &coefficients_chunks, precision);
horiz_convolution_four_rows::<PRECISION>(src_rows, dst_rows, &coefficients_chunks);
}
}
let mut yy = dst_height - dst_height % 4;
while yy < dst_height {
unsafe {
horiz_convolution_8u(
horiz_convolution_one_row::<PRECISION>(
src_image.get_row(yy + offset).unwrap(),
dst_image.get_row_mut(yy).unwrap(),
&coefficients_chunks,
precision,
);
}
yy += 1;
@@ -50,13 +64,12 @@ pub(crate) fn horiz_convolution(
/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.0.len()
/// - precision <= MAX_COEFS_PRECISION
#[target_feature(enable = "sse4.1")]
unsafe fn horiz_convolution_8u4x(
unsafe fn horiz_convolution_four_rows<const PRECISION: i32>(
src_rows: [&[U8x4]; 4],
dst_rows: [&mut &mut [U8x4]; 4],
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
precision: u8,
) {
let initial = _mm_set1_epi32(1 << (precision - 1));
let initial = _mm_set1_epi32(1 << (PRECISION - 1));
let mask_lo = _mm_set_epi8(-1, 7, -1, 3, -1, 6, -1, 2, -1, 5, -1, 1, -1, 4, -1, 0);
let mask_hi = _mm_set_epi8(-1, 15, -1, 11, -1, 14, -1, 10, -1, 13, -1, 9, -1, 12, -1, 8);
let mask = _mm_set_epi8(-1, 7, -1, 3, -1, 6, -1, 2, -1, 5, -1, 1, -1, 4, -1, 0);
@@ -151,15 +164,10 @@ unsafe fn horiz_convolution_8u4x(
sss3 = _mm_add_epi32(sss3, _mm_madd_epi16(pix, mmk));
}
macro_rules! call {
($imm8:expr) => {{
sss0 = _mm_srai_epi32::<$imm8>(sss0);
sss1 = _mm_srai_epi32::<$imm8>(sss1);
sss2 = _mm_srai_epi32::<$imm8>(sss2);
sss3 = _mm_srai_epi32::<$imm8>(sss3);
}};
}
constify_imm8!(precision, call);
sss0 = _mm_srai_epi32::<PRECISION>(sss0);
sss1 = _mm_srai_epi32::<PRECISION>(sss1);
sss2 = _mm_srai_epi32::<PRECISION>(sss2);
sss3 = _mm_srai_epi32::<PRECISION>(sss3);
sss0 = _mm_packs_epi32(sss0, sss0);
sss1 = _mm_packs_epi32(sss1, sss1);
@@ -182,13 +190,12 @@ unsafe fn horiz_convolution_8u4x(
/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.len()
/// - precision <= MAX_COEFS_PRECISION
#[target_feature(enable = "sse4.1")]
unsafe fn horiz_convolution_8u(
unsafe fn horiz_convolution_one_row<const PRECISION: i32>(
src_row: &[U8x4],
dst_row: &mut [U8x4],
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
precision: u8,
) {
let initial = _mm_set1_epi32(1 << (precision - 1));
let initial = _mm_set1_epi32(1 << (PRECISION - 1));
let sh1 = _mm_set_epi8(-1, 11, -1, 3, -1, 10, -1, 2, -1, 9, -1, 1, -1, 8, -1, 0);
let sh2 = _mm_set_epi8(5, 4, 1, 0, 5, 4, 1, 0, 5, 4, 1, 0, 5, 4, 1, 0);
let sh3 = _mm_set_epi8(-1, 15, -1, 7, -1, 14, -1, 6, -1, 13, -1, 5, -1, 12, -1, 4);
@@ -268,13 +275,7 @@ unsafe fn horiz_convolution_8u(
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
}
macro_rules! call {
($imm8:expr) => {{
sss = _mm_srai_epi32::<$imm8>(sss);
}};
}
constify_imm8!(precision, call);
sss = _mm_srai_epi32::<PRECISION>(sss);
sss = _mm_packs_epi32(sss, sss);
*dst_row.get_unchecked_mut(dst_x) =
transmute(_mm_cvtsi128_si32(_mm_packus_epi16(sss, sss)));
src/convolution/vertical_u16/sse4.rs
+4 -4
View File
@@ -110,7 +110,7 @@ unsafe fn vert_convolution_into_one_row_u16<T: PixelExt<Component = u16>>(
let mut dst_ptr = dst_chunk.as_mut_ptr();
for x in 0..2 {
for sum in sums {
_mm_storeu_si128((&mut c_buf).as_mut_ptr() as *mut __m128i, sum[x]);
_mm_storeu_si128(c_buf.as_mut_ptr() as *mut __m128i, sum[x]);
*dst_ptr = normalizer.clip(c_buf[0]);
dst_ptr = dst_ptr.add(1);
*dst_ptr = normalizer.clip(c_buf[1]);
@@ -164,7 +164,7 @@ unsafe fn vert_convolution_into_one_row_u16<T: PixelExt<Component = u16>>(
// sums[i] = _mm_and_si128(sums[i] , mask);
// sums[i] = _mm_srl_epi64(sums[i] , precision_i64);
// _mm_packus_epi32(sums[i] , sums[i] );
_mm_storeu_si128((&mut c_buf).as_mut_ptr() as *mut __m128i, sum);
_mm_storeu_si128(c_buf.as_mut_ptr() as *mut __m128i, sum);
*dst_ptr = normalizer.clip(c_buf[0]);
dst_ptr = dst_ptr.add(1);
*dst_ptr = normalizer.clip(c_buf[1]);
@@ -212,12 +212,12 @@ unsafe fn vert_convolution_into_one_row_u16<T: PixelExt<Component = u16>>(
}
let mut dst_ptr = dst_chunk.as_mut_ptr();
_mm_storeu_si128((&mut c_buf).as_mut_ptr() as *mut __m128i, c01);
_mm_storeu_si128(c_buf.as_mut_ptr() as *mut __m128i, c01);
*dst_ptr = normalizer.clip(c_buf[0]);
dst_ptr = dst_ptr.add(1);
*dst_ptr = normalizer.clip(c_buf[1]);
dst_ptr = dst_ptr.add(1);
_mm_storeu_si128((&mut c_buf).as_mut_ptr() as *mut __m128i, c23);
_mm_storeu_si128(c_buf.as_mut_ptr() as *mut __m128i, c23);
*dst_ptr = normalizer.clip(c_buf[0]);
dst_ptr = dst_ptr.add(1);
*dst_ptr = normalizer.clip(c_buf[1]);
src/convolution/vertical_u8/avx2.rs
+36 -28
View File
@@ -16,20 +16,44 @@ pub(crate) fn vert_convolution<T>(
T: PixelExt<Component = u8>,
{
let normalizer = optimisations::Normalizer16::new(coeffs);
let precision = normalizer.precision();
macro_rules! call {
($imm8:expr) => {{
vert_convolution_p::<T, $imm8>(src_image, dst_image, offset, normalizer);
}};
}
constify_imm8!(precision, call);
}
fn vert_convolution_p<T, const PRECISION: i32>(
src_image: &ImageView<T>,
dst_image: &mut ImageViewMut<T>,
offset: u32,
normalizer: optimisations::Normalizer16,
) where
T: PixelExt<Component = u8>,
{
let coefficients_chunks = normalizer.normalized_chunks();
let src_x = offset as usize * T::count_of_components();
let dst_rows = dst_image.iter_rows_mut();
for (dst_row, coeffs_chunk) in dst_rows.zip(coefficients_chunks) {
unsafe {
vert_convolution_into_one_row_u8(src_image, dst_row, src_x, coeffs_chunk, &normalizer);
vert_convolution_into_one_row::<T, PRECISION>(
src_image,
dst_row,
src_x,
coeffs_chunk,
&normalizer,
);
}
}
}
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn vert_convolution_into_one_row_u8<T>(
unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
src_img: &ImageView<T>,
dst_row: &mut [T],
mut src_x: usize,
@@ -41,10 +65,9 @@ unsafe fn vert_convolution_into_one_row_u8<T>(
let y_start = coeffs_chunk.start;
let coeffs = coeffs_chunk.values;
let max_y = y_start + coeffs.len() as u32;
let precision = normalizer.precision();
let initial = _mm_set1_epi32(1 << (precision - 1));
let initial_256 = _mm256_set1_epi32(1 << (precision - 1));
let initial = _mm_set1_epi32(1 << (PRECISION as u8 - 1));
let initial_256 = _mm256_set1_epi32(1 << (PRECISION as u8 - 1));
let mut dst_u8 = T::components_mut(dst_row);
@@ -104,15 +127,10 @@ unsafe fn vert_convolution_into_one_row_u8<T>(
sss3 = _mm256_add_epi32(sss3, _mm256_madd_epi16(pix, mmk));
}
macro_rules! call {
($imm8:expr) => {{
sss0 = _mm256_srai_epi32::<$imm8>(sss0);
sss1 = _mm256_srai_epi32::<$imm8>(sss1);
sss2 = _mm256_srai_epi32::<$imm8>(sss2);
sss3 = _mm256_srai_epi32::<$imm8>(sss3);
}};
}
constify_imm8!(precision, call);
sss0 = _mm256_srai_epi32::<PRECISION>(sss0);
sss1 = _mm256_srai_epi32::<PRECISION>(sss1);
sss2 = _mm256_srai_epi32::<PRECISION>(sss2);
sss3 = _mm256_srai_epi32::<PRECISION>(sss3);
sss0 = _mm256_packs_epi32(sss0, sss1);
sss2 = _mm256_packs_epi32(sss2, sss3);
@@ -165,13 +183,8 @@ unsafe fn vert_convolution_into_one_row_u8<T>(
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
}
macro_rules! call {
($imm8:expr) => {{
sss0 = _mm_srai_epi32::<$imm8>(sss0);
sss1 = _mm_srai_epi32::<$imm8>(sss1);
}};
}
constify_imm8!(precision, call);
sss0 = _mm_srai_epi32::<PRECISION>(sss0);
sss1 = _mm_srai_epi32::<PRECISION>(sss1);
sss0 = _mm_packs_epi32(sss0, sss1);
sss0 = _mm_packus_epi16(sss0, sss0);
@@ -211,12 +224,7 @@ unsafe fn vert_convolution_into_one_row_u8<T>(
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
}
macro_rules! call {
($imm8:expr) => {{
sss = _mm_srai_epi32::<$imm8>(sss);
}};
}
constify_imm8!(precision, call);
sss = _mm_srai_epi32::<PRECISION>(sss);
sss = _mm_packs_epi32(sss, sss);
let dst_ptr = dst_chunk.as_mut_ptr() as *mut i32;
@@ -230,7 +238,7 @@ unsafe fn vert_convolution_into_one_row_u8<T>(
native::convolution_by_u8(
src_img,
normalizer,
1 << (precision - 1),
1 << (PRECISION as u8 - 1),
dst_u8,
src_x,
y_start,
src/convolution/vertical_u8/sse4.rs
+43 -33
View File
@@ -7,26 +7,52 @@ use crate::simd_utils;
use crate::{ImageView, ImageViewMut};
#[inline]
pub(crate) fn vert_convolution<T: PixelExt<Component = u8>>(
pub(crate) fn vert_convolution<T>(
src_image: &ImageView<T>,
dst_image: &mut ImageViewMut<T>,
offset: u32,
coeffs: Coefficients,
) {
) where
T: PixelExt<Component = u8>,
{
let normalizer = optimisations::Normalizer16::new(coeffs);
let precision = normalizer.precision();
macro_rules! call {
($imm8:expr) => {{
vert_convolution_p::<T, $imm8>(src_image, dst_image, offset, normalizer);
}};
}
constify_imm8!(precision, call);
}
fn vert_convolution_p<T, const PRECISION: i32>(
src_image: &ImageView<T>,
dst_image: &mut ImageViewMut<T>,
offset: u32,
normalizer: optimisations::Normalizer16,
) where
T: PixelExt<Component = u8>,
{
let coefficients_chunks = normalizer.normalized_chunks();
let src_x = offset as usize * T::count_of_components();
let dst_rows = dst_image.iter_rows_mut();
for (dst_row, coeffs_chunk) in dst_rows.zip(coefficients_chunks) {
unsafe {
vert_convolution_into_one_row_u8(src_image, dst_row, src_x, coeffs_chunk, &normalizer);
vert_convolution_into_one_row::<T, PRECISION>(
src_image,
dst_row,
src_x,
coeffs_chunk,
&normalizer,
);
}
}
}
#[target_feature(enable = "sse4.1")]
unsafe fn vert_convolution_into_one_row_u8<T: PixelExt<Component = u8>>(
unsafe fn vert_convolution_into_one_row<T: PixelExt<Component = u8>, const PRECISION: i32>(
src_img: &ImageView<T>,
dst_row: &mut [T],
mut src_x: usize,
@@ -36,10 +62,9 @@ unsafe fn vert_convolution_into_one_row_u8<T: PixelExt<Component = u8>>(
let y_start = coeffs_chunk.start;
let coeffs = coeffs_chunk.values;
let max_y = y_start + coeffs.len() as u32;
let precision = normalizer.precision();
let mut dst_u8 = T::components_mut(dst_row);
let initial = _mm_set1_epi32(1 << (precision - 1));
let initial = _mm_set1_epi32(1 << (PRECISION - 1));
let mut dst_chunks_32 = dst_u8.chunks_exact_mut(32);
for dst_chunk in &mut dst_chunks_32 {
@@ -128,19 +153,14 @@ unsafe fn vert_convolution_into_one_row_u8<T: PixelExt<Component = u8>>(
sss7 = _mm_add_epi32(sss7, _mm_madd_epi16(pix, mmk));
}
macro_rules! call {
($imm8:expr) => {{
sss0 = _mm_srai_epi32::<$imm8>(sss0);
sss1 = _mm_srai_epi32::<$imm8>(sss1);
sss2 = _mm_srai_epi32::<$imm8>(sss2);
sss3 = _mm_srai_epi32::<$imm8>(sss3);
sss4 = _mm_srai_epi32::<$imm8>(sss4);
sss5 = _mm_srai_epi32::<$imm8>(sss5);
sss6 = _mm_srai_epi32::<$imm8>(sss6);
sss7 = _mm_srai_epi32::<$imm8>(sss7);
}};
}
constify_imm8!(precision, call);
sss0 = _mm_srai_epi32::<PRECISION>(sss0);
sss1 = _mm_srai_epi32::<PRECISION>(sss1);
sss2 = _mm_srai_epi32::<PRECISION>(sss2);
sss3 = _mm_srai_epi32::<PRECISION>(sss3);
sss4 = _mm_srai_epi32::<PRECISION>(sss4);
sss5 = _mm_srai_epi32::<PRECISION>(sss5);
sss6 = _mm_srai_epi32::<PRECISION>(sss6);
sss7 = _mm_srai_epi32::<PRECISION>(sss7);
sss0 = _mm_packs_epi32(sss0, sss1);
sss2 = _mm_packs_epi32(sss2, sss3);
@@ -195,13 +215,8 @@ unsafe fn vert_convolution_into_one_row_u8<T: PixelExt<Component = u8>>(
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
}
macro_rules! call {
($imm8:expr) => {{
sss0 = _mm_srai_epi32::<$imm8>(sss0);
sss1 = _mm_srai_epi32::<$imm8>(sss1);
}};
}
constify_imm8!(precision, call);
sss0 = _mm_srai_epi32::<PRECISION>(sss0);
sss1 = _mm_srai_epi32::<PRECISION>(sss1);
sss0 = _mm_packs_epi32(sss0, sss1);
sss0 = _mm_packus_epi16(sss0, sss0);
@@ -241,12 +256,7 @@ unsafe fn vert_convolution_into_one_row_u8<T: PixelExt<Component = u8>>(
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
}
macro_rules! call {
($imm8:expr) => {{
sss = _mm_srai_epi32::<$imm8>(sss);
}};
}
constify_imm8!(precision, call);
sss = _mm_srai_epi32::<PRECISION>(sss);
sss = _mm_packs_epi32(sss, sss);
let dst_ptr = dst_chunk.as_mut_ptr() as *mut i32;
@@ -260,7 +270,7 @@ unsafe fn vert_convolution_into_one_row_u8<T: PixelExt<Component = u8>>(
native::convolution_by_u8(
src_img,
normalizer,
1 << (precision - 1),
1 << (PRECISION - 1),
dst_u8,
src_x,
y_start,