external/fast_image_resize
1
0
Fork 0
mirror of https://github.com/Cykooz/fast_image_resize.git synced 2026-10-08 01:11:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improved speed of MulDiv implementation for U8x4 images.

Browse Source
This commit is contained in:
Kirill Kuzminykh
2022-11-30 20:56:09 +04:00
parent 42866e5b5e
commit a706ebb68c
36 changed files with 754 additions and 577 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+6
View File
@@ -1,3 +1,9 @@
## [Unreleased] - ReleaseDate
### Crate
- Improved speed of `MulDiv` implementation for `U8x4` images.
## [2.3.0] - 2022-11-25
### Crate
Cargo.toml
+3 -3
View File
@@ -96,9 +96,9 @@ opt-level = 3
[profile.release]
opt-level = 3
#incremental = true
#lto = true
#codegen-units = 1
#strip = true
lto = true
codegen-units = 1
strip = true
[profile.test]
benches/bench_alpha.rs
+29 -1
View File
@@ -52,10 +52,24 @@ fn multiplies_alpha(bench: &mut Bench, pixel_type: PixelType, cpu_extensions: Cp
})
},
);
bench.task(
format!(
"Multiplies alpha inplace {:?} {:?}",
pixel_type, cpu_extensions
),
|task| {
let mut data = get_src_image(width, height, pixel_type, pixel);
let mut view = data.view_mut();
task.iter(|| {
alpha_mul_div.multiply_alpha_inplace(&mut view).unwrap();
})
},
);
}
fn divides_alpha(bench: &mut Bench, pixel_type: PixelType, cpu_extensions: CpuExtensions) {
let width = NonZeroU32::new(4096).unwrap();
let width = NonZeroU32::new(4095).unwrap();
let height = NonZeroU32::new(2048).unwrap();
let pixel: &[u8] = match pixel_type {
PixelType::U8x4 => &[128, 64, 0, 128],
@@ -83,6 +97,20 @@ fn divides_alpha(bench: &mut Bench, pixel_type: PixelType, cpu_extensions: CpuEx
})
},
);
bench.task(
format!(
"Divides alpha inplace {:?} {:?}",
pixel_type, cpu_extensions
),
|task| {
let mut data = get_src_image(width, height, pixel_type, pixel);
let mut view = data.view_mut();
task.iter(|| {
alpha_mul_div.divide_alpha_inplace(&mut view).unwrap();
})
},
);
}
fn bench_alpha(bench: &mut Bench) {
src/alpha/u8x4/avx2.rs
+138 -76
View File
@@ -2,77 +2,107 @@ use std::arch::x86_64::*;
use crate::pixels::U8x4;
use crate::simd_utils;
use crate::utils::foreach_with_pre_reading;
use crate::{ImageView, ImageViewMut};
use super::{native, sse4};
use super::sse4;
#[target_feature(enable = "avx2")]
pub(crate) unsafe fn multiply_alpha(
src_image: &ImageView<U8x4>,
dst_image: &mut ImageViewMut<U8x4>,
) {
let width = src_image.width().get() as usize;
let src_rows = src_image.iter_rows(0);
let dst_rows = dst_image.iter_rows_mut();
for (src_row, dst_row) in src_rows.zip(dst_rows) {
multiply_alpha_row(src_row, dst_row, width);
multiply_alpha_row(src_row, dst_row);
}
}
#[target_feature(enable = "avx2")]
pub(crate) unsafe fn multiply_alpha_inplace(image: &mut ImageViewMut<U8x4>) {
let width = image.width().get() as usize;
for dst_row in image.iter_rows_mut() {
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
multiply_alpha_row(src_row, dst_row, width);
for row in image.iter_rows_mut() {
multiply_alpha_row_inplace(row);
}
}
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn multiply_alpha_row(src_row: &[U8x4], dst_row: &mut [U8x4], width: usize) {
unsafe fn multiply_alpha_row(src_row: &[U8x4], dst_row: &mut [U8x4]) {
let src_chunks = src_row.chunks_exact(8);
let src_tail = src_chunks.remainder();
let mut dst_chunks = dst_row.chunks_exact_mut(8);
let src_dst = src_chunks.zip(&mut dst_chunks);
foreach_with_pre_reading(
src_dst,
|(src, dst)| {
let pixels = simd_utils::loadu_si256(src, 0);
let dst_ptr = dst.as_mut_ptr() as *mut __m256i;
(pixels, dst_ptr)
},
|(mut pixels, dst_ptr)| {
pixels = multiply_alpha_8_pixels(pixels);
_mm256_storeu_si256(dst_ptr, pixels);
},
);
if !src_tail.is_empty() {
let dst_tail = dst_chunks.into_remainder();
sse4::multiply_alpha_row(src_tail, dst_tail);
}
}
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn multiply_alpha_row_inplace(row: &mut [U8x4]) {
let mut chunks = row.chunks_exact_mut(8);
foreach_with_pre_reading(
&mut chunks,
|chunk| {
let pixels = simd_utils::loadu_si256(chunk, 0);
let dst_ptr = chunk.as_mut_ptr() as *mut __m256i;
(pixels, dst_ptr)
},
|(mut pixels, dst_ptr)| {
pixels = multiply_alpha_8_pixels(pixels);
_mm256_storeu_si256(dst_ptr, pixels);
},
);
let tail = chunks.into_remainder();
if !tail.is_empty() {
sse4::multiply_alpha_row_inplace(tail);
}
}
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn multiply_alpha_8_pixels(pixels: __m256i) -> __m256i {
let zero = _mm256_setzero_si256();
let half = _mm256_set1_epi16(128);
const MAX_A: i32 = 0xff000000u32 as i32;
let max_alpha = _mm256_set1_epi32(MAX_A);
#[rustfmt::skip]
let factor_mask = _mm256_set_epi8(
let factor_mask = _mm256_set_epi8(
15, 15, 15, 15, 11, 11, 11, 11, 7, 7, 7, 7, 3, 3, 3, 3,
15, 15, 15, 15, 11, 11, 11, 11, 7, 7, 7, 7, 3, 3, 3, 3,
);
let mut x: usize = 0;
while x < width.saturating_sub(7) {
let src_pixels = simd_utils::loadu_si256(src_row, x);
let factor_pixels = _mm256_shuffle_epi8(pixels, factor_mask);
let factor_pixels = _mm256_or_si256(factor_pixels, max_alpha);
let factor_pixels = _mm256_shuffle_epi8(src_pixels, factor_mask);
let factor_pixels = _mm256_or_si256(factor_pixels, max_alpha);
let pix1 = _mm256_unpacklo_epi8(pixels, zero);
let factors = _mm256_unpacklo_epi8(factor_pixels, zero);
let pix1 = _mm256_add_epi16(_mm256_mullo_epi16(pix1, factors), half);
let pix1 = _mm256_add_epi16(pix1, _mm256_srli_epi16::<8>(pix1));
let pix1 = _mm256_srli_epi16::<8>(pix1);
let pix1 = _mm256_unpacklo_epi8(src_pixels, zero);
let factors = _mm256_unpacklo_epi8(factor_pixels, zero);
let pix1 = _mm256_add_epi16(_mm256_mullo_epi16(pix1, factors), half);
let pix1 = _mm256_add_epi16(pix1, _mm256_srli_epi16::<8>(pix1));
let pix1 = _mm256_srli_epi16::<8>(pix1);
let pix2 = _mm256_unpackhi_epi8(pixels, zero);
let factors = _mm256_unpackhi_epi8(factor_pixels, zero);
let pix2 = _mm256_add_epi16(_mm256_mullo_epi16(pix2, factors), half);
let pix2 = _mm256_add_epi16(pix2, _mm256_srli_epi16::<8>(pix2));
let pix2 = _mm256_srli_epi16::<8>(pix2);
let pix2 = _mm256_unpackhi_epi8(src_pixels, zero);
let factors = _mm256_unpackhi_epi8(factor_pixels, zero);
let pix2 = _mm256_add_epi16(_mm256_mullo_epi16(pix2, factors), half);
let pix2 = _mm256_add_epi16(pix2, _mm256_srli_epi16::<8>(pix2));
let pix2 = _mm256_srli_epi16::<8>(pix2);
let dst_pixels = _mm256_packus_epi16(pix1, pix2);
let dst_ptr = dst_row.get_unchecked_mut(x..).as_mut_ptr() as *mut __m256i;
_mm256_storeu_si256(dst_ptr, dst_pixels);
x += 8;
}
let src_tail = &src_row[x..];
let dst_tail = &mut dst_row[x..];
native::multiply_alpha_row(src_tail, dst_tail);
_mm256_packus_epi16(pix1, pix2)
}
// Divide
@@ -89,56 +119,88 @@ pub(crate) unsafe fn divide_alpha(src_image: &ImageView<U8x4>, dst_image: &mut I
#[target_feature(enable = "avx2")]
pub(crate) unsafe fn divide_alpha_inplace(image: &mut ImageViewMut<U8x4>) {
for dst_row in image.iter_rows_mut() {
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
divide_alpha_row(src_row, dst_row);
for row in image.iter_rows_mut() {
divide_alpha_row_inplace(row);
}
}
#[target_feature(enable = "avx2")]
unsafe fn divide_alpha_row(src_row: &[U8x4], dst_row: &mut [U8x4]) {
let zero = _mm256_setzero_si256();
let alpha_mask = _mm256_set1_epi32(0xff000000u32 as i32);
#[rustfmt::skip]
let shuffle1 = _mm256_set_epi8(
5, 4, 5, 4, 5, 4, 5, 4, 1, 0, 1, 0, 1, 0, 1, 0,
5, 4, 5, 4, 5, 4, 5, 4, 1, 0, 1, 0, 1, 0, 1, 0,
);
#[rustfmt::skip]
let shuffle2 = _mm256_set_epi8(
13, 12, 13, 12, 13, 12, 13, 12, 9, 8, 9, 8, 9, 8, 9, 8,
13, 12, 13, 12, 13, 12, 13, 12, 9, 8, 9, 8, 9, 8, 9, 8,
);
let alpha_scale = _mm256_set1_ps(255.0 * 256.0);
let src_chunks = src_row.chunks_exact(8);
let src_remainder = src_chunks.remainder();
let mut dst_chunks = dst_row.chunks_exact_mut(8);
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
let src_pixels = _mm256_loadu_si256(src.as_ptr() as *const __m256i);
let alpha_f32 = _mm256_cvtepi32_ps(_mm256_srli_epi32::<24>(src_pixels));
let scaled_alpha_f32 = _mm256_div_ps(alpha_scale, alpha_f32);
let scaled_alpha_i32 = _mm256_cvtps_epi32(scaled_alpha_f32);
let mma0 = _mm256_shuffle_epi8(scaled_alpha_i32, shuffle1);
let mma1 = _mm256_shuffle_epi8(scaled_alpha_i32, shuffle2);
let pix0 = _mm256_unpacklo_epi8(zero, src_pixels);
let pix1 = _mm256_unpackhi_epi8(zero, src_pixels);
let pix0 = _mm256_mulhi_epu16(pix0, mma0);
let pix1 = _mm256_mulhi_epu16(pix1, mma1);
let alpha = _mm256_and_si256(src_pixels, alpha_mask);
let rgb = _mm256_packus_epi16(pix0, pix1);
let dst_pixels = _mm256_blendv_epi8(rgb, alpha, alpha_mask);
_mm256_storeu_si256(dst.as_mut_ptr() as *mut __m256i, dst_pixels);
}
let src_dst = src_chunks.zip(&mut dst_chunks);
foreach_with_pre_reading(
src_dst,
|(src, dst)| {
let pixels = simd_utils::loadu_si256(src, 0);
let dst_ptr = dst.as_mut_ptr() as *mut __m256i;
(pixels, dst_ptr)
},
|(mut pixels, dst_ptr)| {
pixels = divide_alpha_8_pixels(pixels);
_mm256_storeu_si256(dst_ptr, pixels);
},
);
if !src_remainder.is_empty() {
let dst_reminder = dst_chunks.into_remainder();
sse4::divide_alpha_row(src_remainder, dst_reminder);
}
}
#[target_feature(enable = "avx2")]
unsafe fn divide_alpha_row_inplace(row: &mut [U8x4]) {
let mut chunks = row.chunks_exact_mut(8);
foreach_with_pre_reading(
&mut chunks,
|chunk| {
let pixels = simd_utils::loadu_si256(chunk, 0);
let dst_ptr = chunk.as_mut_ptr() as *mut __m256i;
(pixels, dst_ptr)
},
|(mut pixels, dst_ptr)| {
pixels = divide_alpha_8_pixels(pixels);
_mm256_storeu_si256(dst_ptr, pixels);
},
);
let tail = chunks.into_remainder();
if !tail.is_empty() {
sse4::divide_alpha_row_inplace(tail);
}
}
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn divide_alpha_8_pixels(pixels: __m256i) -> __m256i {
let zero = _mm256_setzero_si256();
let alpha_mask = _mm256_set1_epi32(0xff000000u32 as i32);
#[rustfmt::skip]
let shuffle1 = _mm256_set_epi8(
5, 4, 5, 4, 5, 4, 5, 4, 1, 0, 1, 0, 1, 0, 1, 0,
5, 4, 5, 4, 5, 4, 5, 4, 1, 0, 1, 0, 1, 0, 1, 0,
);
#[rustfmt::skip]
let shuffle2 = _mm256_set_epi8(
13, 12, 13, 12, 13, 12, 13, 12, 9, 8, 9, 8, 9, 8, 9, 8,
13, 12, 13, 12, 13, 12, 13, 12, 9, 8, 9, 8, 9, 8, 9, 8,
);
let alpha_scale = _mm256_set1_ps(255.0 * 256.0);
let alpha_f32 = _mm256_cvtepi32_ps(_mm256_srli_epi32::<24>(pixels));
let scaled_alpha_f32 = _mm256_div_ps(alpha_scale, alpha_f32);
let scaled_alpha_i32 = _mm256_cvtps_epi32(scaled_alpha_f32);
let mma0 = _mm256_shuffle_epi8(scaled_alpha_i32, shuffle1);
let mma1 = _mm256_shuffle_epi8(scaled_alpha_i32, shuffle2);
let pix0 = _mm256_unpacklo_epi8(zero, pixels);
let pix1 = _mm256_unpackhi_epi8(zero, pixels);
let pix0 = _mm256_mulhi_epu16(pix0, mma0);
let pix1 = _mm256_mulhi_epu16(pix1, mma1);
let alpha = _mm256_and_si256(pixels, alpha_mask);
let rgb = _mm256_packus_epi16(pix0, pix1);
_mm256_blendv_epi8(rgb, alpha, alpha_mask)
}
src/alpha/u8x4/native.rs
+47 -29
View File
@@ -7,31 +7,45 @@ pub(crate) fn multiply_alpha(src_image: &ImageView<U8x4>, dst_image: &mut ImageV
let dst_rows = dst_image.iter_rows_mut();
for (src_row, dst_row) in src_rows.zip(dst_rows) {
multiply_alpha_row(src_row, dst_row);
for (src_pixel, dst_pixel) in src_row.iter().zip(dst_row.iter_mut()) {
*dst_pixel = multiply_alpha_pixel(*src_pixel);
}
}
}
pub(crate) fn multiply_alpha_inplace(image: &mut ImageViewMut<U8x4>) {
for dst_row in image.iter_rows_mut() {
let src_row = unsafe { std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len()) };
multiply_alpha_row(src_row, dst_row);
for row in image.iter_rows_mut() {
multiply_alpha_row_inplace(row);
}
}
#[inline(always)]
pub(crate) fn multiply_alpha_row(src_row: &[U8x4], dst_row: &mut [U8x4]) {
for (src_pixel, dst_pixel) in src_row.iter().zip(dst_row) {
let components: [u8; 4] = src_pixel.0.to_le_bytes();
let alpha = components[3];
dst_pixel.0 = u32::from_le_bytes([
mul_div_255(components[0], alpha),
mul_div_255(components[1], alpha),
mul_div_255(components[2], alpha),
alpha,
]);
*dst_pixel = multiply_alpha_pixel(*src_pixel);
}
}
#[inline(always)]
pub(crate) fn multiply_alpha_row_inplace(row: &mut [U8x4]) {
for pixel in row.iter_mut() {
*pixel = multiply_alpha_pixel(*pixel);
}
}
#[inline(always)]
fn multiply_alpha_pixel(mut pixel: U8x4) -> U8x4 {
let components: [u8; 4] = pixel.0.to_le_bytes();
let alpha = components[3];
pixel.0 = u32::from_le_bytes([
mul_div_255(components[0], alpha),
mul_div_255(components[1], alpha),
mul_div_255(components[2], alpha),
alpha,
]);
pixel
}
// Divide
#[inline]
@@ -46,26 +60,30 @@ pub(crate) fn divide_alpha(src_image: &ImageView<U8x4>, dst_image: &mut ImageVie
#[inline]
pub(crate) fn divide_alpha_inplace(image: &mut ImageViewMut<U8x4>) {
for dst_row in image.iter_rows_mut() {
let src_row = unsafe { std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len()) };
divide_alpha_row(src_row, dst_row);
for row in image.iter_rows_mut() {
row.iter_mut().for_each(|pixel| {
*pixel = divide_alpha_pixel(*pixel);
});
}
}
#[inline(always)]
pub(crate) fn divide_alpha_row(src_row: &[U8x4], dst_row: &mut [U8x4]) {
src_row
.iter()
.zip(dst_row)
.for_each(|(src_pixel, dst_pixel)| {
let components: [u8; 4] = src_pixel.0.to_le_bytes();
let alpha = components[3];
let recip_alpha = RECIP_ALPHA[alpha as usize];
dst_pixel.0 = u32::from_le_bytes([
div_and_clip(components[0], recip_alpha),
div_and_clip(components[1], recip_alpha),
div_and_clip(components[2], recip_alpha),
alpha,
]);
});
for (src_pixel, dst_pixel) in src_row.iter().zip(dst_row) {
*dst_pixel = divide_alpha_pixel(*src_pixel);
}
}
#[inline(always)]
fn divide_alpha_pixel(mut pixel: U8x4) -> U8x4 {
let components: [u8; 4] = pixel.0.to_le_bytes();
let alpha = components[3];
let recip_alpha = RECIP_ALPHA[alpha as usize];
pixel.0 = u32::from_le_bytes([
div_and_clip(components[0], recip_alpha),
div_and_clip(components[1], recip_alpha),
div_and_clip(components[2], recip_alpha),
alpha,
]);
pixel
}
src/alpha/u8x4/neon.rs
+148 -55
View File
@@ -2,6 +2,7 @@ use std::arch::aarch64::*;
use crate::neon_utils;
use crate::pixels::U8x4;
use crate::utils::foreach_with_pre_reading;
use crate::{ImageView, ImageViewMut};
use super::native;
@@ -21,36 +22,29 @@ pub(crate) unsafe fn multiply_alpha(
#[target_feature(enable = "neon")]
pub(crate) unsafe fn multiply_alpha_inplace(image: &mut ImageViewMut<U8x4>) {
for dst_row in image.iter_rows_mut() {
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
multiply_alpha_row(src_row, dst_row);
for row in image.iter_rows_mut() {
multiply_alpha_row_inplace(row);
}
}
#[inline]
#[target_feature(enable = "neon")]
#[inline(always)]
unsafe fn multiply_alpha_row(src_row: &[U8x4], dst_row: &mut [U8x4]) {
let zero_u8x16 = vdupq_n_u8(0);
let zero_u8x8 = vdup_n_u8(0);
let src_chunks = src_row.chunks_exact(16);
let src_remainder = src_chunks.remainder();
let mut dst_chunks = dst_row.chunks_exact_mut(16);
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
let mut pixels = neon_utils::load_deintrel_u8x16x4(src, 0);
let alpha_u16 = uint16x8x2_t(
vreinterpretq_u16_u8(vzip1q_u8(pixels.3, zero_u8x16)),
vreinterpretq_u16_u8(vzip2q_u8(pixels.3, zero_u8x16)),
);
pixels.0 = neon_utils::mul_color_to_alpha_u8x16(pixels.0, alpha_u16, zero_u8x16);
pixels.1 = neon_utils::mul_color_to_alpha_u8x16(pixels.1, alpha_u16, zero_u8x16);
pixels.2 = neon_utils::mul_color_to_alpha_u8x16(pixels.2, alpha_u16, zero_u8x16);
let dst_ptr = dst.as_mut_ptr() as *mut u8;
vst4q_u8(dst_ptr, pixels);
}
let src_dst = src_chunks.zip(&mut dst_chunks);
foreach_with_pre_reading(
src_dst,
|(src, dst)| {
let pixels = neon_utils::load_deintrel_u8x16x4(src, 0);
let dst_ptr = dst.as_mut_ptr() as *mut u8;
(pixels, dst_ptr)
},
|(mut pixels, dst_ptr)| {
pixels = multiplies_alpha_16_pixles(pixels);
vst4q_u8(dst_ptr, pixels);
},
);
let src_chunks = src_remainder.chunks_exact(8);
let src_remainder = src_chunks.remainder();
@@ -59,16 +53,7 @@ unsafe fn multiply_alpha_row(src_row: &[U8x4], dst_row: &mut [U8x4]) {
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
let mut pixels = neon_utils::load_deintrel_u8x8x4(src, 0);
let alpha_u8 = pixels.3;
let alpha_u16_lo = vreinterpret_u16_u8(vzip1_u8(alpha_u8, zero_u8x8));
let alpha_u16_hi = vreinterpret_u16_u8(vzip2_u8(alpha_u8, zero_u8x8));
let alpha_u16 = vcombine_u16(alpha_u16_lo, alpha_u16_hi);
pixels.0 = neon_utils::mul_color_to_alpha_u8x8(pixels.0, alpha_u16, zero_u8x8);
pixels.1 = neon_utils::mul_color_to_alpha_u8x8(pixels.1, alpha_u16, zero_u8x8);
pixels.2 = neon_utils::mul_color_to_alpha_u8x8(pixels.2, alpha_u16, zero_u8x8);
pixels = multiplies_alpha_8_pixles(pixels);
let dst_ptr = dst.as_mut_ptr() as *mut u8;
vst4_u8(dst_ptr, pixels);
}
@@ -79,6 +64,63 @@ unsafe fn multiply_alpha_row(src_row: &[U8x4], dst_row: &mut [U8x4]) {
}
}
#[inline(always)]
unsafe fn multiply_alpha_row_inplace(row: &mut [U8x4]) {
let mut chunks = row.chunks_exact_mut(16);
foreach_with_pre_reading(
&mut chunks,
|chunk| {
let pixels = neon_utils::load_deintrel_u8x16x4(chunk, 0);
let dst_ptr = chunk.as_mut_ptr() as *mut u8;
(pixels, dst_ptr)
},
|(mut pixels, dst_ptr)| {
pixels = multiplies_alpha_16_pixles(pixels);
vst4q_u8(dst_ptr, pixels);
},
);
let reminder = chunks.into_remainder();
let mut chunks = reminder.chunks_exact_mut(8);
if let Some(chunk) = chunks.next() {
let mut pixels = neon_utils::load_deintrel_u8x8x4(chunk, 0);
pixels = multiplies_alpha_8_pixles(pixels);
let dst_ptr = chunk.as_mut_ptr() as *mut u8;
vst4_u8(dst_ptr, pixels);
}
let tail = chunks.into_remainder();
if !tail.is_empty() {
native::multiply_alpha_row_inplace(tail);
}
}
#[inline(always)]
unsafe fn multiplies_alpha_16_pixles(mut pixels: uint8x16x4_t) -> uint8x16x4_t {
let zero_u8x16 = vdupq_n_u8(0);
let alpha_u16 = uint16x8x2_t(
vreinterpretq_u16_u8(vzip1q_u8(pixels.3, zero_u8x16)),
vreinterpretq_u16_u8(vzip2q_u8(pixels.3, zero_u8x16)),
);
pixels.0 = neon_utils::mul_color_to_alpha_u8x16(pixels.0, alpha_u16, zero_u8x16);
pixels.1 = neon_utils::mul_color_to_alpha_u8x16(pixels.1, alpha_u16, zero_u8x16);
pixels.2 = neon_utils::mul_color_to_alpha_u8x16(pixels.2, alpha_u16, zero_u8x16);
pixels
}
#[inline(always)]
unsafe fn multiplies_alpha_8_pixles(mut pixels: uint8x8x4_t) -> uint8x8x4_t {
let zero_u8x8 = vdup_n_u8(0);
let alpha_u8 = pixels.3;
let alpha_u16_lo = vreinterpret_u16_u8(vzip1_u8(alpha_u8, zero_u8x8));
let alpha_u16_hi = vreinterpret_u16_u8(vzip2_u8(alpha_u8, zero_u8x8));
let alpha_u16 = vcombine_u16(alpha_u16_lo, alpha_u16_hi);
pixels.0 = neon_utils::mul_color_to_alpha_u8x8(pixels.0, alpha_u16, zero_u8x8);
pixels.1 = neon_utils::mul_color_to_alpha_u8x8(pixels.1, alpha_u16, zero_u8x8);
pixels.2 = neon_utils::mul_color_to_alpha_u8x8(pixels.2, alpha_u16, zero_u8x8);
pixels
}
// Divide
#[target_feature(enable = "neon")]
@@ -93,28 +135,39 @@ pub(crate) unsafe fn divide_alpha(src_image: &ImageView<U8x4>, dst_image: &mut I
#[target_feature(enable = "neon")]
pub(crate) unsafe fn divide_alpha_inplace(image: &mut ImageViewMut<U8x4>) {
for dst_row in image.iter_rows_mut() {
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
divide_alpha_row(src_row, dst_row);
for row in image.iter_rows_mut() {
divide_alpha_row_inline(row);
}
}
#[inline]
#[target_feature(enable = "neon")]
#[inline(always)]
unsafe fn divide_alpha_row(src_row: &[U8x4], dst_row: &mut [U8x4]) {
let src_chunks = src_row.chunks_exact(16);
let src_remainder = src_chunks.remainder();
let mut dst_chunks = dst_row.chunks_exact_mut(16);
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
divide_alpha_16_pixels(src, dst);
}
let src_dst = src_chunks.zip(&mut dst_chunks);
foreach_with_pre_reading(
src_dst,
|(src, dst)| {
let pixels = neon_utils::load_deintrel_u8x16x4(src, 0);
let dst_ptr = dst.as_mut_ptr() as *mut u8;
(pixels, dst_ptr)
},
|(mut pixels, dst_ptr)| {
pixels = divide_alpha_16_pixels(pixels);
vst4q_u8(dst_ptr, pixels);
},
);
let src_chunks = src_remainder.chunks_exact(8);
let src_remainder = src_chunks.remainder();
let dst_reminder = dst_chunks.into_remainder();
let mut dst_chunks = dst_reminder.chunks_exact_mut(8);
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
divide_alpha_8_pixels(src, dst);
let mut pixels = neon_utils::load_deintrel_u8x8x4(src, 0);
pixels = divide_alpha_8_pixels(pixels);
let dst_ptr = dst.as_mut_ptr() as *mut u8;
vst4_u8(dst_ptr, pixels);
}
if !src_remainder.is_empty() {
@@ -126,7 +179,10 @@ unsafe fn divide_alpha_row(src_row: &[U8x4], dst_row: &mut [U8x4]) {
.for_each(|(d, s)| *d = *s);
let mut dst_pixels = [U8x4::new(0); 8];
divide_alpha_8_pixels(src_pixels.as_slice(), dst_pixels.as_mut_slice());
let mut pixels = neon_utils::load_deintrel_u8x8x4(src_pixels.as_slice(), 0);
pixels = divide_alpha_8_pixels(pixels);
let dst_ptr = dst_pixels.as_mut_ptr() as *mut u8;
vst4_u8(dst_ptr, pixels);
dst_pixels
.iter()
@@ -135,12 +191,53 @@ unsafe fn divide_alpha_row(src_row: &[U8x4], dst_row: &mut [U8x4]) {
}
}
#[inline]
#[target_feature(enable = "neon")]
unsafe fn divide_alpha_16_pixels(src: &[U8x4], dst: &mut [U8x4]) {
#[inline(always)]
unsafe fn divide_alpha_row_inline(row: &mut [U8x4]) {
let mut chunks = row.chunks_exact_mut(16);
foreach_with_pre_reading(
&mut chunks,
|chunk| {
let pixels = neon_utils::load_deintrel_u8x16x4(chunk, 0);
let dst_ptr = chunk.as_mut_ptr() as *mut u8;
(pixels, dst_ptr)
},
|(mut pixels, dst_ptr)| {
pixels = divide_alpha_16_pixels(pixels);
vst4q_u8(dst_ptr, pixels);
},
);
let reminder = chunks.into_remainder();
let mut chunks = reminder.chunks_exact_mut(8);
if let Some(chunk) = chunks.next() {
let mut pixels = neon_utils::load_deintrel_u8x8x4(chunk, 0);
pixels = divide_alpha_8_pixels(pixels);
let dst_ptr = chunk.as_mut_ptr() as *mut u8;
vst4_u8(dst_ptr, pixels);
}
let tail = chunks.into_remainder();
if !tail.is_empty() {
let mut src_pixels = [U8x4::new(0); 8];
src_pixels
.iter_mut()
.zip(tail.iter())
.for_each(|(d, s)| *d = *s);
let mut dst_pixels = [U8x4::new(0); 8];
let mut pixels = neon_utils::load_deintrel_u8x8x4(src_pixels.as_slice(), 0);
pixels = divide_alpha_8_pixels(pixels);
let dst_ptr = dst_pixels.as_mut_ptr() as *mut u8;
vst4_u8(dst_ptr, pixels);
dst_pixels.iter().zip(tail).for_each(|(s, d)| *d = *s);
}
}
#[inline(always)]
unsafe fn divide_alpha_16_pixels(mut pixels: uint8x16x4_t) -> uint8x16x4_t {
let zero = vdupq_n_u8(0);
let alpha_scale = vdupq_n_f32(255.0 * 256.0);
let mut pixels = neon_utils::load_deintrel_u8x16x4(src, 0);
let nonzero_alpha_mask = vmvnq_u8(vceqzq_u8(pixels.3));
let alpha_u16_lo = vzip1q_u8(pixels.3, zero);
@@ -174,17 +271,14 @@ unsafe fn divide_alpha_16_pixels(src: &[U8x4], dst: &mut [U8x4]) {
pixels.2 = neon_utils::mul_color_recip_alpha_u8x16(pixels.2, recip_alpha, zero);
pixels.2 = vandq_u8(pixels.2, nonzero_alpha_mask);
let dst_ptr = dst.as_mut_ptr() as *mut u8;
vst4q_u8(dst_ptr, pixels);
pixels
}
#[inline]
#[target_feature(enable = "neon")]
unsafe fn divide_alpha_8_pixels(src: &[U8x4], dst: &mut [U8x4]) {
#[inline(always)]
unsafe fn divide_alpha_8_pixels(mut pixels: uint8x8x4_t) -> uint8x8x4_t {
let zero_u8x8 = vdup_n_u8(0);
let zero_u8x16 = vdupq_n_u8(0);
let alpha_scale = vdupq_n_f32(255.0 * 256.0);
let mut pixels = neon_utils::load_deintrel_u8x8x4(src, 0);
let nonzero_alpha_mask = vmvn_u8(vceqz_u8(pixels.3));
let alpha_u16_lo = vzip1_u8(pixels.3, zero_u8x8);
@@ -208,6 +302,5 @@ unsafe fn divide_alpha_8_pixels(src: &[U8x4], dst: &mut [U8x4]) {
pixels.2 = neon_utils::mul_color_recip_alpha_u8x8(pixels.2, recip_alpha, zero_u8x8);
pixels.2 = vand_u8(pixels.2, nonzero_alpha_mask);
let dst_ptr = dst.as_mut_ptr() as *mut u8;
vst4_u8(dst_ptr, pixels);
pixels
}
src/alpha/u8x4/sse4.rs
+120 -50
View File
@@ -1,6 +1,7 @@
use std::arch::x86_64::*;
use crate::pixels::U8x4;
use crate::utils::foreach_with_pre_reading;
use crate::{ImageView, ImageViewMut};
use super::native;
@@ -20,15 +21,57 @@ pub(crate) unsafe fn multiply_alpha(
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn multiply_alpha_inplace(image: &mut ImageViewMut<U8x4>) {
for dst_row in image.iter_rows_mut() {
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
multiply_alpha_row(src_row, dst_row);
for row in image.iter_rows_mut() {
multiply_alpha_row_inplace(row);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
unsafe fn multiply_alpha_row(src_row: &[U8x4], dst_row: &mut [U8x4]) {
pub(crate) unsafe fn multiply_alpha_row(src_row: &[U8x4], dst_row: &mut [U8x4]) {
let src_chunks = src_row.chunks_exact(4);
let src_remainder = src_chunks.remainder();
let mut dst_chunks = dst_row.chunks_exact_mut(4);
let src_dst = src_chunks.zip(&mut dst_chunks);
foreach_with_pre_reading(
src_dst,
|(src, dst)| {
let pixels = _mm_loadu_si128(src.as_ptr() as *const __m128i);
let dst_ptr = dst.as_mut_ptr() as *mut __m128i;
(pixels, dst_ptr)
},
|(mut pixels, dst_ptr)| {
pixels = multiply_alpha_4_pixels(pixels);
_mm_storeu_si128(dst_ptr, pixels);
},
);
if !src_remainder.is_empty() {
let dst_reminder = dst_chunks.into_remainder();
native::multiply_alpha_row(src_remainder, dst_reminder);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn multiply_alpha_row_inplace(row: &mut [U8x4]) {
let mut chunks = row.chunks_exact_mut(4);
// Using a simple for-loop in this case is faster than implementation with pre-reading
for chunk in &mut chunks {
let mut pixels = _mm_loadu_si128(chunk.as_ptr() as *const __m128i);
pixels = multiply_alpha_4_pixels(pixels);
_mm_storeu_si128(chunk.as_mut_ptr() as *mut __m128i, pixels);
}
let tail = chunks.into_remainder();
if !tail.is_empty() {
native::multiply_alpha_row_inplace(tail);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
unsafe fn multiply_alpha_4_pixels(pixels: __m128i) -> __m128i {
let zero = _mm_setzero_si128();
let half = _mm_set1_epi16(128);
@@ -36,37 +79,22 @@ unsafe fn multiply_alpha_row(src_row: &[U8x4], dst_row: &mut [U8x4]) {
let max_alpha = _mm_set1_epi32(MAX_A);
let factor_mask = _mm_set_epi8(15, 15, 15, 15, 11, 11, 11, 11, 7, 7, 7, 7, 3, 3, 3, 3);
let src_chunks = src_row.chunks_exact(4);
let src_remainder = src_chunks.remainder();
let mut dst_chunks = dst_row.chunks_exact_mut(4);
let factor_pixels = _mm_shuffle_epi8(pixels, factor_mask);
let factor_pixels = _mm_or_si128(factor_pixels, max_alpha);
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
let src_pixels = _mm_loadu_si128(src.as_ptr() as *const __m128i);
let pix1 = _mm_unpacklo_epi8(pixels, zero);
let factors = _mm_unpacklo_epi8(factor_pixels, zero);
let pix1 = _mm_add_epi16(_mm_mullo_epi16(pix1, factors), half);
let pix1 = _mm_add_epi16(pix1, _mm_srli_epi16::<8>(pix1));
let pix1 = _mm_srli_epi16::<8>(pix1);
let factor_pixels = _mm_shuffle_epi8(src_pixels, factor_mask);
let factor_pixels = _mm_or_si128(factor_pixels, max_alpha);
let pix2 = _mm_unpackhi_epi8(pixels, zero);
let factors = _mm_unpackhi_epi8(factor_pixels, zero);
let pix2 = _mm_add_epi16(_mm_mullo_epi16(pix2, factors), half);
let pix2 = _mm_add_epi16(pix2, _mm_srli_epi16::<8>(pix2));
let pix2 = _mm_srli_epi16::<8>(pix2);
let pix1 = _mm_unpacklo_epi8(src_pixels, zero);
let factors = _mm_unpacklo_epi8(factor_pixels, zero);
let pix1 = _mm_add_epi16(_mm_mullo_epi16(pix1, factors), half);
let pix1 = _mm_add_epi16(pix1, _mm_srli_epi16::<8>(pix1));
let pix1 = _mm_srli_epi16::<8>(pix1);
let pix2 = _mm_unpackhi_epi8(src_pixels, zero);
let factors = _mm_unpackhi_epi8(factor_pixels, zero);
let pix2 = _mm_add_epi16(_mm_mullo_epi16(pix2, factors), half);
let pix2 = _mm_add_epi16(pix2, _mm_srli_epi16::<8>(pix2));
let pix2 = _mm_srli_epi16::<8>(pix2);
let dst_pixels = _mm_packus_epi16(pix1, pix2);
_mm_storeu_si128(dst.as_mut_ptr() as *mut __m128i, dst_pixels);
}
if !src_remainder.is_empty() {
let dst_reminder = dst_chunks.into_remainder();
native::multiply_alpha_row(src_remainder, dst_reminder);
}
_mm_packus_epi16(pix1, pix2)
}
// Divide
@@ -75,7 +103,6 @@ unsafe fn multiply_alpha_row(src_row: &[U8x4], dst_row: &mut [U8x4]) {
pub(crate) unsafe fn divide_alpha(src_image: &ImageView<U8x4>, dst_image: &mut ImageViewMut<U8x4>) {
let src_rows = src_image.iter_rows(0);
let dst_rows = dst_image.iter_rows_mut();
for (src_row, dst_row) in src_rows.zip(dst_rows) {
divide_alpha_row(src_row, dst_row);
}
@@ -83,50 +110,94 @@ pub(crate) unsafe fn divide_alpha(src_image: &ImageView<U8x4>, dst_image: &mut I
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn divide_alpha_inplace(image: &mut ImageViewMut<U8x4>) {
for dst_row in image.iter_rows_mut() {
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
divide_alpha_row(src_row, dst_row);
for row in image.iter_rows_mut() {
divide_alpha_row_inplace(row);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn divide_alpha_row(src_row: &[U8x4], dst_row: &mut [U8x4]) {
let src_chunks = src_row.chunks_exact(4);
let src_remainder = src_chunks.remainder();
let mut dst_chunks = dst_row.chunks_exact_mut(4);
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
divide_alpha_four_pixels(src.as_ptr(), dst.as_mut_ptr());
}
let src_dst = src_chunks.zip(&mut dst_chunks);
foreach_with_pre_reading(
src_dst,
|(src, dst)| {
let pixels = _mm_loadu_si128(src.as_ptr() as *const __m128i);
let dst_ptr = dst.as_mut_ptr() as *mut __m128i;
(pixels, dst_ptr)
},
|(mut pixels, dst_ptr)| {
pixels = divide_alpha_4_pixels(pixels);
_mm_storeu_si128(dst_ptr, pixels);
},
);
if !src_remainder.is_empty() {
let dst_reminder = dst_chunks.into_remainder();
let mut src_pixels = [U8x4::new(0); 4];
src_pixels
let mut src_buffer = [U8x4::new(0); 4];
src_buffer
.iter_mut()
.zip(src_remainder)
.for_each(|(d, s)| *d = *s);
let mut dst_pixels = [U8x4::new(0); 4];
divide_alpha_four_pixels(src_pixels.as_ptr(), dst_pixels.as_mut_ptr());
let mut dst_buffer = [U8x4::new(0); 4];
let src_pixels = _mm_loadu_si128(src_buffer.as_ptr() as *const __m128i);
let dst_pixels = divide_alpha_4_pixels(src_pixels);
_mm_storeu_si128(dst_buffer.as_mut_ptr() as *mut __m128i, dst_pixels);
dst_pixels
dst_buffer
.iter()
.zip(dst_reminder)
.for_each(|(s, d)| *d = *s);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
unsafe fn divide_alpha_four_pixels(src: *const U8x4, dst: *mut U8x4) {
pub(crate) unsafe fn divide_alpha_row_inplace(row: &mut [U8x4]) {
let mut chunks = row.chunks_exact_mut(4);
foreach_with_pre_reading(
&mut chunks,
|chunk| {
let pixels = _mm_loadu_si128(chunk.as_ptr() as *const __m128i);
let dst_ptr = chunk.as_mut_ptr() as *mut __m128i;
(pixels, dst_ptr)
},
|(mut pixels, dst_ptr)| {
pixels = divide_alpha_4_pixels(pixels);
_mm_storeu_si128(dst_ptr, pixels);
},
);
let tail = chunks.into_remainder();
if !tail.is_empty() {
let mut src_buffer = [U8x4::new(0); 4];
src_buffer
.iter_mut()
.zip(tail.iter())
.for_each(|(d, s)| *d = *s);
let mut dst_buffer = [U8x4::new(0); 4];
let src_pixels = _mm_loadu_si128(src_buffer.as_ptr() as *const __m128i);
let dst_pixels = divide_alpha_4_pixels(src_pixels);
_mm_storeu_si128(dst_buffer.as_mut_ptr() as *mut __m128i, dst_pixels);
dst_buffer.iter().zip(tail).for_each(|(s, d)| *d = *s);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
unsafe fn divide_alpha_4_pixels(src_pixels: __m128i) -> __m128i {
let zero = _mm_setzero_si128();
let alpha_mask = _mm_set1_epi32(0xff000000u32 as i32);
let shuffle1 = _mm_set_epi8(5, 4, 5, 4, 5, 4, 5, 4, 1, 0, 1, 0, 1, 0, 1, 0);
let shuffle2 = _mm_set_epi8(13, 12, 13, 12, 13, 12, 13, 12, 9, 8, 9, 8, 9, 8, 9, 8);
let alpha_scale = _mm_set1_ps(255.0 * 256.0);
let src_pixels = _mm_loadu_si128(src as *const __m128i);
let alpha_f32 = _mm_cvtepi32_ps(_mm_srli_epi32::<24>(src_pixels));
let scaled_alpha_f32 = _mm_div_ps(alpha_scale, alpha_f32);
let scaled_alpha_i32 = _mm_cvtps_epi32(scaled_alpha_f32);
@@ -141,7 +212,6 @@ unsafe fn divide_alpha_four_pixels(src: *const U8x4, dst: *mut U8x4) {
let alpha = _mm_and_si128(src_pixels, alpha_mask);
let rgb = _mm_packus_epi16(pix0, pix1);
let dst_pixels = _mm_blendv_epi8(rgb, alpha, alpha_mask);
_mm_storeu_si128(dst as *mut __m128i, dst_pixels);
_mm_blendv_epi8(rgb, alpha, alpha_mask)
}
src/convolution/u16x1/avx2.rs
+12 -17
View File
@@ -1,7 +1,6 @@
use std::arch::x86_64::*;
use crate::convolution::{optimisations, Coefficients};
use crate::image_view::{FourRows, FourRowsMut};
use crate::pixels::U16;
use crate::simd_utils;
use crate::{ImageView, ImageViewMut};
@@ -46,15 +45,11 @@ pub(crate) fn horiz_convolution(
/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.0.len()
#[target_feature(enable = "avx2")]
unsafe fn horiz_convolution_four_rows(
src_rows: FourRows<U16>,
dst_rows: FourRowsMut<U16>,
src_rows: [&[U16]; 4],
dst_rows: [&mut &mut [U16]; 4],
coefficients_chunks: &[optimisations::CoefficientsI32Chunk],
normalizer: &optimisations::Normalizer32,
) {
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
let s_rows = [s_row0, s_row1, s_row2, s_row3];
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
let d_rows = [d_row0, d_row1, d_row2, d_row3];
let precision = normalizer.precision();
let half_error = 1i64 << (precision - 1);
let mut ll_buf = [0i64; 4];
@@ -117,8 +112,8 @@ unsafe fn horiz_convolution_four_rows(
for (i, sum) in ll_sum.iter_mut().enumerate() {
let source = _mm256_set_m128i(
simd_utils::loadu_si128(s_rows[i * 2 + 1], x),
simd_utils::loadu_si128(s_rows[i * 2], x),
simd_utils::loadu_si128(src_rows[i * 2 + 1], x),
simd_utils::loadu_si128(src_rows[i * 2], x),
);
let l0l1_i64x4 = _mm256_shuffle_epi8(source, l0l1_shuffle);
@@ -146,8 +141,8 @@ unsafe fn horiz_convolution_four_rows(
for (i, sum) in ll_sum.iter_mut().enumerate() {
let source = _mm256_set_m128i(
simd_utils::loadl_epi64(s_rows[i * 2 + 1], x),
simd_utils::loadl_epi64(s_rows[i * 2], x),
simd_utils::loadl_epi64(src_rows[i * 2 + 1], x),
simd_utils::loadl_epi64(src_rows[i * 2], x),
);
let l0l1_i64x4 = _mm256_shuffle_epi8(source, l0l1_shuffle);
@@ -167,8 +162,8 @@ unsafe fn horiz_convolution_four_rows(
for (i, sum) in ll_sum.iter_mut().enumerate() {
let source = _mm256_set_m128i(
simd_utils::loadl_epi32(s_rows[i * 2 + 1], x),
simd_utils::loadl_epi32(s_rows[i * 2], x),
simd_utils::loadl_epi32(src_rows[i * 2 + 1], x),
simd_utils::loadl_epi32(src_rows[i * 2], x),
);
let l0l1_i64x4 = _mm256_shuffle_epi8(source, l0l1_shuffle);
@@ -183,9 +178,9 @@ unsafe fn horiz_convolution_four_rows(
for (i, sum) in ll_sum.iter_mut().enumerate() {
let source = _mm256_set_epi64x(
0,
s_rows[i * 2 + 1].get_unchecked(x).0 as i64,
src_rows[i * 2 + 1].get_unchecked(x).0 as i64,
0,
s_rows[i * 2].get_unchecked(x).0 as i64,
src_rows[i * 2].get_unchecked(x).0 as i64,
);
*sum = _mm256_add_epi64(*sum, _mm256_mul_epi32(source, coeff0_i64x4));
}
@@ -194,10 +189,10 @@ unsafe fn horiz_convolution_four_rows(
// ll_sum.into_iter().enumerate() executes slowly than ll_sum.iter().enumerate()
for (i, &ll) in ll_sum.iter().enumerate() {
_mm256_storeu_si256((&mut ll_buf).as_mut_ptr() as *mut __m256i, ll);
let dst_pixel = d_rows[i * 2].get_unchecked_mut(dst_x);
let dst_pixel = dst_rows[i * 2].get_unchecked_mut(dst_x);
dst_pixel.0 = normalizer.clip(ll_buf[0] + ll_buf[1] + half_error);
let dst_pixel = d_rows[i * 2 + 1].get_unchecked_mut(dst_x);
let dst_pixel = dst_rows[i * 2 + 1].get_unchecked_mut(dst_x);
dst_pixel.0 = normalizer.clip(ll_buf[2] + ll_buf[3] + half_error);
}
}
src/convolution/u16x1/neon.rs
+6 -12
View File
@@ -1,7 +1,6 @@
use std::arch::aarch64::*;
use crate::convolution::{optimisations, Coefficients};
use crate::image_view::{FourRows, FourRowsMut};
use crate::neon_utils;
use crate::pixels::U16;
use crate::{ImageView, ImageViewMut};
@@ -48,16 +47,11 @@ pub(crate) fn horiz_convolution(
/// - precision <= MAX_COEFS_PRECISION
#[target_feature(enable = "neon")]
unsafe fn horiz_convolution_four_rows(
src_rows: FourRows<U16>,
dst_rows: FourRowsMut<U16>,
src_rows: [&[U16]; 4],
dst_rows: [&mut &mut [U16]; 4],
coefficients_chunks: &[optimisations::CoefficientsI32Chunk],
precision: u8,
) {
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
let s_rows = [s_row0, s_row1, s_row2, s_row3];
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
let d_rows = [d_row0, d_row1, d_row2, d_row3];
let initial = vdupq_n_s64(1i64 << (precision - 2));
let zero_u16x4 = vdup_n_u16(0);
@@ -74,7 +68,7 @@ unsafe fn horiz_convolution_four_rows(
let coeff1 = vget_high_s32(coeffs_i32x4);
for i in 0..4 {
let source = neon_utils::load_u16x4(s_rows[i], x);
let source = neon_utils::load_u16x4(src_rows[i], x);
let mut sss = sss_a[i];
let pix_i32 = vreinterpret_s32_u16(vzip1_u16(source, zero_u16x4));
@@ -93,7 +87,7 @@ unsafe fn horiz_convolution_four_rows(
let coeffs_i32x2 = neon_utils::load_i32x2(k, 0);
for i in 0..4 {
let source = neon_utils::load_u16x2(s_rows[i], x);
let source = neon_utils::load_u16x2(src_rows[i], x);
let pix_i32 = vreinterpret_s32_u16(vzip1_u16(source, zero_u16x4));
sss_a[i] = vmlal_s32(sss_a[i], pix_i32, coeffs_i32x2);
}
@@ -103,7 +97,7 @@ unsafe fn horiz_convolution_four_rows(
if !coeffs.is_empty() {
let coeffs_i32x2 = neon_utils::load_i32x1(coeffs, 0);
for i in 0..4 {
let source = neon_utils::load_u16x1(s_rows[i], x);
let source = neon_utils::load_u16x1(src_rows[i], x);
let pix_i32 = vreinterpret_s32_u16(vzip1_u16(source, zero_u16x4));
sss_a[i] = vmlal_s32(sss_a[i], pix_i32, coeffs_i32x2);
}
@@ -127,7 +121,7 @@ unsafe fn horiz_convolution_four_rows(
for i in 0..4 {
let res = vdupd_lane_s64::<0>(sss_a_i64[i]);
d_rows[i].get_unchecked_mut(dst_x).0 = vqmovns_u32(vqmovund_s64(res));
dst_rows[i].get_unchecked_mut(dst_x).0 = vqmovns_u32(vqmovund_s64(res));
}
}
}
src/convolution/u16x1/sse4.rs
+7 -12
View File
@@ -1,7 +1,6 @@
use std::arch::x86_64::*;
use crate::convolution::{optimisations, Coefficients};
use crate::image_view::{FourRows, FourRowsMut};
use crate::pixels::U16;
use crate::simd_utils;
use crate::{ImageView, ImageViewMut};
@@ -46,15 +45,11 @@ pub(crate) fn horiz_convolution(
/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.0.len()
#[target_feature(enable = "sse4.1")]
unsafe fn horiz_convolution_four_rows(
src_rows: FourRows<U16>,
dst_rows: FourRowsMut<U16>,
src_rows: [&[U16]; 4],
dst_rows: [&mut &mut [U16]; 4],
coefficients_chunks: &[optimisations::CoefficientsI32Chunk],
normalizer: &optimisations::Normalizer32,
) {
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
let s_rows = [s_row0, s_row1, s_row2, s_row3];
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
let d_rows = [d_row0, d_row1, d_row2, d_row3];
let precision = normalizer.precision();
let half_error = 1i64 << (precision - 1);
let mut ll_buf = [0i64; 2];
@@ -100,7 +95,7 @@ unsafe fn horiz_convolution_four_rows(
for i in 0..4 {
let mut sum = ll_sum[i];
let source = simd_utils::loadu_si128(s_rows[i], x);
let source = simd_utils::loadu_si128(src_rows[i], x);
let l0l1_i64x2 = _mm_shuffle_epi8(source, l0l1_shuffle);
sum = _mm_add_epi64(sum, _mm_mul_epi32(l0l1_i64x2, coeff01_i64x2));
@@ -128,7 +123,7 @@ unsafe fn horiz_convolution_four_rows(
for i in 0..4 {
let mut sum = ll_sum[i];
let source = simd_utils::loadl_epi64(s_rows[i], x);
let source = simd_utils::loadl_epi64(src_rows[i], x);
let l0l1_i64x2 = _mm_shuffle_epi8(source, l0l1_shuffle);
sum = _mm_add_epi64(sum, _mm_mul_epi32(l0l1_i64x2, coeff01_i64x2));
@@ -147,7 +142,7 @@ unsafe fn horiz_convolution_four_rows(
for k in coeffs_by_2 {
let coeff01_i64x2 = _mm_set_epi64x(k[1] as i64, k[0] as i64);
for i in 0..4 {
let source = simd_utils::loadl_epi32(s_rows[i], x);
let source = simd_utils::loadl_epi32(src_rows[i], x);
let l_i64x2 = _mm_shuffle_epi8(source, l0l1_shuffle);
ll_sum[i] = _mm_add_epi64(ll_sum[i], _mm_mul_epi32(l_i64x2, coeff01_i64x2));
}
@@ -157,7 +152,7 @@ unsafe fn horiz_convolution_four_rows(
if let Some(&k) = coeffs.first() {
let coeff01_i64x2 = _mm_set_epi64x(0, k as i64);
for i in 0..4 {
let pixel = (*s_rows[i].get_unchecked(x)).0 as i64;
let pixel = (*src_rows[i].get_unchecked(x)).0 as i64;
let source = _mm_set_epi64x(0, pixel);
ll_sum[i] = _mm_add_epi64(ll_sum[i], _mm_mul_epi32(source, coeff01_i64x2));
}
@@ -165,7 +160,7 @@ unsafe fn horiz_convolution_four_rows(
for i in 0..4 {
_mm_storeu_si128((&mut ll_buf).as_mut_ptr() as *mut __m128i, ll_sum[i]);
let dst_pixel = d_rows[i].get_unchecked_mut(dst_x);
let dst_pixel = dst_rows[i].get_unchecked_mut(dst_x);
dst_pixel.0 = normalizer.clip(ll_buf.iter().sum::<i64>() + half_error);
}
}
src/convolution/u16x2/avx2.rs
+10 -15
View File
@@ -1,7 +1,6 @@
use std::arch::x86_64::*;
use crate::convolution::{optimisations, Coefficients};
use crate::image_view::{FourRows, FourRowsMut};
use crate::pixels::U16x2;
use crate::simd_utils;
use crate::{ImageView, ImageViewMut};
@@ -46,15 +45,11 @@ pub(crate) fn horiz_convolution(
/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.0.len()
#[target_feature(enable = "avx2")]
unsafe fn horiz_convolution_four_rows(
src_rows: FourRows<U16x2>,
dst_rows: FourRowsMut<U16x2>,
src_rows: [&[U16x2]; 4],
dst_rows: [&mut &mut [U16x2]; 4],
coefficients_chunks: &[optimisations::CoefficientsI32Chunk],
normalizer: &optimisations::Normalizer32,
) {
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
let s_rows = [s_row0, s_row1, s_row2, s_row3];
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
let d_rows = [d_row0, d_row1, d_row2, d_row3];
let precision = normalizer.precision();
let half_error = 1i64 << (precision - 1);
let mut ll_buf = [0i64; 4];
@@ -113,8 +108,8 @@ unsafe fn horiz_convolution_four_rows(
for (i, sum) in ll_sum.iter_mut().enumerate() {
let source = _mm256_set_m128i(
simd_utils::loadu_si128(s_rows[i * 2 + 1], x),
simd_utils::loadu_si128(s_rows[i * 2], x),
simd_utils::loadu_si128(src_rows[i * 2 + 1], x),
simd_utils::loadu_si128(src_rows[i * 2], x),
);
let pp_i64x4 = _mm256_shuffle_epi8(source, p0_shuffle);
@@ -140,8 +135,8 @@ unsafe fn horiz_convolution_four_rows(
for (i, sum) in ll_sum.iter_mut().enumerate() {
let source = _mm256_set_m128i(
simd_utils::loadl_epi64(s_rows[i * 2 + 1], x),
simd_utils::loadl_epi64(s_rows[i * 2], x),
simd_utils::loadl_epi64(src_rows[i * 2 + 1], x),
simd_utils::loadl_epi64(src_rows[i * 2], x),
);
let pp_i64x4 = _mm256_shuffle_epi8(source, p0_shuffle);
@@ -158,8 +153,8 @@ unsafe fn horiz_convolution_four_rows(
for (i, sum) in ll_sum.iter_mut().enumerate() {
let source = _mm256_set_m128i(
simd_utils::loadl_epi32(s_rows[i * 2 + 1], x),
simd_utils::loadl_epi32(s_rows[i * 2], x),
simd_utils::loadl_epi32(src_rows[i * 2 + 1], x),
simd_utils::loadl_epi32(src_rows[i * 2], x),
);
let pp_i64x4 = _mm256_shuffle_epi8(source, p0_shuffle);
@@ -170,11 +165,11 @@ unsafe fn horiz_convolution_four_rows(
// ll_sum.into_iter().enumerate() executes slowly than ll_sum.iter().enumerate()
for (i, &ll) in ll_sum.iter().enumerate() {
_mm256_storeu_si256((&mut ll_buf).as_mut_ptr() as *mut __m256i, ll);
let dst_pixel = d_rows[i * 2].get_unchecked_mut(dst_x);
let dst_pixel = dst_rows[i * 2].get_unchecked_mut(dst_x);
dst_pixel.0 = [normalizer.clip(ll_buf[0]), normalizer.clip(ll_buf[1])];
let dst_pixel = d_rows[i * 2 + 1].get_unchecked_mut(dst_x);
let dst_pixel = dst_rows[i * 2 + 1].get_unchecked_mut(dst_x);
dst_pixel.0 = [normalizer.clip(ll_buf[2]), normalizer.clip(ll_buf[3])];
}
}
src/convolution/u16x2/neon.rs
+5 -11
View File
@@ -1,7 +1,6 @@
use std::arch::aarch64::*;
use crate::convolution::{optimisations, Coefficients};
use crate::image_view::{FourRows, FourRowsMut};
use crate::neon_utils;
use crate::pixels::U16x2;
use crate::{ImageView, ImageViewMut};
@@ -48,16 +47,11 @@ pub(crate) fn horiz_convolution(
/// - precision <= MAX_COEFS_PRECISION
#[target_feature(enable = "neon")]
unsafe fn horiz_convolution_four_rows(
src_rows: FourRows<U16x2>,
dst_rows: FourRowsMut<U16x2>,
src_rows: [&[U16x2]; 4],
dst_rows: [&mut &mut [U16x2]; 4],
coefficients_chunks: &[optimisations::CoefficientsI32Chunk],
precision: u8,
) {
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
let s_rows = [s_row0, s_row1, s_row2, s_row3];
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
let d_rows = [d_row0, d_row1, d_row2, d_row3];
let initial = vdupq_n_s64(1i64 << (precision - 1));
let zero_u16x8 = vdupq_n_u16(0);
let zero_u16x4 = vdup_n_u16(0);
@@ -76,7 +70,7 @@ unsafe fn horiz_convolution_four_rows(
for i in 0..4 {
let mut sss = sss_a[i];
let source = neon_utils::load_u16x4(s_rows[i], x);
let source = neon_utils::load_u16x4(src_rows[i], x);
let pix_i32 = vreinterpret_s32_u16(vzip1_u16(source, zero_u16x4));
sss = vmlal_s32(sss, pix_i32, coeff0);
let pix_i32 = vreinterpret_s32_u16(vzip2_u16(source, zero_u16x4));
@@ -90,7 +84,7 @@ unsafe fn horiz_convolution_four_rows(
let coeffs_i32x2 = neon_utils::load_i32x1(coeffs, 0);
let coeff = vzip1_s32(coeffs_i32x2, coeffs_i32x2);
for i in 0..4 {
let source = neon_utils::load_u16x2(s_rows[i], x);
let source = neon_utils::load_u16x2(src_rows[i], x);
let pix_i32 = vreinterpret_s32_u16(vzip1_u16(source, zero_u16x4));
sss_a[i] = vmlal_s32(sss_a[i], pix_i32, coeff);
}
@@ -111,7 +105,7 @@ unsafe fn horiz_convolution_four_rows(
vqmovn_s64(sss_a[i]),
vreinterpret_s32_u16(zero_u16x4),
));
d_rows[i].get_unchecked_mut(dst_x).0 = [
dst_rows[i].get_unchecked_mut(dst_x).0 = [
vduph_lane_u16::<0>(res_u16x4),
vduph_lane_u16::<1>(res_u16x4),
];
src/convolution/u16x2/sse4.rs
+6 -11
View File
@@ -1,7 +1,6 @@
use std::arch::x86_64::*;
use crate::convolution::{optimisations, Coefficients};
use crate::image_view::{FourRows, FourRowsMut};
use crate::pixels::U16x2;
use crate::simd_utils;
use crate::{ImageView, ImageViewMut};
@@ -47,15 +46,11 @@ pub(crate) fn horiz_convolution(
/// - precision <= MAX_COEFS_PRECISION
#[target_feature(enable = "sse4.1")]
unsafe fn horiz_convolution_four_rows(
src_rows: FourRows<U16x2>,
dst_rows: FourRowsMut<U16x2>,
src_rows: [&[U16x2]; 4],
dst_rows: [&mut &mut [U16x2]; 4],
coefficients_chunks: &[optimisations::CoefficientsI32Chunk],
normalizer: &optimisations::Normalizer32,
) {
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
let s_rows = [s_row0, s_row1, s_row2, s_row3];
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
let d_rows = [d_row0, d_row1, d_row2, d_row3];
let precision = normalizer.precision();
let half_error = 1i64 << (precision - 1);
let mut ll_buf = [0i64; 2];
@@ -101,7 +96,7 @@ unsafe fn horiz_convolution_four_rows(
for i in 0..4 {
let mut sum = ll_sum[i];
let source = simd_utils::loadu_si128(s_rows[i], x);
let source = simd_utils::loadu_si128(src_rows[i], x);
let p_i64x2 = _mm_shuffle_epi8(source, p0_shuffle);
sum = _mm_add_epi64(sum, _mm_mul_epi32(p_i64x2, coeff0_i64x2));
@@ -129,7 +124,7 @@ unsafe fn horiz_convolution_four_rows(
for i in 0..4 {
let mut sum = ll_sum[i];
let source = simd_utils::loadl_epi64(s_rows[i], x);
let source = simd_utils::loadl_epi64(src_rows[i], x);
let p_i64x2 = _mm_shuffle_epi8(source, p0_shuffle);
sum = _mm_add_epi64(sum, _mm_mul_epi32(p_i64x2, coeff0_i64x2));
@@ -145,7 +140,7 @@ unsafe fn horiz_convolution_four_rows(
if let Some(&k) = coeffs.first() {
let coeff0_i64x2 = _mm_set1_epi64x(k as i64);
for i in 0..4 {
let source = simd_utils::loadl_epi32(s_rows[i], x);
let source = simd_utils::loadl_epi32(src_rows[i], x);
let p_i64x2 = _mm_shuffle_epi8(source, p0_shuffle);
ll_sum[i] = _mm_add_epi64(ll_sum[i], _mm_mul_epi32(p_i64x2, coeff0_i64x2));
}
@@ -153,7 +148,7 @@ unsafe fn horiz_convolution_four_rows(
for i in 0..4 {
_mm_storeu_si128((&mut ll_buf).as_mut_ptr() as *mut __m128i, ll_sum[i]);
let dst_pixel = d_rows[i].get_unchecked_mut(dst_x);
let dst_pixel = dst_rows[i].get_unchecked_mut(dst_x);
dst_pixel.0 = [normalizer.clip(ll_buf[0]), normalizer.clip(ll_buf[1])];
}
}
src/convolution/u16x3/avx2.rs
+7 -11
View File
@@ -1,7 +1,7 @@
use std::arch::x86_64::*;
use crate::convolution::{optimisations, Coefficients};
use crate::image_view::{FourRows, FourRowsMut, ImageView, ImageViewMut};
use crate::image_view::{ImageView, ImageViewMut};
use crate::pixels::U16x3;
use crate::simd_utils;
@@ -45,15 +45,11 @@ pub(crate) fn horiz_convolution(
/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.0.len()
#[target_feature(enable = "avx2")]
unsafe fn horiz_convolution_four_rows(
src_rows: FourRows<U16x3>,
dst_rows: FourRowsMut<U16x3>,
src_rows: [&[U16x3]; 4],
dst_rows: [&mut &mut [U16x3]; 4],
coefficients_chunks: &[optimisations::CoefficientsI32Chunk],
normalizer: &optimisations::Normalizer32,
) {
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
let s_rows = [s_row0, s_row1, s_row2, s_row3];
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
let d_rows = [d_row0, d_row1, d_row2, d_row3];
let precision = normalizer.precision();
let half_error = 1i64 << (precision - 1);
let mut rg_buf = [0i64; 4];
@@ -102,7 +98,7 @@ unsafe fn horiz_convolution_four_rows(
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 11, 10, -1, -1, -1, -1, -1, -1, 5, 4),
);
let width = s_row0.len();
let width = src_rows[0].len();
for (dst_x, coeffs_chunk) in coefficients_chunks.iter().enumerate() {
let mut x: usize = coeffs_chunk.start as usize;
@@ -127,7 +123,7 @@ unsafe fn horiz_convolution_four_rows(
let coeff014_i64x2 = _mm256_set_epi64x(0, k[4] as i64, k[1] as i64, k[0] as i64);
for i in 0..4 {
let source = simd_utils::loadu_si256(s_rows[i], x);
let source = simd_utils::loadu_si256(src_rows[i], x);
let rg03_i64x4 = _mm256_shuffle_epi8(source, rg03_shuffle);
rg_sum[i] =
@@ -155,7 +151,7 @@ unsafe fn horiz_convolution_four_rows(
let coeff_i64x4 = _mm256_set1_epi64x(k as i64);
for i in 0..4 {
let &pixel = s_rows[i].get_unchecked(x);
let &pixel = src_rows[i].get_unchecked(x);
let rgb_i64x4 =
_mm256_set_epi64x(0, pixel.0[2] as i64, pixel.0[1] as i64, pixel.0[0] as i64);
rg_bb_sum[i] =
@@ -168,7 +164,7 @@ unsafe fn horiz_convolution_four_rows(
_mm256_storeu_si256((&mut rg_buf).as_mut_ptr() as *mut __m256i, rg_sum[i]);
_mm256_storeu_si256((&mut rg_bb_buf).as_mut_ptr() as *mut __m256i, rg_bb_sum[i]);
_mm256_storeu_si256((&mut bbb_buf).as_mut_ptr() as *mut __m256i, bbb_sum[i]);
let dst_pixel = d_rows[i].get_unchecked_mut(dst_x);
let dst_pixel = dst_rows[i].get_unchecked_mut(dst_x);
dst_pixel.0[0] = normalizer.clip(rg_buf[0] + rg_buf[2] + rg_bb_buf[0] + half_error);
dst_pixel.0[1] = normalizer.clip(rg_buf[1] + rg_buf[3] + rg_bb_buf[1] + half_error);
dst_pixel.0[2] = normalizer.clip(
src/convolution/u16x3/sse4.rs
+6 -11
View File
@@ -2,7 +2,6 @@ use std::arch::x86_64::*;
use crate::convolution::optimisations::CoefficientsI32Chunk;
use crate::convolution::{optimisations, Coefficients};
use crate::image_view::{FourRows, FourRowsMut};
use crate::pixels::U16x3;
use crate::simd_utils;
use crate::{ImageView, ImageViewMut};
@@ -48,15 +47,11 @@ pub(crate) fn horiz_convolution(
/// - precision <= MAX_COEFS_PRECISION
#[target_feature(enable = "sse4.1")]
unsafe fn horiz_convolution_8u4x(
src_rows: FourRows<U16x3>,
dst_rows: FourRowsMut<U16x3>,
src_rows: [&[U16x3]; 4],
dst_rows: [&mut &mut [U16x3]; 4],
coefficients_chunks: &[CoefficientsI32Chunk],
normalizer: &optimisations::Normalizer32,
) {
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
let s_rows = [s_row0, s_row1, s_row2, s_row3];
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
let d_rows = [d_row0, d_row1, d_row2, d_row3];
let precision = normalizer.precision();
let half_error = 1i64 << (precision - 1);
let mut rg_buf = [0i64; 2];
@@ -81,7 +76,7 @@ unsafe fn horiz_convolution_8u4x(
let rg1_shuffle = _mm_set_epi8(-1, -1, -1, -1, -1, -1, 9, 8, -1, -1, -1, -1, -1, -1, 7, 6);
let bb_shuffle = _mm_set_epi8(-1, -1, -1, -1, -1, -1, 11, 10, -1, -1, -1, -1, -1, -1, 5, 4);
let width = s_row0.len();
let width = src_rows[0].len();
for (dst_x, coeffs_chunk) in coefficients_chunks.iter().enumerate() {
let mut x: usize = coeffs_chunk.start as usize;
@@ -101,7 +96,7 @@ unsafe fn horiz_convolution_8u4x(
let coeff_i64x2 = _mm_set_epi64x(k[1] as i64, k[0] as i64);
for i in 0..4 {
let source = simd_utils::loadu_si128(s_rows[i], x);
let source = simd_utils::loadu_si128(src_rows[i], x);
let rg0_i64x2 = _mm_shuffle_epi8(source, rg0_shuffle);
rg_sum[i] = _mm_add_epi64(rg_sum[i], _mm_mul_epi32(rg0_i64x2, coeff0_i64x2));
@@ -120,7 +115,7 @@ unsafe fn horiz_convolution_8u4x(
let coeff_i64x2 = _mm_set1_epi64x(k as i64);
for i in 0..4 {
let &pixel = s_rows[i].get_unchecked(x);
let &pixel = src_rows[i].get_unchecked(x);
let rg_i64x2 = _mm_set_epi64x(pixel.0[1] as i64, pixel.0[0] as i64);
rg_sum[i] = _mm_add_epi64(rg_sum[i], _mm_mul_epi32(rg_i64x2, coeff_i64x2));
let bb_i64x2 = _mm_set_epi64x(0, pixel.0[2] as i64);
@@ -132,7 +127,7 @@ unsafe fn horiz_convolution_8u4x(
for i in 0..4 {
_mm_storeu_si128((&mut rg_buf).as_mut_ptr() as *mut __m128i, rg_sum[i]);
_mm_storeu_si128((&mut bb_buf).as_mut_ptr() as *mut __m128i, bb_sum[i]);
let dst_pixel = d_rows[i].get_unchecked_mut(dst_x);
let dst_pixel = dst_rows[i].get_unchecked_mut(dst_x);
dst_pixel.0[0] = normalizer.clip(rg_buf[0] + half_error);
dst_pixel.0[1] = normalizer.clip(rg_buf[1] + half_error);
dst_pixel.0[2] = normalizer.clip(bb_buf[0] + bb_buf[1] + half_error);
src/convolution/u16x4/avx2.rs
+8 -13
View File
@@ -1,7 +1,6 @@
use std::arch::x86_64::*;
use crate::convolution::{optimisations, Coefficients};
use crate::image_view::{FourRows, FourRowsMut};
use crate::pixels::U16x4;
use crate::simd_utils;
use crate::{ImageView, ImageViewMut};
@@ -47,15 +46,11 @@ pub(crate) fn horiz_convolution(
/// - precision <= MAX_COEFS_PRECISION
#[target_feature(enable = "avx2")]
unsafe fn horiz_convolution_four_rows(
src_rows: FourRows<U16x4>,
dst_rows: FourRowsMut<U16x4>,
src_rows: [&[U16x4]; 4],
dst_rows: [&mut &mut [U16x4]; 4],
coefficients_chunks: &[optimisations::CoefficientsI32Chunk],
normalizer: &optimisations::Normalizer32,
) {
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
let s_rows = [s_row0, s_row1, s_row2, s_row3];
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
let d_rows = [d_row0, d_row1, d_row2, d_row3];
let precision = normalizer.precision();
let half_error = 1i64 << (precision - 1);
let mut rg_buf = [0i64; 4];
@@ -114,8 +109,8 @@ unsafe fn horiz_convolution_four_rows(
for i in 0..2 {
let source = _mm256_set_m128i(
simd_utils::loadu_si128(s_rows[i * 2 + 1], x),
simd_utils::loadu_si128(s_rows[i * 2], x),
simd_utils::loadu_si128(src_rows[i * 2 + 1], x),
simd_utils::loadu_si128(src_rows[i * 2], x),
);
let mut sum = rg_sum[i];
@@ -140,8 +135,8 @@ unsafe fn horiz_convolution_four_rows(
for i in 0..2 {
let source = _mm256_set_m128i(
simd_utils::loadl_epi64(s_rows[i * 2 + 1], x),
simd_utils::loadl_epi64(s_rows[i * 2], x),
simd_utils::loadl_epi64(src_rows[i * 2 + 1], x),
simd_utils::loadl_epi64(src_rows[i * 2], x),
);
let mut sum = rg_sum[i];
@@ -160,7 +155,7 @@ unsafe fn horiz_convolution_four_rows(
_mm256_storeu_si256((&mut rg_buf).as_mut_ptr() as *mut __m256i, rg_sum[i]);
_mm256_storeu_si256((&mut ba_buf).as_mut_ptr() as *mut __m256i, ba_sum[i]);
let dst_pixel = d_rows[i * 2].get_unchecked_mut(dst_x);
let dst_pixel = dst_rows[i * 2].get_unchecked_mut(dst_x);
dst_pixel.0 = [
normalizer.clip(rg_buf[0]),
normalizer.clip(rg_buf[1]),
@@ -168,7 +163,7 @@ unsafe fn horiz_convolution_four_rows(
normalizer.clip(ba_buf[1]),
];
let dst_pixel = d_rows[i * 2 + 1].get_unchecked_mut(dst_x);
let dst_pixel = dst_rows[i * 2 + 1].get_unchecked_mut(dst_x);
dst_pixel.0 = [
normalizer.clip(rg_buf[2]),
normalizer.clip(rg_buf[3]),
src/convolution/u16x4/neon.rs
+7 -13
View File
@@ -1,7 +1,6 @@
use std::arch::aarch64::*;
use crate::convolution::{optimisations, Coefficients};
use crate::image_view::{FourRows, FourRowsMut};
use crate::neon_utils;
use crate::pixels::U16x4;
use crate::{ImageView, ImageViewMut};
@@ -48,16 +47,11 @@ pub(crate) fn horiz_convolution(
/// - precision <= MAX_COEFS_PRECISION
#[target_feature(enable = "neon")]
unsafe fn horiz_convolution_4_rows(
src_rows: FourRows<U16x4>,
dst_rows: FourRowsMut<U16x4>,
src_rows: [&[U16x4]; 4],
dst_rows: [&mut &mut [U16x4]; 4],
coefficients_chunks: &[optimisations::CoefficientsI32Chunk],
precision: u8,
) {
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
let s_rows = [s_row0, s_row1, s_row2, s_row3];
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
let d_rows = [d_row0, d_row1, d_row2, d_row3];
let initial = vdupq_n_s64(1i64 << (precision - 1));
let zero_u16x8 = vdupq_n_u16(0);
let zero_u16x4 = vdup_n_u16(0);
@@ -80,7 +74,7 @@ unsafe fn horiz_convolution_4_rows(
for i in 0..4 {
let mut sss = sss_a[i];
let source = neon_utils::load_u16x8(s_rows[i], x);
let source = neon_utils::load_u16x8(src_rows[i], x);
let pix_i32 = vreinterpretq_s32_u16(vzip1q_u16(source, zero_u16x8));
sss.0 = vmlal_s32(sss.0, vget_low_s32(pix_i32), coeff0);
@@ -90,7 +84,7 @@ unsafe fn horiz_convolution_4_rows(
sss.0 = vmlal_s32(sss.0, vget_low_s32(pix_i32), coeff1);
sss.1 = vmlal_s32(sss.1, vget_high_s32(pix_i32), coeff1);
let source = neon_utils::load_u16x8(s_rows[i], x + 2);
let source = neon_utils::load_u16x8(src_rows[i], x + 2);
let pix_i32 = vreinterpretq_s32_u16(vzip1q_u16(source, zero_u16x8));
sss.0 = vmlal_s32(sss.0, vget_low_s32(pix_i32), coeff2);
@@ -116,7 +110,7 @@ unsafe fn horiz_convolution_4_rows(
for i in 0..4 {
let mut sss = sss_a[i];
let source = neon_utils::load_u16x8(s_rows[i], x);
let source = neon_utils::load_u16x8(src_rows[i], x);
let pix_i32 = vreinterpretq_s32_u16(vzip1q_u16(source, zero_u16x8));
sss.0 = vmlal_s32(sss.0, vget_low_s32(pix_i32), coeff0);
@@ -136,7 +130,7 @@ unsafe fn horiz_convolution_4_rows(
for i in 0..4 {
let mut sss = sss_a[i];
let source = vcombine_u16(neon_utils::load_u16x4(s_rows[i], x), zero_u16x4);
let source = vcombine_u16(neon_utils::load_u16x4(src_rows[i], x), zero_u16x4);
let pix_i32 = vreinterpretq_s32_u16(vzip1q_u16(source, zero_u16x8));
sss.0 = vmlal_s32(sss.0, vget_low_s32(pix_i32), coeff);
@@ -164,7 +158,7 @@ unsafe fn horiz_convolution_4_rows(
let sss = sss_a[i];
let sss_i32x4 = vcombine_s32(vqmovn_s64(sss.0), vqmovn_s64(sss.1));
let sss_u16x4 = vqmovun_s32(sss_i32x4);
let dst_pix = d_rows[i].get_unchecked_mut(dst_x);
let dst_pix = dst_rows[i].get_unchecked_mut(dst_x);
let ptr = dst_pix as *mut U16x4 as *mut u16;
vst1_u16(ptr, sss_u16x4);
}
src/convolution/u16x4/sse4.rs
+5 -10
View File
@@ -1,7 +1,6 @@
use std::arch::x86_64::*;
use crate::convolution::{optimisations, Coefficients};
use crate::image_view::{FourRows, FourRowsMut};
use crate::pixels::U16x4;
use crate::simd_utils;
use crate::{ImageView, ImageViewMut};
@@ -47,15 +46,11 @@ pub(crate) fn horiz_convolution(
/// - precision <= MAX_COEFS_PRECISION
#[target_feature(enable = "sse4.1")]
unsafe fn horiz_convolution_four_rows(
src_rows: FourRows<U16x4>,
dst_rows: FourRowsMut<U16x4>,
src_rows: [&[U16x4]; 4],
dst_rows: [&mut &mut [U16x4]; 4],
coefficients_chunks: &[optimisations::CoefficientsI32Chunk],
normalizer: &optimisations::Normalizer32,
) {
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
let s_rows = [s_row0, s_row1, s_row2, s_row3];
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
let d_rows = [d_row0, d_row1, d_row2, d_row3];
let precision = normalizer.precision();
let half_error = 1i64 << (precision - 1);
let mut rg_buf = [0i64; 2];
@@ -100,7 +95,7 @@ unsafe fn horiz_convolution_four_rows(
let coeff1_i64x2 = _mm_set1_epi64x(k[1] as i64);
for i in 0..4 {
let source = simd_utils::loadu_si128(s_rows[i], x);
let source = simd_utils::loadu_si128(src_rows[i], x);
let mut sum = rg_sum[i];
let rg_i64x2 = _mm_shuffle_epi8(source, rg0_shuffle);
sum = _mm_add_epi64(sum, _mm_mul_epi32(rg_i64x2, coeff0_i64x2));
@@ -121,7 +116,7 @@ unsafe fn horiz_convolution_four_rows(
if let Some(&k) = coeffs.first() {
let coeff0_i64x2 = _mm_set1_epi64x(k as i64);
for i in 0..4 {
let source = simd_utils::loadl_epi64(s_rows[i], x);
let source = simd_utils::loadl_epi64(src_rows[i], x);
let rg_i64x2 = _mm_shuffle_epi8(source, rg0_shuffle);
rg_sum[i] = _mm_add_epi64(rg_sum[i], _mm_mul_epi32(rg_i64x2, coeff0_i64x2));
let ba_i64x2 = _mm_shuffle_epi8(source, ba0_shuffle);
@@ -132,7 +127,7 @@ unsafe fn horiz_convolution_four_rows(
for i in 0..4 {
_mm_storeu_si128((&mut rg_buf).as_mut_ptr() as *mut __m128i, rg_sum[i]);
_mm_storeu_si128((&mut ba_buf).as_mut_ptr() as *mut __m128i, ba_sum[i]);
let dst_pixel = d_rows[i].get_unchecked_mut(dst_x);
let dst_pixel = dst_rows[i].get_unchecked_mut(dst_x);
dst_pixel.0 = [
normalizer.clip(rg_buf[0]),
normalizer.clip(rg_buf[1]),
src/convolution/u8x1/avx2.rs
+6 -9
View File
@@ -1,7 +1,6 @@
use std::arch::x86_64::*;
use crate::convolution::{optimisations, Coefficients};
use crate::image_view::{FourRows, FourRowsMut};
use crate::pixels::U8;
use crate::simd_utils;
use crate::{ImageView, ImageViewMut};
@@ -48,13 +47,11 @@ pub(crate) fn horiz_convolution(
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn horiz_convolution_8u4x(
src_rows: FourRows<U8>,
dst_rows: FourRowsMut<U8>,
src_rows: [&[U8]; 4],
dst_rows: [&mut &mut [U8]; 4],
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
normalizer: &optimisations::Normalizer16,
) {
let s_rows = [src_rows.0, src_rows.1, src_rows.2, src_rows.3];
let d_rows = [dst_rows.0, dst_rows.1, dst_rows.2, dst_rows.3];
let zero = _mm_setzero_si128();
// 8 components will be added, use only 1/8 of the error
let initial = _mm256_set1_epi32(1 << (normalizer.precision() - 4));
@@ -69,7 +66,7 @@ unsafe fn horiz_convolution_8u4x(
for k in coeffs_by_16 {
let coeffs_i16x16 = _mm256_loadu_si256(k.as_ptr() as *const __m256i);
for i in 0..4 {
let pixels_u8x16 = simd_utils::loadu_si128(s_rows[i], x);
let pixels_u8x16 = simd_utils::loadu_si128(src_rows[i], x);
let pixels_i16x16 = _mm256_cvtepu8_epi16(pixels_u8x16);
result_i32x8x4[i] = _mm256_add_epi32(
result_i32x8x4[i],
@@ -84,7 +81,7 @@ unsafe fn horiz_convolution_8u4x(
if let Some(k) = coeffs_by_8.next() {
let coeffs_i16x8 = _mm_loadu_si128(k.as_ptr() as *const __m128i);
for i in 0..4 {
let pixels_u8x8 = simd_utils::loadl_epi64(s_rows[i], x);
let pixels_u8x8 = simd_utils::loadl_epi64(src_rows[i], x);
let pixels_i16x8 = _mm_cvtepu8_epi16(pixels_u8x8);
result_i32x8x4[i] = _mm256_add_epi32(
result_i32x8x4[i],
@@ -99,14 +96,14 @@ unsafe fn horiz_convolution_8u4x(
for &coeff in reminder8 {
let coeff_i32 = coeff as i32;
for i in 0..4 {
result_i32x4[i] += s_rows[i].get_unchecked(x).0.to_owned() as i32 * coeff_i32;
result_i32x4[i] += src_rows[i].get_unchecked(x).0.to_owned() as i32 * coeff_i32;
}
x += 1;
}
let result_u8x4 = result_i32x4.map(|v| normalizer.clip(v));
for i in 0..4 {
d_rows[i].get_unchecked_mut(dst_x).0 = result_u8x4[i];
dst_rows[i].get_unchecked_mut(dst_x).0 = result_u8x4[i];
}
}
}
src/convolution/u8x1/neon.rs
+8 -14
View File
@@ -1,7 +1,6 @@
use std::arch::aarch64::*;
use crate::convolution::{optimisations, Coefficients};
use crate::image_view::{FourRows, FourRowsMut};
use crate::neon_utils;
use crate::pixels::U8;
use crate::{ImageView, ImageViewMut};
@@ -47,16 +46,11 @@ pub(crate) fn horiz_convolution(
/// - precision <= MAX_COEFS_PRECISION
#[target_feature(enable = "neon")]
unsafe fn horiz_convolution_four_rows(
src_rows: FourRows<U8>,
dst_rows: FourRowsMut<U8>,
src_rows: [&[U8]; 4],
dst_rows: [&mut &mut [U8]; 4],
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
normalizer: &optimisations::Normalizer16,
) {
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
let s_rows = [s_row0, s_row1, s_row2, s_row3];
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
let d_rows = [d_row0, d_row1, d_row2, d_row3];
let precision = normalizer.precision();
let initial = vdupq_n_s32(1 << (precision - 3));
let zero_u8x16 = vdupq_n_u8(0);
@@ -77,7 +71,7 @@ unsafe fn horiz_convolution_four_rows(
let coeff3 = vget_high_s16(coeffs_i16x8x2.1);
for i in 0..4 {
let source = neon_utils::load_u8x16(s_rows[i], x);
let source = neon_utils::load_u8x16(src_rows[i], x);
let mut sss = sss_a[i];
let source_i16 = vreinterpretq_s16_u8(vzip1q_u8(source, zero_u8x16));
@@ -102,7 +96,7 @@ unsafe fn horiz_convolution_four_rows(
let coeff1 = vget_high_s16(coeffs_i16x8);
for i in 0..4 {
let source = neon_utils::load_u8x8(s_rows[i], x);
let source = neon_utils::load_u8x8(src_rows[i], x);
let mut sss = sss_a[i];
let pix = vreinterpret_s16_u8(vzip1_u8(source, zero_u8x8));
@@ -120,7 +114,7 @@ unsafe fn horiz_convolution_four_rows(
if let Some(k) = coeffs_by_4.next() {
let coeffs_i16x4 = neon_utils::load_i16x4(k, 0);
for i in 0..4 {
let source = neon_utils::load_u8x4(s_rows[i], x);
let source = neon_utils::load_u8x4(src_rows[i], x);
sss_a[i] = conv_4_pixels(sss_a[i], coeffs_i16x4, source, zero_u8x8);
}
x += 4;
@@ -131,7 +125,7 @@ unsafe fn horiz_convolution_four_rows(
if let Some(k) = coeffs_by_2.next() {
let coeffs_i16x4 = neon_utils::load_i16x2(k, 0);
for i in 0..4 {
let source = neon_utils::load_u8x2(s_rows[i], x);
let source = neon_utils::load_u8x2(src_rows[i], x);
sss_a[i] = conv_4_pixels(sss_a[i], coeffs_i16x4, source, zero_u8x8);
}
x += 2;
@@ -140,7 +134,7 @@ unsafe fn horiz_convolution_four_rows(
if !coeffs.is_empty() {
let coeffs_i16x4 = neon_utils::load_i16x1(coeffs, 0);
for i in 0..4 {
let source = neon_utils::load_u8x1(s_rows[i], x);
let source = neon_utils::load_u8x1(src_rows[i], x);
sss_a[i] = conv_4_pixels(sss_a[i], coeffs_i16x4, source, zero_u8x8);
}
}
@@ -149,7 +143,7 @@ unsafe fn horiz_convolution_four_rows(
let sss = sss_a[i];
let res_i32x2 = vadd_s32(vget_low_s32(sss), vget_high_s32(sss));
let res = vget_lane_s32::<0>(res_i32x2) + vget_lane_s32::<1>(res_i32x2);
d_rows[i].get_unchecked_mut(dst_x).0 = normalizer.clip(res);
dst_rows[i].get_unchecked_mut(dst_x).0 = normalizer.clip(res);
}
}
}
src/convolution/u8x1/sse4.rs
+6 -9
View File
@@ -1,7 +1,6 @@
use std::arch::x86_64::*;
use crate::convolution::{optimisations, Coefficients};
use crate::image_view::{FourRows, FourRowsMut};
use crate::pixels::U8;
use crate::simd_utils;
use crate::{ImageView, ImageViewMut};
@@ -48,13 +47,11 @@ pub(crate) fn horiz_convolution(
#[inline]
#[target_feature(enable = "sse4.1")]
unsafe fn horiz_convolution_four_rows(
src_rows: FourRows<U8>,
dst_rows: FourRowsMut<U8>,
src_rows: [&[U8]; 4],
dst_rows: [&mut &mut [U8]; 4],
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
normalizer: &optimisations::Normalizer16,
) {
let s_rows = [src_rows.0, src_rows.1, src_rows.2, src_rows.3];
let d_rows = [dst_rows.0, dst_rows.1, dst_rows.2, dst_rows.3];
let zero = _mm_setzero_si128();
let initial = 1 << (normalizer.precision() - 1);
let mut buf = [0, 0, 0, 0, initial];
@@ -69,7 +66,7 @@ unsafe fn horiz_convolution_four_rows(
for k in coeffs_by_8 {
let coeffs_i16x8 = _mm_loadu_si128(k.as_ptr() as *const __m128i);
for i in 0..4 {
let pixels_u8x8 = simd_utils::loadl_epi64(s_rows[i], x);
let pixels_u8x8 = simd_utils::loadl_epi64(src_rows[i], x);
let pixels_i16x8 = _mm_cvtepu8_epi16(pixels_u8x8);
result_i32x4[i] =
_mm_add_epi32(result_i32x4[i], _mm_madd_epi16(pixels_i16x8, coeffs_i16x8));
@@ -82,7 +79,7 @@ unsafe fn horiz_convolution_four_rows(
if let Some(k) = coeffs_by_4.next() {
let coeffs_i16x4 = simd_utils::loadl_epi64(k, 0);
for i in 0..4 {
let pixels_u8x4 = simd_utils::loadl_epi32(s_rows[i], x);
let pixels_u8x4 = simd_utils::loadl_epi32(src_rows[i], x);
let pixels_i16x4 = _mm_cvtepu8_epi16(pixels_u8x4);
result_i32x4[i] =
_mm_add_epi32(result_i32x4[i], _mm_madd_epi16(pixels_i16x4, coeffs_i16x4));
@@ -98,14 +95,14 @@ unsafe fn horiz_convolution_four_rows(
for &coeff in reminder4 {
let coeff_i32 = coeff as i32;
for i in 0..4 {
result_i32x4[i] += s_rows[i].get_unchecked(x).0.to_owned() as i32 * coeff_i32;
result_i32x4[i] += src_rows[i].get_unchecked(x).0.to_owned() as i32 * coeff_i32;
}
x += 1;
}
let result_u8x4 = result_i32x4.map(|v| normalizer.clip(v));
for i in 0..4 {
d_rows[i].get_unchecked_mut(dst_x).0 = result_u8x4[i];
dst_rows[i].get_unchecked_mut(dst_x).0 = result_u8x4[i];
}
}
}
src/convolution/u8x2/avx2.rs
+22 -25
View File
@@ -1,7 +1,6 @@
use std::arch::x86_64::*;
use crate::convolution::{Coefficients, optimisations};
use crate::image_view::{FourRows, FourRowsMut};
use crate::pixels::U8x2;
use crate::simd_utils;
use crate::{ImageView, ImageViewMut};
@@ -53,13 +52,11 @@ pub(crate) fn horiz_convolution(
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn horiz_convolution_four_rows(
src_rows: FourRows<U8x2>,
dst_rows: FourRowsMut<U8x2>,
src_rows: [&[U8x2]; 4],
dst_rows: [&mut &mut [U8x2]; 4],
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
normalizer: &optimisations::Normalizer16,
) {
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
let precision = normalizer.precision();
let initial = _mm256_set1_epi32(1 << (precision - 2));
@@ -102,8 +99,8 @@ unsafe fn horiz_convolution_four_rows(
let mmk1 = simd_utils::ptr_i16_to_256set1_epi64x(k, 4);
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadu_si128(s_row0, x)),
simd_utils::loadu_si128(s_row1, x),
_mm256_castsi128_si256(simd_utils::loadu_si128(src_rows[0], x)),
simd_utils::loadu_si128(src_rows[1], x),
);
let pix = _mm256_shuffle_epi8(source, sh1);
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk0));
@@ -111,8 +108,8 @@ unsafe fn horiz_convolution_four_rows(
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk1));
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadu_si128(s_row2, x)),
simd_utils::loadu_si128(s_row3, x),
_mm256_castsi128_si256(simd_utils::loadu_si128(src_rows[2], x)),
simd_utils::loadu_si128(src_rows[3], x),
);
let pix = _mm256_shuffle_epi8(source, sh1);
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk0));
@@ -129,15 +126,15 @@ unsafe fn horiz_convolution_four_rows(
let mmk = simd_utils::ptr_i16_to_256set1_epi64x(k, 0);
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadl_epi64(s_row0, x)),
simd_utils::loadl_epi64(s_row1, x),
_mm256_castsi128_si256(simd_utils::loadl_epi64(src_rows[0], x)),
simd_utils::loadl_epi64(src_rows[1], x),
);
let pix = _mm256_shuffle_epi8(source, sh1);
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadl_epi64(s_row2, x)),
simd_utils::loadl_epi64(s_row3, x),
_mm256_castsi128_si256(simd_utils::loadl_epi64(src_rows[2], x)),
simd_utils::loadl_epi64(src_rows[3], x),
);
let pix = _mm256_shuffle_epi8(source, sh1);
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
@@ -152,15 +149,15 @@ unsafe fn horiz_convolution_four_rows(
let mmk = simd_utils::ptr_i16_to_256set1_epi32(k, 0);
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadl_epi32(s_row0, x)),
simd_utils::loadl_epi32(s_row1, x),
_mm256_castsi128_si256(simd_utils::loadl_epi32(src_rows[0], x)),
simd_utils::loadl_epi32(src_rows[1], x),
);
let pix = _mm256_shuffle_epi8(source, sh1);
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadl_epi32(s_row2, x)),
simd_utils::loadl_epi32(s_row3, x),
_mm256_castsi128_si256(simd_utils::loadl_epi32(src_rows[2], x)),
simd_utils::loadl_epi32(src_rows[3], x),
);
let pix = _mm256_shuffle_epi8(source, sh1);
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
@@ -174,15 +171,15 @@ unsafe fn horiz_convolution_four_rows(
// [16] xx a0 xx b0 xx g0 xx r0 xx a0 xx b0 xx g0 xx r0
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadl_epi16(s_row0, x)),
simd_utils::loadl_epi16(s_row1, x),
_mm256_castsi128_si256(simd_utils::loadl_epi16(src_rows[0], x)),
simd_utils::loadl_epi16(src_rows[1], x),
);
let pix = _mm256_shuffle_epi8(source, sh1);
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadl_epi16(s_row2, x)),
simd_utils::loadl_epi16(s_row3, x),
_mm256_castsi128_si256(simd_utils::loadl_epi16(src_rows[2], x)),
simd_utils::loadl_epi16(src_rows[3], x),
);
let pix = _mm256_shuffle_epi8(source, sh1);
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
@@ -190,13 +187,13 @@ unsafe fn horiz_convolution_four_rows(
let lo128 = _mm256_extracti128_si256::<0>(sss0);
let hi128 = _mm256_extracti128_si256::<1>(sss0);
set_dst_pixel(lo128, d_row0, dst_x, normalizer);
set_dst_pixel(hi128, d_row1, dst_x, normalizer);
set_dst_pixel(lo128, dst_rows[0], dst_x, normalizer);
set_dst_pixel(hi128, dst_rows[1], dst_x, normalizer);
let lo128 = _mm256_extracti128_si256::<0>(sss1);
let hi128 = _mm256_extracti128_si256::<1>(sss1);
set_dst_pixel(lo128, d_row2, dst_x, normalizer);
set_dst_pixel(hi128, d_row3, dst_x, normalizer);
set_dst_pixel(lo128, dst_rows[2], dst_x, normalizer);
set_dst_pixel(hi128, dst_rows[3], dst_x, normalizer);
}
}
src/convolution/u8x2/neon.rs
+6 -12
View File
@@ -1,7 +1,6 @@
use std::arch::aarch64::*;
use crate::convolution::{optimisations, Coefficients};
use crate::image_view::{FourRows, FourRowsMut};
use crate::neon_utils;
use crate::pixels::U8x2;
use crate::{ImageView, ImageViewMut};
@@ -48,16 +47,11 @@ pub(crate) fn horiz_convolution(
/// - precision <= MAX_COEFS_PRECISION
#[target_feature(enable = "neon")]
unsafe fn horiz_convolution_four_rows(
src_rows: FourRows<U8x2>,
dst_rows: FourRowsMut<U8x2>,
src_rows: [&[U8x2]; 4],
dst_rows: [&mut &mut [U8x2]; 4],
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
precision: u8,
) {
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
let s_rows = [s_row0, s_row1, s_row2, s_row3];
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
let d_rows = [d_row0, d_row1, d_row2, d_row3];
let initial = vdupq_n_s32(1 << (precision - 2));
let zero_u8x16 = vdupq_n_u8(0);
let zero_u8x8 = vdup_n_u8(0);
@@ -79,7 +73,7 @@ unsafe fn horiz_convolution_four_rows(
let coeff3 = vget_high_s16(coeff23);
for i in 0..4 {
let source = neon_utils::load_u8x16(s_rows[i], x);
let source = neon_utils::load_u8x16(src_rows[i], x);
let mut sss = sss_a[i];
let source_i16 = vreinterpretq_s16_u8(vzip1q_u8(source, zero_u8x16));
@@ -105,7 +99,7 @@ unsafe fn horiz_convolution_four_rows(
let coeff1 = vzip2_s16(coeffs_i16x4, coeffs_i16x4);
for i in 0..4 {
let source = neon_utils::load_u8x8(s_rows[i], x);
let source = neon_utils::load_u8x8(src_rows[i], x);
let mut sss = sss_a[i];
let pix = vreinterpret_s16_u8(vzip1_u8(source, zero_u8x8));
@@ -133,7 +127,7 @@ unsafe fn horiz_convolution_four_rows(
for i in 0..4 {
four_pixels
.iter_mut()
.zip(s_rows[i].get_unchecked(x..))
.zip(src_rows[i].get_unchecked(x..))
.for_each(|(d, s)| *d = *s);
let source = neon_utils::load_u8x8(&four_pixels, 0);
let mut sss = sss_a[i];
@@ -165,7 +159,7 @@ unsafe fn horiz_convolution_four_rows(
vqmovn_s32(sss),
vreinterpret_s16_u8(zero_u8x8),
)));
d_rows[i].get_unchecked_mut(dst_x).0 = vget_lane_u16::<0>(s);
dst_rows[i].get_unchecked_mut(dst_x).0 = vget_lane_u16::<0>(s);
}
}
}
src/convolution/u8x2/sse4.rs
+7 -12
View File
@@ -1,7 +1,6 @@
use std::arch::x86_64::*;
use crate::convolution::{optimisations, Coefficients};
use crate::image_view::{FourRows, FourRowsMut};
use crate::pixels::U8x2;
use crate::simd_utils;
use crate::{ImageView, ImageViewMut};
@@ -48,15 +47,11 @@ pub(crate) fn horiz_convolution(
#[inline]
#[target_feature(enable = "sse4.1")]
unsafe fn horiz_convolution_four_rows(
src_rows: FourRows<U8x2>,
dst_rows: FourRowsMut<U8x2>,
src_rows: [&[U8x2]; 4],
dst_rows: [&mut &mut [U8x2]; 4],
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
normalizer: &optimisations::Normalizer16,
) {
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
let s_rows = [s_row0, s_row1, s_row2, s_row3];
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
let d_rows = [d_row0, d_row1, d_row2, d_row3];
let precision = normalizer.precision();
let initial = _mm_set1_epi32(1 << (precision - 2));
@@ -96,7 +91,7 @@ unsafe fn horiz_convolution_four_rows(
let mmk1 = simd_utils::ptr_i16_to_set1_epi64x(k, 4);
for i in 0..4 {
let source = simd_utils::loadu_si128(s_rows[i], x);
let source = simd_utils::loadu_si128(src_rows[i], x);
let pix = _mm_shuffle_epi8(source, sh1);
let tmp_sum = _mm_add_epi32(sss[i], _mm_madd_epi16(pix, mmk0));
let pix = _mm_shuffle_epi8(source, sh2);
@@ -112,7 +107,7 @@ unsafe fn horiz_convolution_four_rows(
let mmk = simd_utils::ptr_i16_to_set1_epi64x(k, 0);
for i in 0..4 {
let source = simd_utils::loadl_epi64(s_rows[i], x);
let source = simd_utils::loadl_epi64(src_rows[i], x);
let pix = _mm_shuffle_epi8(source, sh1);
sss[i] = _mm_add_epi32(sss[i], _mm_madd_epi16(pix, mmk));
}
@@ -126,7 +121,7 @@ unsafe fn horiz_convolution_four_rows(
let mmk = simd_utils::ptr_i16_to_set1_epi32(k, 0);
for i in 0..4 {
let source = simd_utils::loadl_epi32(s_rows[i], x);
let source = simd_utils::loadl_epi32(src_rows[i], x);
let pix = _mm_shuffle_epi8(source, sh1);
sss[i] = _mm_add_epi32(sss[i], _mm_madd_epi16(pix, mmk));
}
@@ -137,14 +132,14 @@ unsafe fn horiz_convolution_four_rows(
let mmk = _mm_set1_epi32(k as i32);
for i in 0..4 {
let source = simd_utils::loadl_epi16(s_rows[i], x);
let source = simd_utils::loadl_epi16(src_rows[i], x);
let pix = _mm_shuffle_epi8(source, sh1);
sss[i] = _mm_add_epi32(sss[i], _mm_madd_epi16(pix, mmk));
}
}
for i in 0..4 {
set_dst_pixel(sss[i], d_rows[i], dst_x, normalizer);
set_dst_pixel(sss[i], dst_rows[i], dst_x, normalizer);
}
}
}
src/convolution/u8x3/avx2.rs
+19 -22
View File
@@ -2,7 +2,6 @@ use std::arch::x86_64::*;
use std::intrinsics::transmute;
use crate::convolution::{optimisations, Coefficients};
use crate::image_view::{FourRows, FourRowsMut};
use crate::pixels::U8x3;
use crate::simd_utils;
use crate::{ImageView, ImageViewMut};
@@ -50,16 +49,14 @@ pub(crate) fn horiz_convolution(
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn horiz_convolution_8u4x(
src_rows: FourRows<U8x3>,
dst_rows: FourRowsMut<U8x3>,
src_rows: [&[U8x3]; 4],
dst_rows: [&mut &mut [U8x3]; 4],
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
precision: u8,
) {
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
let zero = _mm256_setzero_si256();
let initial = _mm256_set1_epi32(1 << (precision - 1));
let src_width = s_row0.len();
let src_width = src_rows[0].len();
/*
|R G B | |R G B | |R G B | |R G B | |R G B | |R |
@@ -108,8 +105,8 @@ unsafe fn horiz_convolution_8u4x(
let mmk1 = simd_utils::ptr_i16_to_256set1_epi32(k, 2);
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadu_si128(s_row0, x)),
simd_utils::loadu_si128(s_row1, x),
_mm256_castsi128_si256(simd_utils::loadu_si128(src_rows[0], x)),
simd_utils::loadu_si128(src_rows[1], x),
);
let pix = _mm256_shuffle_epi8(source, sh1);
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk0));
@@ -117,8 +114,8 @@ unsafe fn horiz_convolution_8u4x(
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk1));
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadu_si128(s_row2, x)),
simd_utils::loadu_si128(s_row3, x),
_mm256_castsi128_si256(simd_utils::loadu_si128(src_rows[2], x)),
simd_utils::loadu_si128(src_rows[3], x),
);
let pix = _mm256_shuffle_epi8(source, sh1);
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk0));
@@ -141,15 +138,15 @@ unsafe fn horiz_convolution_8u4x(
let mmk = simd_utils::ptr_i16_to_256set1_epi32(k, 0);
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadl_epi64(s_row0, x)),
simd_utils::loadl_epi64(s_row1, x),
_mm256_castsi128_si256(simd_utils::loadl_epi64(src_rows[0], x)),
simd_utils::loadl_epi64(src_rows[1], x),
);
let pix = _mm256_shuffle_epi8(source, sh1);
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadl_epi64(s_row2, x)),
simd_utils::loadl_epi64(s_row3, x),
_mm256_castsi128_si256(simd_utils::loadl_epi64(src_rows[2], x)),
simd_utils::loadl_epi64(src_rows[3], x),
);
let pix = _mm256_shuffle_epi8(source, sh1);
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
@@ -168,14 +165,14 @@ unsafe fn horiz_convolution_8u4x(
// [16] xx a0 xx b0 xx g0 xx r0 xx a0 xx b0 xx g0 xx r0
let pix = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::mm_cvtepu8_epi32_u8x3(s_row0, x)),
simd_utils::mm_cvtepu8_epi32_u8x3(s_row1, x),
_mm256_castsi128_si256(simd_utils::mm_cvtepu8_epi32_u8x3(src_rows[0], x)),
simd_utils::mm_cvtepu8_epi32_u8x3(src_rows[1], x),
);
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
let pix = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::mm_cvtepu8_epi32_u8x3(s_row2, x)),
simd_utils::mm_cvtepu8_epi32_u8x3(s_row3, x),
_mm256_castsi128_si256(simd_utils::mm_cvtepu8_epi32_u8x3(src_rows[2], x)),
simd_utils::mm_cvtepu8_epi32_u8x3(src_rows[3], x),
);
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
@@ -197,19 +194,19 @@ unsafe fn horiz_convolution_8u4x(
let pixel: u32 = transmute(_mm_cvtsi128_si32(_mm256_extracti128_si256::<0>(sss0)));
let bytes = pixel.to_le_bytes();
d_row0.get_unchecked_mut(dst_x).0 = [bytes[0], bytes[1], bytes[2]];
dst_rows[0].get_unchecked_mut(dst_x).0 = [bytes[0], bytes[1], bytes[2]];
let pixel: u32 = transmute(_mm_cvtsi128_si32(_mm256_extracti128_si256::<1>(sss0)));
let bytes = pixel.to_le_bytes();
d_row1.get_unchecked_mut(dst_x).0 = [bytes[0], bytes[1], bytes[2]];
dst_rows[1].get_unchecked_mut(dst_x).0 = [bytes[0], bytes[1], bytes[2]];
let pixel: u32 = transmute(_mm_cvtsi128_si32(_mm256_extracti128_si256::<0>(sss1)));
let bytes = pixel.to_le_bytes();
d_row2.get_unchecked_mut(dst_x).0 = [bytes[0], bytes[1], bytes[2]];
dst_rows[2].get_unchecked_mut(dst_x).0 = [bytes[0], bytes[1], bytes[2]];
let pixel: u32 = transmute(_mm_cvtsi128_si32(_mm256_extracti128_si256::<1>(sss1)));
let bytes = pixel.to_le_bytes();
d_row3.get_unchecked_mut(dst_x).0 = [bytes[0], bytes[1], bytes[2]];
dst_rows[3].get_unchecked_mut(dst_x).0 = [bytes[0], bytes[1], bytes[2]];
}
}
src/convolution/u8x3/neon.rs
+6 -12
View File
@@ -1,7 +1,6 @@
use std::arch::aarch64::*;
use crate::convolution::{optimisations, Coefficients};
use crate::image_view::{FourRows, FourRowsMut};
use crate::neon_utils;
use crate::pixels::U8x3;
use crate::{ImageView, ImageViewMut};
@@ -48,16 +47,11 @@ pub(crate) fn horiz_convolution(
/// - precision <= MAX_COEFS_PRECISION
#[target_feature(enable = "neon")]
unsafe fn horiz_convolution_four_rows(
src_rows: FourRows<U8x3>,
dst_rows: FourRowsMut<U8x3>,
src_rows: [&[U8x3]; 4],
dst_rows: [&mut &mut [U8x3]; 4],
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
precision: u8,
) {
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
let s_rows = [s_row0, s_row1, s_row2, s_row3];
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
let d_rows = [d_row0, d_row1, d_row2, d_row3];
let initial = vdupq_n_s32(1 << (precision - 1));
let zero_u8x8 = vdup_n_u8(0);
@@ -71,7 +65,7 @@ unsafe fn horiz_convolution_four_rows(
for k in coeffs_by_8 {
let coeffs_i16x8 = neon_utils::load_i16x8(k, 0);
for i in 0..4 {
sss_a[i] = conv_8_pixels(sss_a[i], coeffs_i16x8, s_rows[i], x, zero_u8x8);
sss_a[i] = conv_8_pixels(sss_a[i], coeffs_i16x8, src_rows[i], x, zero_u8x8);
}
x += 8;
}
@@ -81,7 +75,7 @@ unsafe fn horiz_convolution_four_rows(
if let Some(k) = coeffs_by_4.next() {
let coeffs_i16x4 = neon_utils::load_i16x4(k, 0);
for i in 0..4 {
sss_a[i] = conv_4_pixels(sss_a[i], coeffs_i16x4, s_rows[i], x, zero_u8x8);
sss_a[i] = conv_4_pixels(sss_a[i], coeffs_i16x4, src_rows[i], x, zero_u8x8);
}
x += 4;
}
@@ -99,7 +93,7 @@ unsafe fn horiz_convolution_four_rows(
for i in 0..4 {
four_pixels
.iter_mut()
.zip(s_rows[i].get_unchecked(x..))
.zip(src_rows[i].get_unchecked(x..))
.for_each(|(d, s)| *d = *s);
sss_a[i] = conv_4_pixels(sss_a[i], coeffs_i16x4, &four_pixels, 0, zero_u8x8);
}
@@ -116,7 +110,7 @@ unsafe fn horiz_convolution_four_rows(
constify_imm8!(precision, call);
for i in 0..4 {
store_pixel(sss_a[i], d_rows[i], dst_x, zero_u8x8);
store_pixel(sss_a[i], dst_rows[i], dst_x, zero_u8x8);
}
}
}
src/convolution/u8x3/sse4.rs
+7 -12
View File
@@ -2,7 +2,6 @@ use std::arch::x86_64::*;
use std::intrinsics::transmute;
use crate::convolution::{optimisations, Coefficients};
use crate::image_view::{FourRows, FourRowsMut};
use crate::pixels::U8x3;
use crate::simd_utils;
use crate::{ImageView, ImageViewMut};
@@ -50,18 +49,14 @@ pub(crate) fn horiz_convolution(
#[inline]
#[target_feature(enable = "sse4.1")]
unsafe fn horiz_convolution_8u4x(
src_rows: FourRows<U8x3>,
dst_rows: FourRowsMut<U8x3>,
src_rows: [&[U8x3]; 4],
dst_rows: [&mut &mut [U8x3]; 4],
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
precision: u8,
) {
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
let s_rows = [s_row0, s_row1, s_row2, s_row3];
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
let d_rows = [d_row0, d_row1, d_row2, d_row3];
let zero = _mm_setzero_si128();
let initial = _mm_set1_epi32(1 << (precision - 1));
let src_width = s_row0.len();
let src_width = src_rows[0].len();
/*
|R G B | |R G B | |R G B | |R G B | |R G B | |R |
@@ -109,7 +104,7 @@ unsafe fn horiz_convolution_8u4x(
let mmk0 = simd_utils::ptr_i16_to_set1_epi32(k, 0);
let mmk1 = simd_utils::ptr_i16_to_set1_epi32(k, 2);
for i in 0..4 {
let source = simd_utils::loadu_si128(s_rows[i], x);
let source = simd_utils::loadu_si128(src_rows[i], x);
let pix = _mm_shuffle_epi8(source, sh_lo);
let mut sss = sss_a[i];
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk0));
@@ -136,7 +131,7 @@ unsafe fn horiz_convolution_8u4x(
let mmk = simd_utils::ptr_i16_to_set1_epi32(k, 0);
for i in 0..4 {
let source = simd_utils::loadl_epi64(s_rows[i], x);
let source = simd_utils::loadl_epi64(src_rows[i], x);
let pix = _mm_shuffle_epi8(source, sh_lo);
sss_a[i] = _mm_add_epi32(sss_a[i], _mm_madd_epi16(pix, mmk));
}
@@ -152,7 +147,7 @@ unsafe fn horiz_convolution_8u4x(
for &k in coeffs {
let mmk = _mm_set1_epi32(k as i32);
for i in 0..4 {
let pix = simd_utils::mm_cvtepu8_epi32_u8x3(s_rows[i], x);
let pix = simd_utils::mm_cvtepu8_epi32_u8x3(src_rows[i], x);
sss_a[i] = _mm_add_epi32(sss_a[i], _mm_madd_epi16(pix, mmk));
}
@@ -172,7 +167,7 @@ unsafe fn horiz_convolution_8u4x(
let sss = _mm_packs_epi32(sss_a[i], zero);
let pixel: u32 = transmute(_mm_cvtsi128_si32(_mm_packus_epi16(sss, zero)));
let bytes = pixel.to_le_bytes();
d_rows[i].get_unchecked_mut(dst_x).0 = [bytes[0], bytes[1], bytes[2]];
dst_rows[i].get_unchecked_mut(dst_x).0 = [bytes[0], bytes[1], bytes[2]];
}
}
}
src/convolution/u8x4/avx2.rs
+18 -21
View File
@@ -2,7 +2,6 @@ use std::arch::x86_64::*;
use std::intrinsics::transmute;
use crate::convolution::{optimisations, Coefficients};
use crate::image_view::{FourRows, FourRowsMut};
use crate::pixels::U8x4;
use crate::simd_utils;
use crate::{ImageView, ImageViewMut};
@@ -53,13 +52,11 @@ pub(crate) fn horiz_convolution(
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn horiz_convolution_8u4x(
src_rows: FourRows<U8x4>,
dst_rows: FourRowsMut<U8x4>,
src_rows: [&[U8x4]; 4],
dst_rows: [&mut &mut [U8x4]; 4],
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
precision: u8,
) {
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
let zero = _mm256_setzero_si256();
let initial = _mm256_set1_epi32(1 << (precision - 1));
@@ -89,8 +86,8 @@ unsafe fn horiz_convolution_8u4x(
let mmk1 = simd_utils::ptr_i16_to_256set1_epi32(k, 2);
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadu_si128(s_row0, x)),
simd_utils::loadu_si128(s_row1, x),
_mm256_castsi128_si256(simd_utils::loadu_si128(src_rows[0], x)),
simd_utils::loadu_si128(src_rows[1], x),
);
let pix = _mm256_shuffle_epi8(source, sh1);
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk0));
@@ -98,8 +95,8 @@ unsafe fn horiz_convolution_8u4x(
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk1));
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadu_si128(s_row2, x)),
simd_utils::loadu_si128(s_row3, x),
_mm256_castsi128_si256(simd_utils::loadu_si128(src_rows[2], x)),
simd_utils::loadu_si128(src_rows[3], x),
);
let pix = _mm256_shuffle_epi8(source, sh1);
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk0));
@@ -116,15 +113,15 @@ unsafe fn horiz_convolution_8u4x(
let mmk = simd_utils::ptr_i16_to_256set1_epi32(k, 0);
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadl_epi64(s_row0, x)),
simd_utils::loadl_epi64(s_row1, x),
_mm256_castsi128_si256(simd_utils::loadl_epi64(src_rows[0], x)),
simd_utils::loadl_epi64(src_rows[1], x),
);
let pix = _mm256_shuffle_epi8(source, sh1);
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadl_epi64(s_row2, x)),
simd_utils::loadl_epi64(s_row3, x),
_mm256_castsi128_si256(simd_utils::loadl_epi64(src_rows[2], x)),
simd_utils::loadl_epi64(src_rows[3], x),
);
let pix = _mm256_shuffle_epi8(source, sh1);
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
@@ -138,14 +135,14 @@ unsafe fn horiz_convolution_8u4x(
// [16] xx a0 xx b0 xx g0 xx r0 xx a0 xx b0 xx g0 xx r0
let pix = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::mm_cvtepu8_epi32(s_row0, x)),
simd_utils::mm_cvtepu8_epi32(s_row1, x),
_mm256_castsi128_si256(simd_utils::mm_cvtepu8_epi32(src_rows[0], x)),
simd_utils::mm_cvtepu8_epi32(src_rows[1], x),
);
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
let pix = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::mm_cvtepu8_epi32(s_row2, x)),
simd_utils::mm_cvtepu8_epi32(s_row3, x),
_mm256_castsi128_si256(simd_utils::mm_cvtepu8_epi32(src_rows[2], x)),
simd_utils::mm_cvtepu8_epi32(src_rows[3], x),
);
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
}
@@ -162,13 +159,13 @@ unsafe fn horiz_convolution_8u4x(
sss1 = _mm256_packs_epi32(sss1, zero);
sss0 = _mm256_packus_epi16(sss0, zero);
sss1 = _mm256_packus_epi16(sss1, zero);
*d_row0.get_unchecked_mut(dst_x) =
*dst_rows[0].get_unchecked_mut(dst_x) =
transmute(_mm_cvtsi128_si32(_mm256_extracti128_si256::<0>(sss0)));
*d_row1.get_unchecked_mut(dst_x) =
*dst_rows[1].get_unchecked_mut(dst_x) =
transmute(_mm_cvtsi128_si32(_mm256_extracti128_si256::<1>(sss0)));
*d_row2.get_unchecked_mut(dst_x) =
*dst_rows[2].get_unchecked_mut(dst_x) =
transmute(_mm_cvtsi128_si32(_mm256_extracti128_si256::<0>(sss1)));
*d_row3.get_unchecked_mut(dst_x) =
*dst_rows[3].get_unchecked_mut(dst_x) =
transmute(_mm_cvtsi128_si32(_mm256_extracti128_si256::<1>(sss1)));
}
}
src/convolution/u8x4/neon.rs
+8 -14
View File
@@ -1,7 +1,6 @@
use std::arch::aarch64::*;
use crate::convolution::{optimisations, Coefficients};
use crate::image_view::{FourRows, FourRowsMut};
use crate::neon_utils;
use crate::pixels::U8x4;
use crate::{ImageView, ImageViewMut};
@@ -48,16 +47,11 @@ pub(crate) fn horiz_convolution(
/// - precision <= MAX_COEFS_PRECISION
#[target_feature(enable = "neon")]
unsafe fn horiz_convolution_8u4x(
src_rows: FourRows<U8x4>,
dst_rows: FourRowsMut<U8x4>,
src_rows: [&[U8x4]; 4],
dst_rows: [&mut &mut [U8x4]; 4],
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
precision: u8,
) {
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
let s_rows = [s_row0, s_row1, s_row2, s_row3];
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
let d_rows = [d_row0, d_row1, d_row2, d_row3];
let initial = vdupq_n_s32(1 << (precision - 1));
let zero_u8x16 = vdupq_n_u8(0);
let zero_u8x8 = vdup_n_u8(0);
@@ -83,7 +77,7 @@ unsafe fn horiz_convolution_8u4x(
let coeff7 = vdup_laneq_s16::<7>(coeffs_i16x8);
for i in 0..4 {
let source = neon_utils::load_u8x16(s_rows[i], x);
let source = neon_utils::load_u8x16(src_rows[i], x);
let mut sss = sss_a[i];
let source_i16 = vreinterpretq_s16_u8(vzip1q_u8(source, zero_u8x16));
@@ -98,7 +92,7 @@ unsafe fn horiz_convolution_8u4x(
let pix = vget_high_s16(source_i16);
sss = vmlal_s16(sss, pix, coeff3);
let source = neon_utils::load_u8x16(s_rows[i], x + 4);
let source = neon_utils::load_u8x16(src_rows[i], x + 4);
let source_i16 = vreinterpretq_s16_u8(vzip1q_u8(source, zero_u8x16));
let pix = vget_low_s16(source_i16);
sss = vmlal_s16(sss, pix, coeff4);
@@ -128,7 +122,7 @@ unsafe fn horiz_convolution_8u4x(
let coeff3 = vdup_lane_s16::<3>(coeffs_i16x4);
for i in 0..4 {
let source = neon_utils::load_u8x16(s_rows[i], x);
let source = neon_utils::load_u8x16(src_rows[i], x);
let mut sss = sss_a[i];
let source_i16 = vreinterpretq_s16_u8(vzip1q_u8(source, zero_u8x16));
@@ -156,7 +150,7 @@ unsafe fn horiz_convolution_8u4x(
let coeff1 = vdup_n_s16(k[1]);
for i in 0..4 {
let source = neon_utils::load_u8x8(s_rows[i], x);
let source = neon_utils::load_u8x8(src_rows[i], x);
let mut sss = sss_a[i];
let pix = vreinterpret_s16_u8(vzip1_u8(source, zero_u8x8));
@@ -172,7 +166,7 @@ unsafe fn horiz_convolution_8u4x(
if let Some(&k) = coeffs.first() {
let coeff = vdup_n_s16(k);
for i in 0..4 {
let source = neon_utils::load_u8x4(s_rows[i], x);
let source = neon_utils::load_u8x4(src_rows[i], x);
let pix = vreinterpret_s16_u8(vzip1_u8(source, zero_u8x8));
sss_a[i] = vmlal_s16(sss_a[i], pix, coeff);
}
@@ -191,7 +185,7 @@ unsafe fn horiz_convolution_8u4x(
for i in 0..4 {
let s = vqmovun_s16(vcombine_s16(vqmovn_s32(sss_a[i]), vdup_n_s16(0)));
let s = vreinterpret_u32_u8(s);
d_rows[i].get_unchecked_mut(dst_x).0 = vget_lane_u32::<0>(s);
dst_rows[i].get_unchecked_mut(dst_x).0 = vget_lane_u32::<0>(s);
}
}
}
src/convolution/u8x4/sse4.rs
+18 -21
View File
@@ -2,7 +2,6 @@ use std::arch::x86_64::*;
use std::intrinsics::transmute;
use crate::convolution::{optimisations, Coefficients};
use crate::image_view::{FourRows, FourRowsMut};
use crate::pixels::U8x4;
use crate::simd_utils;
use crate::{ImageView, ImageViewMut};
@@ -52,13 +51,11 @@ pub(crate) fn horiz_convolution(
/// - precision <= MAX_COEFS_PRECISION
#[target_feature(enable = "sse4.1")]
unsafe fn horiz_convolution_8u4x(
src_rows: FourRows<U8x4>,
dst_rows: FourRowsMut<U8x4>,
src_rows: [&[U8x4]; 4],
dst_rows: [&mut &mut [U8x4]; 4],
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
precision: u8,
) {
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
let initial = _mm_set1_epi32(1 << (precision - 1));
let mask_lo = _mm_set_epi8(-1, 7, -1, 3, -1, 6, -1, 2, -1, 5, -1, 1, -1, 4, -1, 0);
let mask_hi = _mm_set_epi8(-1, 15, -1, 11, -1, 14, -1, 10, -1, 13, -1, 9, -1, 12, -1, 8);
@@ -81,7 +78,7 @@ unsafe fn horiz_convolution_8u4x(
let mmk_hi = simd_utils::ptr_i16_to_set1_epi32(k, 2);
// [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0
let mut source = simd_utils::loadu_si128(s_row0, x);
let mut source = simd_utils::loadu_si128(src_rows[0], x);
// [16] a1 a0 b1 b0 g1 g0 r1 r0
let mut pix = _mm_shuffle_epi8(source, mask_lo);
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk_lo));
@@ -89,19 +86,19 @@ unsafe fn horiz_convolution_8u4x(
pix = _mm_shuffle_epi8(source, mask_hi);
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk_hi));
source = simd_utils::loadu_si128(s_row1, x);
source = simd_utils::loadu_si128(src_rows[1], x);
pix = _mm_shuffle_epi8(source, mask_lo);
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk_lo));
pix = _mm_shuffle_epi8(source, mask_hi);
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk_hi));
source = simd_utils::loadu_si128(s_row2, x);
source = simd_utils::loadu_si128(src_rows[2], x);
pix = _mm_shuffle_epi8(source, mask_lo);
sss2 = _mm_add_epi32(sss2, _mm_madd_epi16(pix, mmk_lo));
pix = _mm_shuffle_epi8(source, mask_hi);
sss2 = _mm_add_epi32(sss2, _mm_madd_epi16(pix, mmk_hi));
source = simd_utils::loadu_si128(s_row3, x);
source = simd_utils::loadu_si128(src_rows[3], x);
pix = _mm_shuffle_epi8(source, mask_lo);
sss3 = _mm_add_epi32(sss3, _mm_madd_epi16(pix, mmk_lo));
pix = _mm_shuffle_epi8(source, mask_hi);
@@ -117,20 +114,20 @@ unsafe fn horiz_convolution_8u4x(
let mmk = simd_utils::ptr_i16_to_set1_epi32(k, 0);
// [8] x x x x x x x x a1 b1 g1 r1 a0 b0 g0 r0
let mut pix = simd_utils::loadl_epi64(s_row0, x);
let mut pix = simd_utils::loadl_epi64(src_rows[0], x);
// [16] a1 a0 b1 b0 g1 g0 r1 r0
pix = _mm_shuffle_epi8(pix, mask);
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
pix = simd_utils::loadl_epi64(s_row1, x);
pix = simd_utils::loadl_epi64(src_rows[1], x);
pix = _mm_shuffle_epi8(pix, mask);
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
pix = simd_utils::loadl_epi64(s_row2, x);
pix = simd_utils::loadl_epi64(src_rows[2], x);
pix = _mm_shuffle_epi8(pix, mask);
sss2 = _mm_add_epi32(sss2, _mm_madd_epi16(pix, mmk));
pix = simd_utils::loadl_epi64(s_row3, x);
pix = simd_utils::loadl_epi64(src_rows[3], x);
pix = _mm_shuffle_epi8(pix, mask);
sss3 = _mm_add_epi32(sss3, _mm_madd_epi16(pix, mmk));
@@ -141,16 +138,16 @@ unsafe fn horiz_convolution_8u4x(
// [16] xx k0 xx k0 xx k0 xx k0
let mmk = _mm_set1_epi32(k as i32);
// [16] xx a0 xx b0 xx g0 xx r0
let mut pix = simd_utils::mm_cvtepu8_epi32(s_row0, x);
let mut pix = simd_utils::mm_cvtepu8_epi32(src_rows[0], x);
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
pix = simd_utils::mm_cvtepu8_epi32(s_row1, x);
pix = simd_utils::mm_cvtepu8_epi32(src_rows[1], x);
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
pix = simd_utils::mm_cvtepu8_epi32(s_row2, x);
pix = simd_utils::mm_cvtepu8_epi32(src_rows[2], x);
sss2 = _mm_add_epi32(sss2, _mm_madd_epi16(pix, mmk));
pix = simd_utils::mm_cvtepu8_epi32(s_row3, x);
pix = simd_utils::mm_cvtepu8_epi32(src_rows[3], x);
sss3 = _mm_add_epi32(sss3, _mm_madd_epi16(pix, mmk));
}
@@ -168,13 +165,13 @@ unsafe fn horiz_convolution_8u4x(
sss1 = _mm_packs_epi32(sss1, sss1);
sss2 = _mm_packs_epi32(sss2, sss2);
sss3 = _mm_packs_epi32(sss3, sss3);
*d_row0.get_unchecked_mut(dst_x) =
*dst_rows[0].get_unchecked_mut(dst_x) =
transmute(_mm_cvtsi128_si32(_mm_packus_epi16(sss0, sss0)));
*d_row1.get_unchecked_mut(dst_x) =
*dst_rows[1].get_unchecked_mut(dst_x) =
transmute(_mm_cvtsi128_si32(_mm_packus_epi16(sss1, sss1)));
*d_row2.get_unchecked_mut(dst_x) =
*dst_rows[2].get_unchecked_mut(dst_x) =
transmute(_mm_cvtsi128_si32(_mm_packus_epi16(sss2, sss2)));
*d_row3.get_unchecked_mut(dst_x) =
*dst_rows[3].get_unchecked_mut(dst_x) =
transmute(_mm_cvtsi128_si32(_mm_packus_epi16(sss3, sss3)));
}
}
src/convolution/vertical_u16/sse4.rs
+9 -9
View File
@@ -77,13 +77,13 @@ unsafe fn vert_convolution_into_one_row_u16<T: PixelExt<Component = u16>>(
let coeffs_2 = coeffs.chunks_exact(2);
let coeffs_reminder = coeffs_2.remainder();
for ((s_row0, s_row1), two_coeffs) in src_img.iter_2_rows(y_start, max_y).zip(coeffs_2) {
let s_rows = [T::components(s_row0), T::components(s_row1)];
for (src_rows, two_coeffs) in src_img.iter_2_rows(y_start, max_y).zip(coeffs_2) {
let src_rows = src_rows.map(|row| T::components(row));
for r in 0..2 {
let coeff_i64x2 = _mm_set1_epi64x(two_coeffs[r] as i64);
for x in 0..2 {
let source = simd_utils::loadu_si128(s_rows[r], xx + x * 8);
let source = simd_utils::loadu_si128(src_rows[r], xx + x * 8);
for i in 0..4 {
let c_i64x2 = _mm_shuffle_epi8(source, c_shuffles[i]);
sums[i][x] = _mm_add_epi64(sums[i][x], _mm_mul_epi32(c_i64x2, coeff_i64x2));
@@ -128,15 +128,15 @@ unsafe fn vert_convolution_into_one_row_u16<T: PixelExt<Component = u16>>(
let coeffs_2 = coeffs.chunks_exact(2);
let coeffs_reminder = coeffs_2.remainder();
for ((s_row0, s_row1), two_coeffs) in src_img.iter_2_rows(y_start, max_y).zip(coeffs_2) {
let s_rows = [T::components(s_row0), T::components(s_row1)];
for (src_rows, two_coeffs) in src_img.iter_2_rows(y_start, max_y).zip(coeffs_2) {
let src_rows = src_rows.map(|row| T::components(row));
let coeffs_i64 = [
_mm_set1_epi64x(two_coeffs[0] as i64),
_mm_set1_epi64x(two_coeffs[1] as i64),
];
for r in 0..2 {
let source = simd_utils::loadu_si128(s_rows[r], xx);
let source = simd_utils::loadu_si128(src_rows[r], xx);
for i in 0..4 {
let c_i64x2 = _mm_shuffle_epi8(source, c_shuffles[i]);
sums[i] = _mm_add_epi64(sums[i], _mm_mul_epi32(c_i64x2, coeffs_i64[r]));
@@ -179,14 +179,14 @@ unsafe fn vert_convolution_into_one_row_u16<T: PixelExt<Component = u16>>(
let coeffs_2 = coeffs.chunks_exact(2);
let coeffs_reminder = coeffs_2.remainder();
for ((s_row0, s_row1), two_coeffs) in src_img.iter_2_rows(y_start, max_y).zip(coeffs_2) {
let s_rows = [T::components(s_row0), T::components(s_row1)];
for (src_rows, two_coeffs) in src_img.iter_2_rows(y_start, max_y).zip(coeffs_2) {
let src_rows = src_rows.map(|row| T::components(row));
let coeffs_i64 = [
_mm_set1_epi64x(two_coeffs[0] as i64),
_mm_set1_epi64x(two_coeffs[1] as i64),
];
for r in 0..2 {
let comp_x4 = s_rows[r].get_unchecked(xx..xx + 4);
let comp_x4 = src_rows[r].get_unchecked(xx..xx + 4);
let c_i64x2 = _mm_set_epi64x(comp_x4[1] as i64, comp_x4[0] as i64);
c01 = _mm_add_epi64(c01, _mm_mul_epi32(c_i64x2, coeffs_i64[r]));
let c_i64x2 = _mm_set_epi64x(comp_x4[3] as i64, comp_x4[2] as i64);
src/convolution/vertical_u8/avx2.rs
+9 -9
View File
@@ -55,9 +55,9 @@ unsafe fn vert_convolution_into_one_row_u8<T>(
let mut y: u32 = 0;
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
let components1 = T::components(s_row1);
let components2 = T::components(s_row2);
for src_rows in src_img.iter_2_rows(y_start, max_y) {
let components1 = T::components(src_rows[0]);
let components2 = T::components(src_rows[1]);
// Load two coefficients at once
let mmk = simd_utils::ptr_i16_to_256set1_epi32(coeffs, y as usize);
@@ -127,9 +127,9 @@ unsafe fn vert_convolution_into_one_row_u8<T>(
let mut sss1 = initial; // right row
let mut y: u32 = 0;
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
let components1 = T::components(s_row1);
let components2 = T::components(s_row2);
for src_rows in src_img.iter_2_rows(y_start, max_y) {
let components1 = T::components(src_rows[0]);
let components2 = T::components(src_rows[1]);
// Load two coefficients at once
let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
@@ -180,9 +180,9 @@ unsafe fn vert_convolution_into_one_row_u8<T>(
while x_in_bytes < src_width.saturating_sub(3) {
let mut sss = initial;
let mut y: u32 = 0;
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
let components1 = T::components(s_row1);
let components2 = T::components(s_row2);
for src_rows in src_img.iter_2_rows(y_start, max_y) {
let components1 = T::components(src_rows[0]);
let components2 = T::components(src_rows[1]);
// Load two coefficients at once
let two_coeffs = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
src/convolution/vertical_u8/sse4.rs
+9 -9
View File
@@ -52,9 +52,9 @@ pub(crate) unsafe fn vert_convolution_into_one_row_u8<T: PixelExt<Component = u8
let mut y: u32 = 0;
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
let components1 = T::components(s_row1);
let components2 = T::components(s_row2);
for src_rows in src_img.iter_2_rows(y_start, max_y) {
let components1 = T::components(src_rows[0]);
let components2 = T::components(src_rows[1]);
// Load two coefficients at once
let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
@@ -159,9 +159,9 @@ pub(crate) unsafe fn vert_convolution_into_one_row_u8<T: PixelExt<Component = u8
let mut sss1 = initial; // right row
let mut y: u32 = 0;
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
let components1 = T::components(s_row1);
let components2 = T::components(s_row2);
for src_rows in src_img.iter_2_rows(y_start, max_y) {
let components1 = T::components(src_rows[0]);
let components2 = T::components(src_rows[1]);
// Load two coefficients at once
let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
@@ -211,9 +211,9 @@ pub(crate) unsafe fn vert_convolution_into_one_row_u8<T: PixelExt<Component = u8
let mut sss = initial;
let mut y: u32 = 0;
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
let components1 = T::components(s_row1);
let components2 = T::components(s_row2);
for src_rows in src_img.iter_2_rows(y_start, max_y) {
let components1 = T::components(src_rows[0]);
let components2 = T::components(src_rows[1]);
// Load two coefficients at once
let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
src/image_view.rs
+7 -17
View File
@@ -5,16 +5,6 @@ use std::slice;
use crate::pixels::{GetCount, IntoPixelComponent, PixelComponent, PixelExt};
use crate::{CropBoxError, DifferentDimensionsError, ImageBufferError, ImageRowsError, PixelType};
pub(crate) type RowMut<'a, 'b, T> = &'a mut &'b mut [T];
pub(crate) type TwoRows<'a, T> = (&'a [T], &'a [T]);
pub(crate) type FourRows<'a, T> = (&'a [T], &'a [T], &'a [T], &'a [T]);
pub(crate) type FourRowsMut<'a, 'b, T> = (
&'a mut &'b mut [T],
&'a mut &'b mut [T],
&'a mut &'b mut [T],
&'a mut &'b mut [T],
);
/// Parameters of crop box that may be used with [`ImageView`]
/// and [`DynamicImageView`](crate::DynamicImageView)
#[derive(Debug, Clone, Copy)]
@@ -211,12 +201,12 @@ where
&'s self,
start_y: u32,
max_y: u32,
) -> impl Iterator<Item = FourRows<'a, P>> + 's {
) -> impl Iterator<Item = [&'a [P]; 4]> + 's {
let start_y = start_y as usize;
let max_y = max_y.min(self.height.get()) as usize;
let rows = self.rows.get(start_y..max_y).unwrap_or(&[]);
rows.chunks_exact(4).map(|rows| match *rows {
[r0, r1, r2, r3] => (r0, r1, r2, r3),
[r0, r1, r2, r3] => [r0, r1, r2, r3],
_ => unreachable!(),
})
}
@@ -226,12 +216,12 @@ where
&'s self,
start_y: u32,
max_y: u32,
) -> impl Iterator<Item = TwoRows<'a, P>> + 's {
) -> impl Iterator<Item = [&'a [P]; 2]> + 's {
let start_y = start_y as usize;
let max_y = max_y.min(self.height.get()) as usize;
let rows = self.rows.get(start_y..max_y).unwrap_or(&[]);
rows.chunks_exact(2).map(|rows| match *rows {
[r0, r1] => (r0, r1),
[r0, r1] => [r0, r1],
_ => unreachable!(),
})
}
@@ -370,15 +360,15 @@ where
#[inline(always)]
pub(crate) fn iter_4_rows_mut<'s>(
&'s mut self,
) -> impl Iterator<Item = FourRowsMut<'s, 'a, P>> {
) -> impl Iterator<Item = [&'s mut &'a mut [P]; 4]> {
self.rows.chunks_exact_mut(4).map(|rows| match rows {
[a, b, c, d] => (a, b, c, d),
[a, b, c, d] => [a, b, c, d],
_ => unreachable!(),
})
}
#[inline(always)]
pub(crate) fn get_row_mut<'s>(&'s mut self, y: u32) -> Option<RowMut<'s, 'a, P>> {
pub(crate) fn get_row_mut<'s>(&'s mut self, y: u32) -> Option<&'s mut &'a mut [P]> {
self.rows.get_mut(y as usize)
}
src/lib.rs
+1
View File
@@ -29,3 +29,4 @@ pub mod pixels;
mod resizer;
#[cfg(target_arch = "x86_64")]
mod simd_utils;
mod utils;
src/utils.rs
+18
View File
@@ -0,0 +1,18 @@
/// Pre-reading data from memory increases speed slightly for some operations
#[inline(always)]
pub(crate) fn foreach_with_pre_reading<D, I>(
mut iter: impl Iterator<Item = I>,
read_data: fn(src: I) -> D,
process_data: fn(data: D),
) {
let mut next_data: D;
if let Some(src) = iter.next() {
next_data = read_data(src);
for src in iter {
let data = next_data;
next_data = read_data(src);
process_data(data);
}
process_data(next_data);
}
}