mirror of
https://github.com/Cykooz/fast_image_resize.git
synced 2026-10-08 01:11:09 +00:00
Improved speed of MulDiv implementation for U16x4 images.
This commit is contained in:
Kirill Kuzminykh
+1
-1
@@ -2,7 +2,7 @@
|
||||
|
||||
### Crate
|
||||
|
||||
- Improved speed of `MulDiv` implementation for `U8x2` and `U8x4` images.
|
||||
- Improved speed of `MulDiv` implementation for `U8x2`, `U8x4` and `U16x4` images.
|
||||
- Excluded possibility of unnecessary operations during resize
|
||||
of cropped image by convolution algorithm.
|
||||
|
||||
|
||||
+122
-50
@@ -1,6 +1,7 @@
|
||||
use std::arch::x86_64::*;
|
||||
|
||||
use crate::pixels::U16x4;
|
||||
use crate::utils::foreach_with_pre_reading;
|
||||
use crate::{ImageView, ImageViewMut};
|
||||
|
||||
use super::sse4;
|
||||
@@ -20,15 +21,62 @@ pub(crate) unsafe fn multiply_alpha(
|
||||
|
||||
#[target_feature(enable = "avx2")]
|
||||
pub(crate) unsafe fn multiply_alpha_inplace(image: &mut ImageViewMut<U16x4>) {
|
||||
for dst_row in image.iter_rows_mut() {
|
||||
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
|
||||
multiply_alpha_row(src_row, dst_row);
|
||||
for row in image.iter_rows_mut() {
|
||||
multiply_alpha_row_inplace(row);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[target_feature(enable = "avx2")]
|
||||
pub(crate) unsafe fn multiply_alpha_row(src_row: &[U16x4], dst_row: &mut [U16x4]) {
|
||||
let src_chunks = src_row.chunks_exact(4);
|
||||
let src_remainder = src_chunks.remainder();
|
||||
let mut dst_chunks = dst_row.chunks_exact_mut(4);
|
||||
let src_dst = src_chunks.zip(&mut dst_chunks);
|
||||
foreach_with_pre_reading(
|
||||
src_dst,
|
||||
|(src, dst)| {
|
||||
let pixels = _mm256_loadu_si256(src.as_ptr() as *const __m256i);
|
||||
let dst_ptr = dst.as_mut_ptr() as *mut __m256i;
|
||||
(pixels, dst_ptr)
|
||||
},
|
||||
|(mut pixels, dst_ptr)| {
|
||||
pixels = multiply_alpha_4_pixels(pixels);
|
||||
_mm256_storeu_si256(dst_ptr, pixels);
|
||||
},
|
||||
);
|
||||
|
||||
if !src_remainder.is_empty() {
|
||||
let dst_reminder = dst_chunks.into_remainder();
|
||||
sse4::multiply_alpha_row(src_remainder, dst_reminder);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[target_feature(enable = "avx2")]
|
||||
pub(crate) unsafe fn multiply_alpha_row_inplace(row: &mut [U16x4]) {
|
||||
let mut chunks = row.chunks_exact_mut(4);
|
||||
foreach_with_pre_reading(
|
||||
&mut chunks,
|
||||
|chunk| {
|
||||
let pixels = _mm256_loadu_si256(chunk.as_ptr() as *const __m256i);
|
||||
let dst_ptr = chunk.as_mut_ptr() as *mut __m256i;
|
||||
(pixels, dst_ptr)
|
||||
},
|
||||
|(mut pixels, dst_ptr)| {
|
||||
pixels = multiply_alpha_4_pixels(pixels);
|
||||
_mm256_storeu_si256(dst_ptr, pixels);
|
||||
},
|
||||
);
|
||||
let reminder = chunks.into_remainder();
|
||||
if !reminder.is_empty() {
|
||||
sse4::multiply_alpha_row_inplace(reminder);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[target_feature(enable = "avx2")]
|
||||
unsafe fn multiply_alpha_4_pixels(pixels: __m256i) -> __m256i {
|
||||
let zero = _mm256_setzero_si256();
|
||||
let half = _mm256_set1_epi32(0x8000);
|
||||
|
||||
@@ -43,37 +91,22 @@ pub(crate) unsafe fn multiply_alpha_row(src_row: &[U16x4], dst_row: &mut [U16x4]
|
||||
_mm_set_epi8(15, 14, 15, 14, 15, 14, 15, 14, 7, 6, 7, 6, 7, 6, 7, 6),
|
||||
);
|
||||
|
||||
let src_chunks = src_row.chunks_exact(4);
|
||||
let src_remainder = src_chunks.remainder();
|
||||
let mut dst_chunks = dst_row.chunks_exact_mut(4);
|
||||
let factor_pixels = _mm256_shuffle_epi8(pixels, factor_mask);
|
||||
let factor_pixels = _mm256_or_si256(factor_pixels, max_alpha);
|
||||
|
||||
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
|
||||
let src_pixels = _mm256_loadu_si256(src.as_ptr() as *const __m256i);
|
||||
let src_i32_lo = _mm256_unpacklo_epi16(pixels, zero);
|
||||
let factors = _mm256_unpacklo_epi16(factor_pixels, zero);
|
||||
let src_i32_lo = _mm256_add_epi32(_mm256_mullo_epi32(src_i32_lo, factors), half);
|
||||
let dst_i32_lo = _mm256_add_epi32(src_i32_lo, _mm256_srli_epi32::<16>(src_i32_lo));
|
||||
let dst_i32_lo = _mm256_srli_epi32::<16>(dst_i32_lo);
|
||||
|
||||
let factor_pixels = _mm256_shuffle_epi8(src_pixels, factor_mask);
|
||||
let factor_pixels = _mm256_or_si256(factor_pixels, max_alpha);
|
||||
let src_i32_hi = _mm256_unpackhi_epi16(pixels, zero);
|
||||
let factors = _mm256_unpackhi_epi16(factor_pixels, zero);
|
||||
let src_i32_hi = _mm256_add_epi32(_mm256_mullo_epi32(src_i32_hi, factors), half);
|
||||
let dst_i32_hi = _mm256_add_epi32(src_i32_hi, _mm256_srli_epi32::<16>(src_i32_hi));
|
||||
let dst_i32_hi = _mm256_srli_epi32::<16>(dst_i32_hi);
|
||||
|
||||
let src_i32_lo = _mm256_unpacklo_epi16(src_pixels, zero);
|
||||
let factors = _mm256_unpacklo_epi16(factor_pixels, zero);
|
||||
let src_i32_lo = _mm256_add_epi32(_mm256_mullo_epi32(src_i32_lo, factors), half);
|
||||
let dst_i32_lo = _mm256_add_epi32(src_i32_lo, _mm256_srli_epi32::<16>(src_i32_lo));
|
||||
let dst_i32_lo = _mm256_srli_epi32::<16>(dst_i32_lo);
|
||||
|
||||
let src_i32_hi = _mm256_unpackhi_epi16(src_pixels, zero);
|
||||
let factors = _mm256_unpackhi_epi16(factor_pixels, zero);
|
||||
let src_i32_hi = _mm256_add_epi32(_mm256_mullo_epi32(src_i32_hi, factors), half);
|
||||
let dst_i32_hi = _mm256_add_epi32(src_i32_hi, _mm256_srli_epi32::<16>(src_i32_hi));
|
||||
let dst_i32_hi = _mm256_srli_epi32::<16>(dst_i32_hi);
|
||||
|
||||
let dst_pixels = _mm256_packus_epi32(dst_i32_lo, dst_i32_hi);
|
||||
|
||||
_mm256_storeu_si256(dst.as_mut_ptr() as *mut __m256i, dst_pixels);
|
||||
}
|
||||
|
||||
if !src_remainder.is_empty() {
|
||||
let dst_reminder = dst_chunks.into_remainder();
|
||||
sse4::multiply_alpha_row(src_remainder, dst_reminder);
|
||||
}
|
||||
_mm256_packus_epi32(dst_i32_lo, dst_i32_hi)
|
||||
}
|
||||
|
||||
// Divide
|
||||
@@ -93,9 +126,8 @@ pub(crate) unsafe fn divide_alpha(
|
||||
|
||||
#[target_feature(enable = "avx2")]
|
||||
pub(crate) unsafe fn divide_alpha_inplace(image: &mut ImageViewMut<U16x4>) {
|
||||
for dst_row in image.iter_rows_mut() {
|
||||
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
|
||||
divide_alpha_row(src_row, dst_row);
|
||||
for row in image.iter_rows_mut() {
|
||||
divide_alpha_row_inplace(row);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -104,10 +136,19 @@ pub(crate) unsafe fn divide_alpha_row(src_row: &[U16x4], dst_row: &mut [U16x4])
|
||||
let src_chunks = src_row.chunks_exact(4);
|
||||
let src_remainder = src_chunks.remainder();
|
||||
let mut dst_chunks = dst_row.chunks_exact_mut(4);
|
||||
|
||||
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
|
||||
divide_alpha_four_pixels(src.as_ptr(), dst.as_mut_ptr());
|
||||
}
|
||||
let src_dst = src_chunks.zip(&mut dst_chunks);
|
||||
foreach_with_pre_reading(
|
||||
src_dst,
|
||||
|(src, dst)| {
|
||||
let pixels = _mm256_loadu_si256(src.as_ptr() as *const __m256i);
|
||||
let dst_ptr = dst.as_mut_ptr() as *mut __m256i;
|
||||
(pixels, dst_ptr)
|
||||
},
|
||||
|(mut pixels, dst_ptr)| {
|
||||
pixels = divide_alpha_4_pixels(pixels);
|
||||
_mm256_storeu_si256(dst_ptr, pixels);
|
||||
},
|
||||
);
|
||||
|
||||
if !src_remainder.is_empty() {
|
||||
let dst_reminder = dst_chunks.into_remainder();
|
||||
@@ -118,7 +159,9 @@ pub(crate) unsafe fn divide_alpha_row(src_row: &[U16x4], dst_row: &mut [U16x4])
|
||||
.for_each(|(d, s)| *d = *s);
|
||||
|
||||
let mut dst_pixels = [U16x4::new([0, 0, 0, 0]); 4];
|
||||
divide_alpha_four_pixels(src_pixels.as_ptr(), dst_pixels.as_mut_ptr());
|
||||
let mut pixels = _mm256_loadu_si256(src_pixels.as_ptr() as *const __m256i);
|
||||
pixels = divide_alpha_4_pixels(pixels);
|
||||
_mm256_storeu_si256(dst_pixels.as_mut_ptr() as *mut __m256i, pixels);
|
||||
|
||||
dst_pixels
|
||||
.iter()
|
||||
@@ -127,9 +170,42 @@ pub(crate) unsafe fn divide_alpha_row(src_row: &[U16x4], dst_row: &mut [U16x4])
|
||||
}
|
||||
}
|
||||
|
||||
#[target_feature(enable = "avx2")]
|
||||
pub(crate) unsafe fn divide_alpha_row_inplace(row: &mut [U16x4]) {
|
||||
let mut chunks = row.chunks_exact_mut(4);
|
||||
foreach_with_pre_reading(
|
||||
&mut chunks,
|
||||
|chunk| {
|
||||
let pixels = _mm256_loadu_si256(chunk.as_ptr() as *const __m256i);
|
||||
let dst_ptr = chunk.as_mut_ptr() as *mut __m256i;
|
||||
(pixels, dst_ptr)
|
||||
},
|
||||
|(mut pixels, dst_ptr)| {
|
||||
pixels = divide_alpha_4_pixels(pixels);
|
||||
_mm256_storeu_si256(dst_ptr, pixels);
|
||||
},
|
||||
);
|
||||
|
||||
let reminder = chunks.into_remainder();
|
||||
if !reminder.is_empty() {
|
||||
let mut src_pixels = [U16x4::new([0, 0, 0, 0]); 4];
|
||||
src_pixels
|
||||
.iter_mut()
|
||||
.zip(reminder.iter())
|
||||
.for_each(|(d, s)| *d = *s);
|
||||
|
||||
let mut dst_pixels = [U16x4::new([0, 0, 0, 0]); 4];
|
||||
let mut pixels = _mm256_loadu_si256(src_pixels.as_ptr() as *const __m256i);
|
||||
pixels = divide_alpha_4_pixels(pixels);
|
||||
_mm256_storeu_si256(dst_pixels.as_mut_ptr() as *mut __m256i, pixels);
|
||||
|
||||
dst_pixels.iter().zip(reminder).for_each(|(s, d)| *d = *s);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[target_feature(enable = "avx2")]
|
||||
unsafe fn divide_alpha_four_pixels(src: *const U16x4, dst: *mut U16x4) {
|
||||
unsafe fn divide_alpha_4_pixels(pixels: __m256i) -> __m256i {
|
||||
let zero = _mm256_setzero_si256();
|
||||
let alpha_mask = _mm256_set1_epi64x(0xffff000000000000u64 as i64);
|
||||
let alpha_max = _mm256_set1_ps(65535.0);
|
||||
@@ -150,13 +226,11 @@ unsafe fn divide_alpha_four_pixels(src: *const U16x4, dst: *mut U16x4) {
|
||||
),
|
||||
);
|
||||
|
||||
let src_pixels = _mm256_loadu_si256(src as *const __m256i);
|
||||
let alpha0_f32x8 = _mm256_cvtepi32_ps(_mm256_shuffle_epi8(pixels, alpha32_sh0));
|
||||
let alpha1_f32x8 = _mm256_cvtepi32_ps(_mm256_shuffle_epi8(pixels, alpha32_sh1));
|
||||
|
||||
let alpha0_f32x8 = _mm256_cvtepi32_ps(_mm256_shuffle_epi8(src_pixels, alpha32_sh0));
|
||||
let alpha1_f32x8 = _mm256_cvtepi32_ps(_mm256_shuffle_epi8(src_pixels, alpha32_sh1));
|
||||
|
||||
let pix0_f32x8 = _mm256_cvtepi32_ps(_mm256_unpacklo_epi16(src_pixels, zero));
|
||||
let pix1_f32x8 = _mm256_cvtepi32_ps(_mm256_unpackhi_epi16(src_pixels, zero));
|
||||
let pix0_f32x8 = _mm256_cvtepi32_ps(_mm256_unpacklo_epi16(pixels, zero));
|
||||
let pix1_f32x8 = _mm256_cvtepi32_ps(_mm256_unpackhi_epi16(pixels, zero));
|
||||
|
||||
let scaled_pix0_f32x8 = _mm256_mul_ps(pix0_f32x8, alpha_max);
|
||||
let scaled_pix1_f32x8 = _mm256_mul_ps(pix1_f32x8, alpha_max);
|
||||
@@ -165,8 +239,6 @@ unsafe fn divide_alpha_four_pixels(src: *const U16x4, dst: *mut U16x4) {
|
||||
let divided_pix1_i32x8 = _mm256_cvtps_epi32(_mm256_div_ps(scaled_pix1_f32x8, alpha1_f32x8));
|
||||
|
||||
let two_pixels_i16x16 = _mm256_packus_epi32(divided_pix0_i32x8, divided_pix1_i32x8);
|
||||
let alpha = _mm256_and_si256(src_pixels, alpha_mask);
|
||||
let dst_pixels = _mm256_blendv_epi8(two_pixels_i16x16, alpha, alpha_mask);
|
||||
|
||||
_mm256_storeu_si256(dst as *mut __m256i, dst_pixels);
|
||||
let alpha = _mm256_and_si256(pixels, alpha_mask);
|
||||
_mm256_blendv_epi8(two_pixels_i16x16, alpha, alpha_mask)
|
||||
}
|
||||
|
||||
@@ -23,8 +23,8 @@ impl AlphaMulDiv for U16x4 {
|
||||
CpuExtensions::Avx2 => unsafe { avx2::multiply_alpha(src_image, dst_image) },
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Sse4_1 => unsafe { sse4::multiply_alpha(src_image, dst_image) },
|
||||
// Custom Neon implementation of alpha multiplying are little bit slower than code
|
||||
// generated by Rust compiler for native implementation.
|
||||
#[cfg(target_arch = "aarch64")]
|
||||
CpuExtensions::Neon => unsafe { neon::multiply_alpha(src_image, dst_image) },
|
||||
_ => native::multiply_alpha(src_image, dst_image),
|
||||
}
|
||||
}
|
||||
@@ -35,8 +35,8 @@ impl AlphaMulDiv for U16x4 {
|
||||
CpuExtensions::Avx2 => unsafe { avx2::multiply_alpha_inplace(image) },
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Sse4_1 => unsafe { sse4::multiply_alpha_inplace(image) },
|
||||
// Custom Neon implementation of alpha multiplying are little bit slower than code
|
||||
// generated by Rust compiler for native implementation.
|
||||
#[cfg(target_arch = "aarch64")]
|
||||
CpuExtensions::Neon => unsafe { neon::multiply_alpha_inplace(image) },
|
||||
_ => native::multiply_alpha_inplace(image),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,9 +12,8 @@ pub(crate) fn multiply_alpha(src_image: &ImageView<U16x4>, dst_image: &mut Image
|
||||
}
|
||||
|
||||
pub(crate) fn multiply_alpha_inplace(image: &mut ImageViewMut<U16x4>) {
|
||||
for dst_row in image.iter_rows_mut() {
|
||||
let src_row = unsafe { std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len()) };
|
||||
multiply_alpha_row(src_row, dst_row);
|
||||
for row in image.iter_rows_mut() {
|
||||
multiply_alpha_row_inplace(row);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,6 +31,20 @@ pub(crate) fn multiply_alpha_row(src_row: &[U16x4], dst_row: &mut [U16x4]) {
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub(crate) fn multiply_alpha_row_inplace(row: &mut [U16x4]) {
|
||||
for pixel in row {
|
||||
let components: [u16; 4] = pixel.0;
|
||||
let alpha = components[3];
|
||||
pixel.0 = [
|
||||
mul_div_65535(components[0], alpha),
|
||||
mul_div_65535(components[1], alpha),
|
||||
mul_div_65535(components[2], alpha),
|
||||
alpha,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
// Divide
|
||||
|
||||
#[inline]
|
||||
@@ -46,9 +59,8 @@ pub(crate) fn divide_alpha(src_image: &ImageView<U16x4>, dst_image: &mut ImageVi
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn divide_alpha_inplace(image: &mut ImageViewMut<U16x4>) {
|
||||
for dst_row in image.iter_rows_mut() {
|
||||
let src_row = unsafe { std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len()) };
|
||||
divide_alpha_row(src_row, dst_row);
|
||||
for row in image.iter_rows_mut() {
|
||||
divide_alpha_row_inplace(row);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,3 +81,18 @@ pub(crate) fn divide_alpha_row(src_row: &[U16x4], dst_row: &mut [U16x4]) {
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub(crate) fn divide_alpha_row_inplace(row: &mut [U16x4]) {
|
||||
row.iter_mut().for_each(|pixel| {
|
||||
let components: [u16; 4] = pixel.0;
|
||||
let alpha = components[3];
|
||||
let recip_alpha = RECIP_ALPHA16[alpha as usize];
|
||||
pixel.0 = [
|
||||
div_and_clip16(components[0], recip_alpha),
|
||||
div_and_clip16(components[1], recip_alpha),
|
||||
div_and_clip16(components[2], recip_alpha),
|
||||
alpha,
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
+116
-37
@@ -2,6 +2,7 @@ use std::arch::aarch64::*;
|
||||
|
||||
use crate::neon_utils;
|
||||
use crate::pixels::U16x4;
|
||||
use crate::utils::foreach_with_pre_reading;
|
||||
use crate::{ImageView, ImageViewMut};
|
||||
|
||||
use super::native;
|
||||
@@ -21,33 +22,38 @@ pub(crate) unsafe fn multiply_alpha(
|
||||
|
||||
#[target_feature(enable = "neon")]
|
||||
pub(crate) unsafe fn multiply_alpha_inplace(image: &mut ImageViewMut<U16x4>) {
|
||||
for dst_row in image.iter_rows_mut() {
|
||||
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
|
||||
multiply_alpha_row(src_row, dst_row);
|
||||
for row in image.iter_rows_mut() {
|
||||
multiply_alpha_row_inplace(row);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[target_feature(enable = "neon")]
|
||||
#[inline(always)]
|
||||
unsafe fn multiply_alpha_row(src_row: &[U16x4], dst_row: &mut [U16x4]) {
|
||||
let src_chunks = src_row.chunks_exact(8);
|
||||
let src_remainder = src_chunks.remainder();
|
||||
let mut dst_chunks = dst_row.chunks_exact_mut(8);
|
||||
|
||||
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
|
||||
let mut pixels = neon_utils::load_deintrel_u16x8x4(src, 0);
|
||||
pixels.0 = multiply_color_to_alpha_u16x8(pixels.0, pixels.3);
|
||||
pixels.1 = multiply_color_to_alpha_u16x8(pixels.1, pixels.3);
|
||||
pixels.2 = multiply_color_to_alpha_u16x8(pixels.2, pixels.3);
|
||||
let dst_ptr = dst.as_mut_ptr() as *mut u16;
|
||||
vst4q_u16(dst_ptr, pixels);
|
||||
}
|
||||
let src_dst = src_chunks.zip(&mut dst_chunks);
|
||||
foreach_with_pre_reading(
|
||||
src_dst,
|
||||
|(src, dst)| {
|
||||
let pixels = neon_utils::load_deintrel_u16x8x4(src, 0);
|
||||
let dst_ptr = dst.as_mut_ptr() as *mut u16;
|
||||
(pixels, dst_ptr)
|
||||
},
|
||||
|(mut pixels, dst_ptr)| {
|
||||
pixels.0 = multiply_color_to_alpha_u16x8(pixels.0, pixels.3);
|
||||
pixels.1 = multiply_color_to_alpha_u16x8(pixels.1, pixels.3);
|
||||
pixels.2 = multiply_color_to_alpha_u16x8(pixels.2, pixels.3);
|
||||
vst4q_u16(dst_ptr, pixels);
|
||||
},
|
||||
);
|
||||
|
||||
let src_chunks = src_remainder.chunks_exact(4);
|
||||
let src_remainder = src_chunks.remainder();
|
||||
let dst_reminder = dst_chunks.into_remainder();
|
||||
let mut dst_chunks = dst_reminder.chunks_exact_mut(4);
|
||||
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
|
||||
let mut src_dst = src_chunks.zip(&mut dst_chunks);
|
||||
if let Some((src, dst)) = src_dst.next() {
|
||||
let mut pixels = neon_utils::load_deintrel_u16x4x4(src, 0);
|
||||
pixels.0 = multiply_color_to_alpha_u16x4(pixels.0, pixels.3);
|
||||
pixels.1 = multiply_color_to_alpha_u16x4(pixels.1, pixels.3);
|
||||
@@ -62,8 +68,42 @@ unsafe fn multiply_alpha_row(src_row: &[U16x4], dst_row: &mut [U16x4]) {
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[target_feature(enable = "neon")]
|
||||
#[inline(always)]
|
||||
unsafe fn multiply_alpha_row_inplace(row: &mut [U16x4]) {
|
||||
let mut chunks = row.chunks_exact_mut(8);
|
||||
foreach_with_pre_reading(
|
||||
&mut chunks,
|
||||
|chunk| {
|
||||
let pixels = neon_utils::load_deintrel_u16x8x4(chunk, 0);
|
||||
let dst_ptr = chunk.as_mut_ptr() as *mut u16;
|
||||
(pixels, dst_ptr)
|
||||
},
|
||||
|(mut pixels, dst_ptr)| {
|
||||
pixels.0 = multiply_color_to_alpha_u16x8(pixels.0, pixels.3);
|
||||
pixels.1 = multiply_color_to_alpha_u16x8(pixels.1, pixels.3);
|
||||
pixels.2 = multiply_color_to_alpha_u16x8(pixels.2, pixels.3);
|
||||
vst4q_u16(dst_ptr, pixels);
|
||||
},
|
||||
);
|
||||
|
||||
let reminder = chunks.into_remainder();
|
||||
let mut chunks = reminder.chunks_exact_mut(4);
|
||||
if let Some(chunk) = chunks.next() {
|
||||
let mut pixels = neon_utils::load_deintrel_u16x4x4(chunk, 0);
|
||||
pixels.0 = multiply_color_to_alpha_u16x4(pixels.0, pixels.3);
|
||||
pixels.1 = multiply_color_to_alpha_u16x4(pixels.1, pixels.3);
|
||||
pixels.2 = multiply_color_to_alpha_u16x4(pixels.2, pixels.3);
|
||||
let dst_ptr = chunk.as_mut_ptr() as *mut u16;
|
||||
vst4_u16(dst_ptr, pixels);
|
||||
}
|
||||
|
||||
let reminder = chunks.into_remainder();
|
||||
if !reminder.is_empty() {
|
||||
native::multiply_alpha_row_inplace(reminder);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
unsafe fn multiply_color_to_alpha_u16x8(color: uint16x8_t, alpha: uint16x8_t) -> uint16x8_t {
|
||||
let rounder = vdupq_n_u32(0x8000);
|
||||
let color_lo_u32 = vmlal_u16(rounder, vget_low_u16(color), vget_low_u16(alpha));
|
||||
@@ -73,8 +113,7 @@ unsafe fn multiply_color_to_alpha_u16x8(color: uint16x8_t, alpha: uint16x8_t) ->
|
||||
vcombine_u16(color_lo_u32, color_hi_u32)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[target_feature(enable = "neon")]
|
||||
#[inline(always)]
|
||||
unsafe fn multiply_color_to_alpha_u16x4(color: uint16x4_t, alpha: uint16x4_t) -> uint16x4_t {
|
||||
let rounder = vdupq_n_u32(0x8000);
|
||||
let color_u32 = vmlal_u16(rounder, color, alpha);
|
||||
@@ -98,21 +137,29 @@ pub(crate) unsafe fn divide_alpha(
|
||||
|
||||
#[target_feature(enable = "neon")]
|
||||
pub(crate) unsafe fn divide_alpha_inplace(image: &mut ImageViewMut<U16x4>) {
|
||||
for dst_row in image.iter_rows_mut() {
|
||||
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
|
||||
divide_alpha_row(src_row, dst_row);
|
||||
for row in image.iter_rows_mut() {
|
||||
divide_alpha_row_inplace(row);
|
||||
}
|
||||
}
|
||||
|
||||
#[target_feature(enable = "neon")]
|
||||
#[inline(always)]
|
||||
pub(crate) unsafe fn divide_alpha_row(src_row: &[U16x4], dst_row: &mut [U16x4]) {
|
||||
let src_chunks = src_row.chunks_exact(8);
|
||||
let src_remainder = src_chunks.remainder();
|
||||
let mut dst_chunks = dst_row.chunks_exact_mut(8);
|
||||
|
||||
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
|
||||
divide_alpha_8_pixels(src, dst);
|
||||
}
|
||||
let src_dst = src_chunks.zip(&mut dst_chunks);
|
||||
foreach_with_pre_reading(
|
||||
src_dst,
|
||||
|(src, dst)| {
|
||||
let pixels = neon_utils::load_deintrel_u16x8x4(src, 0);
|
||||
let dst_ptr = dst.as_mut_ptr() as *mut u16;
|
||||
(pixels, dst_ptr)
|
||||
},
|
||||
|(mut pixels, dst_ptr)| {
|
||||
pixels = divide_alpha_8_pixels(pixels);
|
||||
vst4q_u16(dst_ptr, pixels);
|
||||
},
|
||||
);
|
||||
|
||||
if !src_remainder.is_empty() {
|
||||
let dst_reminder = dst_chunks.into_remainder();
|
||||
@@ -123,7 +170,10 @@ pub(crate) unsafe fn divide_alpha_row(src_row: &[U16x4], dst_row: &mut [U16x4])
|
||||
.for_each(|(d, s)| *d = *s);
|
||||
|
||||
let mut dst_pixels = [U16x4::new([0; 4]); 8];
|
||||
divide_alpha_8_pixels(src_pixels.as_slice(), dst_pixels.as_mut_slice());
|
||||
let mut pixels = neon_utils::load_deintrel_u16x8x4(&src_pixels, 0);
|
||||
pixels = divide_alpha_8_pixels(pixels);
|
||||
let dst_ptr = dst_pixels.as_mut_ptr() as *mut u16;
|
||||
vst4q_u16(dst_ptr, pixels);
|
||||
|
||||
dst_pixels
|
||||
.iter()
|
||||
@@ -132,12 +182,44 @@ pub(crate) unsafe fn divide_alpha_row(src_row: &[U16x4], dst_row: &mut [U16x4])
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[target_feature(enable = "neon")]
|
||||
unsafe fn divide_alpha_8_pixels(src: &[U16x4], dst: &mut [U16x4]) {
|
||||
#[inline(always)]
|
||||
pub(crate) unsafe fn divide_alpha_row_inplace(row: &mut [U16x4]) {
|
||||
let mut chunks = row.chunks_exact_mut(8);
|
||||
foreach_with_pre_reading(
|
||||
&mut chunks,
|
||||
|chunk| {
|
||||
let pixels = neon_utils::load_deintrel_u16x8x4(chunk, 0);
|
||||
let dst_ptr = chunk.as_mut_ptr() as *mut u16;
|
||||
(pixels, dst_ptr)
|
||||
},
|
||||
|(mut pixels, dst_ptr)| {
|
||||
pixels = divide_alpha_8_pixels(pixels);
|
||||
vst4q_u16(dst_ptr, pixels);
|
||||
},
|
||||
);
|
||||
|
||||
let reminder = chunks.into_remainder();
|
||||
if !reminder.is_empty() {
|
||||
let mut src_pixels = [U16x4::new([0; 4]); 8];
|
||||
src_pixels
|
||||
.iter_mut()
|
||||
.zip(reminder.iter())
|
||||
.for_each(|(d, s)| *d = *s);
|
||||
|
||||
let mut dst_pixels = [U16x4::new([0; 4]); 8];
|
||||
let mut pixels = neon_utils::load_deintrel_u16x8x4(&src_pixels, 0);
|
||||
pixels = divide_alpha_8_pixels(pixels);
|
||||
let dst_ptr = dst_pixels.as_mut_ptr() as *mut u16;
|
||||
vst4q_u16(dst_ptr, pixels);
|
||||
|
||||
dst_pixels.iter().zip(reminder).for_each(|(s, d)| *d = *s);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
unsafe fn divide_alpha_8_pixels(mut pixels: uint16x8x4_t) -> uint16x8x4_t {
|
||||
let zero = vdupq_n_u16(0);
|
||||
let alpha_scale = vdupq_n_f32(65535.0);
|
||||
let mut pixels = neon_utils::load_deintrel_u16x8x4(src, 0);
|
||||
let nonzero_alpha_mask = vmvnq_u16(vceqzq_u16(pixels.3));
|
||||
|
||||
// Low
|
||||
@@ -156,13 +238,10 @@ unsafe fn divide_alpha_8_pixels(src: &[U16x4], dst: &mut [U16x4]) {
|
||||
pixels.1 = vandq_u16(pixels.1, nonzero_alpha_mask);
|
||||
pixels.2 = mul_color_recip_alpha(pixels.2, recip_alpha_lo_f32, recip_alpha_hi_f32, zero);
|
||||
pixels.2 = vandq_u16(pixels.2, nonzero_alpha_mask);
|
||||
|
||||
let dst_ptr = dst.as_mut_ptr() as *mut u16;
|
||||
vst4q_u16(dst_ptr, pixels);
|
||||
pixels
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[target_feature(enable = "neon")]
|
||||
#[inline(always)]
|
||||
unsafe fn mul_color_recip_alpha(
|
||||
color: uint16x8_t,
|
||||
recip_alpha_lo: float32x4_t,
|
||||
|
||||
+119
-51
@@ -1,6 +1,7 @@
|
||||
use std::arch::x86_64::*;
|
||||
|
||||
use crate::pixels::U16x4;
|
||||
use crate::utils::foreach_with_pre_reading;
|
||||
use crate::{ImageView, ImageViewMut};
|
||||
|
||||
use super::native;
|
||||
@@ -20,18 +21,65 @@ pub(crate) unsafe fn multiply_alpha(
|
||||
|
||||
#[target_feature(enable = "sse4.1")]
|
||||
pub(crate) unsafe fn multiply_alpha_inplace(image: &mut ImageViewMut<U16x4>) {
|
||||
for dst_row in image.iter_rows_mut() {
|
||||
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
|
||||
multiply_alpha_row(src_row, dst_row);
|
||||
for row in image.iter_rows_mut() {
|
||||
multiply_alpha_row_inplace(row);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[target_feature(enable = "sse4.1")]
|
||||
pub(crate) unsafe fn multiply_alpha_row(src_row: &[U16x4], dst_row: &mut [U16x4]) {
|
||||
let src_chunks = src_row.chunks_exact(2);
|
||||
let src_remainder = src_chunks.remainder();
|
||||
let mut dst_chunks = dst_row.chunks_exact_mut(2);
|
||||
let src_dst = src_chunks.zip(&mut dst_chunks);
|
||||
foreach_with_pre_reading(
|
||||
src_dst,
|
||||
|(src, dst)| {
|
||||
let pixels = _mm_loadu_si128(src.as_ptr() as *const __m128i);
|
||||
let dst_ptr = dst.as_mut_ptr() as *mut __m128i;
|
||||
(pixels, dst_ptr)
|
||||
},
|
||||
|(mut pixels, dst_ptr)| {
|
||||
pixels = multiply_alpha_2_pixels(pixels);
|
||||
_mm_storeu_si128(dst_ptr, pixels);
|
||||
},
|
||||
);
|
||||
|
||||
if !src_remainder.is_empty() {
|
||||
let dst_reminder = dst_chunks.into_remainder();
|
||||
native::multiply_alpha_row(src_remainder, dst_reminder);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[target_feature(enable = "sse4.1")]
|
||||
pub(crate) unsafe fn multiply_alpha_row_inplace(row: &mut [U16x4]) {
|
||||
let mut chunks = row.chunks_exact_mut(2);
|
||||
foreach_with_pre_reading(
|
||||
&mut chunks,
|
||||
|chunk| {
|
||||
let pixels = _mm_loadu_si128(chunk.as_ptr() as *const __m128i);
|
||||
let dst_ptr = chunk.as_mut_ptr() as *mut __m128i;
|
||||
(pixels, dst_ptr)
|
||||
},
|
||||
|(mut pixels, dst_ptr)| {
|
||||
pixels = multiply_alpha_2_pixels(pixels);
|
||||
_mm_storeu_si128(dst_ptr, pixels);
|
||||
},
|
||||
);
|
||||
|
||||
let remainder = chunks.into_remainder();
|
||||
if !remainder.is_empty() {
|
||||
native::multiply_alpha_row_inplace(remainder);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[target_feature(enable = "sse4.1")]
|
||||
unsafe fn multiply_alpha_2_pixels(pixels: __m128i) -> __m128i {
|
||||
let zero = _mm_setzero_si128();
|
||||
let half = _mm_set1_epi32(0x8000);
|
||||
|
||||
const MAX_A: i64 = 0xffff000000000000u64 as i64;
|
||||
let max_alpha = _mm_set1_epi64x(MAX_A);
|
||||
/*
|
||||
@@ -40,37 +88,22 @@ pub(crate) unsafe fn multiply_alpha_row(src_row: &[U16x4], dst_row: &mut [U16x4]
|
||||
*/
|
||||
let factor_mask = _mm_set_epi8(15, 14, 15, 14, 15, 14, 15, 14, 7, 6, 7, 6, 7, 6, 7, 6);
|
||||
|
||||
let src_chunks = src_row.chunks_exact(2);
|
||||
let src_remainder = src_chunks.remainder();
|
||||
let mut dst_chunks = dst_row.chunks_exact_mut(2);
|
||||
let factor_pixels = _mm_shuffle_epi8(pixels, factor_mask);
|
||||
let factor_pixels = _mm_or_si128(factor_pixels, max_alpha);
|
||||
|
||||
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
|
||||
let src_pixels = _mm_loadu_si128(src.as_ptr() as *const __m128i);
|
||||
let src_i32_lo = _mm_unpacklo_epi16(pixels, zero);
|
||||
let factors = _mm_unpacklo_epi16(factor_pixels, zero);
|
||||
let src_i32_lo = _mm_add_epi32(_mm_mullo_epi32(src_i32_lo, factors), half);
|
||||
let dst_i32_lo = _mm_add_epi32(src_i32_lo, _mm_srli_epi32::<16>(src_i32_lo));
|
||||
let dst_i32_lo = _mm_srli_epi32::<16>(dst_i32_lo);
|
||||
|
||||
let factor_pixels = _mm_shuffle_epi8(src_pixels, factor_mask);
|
||||
let factor_pixels = _mm_or_si128(factor_pixels, max_alpha);
|
||||
let src_i32_hi = _mm_unpackhi_epi16(pixels, zero);
|
||||
let factors = _mm_unpackhi_epi16(factor_pixels, zero);
|
||||
let src_i32_hi = _mm_add_epi32(_mm_mullo_epi32(src_i32_hi, factors), half);
|
||||
let dst_i32_hi = _mm_add_epi32(src_i32_hi, _mm_srli_epi32::<16>(src_i32_hi));
|
||||
let dst_i32_hi = _mm_srli_epi32::<16>(dst_i32_hi);
|
||||
|
||||
let src_i32_lo = _mm_unpacklo_epi16(src_pixels, zero);
|
||||
let factors = _mm_unpacklo_epi16(factor_pixels, zero);
|
||||
let src_i32_lo = _mm_add_epi32(_mm_mullo_epi32(src_i32_lo, factors), half);
|
||||
let dst_i32_lo = _mm_add_epi32(src_i32_lo, _mm_srli_epi32::<16>(src_i32_lo));
|
||||
let dst_i32_lo = _mm_srli_epi32::<16>(dst_i32_lo);
|
||||
|
||||
let src_i32_hi = _mm_unpackhi_epi16(src_pixels, zero);
|
||||
let factors = _mm_unpackhi_epi16(factor_pixels, zero);
|
||||
let src_i32_hi = _mm_add_epi32(_mm_mullo_epi32(src_i32_hi, factors), half);
|
||||
let dst_i32_hi = _mm_add_epi32(src_i32_hi, _mm_srli_epi32::<16>(src_i32_hi));
|
||||
let dst_i32_hi = _mm_srli_epi32::<16>(dst_i32_hi);
|
||||
|
||||
let dst_pixels = _mm_packus_epi32(dst_i32_lo, dst_i32_hi);
|
||||
|
||||
_mm_storeu_si128(dst.as_mut_ptr() as *mut __m128i, dst_pixels);
|
||||
}
|
||||
|
||||
if !src_remainder.is_empty() {
|
||||
let dst_reminder = dst_chunks.into_remainder();
|
||||
native::multiply_alpha_row(src_remainder, dst_reminder);
|
||||
}
|
||||
_mm_packus_epi32(dst_i32_lo, dst_i32_hi)
|
||||
}
|
||||
|
||||
// Divide
|
||||
@@ -90,9 +123,8 @@ pub(crate) unsafe fn divide_alpha(
|
||||
|
||||
#[target_feature(enable = "sse4.1")]
|
||||
pub(crate) unsafe fn divide_alpha_inplace(image: &mut ImageViewMut<U16x4>) {
|
||||
for dst_row in image.iter_rows_mut() {
|
||||
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
|
||||
divide_alpha_row(src_row, dst_row);
|
||||
for row in image.iter_rows_mut() {
|
||||
divide_alpha_row_inplace(row);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,15 +133,27 @@ pub(crate) unsafe fn divide_alpha_row(src_row: &[U16x4], dst_row: &mut [U16x4])
|
||||
let src_chunks = src_row.chunks_exact(2);
|
||||
let src_remainder = src_chunks.remainder();
|
||||
let mut dst_chunks = dst_row.chunks_exact_mut(2);
|
||||
|
||||
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
|
||||
divide_alpha_two_pixels(src.as_ptr(), dst.as_mut_ptr());
|
||||
}
|
||||
let src_dst = src_chunks.zip(&mut dst_chunks);
|
||||
foreach_with_pre_reading(
|
||||
src_dst,
|
||||
|(src, dst)| {
|
||||
let pixels = _mm_loadu_si128(src.as_ptr() as *const __m128i);
|
||||
let dst_ptr = dst.as_mut_ptr() as *mut __m128i;
|
||||
(pixels, dst_ptr)
|
||||
},
|
||||
|(mut pixels, dst_ptr)| {
|
||||
pixels = divide_alpha_2_pixels(pixels);
|
||||
_mm_storeu_si128(dst_ptr, pixels);
|
||||
},
|
||||
);
|
||||
|
||||
if let Some(src) = src_remainder.first() {
|
||||
let src_pixels = [*src, U16x4::new([0, 0, 0, 0])];
|
||||
let mut dst_pixels = [U16x4::new([0, 0, 0, 0]); 2];
|
||||
divide_alpha_two_pixels(src_pixels.as_ptr(), dst_pixels.as_mut_ptr());
|
||||
|
||||
let mut pixels = _mm_loadu_si128(src_pixels.as_ptr() as *const __m128i);
|
||||
pixels = divide_alpha_2_pixels(pixels);
|
||||
_mm_storeu_si128(dst_pixels.as_mut_ptr() as *mut __m128i, pixels);
|
||||
|
||||
let dst_reminder = dst_chunks.into_remainder();
|
||||
if let Some(dst) = dst_reminder.get_mut(0) {
|
||||
@@ -118,9 +162,37 @@ pub(crate) unsafe fn divide_alpha_row(src_row: &[U16x4], dst_row: &mut [U16x4])
|
||||
}
|
||||
}
|
||||
|
||||
#[target_feature(enable = "sse4.1")]
|
||||
pub(crate) unsafe fn divide_alpha_row_inplace(row: &mut [U16x4]) {
|
||||
let mut chunks = row.chunks_exact_mut(2);
|
||||
foreach_with_pre_reading(
|
||||
&mut chunks,
|
||||
|chunk| {
|
||||
let pixels = _mm_loadu_si128(chunk.as_ptr() as *const __m128i);
|
||||
let dst_ptr = chunk.as_mut_ptr() as *mut __m128i;
|
||||
(pixels, dst_ptr)
|
||||
},
|
||||
|(mut pixels, dst_ptr)| {
|
||||
pixels = divide_alpha_2_pixels(pixels);
|
||||
_mm_storeu_si128(dst_ptr, pixels);
|
||||
},
|
||||
);
|
||||
|
||||
let reminder = chunks.into_remainder();
|
||||
if let Some(pixel) = reminder.first_mut() {
|
||||
let src_pixels = [*pixel, U16x4::new([0, 0, 0, 0])];
|
||||
let mut dst_pixels = [U16x4::new([0, 0, 0, 0]); 2];
|
||||
|
||||
let mut pixels = _mm_loadu_si128(src_pixels.as_ptr() as *const __m128i);
|
||||
pixels = divide_alpha_2_pixels(pixels);
|
||||
_mm_storeu_si128(dst_pixels.as_mut_ptr() as *mut __m128i, pixels);
|
||||
*pixel = dst_pixels[0];
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[target_feature(enable = "sse4.1")]
|
||||
unsafe fn divide_alpha_two_pixels(src: *const U16x4, dst: *mut U16x4) {
|
||||
unsafe fn divide_alpha_2_pixels(pixels: __m128i) -> __m128i {
|
||||
let zero = _mm_setzero_si128();
|
||||
let alpha_mask = _mm_set1_epi64x(0xffff000000000000u64 as i64);
|
||||
let alpha_max = _mm_set1_ps(65535.0);
|
||||
@@ -133,13 +205,11 @@ unsafe fn divide_alpha_two_pixels(src: *const U16x4, dst: *mut U16x4) {
|
||||
-1, -1, 15, 14, -1, -1, 15, 14, -1, -1, 15, 14, -1, -1, 15, 14,
|
||||
);
|
||||
|
||||
let src_pixels = _mm_loadu_si128(src as *const __m128i);
|
||||
let alpha0_f32x4 = _mm_cvtepi32_ps(_mm_shuffle_epi8(pixels, alpha32_sh0));
|
||||
let alpha1_f32x4 = _mm_cvtepi32_ps(_mm_shuffle_epi8(pixels, alpha32_sh1));
|
||||
|
||||
let alpha0_f32x4 = _mm_cvtepi32_ps(_mm_shuffle_epi8(src_pixels, alpha32_sh0));
|
||||
let alpha1_f32x4 = _mm_cvtepi32_ps(_mm_shuffle_epi8(src_pixels, alpha32_sh1));
|
||||
|
||||
let pix0_f32x4 = _mm_cvtepi32_ps(_mm_unpacklo_epi16(src_pixels, zero));
|
||||
let pix1_f32x4 = _mm_cvtepi32_ps(_mm_unpackhi_epi16(src_pixels, zero));
|
||||
let pix0_f32x4 = _mm_cvtepi32_ps(_mm_unpacklo_epi16(pixels, zero));
|
||||
let pix1_f32x4 = _mm_cvtepi32_ps(_mm_unpackhi_epi16(pixels, zero));
|
||||
|
||||
let scaled_pix0_f32x4 = _mm_mul_ps(pix0_f32x4, alpha_max);
|
||||
let scaled_pix1_f32x4 = _mm_mul_ps(pix1_f32x4, alpha_max);
|
||||
@@ -148,8 +218,6 @@ unsafe fn divide_alpha_two_pixels(src: *const U16x4, dst: *mut U16x4) {
|
||||
let divided_pix1_i32x4 = _mm_cvtps_epi32(_mm_div_ps(scaled_pix1_f32x4, alpha1_f32x4));
|
||||
|
||||
let two_pixels_i16x8 = _mm_packus_epi32(divided_pix0_i32x4, divided_pix1_i32x4);
|
||||
let alpha = _mm_and_si128(src_pixels, alpha_mask);
|
||||
let dst_pixels = _mm_blendv_epi8(two_pixels_i16x8, alpha, alpha_mask);
|
||||
|
||||
_mm_storeu_si128(dst as *mut __m128i, dst_pixels);
|
||||
let alpha = _mm_and_si128(pixels, alpha_mask);
|
||||
_mm_blendv_epi8(two_pixels_i16x8, alpha, alpha_mask)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user