external/fast_image_resize
1
0
Fork 0
mirror of https://github.com/Cykooz/fast_image_resize.git synced 2026-10-08 01:11:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improved speed of MulDiv implementation for U8x2 images.

Browse Source
This commit is contained in:
Kirill Kuzminykh
2022-12-10 16:57:49 +04:00
parent b6e36cbd86
commit d59ff2d69f
5 changed files with 434 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+1 -1
View File
@@ -2,7 +2,7 @@
### Crate
- Improved speed of `MulDiv` implementation for `U8x4` images.
- Improved speed of `MulDiv` implementation for `U8x2` and `U8x4` images.
- Excluded possibility of unnecessary operations during resize
of cropped image by convolution algorithm.
src/alpha/u8x2/avx2.rs
+110 -55
View File
@@ -2,6 +2,7 @@ use std::arch::x86_64::*;
use crate::pixels::U8x2;
use crate::simd_utils;
use crate::utils::foreach_with_pre_reading;
use crate::{ImageView, ImageViewMut};
use super::sse4;
@@ -11,30 +12,73 @@ pub(crate) unsafe fn multiply_alpha(
src_image: &ImageView<U8x2>,
dst_image: &mut ImageViewMut<U8x2>,
) {
let width = src_image.width().get() as usize;
let src_rows = src_image.iter_rows(0);
let dst_rows = dst_image.iter_rows_mut();
for (src_row, dst_row) in src_rows.zip(dst_rows) {
multiply_alpha_row(src_row, dst_row, width);
multiply_alpha_row(src_row, dst_row);
}
}
#[target_feature(enable = "avx2")]
pub(crate) unsafe fn multiply_alpha_inplace(image: &mut ImageViewMut<U8x2>) {
let width = image.width().get() as usize;
for dst_row in image.iter_rows_mut() {
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
multiply_alpha_row(src_row, dst_row, width);
for row in image.iter_rows_mut() {
multiply_alpha_row_inplace(row);
}
}
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn multiply_alpha_row(src_row: &[U8x2], dst_row: &mut [U8x2], width: usize) {
unsafe fn multiply_alpha_row(src_row: &[U8x2], dst_row: &mut [U8x2]) {
let src_chunks = src_row.chunks_exact(16);
let src_tail = src_chunks.remainder();
let mut dst_chunks = dst_row.chunks_exact_mut(16);
let src_dst = src_chunks.zip(&mut dst_chunks);
foreach_with_pre_reading(
src_dst,
|(src, dst)| {
let pixels = simd_utils::loadu_si256(src, 0);
let dst_ptr = dst.as_mut_ptr() as *mut __m256i;
(pixels, dst_ptr)
},
|(mut pixels, dst_ptr)| {
pixels = multiply_alpha_16_pixels(pixels);
_mm256_storeu_si256(dst_ptr, pixels);
},
);
if !src_tail.is_empty() {
let dst_tail = dst_chunks.into_remainder();
sse4::multiply_alpha_row(src_tail, dst_tail);
}
}
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn multiply_alpha_row_inplace(row: &mut [U8x2]) {
let mut chunks = row.chunks_exact_mut(16);
foreach_with_pre_reading(
&mut chunks,
|chunk| {
let pixels = simd_utils::loadu_si256(chunk, 0);
let dst_ptr = chunk.as_mut_ptr() as *mut __m256i;
(pixels, dst_ptr)
},
|(mut pixels, dst_ptr)| {
pixels = multiply_alpha_16_pixels(pixels);
_mm256_storeu_si256(dst_ptr, pixels);
},
);
let tail = chunks.into_remainder();
if !tail.is_empty() {
sse4::multiply_alpha_row_inplace(tail);
}
}
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn multiply_alpha_16_pixels(pixels: __m256i) -> __m256i {
let zero = _mm256_setzero_si256();
let half = _mm256_set1_epi16(128);
const MAX_A: i16 = 0xff00u16 as i16;
let max_alpha = _mm256_set1_epi16(MAX_A);
/*
@@ -42,41 +86,27 @@ unsafe fn multiply_alpha_row(src_row: &[U8x2], dst_row: &mut [U8x2], width: usiz
|00 01| |02 03| |04 05| |06 07| |08 09| |10 11| |12 13| |14 15|
*/
#[rustfmt::skip]
let factor_mask = _mm256_set_epi8(
let factor_mask = _mm256_set_epi8(
15, 15, 13, 13, 11, 11, 9, 9, 7, 7, 5, 5, 3, 3, 1, 1,
15, 15, 13, 13, 11, 11, 9, 9, 7, 7, 5, 5, 3, 3, 1, 1
);
let mut x: usize = 0;
while x < width.saturating_sub(15) {
let src_pixels = simd_utils::loadu_si256(src_row, x);
let factor_pixels = _mm256_shuffle_epi8(pixels, factor_mask);
let factor_pixels = _mm256_or_si256(factor_pixels, max_alpha);
let factor_pixels = _mm256_shuffle_epi8(src_pixels, factor_mask);
let factor_pixels = _mm256_or_si256(factor_pixels, max_alpha);
let src_i16_lo = _mm256_unpacklo_epi8(pixels, zero);
let factors = _mm256_unpacklo_epi8(factor_pixels, zero);
let src_i16_lo = _mm256_add_epi16(_mm256_mullo_epi16(src_i16_lo, factors), half);
let dst_i16_lo = _mm256_add_epi16(src_i16_lo, _mm256_srli_epi16::<8>(src_i16_lo));
let dst_i16_lo = _mm256_srli_epi16::<8>(dst_i16_lo);
let src_i16_lo = _mm256_unpacklo_epi8(src_pixels, zero);
let factors = _mm256_unpacklo_epi8(factor_pixels, zero);
let src_i16_lo = _mm256_add_epi16(_mm256_mullo_epi16(src_i16_lo, factors), half);
let dst_i16_lo = _mm256_add_epi16(src_i16_lo, _mm256_srli_epi16::<8>(src_i16_lo));
let dst_i16_lo = _mm256_srli_epi16::<8>(dst_i16_lo);
let src_i16_hi = _mm256_unpackhi_epi8(pixels, zero);
let factors = _mm256_unpackhi_epi8(factor_pixels, zero);
let src_i16_hi = _mm256_add_epi16(_mm256_mullo_epi16(src_i16_hi, factors), half);
let dst_i16_hi = _mm256_add_epi16(src_i16_hi, _mm256_srli_epi16::<8>(src_i16_hi));
let dst_i16_hi = _mm256_srli_epi16::<8>(dst_i16_hi);
let src_i16_hi = _mm256_unpackhi_epi8(src_pixels, zero);
let factors = _mm256_unpackhi_epi8(factor_pixels, zero);
let src_i16_hi = _mm256_add_epi16(_mm256_mullo_epi16(src_i16_hi, factors), half);
let dst_i16_hi = _mm256_add_epi16(src_i16_hi, _mm256_srli_epi16::<8>(src_i16_hi));
let dst_i16_hi = _mm256_srli_epi16::<8>(dst_i16_hi);
let dst_pixels = _mm256_packus_epi16(dst_i16_lo, dst_i16_hi);
let dst_ptr = dst_row.get_unchecked_mut(x..).as_mut_ptr() as *mut __m256i;
_mm256_storeu_si256(dst_ptr, dst_pixels);
x += 16;
}
let src_tail = &src_row[x..];
let dst_tail = &mut dst_row[x..];
sse4::multiply_alpha_row(src_tail, dst_tail);
_mm256_packus_epi16(dst_i16_lo, dst_i16_hi)
}
// Divide
@@ -93,9 +123,8 @@ pub(crate) unsafe fn divide_alpha(src_image: &ImageView<U8x2>, dst_image: &mut I
#[target_feature(enable = "avx2")]
pub(crate) unsafe fn divide_alpha_inplace(image: &mut ImageViewMut<U8x2>) {
for dst_row in image.iter_rows_mut() {
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
divide_alpha_row(src_row, dst_row);
for row in image.iter_rows_mut() {
divide_alpha_row_inplace(row);
}
}
@@ -104,47 +133,73 @@ unsafe fn divide_alpha_row(src_row: &[U8x2], dst_row: &mut [U8x2]) {
let src_chunks = src_row.chunks_exact(16);
let src_remainder = src_chunks.remainder();
let mut dst_chunks = dst_row.chunks_exact_mut(16);
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
divide_alpha_sixteen_pixels(src.as_ptr(), dst.as_mut_ptr());
}
let src_dst = src_chunks.zip(&mut dst_chunks);
foreach_with_pre_reading(
src_dst,
|(src, dst)| {
let pixels = simd_utils::loadu_si256(src, 0);
let dst_ptr = dst.as_mut_ptr() as *mut __m256i;
(pixels, dst_ptr)
},
|(mut pixels, dst_ptr)| {
pixels = divide_alpha_16_pixels(pixels);
_mm256_storeu_si256(dst_ptr, pixels);
},
);
if !src_remainder.is_empty() {
let dst_reminder = dst_chunks.into_remainder();
sse4::divide_alpha_row(src_remainder, dst_reminder);
}
}
#[target_feature(enable = "avx2")]
unsafe fn divide_alpha_row_inplace(row: &mut [U8x2]) {
let mut chunks = row.chunks_exact_mut(16);
foreach_with_pre_reading(
&mut chunks,
|chunk| {
let pixels = simd_utils::loadu_si256(chunk, 0);
let dst_ptr = chunk.as_mut_ptr() as *mut __m256i;
(pixels, dst_ptr)
},
|(mut pixels, dst_ptr)| {
pixels = divide_alpha_16_pixels(pixels);
_mm256_storeu_si256(dst_ptr, pixels);
},
);
let reminder = chunks.into_remainder();
if !reminder.is_empty() {
sse4::divide_alpha_row_inplace(reminder);
}
}
#[inline]
#[target_feature(enable = "avx2")]
unsafe fn divide_alpha_sixteen_pixels(src: *const U8x2, dst: *mut U8x2) {
unsafe fn divide_alpha_16_pixels(pixels: __m256i) -> __m256i {
let alpha_mask = _mm256_set1_epi16(0xff00u16 as i16);
let luma_mask = _mm256_set1_epi16(0xff);
#[rustfmt::skip]
let alpha32_sh_lo = _mm256_set_epi8(
let alpha32_sh_lo = _mm256_set_epi8(
-1, -1, -1, 7, -1, -1, -1, 5, -1, -1, -1, 3, -1, -1, -1, 1,
-1, -1, -1, 7, -1, -1, -1, 5, -1, -1, -1, 3, -1, -1, -1, 1,
);
#[rustfmt::skip]
let alpha32_sh_hi = _mm256_set_epi8(
let alpha32_sh_hi = _mm256_set_epi8(
-1, -1, -1, 15, -1, -1, -1, 13, -1, -1, -1, 11, -1, -1, -1, 9,
-1, -1, -1, 15, -1, -1, -1, 13, -1, -1, -1, 11, -1, -1, -1, 9,
);
let alpha_scale = _mm256_set1_ps(255.0 * 256.0);
let src_pixels = _mm256_loadu_si256(src as *const __m256i);
let alpha_lo_f32 = _mm256_cvtepi32_ps(_mm256_shuffle_epi8(src_pixels, alpha32_sh_lo));
let alpha_lo_f32 = _mm256_cvtepi32_ps(_mm256_shuffle_epi8(pixels, alpha32_sh_lo));
let scaled_alpha_lo_i32 = _mm256_cvtps_epi32(_mm256_div_ps(alpha_scale, alpha_lo_f32));
let alpha_hi_f32 = _mm256_cvtepi32_ps(_mm256_shuffle_epi8(src_pixels, alpha32_sh_hi));
let alpha_hi_f32 = _mm256_cvtepi32_ps(_mm256_shuffle_epi8(pixels, alpha32_sh_hi));
let scaled_alpha_hi_i32 = _mm256_cvtps_epi32(_mm256_div_ps(alpha_scale, alpha_hi_f32));
let scaled_alpha_i16 = _mm256_packus_epi32(scaled_alpha_lo_i32, scaled_alpha_hi_i32);
let luma_i16 = _mm256_and_si256(src_pixels, luma_mask);
let luma_i16 = _mm256_and_si256(pixels, luma_mask);
let scaled_luma_i16 = _mm256_mullo_epi16(luma_i16, scaled_alpha_i16);
let scaled_luma_i16 = _mm256_srli_epi16::<8>(scaled_luma_i16);
let alpha = _mm256_and_si256(src_pixels, alpha_mask);
let dst_pixels = _mm256_blendv_epi8(scaled_luma_i16, alpha, alpha_mask);
_mm256_storeu_si256(dst as *mut __m256i, dst_pixels);
let alpha = _mm256_and_si256(pixels, alpha_mask);
_mm256_blendv_epi8(scaled_luma_i16, alpha, alpha_mask)
}
src/alpha/u8x2/native.rs
+11 -3
View File
@@ -12,9 +12,8 @@ pub(crate) fn multiply_alpha(src_image: &ImageView<U8x2>, dst_image: &mut ImageV
}
pub(crate) fn multiply_alpha_inplace(image: &mut ImageViewMut<U8x2>) {
for dst_row in image.iter_rows_mut() {
let src_row = unsafe { std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len()) };
multiply_alpha_row(src_row, dst_row);
for row in image.iter_rows_mut() {
multiply_alpha_row_inplace(row);
}
}
@@ -27,6 +26,15 @@ pub(crate) fn multiply_alpha_row(src_row: &[U8x2], dst_row: &mut [U8x2]) {
}
}
#[inline(always)]
pub(crate) fn multiply_alpha_row_inplace(row: &mut [U8x2]) {
for pixel in row {
let components: [u8; 2] = pixel.0.to_le_bytes();
let alpha = components[1];
pixel.0 = u16::from_le_bytes([mul_div_255(components[0], alpha), alpha]);
}
}
// Divide
#[inline]
src/alpha/u8x2/neon.rs
+194 -89
View File
@@ -2,6 +2,7 @@ use std::arch::aarch64::*;
use crate::neon_utils;
use crate::pixels::U8x2;
use crate::utils::foreach_with_pre_reading;
use crate::{ImageView, ImageViewMut};
use super::native;
@@ -19,75 +20,40 @@ pub(crate) unsafe fn multiply_alpha(
}
}
#[target_feature(enable = "neon")]
pub(crate) unsafe fn multiply_alpha_inplace(image: &mut ImageViewMut<U8x2>) {
for dst_row in image.iter_rows_mut() {
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
multiply_alpha_row(src_row, dst_row);
for row in image.iter_rows_mut() {
multiply_alpha_row_inplace(row);
}
}
#[inline]
#[target_feature(enable = "neon")]
#[inline(always)]
unsafe fn multiply_alpha_row(src_row: &[U8x2], dst_row: &mut [U8x2]) {
let zero_u8x16 = vdupq_n_u8(0);
let zero_u8x8 = vdup_n_u8(0);
let src_chunks = src_row.chunks_exact(64);
let src_chunks = src_row.chunks_exact(32);
let src_remainder = src_chunks.remainder();
let mut dst_chunks = dst_row.chunks_exact_mut(64);
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
let mut pixels = neon_utils::load_deintrel_u8x16x4(src, 0);
let alpha_u16 = uint16x8x2_t(
vreinterpretq_u16_u8(vzip1q_u8(pixels.1, zero_u8x16)),
vreinterpretq_u16_u8(vzip2q_u8(pixels.1, zero_u8x16)),
);
pixels.0 = neon_utils::mul_color_to_alpha_u8x16(pixels.0, alpha_u16, zero_u8x16);
let alpha_u16 = uint16x8x2_t(
vreinterpretq_u16_u8(vzip1q_u8(pixels.3, zero_u8x16)),
vreinterpretq_u16_u8(vzip2q_u8(pixels.3, zero_u8x16)),
);
pixels.2 = neon_utils::mul_color_to_alpha_u8x16(pixels.2, alpha_u16, zero_u8x16);
let dst_ptr = dst.as_mut_ptr() as *mut u8;
vst4q_u8(dst_ptr, pixels);
}
let src_chunks = src_remainder.chunks_exact(32);
let src_remainder = src_chunks.remainder();
let dst_reminder = dst_chunks.into_remainder();
let mut dst_chunks = dst_reminder.chunks_exact_mut(32);
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
let mut pixels = neon_utils::load_deintrel_u8x16x2(src, 0);
let alpha_u16 = uint16x8x2_t(
vreinterpretq_u16_u8(vzip1q_u8(pixels.1, zero_u8x16)),
vreinterpretq_u16_u8(vzip2q_u8(pixels.1, zero_u8x16)),
);
pixels.0 = neon_utils::mul_color_to_alpha_u8x16(pixels.0, alpha_u16, zero_u8x16);
let dst_ptr = dst.as_mut_ptr() as *mut u8;
vst2q_u8(dst_ptr, pixels);
}
let mut dst_chunks = dst_row.chunks_exact_mut(32);
let src_dst = src_chunks.zip(&mut dst_chunks);
foreach_with_pre_reading(
src_dst,
|(src, dst)| {
let pixels = neon_utils::load_deintrel_u8x16x2(src, 0);
let dst_ptr = dst.as_mut_ptr() as *mut u8;
(pixels, dst_ptr)
},
|(mut pixels, dst_ptr)| {
pixels = multiplies_alpha_32_pixels(pixels);
vst2q_u8(dst_ptr, pixels);
},
);
let src_chunks = src_remainder.chunks_exact(16);
let src_remainder = src_chunks.remainder();
let dst_reminder = dst_chunks.into_remainder();
let mut dst_chunks = dst_reminder.chunks_exact_mut(16);
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
let mut src_dst = src_chunks.zip(&mut dst_chunks);
if let Some((src, dst)) = src_dst.next() {
let mut pixels = neon_utils::load_deintrel_u8x8x4(src, 0);
let alpha_u16_lo = vreinterpret_u16_u8(vzip1_u8(pixels.1, zero_u8x8));
let alpha_u16_hi = vreinterpret_u16_u8(vzip2_u8(pixels.1, zero_u8x8));
let alpha_u16 = vcombine_u16(alpha_u16_lo, alpha_u16_hi);
pixels.0 = neon_utils::mul_color_to_alpha_u8x8(pixels.0, alpha_u16, zero_u8x8);
let alpha_u16_lo = vreinterpret_u16_u8(vzip1_u8(pixels.3, zero_u8x8));
let alpha_u16_hi = vreinterpret_u16_u8(vzip2_u8(pixels.3, zero_u8x8));
let alpha_u16 = vcombine_u16(alpha_u16_lo, alpha_u16_hi);
pixels.2 = neon_utils::mul_color_to_alpha_u8x8(pixels.2, alpha_u16, zero_u8x8);
pixels = multiplies_alpha_16_pixels(pixels);
let dst_ptr = dst.as_mut_ptr() as *mut u8;
vst4_u8(dst_ptr, pixels);
}
@@ -96,14 +62,10 @@ unsafe fn multiply_alpha_row(src_row: &[U8x2], dst_row: &mut [U8x2]) {
let src_remainder = src_chunks.remainder();
let dst_reminder = dst_chunks.into_remainder();
let mut dst_chunks = dst_reminder.chunks_exact_mut(8);
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
let mut src_dst = src_chunks.zip(&mut dst_chunks);
if let Some((src, dst)) = src_dst.next() {
let mut pixels = neon_utils::load_deintrel_u8x8x2(src, 0);
let alpha_u16_lo = vreinterpret_u16_u8(vzip1_u8(pixels.1, zero_u8x8));
let alpha_u16_hi = vreinterpret_u16_u8(vzip2_u8(pixels.1, zero_u8x8));
let alpha_u16 = vcombine_u16(alpha_u16_lo, alpha_u16_hi);
pixels.0 = neon_utils::mul_color_to_alpha_u8x8(pixels.0, alpha_u16, zero_u8x8);
pixels = multiplies_alpha_8_pixels(pixels);
let dst_ptr = dst.as_mut_ptr() as *mut u8;
vst2_u8(dst_ptr, pixels);
}
@@ -114,6 +76,99 @@ unsafe fn multiply_alpha_row(src_row: &[U8x2], dst_row: &mut [U8x2]) {
}
}
#[inline(always)]
unsafe fn multiply_alpha_row_inplace(row: &mut [U8x2]) {
let mut chunks = row.chunks_exact_mut(32);
foreach_with_pre_reading(
&mut chunks,
|chunk| {
let pixels = neon_utils::load_deintrel_u8x16x2(chunk, 0);
let dst_ptr = chunk.as_mut_ptr() as *mut u8;
(pixels, dst_ptr)
},
|(mut pixels, dst_ptr)| {
pixels = multiplies_alpha_32_pixels(pixels);
vst2q_u8(dst_ptr, pixels);
},
);
let reminder = chunks.into_remainder();
let mut chunks = reminder.chunks_exact_mut(16);
for chunk in &mut chunks {
let mut pixels = neon_utils::load_deintrel_u8x8x4(chunk, 0);
pixels = multiplies_alpha_16_pixels(pixels);
let chunk_ptr = chunk.as_mut_ptr() as *mut u8;
vst4_u8(chunk_ptr, pixels);
}
let reminder = chunks.into_remainder();
let mut chunks = reminder.chunks_exact_mut(8);
for chunk in &mut chunks {
let mut pixels = neon_utils::load_deintrel_u8x8x2(chunk, 0);
pixels = multiplies_alpha_8_pixels(pixels);
let chunk_ptr = chunk.as_mut_ptr() as *mut u8;
vst2_u8(chunk_ptr, pixels);
}
let reminder = chunks.into_remainder();
if !reminder.is_empty() {
native::multiply_alpha_row_inplace(reminder);
}
}
// #[inline(always)]
// unsafe fn multiplies_alpha_64_pixels(mut pixels: uint8x16x4_t) -> uint8x16x4_t {
// let zero_u8x16 = vdupq_n_u8(0);
// let alpha_u16 = uint16x8x2_t(
// vreinterpretq_u16_u8(vzip1q_u8(pixels.1, zero_u8x16)),
// vreinterpretq_u16_u8(vzip2q_u8(pixels.1, zero_u8x16)),
// );
// pixels.0 = neon_utils::mul_color_to_alpha_u8x16(pixels.0, alpha_u16, zero_u8x16);
// let alpha_u16 = uint16x8x2_t(
// vreinterpretq_u16_u8(vzip1q_u8(pixels.3, zero_u8x16)),
// vreinterpretq_u16_u8(vzip2q_u8(pixels.3, zero_u8x16)),
// );
// pixels.2 = neon_utils::mul_color_to_alpha_u8x16(pixels.2, alpha_u16, zero_u8x16);
// pixels
// }
#[inline(always)]
unsafe fn multiplies_alpha_32_pixels(mut pixels: uint8x16x2_t) -> uint8x16x2_t {
let zero_u8x16 = vdupq_n_u8(0);
let alpha_u16 = uint16x8x2_t(
vreinterpretq_u16_u8(vzip1q_u8(pixels.1, zero_u8x16)),
vreinterpretq_u16_u8(vzip2q_u8(pixels.1, zero_u8x16)),
);
pixels.0 = neon_utils::mul_color_to_alpha_u8x16(pixels.0, alpha_u16, zero_u8x16);
pixels
}
#[inline(always)]
unsafe fn multiplies_alpha_16_pixels(mut pixels: uint8x8x4_t) -> uint8x8x4_t {
let zero_u8x8 = vdup_n_u8(0);
let alpha_u16_lo = vreinterpret_u16_u8(vzip1_u8(pixels.1, zero_u8x8));
let alpha_u16_hi = vreinterpret_u16_u8(vzip2_u8(pixels.1, zero_u8x8));
let alpha_u16 = vcombine_u16(alpha_u16_lo, alpha_u16_hi);
pixels.0 = neon_utils::mul_color_to_alpha_u8x8(pixels.0, alpha_u16, zero_u8x8);
let alpha_u16_lo = vreinterpret_u16_u8(vzip1_u8(pixels.3, zero_u8x8));
let alpha_u16_hi = vreinterpret_u16_u8(vzip2_u8(pixels.3, zero_u8x8));
let alpha_u16 = vcombine_u16(alpha_u16_lo, alpha_u16_hi);
pixels.2 = neon_utils::mul_color_to_alpha_u8x8(pixels.2, alpha_u16, zero_u8x8);
pixels
}
#[inline(always)]
unsafe fn multiplies_alpha_8_pixels(mut pixels: uint8x8x2_t) -> uint8x8x2_t {
let zero_u8x8 = vdup_n_u8(0);
let alpha_u16_lo = vreinterpret_u16_u8(vzip1_u8(pixels.1, zero_u8x8));
let alpha_u16_hi = vreinterpret_u16_u8(vzip2_u8(pixels.1, zero_u8x8));
let alpha_u16 = vcombine_u16(alpha_u16_lo, alpha_u16_hi);
pixels.0 = neon_utils::mul_color_to_alpha_u8x8(pixels.0, alpha_u16, zero_u8x8);
pixels
}
// Divide
#[target_feature(enable = "neon")]
@@ -128,28 +183,40 @@ pub(crate) unsafe fn divide_alpha(src_image: &ImageView<U8x2>, dst_image: &mut I
#[target_feature(enable = "neon")]
pub(crate) unsafe fn divide_alpha_inplace(image: &mut ImageViewMut<U8x2>) {
for dst_row in image.iter_rows_mut() {
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
divide_alpha_row(src_row, dst_row);
for row in image.iter_rows_mut() {
divide_alpha_row_inplace(row);
}
}
#[inline]
#[target_feature(enable = "neon")]
#[inline(always)]
unsafe fn divide_alpha_row(src_row: &[U8x2], dst_row: &mut [U8x2]) {
let src_chunks = src_row.chunks_exact(16);
let src_remainder = src_chunks.remainder();
let mut dst_chunks = dst_row.chunks_exact_mut(16);
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
divide_alpha_16_pixels(src, dst);
}
let src_dst = src_chunks.zip(&mut dst_chunks);
foreach_with_pre_reading(
src_dst,
|(src, dst)| {
let pixels = neon_utils::load_deintrel_u8x16x2(src, 0);
let dst_ptr = dst.as_mut_ptr() as *mut u8;
(pixels, dst_ptr)
},
|(mut pixels, dst_ptr)| {
pixels = divide_alpha_16_pixels(pixels);
vst2q_u8(dst_ptr, pixels);
},
);
let src_chunks = src_remainder.chunks_exact(8);
let src_remainder = src_chunks.remainder();
let dst_reminder = dst_chunks.into_remainder();
let mut dst_chunks = dst_reminder.chunks_exact_mut(8);
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
divide_alpha_8_pixels(src, dst);
let mut src_dst = src_chunks.zip(&mut dst_chunks);
if let Some((src, dst)) = src_dst.next() {
let mut pixels = neon_utils::load_deintrel_u8x8x2(src, 0);
pixels = divide_alpha_8_pixels(pixels);
let dst_ptr = dst.as_mut_ptr() as *mut u8;
vst2_u8(dst_ptr, pixels);
}
if !src_remainder.is_empty() {
@@ -161,7 +228,10 @@ unsafe fn divide_alpha_row(src_row: &[U8x2], dst_row: &mut [U8x2]) {
.for_each(|(d, s)| *d = *s);
let mut dst_pixels = [U8x2::new(0); 8];
divide_alpha_8_pixels(src_pixels.as_slice(), dst_pixels.as_mut_slice());
let mut pixels = neon_utils::load_deintrel_u8x8x2(&src_pixels, 0);
pixels = divide_alpha_8_pixels(pixels);
let dst_ptr = dst_pixels.as_mut_ptr() as *mut u8;
vst2_u8(dst_ptr, pixels);
dst_pixels
.iter()
@@ -170,12 +240,53 @@ unsafe fn divide_alpha_row(src_row: &[U8x2], dst_row: &mut [U8x2]) {
}
}
#[inline]
#[target_feature(enable = "neon")]
unsafe fn divide_alpha_16_pixels(src: &[U8x2], dst: &mut [U8x2]) {
#[inline(always)]
unsafe fn divide_alpha_row_inplace(row: &mut [U8x2]) {
let mut chunks = row.chunks_exact_mut(16);
foreach_with_pre_reading(
&mut chunks,
|chunk| {
let pixels = neon_utils::load_deintrel_u8x16x2(chunk, 0);
let dst_ptr = chunk.as_mut_ptr() as *mut u8;
(pixels, dst_ptr)
},
|(mut pixels, dst_ptr)| {
pixels = divide_alpha_16_pixels(pixels);
vst2q_u8(dst_ptr, pixels);
},
);
let reminder = chunks.into_remainder();
let mut chunks = reminder.chunks_exact_mut(8);
if let Some(chunk) = chunks.next() {
let mut pixels = neon_utils::load_deintrel_u8x8x2(chunk, 0);
pixels = divide_alpha_8_pixels(pixels);
let chunk_ptr = chunk.as_mut_ptr() as *mut u8;
vst2_u8(chunk_ptr, pixels);
}
let reminder = chunks.into_remainder();
if !reminder.is_empty() {
let mut src_pixels = [U8x2::new(0); 8];
src_pixels
.iter_mut()
.zip(reminder.iter())
.for_each(|(d, s)| *d = *s);
let mut dst_pixels = [U8x2::new(0); 8];
let mut pixels = neon_utils::load_deintrel_u8x8x2(&src_pixels, 0);
pixels = divide_alpha_8_pixels(pixels);
let dst_ptr = dst_pixels.as_mut_ptr() as *mut u8;
vst2_u8(dst_ptr, pixels);
dst_pixels.iter().zip(reminder).for_each(|(s, d)| *d = *s);
}
}
#[inline(always)]
unsafe fn divide_alpha_16_pixels(mut pixels: uint8x16x2_t) -> uint8x16x2_t {
let zero = vdupq_n_u8(0);
let alpha_scale = vdupq_n_f32(255.0 * 256.0);
let mut pixels = neon_utils::load_deintrel_u8x16x2(src, 0);
let nonzero_alpha_mask = vmvnq_u8(vceqzq_u8(pixels.1));
let alpha_u16_lo = vzip1q_u8(pixels.1, zero);
@@ -204,18 +315,14 @@ unsafe fn divide_alpha_16_pixels(src: &[U8x2], dst: &mut [U8x2]) {
pixels.0 = neon_utils::mul_color_recip_alpha_u8x16(pixels.0, recip_alpha, zero);
pixels.0 = vandq_u8(pixels.0, nonzero_alpha_mask);
let dst_ptr = dst.as_mut_ptr() as *mut u8;
vst2q_u8(dst_ptr, pixels);
pixels
}
#[inline]
#[target_feature(enable = "neon")]
unsafe fn divide_alpha_8_pixels(src: &[U8x2], dst: &mut [U8x2]) {
#[inline(always)]
unsafe fn divide_alpha_8_pixels(mut pixels: uint8x8x2_t) -> uint8x8x2_t {
let zero_u8x8 = vdup_n_u8(0);
let zero_u8x16 = vdupq_n_u8(0);
let alpha_scale = vdupq_n_f32(255.0 * 256.0);
let mut pixels = neon_utils::load_deintrel_u8x8x2(src, 0);
let nonzero_alpha_mask = vmvn_u8(vceqz_u8(pixels.1));
let alpha_u16_lo = vzip1_u8(pixels.1, zero_u8x8);
@@ -234,7 +341,5 @@ unsafe fn divide_alpha_8_pixels(src: &[U8x2], dst: &mut [U8x2]) {
pixels.0 = neon_utils::mul_color_recip_alpha_u8x8(pixels.0, recip_alpha, zero_u8x8);
pixels.0 = vand_u8(pixels.0, nonzero_alpha_mask);
let dst_ptr = dst.as_mut_ptr() as *mut u8;
vst2_u8(dst_ptr, pixels);
pixels
}
src/alpha/u8x2/sse4.rs
+118 -49
View File
@@ -1,6 +1,7 @@
use std::arch::x86_64::*;
use crate::pixels::U8x2;
use crate::utils::foreach_with_pre_reading;
use crate::{ImageView, ImageViewMut};
use super::native;
@@ -20,18 +21,59 @@ pub(crate) unsafe fn multiply_alpha(
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn multiply_alpha_inplace(image: &mut ImageViewMut<U8x2>) {
for dst_row in image.iter_rows_mut() {
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
multiply_alpha_row(src_row, dst_row);
for row in image.iter_rows_mut() {
multiply_alpha_row_inplace(row);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn multiply_alpha_row(src_row: &[U8x2], dst_row: &mut [U8x2]) {
let src_chunks = src_row.chunks_exact(8);
let src_remainder = src_chunks.remainder();
let mut dst_chunks = dst_row.chunks_exact_mut(8);
let src_dst = src_chunks.zip(&mut dst_chunks);
foreach_with_pre_reading(
src_dst,
|(src, dst)| {
let pixels = _mm_loadu_si128(src.as_ptr() as *const __m128i);
let dst_ptr = dst.as_mut_ptr() as *mut __m128i;
(pixels, dst_ptr)
},
|(mut pixels, dst_ptr)| {
pixels = multiplies_alpha_8_pixels(pixels);
_mm_storeu_si128(dst_ptr, pixels);
},
);
if !src_remainder.is_empty() {
let dst_reminder = dst_chunks.into_remainder();
native::multiply_alpha_row(src_remainder, dst_reminder);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn multiply_alpha_row_inplace(row: &mut [U8x2]) {
let mut chunks = row.chunks_exact_mut(8);
// Using a simple for-loop in this case is faster than implementation with pre-reading
for chunk in &mut chunks {
let src_pixels = _mm_loadu_si128(chunk.as_ptr() as *const __m128i);
let dst_pixels = multiplies_alpha_8_pixels(src_pixels);
_mm_storeu_si128(chunk.as_mut_ptr() as *mut __m128i, dst_pixels);
}
let reminder = chunks.into_remainder();
if !reminder.is_empty() {
native::multiply_alpha_row_inplace(reminder);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
unsafe fn multiplies_alpha_8_pixels(pixels: __m128i) -> __m128i {
let zero = _mm_setzero_si128();
let half = _mm_set1_epi16(128);
const MAX_A: i16 = 0xff00u16 as i16;
let max_alpha = _mm_set1_epi16(MAX_A);
/*
@@ -40,37 +82,22 @@ pub(crate) unsafe fn multiply_alpha_row(src_row: &[U8x2], dst_row: &mut [U8x2])
*/
let factor_mask = _mm_set_epi8(15, 15, 13, 13, 11, 11, 9, 9, 7, 7, 5, 5, 3, 3, 1, 1);
let src_chunks = src_row.chunks_exact(8);
let src_remainder = src_chunks.remainder();
let mut dst_chunks = dst_row.chunks_exact_mut(8);
let factor_pixels = _mm_shuffle_epi8(pixels, factor_mask);
let factor_pixels = _mm_or_si128(factor_pixels, max_alpha);
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
let src_pixels = _mm_loadu_si128(src.as_ptr() as *const __m128i);
let src_i16_lo = _mm_unpacklo_epi8(pixels, zero);
let factors = _mm_unpacklo_epi8(factor_pixels, zero);
let src_i16_lo = _mm_add_epi16(_mm_mullo_epi16(src_i16_lo, factors), half);
let dst_i16_lo = _mm_add_epi16(src_i16_lo, _mm_srli_epi16::<8>(src_i16_lo));
let dst_i16_lo = _mm_srli_epi16::<8>(dst_i16_lo);
let factor_pixels = _mm_shuffle_epi8(src_pixels, factor_mask);
let factor_pixels = _mm_or_si128(factor_pixels, max_alpha);
let src_i16_hi = _mm_unpackhi_epi8(pixels, zero);
let factors = _mm_unpackhi_epi8(factor_pixels, zero);
let src_i16_hi = _mm_add_epi16(_mm_mullo_epi16(src_i16_hi, factors), half);
let dst_i16_hi = _mm_add_epi16(src_i16_hi, _mm_srli_epi16::<8>(src_i16_hi));
let dst_i16_hi = _mm_srli_epi16::<8>(dst_i16_hi);
let src_i16_lo = _mm_unpacklo_epi8(src_pixels, zero);
let factors = _mm_unpacklo_epi8(factor_pixels, zero);
let src_i16_lo = _mm_add_epi16(_mm_mullo_epi16(src_i16_lo, factors), half);
let dst_i16_lo = _mm_add_epi16(src_i16_lo, _mm_srli_epi16::<8>(src_i16_lo));
let dst_i16_lo = _mm_srli_epi16::<8>(dst_i16_lo);
let src_i16_hi = _mm_unpackhi_epi8(src_pixels, zero);
let factors = _mm_unpackhi_epi8(factor_pixels, zero);
let src_i16_hi = _mm_add_epi16(_mm_mullo_epi16(src_i16_hi, factors), half);
let dst_i16_hi = _mm_add_epi16(src_i16_hi, _mm_srli_epi16::<8>(src_i16_hi));
let dst_i16_hi = _mm_srli_epi16::<8>(dst_i16_hi);
let dst_pixels = _mm_packus_epi16(dst_i16_lo, dst_i16_hi);
_mm_storeu_si128(dst.as_mut_ptr() as *mut __m128i, dst_pixels);
}
if !src_remainder.is_empty() {
let dst_reminder = dst_chunks.into_remainder();
native::multiply_alpha_row(src_remainder, dst_reminder);
}
_mm_packus_epi16(dst_i16_lo, dst_i16_hi)
}
// Divide
@@ -87,21 +114,30 @@ pub(crate) unsafe fn divide_alpha(src_image: &ImageView<U8x2>, dst_image: &mut I
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn divide_alpha_inplace(image: &mut ImageViewMut<U8x2>) {
for dst_row in image.iter_rows_mut() {
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
divide_alpha_row(src_row, dst_row);
for row in image.iter_rows_mut() {
divide_alpha_row_inplace(row);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
pub(crate) unsafe fn divide_alpha_row(src_row: &[U8x2], dst_row: &mut [U8x2]) {
let src_chunks = src_row.chunks_exact(8);
let src_remainder = src_chunks.remainder();
let mut dst_chunks = dst_row.chunks_exact_mut(8);
for (src, dst) in src_chunks.zip(&mut dst_chunks) {
divide_alpha_eight_pixels(src.as_ptr(), dst.as_mut_ptr());
}
let src_dst = src_chunks.zip(&mut dst_chunks);
foreach_with_pre_reading(
src_dst,
|(src, dst)| {
let pixels = _mm_loadu_si128(src.as_ptr() as *const __m128i);
let dst_ptr = dst.as_mut_ptr() as *mut __m128i;
(pixels, dst_ptr)
},
|(mut pixels, dst_ptr)| {
pixels = divide_alpha_8_pixels(pixels);
_mm_storeu_si128(dst_ptr, pixels);
},
);
if !src_remainder.is_empty() {
let dst_reminder = dst_chunks.into_remainder();
@@ -112,7 +148,9 @@ pub(crate) unsafe fn divide_alpha_row(src_row: &[U8x2], dst_row: &mut [U8x2]) {
.for_each(|(d, s)| *d = *s);
let mut dst_pixels = [U8x2::new(0); 8];
divide_alpha_eight_pixels(src_pixels.as_ptr(), dst_pixels.as_mut_ptr());
let mut pixels = _mm_loadu_si128(src_pixels.as_ptr() as *const __m128i);
pixels = divide_alpha_8_pixels(pixels);
_mm_storeu_si128(dst_pixels.as_mut_ptr() as *mut __m128i, pixels);
dst_pixels
.iter()
@@ -123,7 +161,41 @@ pub(crate) unsafe fn divide_alpha_row(src_row: &[U8x2], dst_row: &mut [U8x2]) {
#[inline]
#[target_feature(enable = "sse4.1")]
unsafe fn divide_alpha_eight_pixels(src: *const U8x2, dst: *mut U8x2) {
pub(crate) unsafe fn divide_alpha_row_inplace(row: &mut [U8x2]) {
let mut chunks = row.chunks_exact_mut(8);
foreach_with_pre_reading(
&mut chunks,
|chunk| {
let pixels = _mm_loadu_si128(chunk.as_ptr() as *const __m128i);
let dst_ptr = chunk.as_mut_ptr() as *mut __m128i;
(pixels, dst_ptr)
},
|(mut pixels, dst_ptr)| {
pixels = divide_alpha_8_pixels(pixels);
_mm_storeu_si128(dst_ptr, pixels);
},
);
let reminder = chunks.into_remainder();
if !reminder.is_empty() {
let mut src_pixels = [U8x2::new(0); 8];
src_pixels
.iter_mut()
.zip(reminder.iter())
.for_each(|(d, s)| *d = *s);
let mut dst_pixels = [U8x2::new(0); 8];
let mut pixels = _mm_loadu_si128(src_pixels.as_ptr() as *const __m128i);
pixels = divide_alpha_8_pixels(pixels);
_mm_storeu_si128(dst_pixels.as_mut_ptr() as *mut __m128i, pixels);
dst_pixels.iter().zip(reminder).for_each(|(s, d)| *d = *s);
}
}
#[inline]
#[target_feature(enable = "sse4.1")]
unsafe fn divide_alpha_8_pixels(pixels: __m128i) -> __m128i {
let alpha_mask = _mm_set1_epi16(0xff00u16 as i16);
let luma_mask = _mm_set1_epi16(0xff);
let alpha32_sh_lo = _mm_set_epi8(-1, -1, -1, 7, -1, -1, -1, 5, -1, -1, -1, 3, -1, -1, -1, 1);
@@ -132,19 +204,16 @@ unsafe fn divide_alpha_eight_pixels(src: *const U8x2, dst: *mut U8x2) {
);
let alpha_scale = _mm_set1_ps(255.0 * 256.0);
let src_pixels = _mm_loadu_si128(src as *const __m128i);
let alpha_lo_f32 = _mm_cvtepi32_ps(_mm_shuffle_epi8(src_pixels, alpha32_sh_lo));
let alpha_lo_f32 = _mm_cvtepi32_ps(_mm_shuffle_epi8(pixels, alpha32_sh_lo));
let scaled_alpha_lo_i32 = _mm_cvtps_epi32(_mm_div_ps(alpha_scale, alpha_lo_f32));
let alpha_hi_f32 = _mm_cvtepi32_ps(_mm_shuffle_epi8(src_pixels, alpha32_sh_hi));
let alpha_hi_f32 = _mm_cvtepi32_ps(_mm_shuffle_epi8(pixels, alpha32_sh_hi));
let scaled_alpha_hi_i32 = _mm_cvtps_epi32(_mm_div_ps(alpha_scale, alpha_hi_f32));
let scaled_alpha_i16 = _mm_packus_epi32(scaled_alpha_lo_i32, scaled_alpha_hi_i32);
let luma_i16 = _mm_and_si128(src_pixels, luma_mask);
let luma_i16 = _mm_and_si128(pixels, luma_mask);
let scaled_luma_i16 = _mm_mullo_epi16(luma_i16, scaled_alpha_i16);
let scaled_luma_i16 = _mm_srli_epi16::<8>(scaled_luma_i16);
let alpha = _mm_and_si128(src_pixels, alpha_mask);
let dst_pixels = _mm_blendv_epi8(scaled_luma_i16, alpha, alpha_mask);
_mm_storeu_si128(dst as *mut __m128i, dst_pixels);
let alpha = _mm_and_si128(pixels, alpha_mask);
_mm_blendv_epi8(scaled_luma_i16, alpha, alpha_mask)
}