Added support of compilation for architectures other than x86_64.

This commit is contained in:
Kirill Kuzminykh
2021-10-09 14:42:26 +03:00
parent 52acf313c2
commit 071d5c3e31
19 changed files with 497 additions and 393 deletions
+4
View File
@@ -1,3 +1,7 @@
## [Unreleased] - ReleaseDate
- Added support of compilation for architectures other than x86_64.
## [0.3.0] - 2021-08-28
- Added method `SrcImageView.set_crop_box_to_fit_dst_size()`.
+29 -9
View File
@@ -16,6 +16,7 @@ fn get_src_image(width: NonZeroU32, height: NonZeroU32, pixel: u32) -> ImageData
ImageData::from_vec_u32(width, height, buffer, PixelType::U8x4).unwrap()
}
#[cfg(target_arch = "x86_64")]
fn multiplies_alpha_avx2(bench: &mut Bench) {
let width = NonZeroU32::new(4096).unwrap();
let height = NonZeroU32::new(2048).unwrap();
@@ -37,6 +38,7 @@ fn multiplies_alpha_avx2(bench: &mut Bench) {
});
}
#[cfg(target_arch = "x86_64")]
fn multiplies_alpha_sse2(bench: &mut Bench) {
let width = NonZeroU32::new(4096).unwrap();
let height = NonZeroU32::new(2048).unwrap();
@@ -79,6 +81,7 @@ fn multiplies_alpha_native(bench: &mut Bench) {
});
}
#[cfg(target_arch = "x86_64")]
fn divides_alpha_avx2(bench: &mut Bench) {
let width = NonZeroU32::new(4096).unwrap();
let height = NonZeroU32::new(2048).unwrap();
@@ -100,6 +103,7 @@ fn divides_alpha_avx2(bench: &mut Bench) {
});
}
#[cfg(target_arch = "x86_64")]
fn divides_alpha_sse2(bench: &mut Bench) {
let width = NonZeroU32::new(4096).unwrap();
let height = NonZeroU32::new(2048).unwrap();
@@ -142,12 +146,28 @@ fn divides_alpha_native(bench: &mut Bench) {
});
}
glassbench!(
"Alpha",
multiplies_alpha_avx2,
multiplies_alpha_sse2,
multiplies_alpha_native,
divides_alpha_avx2,
divides_alpha_sse2,
divides_alpha_native,
);
pub fn main() {
use glassbench::*;
let name = env!("CARGO_CRATE_NAME");
let cmd = Command::read();
if cmd.include_bench(name) {
let mut bench = create_bench(name, "Alpha", &cmd);
#[cfg(target_arch = "x86_64")]
{
multiplies_alpha_avx2(&mut bench);
multiplies_alpha_sse2(&mut bench);
}
multiplies_alpha_native(&mut bench);
#[cfg(target_arch = "x86_64")]
{
divides_alpha_avx2(&mut bench);
divides_alpha_sse2(&mut bench);
}
divides_alpha_native(&mut bench);
if let Err(e) = after_bench(&mut bench, &cmd) {
eprintln!("{:?}", e);
}
} else {
println!("skipping bench {:?}", &name);
}
}
+7 -11
View File
@@ -63,17 +63,13 @@ pub fn bench_downscale_rgb(bench: &mut Bench) {
}
// fast_image_resize crate;
for cpu_ext in [
CpuExtensions::None,
CpuExtensions::Sse4_1,
CpuExtensions::Avx2,
] {
let ext_name = match cpu_ext {
CpuExtensions::None => "rust",
CpuExtensions::Sse4_1 => "sse4.1",
CpuExtensions::Avx2 => "avx2",
_ => continue,
};
let mut cpu_ext_and_name = vec![(CpuExtensions::None, "rust")];
#[cfg(target_arch = "x86_64")]
{
cpu_ext_and_name.push((CpuExtensions::Sse4_1, "sse4.1"));
cpu_ext_and_name.push((CpuExtensions::Avx2, "avx2"));
}
for (cpu_ext, ext_name) in cpu_ext_and_name {
for alg_name in alg_names {
let src_rgba_image = utils::get_big_rgba_image();
let src_image_data = ImageData::from_vec_u8(
+7 -11
View File
@@ -64,17 +64,13 @@ pub fn bench_downscale_rgba(bench: &mut Bench) {
}
// fast_image_resize crate;
for cpu_ext in [
CpuExtensions::None,
CpuExtensions::Sse4_1,
CpuExtensions::Avx2,
] {
let ext_name = match cpu_ext {
CpuExtensions::None => "rust",
CpuExtensions::Sse4_1 => "sse4.1",
CpuExtensions::Avx2 => "avx2",
_ => continue,
};
let mut cpu_ext_and_name = vec![(CpuExtensions::None, "rust")];
#[cfg(target_arch = "x86_64")]
{
cpu_ext_and_name.push((CpuExtensions::Sse4_1, "sse4.1"));
cpu_ext_and_name.push((CpuExtensions::Avx2, "avx2"));
}
for (cpu_ext, ext_name) in cpu_ext_and_name {
for alg_name in alg_names {
let resize_alg = match alg_name {
"Nearest" => ResizeAlg::Nearest,
+27 -10
View File
@@ -97,6 +97,7 @@ fn lanczos3_wo_simd_bench(bench: &mut Bench) {
});
}
#[cfg(target_arch = "x86_64")]
fn sse4_lanczos3_bench(bench: &mut Bench) {
let image = get_big_source_image();
let mut res_image = ImageData::new(
@@ -117,6 +118,7 @@ fn sse4_lanczos3_bench(bench: &mut Bench) {
});
}
#[cfg(target_arch = "x86_64")]
fn avx2_lanczos3_bench(bench: &mut Bench) {
let image = get_big_source_image();
let mut res_image = ImageData::new(
@@ -137,6 +139,7 @@ fn avx2_lanczos3_bench(bench: &mut Bench) {
});
}
#[cfg(target_arch = "x86_64")]
fn avx2_supersampling_lanczos3_bench(bench: &mut Bench) {
let image = get_big_source_image();
let mut res_image = ImageData::new(
@@ -157,6 +160,7 @@ fn avx2_supersampling_lanczos3_bench(bench: &mut Bench) {
});
}
#[cfg(target_arch = "x86_64")]
fn avx2_lanczos3_upscale_bench(bench: &mut Bench) {
let image = get_small_source_image();
let mut res_image = ImageData::new(
@@ -197,13 +201,26 @@ fn native_lanczos3_i32_bench(bench: &mut Bench) {
});
}
glassbench!(
"Resize",
nearest_wo_simd_bench,
lanczos3_wo_simd_bench,
sse4_lanczos3_bench,
avx2_lanczos3_bench,
avx2_supersampling_lanczos3_bench,
avx2_lanczos3_upscale_bench,
native_lanczos3_i32_bench,
);
pub fn main() {
use glassbench::*;
let name = env!("CARGO_CRATE_NAME");
let cmd = Command::read();
if cmd.include_bench(name) {
let mut bench = create_bench(name, "Resize", &cmd);
#[cfg(target_arch = "x86_64")]
{
sse4_lanczos3_bench(&mut bench);
avx2_lanczos3_bench(&mut bench);
avx2_supersampling_lanczos3_bench(&mut bench);
avx2_lanczos3_upscale_bench(&mut bench);
}
nearest_wo_simd_bench(&mut bench);
lanczos3_wo_simd_bench(&mut bench);
native_lanczos3_i32_bench(&mut bench);
if let Err(e) = after_bench(&mut bench, &cmd) {
eprintln!("{:?}", e);
}
} else {
println!("skipping bench {:?}", &name);
}
}
+75
View File
@@ -0,0 +1,75 @@
use std::arch::x86_64::*;
use crate::alpha::native;
use crate::simd_utils;
use crate::{DstImageView, SrcImageView};
pub(crate) fn divide_alpha_avx2(src_image: &SrcImageView, dst_image: &mut DstImageView) {
let width = src_image.width().get();
let src_rows = src_image.iter_rows(0, src_image.height().get());
let dst_rows = dst_image.iter_rows_mut();
for (src_row, dst_row) in src_rows.zip(dst_rows) {
unsafe {
divide_alpha_row_avx2(src_row, dst_row, width as usize);
}
}
}
pub(crate) fn divide_alpha_inplace_avx2(image: &mut DstImageView) {
let width = image.width().get() as usize;
for dst_row in image.iter_rows_mut() {
unsafe {
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
divide_alpha_row_avx2(src_row, dst_row, width);
}
}
}
#[target_feature(enable = "avx2")]
unsafe fn divide_alpha_row_avx2(src_row: &[u32], dst_row: &mut [u32], width: usize) {
let mut x: usize = 0;
let zero = _mm256_setzero_si256();
#[rustfmt::skip]
let alpha_mask = _mm256_set1_epi32(0xff000000u32 as i32);
#[rustfmt::skip]
let shuffle1 = _mm256_set_epi8(
5, 4, 5, 4, 5, 4, 5, 4, 1, 0, 1, 0, 1, 0, 1, 0,
5, 4, 5, 4, 5, 4, 5, 4, 1, 0, 1, 0, 1, 0, 1, 0,
);
#[rustfmt::skip]
let shuffle2 = _mm256_set_epi8(
13, 12, 13, 12, 13, 12, 13, 12, 9, 8, 9, 8, 9, 8, 9, 8,
13, 12, 13, 12, 13, 12, 13, 12, 9, 8, 9, 8, 9, 8, 9, 8,
);
let alpha_scale = _mm256_set1_ps(255.0 * 256.0);
while x < width.saturating_sub(7) {
let mut source = simd_utils::loadu_si256(src_row, x);
let alpha_f32 = _mm256_cvtepi32_ps(_mm256_srli_epi32::<24>(source));
let scaled_alpha_f32 = _mm256_mul_ps(alpha_scale, _mm256_rcp_ps(alpha_f32));
let scaled_alpha_i32 = _mm256_cvtps_epi32(scaled_alpha_f32);
let mma0 = _mm256_shuffle_epi8(scaled_alpha_i32, shuffle1);
let mma1 = _mm256_shuffle_epi8(scaled_alpha_i32, shuffle2);
let mut pix0 = _mm256_unpacklo_epi8(zero, source);
let mut pix1 = _mm256_unpackhi_epi8(zero, source);
pix0 = _mm256_mulhi_epu16(pix0, mma0);
pix1 = _mm256_mulhi_epu16(pix1, mma1);
let alpha = _mm256_and_si256(source, alpha_mask);
source = _mm256_packus_epi16(pix0, pix1);
source = _mm256_blendv_epi8(source, alpha, alpha_mask);
let dst_ptr = dst_row.get_unchecked_mut(x..).as_mut_ptr() as *mut __m256i;
_mm256_storeu_si256(dst_ptr, source);
x += 8;
}
let src_tail = &src_row[x..];
let dst_tail = &mut dst_row[x..];
native::divide_alpha_row_native(src_tail, dst_tail);
}
+5
View File
@@ -0,0 +1,5 @@
pub(crate) use div::{divide_alpha_avx2, divide_alpha_inplace_avx2};
pub(crate) use mul::{multiply_alpha_avx2, multiply_alpha_inplace_avx2};
mod div;
mod mul;
+70
View File
@@ -0,0 +1,70 @@
use std::arch::x86_64::*;
use crate::alpha::native;
use crate::{simd_utils, DstImageView, SrcImageView};
pub(crate) fn multiply_alpha_avx2(src_image: &SrcImageView, dst_image: &mut DstImageView) {
let width = src_image.width().get() as usize;
let src_rows = src_image.iter_rows(0, src_image.height().get());
let dst_rows = dst_image.iter_rows_mut();
for (src_row, dst_row) in src_rows.zip(dst_rows) {
unsafe {
multiply_alpha_row_avx2(src_row, dst_row, width);
}
}
}
pub(crate) fn multiply_alpha_inplace_avx2(image: &mut DstImageView) {
let width = image.width().get() as usize;
for dst_row in image.iter_rows_mut() {
unsafe {
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
multiply_alpha_row_avx2(src_row, dst_row, width);
}
}
}
/// https://github.com/Wizermil/premultiply_alpha/blob/master/premultiply_alpha/premultiply_alpha.hpp#L232
#[target_feature(enable = "avx2")]
unsafe fn multiply_alpha_row_avx2(src_row: &[u32], dst_row: &mut [u32], width: usize) {
let mask_alpha_color_odd_255 = _mm256_set1_epi32(0xff000000u32 as i32);
let div_255 = _mm256_set1_epi16(0x8081u16 as i16);
#[rustfmt::skip]
let mask_shuffle_alpha = _mm256_set_epi8(
15, -1, 15, -1, 11, -1, 11, -1, 7, -1, 7, -1, 3, -1, 3, -1,
15, -1, 15, -1, 11, -1, 11, -1, 7, -1, 7, -1, 3, -1, 3, -1,
);
#[rustfmt::skip]
let mask_shuffle_color_odd = _mm256_set_epi8(
-1, -1, 13, -1, -1, -1, 9, -1, -1, -1, 5, -1, -1, -1, 1, -1,
-1, -1, 13, -1, -1, -1, 9, -1, -1, -1, 5, -1, -1, -1, 1, -1,
);
let mut x: usize = 0;
while x < width.saturating_sub(7) {
let mut color = simd_utils::loadu_si256(src_row, x);
let alpha = _mm256_shuffle_epi8(color, mask_shuffle_alpha);
let mut color_even = _mm256_slli_epi16::<8>(color);
let mut color_odd = _mm256_shuffle_epi8(color, mask_shuffle_color_odd);
color_odd = _mm256_or_si256(color_odd, mask_alpha_color_odd_255);
color_odd = _mm256_mulhi_epu16(color_odd, alpha);
color_even = _mm256_mulhi_epu16(color_even, alpha);
color_odd = _mm256_srli_epi16::<7>(_mm256_mulhi_epu16(color_odd, div_255));
color_even = _mm256_srli_epi16::<7>(_mm256_mulhi_epu16(color_even, div_255));
color = _mm256_or_si256(color_even, _mm256_slli_epi16::<8>(color_odd));
let dst_ptr = dst_row.get_unchecked_mut(x..).as_mut_ptr() as *mut __m256i;
_mm256_storeu_si256(dst_ptr, color);
x += 8;
}
let src_tail = &src_row[x..];
let dst_tail = &mut dst_row[x..];
native::multiply_alpha_row_native(src_tail, dst_tail);
}
-175
View File
@@ -1,175 +0,0 @@
use std::arch::x86_64::*;
use crate::simd_utils;
use crate::{DstImageView, SrcImageView};
pub(crate) fn divide_alpha_avx2(src_image: &SrcImageView, dst_image: &mut DstImageView) {
let width = src_image.width().get();
let src_rows = src_image.iter_rows(0, src_image.height().get());
let dst_rows = dst_image.iter_rows_mut();
for (src_row, dst_row) in src_rows.zip(dst_rows) {
unsafe {
divide_alpha_row_avx2(src_row, dst_row, width as usize);
}
}
}
pub(crate) fn divide_alpha_inplace_avx2(image: &mut DstImageView) {
let width = image.width().get() as usize;
for dst_row in image.iter_rows_mut() {
unsafe {
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
divide_alpha_row_avx2(src_row, dst_row, width);
}
}
}
pub(crate) fn divide_alpha_sse2(src_image: &SrcImageView, dst_image: &mut DstImageView) {
let width = src_image.width().get() as usize;
let src_rows = src_image.iter_rows(0, src_image.height().get());
let dst_rows = dst_image.iter_rows_mut();
for (src_row, dst_row) in src_rows.zip(dst_rows) {
unsafe {
divide_alpha_row_sse2(src_row, dst_row, width);
}
}
}
pub(crate) fn divide_alpha_inplace_sse2(image: &mut DstImageView) {
let width = image.width().get() as usize;
for dst_row in image.iter_rows_mut() {
unsafe {
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
divide_alpha_row_sse2(src_row, dst_row, width);
}
}
}
pub(crate) fn divide_alpha_native(src_image: &SrcImageView, dst_image: &mut DstImageView) {
let src_rows = src_image.iter_rows(0, src_image.height().get());
let dst_rows = dst_image.iter_rows_mut();
for (src_row, dst_row) in src_rows.zip(dst_rows) {
divide_alpha_row_native(src_row, dst_row);
}
}
pub(crate) fn divide_alpha_inplace_native(image: &mut DstImageView) {
for dst_row in image.iter_rows_mut() {
let src_row = unsafe { std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len()) };
divide_alpha_row_native(src_row, dst_row);
}
}
#[target_feature(enable = "avx2")]
unsafe fn divide_alpha_row_avx2(src_row: &[u32], dst_row: &mut [u32], width: usize) {
let mut x: usize = 0;
let zero = _mm256_setzero_si256();
#[rustfmt::skip]
let alpha_mask = _mm256_set1_epi32(0xff000000u32 as i32);
#[rustfmt::skip]
let shuffle1 = _mm256_set_epi8(
5, 4, 5, 4, 5, 4, 5, 4, 1, 0, 1, 0, 1, 0, 1, 0,
5, 4, 5, 4, 5, 4, 5, 4, 1, 0, 1, 0, 1, 0, 1, 0,
);
#[rustfmt::skip]
let shuffle2 = _mm256_set_epi8(
13, 12, 13, 12, 13, 12, 13, 12, 9, 8, 9, 8, 9, 8, 9, 8,
13, 12, 13, 12, 13, 12, 13, 12, 9, 8, 9, 8, 9, 8, 9, 8,
);
let alpha_scale = _mm256_set1_ps(255.0 * 256.0);
while x < width.saturating_sub(7) {
let mut source = simd_utils::loadu_si256(src_row, x);
let alpha_f32 = _mm256_cvtepi32_ps(_mm256_srli_epi32(source, 24));
let scaled_alpha_f32 = _mm256_mul_ps(alpha_scale, _mm256_rcp_ps(alpha_f32));
let scaled_alpha_i32 = _mm256_cvtps_epi32(scaled_alpha_f32);
let mma0 = _mm256_shuffle_epi8(scaled_alpha_i32, shuffle1);
let mma1 = _mm256_shuffle_epi8(scaled_alpha_i32, shuffle2);
let mut pix0 = _mm256_unpacklo_epi8(zero, source);
let mut pix1 = _mm256_unpackhi_epi8(zero, source);
pix0 = _mm256_mulhi_epu16(pix0, mma0);
pix1 = _mm256_mulhi_epu16(pix1, mma1);
let alpha = _mm256_and_si256(source, alpha_mask);
source = _mm256_packus_epi16(pix0, pix1);
source = _mm256_blendv_epi8(source, alpha, alpha_mask);
let dst_ptr = dst_row.get_unchecked_mut(x..).as_mut_ptr() as *mut __m256i;
_mm256_storeu_si256(dst_ptr, source);
x += 8;
}
let src_tail = &src_row[x..];
let dst_tail = &mut dst_row[x..];
divide_alpha_row_native(src_tail, dst_tail);
}
#[target_feature(enable = "sse2")]
unsafe fn divide_alpha_row_sse2(src_row: &[u32], dst_row: &mut [u32], width: usize) {
let zero = _mm_setzero_si128();
let alpha_mask = _mm_set1_epi32(0xff000000u32 as i32);
let shuffle0 = _mm_set_epi8(5, 4, 5, 4, 5, 4, 5, 4, 1, 0, 1, 0, 1, 0, 1, 0);
let shuffle1 = _mm_set_epi8(13, 12, 13, 12, 13, 12, 13, 12, 9, 8, 9, 8, 9, 8, 9, 8);
let alpha_scale = _mm_set1_ps(255.0 * 256.0);
let mut x: usize = 0;
while x < width.saturating_sub(3) {
let mut source = simd_utils::loadu_si128(src_row, x);
let alpha = _mm_and_si128(source, alpha_mask);
let alpha_f32 = _mm_cvtepi32_ps(_mm_srli_epi32(source, 24));
let scaled_recip_alpha_f32 = _mm_mul_ps(alpha_scale, _mm_rcp_ps(alpha_f32));
let scaled_recip_alpha_i32 = _mm_cvtps_epi32(scaled_recip_alpha_f32);
let mma0 = _mm_shuffle_epi8(scaled_recip_alpha_i32, shuffle0);
let mma1 = _mm_shuffle_epi8(scaled_recip_alpha_i32, shuffle1);
let mut pix0 = _mm_unpacklo_epi8(zero, source);
let mut pix1 = _mm_unpackhi_epi8(zero, source);
pix0 = _mm_mulhi_epu16(pix0, mma0);
pix1 = _mm_mulhi_epu16(pix1, mma1);
source = _mm_packus_epi16(pix0, pix1);
source = _mm_blendv_epi8(source, alpha, alpha_mask);
let dst_ptr = dst_row.get_unchecked_mut(x..).as_mut_ptr() as *mut __m128i;
_mm_storeu_si128(dst_ptr, source);
x += 4;
}
let src_tail = &src_row[x..];
let dst_tail = &mut dst_row[x..];
divide_alpha_row_native(src_tail, dst_tail);
}
#[inline(always)]
fn div_and_clip(v: u8, rev_alpha: f32) -> u8 {
let res = v as f32 * rev_alpha;
res.min(255.) as u8
}
#[inline(always)]
fn divide_alpha_row_native(src_row: &[u32], dst_row: &mut [u32]) {
src_row
.iter()
.zip(dst_row)
.for_each(|(src_pixel, dst_pixel)| {
let components: [u8; 4] = src_pixel.to_le_bytes();
let alpha = components[3];
let recip_alpha = if alpha == 0 { 0. } else { 255. / alpha as f32 };
let res = [
div_and_clip(components[0], recip_alpha),
div_and_clip(components[1], recip_alpha),
div_and_clip(components[2], recip_alpha),
alpha,
];
*dst_pixel = u32::from_le_bytes(res);
});
}
+32 -15
View File
@@ -3,8 +3,11 @@ use thiserror::Error;
use crate::{CpuExtensions, PixelType};
use crate::{DstImageView, SrcImageView};
mod div;
mod mul;
#[cfg(target_arch = "x86_64")]
mod avx2;
mod native;
#[cfg(target_arch = "x86_64")]
mod sse2;
#[derive(Error, Debug, Clone, Copy)]
#[non_exhaustive]
@@ -27,7 +30,7 @@ pub enum MulDivImageError {
/// Methods of this structure used to multiplies or divides RGB-channels
/// by alpha-channel.
///
/// By default instance of `MulDiv` created with best CPU-extensions provided by your CPU.
/// By default, instance of `MulDiv` created with best CPU-extensions provided by your CPU.
/// You can change this by use method [MulDiv::set_cpu_extensions].
///
/// # Examples
@@ -71,9 +74,14 @@ impl MulDiv {
) -> Result<(), MulDivImagesError> {
self.assert_images(src_image, dst_image)?;
match self.cpu_extensions {
CpuExtensions::Avx2 => mul::multiply_alpha_avx2(src_image, dst_image),
// CpuExtensions::Sse2 => mul::multiply_alpha_sse2(src_image, dst_image),
_ => mul::multiply_alpha_native(src_image, dst_image),
#[cfg(target_arch = "x86_64")]
CpuExtensions::Avx2 => avx2::multiply_alpha_avx2(src_image, dst_image),
// WARNING: SSE2 implementation is drastically slower than native version
// #[cfg(target_arch = "x86_64")]
// CpuExtensions::Sse4_1 | CpuExtensions::Sse2 => {
// sse2::multiply_alpha_sse2(src_image, dst_image)
// }
_ => native::multiply_alpha_native(src_image, dst_image),
}
Ok(())
}
@@ -82,9 +90,14 @@ impl MulDiv {
pub fn multiply_alpha_inplace(&self, image: &mut DstImageView) -> Result<(), MulDivImageError> {
self.assert_image(image)?;
match self.cpu_extensions {
CpuExtensions::Avx2 => mul::multiply_alpha_inplace_avx2(image),
// CpuExtensions::Sse2 => mul::multiply_alpha_sse2(src_image, dst_image),
_ => mul::multiply_alpha_inplace_native(image),
#[cfg(target_arch = "x86_64")]
CpuExtensions::Avx2 => avx2::multiply_alpha_inplace_avx2(image),
// WARNING: SSE2 implementation is drastically slower than native version
// #[cfg(target_arch = "x86_64")]
// CpuExtensions::Sse4_1 | CpuExtensions::Sse2 => {
// sse2::multiply_alpha_sse2(src_image, dst_image)
// }
_ => native::multiply_alpha_inplace_native(image),
}
Ok(())
}
@@ -98,11 +111,13 @@ impl MulDiv {
) -> Result<(), MulDivImagesError> {
self.assert_images(src_image, dst_image)?;
match self.cpu_extensions {
CpuExtensions::Avx2 => div::divide_alpha_avx2(src_image, dst_image),
#[cfg(target_arch = "x86_64")]
CpuExtensions::Avx2 => avx2::divide_alpha_avx2(src_image, dst_image),
#[cfg(target_arch = "x86_64")]
CpuExtensions::Sse4_1 | CpuExtensions::Sse2 => {
div::divide_alpha_sse2(src_image, dst_image)
sse2::divide_alpha_sse2(src_image, dst_image)
}
_ => div::divide_alpha_native(src_image, dst_image),
_ => native::divide_alpha_native(src_image, dst_image),
}
Ok(())
}
@@ -111,9 +126,11 @@ impl MulDiv {
pub fn divide_alpha_inplace(&self, image: &mut DstImageView) -> Result<(), MulDivImageError> {
self.assert_image(image)?;
match self.cpu_extensions {
CpuExtensions::Avx2 => div::divide_alpha_inplace_avx2(image),
CpuExtensions::Sse4_1 | CpuExtensions::Sse2 => div::divide_alpha_inplace_sse2(image),
_ => div::divide_alpha_inplace_native(image),
#[cfg(target_arch = "x86_64")]
CpuExtensions::Avx2 => avx2::divide_alpha_inplace_avx2(image),
#[cfg(target_arch = "x86_64")]
CpuExtensions::Sse4_1 | CpuExtensions::Sse2 => sse2::divide_alpha_inplace_sse2(image),
_ => native::divide_alpha_inplace_native(image),
}
Ok(())
}
-161
View File
@@ -1,161 +0,0 @@
use std::arch::x86_64::*;
use crate::simd_utils;
use crate::{DstImageView, SrcImageView};
pub(crate) fn multiply_alpha_avx2(src_image: &SrcImageView, dst_image: &mut DstImageView) {
let width = src_image.width().get() as usize;
let src_rows = src_image.iter_rows(0, src_image.height().get());
let dst_rows = dst_image.iter_rows_mut();
for (src_row, dst_row) in src_rows.zip(dst_rows) {
unsafe {
multiply_alpha_row_avx2(src_row, dst_row, width);
}
}
}
pub(crate) fn multiply_alpha_inplace_avx2(image: &mut DstImageView) {
let width = image.width().get() as usize;
for dst_row in image.iter_rows_mut() {
unsafe {
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
multiply_alpha_row_avx2(src_row, dst_row, width);
}
}
}
#[allow(dead_code)]
pub(crate) fn multiply_alpha_sse2(src_image: &SrcImageView, dst_image: &mut DstImageView) {
let width = src_image.width().get() as usize;
let src_rows = src_image.iter_rows(0, src_image.height().get());
let dst_rows = dst_image.iter_rows_mut();
for (src_row, dst_row) in src_rows.zip(dst_rows) {
unsafe {
multiply_alpha_row_sse2(src_row, dst_row, width);
}
}
}
pub(crate) fn multiply_alpha_native(src_image: &SrcImageView, dst_image: &mut DstImageView) {
let src_rows = src_image.iter_rows(0, src_image.height().get());
let dst_rows = dst_image.iter_rows_mut();
for (src_row, dst_row) in src_rows.zip(dst_rows) {
multiply_alpha_row_native(src_row, dst_row);
}
}
pub(crate) fn multiply_alpha_inplace_native(image: &mut DstImageView) {
for dst_row in image.iter_rows_mut() {
let src_row = unsafe { std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len()) };
multiply_alpha_row_native(src_row, dst_row);
}
}
/// https://github.com/Wizermil/premultiply_alpha/blob/master/premultiply_alpha/premultiply_alpha.hpp#L232
#[target_feature(enable = "avx2")]
unsafe fn multiply_alpha_row_avx2(src_row: &[u32], dst_row: &mut [u32], width: usize) {
let mask_alpha_color_odd_255 = _mm256_set1_epi32(0xff000000u32 as i32);
let div_255 = _mm256_set1_epi16(0x8081u16 as i16);
#[rustfmt::skip]
let mask_shuffle_alpha = _mm256_set_epi8(
15, -1, 15, -1, 11, -1, 11, -1, 7, -1, 7, -1, 3, -1, 3, -1,
15, -1, 15, -1, 11, -1, 11, -1, 7, -1, 7, -1, 3, -1, 3, -1,
);
#[rustfmt::skip]
let mask_shuffle_color_odd = _mm256_set_epi8(
-1, -1, 13, -1, -1, -1, 9, -1, -1, -1, 5, -1, -1, -1, 1, -1,
-1, -1, 13, -1, -1, -1, 9, -1, -1, -1, 5, -1, -1, -1, 1, -1,
);
let mut x: usize = 0;
while x < width.saturating_sub(7) {
let mut color = simd_utils::loadu_si256(src_row, x);
let alpha = _mm256_shuffle_epi8(color, mask_shuffle_alpha);
let mut color_even = _mm256_slli_epi16(color, 8);
let mut color_odd = _mm256_shuffle_epi8(color, mask_shuffle_color_odd);
color_odd = _mm256_or_si256(color_odd, mask_alpha_color_odd_255);
color_odd = _mm256_mulhi_epu16(color_odd, alpha);
color_even = _mm256_mulhi_epu16(color_even, alpha);
color_odd = _mm256_srli_epi16(_mm256_mulhi_epu16(color_odd, div_255), 7);
color_even = _mm256_srli_epi16(_mm256_mulhi_epu16(color_even, div_255), 7);
color = _mm256_or_si256(color_even, _mm256_slli_epi16(color_odd, 8));
let dst_ptr = dst_row.get_unchecked_mut(x..).as_mut_ptr() as *mut __m256i;
_mm256_storeu_si256(dst_ptr, color);
x += 8;
}
let src_tail = &src_row[x..];
let dst_tail = &mut dst_row[x..];
multiply_alpha_row_native(src_tail, dst_tail);
}
/// https://github.com/Wizermil/premultiply_alpha/blob/master/premultiply_alpha/premultiply_alpha.hpp#L108
/// This implementation is twice slowly than native version.
#[allow(dead_code)]
#[target_feature(enable = "sse2")]
unsafe fn multiply_alpha_row_sse2(src_row: &[u32], dst_row: &mut [u32], width: usize) {
let mask_alpha_color_odd_255 = _mm_set1_epi32(0xff000000u32 as i32);
let div_255 = _mm_set1_epi16(0x8081u16 as i16);
let mask_shuffle_alpha =
_mm_set_epi8(15, -1, 15, -1, 11, -1, 11, -1, 7, -1, 7, -1, 3, -1, 3, -1);
let mask_shuffle_color_odd =
_mm_set_epi8(-1, -1, 13, -1, -1, -1, 9, -1, -1, -1, 5, -1, -1, -1, 1, -1);
let mut x: usize = 0;
while x < width.saturating_sub(3) {
let mut color = simd_utils::loadu_si128(src_row, x);
let alpha = _mm_shuffle_epi8(color, mask_shuffle_alpha);
let mut color_even = _mm_slli_epi16(color, 8);
let mut color_odd = _mm_shuffle_epi8(color, mask_shuffle_color_odd);
color_odd = _mm_or_si128(color_odd, mask_alpha_color_odd_255);
color_odd = _mm_mulhi_epu16(color_odd, alpha);
color_even = _mm_mulhi_epu16(color_even, alpha);
color_odd = _mm_srli_epi16(_mm_mulhi_epu16(color_odd, div_255), 7);
color_even = _mm_srli_epi16(_mm_mulhi_epu16(color_even, div_255), 7);
color = _mm_or_si128(color_even, _mm_slli_epi16(color_odd, 8));
let dst_ptr = dst_row.get_unchecked_mut(x..).as_mut_ptr() as *mut __m128i;
_mm_storeu_si128(dst_ptr, color);
x += 4;
}
let src_tail = &src_row[x..];
let dst_tail = &mut dst_row[x..];
multiply_alpha_row_native(src_tail, dst_tail);
}
#[inline(always)]
fn multiply_alpha_row_native(src_row: &[u32], dst_row: &mut [u32]) {
for (src_pixel, dst_pixel) in src_row.iter().zip(dst_row) {
let components: [u8; 4] = src_pixel.to_le_bytes();
let alpha = components[3];
let res: [u8; 4] = [
mul_div_255(components[0], alpha),
mul_div_255(components[1], alpha),
mul_div_255(components[2], alpha),
alpha,
];
*dst_pixel = u32::from_le_bytes(res);
}
}
#[inline(always)]
fn mul_div_255(a: u8, b: u8) -> u8 {
let tmp = a as u32 * b as u32 + 128;
(((tmp >> 8) + tmp) >> 8) as u8
}
+42
View File
@@ -0,0 +1,42 @@
use crate::{DstImageView, SrcImageView};
pub(crate) fn divide_alpha_native(src_image: &SrcImageView, dst_image: &mut DstImageView) {
let src_rows = src_image.iter_rows(0, src_image.height().get());
let dst_rows = dst_image.iter_rows_mut();
for (src_row, dst_row) in src_rows.zip(dst_rows) {
divide_alpha_row_native(src_row, dst_row);
}
}
pub(crate) fn divide_alpha_inplace_native(image: &mut DstImageView) {
for dst_row in image.iter_rows_mut() {
let src_row = unsafe { std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len()) };
divide_alpha_row_native(src_row, dst_row);
}
}
#[inline(always)]
pub(crate) fn div_and_clip(v: u8, rev_alpha: f32) -> u8 {
let res = v as f32 * rev_alpha;
res.min(255.) as u8
}
#[inline(always)]
pub(crate) fn divide_alpha_row_native(src_row: &[u32], dst_row: &mut [u32]) {
src_row
.iter()
.zip(dst_row)
.for_each(|(src_pixel, dst_pixel)| {
let components: [u8; 4] = src_pixel.to_le_bytes();
let alpha = components[3];
let recip_alpha = if alpha == 0 { 0. } else { 255. / alpha as f32 };
let res = [
div_and_clip(components[0], recip_alpha),
div_and_clip(components[1], recip_alpha),
div_and_clip(components[2], recip_alpha),
alpha,
];
*dst_pixel = u32::from_le_bytes(res);
});
}
+7
View File
@@ -0,0 +1,7 @@
pub(crate) use div::{divide_alpha_inplace_native, divide_alpha_native, divide_alpha_row_native};
pub(crate) use mul::{
multiply_alpha_inplace_native, multiply_alpha_native, multiply_alpha_row_native,
};
mod div;
mod mul;
+38
View File
@@ -0,0 +1,38 @@
use crate::{DstImageView, SrcImageView};
pub(crate) fn multiply_alpha_native(src_image: &SrcImageView, dst_image: &mut DstImageView) {
let src_rows = src_image.iter_rows(0, src_image.height().get());
let dst_rows = dst_image.iter_rows_mut();
for (src_row, dst_row) in src_rows.zip(dst_rows) {
multiply_alpha_row_native(src_row, dst_row);
}
}
pub(crate) fn multiply_alpha_inplace_native(image: &mut DstImageView) {
for dst_row in image.iter_rows_mut() {
let src_row = unsafe { std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len()) };
multiply_alpha_row_native(src_row, dst_row);
}
}
#[inline(always)]
pub(crate) fn multiply_alpha_row_native(src_row: &[u32], dst_row: &mut [u32]) {
for (src_pixel, dst_pixel) in src_row.iter().zip(dst_row) {
let components: [u8; 4] = src_pixel.to_le_bytes();
let alpha = components[3];
let res: [u8; 4] = [
mul_div_255(components[0], alpha),
mul_div_255(components[1], alpha),
mul_div_255(components[2], alpha),
alpha,
];
*dst_pixel = u32::from_le_bytes(res);
}
}
#[inline(always)]
pub(crate) fn mul_div_255(a: u8, b: u8) -> u8 {
let tmp = a as u32 * b as u32 + 128;
(((tmp >> 8) + tmp) >> 8) as u8
}
+65
View File
@@ -0,0 +1,65 @@
use std::arch::x86_64::*;
use crate::alpha::native;
use crate::simd_utils;
use crate::{DstImageView, SrcImageView};
pub(crate) fn divide_alpha_sse2(src_image: &SrcImageView, dst_image: &mut DstImageView) {
let width = src_image.width().get() as usize;
let src_rows = src_image.iter_rows(0, src_image.height().get());
let dst_rows = dst_image.iter_rows_mut();
for (src_row, dst_row) in src_rows.zip(dst_rows) {
unsafe {
divide_alpha_row_sse2(src_row, dst_row, width);
}
}
}
pub(crate) fn divide_alpha_inplace_sse2(image: &mut DstImageView) {
let width = image.width().get() as usize;
for dst_row in image.iter_rows_mut() {
unsafe {
let src_row = std::slice::from_raw_parts(dst_row.as_ptr(), dst_row.len());
divide_alpha_row_sse2(src_row, dst_row, width);
}
}
}
#[target_feature(enable = "sse2")]
unsafe fn divide_alpha_row_sse2(src_row: &[u32], dst_row: &mut [u32], width: usize) {
let zero = _mm_setzero_si128();
let alpha_mask = _mm_set1_epi32(0xff000000u32 as i32);
let shuffle0 = _mm_set_epi8(5, 4, 5, 4, 5, 4, 5, 4, 1, 0, 1, 0, 1, 0, 1, 0);
let shuffle1 = _mm_set_epi8(13, 12, 13, 12, 13, 12, 13, 12, 9, 8, 9, 8, 9, 8, 9, 8);
let alpha_scale = _mm_set1_ps(255.0 * 256.0);
let mut x: usize = 0;
while x < width.saturating_sub(3) {
let mut source = simd_utils::loadu_si128(src_row, x);
let alpha = _mm_and_si128(source, alpha_mask);
let alpha_f32 = _mm_cvtepi32_ps(_mm_srli_epi32::<24>(source));
let scaled_recip_alpha_f32 = _mm_mul_ps(alpha_scale, _mm_rcp_ps(alpha_f32));
let scaled_recip_alpha_i32 = _mm_cvtps_epi32(scaled_recip_alpha_f32);
let mma0 = _mm_shuffle_epi8(scaled_recip_alpha_i32, shuffle0);
let mma1 = _mm_shuffle_epi8(scaled_recip_alpha_i32, shuffle1);
let mut pix0 = _mm_unpacklo_epi8(zero, source);
let mut pix1 = _mm_unpackhi_epi8(zero, source);
pix0 = _mm_mulhi_epu16(pix0, mma0);
pix1 = _mm_mulhi_epu16(pix1, mma1);
source = _mm_packus_epi16(pix0, pix1);
source = _mm_blendv_epi8(source, alpha, alpha_mask);
let dst_ptr = dst_row.get_unchecked_mut(x..).as_mut_ptr() as *mut __m128i;
_mm_storeu_si128(dst_ptr, source);
x += 4;
}
let src_tail = &src_row[x..];
let dst_tail = &mut dst_row[x..];
native::divide_alpha_row_native(src_tail, dst_tail);
}
+5
View File
@@ -0,0 +1,5 @@
pub(crate) use div::{divide_alpha_inplace_sse2, divide_alpha_sse2};
pub(crate) use mul::multiply_alpha_sse2;
mod div;
mod mul;
+59
View File
@@ -0,0 +1,59 @@
use std::arch::x86_64::*;
use crate::alpha::native;
use crate::simd_utils;
use crate::{DstImageView, SrcImageView};
#[allow(dead_code)]
pub(crate) fn multiply_alpha_sse2(src_image: &SrcImageView, dst_image: &mut DstImageView) {
let width = src_image.width().get() as usize;
let src_rows = src_image.iter_rows(0, src_image.height().get());
let dst_rows = dst_image.iter_rows_mut();
for (src_row, dst_row) in src_rows.zip(dst_rows) {
unsafe {
multiply_alpha_row_sse2(src_row, dst_row, width);
}
}
}
/// https://github.com/Wizermil/premultiply_alpha/blob/master/premultiply_alpha/premultiply_alpha.hpp#L108
/// This implementation is twice slowly than native version.
#[allow(dead_code)]
#[target_feature(enable = "sse2")]
unsafe fn multiply_alpha_row_sse2(src_row: &[u32], dst_row: &mut [u32], width: usize) {
let mask_alpha_color_odd_255 = _mm_set1_epi32(0xff000000u32 as i32);
let div_255 = _mm_set1_epi16(0x8081u16 as i16);
let mask_shuffle_alpha =
_mm_set_epi8(15, -1, 15, -1, 11, -1, 11, -1, 7, -1, 7, -1, 3, -1, 3, -1);
let mask_shuffle_color_odd =
_mm_set_epi8(-1, -1, 13, -1, -1, -1, 9, -1, -1, -1, 5, -1, -1, -1, 1, -1);
let mut x: usize = 0;
while x < width.saturating_sub(3) {
let mut color = simd_utils::loadu_si128(src_row, x);
let alpha = _mm_shuffle_epi8(color, mask_shuffle_alpha);
let mut color_even = _mm_slli_epi16::<8>(color);
let mut color_odd = _mm_shuffle_epi8(color, mask_shuffle_color_odd);
color_odd = _mm_or_si128(color_odd, mask_alpha_color_odd_255);
color_odd = _mm_mulhi_epu16(color_odd, alpha);
color_even = _mm_mulhi_epu16(color_even, alpha);
color_odd = _mm_srli_epi16::<7>(_mm_mulhi_epu16(color_odd, div_255));
color_even = _mm_srli_epi16::<7>(_mm_mulhi_epu16(color_even, div_255));
color = _mm_or_si128(color_even, _mm_slli_epi16::<8>(color_odd));
let dst_ptr = dst_row.get_unchecked_mut(x..).as_mut_ptr() as *mut __m128i;
_mm_storeu_si128(dst_ptr, color);
x += 4;
}
let src_tail = &src_row[x..];
let dst_tail = &mut dst_row[x..];
native::multiply_alpha_row_native(src_tail, dst_tail);
}
+4
View File
@@ -1,8 +1,10 @@
use std::num::NonZeroU32;
#[cfg(target_arch = "x86_64")]
pub use avx2::Avx2U8x4;
pub use filters::{get_filter_func, FilterType};
pub use native::{NativeF32, NativeI32, NativeU8x4};
#[cfg(target_arch = "x86_64")]
pub use sse4::Sse4U8x4;
use crate::image_view::{DstImageView, SrcImageView};
@@ -10,10 +12,12 @@ use crate::image_view::{DstImageView, SrcImageView};
#[macro_use]
mod macros;
#[cfg(target_arch = "x86_64")]
mod avx2;
mod filters;
mod native;
mod optimisations;
#[cfg(target_arch = "x86_64")]
mod sse4;
pub trait Convolution {
+21 -1
View File
@@ -7,12 +7,16 @@ use crate::image_view::{DstImageView, PixelType, SrcImageView};
#[derive(Debug, Clone, Copy)]
pub enum CpuExtensions {
None,
#[cfg(target_arch = "x86_64")]
Sse2,
#[cfg(target_arch = "x86_64")]
Sse4_1,
#[cfg(target_arch = "x86_64")]
Avx2,
}
impl Default for CpuExtensions {
#[cfg(target_arch = "x86_64")]
fn default() -> Self {
if is_x86_feature_detected!("avx2") {
Self::Avx2
@@ -24,9 +28,15 @@ impl Default for CpuExtensions {
Self::None
}
}
#[cfg(not(target_arch = "x86_64"))]
fn default() -> Self {
Self::None
}
}
impl CpuExtensions {
#[cfg(target_arch = "x86_64")]
#[inline]
fn get_resampler(&self, pixel_type: PixelType) -> &dyn Convolution {
match pixel_type {
@@ -39,6 +49,16 @@ impl CpuExtensions {
PixelType::F32 => &convolution::NativeF32,
}
}
#[cfg(not(target_arch = "x86_64"))]
#[inline]
fn get_resampler(&self, pixel_type: PixelType) -> &dyn Convolution {
match pixel_type {
PixelType::U8x4 => &convolution::NativeU8x4,
PixelType::I32 => &convolution::NativeI32,
PixelType::F32 => &convolution::NativeF32,
}
}
}
#[derive(Debug, Clone, Copy)]
@@ -67,7 +87,7 @@ pub struct Resizer {
impl Resizer {
/// Creates instance of `Resizer`
///
/// By default instance of `Resizer` created with best CPU-extensions provided by your CPU.
/// By default, instance of `Resizer` created with best CPU-extensions provided by your CPU.
/// You can change this by use method [Resizer::set_cpu_extensions].
pub fn new(algorithm: ResizeAlg) -> Self {
Self {