external/fast_image_resize
1
0
Fork 0
mirror of https://github.com/Cykooz/fast_image_resize.git synced 2026-10-08 01:11:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added support of optimisation with helps of NEON SIMD for convolution of U16x3 images.

Browse Source
This commit is contained in:
Kirill Kuzminykh
2022-11-22 23:50:56 +04:00
parent 781c168ba8
commit 788297ce67
8 changed files with 217 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+1
View File
@@ -4,6 +4,7 @@
- Added support of optimisation with helps of `NEON SIMD` for convolution of `U16` images.
- Added support of optimisation with helps of `NEON SIMD` for convolution of `U16x2` images.
- Added support of optimisation with helps of `NEON SIMD` for convolution of `U16x3` images.
- Improved optimisation of convolution with helps of `NEON SIMD` for `U8` images.
## [2.2.0] - 2022-11-18
Cargo.lock
Generated
+20 -21
View File
@@ -27,9 +27,9 @@ dependencies = [
[[package]]
name = "aho-corasick"
version = "0.7.19"
version = "0.7.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e"
checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac"
dependencies = [
"memchr",
]
@@ -145,9 +145,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]]
name = "cc"
version = "1.0.76"
version = "1.0.77"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76a284da2e6fe2092f2353e51713435363112dfd60030e22add80be333fb928f"
checksum = "e9f73505338f7d905b19d18738976aae232eb46b8efc15554ffc56deb5d9ebe4"
dependencies = [
"jobserver",
]
@@ -310,9 +310,9 @@ dependencies = [
[[package]]
name = "crossbeam-epoch"
version = "0.9.11"
version = "0.9.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f916dfc5d356b0ed9dae65f1db9fc9770aa2851d2662b988ccf4fe3516e86348"
checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a"
dependencies = [
"autocfg",
"cfg-if",
@@ -323,9 +323,9 @@ dependencies = [
[[package]]
name = "crossbeam-queue"
version = "0.3.6"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cd42583b04998a5363558e5f9291ee5a5ff6b49944332103f251e7479a82aa7"
checksum = "d1cfb3ea8a53f37c40dea2c7bedcbd88bdfae54f5e2175d6ecaff1c988353add"
dependencies = [
"cfg-if",
"crossbeam-utils",
@@ -333,9 +333,9 @@ dependencies = [
[[package]]
name = "crossbeam-utils"
version = "0.8.12"
version = "0.8.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edbafec5fa1f196ca66527c1b12c2ec4745ca14b50f1ad8f9f6f720b55d11fac"
checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f"
dependencies = [
"cfg-if",
]
@@ -913,9 +913,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
[[package]]
name = "memoffset"
version = "0.6.5"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
dependencies = [
"autocfg",
]
@@ -1038,9 +1038,9 @@ dependencies = [
[[package]]
name = "os_str_bytes"
version = "6.4.0"
version = "6.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b5bf27447411e9ee3ff51186bf7a08e16c341efdde93f4d823e8844429bed7e"
checksum = "9b7820b9daea5457c9f21c69448905d723fbd21136ccf521748f23fd49e723ee"
[[package]]
name = "parking_lot"
@@ -1168,11 +1168,10 @@ dependencies = [
[[package]]
name = "rayon"
version = "1.5.3"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d"
checksum = "1e060280438193c554f654141c9ea9417886713b7acd75974c85b18a69a88e0b"
dependencies = [
"autocfg",
"crossbeam-deque",
"either",
"rayon-core",
@@ -1180,9 +1179,9 @@ dependencies = [
[[package]]
name = "rayon-core"
version = "1.9.3"
version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f"
checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
@@ -1336,9 +1335,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.88"
version = "1.0.89"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e8b3801309262e8184d9687fb697586833e939767aea0dda89f5a8e650e8bd7"
checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db"
dependencies = [
"itoa 1.0.4",
"ryu",
README.md
+3 -3
View File
@@ -16,9 +16,9 @@ Supported pixel formats and available optimisations:
| U8x2 | Two `u8` components per pixel (e.g. LA) | + | + | + | + |
| U8x3 | Three `u8` components per pixel (e.g. RGB) | + | + | + | + |
| U8x4 | Four `u8` components per pixel (e.g. RGBA, RGBx, CMYK) | + | + | + | + |
| U16 | One `u16` components per pixel (e.g. L16) | + | + | + | - |
| U16x2 | Two `u16` components per pixel (e.g. LA16) | + | + | + | - |
| U16x3 | Three `u16` components per pixel (e.g. RGB16) | + | + | + | - |
| U16 | One `u16` components per pixel (e.g. L16) | + | + | + | + |
| U16x2 | Two `u16` components per pixel (e.g. LA16) | + | + | + | + |
| U16x3 | Three `u16` components per pixel (e.g. RGB16) | + | + | + | + |
| U16x4 | Four `u16` components per pixel (e.g. RGBA16, RGBx16, CMYK16) | + | + | + | + |
| I32 | One `i32` component per pixel | + | - | - | - |
| F32 | One `f32` component per pixel | + | - | - | - |
resizer/Cargo.toml
-1
View File
@@ -3,7 +3,6 @@ name = "resizer"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
fast_image_resize = {path=".."}
src/convolution/u16x3/mod.rs
+4
View File
@@ -8,6 +8,8 @@ use super::{Coefficients, Convolution};
#[cfg(target_arch = "x86_64")]
mod avx2;
mod native;
#[cfg(target_arch = "aarch64")]
mod neon;
#[cfg(target_arch = "x86_64")]
mod sse4;
@@ -24,6 +26,8 @@ impl Convolution for U16x3 {
CpuExtensions::Avx2 => avx2::horiz_convolution(src_image, dst_image, offset, coeffs),
#[cfg(target_arch = "x86_64")]
CpuExtensions::Sse4_1 => sse4::horiz_convolution(src_image, dst_image, offset, coeffs),
#[cfg(target_arch = "aarch64")]
CpuExtensions::Neon => neon::horiz_convolution(src_image, dst_image, offset, coeffs),
_ => native::horiz_convolution(src_image, dst_image, offset, coeffs),
}
}
src/convolution/u16x3/neon.rs
+160
View File
@@ -0,0 +1,160 @@
use std::arch::aarch64::*;
use crate::convolution::{optimisations, Coefficients};
use crate::neon_utils;
use crate::pixels::U16x3;
use crate::{ImageView, ImageViewMut};
#[inline]
pub(crate) fn horiz_convolution(
src_image: &ImageView<U16x3>,
dst_image: &mut ImageViewMut<U16x3>,
offset: u32,
coeffs: Coefficients,
) {
let normalizer = optimisations::Normalizer32::new(coeffs);
let precision = normalizer.precision();
let coefficients_chunks = normalizer.normalized_chunks();
let src_iter = src_image.iter_rows(offset);
let dst_iter = dst_image.iter_rows_mut();
for (src_row, dst_row) in src_iter.zip(dst_iter) {
unsafe {
horiz_convolution_row(src_row, dst_row, &coefficients_chunks, precision);
}
}
}
/// For safety, it is necessary to ensure the following conditions:
/// - bounds.len() == dst_row.len()
/// - coefficients_chunks.len() == dst_row.len()
/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.len()
/// - precision <= MAX_COEFS_PRECISION
#[target_feature(enable = "neon")]
unsafe fn horiz_convolution_row(
src_row: &[U16x3],
dst_row: &mut [U16x3],
coefficients_chunks: &[optimisations::CoefficientsI32Chunk],
precision: u8,
) {
let initial = vdupq_n_s64(1i64 << (precision - 2));
let zero_u16x8 = vdupq_n_u16(0);
let zero_u16x4 = vdup_n_u16(0);
for (dst_x, &coeffs_chunk) in coefficients_chunks.iter().enumerate() {
let mut x: usize = coeffs_chunk.start as usize;
let mut sss = [initial; 3];
let mut coeffs = coeffs_chunk.values;
let coeffs_by_8 = coeffs.chunks_exact(8);
coeffs = coeffs_by_8.remainder();
for k in coeffs_by_8 {
let coeffs_i32x4x2 = neon_utils::load_i32x4x2(k, 0);
let source = neon_utils::load_deintrel_u16x8x3(src_row, x);
sss[0] = conv_8_comp(sss[0], source.0, coeffs_i32x4x2, zero_u16x8);
sss[1] = conv_8_comp(sss[1], source.1, coeffs_i32x4x2, zero_u16x8);
sss[2] = conv_8_comp(sss[2], source.2, coeffs_i32x4x2, zero_u16x8);
x += 8;
}
let mut coeffs_by_4 = coeffs.chunks_exact(4);
coeffs = coeffs_by_4.remainder();
if let Some(k) = coeffs_by_4.next() {
let coeffs_i32x4 = neon_utils::load_i32x4(k, 0);
let source = neon_utils::load_deintrel_u16x4x3(src_row, x);
sss[0] = conv_4_comp(sss[0], source.0, coeffs_i32x4, zero_u16x4);
sss[1] = conv_4_comp(sss[1], source.1, coeffs_i32x4, zero_u16x4);
sss[2] = conv_4_comp(sss[2], source.2, coeffs_i32x4, zero_u16x4);
x += 4;
}
let mut coeffs_by_2 = coeffs.chunks_exact(2);
coeffs = coeffs_by_2.remainder();
if let Some(k) = coeffs_by_2.next() {
let coeffs_i32x2 = neon_utils::load_i32x2(k, 0);
let source = neon_utils::load_deintrel_u16x2x3(src_row, x);
sss[0] = conv_2_comp(sss[0], source.0, coeffs_i32x2, zero_u16x4);
sss[1] = conv_2_comp(sss[1], source.1, coeffs_i32x2, zero_u16x4);
sss[2] = conv_2_comp(sss[2], source.2, coeffs_i32x2, zero_u16x4);
x += 2;
}
if !coeffs.is_empty() {
let coeffs_i32x2 = neon_utils::load_i32x1(coeffs, 0);
let source = neon_utils::load_deintrel_u16x1x3(src_row, x);
sss[0] = conv_2_comp(sss[0], source.0, coeffs_i32x2, zero_u16x4);
sss[1] = conv_2_comp(sss[1], source.1, coeffs_i32x2, zero_u16x4);
sss[2] = conv_2_comp(sss[2], source.2, coeffs_i32x2, zero_u16x4);
}
let mut sss_i64 = [
vadd_s64(vget_low_s64(sss[0]), vget_high_s64(sss[0])),
vadd_s64(vget_low_s64(sss[1]), vget_high_s64(sss[1])),
vadd_s64(vget_low_s64(sss[2]), vget_high_s64(sss[2])),
];
macro_rules! call {
($imm8:expr) => {{
sss_i64[0] = vshr_n_s64::<$imm8>(sss_i64[0]);
sss_i64[1] = vshr_n_s64::<$imm8>(sss_i64[1]);
sss_i64[2] = vshr_n_s64::<$imm8>(sss_i64[2]);
}};
}
constify_64_imm8!(precision, call);
dst_row.get_unchecked_mut(dst_x).0 = [
vqmovns_u32(vqmovund_s64(vdupd_lane_s64::<0>(sss_i64[0]))),
vqmovns_u32(vqmovund_s64(vdupd_lane_s64::<0>(sss_i64[1]))),
vqmovns_u32(vqmovund_s64(vdupd_lane_s64::<0>(sss_i64[2]))),
];
}
}
#[inline(always)]
unsafe fn conv_8_comp(
mut sss: int64x2_t,
source: uint16x8_t,
coeffs: int32x4x2_t,
zero_u16x8: uint16x8_t,
) -> int64x2_t {
let pix_i32 = vreinterpretq_s32_u16(vzip1q_u16(source, zero_u16x8));
sss = vmlal_s32(sss, vget_low_s32(pix_i32), vget_low_s32(coeffs.0));
sss = vmlal_s32(sss, vget_high_s32(pix_i32), vget_high_s32(coeffs.0));
let pix_i32 = vreinterpretq_s32_u16(vzip2q_u16(source, zero_u16x8));
sss = vmlal_s32(sss, vget_low_s32(pix_i32), vget_low_s32(coeffs.1));
sss = vmlal_s32(sss, vget_high_s32(pix_i32), vget_high_s32(coeffs.1));
sss
}
#[inline(always)]
unsafe fn conv_4_comp(
mut sss: int64x2_t,
source: uint16x4_t,
coeffs: int32x4_t,
zero_u16x4: uint16x4_t,
) -> int64x2_t {
let pix_i32 = vreinterpret_s32_u16(vzip1_u16(source, zero_u16x4));
sss = vmlal_s32(sss, pix_i32, vget_low_s32(coeffs));
let pix_i32 = vreinterpret_s32_u16(vzip2_u16(source, zero_u16x4));
sss = vmlal_s32(sss, pix_i32, vget_high_s32(coeffs));
sss
}
#[inline(always)]
unsafe fn conv_2_comp(
mut sss: int64x2_t,
source: uint16x4_t,
coeffs: int32x2_t,
zero_u16x4: uint16x4_t,
) -> int64x2_t {
let pix_i32 = vreinterpret_s32_u16(vzip1_u16(source, zero_u16x4));
sss = vmlal_s32(sss, pix_i32, coeffs);
sss
}
src/neon_utils.rs
+28
View File
@@ -100,11 +100,39 @@ pub unsafe fn load_u16x8x4<T>(buf: &[T], index: usize) -> uint16x8x4_t {
vld1q_u16_x4(buf.get_unchecked(index..).as_ptr() as *const u16)
}
#[inline(always)]
pub unsafe fn load_deintrel_u16x1x3<T>(buf: &[T], index: usize) -> uint16x4x3_t {
let mut arr = [0u16; 12];
let src_ptr = buf.get_unchecked(index..).as_ptr() as *const u16;
let src_slice = std::slice::from_raw_parts(src_ptr, 3);
arr[0..3].copy_from_slice(src_slice);
vld3_u16(arr.as_ptr())
}
#[inline(always)]
pub unsafe fn load_deintrel_u16x2x3<T>(buf: &[T], index: usize) -> uint16x4x3_t {
let mut arr = [0u16; 12];
let src_ptr = buf.get_unchecked(index..).as_ptr() as *const u16;
let src_slice = std::slice::from_raw_parts(src_ptr, 6);
arr[0..6].copy_from_slice(src_slice);
vld3_u16(arr.as_ptr())
}
#[inline(always)]
pub unsafe fn load_deintrel_u16x4x3<T>(buf: &[T], index: usize) -> uint16x4x3_t {
vld3_u16(buf.get_unchecked(index..).as_ptr() as *const u16)
}
#[inline(always)]
pub unsafe fn load_deintrel_u16x4x4<T>(buf: &[T], index: usize) -> uint16x4x4_t {
vld4_u16(buf.get_unchecked(index..).as_ptr() as *const u16)
}
#[inline(always)]
pub unsafe fn load_deintrel_u16x8x3<T>(buf: &[T], index: usize) -> uint16x8x3_t {
vld3q_u16(buf.get_unchecked(index..).as_ptr() as *const u16)
}
#[inline(always)]
pub unsafe fn load_deintrel_u16x8x4<T>(buf: &[T], index: usize) -> uint16x8x4_t {
vld4q_u16(buf.get_unchecked(index..).as_ptr() as *const u16)
testing/Cargo.toml
+1 -2
View File
@@ -3,11 +3,10 @@ name = "testing"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
fast_image_resize = {path=".."}
image = "0.24.4"
image = "0.24.5"
[package.metadata.release]