mirror of
https://github.com/Cykooz/fast_image_resize.git
synced 2026-10-08 01:11:09 +00:00
Added support of optimisation with helps of NEON SIMD for convolution of U16 images.
This commit is contained in:
Kirill Kuzminykh
@@ -1,3 +1,9 @@
|
||||
## [Unreleased] - ReleaseDate
|
||||
|
||||
### Crate
|
||||
|
||||
- Added support of optimisation with helps of `NEON SIMD` for convolution of `U16` images.
|
||||
|
||||
## [2.2.0] - 2022-11-18
|
||||
|
||||
### Crate
|
||||
|
||||
Generated
+45
-45
@@ -133,9 +133,9 @@ checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba"
|
||||
|
||||
[[package]]
|
||||
name = "bytemuck"
|
||||
version = "1.12.1"
|
||||
version = "1.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2f5715e491b5a1598fc2bef5a606847b5dc1d48ea625bd3c02c00de8285591da"
|
||||
checksum = "aaa3a8d9a1ca92e282c96a32d6511b695d7d994d1d102ba85d279f9b2756947f"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
@@ -145,9 +145,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.73"
|
||||
version = "1.0.76"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
|
||||
checksum = "76a284da2e6fe2092f2353e51713435363112dfd60030e22add80be333fb928f"
|
||||
dependencies = [
|
||||
"jobserver",
|
||||
]
|
||||
@@ -160,9 +160,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
|
||||
[[package]]
|
||||
name = "chrono"
|
||||
version = "0.4.22"
|
||||
version = "0.4.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1"
|
||||
checksum = "16b0a3d9ed01224b22057780a37bb8c5dbfe1be8ba48678e7bf57ec4b385411f"
|
||||
dependencies = [
|
||||
"iana-time-zone",
|
||||
"js-sys",
|
||||
@@ -176,9 +176,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.0.18"
|
||||
version = "4.0.26"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "335867764ed2de42325fafe6d18b8af74ba97ee0c590fa016f157535b42ab04b"
|
||||
checksum = "2148adefda54e14492fb9bddcc600b4344c5d1a3123bd666dcb939c6f0e0e57e"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"bitflags",
|
||||
@@ -201,9 +201,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap_derive"
|
||||
version = "4.0.18"
|
||||
version = "4.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "16a1b0f6422af32d5da0c58e2703320f379216ee70198241c84173a8c5ac28f3"
|
||||
checksum = "0177313f9f02afc995627906bbd8967e2be069f5261954222dac78290c2b9014"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro-error",
|
||||
@@ -416,9 +416,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cxx"
|
||||
version = "1.0.80"
|
||||
version = "1.0.82"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6b7d4e43b25d3c994662706a1d4fcfc32aaa6afd287502c111b237093bb23f3a"
|
||||
checksum = "d4a41a86530d0fe7f5d9ea779916b7cadd2d4f9add748b99c2c029cbbdfaf453"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cxxbridge-flags",
|
||||
@@ -428,9 +428,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cxx-build"
|
||||
version = "1.0.80"
|
||||
version = "1.0.82"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "84f8829ddc213e2c1368e51a2564c552b65a8cb6a28f31e576270ac81d5e5827"
|
||||
checksum = "06416d667ff3e3ad2df1cd8cd8afae5da26cf9cec4d0825040f88b5ca659a2f0"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"codespan-reporting",
|
||||
@@ -443,15 +443,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cxxbridge-flags"
|
||||
version = "1.0.80"
|
||||
version = "1.0.82"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e72537424b474af1460806647c41d4b6d35d09ef7fe031c5c2fa5766047cc56a"
|
||||
checksum = "820a9a2af1669deeef27cb271f476ffd196a2c4b6731336011e0ba63e2c7cf71"
|
||||
|
||||
[[package]]
|
||||
name = "cxxbridge-macro"
|
||||
version = "1.0.80"
|
||||
version = "1.0.82"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "309e4fb93eed90e1e14bea0da16b209f81813ba9fc7830c20ed151dd7bc0a4d7"
|
||||
checksum = "a08a6e2fcc370a089ad3b4aaf54db3b1b4cee38ddabce5896b33eb693275f470"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -487,9 +487,9 @@ checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797"
|
||||
|
||||
[[package]]
|
||||
name = "env_logger"
|
||||
version = "0.9.1"
|
||||
version = "0.9.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c90bf5f19754d10198ccb95b70664fc925bd1fc090a0fd9a6ebc54acc8cd6272"
|
||||
checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"humantime",
|
||||
@@ -527,9 +527,9 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
|
||||
|
||||
[[package]]
|
||||
name = "fallible_collections"
|
||||
version = "0.4.5"
|
||||
version = "0.4.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c195cf4b2285d3c993eb887b4dc56b0d5728bbe1d0f9a99c0ac6bec2da3e4d85"
|
||||
checksum = "3f57ccc32870366ae684be48b32a1a2e196f98a42a9b4361fe77e13fd4a34755"
|
||||
dependencies = [
|
||||
"hashbrown 0.12.3",
|
||||
]
|
||||
@@ -725,9 +725,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.51"
|
||||
version = "0.1.53"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f5a6ef98976b22b3b7f2f3a806f858cb862044cfa66805aa3ad84cb3d3b785ed"
|
||||
checksum = "64c122667b287044802d6ce17ee2ddf13207ed924c712de9a66a5814d5b64765"
|
||||
dependencies = [
|
||||
"android_system_properties",
|
||||
"core-foundation-sys",
|
||||
@@ -759,9 +759,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "image"
|
||||
version = "0.24.4"
|
||||
version = "0.24.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bd8e4fb07cf672b1642304e731ef8a6a4c7891d67bb4fd4f5ce58cd6ed86803c"
|
||||
checksum = "69b7ea949b537b0fd0af141fff8c77690f2ce96f4f41f042ccb6c69c6c965945"
|
||||
dependencies = [
|
||||
"bytemuck",
|
||||
"byteorder",
|
||||
@@ -808,9 +808,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "jpeg-decoder"
|
||||
version = "0.2.6"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9478aa10f73e7528198d75109c8be5cd7d15fb530238040148d5f9a22d4c5b3b"
|
||||
checksum = "bc0000e42512c92e31c2252315bda326620a4e034105e900c98ec492fa077b3e"
|
||||
dependencies = [
|
||||
"rayon",
|
||||
]
|
||||
@@ -1012,9 +1012,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "num_cpus"
|
||||
version = "1.13.1"
|
||||
version = "1.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1"
|
||||
checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
@@ -1022,9 +1022,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.15.0"
|
||||
version = "1.16.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1"
|
||||
checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"
|
||||
|
||||
[[package]]
|
||||
name = "open"
|
||||
@@ -1038,9 +1038,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "os_str_bytes"
|
||||
version = "6.3.0"
|
||||
version = "6.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff"
|
||||
checksum = "7b5bf27447411e9ee3ff51186bf7a08e16c341efdde93f4d823e8844429bed7e"
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
@@ -1105,14 +1105,14 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
|
||||
|
||||
[[package]]
|
||||
name = "png"
|
||||
version = "0.17.6"
|
||||
version = "0.17.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f0e7f4c94ec26ff209cee506314212639d6c91b80afb82984819fafce9df01c"
|
||||
checksum = "5d708eaf860a19b19ce538740d2b4bdeeb8337fa53f7738455e706623ad5c638"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"crc32fast",
|
||||
"flate2",
|
||||
"miniz_oxide 0.5.4",
|
||||
"miniz_oxide 0.6.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1212,9 +1212,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.6.0"
|
||||
version = "1.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b"
|
||||
checksum = "e076559ef8e241f2ae3479e36f97bd5741c0330689e217ad51ce2c76808b868a"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
@@ -1229,9 +1229,9 @@ checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.6.27"
|
||||
version = "0.6.28"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
|
||||
checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
|
||||
|
||||
[[package]]
|
||||
name = "remove_dir_all"
|
||||
@@ -1336,9 +1336,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.87"
|
||||
version = "1.0.88"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ce777b7b150d76b9cf60d28b55f5847135a003f7d7350c6be7a773508ce7d45"
|
||||
checksum = "8e8b3801309262e8184d9687fb697586833e939767aea0dda89f5a8e650e8bd7"
|
||||
dependencies = [
|
||||
"itoa 1.0.4",
|
||||
"ryu",
|
||||
@@ -1495,9 +1495,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tiff"
|
||||
version = "0.7.3"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7259662e32d1e219321eb309d5f9d898b779769d81b76e762c07c8e5d38fcb65"
|
||||
checksum = "f17def29300a156c19ae30814710d9c63cd50288a49c6fd3a10ccfbe4cf886fd"
|
||||
dependencies = [
|
||||
"flate2",
|
||||
"jpeg-decoder",
|
||||
|
||||
+2
-2
@@ -26,10 +26,10 @@ thiserror = "1.0.37"
|
||||
|
||||
[dev-dependencies]
|
||||
glassbench = "0.3.3"
|
||||
image = "0.24.4"
|
||||
image = "0.24.5"
|
||||
resize = "0.7.4"
|
||||
rgb = "0.8.34"
|
||||
png = "0.17.6"
|
||||
png = "0.17.7"
|
||||
nix = { version = "0.25.0", default-features = false, features = ["sched"] }
|
||||
testing = {path= "testing" }
|
||||
|
||||
|
||||
+5
-5
@@ -7,13 +7,13 @@ edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
fast_image_resize = {path=".."}
|
||||
image = "0.24.4"
|
||||
clap = { version = "4.0.9", features = ["derive"] }
|
||||
image = "0.24.5"
|
||||
clap = { version = "4.0.26", features = ["derive"] }
|
||||
log = "0.4.17"
|
||||
env_logger = "0.9.1"
|
||||
anyhow = "1.0.65"
|
||||
env_logger = "0.9.3"
|
||||
anyhow = "1.0.66"
|
||||
clap-verbosity-flag = "2.0.0"
|
||||
once_cell = "1.15.0"
|
||||
once_cell = "1.16.0"
|
||||
|
||||
|
||||
[package.metadata.release]
|
||||
|
||||
@@ -8,6 +8,8 @@ use super::{Coefficients, Convolution};
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
mod avx2;
|
||||
mod native;
|
||||
#[cfg(target_arch = "aarch64")]
|
||||
mod neon;
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
mod sse4;
|
||||
|
||||
@@ -24,6 +26,8 @@ impl Convolution for U16 {
|
||||
CpuExtensions::Avx2 => avx2::horiz_convolution(src_image, dst_image, offset, coeffs),
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Sse4_1 => sse4::horiz_convolution(src_image, dst_image, offset, coeffs),
|
||||
#[cfg(target_arch = "aarch64")]
|
||||
CpuExtensions::Neon => neon::horiz_convolution(src_image, dst_image, offset, coeffs),
|
||||
_ => native::horiz_convolution(src_image, dst_image, offset, coeffs),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
use std::arch::aarch64::*;
|
||||
|
||||
use crate::convolution::{optimisations, Coefficients};
|
||||
use crate::image_view::{FourRows, FourRowsMut};
|
||||
use crate::neon_utils;
|
||||
use crate::pixels::U16;
|
||||
use crate::{ImageView, ImageViewMut};
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn horiz_convolution(
|
||||
src_image: &ImageView<U16>,
|
||||
dst_image: &mut ImageViewMut<U16>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
let normalizer = optimisations::Normalizer32::new(coeffs);
|
||||
let precision = normalizer.precision();
|
||||
let coefficients_chunks = normalizer.normalized_chunks();
|
||||
let dst_height = dst_image.height().get();
|
||||
|
||||
let src_iter = src_image.iter_4_rows(offset, dst_height + offset);
|
||||
let dst_iter = dst_image.iter_4_rows_mut();
|
||||
for (src_rows, dst_rows) in src_iter.zip(dst_iter) {
|
||||
unsafe {
|
||||
horiz_convolution_four_rows(src_rows, dst_rows, &coefficients_chunks, precision);
|
||||
}
|
||||
}
|
||||
|
||||
let mut yy = dst_height - dst_height % 4;
|
||||
while yy < dst_height {
|
||||
unsafe {
|
||||
horiz_convolution_row(
|
||||
src_image.get_row(yy + offset).unwrap(),
|
||||
dst_image.get_row_mut(yy).unwrap(),
|
||||
&coefficients_chunks,
|
||||
precision,
|
||||
);
|
||||
}
|
||||
yy += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/// For safety, it is necessary to ensure the following conditions:
|
||||
/// - length of all rows in src_rows must be equal
|
||||
/// - length of all rows in dst_rows must be equal
|
||||
/// - coefficients_chunks.len() == dst_rows.0.len()
|
||||
/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.0.len()
|
||||
/// - precision <= MAX_COEFS_PRECISION
|
||||
#[target_feature(enable = "neon")]
|
||||
unsafe fn horiz_convolution_four_rows(
|
||||
src_rows: FourRows<U16>,
|
||||
dst_rows: FourRowsMut<U16>,
|
||||
coefficients_chunks: &[optimisations::CoefficientsI32Chunk],
|
||||
precision: u8,
|
||||
) {
|
||||
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
|
||||
let s_rows = [s_row0, s_row1, s_row2, s_row3];
|
||||
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
|
||||
let d_rows = [d_row0, d_row1, d_row2, d_row3];
|
||||
|
||||
let initial = vdupq_n_s64(1i64 << (precision - 2));
|
||||
let zero_u16x4 = vdup_n_u16(0);
|
||||
|
||||
for (dst_x, coeffs_chunk) in coefficients_chunks.iter().enumerate() {
|
||||
let mut x: usize = coeffs_chunk.start as usize;
|
||||
let mut sss_a = [initial; 4];
|
||||
let mut coeffs = coeffs_chunk.values;
|
||||
|
||||
let coeffs_by_4 = coeffs.chunks_exact(4);
|
||||
coeffs = coeffs_by_4.remainder();
|
||||
for k in coeffs_by_4 {
|
||||
let coeffs_i32x4 = neon_utils::load_i32x4(k, 0);
|
||||
let coeff0 = vget_low_s32(coeffs_i32x4);
|
||||
let coeff1 = vget_high_s32(coeffs_i32x4);
|
||||
|
||||
for i in 0..4 {
|
||||
let source = neon_utils::load_u16x4(s_rows[i], x);
|
||||
let mut sss = sss_a[i];
|
||||
|
||||
let pix_i32 = vreinterpret_s32_u16(vzip1_u16(source, zero_u16x4));
|
||||
sss = vmlal_s32(sss, pix_i32, coeff0);
|
||||
let pix_i32 = vreinterpret_s32_u16(vzip2_u16(source, zero_u16x4));
|
||||
sss = vmlal_s32(sss, pix_i32, coeff1);
|
||||
|
||||
sss_a[i] = sss;
|
||||
}
|
||||
x += 4;
|
||||
}
|
||||
|
||||
let mut coeffs_by_2 = coeffs.chunks_exact(2);
|
||||
coeffs = coeffs_by_2.remainder();
|
||||
if let Some(k) = coeffs_by_2.next() {
|
||||
let coeffs_i32x2 = neon_utils::load_i32x2(k, 0);
|
||||
|
||||
for i in 0..4 {
|
||||
let source = neon_utils::load_u16x2(s_rows[i], x);
|
||||
let pix_i32 = vreinterpret_s32_u16(vzip1_u16(source, zero_u16x4));
|
||||
sss_a[i] = vmlal_s32(sss_a[i], pix_i32, coeffs_i32x2);
|
||||
}
|
||||
x += 2;
|
||||
}
|
||||
|
||||
if !coeffs.is_empty() {
|
||||
let coeffs_i32x2 = neon_utils::load_i32x1(coeffs, 0);
|
||||
for i in 0..4 {
|
||||
let source = neon_utils::load_u16x1(s_rows[i], x);
|
||||
let pix_i32 = vreinterpret_s32_u16(vzip1_u16(source, zero_u16x4));
|
||||
sss_a[i] = vmlal_s32(sss_a[i], pix_i32, coeffs_i32x2);
|
||||
}
|
||||
}
|
||||
|
||||
let mut sss_a_i64 = [
|
||||
vadd_s64(vget_low_s64(sss_a[0]), vget_high_s64(sss_a[0])),
|
||||
vadd_s64(vget_low_s64(sss_a[1]), vget_high_s64(sss_a[1])),
|
||||
vadd_s64(vget_low_s64(sss_a[2]), vget_high_s64(sss_a[2])),
|
||||
vadd_s64(vget_low_s64(sss_a[3]), vget_high_s64(sss_a[3])),
|
||||
];
|
||||
macro_rules! call {
|
||||
($imm8:expr) => {{
|
||||
sss_a_i64[0] = vshr_n_s64::<$imm8>(sss_a_i64[0]);
|
||||
sss_a_i64[1] = vshr_n_s64::<$imm8>(sss_a_i64[1]);
|
||||
sss_a_i64[2] = vshr_n_s64::<$imm8>(sss_a_i64[2]);
|
||||
sss_a_i64[3] = vshr_n_s64::<$imm8>(sss_a_i64[3]);
|
||||
}};
|
||||
}
|
||||
constify_64_imm8!(precision, call);
|
||||
|
||||
for i in 0..4 {
|
||||
let res = vdupd_lane_s64::<0>(sss_a_i64[i]);
|
||||
d_rows[i].get_unchecked_mut(dst_x).0 = vqmovns_u32(vqmovund_s64(res));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// For safety, it is necessary to ensure the following conditions:
|
||||
/// - bounds.len() == dst_row.len()
|
||||
/// - coefficients_chunks.len() == dst_row.len()
|
||||
/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.len()
|
||||
/// - precision <= MAX_COEFS_PRECISION
|
||||
#[target_feature(enable = "neon")]
|
||||
unsafe fn horiz_convolution_row(
|
||||
src_row: &[U16],
|
||||
dst_row: &mut [U16],
|
||||
coefficients_chunks: &[optimisations::CoefficientsI32Chunk],
|
||||
precision: u8,
|
||||
) {
|
||||
let initial = vdupq_n_s64(1i64 << (precision - 2));
|
||||
let zero_u16x8 = vdupq_n_u16(0);
|
||||
let zero_u16x4 = vdup_n_u16(0);
|
||||
|
||||
for (dst_x, &coeffs_chunk) in coefficients_chunks.iter().enumerate() {
|
||||
let mut x: usize = coeffs_chunk.start as usize;
|
||||
let mut sss = initial;
|
||||
let mut coeffs = coeffs_chunk.values;
|
||||
|
||||
let coeffs_by_8 = coeffs.chunks_exact(8);
|
||||
coeffs = coeffs_by_8.remainder();
|
||||
for k in coeffs_by_8 {
|
||||
let coeffs_i32x4x2 = neon_utils::load_i32x4x2(k, 0);
|
||||
let source = neon_utils::load_u16x8(src_row, x);
|
||||
|
||||
let pix_i32 = vreinterpretq_s32_u16(vzip1q_u16(source, zero_u16x8));
|
||||
sss = vmlal_s32(sss, vget_low_s32(pix_i32), vget_low_s32(coeffs_i32x4x2.0));
|
||||
sss = vmlal_s32(sss, vget_high_s32(pix_i32), vget_high_s32(coeffs_i32x4x2.0));
|
||||
|
||||
let pix_i32 = vreinterpretq_s32_u16(vzip2q_u16(source, zero_u16x8));
|
||||
sss = vmlal_s32(sss, vget_low_s32(pix_i32), vget_low_s32(coeffs_i32x4x2.1));
|
||||
sss = vmlal_s32(sss, vget_high_s32(pix_i32), vget_high_s32(coeffs_i32x4x2.1));
|
||||
|
||||
x += 8;
|
||||
}
|
||||
|
||||
let mut coeffs_by_4 = coeffs.chunks_exact(4);
|
||||
coeffs = coeffs_by_4.remainder();
|
||||
if let Some(k) = coeffs_by_4.next() {
|
||||
let coeffs_i32x4 = neon_utils::load_i32x4(k, 0);
|
||||
let source = neon_utils::load_u16x4(src_row, x);
|
||||
|
||||
let pix_i32 = vreinterpret_s32_u16(vzip1_u16(source, zero_u16x4));
|
||||
sss = vmlal_s32(sss, pix_i32, vget_low_s32(coeffs_i32x4));
|
||||
let pix_i32 = vreinterpret_s32_u16(vzip2_u16(source, zero_u16x4));
|
||||
sss = vmlal_s32(sss, pix_i32, vget_high_s32(coeffs_i32x4));
|
||||
|
||||
x += 4;
|
||||
}
|
||||
|
||||
let mut coeffs_by_2 = coeffs.chunks_exact(2);
|
||||
coeffs = coeffs_by_2.remainder();
|
||||
if let Some(k) = coeffs_by_2.next() {
|
||||
let coeffs_i32x2 = neon_utils::load_i32x2(k, 0);
|
||||
let source = neon_utils::load_u16x2(src_row, x);
|
||||
let pix_i32 = vreinterpret_s32_u16(vzip1_u16(source, zero_u16x4));
|
||||
sss = vmlal_s32(sss, pix_i32, coeffs_i32x2);
|
||||
x += 2;
|
||||
}
|
||||
|
||||
if !coeffs.is_empty() {
|
||||
let coeffs_i32x2 = neon_utils::load_i32x1(coeffs, 0);
|
||||
let source = neon_utils::load_u16x1(src_row, x);
|
||||
let pix_i32 = vreinterpret_s32_u16(vzip1_u16(source, zero_u16x4));
|
||||
sss = vmlal_s32(sss, pix_i32, coeffs_i32x2);
|
||||
}
|
||||
|
||||
let mut sss_i64 = vadd_s64(vget_low_s64(sss), vget_high_s64(sss));
|
||||
macro_rules! call {
|
||||
($imm8:expr) => {{
|
||||
sss_i64 = vshr_n_s64::<$imm8>(sss_i64);
|
||||
}};
|
||||
}
|
||||
constify_64_imm8!(precision, call);
|
||||
|
||||
let res = vdupd_lane_s64::<0>(sss_i64);
|
||||
dst_row.get_unchecked_mut(dst_x).0 = vqmovns_u32(vqmovund_s64(res));
|
||||
}
|
||||
}
|
||||
@@ -56,6 +56,18 @@ pub unsafe fn load_deintrel_u8x16x4<T>(buf: &[T], index: usize) -> uint8x16x4_t
|
||||
vld4q_u8(buf.get_unchecked(index..).as_ptr() as *const u8)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub unsafe fn load_u16x1<T>(buf: &[T], index: usize) -> uint16x4_t {
|
||||
let ptr = buf.get_unchecked(index..).as_ptr() as *const u16;
|
||||
vcreate_u16(*ptr as u64)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub unsafe fn load_u16x2<T>(buf: &[T], index: usize) -> uint16x4_t {
|
||||
let ptr = buf.get_unchecked(index..).as_ptr() as *const u32;
|
||||
vcreate_u16(*ptr as u64)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub unsafe fn load_u16x4<T>(buf: &[T], index: usize) -> uint16x4_t {
|
||||
vld1_u16(buf.get_unchecked(index..).as_ptr() as *const u16)
|
||||
@@ -86,6 +98,12 @@ pub unsafe fn load_deintrel_u16x8x4<T>(buf: &[T], index: usize) -> uint16x8x4_t
|
||||
vld4q_u16(buf.get_unchecked(index..).as_ptr() as *const u16)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub unsafe fn load_i32x1<T>(buf: &[T], index: usize) -> int32x2_t {
|
||||
let ptr = buf.get_unchecked(index..).as_ptr() as *const u32;
|
||||
vcreate_s32(*ptr as u64)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub unsafe fn load_i32x2<T>(buf: &[T], index: usize) -> int32x2_t {
|
||||
vld1_s32(buf.get_unchecked(index..).as_ptr() as *const i32)
|
||||
|
||||
Reference in New Issue
Block a user