mirror of
https://github.com/Cykooz/fast_image_resize.git
synced 2026-10-08 01:11:09 +00:00
Excluded possibility of unnecessary operations during resize of cropped image by convolution algorithm.
This commit is contained in:
Kirill Kuzminykh
@@ -3,6 +3,8 @@
|
||||
### Crate
|
||||
|
||||
- Improved speed of `MulDiv` implementation for `U8x4` images.
|
||||
- Excluded possibility of unnecessary operations during resize
|
||||
of cropped image by convolution algorithm.
|
||||
|
||||
## [2.3.0] - 2022-11-25
|
||||
|
||||
|
||||
Generated
+112
-42
@@ -84,7 +84,7 @@ version = "0.2.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"hermit-abi 0.1.19",
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
@@ -176,14 +176,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.0.26"
|
||||
version = "4.0.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2148adefda54e14492fb9bddcc600b4344c5d1a3123bd666dcb939c6f0e0e57e"
|
||||
checksum = "4d63b9e9c07271b9957ad22c173bae2a4d9a81127680962039296abcd2f8251d"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"bitflags",
|
||||
"clap_derive",
|
||||
"clap_lex",
|
||||
"is-terminal",
|
||||
"once_cell",
|
||||
"strsim",
|
||||
"termcolor",
|
||||
@@ -416,9 +416,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cxx"
|
||||
version = "1.0.82"
|
||||
version = "1.0.83"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d4a41a86530d0fe7f5d9ea779916b7cadd2d4f9add748b99c2c029cbbdfaf453"
|
||||
checksum = "bdf07d07d6531bfcdbe9b8b739b104610c6508dcc4d63b410585faf338241daf"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cxxbridge-flags",
|
||||
@@ -428,9 +428,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cxx-build"
|
||||
version = "1.0.82"
|
||||
version = "1.0.83"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06416d667ff3e3ad2df1cd8cd8afae5da26cf9cec4d0825040f88b5ca659a2f0"
|
||||
checksum = "d2eb5b96ecdc99f72657332953d4d9c50135af1bac34277801cc3937906ebd39"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"codespan-reporting",
|
||||
@@ -443,15 +443,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "cxxbridge-flags"
|
||||
version = "1.0.82"
|
||||
version = "1.0.83"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "820a9a2af1669deeef27cb271f476ffd196a2c4b6731336011e0ba63e2c7cf71"
|
||||
checksum = "ac040a39517fd1674e0f32177648334b0f4074625b5588a64519804ba0553b12"
|
||||
|
||||
[[package]]
|
||||
name = "cxxbridge-macro"
|
||||
version = "1.0.82"
|
||||
version = "1.0.83"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a08a6e2fcc370a089ad3b4aaf54db3b1b4cee38ddabce5896b33eb693275f470"
|
||||
checksum = "1362b0ddcfc4eb0a1f57b68bd77dd99f0e826958a96abd0ae9bd092e114ffed6"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -498,6 +498,27 @@ dependencies = [
|
||||
"termcolor",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "errno"
|
||||
version = "0.2.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1"
|
||||
dependencies = [
|
||||
"errno-dragonfly",
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "errno-dragonfly"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "exr"
|
||||
version = "1.5.2"
|
||||
@@ -508,7 +529,7 @@ dependencies = [
|
||||
"flume",
|
||||
"half",
|
||||
"lebe",
|
||||
"miniz_oxide 0.6.2",
|
||||
"miniz_oxide",
|
||||
"smallvec",
|
||||
"threadpool",
|
||||
]
|
||||
@@ -538,6 +559,7 @@ dependencies = [
|
||||
name = "fast_image_resize"
|
||||
version = "2.3.0"
|
||||
dependencies = [
|
||||
"fast_image_resize",
|
||||
"glassbench",
|
||||
"image",
|
||||
"nix",
|
||||
@@ -566,12 +588,12 @@ checksum = "9544f10105d33957765016b8a9baea7e689bf1f0f2f32c2fa2f568770c38d2b3"
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.0.24"
|
||||
version = "1.0.25"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f82b0f4c27ad9f8bfd1f3208d882da2b09c301bc1c828fd3a00d0216d2fbbff6"
|
||||
checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide 0.5.4",
|
||||
"miniz_oxide",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -717,6 +739,15 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hermit-abi"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "humantime"
|
||||
version = "2.1.0"
|
||||
@@ -785,6 +816,28 @@ dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "io-lifetimes"
|
||||
version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "46112a93252b123d31a119a8d1a1ac19deac4fac6e0e8b0df58f0d4e5870e63c"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "is-terminal"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "927609f78c2913a6f6ac3c27a4fe87f43e2a35367c0c4b0f8265e8f49a104330"
|
||||
dependencies = [
|
||||
"hermit-abi 0.2.6",
|
||||
"io-lifetimes",
|
||||
"rustix",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "0.4.8"
|
||||
@@ -838,9 +891,9 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.137"
|
||||
version = "0.2.138"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89"
|
||||
checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8"
|
||||
|
||||
[[package]]
|
||||
name = "libgit2-sys"
|
||||
@@ -886,6 +939,12 @@ dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f9f08d8963a6c613f4b1a78f4f4a4dbfadf8e6545b2d72861731e4858b8b47f"
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.9"
|
||||
@@ -929,15 +988,6 @@ dependencies = [
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.5.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34"
|
||||
dependencies = [
|
||||
"adler",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.6.2"
|
||||
@@ -970,14 +1020,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "nix"
|
||||
version = "0.25.0"
|
||||
version = "0.26.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e322c04a9e3440c327fca7b6c8a63e6890a32fa2ad689db972425f07e0d22abb"
|
||||
checksum = "46a58d1d356c6597d08cde02c2f09d785b09e28711837b1ed667dc652c08a694"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"bitflags",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"static_assertions",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1016,7 +1066,7 @@ version = "1.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"hermit-abi 0.1.19",
|
||||
"libc",
|
||||
]
|
||||
|
||||
@@ -1054,9 +1104,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.9.4"
|
||||
version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4dc9e0dc2adc1c69d09143aff38d3d30c5c3f0df0dad82e6d25547af174ebec0"
|
||||
checksum = "7ff9f3fef3968a3ec5945535ed654cb38ff72d7495a25619e2247fb15a2ed9ba"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
@@ -1112,7 +1162,7 @@ dependencies = [
|
||||
"bitflags",
|
||||
"crc32fast",
|
||||
"flate2",
|
||||
"miniz_oxide 0.6.2",
|
||||
"miniz_oxide",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1289,6 +1339,20 @@ dependencies = [
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "0.36.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cb93e85278e08bb5788653183213d3a60fc242b10cb9be96586f5a73dcb67c23"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"errno",
|
||||
"io-lifetimes",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ryu"
|
||||
version = "1.0.11"
|
||||
@@ -1315,18 +1379,18 @@ checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.147"
|
||||
version = "1.0.149"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965"
|
||||
checksum = "256b9932320c590e707b94576e3cc1f7c9024d0ee6612dfbcf1cb106cbe8e055"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.147"
|
||||
version = "1.0.149"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852"
|
||||
checksum = "b4eae9b04cbffdfd550eb462ed33bc6a1b68c935127d008b27444d08380f94e4"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -1389,6 +1453,12 @@ dependencies = [
|
||||
"lock_api",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "static_assertions"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.10.0"
|
||||
@@ -1409,9 +1479,9 @@ checksum = "e72d8b19ab05827afefcca66bf47040c1e66a0901eb814784c77d4ec118bd309"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.103"
|
||||
version = "1.0.105"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d"
|
||||
checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -1505,9 +1575,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "time"
|
||||
version = "0.1.44"
|
||||
version = "0.1.45"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255"
|
||||
checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"wasi 0.10.0+wasi-snapshot-preview1",
|
||||
|
||||
+7
-4
@@ -23,15 +23,18 @@ exclude = ["/data"]
|
||||
num-traits = "0.2.15"
|
||||
thiserror = "1.0.37"
|
||||
|
||||
[features]
|
||||
for_test = []
|
||||
|
||||
[dev-dependencies]
|
||||
fast_image_resize = { path = ".", features = ["for_test"] }
|
||||
glassbench = "0.3.3"
|
||||
image = "0.24.5"
|
||||
resize = "0.7.4"
|
||||
rgb = "0.8.34"
|
||||
png = "0.17.7"
|
||||
nix = { version = "0.25.0", default-features = false, features = ["sched"] }
|
||||
testing = {path= "testing" }
|
||||
nix = { version = "0.26.1", default-features = false, features = ["sched"] }
|
||||
testing = { path = "testing" }
|
||||
|
||||
|
||||
[[bench]]
|
||||
@@ -107,8 +110,8 @@ opt-level = 3
|
||||
|
||||
[package.metadata.release]
|
||||
pre-release-replacements = [
|
||||
{file="CHANGELOG.md", search="Unreleased", replace="{{version}}"},
|
||||
{file="CHANGELOG.md", search="ReleaseDate", replace="{{date}}"}
|
||||
{ file = "CHANGELOG.md", search = "Unreleased", replace = "{{version}}" },
|
||||
{ file = "CHANGELOG.md", search = "ReleaseDate", replace = "{{date}}" }
|
||||
]
|
||||
|
||||
# Header of next release in CHANGELOG.md:
|
||||
|
||||
@@ -20,9 +20,10 @@ impl Convolution for F32 {
|
||||
fn vert_convolution(
|
||||
src_image: &ImageView<Self>,
|
||||
dst_image: &mut ImageViewMut<Self>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
_cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
native::vert_convolution(src_image, dst_image, coeffs);
|
||||
native::vert_convolution(src_image, dst_image, offset, coeffs);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,20 +27,24 @@ pub(crate) fn horiz_convolution(
|
||||
pub(crate) fn vert_convolution(
|
||||
src_image: &ImageView<F32>,
|
||||
dst_image: &mut ImageViewMut<F32>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
let coefficients_chunks = coeffs.get_chunks();
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
let start_src_x = offset as usize;
|
||||
for (&coeffs_chunk, dst_row) in coefficients_chunks.iter().zip(dst_rows) {
|
||||
let first_y_src = coeffs_chunk.start;
|
||||
for (x_src, dst_pixel) in dst_row.iter_mut().enumerate() {
|
||||
let mut src_x = start_src_x;
|
||||
for dst_pixel in dst_row.iter_mut() {
|
||||
let mut ss = 0.;
|
||||
let src_rows = src_image.iter_rows(first_y_src);
|
||||
for (src_row, &k) in src_rows.zip(coeffs_chunk.values) {
|
||||
let src_pixel = unsafe { src_row.get_unchecked(x_src as usize) };
|
||||
let src_pixel = unsafe { src_row.get_unchecked(src_x) };
|
||||
ss += src_pixel.0 as f64 * k;
|
||||
}
|
||||
dst_pixel.0 = ss.round() as f32;
|
||||
src_x += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,9 +20,10 @@ impl Convolution for I32 {
|
||||
fn vert_convolution(
|
||||
src_image: &ImageView<Self>,
|
||||
dst_image: &mut ImageViewMut<Self>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
_cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
native::vert_convolution(src_image, dst_image, coeffs);
|
||||
native::vert_convolution(src_image, dst_image, offset, coeffs);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,20 +27,24 @@ pub(crate) fn horiz_convolution(
|
||||
pub(crate) fn vert_convolution(
|
||||
src_image: &ImageView<I32>,
|
||||
dst_image: &mut ImageViewMut<I32>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
let coefficients_chunks = coeffs.get_chunks();
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
let start_src_x = offset as usize;
|
||||
for (&coeffs_chunk, dst_row) in coefficients_chunks.iter().zip(dst_rows) {
|
||||
let first_y_src = coeffs_chunk.start;
|
||||
for (x_src, dst_pixel) in dst_row.iter_mut().enumerate() {
|
||||
let mut src_x = start_src_x;
|
||||
for dst_pixel in dst_row.iter_mut() {
|
||||
let mut ss = 0.;
|
||||
let src_rows = src_image.iter_rows(first_y_src);
|
||||
for (src_row, &k) in src_rows.zip(coeffs_chunk.values) {
|
||||
let src_pixel = unsafe { src_row.get_unchecked(x_src as usize) };
|
||||
let src_pixel = unsafe { src_row.get_unchecked(src_x) };
|
||||
ss += src_pixel.0 as f64 * k;
|
||||
}
|
||||
dst_pixel.0 = ss.round() as i32;
|
||||
src_x += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,6 +39,7 @@ where
|
||||
fn vert_convolution(
|
||||
src_image: &ImageView<Self>,
|
||||
dst_image: &mut ImageViewMut<Self>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
cpu_extensions: CpuExtensions,
|
||||
);
|
||||
|
||||
@@ -35,9 +35,10 @@ impl Convolution for U16 {
|
||||
fn vert_convolution(
|
||||
src_image: &ImageView<Self>,
|
||||
dst_image: &mut ImageViewMut<Self>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
vert_convolution_u16(src_image, dst_image, coeffs, cpu_extensions);
|
||||
vert_convolution_u16(src_image, dst_image, offset, coeffs, cpu_extensions);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,9 +35,10 @@ impl Convolution for U16x2 {
|
||||
fn vert_convolution(
|
||||
src_image: &ImageView<Self>,
|
||||
dst_image: &mut ImageViewMut<Self>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
vert_convolution_u16(src_image, dst_image, coeffs, cpu_extensions);
|
||||
vert_convolution_u16(src_image, dst_image, offset, coeffs, cpu_extensions);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,9 +35,10 @@ impl Convolution for U16x3 {
|
||||
fn vert_convolution(
|
||||
src_image: &ImageView<Self>,
|
||||
dst_image: &mut ImageViewMut<Self>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
vert_convolution_u16(src_image, dst_image, coeffs, cpu_extensions);
|
||||
vert_convolution_u16(src_image, dst_image, offset, coeffs, cpu_extensions);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,9 +35,10 @@ impl Convolution for U16x4 {
|
||||
fn vert_convolution(
|
||||
src_image: &ImageView<Self>,
|
||||
dst_image: &mut ImageViewMut<Self>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
vert_convolution_u16(src_image, dst_image, coeffs, cpu_extensions);
|
||||
vert_convolution_u16(src_image, dst_image, offset, coeffs, cpu_extensions);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,9 +35,10 @@ impl Convolution for U8 {
|
||||
fn vert_convolution(
|
||||
src_image: &ImageView<Self>,
|
||||
dst_image: &mut ImageViewMut<Self>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
vert_convolution_u8(src_image, dst_image, coeffs, cpu_extensions);
|
||||
vert_convolution_u8(src_image, dst_image, offset, coeffs, cpu_extensions);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,9 +35,10 @@ impl Convolution for U8x2 {
|
||||
fn vert_convolution(
|
||||
src_image: &ImageView<Self>,
|
||||
dst_image: &mut ImageViewMut<Self>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
vert_convolution_u8(src_image, dst_image, coeffs, cpu_extensions);
|
||||
vert_convolution_u8(src_image, dst_image, offset, coeffs, cpu_extensions);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,9 +35,10 @@ impl Convolution for U8x3 {
|
||||
fn vert_convolution(
|
||||
src_image: &ImageView<Self>,
|
||||
dst_image: &mut ImageViewMut<Self>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
vert_convolution_u8(src_image, dst_image, coeffs, cpu_extensions);
|
||||
vert_convolution_u8(src_image, dst_image, offset, coeffs, cpu_extensions);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,9 +35,10 @@ impl Convolution for U8x4 {
|
||||
fn vert_convolution(
|
||||
src_image: &ImageView<Self>,
|
||||
dst_image: &mut ImageViewMut<Self>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
vert_convolution_u8(src_image, dst_image, coeffs, cpu_extensions);
|
||||
vert_convolution_u8(src_image, dst_image, offset, coeffs, cpu_extensions);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,17 +8,19 @@ use crate::{ImageView, ImageViewMut};
|
||||
pub(crate) fn vert_convolution<T>(
|
||||
src_image: &ImageView<T>,
|
||||
dst_image: &mut ImageViewMut<T>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
) where
|
||||
T: PixelExt<Component = u16>,
|
||||
{
|
||||
let normalizer = optimisations::Normalizer32::new(coeffs);
|
||||
let coefficients_chunks = normalizer.normalized_chunks();
|
||||
let src_x = offset as usize * T::count_of_components();
|
||||
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
for (dst_row, coeffs_chunk) in dst_rows.zip(coefficients_chunks) {
|
||||
unsafe {
|
||||
vert_convolution_into_one_row_u16(src_image, dst_row, coeffs_chunk, &normalizer);
|
||||
vert_convolution_into_one_row_u16(src_image, dst_row, src_x, coeffs_chunk, &normalizer);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -27,16 +29,15 @@ pub(crate) fn vert_convolution<T>(
|
||||
pub(crate) unsafe fn vert_convolution_into_one_row_u16<T>(
|
||||
src_img: &ImageView<T>,
|
||||
dst_row: &mut [T],
|
||||
mut src_x: usize,
|
||||
coeffs_chunk: optimisations::CoefficientsI32Chunk,
|
||||
normalizer: &optimisations::Normalizer32,
|
||||
) where
|
||||
T: PixelExt<Component = u16>,
|
||||
{
|
||||
let mut xx: usize = 0;
|
||||
let src_width = src_img.width().get() as usize * T::count_of_components();
|
||||
let y_start = coeffs_chunk.start;
|
||||
let coeffs = coeffs_chunk.values;
|
||||
let dst_components = T::components_mut(dst_row);
|
||||
let mut dst_u16 = T::components_mut(dst_row);
|
||||
|
||||
/*
|
||||
|R G B | |R G B | |R G | - |B | |R G B | |R G B | |R |
|
||||
@@ -86,15 +87,16 @@ pub(crate) unsafe fn vert_convolution_into_one_row_u16<T>(
|
||||
let initial = _mm256_set1_epi64x(1 << (precision - 1));
|
||||
let mut comp_buf = [0i64; 4];
|
||||
|
||||
// 16 components in one register - 1 = 15
|
||||
while xx < src_width.saturating_sub(15) {
|
||||
// 16 components in one register
|
||||
let mut dst_chunks_16 = dst_u16.chunks_exact_mut(16);
|
||||
for dst_chunk in &mut dst_chunks_16 {
|
||||
// 16 components / 4 per register = 4 registers
|
||||
let mut sum = [initial; 4];
|
||||
|
||||
for (s_row, &coeff) in src_img.iter_rows(y_start).zip(coeffs) {
|
||||
let components = T::components(s_row);
|
||||
let coeff_i64x4 = _mm256_set1_epi64x(coeff as i64);
|
||||
let source = simd_utils::loadu_si256(components, xx);
|
||||
let source = simd_utils::loadu_si256(components, src_x);
|
||||
for i in 0..4 {
|
||||
let comp_i64x4 = _mm256_shuffle_epi8(source, shuffles[i]);
|
||||
sum[i] = _mm256_add_epi64(sum[i], _mm256_mul_epi32(comp_i64x4, coeff_i64x4));
|
||||
@@ -102,28 +104,34 @@ pub(crate) unsafe fn vert_convolution_into_one_row_u16<T>(
|
||||
}
|
||||
|
||||
for i in 0..4 {
|
||||
_mm256_storeu_si256((&mut comp_buf).as_mut_ptr() as *mut __m256i, sum[i]);
|
||||
let component = dst_components.get_unchecked_mut(xx + i * 2);
|
||||
_mm256_storeu_si256(comp_buf.as_mut_ptr() as *mut __m256i, sum[i]);
|
||||
let component = dst_chunk.get_unchecked_mut(i * 2);
|
||||
*component = normalizer.clip(comp_buf[0]);
|
||||
let component = dst_components.get_unchecked_mut(xx + i * 2 + 1);
|
||||
let component = dst_chunk.get_unchecked_mut(i * 2 + 1);
|
||||
*component = normalizer.clip(comp_buf[1]);
|
||||
let component = dst_components.get_unchecked_mut(xx + i * 2 + 8);
|
||||
let component = dst_chunk.get_unchecked_mut(i * 2 + 8);
|
||||
*component = normalizer.clip(comp_buf[2]);
|
||||
let component = dst_components.get_unchecked_mut(xx + i * 2 + 9);
|
||||
let component = dst_chunk.get_unchecked_mut(i * 2 + 9);
|
||||
*component = normalizer.clip(comp_buf[3]);
|
||||
}
|
||||
|
||||
xx += 16;
|
||||
src_x += 16;
|
||||
}
|
||||
|
||||
if xx < src_width {
|
||||
dst_u16 = dst_chunks_16.into_remainder();
|
||||
if !dst_u16.is_empty() {
|
||||
// 16 components / 4 per register = 4 registers
|
||||
let mut sum = [initial; 4];
|
||||
let mut buf = [0u16; 16];
|
||||
|
||||
for (s_row, &coeff) in src_img.iter_rows(y_start).zip(coeffs) {
|
||||
let components = T::components(s_row);
|
||||
for (i, &v) in components.get_unchecked(xx..).iter().enumerate() {
|
||||
for (i, &v) in components
|
||||
.get_unchecked(src_x..)
|
||||
.iter()
|
||||
.take(dst_u16.len())
|
||||
.enumerate()
|
||||
{
|
||||
buf[i] = v;
|
||||
}
|
||||
let coeff_i64x4 = _mm256_set1_epi64x(coeff as i64);
|
||||
@@ -135,7 +143,7 @@ pub(crate) unsafe fn vert_convolution_into_one_row_u16<T>(
|
||||
}
|
||||
|
||||
for i in 0..4 {
|
||||
_mm256_storeu_si256((&mut comp_buf).as_mut_ptr() as *mut __m256i, sum[i]);
|
||||
_mm256_storeu_si256(comp_buf.as_mut_ptr() as *mut __m256i, sum[i]);
|
||||
let component = buf.get_unchecked_mut(i * 2);
|
||||
*component = normalizer.clip(comp_buf[0]);
|
||||
let component = buf.get_unchecked_mut(i * 2 + 1);
|
||||
@@ -145,11 +153,7 @@ pub(crate) unsafe fn vert_convolution_into_one_row_u16<T>(
|
||||
let component = buf.get_unchecked_mut(i * 2 + 9);
|
||||
*component = normalizer.clip(comp_buf[3]);
|
||||
}
|
||||
for (i, v) in dst_components
|
||||
.get_unchecked_mut(xx..)
|
||||
.iter_mut()
|
||||
.enumerate()
|
||||
{
|
||||
for (i, v) in dst_u16.iter_mut().enumerate() {
|
||||
*v = buf[i];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,16 +14,21 @@ pub(crate) mod sse4;
|
||||
pub(crate) fn vert_convolution_u16<T: PixelExt<Component = u16>>(
|
||||
src_image: &ImageView<T>,
|
||||
dst_image: &mut ImageViewMut<T>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
// Check safety conditions
|
||||
debug_assert!(src_image.width().get() - offset >= dst_image.width().get());
|
||||
debug_assert_eq!(coeffs.bounds.len(), dst_image.height().get() as usize);
|
||||
|
||||
match cpu_extensions {
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Avx2 => avx2::vert_convolution(src_image, dst_image, coeffs),
|
||||
CpuExtensions::Avx2 => avx2::vert_convolution(src_image, dst_image, offset, coeffs),
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Sse4_1 => sse4::vert_convolution(src_image, dst_image, coeffs),
|
||||
CpuExtensions::Sse4_1 => sse4::vert_convolution(src_image, dst_image, offset, coeffs),
|
||||
#[cfg(target_arch = "aarch64")]
|
||||
CpuExtensions::Neon => neon::vert_convolution(src_image, dst_image, coeffs),
|
||||
_ => native::vert_convolution(src_image, dst_image, coeffs),
|
||||
CpuExtensions::Neon => neon::vert_convolution(src_image, dst_image, offset, coeffs),
|
||||
_ => native::vert_convolution(src_image, dst_image, offset, coeffs),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,16 +6,14 @@ use crate::{ImageView, ImageViewMut};
|
||||
pub(crate) fn vert_convolution<T: PixelExt<Component = u16>>(
|
||||
src_image: &ImageView<T>,
|
||||
dst_image: &mut ImageViewMut<T>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
// Check safety conditions
|
||||
debug_assert_eq!(src_image.width(), dst_image.width());
|
||||
debug_assert_eq!(coeffs.bounds.len(), dst_image.height().get() as usize);
|
||||
|
||||
let normalizer = optimisations::Normalizer32::new(coeffs);
|
||||
let coefficients_chunks = normalizer.normalized_chunks();
|
||||
let precision = normalizer.precision();
|
||||
let initial: i64 = 1 << (precision - 1);
|
||||
let x_src = offset as usize * T::count_of_components();
|
||||
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
let coeffs_chunks_iter = coefficients_chunks.into_iter();
|
||||
@@ -29,7 +27,7 @@ pub(crate) fn vert_convolution<T: PixelExt<Component = u16>>(
|
||||
&normalizer,
|
||||
initial,
|
||||
dst_components,
|
||||
0,
|
||||
x_src,
|
||||
first_y_src,
|
||||
ks,
|
||||
);
|
||||
@@ -46,7 +44,7 @@ pub(crate) fn convolution_by_u16<T: PixelExt<Component = u16>>(
|
||||
first_y_src: u32,
|
||||
ks: &[i32],
|
||||
) -> usize {
|
||||
for dst_component in dst_components.iter_mut().skip(x_src) {
|
||||
for dst_component in dst_components.iter_mut() {
|
||||
let mut ss = initial;
|
||||
let src_rows = src_image.iter_rows(first_y_src);
|
||||
for (&k, src_row) in ks.iter().zip(src_rows) {
|
||||
|
||||
@@ -9,16 +9,14 @@ use crate::{ImageView, ImageViewMut};
|
||||
pub(crate) fn vert_convolution<T: PixelExt<Component = u16>>(
|
||||
src_image: &ImageView<T>,
|
||||
dst_image: &mut ImageViewMut<T>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
// Check safety conditions
|
||||
debug_assert_eq!(src_image.width(), dst_image.width());
|
||||
debug_assert_eq!(coeffs.bounds.len(), dst_image.height().get() as usize);
|
||||
|
||||
let normalizer = optimisations::Normalizer32::new(coeffs);
|
||||
let coefficients_chunks = normalizer.normalized_chunks();
|
||||
let precision = normalizer.precision();
|
||||
let initial = 1i64 << (precision - 1);
|
||||
let start_src_x = offset as usize * T::count_of_components();
|
||||
|
||||
let mut tmp_dst = vec![0i64; dst_image.width().get() as usize * T::count_of_components()];
|
||||
let tmp_buf = tmp_dst.as_mut_slice();
|
||||
@@ -26,7 +24,7 @@ pub(crate) fn vert_convolution<T: PixelExt<Component = u16>>(
|
||||
for (dst_row, coeffs_chunk) in dst_rows.zip(coefficients_chunks) {
|
||||
tmp_buf.fill(initial);
|
||||
unsafe {
|
||||
vert_convolution_into_one_row_i64(src_image, tmp_buf, coeffs_chunk);
|
||||
vert_convolution_into_one_row_i64(src_image, tmp_buf, start_src_x, coeffs_chunk);
|
||||
let dst_comp = T::components_mut(dst_row);
|
||||
macro_rules! call {
|
||||
($imm8:expr) => {{
|
||||
@@ -42,6 +40,7 @@ pub(crate) fn vert_convolution<T: PixelExt<Component = u16>>(
|
||||
unsafe fn vert_convolution_into_one_row_i64<T: PixelExt<Component = u16>>(
|
||||
src_img: &ImageView<T>,
|
||||
dst_buf: &mut [i64],
|
||||
start_src_x: usize,
|
||||
coeffs_chunk: optimisations::CoefficientsI32Chunk,
|
||||
) {
|
||||
let width = dst_buf.len();
|
||||
@@ -55,65 +54,70 @@ unsafe fn vert_convolution_into_one_row_i64<T: PixelExt<Component = u16>>(
|
||||
let components = T::components(s_row);
|
||||
let coeff_i32x2 = vdup_n_s32(coeff);
|
||||
|
||||
let mut x: usize = 0;
|
||||
while x < width.saturating_sub(31) {
|
||||
let source = neon_utils::load_u16x8x4(components, x);
|
||||
let mut dst_x: usize = 0;
|
||||
let mut src_x = start_src_x;
|
||||
while dst_x < width.saturating_sub(31) {
|
||||
let source = neon_utils::load_u16x8x4(components, src_x);
|
||||
|
||||
for s in [source.0, source.1, source.2, source.3] {
|
||||
let mut accum = neon_utils::load_i64x2x4(dst_buf, x);
|
||||
let mut accum = neon_utils::load_i64x2x4(dst_buf, dst_x);
|
||||
let pix = vreinterpretq_s32_u16(vzip1q_u16(s, zero_u16x8));
|
||||
accum.0 = vmlal_s32(accum.0, vget_low_s32(pix), coeff_i32x2);
|
||||
accum.1 = vmlal_s32(accum.1, vget_high_s32(pix), coeff_i32x2);
|
||||
let pix = vreinterpretq_s32_u16(vzip2q_u16(s, zero_u16x8));
|
||||
accum.2 = vmlal_s32(accum.2, vget_low_s32(pix), coeff_i32x2);
|
||||
accum.3 = vmlal_s32(accum.3, vget_high_s32(pix), coeff_i32x2);
|
||||
neon_utils::store_i64x2x4(dst_buf, x, accum);
|
||||
x += 8;
|
||||
neon_utils::store_i64x2x4(dst_buf, dst_x, accum);
|
||||
dst_x += 8;
|
||||
src_x += 8;
|
||||
}
|
||||
}
|
||||
|
||||
if x < width.saturating_sub(15) {
|
||||
let source = neon_utils::load_u16x8x2(components, x);
|
||||
if dst_x < width.saturating_sub(15) {
|
||||
let source = neon_utils::load_u16x8x2(components, src_x);
|
||||
|
||||
for s in [source.0, source.1] {
|
||||
let mut accum = neon_utils::load_i64x2x4(dst_buf, x);
|
||||
let mut accum = neon_utils::load_i64x2x4(dst_buf, dst_x);
|
||||
let pix = vreinterpretq_s32_u16(vzip1q_u16(s, zero_u16x8));
|
||||
accum.0 = vmlal_s32(accum.0, vget_low_s32(pix), coeff_i32x2);
|
||||
accum.1 = vmlal_s32(accum.1, vget_high_s32(pix), coeff_i32x2);
|
||||
let pix = vreinterpretq_s32_u16(vzip2q_u16(s, zero_u16x8));
|
||||
accum.2 = vmlal_s32(accum.2, vget_low_s32(pix), coeff_i32x2);
|
||||
accum.3 = vmlal_s32(accum.3, vget_high_s32(pix), coeff_i32x2);
|
||||
neon_utils::store_i64x2x4(dst_buf, x, accum);
|
||||
x += 8;
|
||||
neon_utils::store_i64x2x4(dst_buf, dst_x, accum);
|
||||
dst_x += 8;
|
||||
src_x += 8;
|
||||
}
|
||||
}
|
||||
|
||||
if x < width.saturating_sub(7) {
|
||||
let s = neon_utils::load_u16x8(components, x);
|
||||
let mut accum = neon_utils::load_i64x2x4(dst_buf, x);
|
||||
if dst_x < width.saturating_sub(7) {
|
||||
let s = neon_utils::load_u16x8(components, src_x);
|
||||
let mut accum = neon_utils::load_i64x2x4(dst_buf, dst_x);
|
||||
let pix = vreinterpretq_s32_u16(vzip1q_u16(s, zero_u16x8));
|
||||
accum.0 = vmlal_s32(accum.0, vget_low_s32(pix), coeff_i32x2);
|
||||
accum.1 = vmlal_s32(accum.1, vget_high_s32(pix), coeff_i32x2);
|
||||
let pix = vreinterpretq_s32_u16(vzip2q_u16(s, zero_u16x8));
|
||||
accum.2 = vmlal_s32(accum.2, vget_low_s32(pix), coeff_i32x2);
|
||||
accum.3 = vmlal_s32(accum.3, vget_high_s32(pix), coeff_i32x2);
|
||||
neon_utils::store_i64x2x4(dst_buf, x, accum);
|
||||
x += 8;
|
||||
neon_utils::store_i64x2x4(dst_buf, dst_x, accum);
|
||||
dst_x += 8;
|
||||
src_x += 8;
|
||||
}
|
||||
|
||||
if x < width.saturating_sub(3) {
|
||||
let s = vcombine_u16(neon_utils::load_u16x4(components, x), zero_u16x4);
|
||||
let mut accum = neon_utils::load_i64x2x2(dst_buf, x);
|
||||
if dst_x < width.saturating_sub(3) {
|
||||
let s = vcombine_u16(neon_utils::load_u16x4(components, src_x), zero_u16x4);
|
||||
let mut accum = neon_utils::load_i64x2x2(dst_buf, dst_x);
|
||||
let pix = vreinterpretq_s32_u16(vzip1q_u16(s, zero_u16x8));
|
||||
accum.0 = vmlal_s32(accum.0, vget_low_s32(pix), coeff_i32x2);
|
||||
accum.1 = vmlal_s32(accum.1, vget_high_s32(pix), coeff_i32x2);
|
||||
neon_utils::store_i64x2x2(dst_buf, x, accum);
|
||||
x += 4;
|
||||
neon_utils::store_i64x2x2(dst_buf, dst_x, accum);
|
||||
dst_x += 4;
|
||||
src_x += 4;
|
||||
}
|
||||
|
||||
let coeff = coeff as i64;
|
||||
let tmp_tail = dst_buf.iter_mut().skip(x);
|
||||
let comp_tail = components.iter().skip(x);
|
||||
let tmp_tail = dst_buf.iter_mut().skip(dst_x);
|
||||
let comp_tail = components.iter().skip(src_x);
|
||||
for (accum, &comp) in tmp_tail.zip(comp_tail) {
|
||||
*accum += coeff * comp as i64;
|
||||
}
|
||||
|
||||
@@ -10,15 +10,17 @@ use crate::{ImageView, ImageViewMut};
|
||||
pub(crate) fn vert_convolution<T: PixelExt<Component = u16>>(
|
||||
src_image: &ImageView<T>,
|
||||
dst_image: &mut ImageViewMut<T>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
let normalizer = optimisations::Normalizer32::new(coeffs);
|
||||
let coefficients_chunks = normalizer.normalized_chunks();
|
||||
let src_x = offset as usize * T::count_of_components();
|
||||
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
for (dst_row, coeffs_chunk) in dst_rows.zip(coefficients_chunks) {
|
||||
unsafe {
|
||||
vert_convolution_into_one_row_u16(src_image, dst_row, coeffs_chunk, &normalizer);
|
||||
vert_convolution_into_one_row_u16(src_image, dst_row, src_x, coeffs_chunk, &normalizer);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -27,16 +29,14 @@ pub(crate) fn vert_convolution<T: PixelExt<Component = u16>>(
|
||||
unsafe fn vert_convolution_into_one_row_u16<T: PixelExt<Component = u16>>(
|
||||
src_img: &ImageView<T>,
|
||||
dst_row: &mut [T],
|
||||
mut src_x: usize,
|
||||
coeffs_chunk: CoefficientsI32Chunk,
|
||||
normalizer: &optimisations::Normalizer32,
|
||||
) {
|
||||
let mut xx: usize = 0;
|
||||
let src_width = src_img.width().get() as usize * T::count_of_components();
|
||||
let y_start = coeffs_chunk.start;
|
||||
let coeffs = coeffs_chunk.values;
|
||||
let max_y = y_start + coeffs.len() as u32;
|
||||
let dst_components = T::components_mut(dst_row);
|
||||
let mut dst_ptr_u16 = dst_components.as_mut_ptr() as *mut u16;
|
||||
let mut dst_u16 = T::components_mut(dst_row);
|
||||
|
||||
/*
|
||||
|0 1 2 3 4 5 6 7 |
|
||||
@@ -69,8 +69,8 @@ unsafe fn vert_convolution_into_one_row_u16<T: PixelExt<Component = u16>>(
|
||||
let initial = _mm_set1_epi64x(1 << (precision - 1));
|
||||
let mut c_buf = [0i64; 2];
|
||||
|
||||
// 16 components - 1 = 15
|
||||
while xx < src_width.saturating_sub(15) {
|
||||
let mut dst_chunks_16 = dst_u16.chunks_exact_mut(16);
|
||||
for dst_chunk in &mut dst_chunks_16 {
|
||||
let mut sums = [[initial; 2], [initial; 2], [initial; 2], [initial; 2]];
|
||||
|
||||
let mut y: u32 = 0;
|
||||
@@ -83,7 +83,7 @@ unsafe fn vert_convolution_into_one_row_u16<T: PixelExt<Component = u16>>(
|
||||
for r in 0..2 {
|
||||
let coeff_i64x2 = _mm_set1_epi64x(two_coeffs[r] as i64);
|
||||
for x in 0..2 {
|
||||
let source = simd_utils::loadu_si128(src_rows[r], xx + x * 8);
|
||||
let source = simd_utils::loadu_si128(src_rows[r], src_x + x * 8);
|
||||
for i in 0..4 {
|
||||
let c_i64x2 = _mm_shuffle_epi8(source, c_shuffles[i]);
|
||||
sums[i][x] = _mm_add_epi64(sums[i][x], _mm_mul_epi32(c_i64x2, coeff_i64x2));
|
||||
@@ -99,7 +99,7 @@ unsafe fn vert_convolution_into_one_row_u16<T: PixelExt<Component = u16>>(
|
||||
let coeff_i64x2 = _mm_set1_epi64x(k as i64);
|
||||
|
||||
for x in 0..2 {
|
||||
let source = simd_utils::loadu_si128(components, xx + x * 8);
|
||||
let source = simd_utils::loadu_si128(components, src_x + x * 8);
|
||||
for i in 0..4 {
|
||||
let c_i64x2 = _mm_shuffle_epi8(source, c_shuffles[i]);
|
||||
sums[i][x] = _mm_add_epi64(sums[i][x], _mm_mul_epi32(c_i64x2, coeff_i64x2));
|
||||
@@ -107,21 +107,23 @@ unsafe fn vert_convolution_into_one_row_u16<T: PixelExt<Component = u16>>(
|
||||
}
|
||||
}
|
||||
|
||||
let mut dst_ptr = dst_chunk.as_mut_ptr();
|
||||
for x in 0..2 {
|
||||
for sum in sums {
|
||||
_mm_storeu_si128((&mut c_buf).as_mut_ptr() as *mut __m128i, sum[x]);
|
||||
*dst_ptr_u16 = normalizer.clip(c_buf[0]);
|
||||
dst_ptr_u16 = dst_ptr_u16.add(1);
|
||||
*dst_ptr_u16 = normalizer.clip(c_buf[1]);
|
||||
dst_ptr_u16 = dst_ptr_u16.add(1);
|
||||
*dst_ptr = normalizer.clip(c_buf[0]);
|
||||
dst_ptr = dst_ptr.add(1);
|
||||
*dst_ptr = normalizer.clip(c_buf[1]);
|
||||
dst_ptr = dst_ptr.add(1);
|
||||
}
|
||||
}
|
||||
|
||||
xx += 16;
|
||||
src_x += 16;
|
||||
}
|
||||
|
||||
// 8 components - 1 = 7
|
||||
while xx < src_width.saturating_sub(7) {
|
||||
dst_u16 = dst_chunks_16.into_remainder();
|
||||
let mut dst_chunks_8 = dst_u16.chunks_exact_mut(8);
|
||||
if let Some(dst_chunk) = dst_chunks_8.next() {
|
||||
let mut sums = [initial, initial, initial, initial];
|
||||
|
||||
let mut y: u32 = 0;
|
||||
@@ -136,7 +138,7 @@ unsafe fn vert_convolution_into_one_row_u16<T: PixelExt<Component = u16>>(
|
||||
];
|
||||
|
||||
for r in 0..2 {
|
||||
let source = simd_utils::loadu_si128(src_rows[r], xx);
|
||||
let source = simd_utils::loadu_si128(src_rows[r], src_x);
|
||||
for i in 0..4 {
|
||||
let c_i64x2 = _mm_shuffle_epi8(source, c_shuffles[i]);
|
||||
sums[i] = _mm_add_epi64(sums[i], _mm_mul_epi32(c_i64x2, coeffs_i64[r]));
|
||||
@@ -149,30 +151,32 @@ unsafe fn vert_convolution_into_one_row_u16<T: PixelExt<Component = u16>>(
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let components = T::components(s_row);
|
||||
let coeff_i64x2 = _mm_set1_epi64x(k as i64);
|
||||
let source = simd_utils::loadu_si128(components, xx);
|
||||
let source = simd_utils::loadu_si128(components, src_x);
|
||||
for i in 0..4 {
|
||||
let c_i64x2 = _mm_shuffle_epi8(source, c_shuffles[i]);
|
||||
sums[i] = _mm_add_epi64(sums[i], _mm_mul_epi32(c_i64x2, coeff_i64x2));
|
||||
}
|
||||
}
|
||||
|
||||
let mut dst_ptr = dst_chunk.as_mut_ptr();
|
||||
for sum in sums {
|
||||
// let mask = _mm_cmpgt_epi64(sums[i], zero);
|
||||
// sums[i] = _mm_and_si128(sums[i] , mask);
|
||||
// sums[i] = _mm_srl_epi64(sums[i] , precision_i64);
|
||||
// _mm_packus_epi32(sums[i] , sums[i] );
|
||||
_mm_storeu_si128((&mut c_buf).as_mut_ptr() as *mut __m128i, sum);
|
||||
*dst_ptr_u16 = normalizer.clip(c_buf[0]);
|
||||
dst_ptr_u16 = dst_ptr_u16.add(1);
|
||||
*dst_ptr_u16 = normalizer.clip(c_buf[1]);
|
||||
dst_ptr_u16 = dst_ptr_u16.add(1);
|
||||
*dst_ptr = normalizer.clip(c_buf[0]);
|
||||
dst_ptr = dst_ptr.add(1);
|
||||
*dst_ptr = normalizer.clip(c_buf[1]);
|
||||
dst_ptr = dst_ptr.add(1);
|
||||
}
|
||||
|
||||
xx += 8;
|
||||
src_x += 8;
|
||||
}
|
||||
|
||||
// 4 components - 1 = 3
|
||||
while xx < src_width.saturating_sub(3) {
|
||||
dst_u16 = dst_chunks_8.into_remainder();
|
||||
let mut dst_chunks_4 = dst_u16.chunks_exact_mut(4);
|
||||
if let Some(dst_chunk) = dst_chunks_4.next() {
|
||||
let mut c01 = initial;
|
||||
let mut c23 = initial;
|
||||
let mut y: u32 = 0;
|
||||
@@ -186,7 +190,7 @@ unsafe fn vert_convolution_into_one_row_u16<T: PixelExt<Component = u16>>(
|
||||
_mm_set1_epi64x(two_coeffs[1] as i64),
|
||||
];
|
||||
for r in 0..2 {
|
||||
let comp_x4 = src_rows[r].get_unchecked(xx..xx + 4);
|
||||
let comp_x4 = src_rows[r].get_unchecked(src_x..src_x + 4);
|
||||
let c_i64x2 = _mm_set_epi64x(comp_x4[1] as i64, comp_x4[0] as i64);
|
||||
c01 = _mm_add_epi64(c01, _mm_mul_epi32(c_i64x2, coeffs_i64[r]));
|
||||
let c_i64x2 = _mm_set_epi64x(comp_x4[3] as i64, comp_x4[2] as i64);
|
||||
@@ -200,37 +204,32 @@ unsafe fn vert_convolution_into_one_row_u16<T: PixelExt<Component = u16>>(
|
||||
let components = T::components(s_row);
|
||||
let coeff_i64x2 = _mm_set1_epi64x(k as i64);
|
||||
|
||||
let comp_x4 = components.get_unchecked(xx..xx + 4);
|
||||
let comp_x4 = components.get_unchecked(src_x..src_x + 4);
|
||||
let c_i64x2 = _mm_set_epi64x(comp_x4[1] as i64, comp_x4[0] as i64);
|
||||
c01 = _mm_add_epi64(c01, _mm_mul_epi32(c_i64x2, coeff_i64x2));
|
||||
let c_i64x2 = _mm_set_epi64x(comp_x4[3] as i64, comp_x4[2] as i64);
|
||||
c23 = _mm_add_epi64(c23, _mm_mul_epi32(c_i64x2, coeff_i64x2));
|
||||
}
|
||||
|
||||
let mut dst_ptr = dst_chunk.as_mut_ptr();
|
||||
_mm_storeu_si128((&mut c_buf).as_mut_ptr() as *mut __m128i, c01);
|
||||
*dst_ptr_u16 = normalizer.clip(c_buf[0]);
|
||||
dst_ptr_u16 = dst_ptr_u16.add(1);
|
||||
*dst_ptr_u16 = normalizer.clip(c_buf[1]);
|
||||
dst_ptr_u16 = dst_ptr_u16.add(1);
|
||||
*dst_ptr = normalizer.clip(c_buf[0]);
|
||||
dst_ptr = dst_ptr.add(1);
|
||||
*dst_ptr = normalizer.clip(c_buf[1]);
|
||||
dst_ptr = dst_ptr.add(1);
|
||||
_mm_storeu_si128((&mut c_buf).as_mut_ptr() as *mut __m128i, c23);
|
||||
*dst_ptr_u16 = normalizer.clip(c_buf[0]);
|
||||
dst_ptr_u16 = dst_ptr_u16.add(1);
|
||||
*dst_ptr_u16 = normalizer.clip(c_buf[1]);
|
||||
dst_ptr_u16 = dst_ptr_u16.add(1);
|
||||
*dst_ptr = normalizer.clip(c_buf[0]);
|
||||
dst_ptr = dst_ptr.add(1);
|
||||
*dst_ptr = normalizer.clip(c_buf[1]);
|
||||
|
||||
xx += 4;
|
||||
src_x += 4;
|
||||
}
|
||||
|
||||
if xx < src_width {
|
||||
dst_u16 = dst_chunks_4.into_remainder();
|
||||
if !dst_u16.is_empty() {
|
||||
let initial = 1 << (precision - 1);
|
||||
convolution_by_u16(
|
||||
src_img,
|
||||
normalizer,
|
||||
initial,
|
||||
dst_components,
|
||||
xx,
|
||||
y_start,
|
||||
coeffs,
|
||||
src_img, normalizer, initial, dst_u16, src_x, y_start, coeffs,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use std::arch::x86_64::*;
|
||||
|
||||
use crate::convolution::vertical_u8::native;
|
||||
use crate::convolution::{optimisations, Coefficients};
|
||||
use crate::pixels::PixelExt;
|
||||
use crate::simd_utils;
|
||||
@@ -9,17 +10,19 @@ use crate::{ImageView, ImageViewMut};
|
||||
pub(crate) fn vert_convolution<T>(
|
||||
src_image: &ImageView<T>,
|
||||
dst_image: &mut ImageViewMut<T>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
) where
|
||||
T: PixelExt<Component = u8>,
|
||||
{
|
||||
let normalizer = optimisations::Normalizer16::new(coeffs);
|
||||
let coefficients_chunks = normalizer.normalized_chunks();
|
||||
let src_x = offset as usize * T::count_of_components();
|
||||
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
for (dst_row, coeffs_chunk) in dst_rows.zip(coefficients_chunks) {
|
||||
unsafe {
|
||||
vert_convolution_into_one_row_u8(src_image, dst_row, coeffs_chunk, &normalizer);
|
||||
vert_convolution_into_one_row_u8(src_image, dst_row, src_x, coeffs_chunk, &normalizer);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -29,12 +32,12 @@ pub(crate) fn vert_convolution<T>(
|
||||
unsafe fn vert_convolution_into_one_row_u8<T>(
|
||||
src_img: &ImageView<T>,
|
||||
dst_row: &mut [T],
|
||||
mut src_x: usize,
|
||||
coeffs_chunk: optimisations::CoefficientsI16Chunk,
|
||||
normalizer: &optimisations::Normalizer16,
|
||||
) where
|
||||
T: PixelExt<Component = u8>,
|
||||
{
|
||||
let src_width = src_img.width().get() as usize * T::count_of_components();
|
||||
let y_start = coeffs_chunk.start;
|
||||
let coeffs = coeffs_chunk.values;
|
||||
let max_y = y_start + coeffs.len() as u32;
|
||||
@@ -43,11 +46,11 @@ unsafe fn vert_convolution_into_one_row_u8<T>(
|
||||
let initial = _mm_set1_epi32(1 << (precision - 1));
|
||||
let initial_256 = _mm256_set1_epi32(1 << (precision - 1));
|
||||
|
||||
let mut x_in_bytes: usize = 0;
|
||||
let dst_ptr_u8 = T::components_mut(dst_row).as_mut_ptr() as *mut u8;
|
||||
let mut dst_u8 = T::components_mut(dst_row);
|
||||
|
||||
// 32 components in one register - 1 = 31
|
||||
while x_in_bytes < src_width.saturating_sub(31) {
|
||||
// 32 components in one register
|
||||
let mut dst_chunks_32 = dst_u8.chunks_exact_mut(32);
|
||||
for dst_chunk in &mut dst_chunks_32 {
|
||||
let mut sss0 = initial_256;
|
||||
let mut sss1 = initial_256;
|
||||
let mut sss2 = initial_256;
|
||||
@@ -62,8 +65,8 @@ unsafe fn vert_convolution_into_one_row_u8<T>(
|
||||
// Load two coefficients at once
|
||||
let mmk = simd_utils::ptr_i16_to_256set1_epi32(coeffs, y as usize);
|
||||
|
||||
let source1 = simd_utils::loadu_si256(components1, x_in_bytes); // top line
|
||||
let source2 = simd_utils::loadu_si256(components2, x_in_bytes); // bottom line
|
||||
let source1 = simd_utils::loadu_si256(components1, src_x); // top line
|
||||
let source2 = simd_utils::loadu_si256(components2, src_x); // bottom line
|
||||
|
||||
let source = _mm256_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
|
||||
@@ -85,7 +88,7 @@ unsafe fn vert_convolution_into_one_row_u8<T>(
|
||||
let components = T::components(s_row);
|
||||
let mmk = _mm256_set1_epi32(k as i32);
|
||||
|
||||
let source1 = simd_utils::loadu_si256(components, x_in_bytes); // top line
|
||||
let source1 = simd_utils::loadu_si256(components, src_x); // top line
|
||||
let source2 = _mm256_setzero_si256(); // bottom line is empty
|
||||
|
||||
let source = _mm256_unpacklo_epi8(source1, source2);
|
||||
@@ -115,14 +118,16 @@ unsafe fn vert_convolution_into_one_row_u8<T>(
|
||||
sss2 = _mm256_packs_epi32(sss2, sss3);
|
||||
sss0 = _mm256_packus_epi16(sss0, sss2);
|
||||
|
||||
let dst_ptr = dst_ptr_u8.add(x_in_bytes) as *mut __m256i;
|
||||
let dst_ptr = dst_chunk.as_mut_ptr() as *mut __m256i;
|
||||
_mm256_storeu_si256(dst_ptr, sss0);
|
||||
|
||||
x_in_bytes += 32;
|
||||
src_x += 32;
|
||||
}
|
||||
|
||||
// 8 components in half of SSE register - 1 = 7
|
||||
while x_in_bytes < src_width.saturating_sub(7) {
|
||||
// 8 components in half of SSE register
|
||||
dst_u8 = dst_chunks_32.into_remainder();
|
||||
let mut dst_chunks_8 = dst_u8.chunks_exact_mut(8);
|
||||
for dst_chunk in &mut dst_chunks_8 {
|
||||
let mut sss0 = initial; // left row
|
||||
let mut sss1 = initial; // right row
|
||||
let mut y: u32 = 0;
|
||||
@@ -133,8 +138,8 @@ unsafe fn vert_convolution_into_one_row_u8<T>(
|
||||
// Load two coefficients at once
|
||||
let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
|
||||
|
||||
let source1 = simd_utils::loadl_epi64(components1, x_in_bytes); // top line
|
||||
let source2 = simd_utils::loadl_epi64(components2, x_in_bytes); // bottom line
|
||||
let source1 = simd_utils::loadl_epi64(components1, src_x); // top line
|
||||
let source2 = simd_utils::loadl_epi64(components2, src_x); // bottom line
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
@@ -150,7 +155,7 @@ unsafe fn vert_convolution_into_one_row_u8<T>(
|
||||
let components = T::components(s_row);
|
||||
let mmk = _mm_set1_epi32(k as i32);
|
||||
|
||||
let source1 = simd_utils::loadl_epi64(components, x_in_bytes); // top line
|
||||
let source1 = simd_utils::loadl_epi64(components, src_x); // top line
|
||||
let source2 = _mm_setzero_si128(); // bottom line is empty
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, source2);
|
||||
@@ -171,13 +176,15 @@ unsafe fn vert_convolution_into_one_row_u8<T>(
|
||||
sss0 = _mm_packs_epi32(sss0, sss1);
|
||||
sss0 = _mm_packus_epi16(sss0, sss0);
|
||||
|
||||
let dst_ptr = dst_ptr_u8.add(x_in_bytes) as *mut __m128i;
|
||||
let dst_ptr = dst_chunk.as_mut_ptr() as *mut __m128i;
|
||||
_mm_storel_epi64(dst_ptr, sss0);
|
||||
|
||||
x_in_bytes += 8;
|
||||
src_x += 8;
|
||||
}
|
||||
|
||||
while x_in_bytes < src_width.saturating_sub(3) {
|
||||
dst_u8 = dst_chunks_8.into_remainder();
|
||||
let mut dst_chunks_4 = dst_u8.chunks_exact_mut(4);
|
||||
if let Some(dst_chunk) = dst_chunks_4.next() {
|
||||
let mut sss = initial;
|
||||
let mut y: u32 = 0;
|
||||
for src_rows in src_img.iter_2_rows(y_start, max_y) {
|
||||
@@ -186,8 +193,8 @@ unsafe fn vert_convolution_into_one_row_u8<T>(
|
||||
// Load two coefficients at once
|
||||
let two_coeffs = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
|
||||
|
||||
let row1 = simd_utils::mm_cvtsi32_si128_from_u8(components1, x_in_bytes); // top line
|
||||
let row2 = simd_utils::mm_cvtsi32_si128_from_u8(components2, x_in_bytes); // bottom line
|
||||
let row1 = simd_utils::mm_cvtsi32_si128_from_u8(components1, src_x); // top line
|
||||
let row2 = simd_utils::mm_cvtsi32_si128_from_u8(components2, src_x); // bottom line
|
||||
|
||||
let pixels_u8 = _mm_unpacklo_epi8(row1, row2);
|
||||
let pixels_i16 = _mm_unpacklo_epi8(pixels_u8, _mm_setzero_si128());
|
||||
@@ -199,7 +206,7 @@ unsafe fn vert_convolution_into_one_row_u8<T>(
|
||||
if let Some(&k) = coeffs.get(y as usize) {
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let components = T::components(s_row);
|
||||
let pix = simd_utils::mm_cvtepu8_epi32_from_u8(components, x_in_bytes);
|
||||
let pix = simd_utils::mm_cvtepu8_epi32_from_u8(components, src_x);
|
||||
let mmk = _mm_set1_epi32(k as i32);
|
||||
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
|
||||
}
|
||||
@@ -212,27 +219,22 @@ unsafe fn vert_convolution_into_one_row_u8<T>(
|
||||
constify_imm8!(precision, call);
|
||||
|
||||
sss = _mm_packs_epi32(sss, sss);
|
||||
let dst_ptr_i32 = dst_ptr_u8.add(x_in_bytes) as *mut i32;
|
||||
*dst_ptr_i32 = _mm_cvtsi128_si32(_mm_packus_epi16(sss, sss));
|
||||
let dst_ptr = dst_chunk.as_mut_ptr() as *mut i32;
|
||||
*dst_ptr = _mm_cvtsi128_si32(_mm_packus_epi16(sss, sss));
|
||||
|
||||
x_in_bytes += 4;
|
||||
src_x += 4;
|
||||
}
|
||||
|
||||
if x_in_bytes < src_width {
|
||||
let dst_u8 =
|
||||
std::slice::from_raw_parts_mut(dst_ptr_u8.add(x_in_bytes), src_width - x_in_bytes);
|
||||
|
||||
for dst_pixel in dst_u8 {
|
||||
let mut ss0 = 1 << (precision - 1);
|
||||
for (dy, &k) in coeffs.iter().enumerate() {
|
||||
if let Some(src_row) = src_img.get_row(y_start + dy as u32) {
|
||||
let src_ptr = src_row.as_ptr() as *const u8;
|
||||
let src_component = *src_ptr.add(x_in_bytes);
|
||||
ss0 += src_component as i32 * (k as i32);
|
||||
}
|
||||
}
|
||||
*dst_pixel = normalizer.clip(ss0);
|
||||
x_in_bytes += 1;
|
||||
}
|
||||
dst_u8 = dst_chunks_4.into_remainder();
|
||||
if !dst_u8.is_empty() {
|
||||
native::convolution_by_u8(
|
||||
src_img,
|
||||
normalizer,
|
||||
1 << (precision - 1),
|
||||
dst_u8,
|
||||
src_x,
|
||||
y_start,
|
||||
coeffs,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,16 +14,21 @@ pub(crate) mod sse4;
|
||||
pub(crate) fn vert_convolution_u8<T: PixelExt<Component = u8>>(
|
||||
src_image: &ImageView<T>,
|
||||
dst_image: &mut ImageViewMut<T>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
// Check safety conditions
|
||||
debug_assert!(src_image.width().get() - offset >= dst_image.width().get());
|
||||
debug_assert_eq!(coeffs.bounds.len(), dst_image.height().get() as usize);
|
||||
|
||||
match cpu_extensions {
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Avx2 => avx2::vert_convolution(src_image, dst_image, coeffs),
|
||||
CpuExtensions::Avx2 => avx2::vert_convolution(src_image, dst_image, offset, coeffs),
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Sse4_1 => sse4::vert_convolution(src_image, dst_image, coeffs),
|
||||
CpuExtensions::Sse4_1 => sse4::vert_convolution(src_image, dst_image, offset, coeffs),
|
||||
#[cfg(target_arch = "aarch64")]
|
||||
CpuExtensions::Neon => neon::vert_convolution(src_image, dst_image, coeffs),
|
||||
_ => native::vert_convolution(src_image, dst_image, coeffs),
|
||||
CpuExtensions::Neon => neon::vert_convolution(src_image, dst_image, offset, coeffs),
|
||||
_ => native::vert_convolution(src_image, dst_image, offset, coeffs),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,25 +6,23 @@ use crate::{ImageView, ImageViewMut};
|
||||
pub(crate) fn vert_convolution<T>(
|
||||
src_image: &ImageView<T>,
|
||||
dst_image: &mut ImageViewMut<T>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
) where
|
||||
T: PixelExt<Component = u8>,
|
||||
{
|
||||
// Check safety conditions
|
||||
debug_assert_eq!(src_image.width(), dst_image.width());
|
||||
debug_assert_eq!(coeffs.bounds.len(), dst_image.height().get() as usize);
|
||||
|
||||
let normalizer = optimisations::Normalizer16::new(coeffs);
|
||||
let coefficients_chunks = normalizer.normalized_chunks();
|
||||
let precision = normalizer.precision();
|
||||
let initial = 1 << (precision - 1);
|
||||
let src_x_initial = offset as usize * T::count_of_components();
|
||||
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
let coeffs_chunks_iter = coefficients_chunks.into_iter();
|
||||
for (coeffs_chunk, dst_row) in coeffs_chunks_iter.zip(dst_rows) {
|
||||
let first_y_src = coeffs_chunk.start;
|
||||
let ks = coeffs_chunk.values;
|
||||
let mut x_src: usize = 0;
|
||||
let mut x_src = src_x_initial;
|
||||
let dst_components = T::components_mut(dst_row);
|
||||
let (head, dst_chunks, tail) = unsafe { dst_components.align_to_mut::<u32>() };
|
||||
|
||||
@@ -75,7 +73,7 @@ pub(crate) fn vert_convolution<T>(
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn convolution_by_u8<T>(
|
||||
pub(crate) fn convolution_by_u8<T>(
|
||||
src_image: &ImageView<T>,
|
||||
normalizer: &optimisations::Normalizer16,
|
||||
initial: i32,
|
||||
|
||||
@@ -9,16 +9,14 @@ use crate::{ImageView, ImageViewMut};
|
||||
pub(crate) fn vert_convolution<T: PixelExt<Component = u8>>(
|
||||
src_image: &ImageView<T>,
|
||||
dst_image: &mut ImageViewMut<T>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
// Check safety conditions
|
||||
debug_assert_eq!(src_image.width(), dst_image.width());
|
||||
debug_assert_eq!(coeffs.bounds.len(), dst_image.height().get() as usize);
|
||||
|
||||
let normalizer = optimisations::Normalizer16::new(coeffs);
|
||||
let coefficients_chunks = normalizer.normalized_chunks();
|
||||
let precision = normalizer.precision();
|
||||
let initial = 1 << (precision - 1);
|
||||
let start_src_x = offset as usize * T::count_of_components();
|
||||
|
||||
let mut tmp_dst = vec![0i32; dst_image.width().get() as usize * T::count_of_components()];
|
||||
let tmp_buf = tmp_dst.as_mut_slice();
|
||||
@@ -26,7 +24,7 @@ pub(crate) fn vert_convolution<T: PixelExt<Component = u8>>(
|
||||
for (dst_row, coeffs_chunk) in dst_rows.zip(coefficients_chunks) {
|
||||
tmp_buf.fill(initial);
|
||||
unsafe {
|
||||
vert_convolution_into_one_row_i32(src_image, tmp_buf, coeffs_chunk);
|
||||
vert_convolution_into_one_row_i32(src_image, tmp_buf, start_src_x, coeffs_chunk);
|
||||
let dst_comp = T::components_mut(dst_row);
|
||||
macro_rules! call {
|
||||
($imm8:expr) => {{
|
||||
@@ -42,6 +40,7 @@ pub(crate) fn vert_convolution<T: PixelExt<Component = u8>>(
|
||||
unsafe fn vert_convolution_into_one_row_i32<T: PixelExt<Component = u8>>(
|
||||
src_img: &ImageView<T>,
|
||||
dst_buf: &mut [i32],
|
||||
start_src_x: usize,
|
||||
coeffs_chunk: optimisations::CoefficientsI16Chunk,
|
||||
) {
|
||||
let width = dst_buf.len();
|
||||
@@ -55,74 +54,80 @@ unsafe fn vert_convolution_into_one_row_i32<T: PixelExt<Component = u8>>(
|
||||
let components = T::components(s_row);
|
||||
let coeff_i16x4 = vdup_n_s16(coeff);
|
||||
|
||||
let mut x: usize = 0;
|
||||
while x < width.saturating_sub(63) {
|
||||
let source = neon_utils::load_u8x16x4(components, x);
|
||||
let mut dst_x: usize = 0;
|
||||
let mut src_x = start_src_x;
|
||||
while dst_x < width.saturating_sub(63) {
|
||||
let source = neon_utils::load_u8x16x4(components, src_x);
|
||||
|
||||
for s in [source.0, source.1, source.2, source.3] {
|
||||
let mut accum = neon_utils::load_i32x4x4(dst_buf, x);
|
||||
let mut accum = neon_utils::load_i32x4x4(dst_buf, dst_x);
|
||||
let pix = vreinterpretq_s16_u8(vzip1q_u8(s, zero_u8x16));
|
||||
accum.0 = vmlal_s16(accum.0, vget_low_s16(pix), coeff_i16x4);
|
||||
accum.1 = vmlal_s16(accum.1, vget_high_s16(pix), coeff_i16x4);
|
||||
let pix = vreinterpretq_s16_u8(vzip2q_u8(s, zero_u8x16));
|
||||
accum.2 = vmlal_s16(accum.2, vget_low_s16(pix), coeff_i16x4);
|
||||
accum.3 = vmlal_s16(accum.3, vget_high_s16(pix), coeff_i16x4);
|
||||
neon_utils::store_i32x4x4(dst_buf, x, accum);
|
||||
x += 16;
|
||||
neon_utils::store_i32x4x4(dst_buf, dst_x, accum);
|
||||
dst_x += 16;
|
||||
src_x += 16;
|
||||
}
|
||||
}
|
||||
|
||||
if x < width.saturating_sub(31) {
|
||||
let source = neon_utils::load_u8x16x2(components, x);
|
||||
if dst_x < width.saturating_sub(31) {
|
||||
let source = neon_utils::load_u8x16x2(components, src_x);
|
||||
|
||||
for s in [source.0, source.1] {
|
||||
let mut accum = neon_utils::load_i32x4x4(dst_buf, x);
|
||||
let mut accum = neon_utils::load_i32x4x4(dst_buf, dst_x);
|
||||
let pix = vreinterpretq_s16_u8(vzip1q_u8(s, zero_u8x16));
|
||||
accum.0 = vmlal_s16(accum.0, vget_low_s16(pix), coeff_i16x4);
|
||||
accum.1 = vmlal_s16(accum.1, vget_high_s16(pix), coeff_i16x4);
|
||||
let pix = vreinterpretq_s16_u8(vzip2q_u8(s, zero_u8x16));
|
||||
accum.2 = vmlal_s16(accum.2, vget_low_s16(pix), coeff_i16x4);
|
||||
accum.3 = vmlal_s16(accum.3, vget_high_s16(pix), coeff_i16x4);
|
||||
neon_utils::store_i32x4x4(dst_buf, x, accum);
|
||||
x += 16;
|
||||
neon_utils::store_i32x4x4(dst_buf, dst_x, accum);
|
||||
dst_x += 16;
|
||||
src_x += 16;
|
||||
}
|
||||
}
|
||||
|
||||
if x < width.saturating_sub(15) {
|
||||
let s = neon_utils::load_u8x16(components, x);
|
||||
let mut accum = neon_utils::load_i32x4x4(dst_buf, x);
|
||||
if dst_x < width.saturating_sub(15) {
|
||||
let s = neon_utils::load_u8x16(components, src_x);
|
||||
let mut accum = neon_utils::load_i32x4x4(dst_buf, dst_x);
|
||||
let pix = vreinterpretq_s16_u8(vzip1q_u8(s, zero_u8x16));
|
||||
accum.0 = vmlal_s16(accum.0, vget_low_s16(pix), coeff_i16x4);
|
||||
accum.1 = vmlal_s16(accum.1, vget_high_s16(pix), coeff_i16x4);
|
||||
let pix = vreinterpretq_s16_u8(vzip2q_u8(s, zero_u8x16));
|
||||
accum.2 = vmlal_s16(accum.2, vget_low_s16(pix), coeff_i16x4);
|
||||
accum.3 = vmlal_s16(accum.3, vget_high_s16(pix), coeff_i16x4);
|
||||
neon_utils::store_i32x4x4(dst_buf, x, accum);
|
||||
x += 16;
|
||||
neon_utils::store_i32x4x4(dst_buf, dst_x, accum);
|
||||
dst_x += 16;
|
||||
src_x += 16;
|
||||
}
|
||||
|
||||
if x < width.saturating_sub(7) {
|
||||
let s = vcombine_u8(neon_utils::load_u8x8(components, x), zero_u8x8);
|
||||
let mut accum = neon_utils::load_i32x4x2(dst_buf, x);
|
||||
if dst_x < width.saturating_sub(7) {
|
||||
let s = vcombine_u8(neon_utils::load_u8x8(components, src_x), zero_u8x8);
|
||||
let mut accum = neon_utils::load_i32x4x2(dst_buf, dst_x);
|
||||
let pix = vreinterpretq_s16_u8(vzip1q_u8(s, zero_u8x16));
|
||||
accum.0 = vmlal_s16(accum.0, vget_low_s16(pix), coeff_i16x4);
|
||||
accum.1 = vmlal_s16(accum.1, vget_high_s16(pix), coeff_i16x4);
|
||||
neon_utils::store_i32x4x2(dst_buf, x, accum);
|
||||
x += 8;
|
||||
neon_utils::store_i32x4x2(dst_buf, dst_x, accum);
|
||||
dst_x += 8;
|
||||
src_x += 8;
|
||||
}
|
||||
|
||||
if x < width.saturating_sub(3) {
|
||||
let s = neon_utils::create_u8x16_from_one_u32(components, x);
|
||||
let mut accum = neon_utils::load_i32x4(dst_buf, x);
|
||||
if dst_x < width.saturating_sub(3) {
|
||||
let s = neon_utils::create_u8x16_from_one_u32(components, src_x);
|
||||
let mut accum = neon_utils::load_i32x4(dst_buf, dst_x);
|
||||
let pix = vreinterpretq_s16_u8(vzip1q_u8(s, zero_u8x16));
|
||||
accum = vmlal_s16(accum, vget_low_s16(pix), coeff_i16x4);
|
||||
neon_utils::store_i32x4(dst_buf, x, accum);
|
||||
x += 4
|
||||
neon_utils::store_i32x4(dst_buf, dst_x, accum);
|
||||
dst_x += 4;
|
||||
src_x += 4;
|
||||
}
|
||||
|
||||
let coeff = coeff as i32;
|
||||
let tmp_tail = dst_buf.iter_mut().skip(x);
|
||||
let comp_tail = components.iter().skip(x);
|
||||
let tmp_tail = dst_buf.iter_mut().skip(dst_x);
|
||||
let comp_tail = components.iter().skip(src_x);
|
||||
for (accum, &comp) in tmp_tail.zip(comp_tail) {
|
||||
*accum += coeff * comp as i32;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
use std::arch::x86_64::*;
|
||||
|
||||
use crate::convolution::vertical_u8::native;
|
||||
use crate::convolution::{optimisations, Coefficients};
|
||||
use crate::pixels::PixelExt;
|
||||
use crate::simd_utils;
|
||||
@@ -9,15 +10,17 @@ use crate::{ImageView, ImageViewMut};
|
||||
pub(crate) fn vert_convolution<T: PixelExt<Component = u8>>(
|
||||
src_image: &ImageView<T>,
|
||||
dst_image: &mut ImageViewMut<T>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
let normalizer = optimisations::Normalizer16::new(coeffs);
|
||||
let coefficients_chunks = normalizer.normalized_chunks();
|
||||
let src_x = offset as usize * T::count_of_components();
|
||||
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
for (dst_row, coeffs_chunk) in dst_rows.zip(coefficients_chunks) {
|
||||
unsafe {
|
||||
vert_convolution_into_one_row_u8(src_image, dst_row, coeffs_chunk, &normalizer);
|
||||
vert_convolution_into_one_row_u8(src_image, dst_row, src_x, coeffs_chunk, &normalizer);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -26,21 +29,20 @@ pub(crate) fn vert_convolution<T: PixelExt<Component = u8>>(
|
||||
pub(crate) unsafe fn vert_convolution_into_one_row_u8<T: PixelExt<Component = u8>>(
|
||||
src_img: &ImageView<T>,
|
||||
dst_row: &mut [T],
|
||||
mut src_x: usize,
|
||||
coeffs_chunk: optimisations::CoefficientsI16Chunk,
|
||||
normalizer: &optimisations::Normalizer16,
|
||||
) {
|
||||
let mut xx: usize = 0;
|
||||
let src_width = src_img.width().get() as usize * T::count_of_components();
|
||||
let y_start = coeffs_chunk.start;
|
||||
let coeffs = coeffs_chunk.values;
|
||||
let max_y = y_start + coeffs.len() as u32;
|
||||
let precision = normalizer.precision();
|
||||
let dst_ptr_u8 = T::components_mut(dst_row).as_mut_ptr() as *mut u8;
|
||||
let mut dst_u8 = T::components_mut(dst_row);
|
||||
|
||||
let initial = _mm_set1_epi32(1 << (precision - 1));
|
||||
|
||||
// 32 components in two registers - 1 = 31
|
||||
while xx < src_width.saturating_sub(31) {
|
||||
let mut dst_chunks_32 = dst_u8.chunks_exact_mut(32);
|
||||
for dst_chunk in &mut dst_chunks_32 {
|
||||
let mut sss0 = initial;
|
||||
let mut sss1 = initial;
|
||||
let mut sss2 = initial;
|
||||
@@ -59,8 +61,8 @@ pub(crate) unsafe fn vert_convolution_into_one_row_u8<T: PixelExt<Component = u8
|
||||
// Load two coefficients at once
|
||||
let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
|
||||
|
||||
let source1 = simd_utils::loadu_si128(components1, xx); // top line
|
||||
let source2 = simd_utils::loadu_si128(components2, xx); // bottom line
|
||||
let source1 = simd_utils::loadu_si128(components1, src_x); // top line
|
||||
let source2 = simd_utils::loadu_si128(components2, src_x); // bottom line
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
@@ -74,8 +76,8 @@ pub(crate) unsafe fn vert_convolution_into_one_row_u8<T: PixelExt<Component = u8
|
||||
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss3 = _mm_add_epi32(sss3, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
let source1 = simd_utils::loadu_si128(components1, xx + 16); // top line
|
||||
let source2 = simd_utils::loadu_si128(components2, xx + 16); // bottom line
|
||||
let source1 = simd_utils::loadu_si128(components1, src_x + 16); // top line
|
||||
let source2 = simd_utils::loadu_si128(components2, src_x + 16); // bottom line
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
@@ -97,7 +99,7 @@ pub(crate) unsafe fn vert_convolution_into_one_row_u8<T: PixelExt<Component = u8
|
||||
let components = T::components(s_row);
|
||||
let mmk = _mm_set1_epi32(k as i32);
|
||||
|
||||
let source1 = simd_utils::loadu_si128(components, xx); // top line
|
||||
let source1 = simd_utils::loadu_si128(components, src_x); // top line
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, _mm_setzero_si128());
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
@@ -111,7 +113,7 @@ pub(crate) unsafe fn vert_convolution_into_one_row_u8<T: PixelExt<Component = u8
|
||||
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss3 = _mm_add_epi32(sss3, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
let source1 = simd_utils::loadu_si128(components, xx + 16); // top line
|
||||
let source1 = simd_utils::loadu_si128(components, src_x + 16); // top line
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, _mm_setzero_si128());
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
@@ -143,18 +145,20 @@ pub(crate) unsafe fn vert_convolution_into_one_row_u8<T: PixelExt<Component = u8
|
||||
sss0 = _mm_packs_epi32(sss0, sss1);
|
||||
sss2 = _mm_packs_epi32(sss2, sss3);
|
||||
sss0 = _mm_packus_epi16(sss0, sss2);
|
||||
let dst_ptr = dst_ptr_u8.add(xx) as *mut __m128i;
|
||||
let dst_ptr = dst_chunk.as_mut_ptr() as *mut __m128i;
|
||||
_mm_storeu_si128(dst_ptr, sss0);
|
||||
sss4 = _mm_packs_epi32(sss4, sss5);
|
||||
sss6 = _mm_packs_epi32(sss6, sss7);
|
||||
sss4 = _mm_packus_epi16(sss4, sss6);
|
||||
let dst_ptr = dst_ptr_u8.add(xx + 16) as *mut __m128i;
|
||||
let dst_ptr = dst_ptr.add(1);
|
||||
_mm_storeu_si128(dst_ptr, sss4);
|
||||
|
||||
xx += 32;
|
||||
src_x += 32;
|
||||
}
|
||||
|
||||
while xx < src_width.saturating_sub(7) {
|
||||
dst_u8 = dst_chunks_32.into_remainder();
|
||||
let mut dst_chunks_8 = dst_u8.chunks_exact_mut(8);
|
||||
for dst_chunk in &mut dst_chunks_8 {
|
||||
let mut sss0 = initial; // left row
|
||||
let mut sss1 = initial; // right row
|
||||
let mut y: u32 = 0;
|
||||
@@ -165,8 +169,8 @@ pub(crate) unsafe fn vert_convolution_into_one_row_u8<T: PixelExt<Component = u8
|
||||
// Load two coefficients at once
|
||||
let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
|
||||
|
||||
let source1 = simd_utils::loadl_epi64(components1, xx); // top line
|
||||
let source2 = simd_utils::loadl_epi64(components2, xx); // bottom line
|
||||
let source1 = simd_utils::loadl_epi64(components1, src_x); // top line
|
||||
let source2 = simd_utils::loadl_epi64(components2, src_x); // bottom line
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
@@ -182,7 +186,7 @@ pub(crate) unsafe fn vert_convolution_into_one_row_u8<T: PixelExt<Component = u8
|
||||
let components = T::components(s_row);
|
||||
let mmk = _mm_set1_epi32(k as i32);
|
||||
|
||||
let source1 = simd_utils::loadl_epi64(components, xx); // top line
|
||||
let source1 = simd_utils::loadl_epi64(components, src_x); // top line
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, _mm_setzero_si128());
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
@@ -201,13 +205,15 @@ pub(crate) unsafe fn vert_convolution_into_one_row_u8<T: PixelExt<Component = u8
|
||||
|
||||
sss0 = _mm_packs_epi32(sss0, sss1);
|
||||
sss0 = _mm_packus_epi16(sss0, sss0);
|
||||
let dst_ptr = dst_ptr_u8.add(xx) as *mut __m128i;
|
||||
let dst_ptr = dst_chunk.as_mut_ptr() as *mut __m128i;
|
||||
_mm_storel_epi64(dst_ptr, sss0);
|
||||
|
||||
xx += 8;
|
||||
src_x += 8;
|
||||
}
|
||||
|
||||
while xx < src_width.saturating_sub(3) {
|
||||
dst_u8 = dst_chunks_8.into_remainder();
|
||||
let mut dst_chunks_4 = dst_u8.chunks_exact_mut(4);
|
||||
if let Some(dst_chunk) = dst_chunks_4.next() {
|
||||
let mut sss = initial;
|
||||
let mut y: u32 = 0;
|
||||
|
||||
@@ -217,8 +223,8 @@ pub(crate) unsafe fn vert_convolution_into_one_row_u8<T: PixelExt<Component = u8
|
||||
// Load two coefficients at once
|
||||
let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
|
||||
|
||||
let source1 = simd_utils::mm_cvtsi32_si128_from_u8(components1, xx); // top line
|
||||
let source2 = simd_utils::mm_cvtsi32_si128_from_u8(components2, xx); // bottom line
|
||||
let source1 = simd_utils::mm_cvtsi32_si128_from_u8(components1, src_x); // top line
|
||||
let source2 = simd_utils::mm_cvtsi32_si128_from_u8(components2, src_x); // bottom line
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
@@ -230,7 +236,7 @@ pub(crate) unsafe fn vert_convolution_into_one_row_u8<T: PixelExt<Component = u8
|
||||
if let Some(&k) = coeffs.get(y as usize) {
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let components = T::components(s_row);
|
||||
let pix = simd_utils::mm_cvtepu8_epi32_from_u8(components, xx);
|
||||
let pix = simd_utils::mm_cvtepu8_epi32_from_u8(components, src_x);
|
||||
let mmk = _mm_set1_epi32(k as i32);
|
||||
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
|
||||
}
|
||||
@@ -243,26 +249,22 @@ pub(crate) unsafe fn vert_convolution_into_one_row_u8<T: PixelExt<Component = u8
|
||||
constify_imm8!(precision, call);
|
||||
|
||||
sss = _mm_packs_epi32(sss, sss);
|
||||
let dst_ptr = dst_ptr_u8.add(xx) as *mut i32;
|
||||
let dst_ptr = dst_chunk.as_mut_ptr() as *mut i32;
|
||||
*dst_ptr = _mm_cvtsi128_si32(_mm_packus_epi16(sss, sss));
|
||||
|
||||
xx += 4;
|
||||
src_x += 4;
|
||||
}
|
||||
|
||||
if xx < src_width {
|
||||
let dst_u8 = std::slice::from_raw_parts_mut(dst_ptr_u8.add(xx), src_width - xx);
|
||||
|
||||
for dst_pixel in dst_u8 {
|
||||
let mut ss0 = 1 << (precision - 1);
|
||||
for (dy, &k) in coeffs.iter().enumerate() {
|
||||
if let Some(src_row) = src_img.get_row(y_start + dy as u32) {
|
||||
let src_ptr = src_row.as_ptr() as *const u8;
|
||||
let src_component = *src_ptr.add(xx);
|
||||
ss0 += src_component as i32 * (k as i32);
|
||||
}
|
||||
}
|
||||
*dst_pixel = normalizer.clip(ss0);
|
||||
xx += 1;
|
||||
}
|
||||
dst_u8 = dst_chunks_4.into_remainder();
|
||||
if !dst_u8.is_empty() {
|
||||
native::convolution_by_u8(
|
||||
src_img,
|
||||
normalizer,
|
||||
1 << (precision - 1),
|
||||
dst_u8,
|
||||
src_x,
|
||||
y_start,
|
||||
coeffs,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use std::fmt::Debug;
|
||||
use std::mem::ManuallyDrop;
|
||||
use std::num::NonZeroU32;
|
||||
use std::slice;
|
||||
|
||||
@@ -389,6 +390,31 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a, P> From<ImageViewMut<'a, P>> for ImageView<'a, P>
|
||||
where
|
||||
P: PixelExt,
|
||||
{
|
||||
fn from(view: ImageViewMut<'a, P>) -> Self {
|
||||
let rows = {
|
||||
let mut old_rows = ManuallyDrop::new(view.rows);
|
||||
let (ptr, length, capacity) =
|
||||
(old_rows.as_mut_ptr(), old_rows.len(), old_rows.capacity());
|
||||
unsafe { Vec::from_raw_parts(ptr as *mut &[P], length, capacity) }
|
||||
};
|
||||
ImageView {
|
||||
width: view.width,
|
||||
height: view.height,
|
||||
crop_box: CropBox {
|
||||
left: 0,
|
||||
top: 0,
|
||||
width: view.width,
|
||||
height: view.height,
|
||||
},
|
||||
rows,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn check_rows_count_and_size<T>(
|
||||
width: NonZeroU32,
|
||||
height: NonZeroU32,
|
||||
|
||||
+5
-1
@@ -15,6 +15,9 @@ pub use resizer::{CpuExtensions, ResizeAlg, Resizer};
|
||||
|
||||
pub use crate::image::Image;
|
||||
|
||||
#[macro_use]
|
||||
mod utils;
|
||||
|
||||
mod alpha;
|
||||
mod color;
|
||||
mod convolution;
|
||||
@@ -29,4 +32,5 @@ pub mod pixels;
|
||||
mod resizer;
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
mod simd_utils;
|
||||
mod utils;
|
||||
#[cfg(feature = "for_test")]
|
||||
pub mod testing;
|
||||
|
||||
+44
-25
@@ -269,41 +269,44 @@ fn resample_convolution<P>(
|
||||
let dst_height = dst_image.height();
|
||||
let (filter_fn, filter_support) = convolution::get_filter_func(filter_type);
|
||||
|
||||
let need_horizontal = dst_width != src_image.width() || crop_box.width != src_image.width();
|
||||
let need_vertical = dst_height != src_image.height() || crop_box.height != src_image.height();
|
||||
|
||||
let mut vert_coeffs = convolution::precompute_coefficients(
|
||||
src_image.height(),
|
||||
crop_box.top as f64,
|
||||
crop_box.top as f64 + crop_box.height.get() as f64,
|
||||
dst_height,
|
||||
filter_fn,
|
||||
filter_support,
|
||||
);
|
||||
|
||||
if need_horizontal {
|
||||
let horiz_coeffs = convolution::precompute_coefficients(
|
||||
let need_horizontal = dst_width != crop_box.width;
|
||||
let horiz_coeffs = need_horizontal.then(|| {
|
||||
test_log!("compute horizontal convolution coefficients");
|
||||
convolution::precompute_coefficients(
|
||||
src_image.width(),
|
||||
crop_box.left as f64,
|
||||
crop_box.left as f64 + crop_box.width.get() as f64,
|
||||
dst_width,
|
||||
filter_fn,
|
||||
filter_support,
|
||||
);
|
||||
)
|
||||
});
|
||||
|
||||
// First used row in the source image
|
||||
let y_first = vert_coeffs.bounds[0].start;
|
||||
let need_vertical = dst_height != crop_box.height;
|
||||
let vert_coeffs = need_vertical.then(|| {
|
||||
test_log!("compute vertical convolution coefficients");
|
||||
convolution::precompute_coefficients(
|
||||
src_image.height(),
|
||||
crop_box.top as f64,
|
||||
crop_box.top as f64 + crop_box.height.get() as f64,
|
||||
dst_height,
|
||||
filter_fn,
|
||||
filter_support,
|
||||
)
|
||||
});
|
||||
|
||||
if need_vertical {
|
||||
match (horiz_coeffs, vert_coeffs) {
|
||||
(Some(horiz_coeffs), Some(mut vert_coeffs)) => {
|
||||
let y_first = vert_coeffs.bounds[0].start;
|
||||
// Last used row in the source image
|
||||
let last_y_bound = vert_coeffs.bounds.last().unwrap();
|
||||
let y_last = last_y_bound.start + last_y_bound.size;
|
||||
|
||||
let temp_height = NonZeroU32::new(y_last - y_first).unwrap();
|
||||
let mut temp_image = get_temp_image_from_buffer(temp_buffer, dst_width, temp_height);
|
||||
let mut tmp_dst_view = temp_image.dst_view();
|
||||
P::horiz_convolution(
|
||||
src_image,
|
||||
&mut temp_image.dst_view(),
|
||||
&mut tmp_dst_view,
|
||||
y_first,
|
||||
horiz_coeffs,
|
||||
cpu_extensions,
|
||||
@@ -315,16 +318,32 @@ fn resample_convolution<P>(
|
||||
.iter_mut()
|
||||
.for_each(|b| b.start -= y_first);
|
||||
P::vert_convolution(
|
||||
&temp_image.src_view(),
|
||||
&tmp_dst_view.into(),
|
||||
dst_image,
|
||||
0,
|
||||
vert_coeffs,
|
||||
cpu_extensions,
|
||||
);
|
||||
} else {
|
||||
P::horiz_convolution(src_image, dst_image, y_first, horiz_coeffs, cpu_extensions);
|
||||
}
|
||||
} else if need_vertical {
|
||||
P::vert_convolution(src_image, dst_image, vert_coeffs, cpu_extensions);
|
||||
(Some(horiz_coeffs), None) => {
|
||||
P::horiz_convolution(
|
||||
src_image,
|
||||
dst_image,
|
||||
crop_box.top,
|
||||
horiz_coeffs,
|
||||
cpu_extensions,
|
||||
);
|
||||
}
|
||||
(None, Some(vert_coeffs)) => {
|
||||
P::vert_convolution(
|
||||
src_image,
|
||||
dst_image,
|
||||
crop_box.left,
|
||||
vert_coeffs,
|
||||
cpu_extensions,
|
||||
);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
use std::cell::RefCell;
|
||||
|
||||
thread_local!(static TEST_LOGS: RefCell<Vec<String>> = RefCell::new(Vec::new()));
|
||||
|
||||
pub fn log_message(msg: &str) {
|
||||
TEST_LOGS.with(|f| {
|
||||
let mut logs = f.borrow_mut();
|
||||
logs.push(msg.to_string());
|
||||
});
|
||||
}
|
||||
|
||||
pub fn logs_contain(msg: &str) -> bool {
|
||||
TEST_LOGS.with(|f| {
|
||||
let logs = f.borrow();
|
||||
for line in logs.iter() {
|
||||
if line.contains(msg) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
false
|
||||
})
|
||||
}
|
||||
|
||||
pub fn clear_log() {
|
||||
TEST_LOGS.with(|f| {
|
||||
let mut logs = f.borrow_mut();
|
||||
logs.clear();
|
||||
})
|
||||
}
|
||||
@@ -16,3 +16,13 @@ pub(crate) fn foreach_with_pre_reading<D, I>(
|
||||
process_data(next_data);
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! test_log {
|
||||
($s:expr) => {
|
||||
#[cfg(feature = "for_test")]
|
||||
{
|
||||
use crate::testing::log_message;
|
||||
log_message($s);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
+161
-2
@@ -3,8 +3,8 @@ use std::fmt::Debug;
|
||||
|
||||
use fast_image_resize::pixels::*;
|
||||
use fast_image_resize::{
|
||||
CpuExtensions, CropBox, DifferentTypesOfPixelsError, DynamicImageView, FilterType, Image,
|
||||
PixelType, ResizeAlg, Resizer,
|
||||
testing as fr_testing, CpuExtensions, CropBox, DifferentTypesOfPixelsError, DynamicImageView,
|
||||
FilterType, Image, PixelType, ResizeAlg, Resizer,
|
||||
};
|
||||
use testing::{cpu_ext_into_str, nonzero, PixelTestingExt};
|
||||
|
||||
@@ -83,6 +83,165 @@ fn resize_to_same_size_after_cropping() {
|
||||
assert!(matches!(cropped_buffer.cmp(&dst_buffer), Ordering::Equal));
|
||||
}
|
||||
|
||||
/// In this test, we check that resizer won't use horizontal convolution
|
||||
/// if width of destination image is equal to width of cropped source image.
|
||||
fn resize_to_same_width<const C: usize>(
|
||||
pixel_type: PixelType,
|
||||
cpu_extensions: CpuExtensions,
|
||||
create_pixel: fn(v: u8) -> [u8; C],
|
||||
) {
|
||||
fr_testing::clear_log();
|
||||
let width = nonzero(100);
|
||||
let height = nonzero(80);
|
||||
let src_width = nonzero(120);
|
||||
let src_height = nonzero(100);
|
||||
// Image columns are made up of pixels of the same color.
|
||||
let buffer: Vec<u8> = (0..12000)
|
||||
.flat_map(|v| create_pixel((v % 120) as u8))
|
||||
.collect();
|
||||
let src_image = Image::from_vec_u8(src_width, src_height, buffer, pixel_type).unwrap();
|
||||
let mut src_view = src_image.view();
|
||||
src_view
|
||||
.set_crop_box(CropBox {
|
||||
left: 10,
|
||||
top: 0,
|
||||
width,
|
||||
height: src_height,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let mut dst_image = Image::new(width, height, pixel_type);
|
||||
let mut resizer = Resizer::new(ResizeAlg::Convolution(FilterType::Lanczos3));
|
||||
unsafe {
|
||||
resizer.set_cpu_extensions(cpu_extensions);
|
||||
}
|
||||
resizer
|
||||
.resize(&src_view, &mut dst_image.view_mut())
|
||||
.unwrap();
|
||||
let expected_result: Vec<u8> = (0..8000u32)
|
||||
.flat_map(|v| create_pixel((10 + v % 100) as u8))
|
||||
.collect();
|
||||
let dst_buffer = dst_image.into_vec();
|
||||
assert!(
|
||||
matches!(expected_result.cmp(&dst_buffer), Ordering::Equal),
|
||||
"Resizing result is not equal to expected ones ({:?}, {:?})",
|
||||
pixel_type,
|
||||
cpu_extensions
|
||||
);
|
||||
|
||||
assert!(fr_testing::logs_contain(
|
||||
"compute vertical convolution coefficients"
|
||||
));
|
||||
assert!(!fr_testing::logs_contain(
|
||||
"compute horizontal convolution coefficients"
|
||||
));
|
||||
}
|
||||
|
||||
/// In this test, we check that resizer won't use vertical convolution
|
||||
/// if height of destination image is equal to height of cropped source image.
|
||||
fn resize_to_same_height<const C: usize>(
|
||||
pixel_type: PixelType,
|
||||
cpu_extensions: CpuExtensions,
|
||||
create_pixel: fn(v: u8) -> [u8; C],
|
||||
) {
|
||||
fr_testing::clear_log();
|
||||
let width = nonzero(100);
|
||||
let height = nonzero(80);
|
||||
let src_width = nonzero(120);
|
||||
let src_height = nonzero(100);
|
||||
// Image rows are made up of pixels of the same color.
|
||||
let buffer: Vec<u8> = (0..12000)
|
||||
.flat_map(|v| create_pixel((v / 120) as u8))
|
||||
.collect();
|
||||
let src_image = Image::from_vec_u8(src_width, src_height, buffer, pixel_type).unwrap();
|
||||
let mut src_view = src_image.view();
|
||||
src_view
|
||||
.set_crop_box(CropBox {
|
||||
left: 0,
|
||||
top: 10,
|
||||
width: src_width,
|
||||
height,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let mut dst_image = Image::new(width, height, pixel_type);
|
||||
let mut resizer = Resizer::new(ResizeAlg::Convolution(FilterType::Lanczos3));
|
||||
unsafe {
|
||||
resizer.set_cpu_extensions(cpu_extensions);
|
||||
}
|
||||
resizer
|
||||
.resize(&src_view, &mut dst_image.view_mut())
|
||||
.unwrap();
|
||||
let expected_result: Vec<u8> = (0..8000u32)
|
||||
.flat_map(|v| create_pixel((10 + v / 100) as u8))
|
||||
.collect();
|
||||
let dst_buffer = dst_image.into_vec();
|
||||
assert!(
|
||||
matches!(expected_result.cmp(&dst_buffer), Ordering::Equal),
|
||||
"Resizing result is not equal to expected ones ({:?}, {:?})",
|
||||
pixel_type,
|
||||
cpu_extensions
|
||||
);
|
||||
|
||||
assert!(!fr_testing::logs_contain(
|
||||
"compute vertical convolution coefficients"
|
||||
));
|
||||
assert!(fr_testing::logs_contain(
|
||||
"compute horizontal convolution coefficients"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resize_to_same_width_after_cropping() {
|
||||
let mut cpu_extensions_vec = vec![CpuExtensions::None];
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
{
|
||||
cpu_extensions_vec.push(CpuExtensions::Sse4_1);
|
||||
cpu_extensions_vec.push(CpuExtensions::Avx2);
|
||||
}
|
||||
#[cfg(target_arch = "aarch64")]
|
||||
{
|
||||
cpu_extensions_vec.push(CpuExtensions::Neon);
|
||||
}
|
||||
for cpu_extensions in cpu_extensions_vec {
|
||||
if !cpu_extensions.is_supported() {
|
||||
continue;
|
||||
}
|
||||
resize_to_same_width(PixelType::U8, cpu_extensions, |v| [v]);
|
||||
resize_to_same_width(PixelType::U8x2, cpu_extensions, |v| [v; 2]);
|
||||
resize_to_same_width(PixelType::U8x3, cpu_extensions, |v| [v; 3]);
|
||||
resize_to_same_width(PixelType::U8x4, cpu_extensions, |v| [v; 4]);
|
||||
resize_to_same_width(PixelType::U16, cpu_extensions, |v| [v, 0]);
|
||||
resize_to_same_width(PixelType::U16x2, cpu_extensions, |v| [v, 0, v, 0]);
|
||||
resize_to_same_width(PixelType::U16x3, cpu_extensions, |v| [v, 0, v, 0, v, 0]);
|
||||
resize_to_same_width(PixelType::U16x4, cpu_extensions, |v| {
|
||||
[v, 0, v, 0, v, 0, v, 0]
|
||||
});
|
||||
resize_to_same_height(PixelType::U8, cpu_extensions, |v| [v]);
|
||||
resize_to_same_height(PixelType::U8x2, cpu_extensions, |v| [v; 2]);
|
||||
resize_to_same_height(PixelType::U8x3, cpu_extensions, |v| [v; 3]);
|
||||
resize_to_same_height(PixelType::U8x4, cpu_extensions, |v| [v; 4]);
|
||||
resize_to_same_height(PixelType::U16, cpu_extensions, |v| [v, 0]);
|
||||
resize_to_same_height(PixelType::U16x2, cpu_extensions, |v| [v, 0, v, 0]);
|
||||
resize_to_same_height(PixelType::U16x3, cpu_extensions, |v| [v, 0, v, 0, v, 0]);
|
||||
resize_to_same_height(PixelType::U16x4, cpu_extensions, |v| {
|
||||
[v, 0, v, 0, v, 0, v, 0]
|
||||
});
|
||||
}
|
||||
resize_to_same_width(PixelType::I32, CpuExtensions::None, |v| {
|
||||
(v as i32).to_le_bytes()
|
||||
});
|
||||
resize_to_same_width(PixelType::F32, CpuExtensions::None, |v| {
|
||||
(v as f32).to_le_bytes()
|
||||
});
|
||||
resize_to_same_height(PixelType::I32, CpuExtensions::None, |v| {
|
||||
(v as i32).to_le_bytes()
|
||||
});
|
||||
resize_to_same_height(PixelType::F32, CpuExtensions::None, |v| {
|
||||
(v as f32).to_le_bytes()
|
||||
});
|
||||
}
|
||||
|
||||
trait ResizeTest<const CC: usize> {
|
||||
fn downscale_test(resize_alg: ResizeAlg, cpu_extensions: CpuExtensions, checksum: [u64; CC]);
|
||||
fn upscale_test(resize_alg: ResizeAlg, cpu_extensions: CpuExtensions, checksum: [u64; CC]);
|
||||
|
||||
Reference in New Issue
Block a user