Improved speed (about 9%) of SSE4.1 implementation for vertical convolution pass for pixel types based on u8 components.

This commit is contained in:
Kirill Kuzminykh
2024-10-20 00:02:25 +03:00
parent eaa54c5e87
commit faa65e30bb
14 changed files with 154 additions and 155 deletions
+5
View File
@@ -1,5 +1,10 @@
## [Unreleased] - ReleaseDate
### Changed
- Improved speed (about 9%) of `SSE4.1` implementation for vertical
convolution pass for pixel types based on `u8` components.
### Fixed
- `is_aarch64_feature_detected()` is used now in
Generated
+72 -72
View File
@@ -101,9 +101,9 @@ dependencies = [
[[package]]
name = "anyhow"
version = "1.0.89"
version = "1.0.90"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6"
checksum = "37bf3594c4c988a53154954629820791dde498571819ae4ca50ca811e060cc95"
[[package]]
name = "arbitrary"
@@ -130,9 +130,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "autocfg"
version = "1.3.0"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26"
[[package]]
name = "av1-grain"
@@ -150,9 +150,9 @@ dependencies = [
[[package]]
name = "avif-serialize"
version = "0.8.1"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "876c75a42f6364451a033496a14c44bffe41f5f4a8236f697391f11024e596d2"
checksum = "e335041290c43101ca215eed6f43ec437eb5a42125573f600fc3fa42b9bddd62"
dependencies = [
"arrayvec",
]
@@ -202,9 +202,9 @@ dependencies = [
[[package]]
name = "built"
version = "0.7.4"
version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "236e6289eda5a812bc6b53c3b024039382a2895fbbeef2d748b2931546d392c4"
checksum = "c360505aed52b7ec96a3636c3f039d99103c37d1d9b4f7a8c743d3ea9ffcd03b"
[[package]]
name = "bumpalo"
@@ -214,9 +214,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"
[[package]]
name = "bytemuck"
version = "1.18.0"
version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94bbb0ad554ad961ddc5da507a12a29b14e4ae5bda06b19f575a3e6079d2e2ae"
checksum = "8334215b81e418a0a7bdb8ef0849474f40bb10c8b71f1c4ed315cff49f32494d"
[[package]]
name = "byteorder"
@@ -238,9 +238,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cc"
version = "1.1.21"
version = "1.1.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07b1695e2c7e8fc85310cde85aeaab7e3097f593c91d209d3f9df76c928100f0"
checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f"
dependencies = [
"jobserver",
"libc",
@@ -332,9 +332,9 @@ dependencies = [
[[package]]
name = "clap"
version = "4.5.18"
version = "4.5.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3"
checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8"
dependencies = [
"clap_builder",
"clap_derive",
@@ -352,9 +352,9 @@ dependencies = [
[[package]]
name = "clap_builder"
version = "4.5.18"
version = "4.5.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b"
checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54"
dependencies = [
"anstream",
"anstyle",
@@ -614,9 +614,9 @@ dependencies = [
[[package]]
name = "flume"
version = "0.11.0"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181"
checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095"
dependencies = [
"spin",
]
@@ -688,9 +688,9 @@ dependencies = [
[[package]]
name = "hashbrown"
version = "0.14.5"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb"
[[package]]
name = "heck"
@@ -760,9 +760,9 @@ dependencies = [
[[package]]
name = "image"
version = "0.25.2"
version = "0.25.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "99314c8a2152b8ddb211f924cdae532d8c5e4c8bb54728e12fff1b0cd5963a10"
checksum = "bc144d44a31d753b02ce64093d532f55ff8dc4ebf2ffb8a63c0dda691385acae"
dependencies = [
"bytemuck",
"byteorder-lite",
@@ -783,9 +783,9 @@ dependencies = [
[[package]]
name = "image-webp"
version = "0.1.3"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f79afb8cbee2ef20f59ccd477a218c12a93943d075b492015ecb1bb81f8ee904"
checksum = "e031e8e3d94711a9ccb5d6ea357439ef3dcbed361798bd4071dc4d9793fbe22f"
dependencies = [
"byteorder-lite",
"quick-error",
@@ -793,15 +793,15 @@ dependencies = [
[[package]]
name = "imgref"
version = "1.10.1"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "44feda355f4159a7c757171a77de25daf6411e217b4cabd03bd6650690468126"
checksum = "d0263a3d970d5c054ed9312c0057b4f3bde9c0b33836d3637361d4a9e6e7a408"
[[package]]
name = "indexmap"
version = "2.5.0"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5"
checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da"
dependencies = [
"equivalent",
"hashbrown",
@@ -885,9 +885,9 @@ checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0"
[[package]]
name = "js-sys"
version = "0.3.70"
version = "0.3.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a"
checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9"
dependencies = [
"wasm-bindgen",
]
@@ -906,9 +906,9 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8"
[[package]]
name = "libc"
version = "0.2.159"
version = "0.2.161"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5"
checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1"
[[package]]
name = "libfuzzer-sys"
@@ -1094,9 +1094,9 @@ dependencies = [
[[package]]
name = "once_cell"
version = "1.19.0"
version = "1.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
[[package]]
name = "oorandom"
@@ -1127,9 +1127,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
[[package]]
name = "pest"
version = "2.7.13"
version = "2.7.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdbef9d1d47087a895abd220ed25eb4ad973a5e26f6a4367b038c25e28dfc2d9"
checksum = "879952a81a83930934cbf1786752d6dedc3b1f29e8f8fb2ad1d0a36f377cf442"
dependencies = [
"memchr",
"thiserror",
@@ -1138,9 +1138,9 @@ dependencies = [
[[package]]
name = "pest_derive"
version = "2.7.13"
version = "2.7.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4d3a6e3394ec80feb3b6393c725571754c6188490265c61aaf260810d6b95aa0"
checksum = "d214365f632b123a47fd913301e14c946c61d1c183ee245fa76eb752e59a02dd"
dependencies = [
"pest",
"pest_generator",
@@ -1148,9 +1148,9 @@ dependencies = [
[[package]]
name = "pest_generator"
version = "2.7.13"
version = "2.7.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94429506bde1ca69d1b5601962c73f4172ab4726571a59ea95931218cb0e930e"
checksum = "eb55586734301717aea2ac313f50b2eb8f60d2fc3dc01d190eefa2e625f60c4e"
dependencies = [
"pest",
"pest_meta",
@@ -1161,9 +1161,9 @@ dependencies = [
[[package]]
name = "pest_meta"
version = "2.7.13"
version = "2.7.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac8a071862e93690b6e34e9a5fb8e33ff3734473ac0245b27232222c4906a33f"
checksum = "b75da2a70cf4d9cb76833c990ac9cd3923c9a8905a8929789ce347c84564d03d"
dependencies = [
"once_cell",
"pest",
@@ -1238,27 +1238,27 @@ dependencies = [
[[package]]
name = "proc-macro2"
version = "1.0.86"
version = "1.0.88"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
checksum = "7c3a7fc5db1e57d5a779a352c8cdb57b29aa4c40cc69c3a68a7fedc815fbf2f9"
dependencies = [
"unicode-ident",
]
[[package]]
name = "profiling"
version = "1.0.15"
version = "1.0.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43d84d1d7a6ac92673717f9f6d1518374ef257669c24ebc5ac25d5033828be58"
checksum = "afbdc74edc00b6f6a218ca6a5364d6226a259d4b8ea1af4a0ea063f27e179f4d"
dependencies = [
"profiling-procmacros",
]
[[package]]
name = "profiling-procmacros"
version = "1.0.15"
version = "1.0.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8021cf59c8ec9c432cfc2526ac6b8aa508ecaf29cd415f271b8406c1b851c3fd"
checksum = "a65f2e60fbf1063868558d69c6beacf412dc755f9fc020f514b7955fc914fe30"
dependencies = [
"quote",
"syn",
@@ -1355,9 +1355,9 @@ dependencies = [
[[package]]
name = "ravif"
version = "0.11.10"
version = "0.11.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8f0bfd976333248de2078d350bfdf182ff96e168a24d23d2436cef320dd4bdd"
checksum = "2413fd96bd0ea5cdeeb37eaf446a22e6ed7b981d792828721e74ded1980a45c6"
dependencies = [
"avif-serialize",
"imgref",
@@ -1389,9 +1389,9 @@ dependencies = [
[[package]]
name = "regex"
version = "1.10.6"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619"
checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8"
dependencies = [
"aho-corasick",
"memchr",
@@ -1401,9 +1401,9 @@ dependencies = [
[[package]]
name = "regex-automata"
version = "0.4.7"
version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df"
checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3"
dependencies = [
"aho-corasick",
"memchr",
@@ -1412,9 +1412,9 @@ dependencies = [
[[package]]
name = "regex-syntax"
version = "0.8.4"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
[[package]]
name = "resize"
@@ -1492,9 +1492,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.128"
version = "1.0.132"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8"
checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03"
dependencies = [
"itoa",
"memchr",
@@ -1582,9 +1582,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
name = "syn"
version = "2.0.77"
version = "2.0.80"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed"
checksum = "e6e185e337f816bc8da115b8afcb3324006ccc82eeaddf35113888d3bd8e44ac"
dependencies = [
"proc-macro2",
"quote",
@@ -1723,9 +1723,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
[[package]]
name = "ucd-trie"
version = "0.1.6"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9"
checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971"
[[package]]
name = "unic-char-property"
@@ -1830,9 +1830,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
[[package]]
name = "wasm-bindgen"
version = "0.2.93"
version = "0.2.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5"
checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e"
dependencies = [
"cfg-if",
"once_cell",
@@ -1841,9 +1841,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.93"
version = "0.2.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b"
checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358"
dependencies = [
"bumpalo",
"log",
@@ -1856,9 +1856,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.93"
version = "0.2.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf"
checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
@@ -1866,9 +1866,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.93"
version = "0.2.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836"
checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68"
dependencies = [
"proc-macro2",
"quote",
@@ -1879,9 +1879,9 @@ dependencies = [
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.93"
version = "0.2.95"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484"
checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d"
[[package]]
name = "weezl"
-1
View File
@@ -162,7 +162,6 @@ codegen-units = 256 # max threading
# release build for procmacros - same config as debug build for procmacros
[profile.release.build-override]
opt-level = 2
codegen-units = 256
debug = false # when possible
+1 -1
View File
@@ -133,7 +133,7 @@ unsafe fn horiz_convolution_four_rows(
let reminder = coeffs_by_2.remainder();
for k in coeffs_by_2 {
let mmk = simd_utils::ptr_i16_to_256set1_epi32(k, 0);
let mmk = simd_utils::mm256_load_and_clone_i16x2(k);
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadl_epi32(src_rows[0], x)),
+1 -1
View File
@@ -110,7 +110,7 @@ unsafe fn horiz_convolution_four_rows(
let reminder = coeffs_by_2.remainder();
for k in coeffs_by_2 {
let mmk = simd_utils::ptr_i16_to_set1_epi32(k, 0);
let mmk = simd_utils::mm_load_and_clone_i16x2(k);
for i in 0..4 {
let source = simd_utils::loadl_epi32(src_rows[i], x);
+4 -4
View File
@@ -108,8 +108,8 @@ unsafe fn horiz_convolution_four_rows<const PRECISION: i32>(
let coeffs_by_4 = coeffs.chunks_exact(4);
for k in coeffs_by_4 {
let mmk0 = simd_utils::ptr_i16_to_256set1_epi32(k, 0);
let mmk1 = simd_utils::ptr_i16_to_256set1_epi32(k, 2);
let mmk0 = simd_utils::mm256_load_and_clone_i16x2(k);
let mmk1 = simd_utils::mm256_load_and_clone_i16x2(&k[2..]);
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadu_si128(src_rows[0], x)),
@@ -142,7 +142,7 @@ unsafe fn horiz_convolution_four_rows<const PRECISION: i32>(
let coeffs_by_2 = coeffs[x - x_start..].chunks_exact(2);
for k in coeffs_by_2 {
let mmk = simd_utils::ptr_i16_to_256set1_epi32(k, 0);
let mmk = simd_utils::mm256_load_and_clone_i16x2(k);
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadl_epi64(src_rows[0], x)),
@@ -344,7 +344,7 @@ unsafe fn horiz_convolution_one_row<const PRECISION: i32>(
let coeffs_by_2 = coeffs[x - x_start..].chunks_exact(2);
for k in coeffs_by_2 {
let mmk = simd_utils::ptr_i16_to_set1_epi32(k, 0);
let mmk = simd_utils::mm_load_and_clone_i16x2(k);
let source = simd_utils::loadl_epi64(src_row, x);
let pix = _mm_shuffle_epi8(source, sh7);
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
+4 -4
View File
@@ -108,8 +108,8 @@ unsafe fn horiz_convolution_four_rows<const PRECISION: i32>(
let coeffs_by_4 = coeffs.chunks_exact(4);
for k in coeffs_by_4 {
let mmk0 = simd_utils::ptr_i16_to_set1_epi32(k, 0);
let mmk1 = simd_utils::ptr_i16_to_set1_epi32(k, 2);
let mmk0 = simd_utils::mm_load_and_clone_i16x2(k);
let mmk1 = simd_utils::mm_load_and_clone_i16x2(&k[2..]);
for i in 0..4 {
let source = simd_utils::loadu_si128(src_rows[i], x);
let pix = _mm_shuffle_epi8(source, sh_lo);
@@ -135,7 +135,7 @@ unsafe fn horiz_convolution_four_rows<const PRECISION: i32>(
let coeffs_by_2 = coeffs[x - x_start..].chunks_exact(2);
for k in coeffs_by_2 {
let mmk = simd_utils::ptr_i16_to_set1_epi32(k, 0);
let mmk = simd_utils::mm_load_and_clone_i16x2(k);
for i in 0..4 {
let source = simd_utils::loadl_epi64(src_rows[i], x);
@@ -261,7 +261,7 @@ unsafe fn horiz_convolution_one_row<const PRECISION: i32>(
let coeffs_by_2 = coeffs[x - x_start..].chunks_exact(2);
for k in coeffs_by_2 {
let mmk = simd_utils::ptr_i16_to_set1_epi32(k, 0);
let mmk = simd_utils::mm_load_and_clone_i16x2(k);
let source = simd_utils::loadl_epi64(src_row, x);
let pix = _mm_shuffle_epi8(source, pix_sh1);
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
+4 -4
View File
@@ -90,8 +90,8 @@ unsafe fn horiz_convolution_four_rows<const PRECISION: i32>(
let reminder1 = coeffs_by_4.remainder();
for k in coeffs_by_4 {
let mmk0 = simd_utils::ptr_i16_to_256set1_epi32(k, 0);
let mmk1 = simd_utils::ptr_i16_to_256set1_epi32(k, 2);
let mmk0 = simd_utils::mm256_load_and_clone_i16x2(k);
let mmk1 = simd_utils::mm256_load_and_clone_i16x2(&k[2..]);
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadu_si128(src_rows[0], x)),
@@ -118,7 +118,7 @@ unsafe fn horiz_convolution_four_rows<const PRECISION: i32>(
let reminder2 = coeffs_by_2.remainder();
for k in coeffs_by_2 {
let mmk = simd_utils::ptr_i16_to_256set1_epi32(k, 0);
let mmk = simd_utils::mm256_load_and_clone_i16x2(k);
let source = _mm256_inserti128_si256::<1>(
_mm256_castsi128_si256(simd_utils::loadl_epi64(src_rows[0], x)),
@@ -276,7 +276,7 @@ unsafe fn horiz_convolution_one_row<const PRECISION: i32>(
let reminder1 = coeffs_by_2.remainder();
for k in coeffs_by_2 {
let mmk = simd_utils::ptr_i16_to_set1_epi32(k, 0);
let mmk = simd_utils::mm_load_and_clone_i16x2(k);
let source = simd_utils::loadl_epi64(src_row, x);
let pix = _mm_shuffle_epi8(source, sh7);
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
+4 -4
View File
@@ -80,8 +80,8 @@ unsafe fn horiz_convolution_four_rows<const PRECISION: i32>(
let reminder1 = coeffs_by_4.remainder();
for k in coeffs_by_4 {
let mmk_lo = simd_utils::ptr_i16_to_set1_epi32(k, 0);
let mmk_hi = simd_utils::ptr_i16_to_set1_epi32(k, 2);
let mmk_lo = simd_utils::mm_load_and_clone_i16x2(k);
let mmk_hi = simd_utils::mm_load_and_clone_i16x2(&k[2..]);
// [8] a3 b3 g3 r3 a2 b2 g2 r2 a1 b1 g1 r1 a0 b0 g0 r0
let mut source = simd_utils::loadu_si128(src_rows[0], x);
@@ -117,7 +117,7 @@ unsafe fn horiz_convolution_four_rows<const PRECISION: i32>(
for k in coeffs_by_2 {
// [16] k1 k0 k1 k0 k1 k0 k1 k0
let mmk = simd_utils::ptr_i16_to_set1_epi32(k, 0);
let mmk = simd_utils::mm_load_and_clone_i16x2(k);
// [8] x x x x x x x x a1 b1 g1 r1 a0 b0 g0 r0
let mut pix = simd_utils::loadl_epi64(src_rows[0], x);
@@ -254,7 +254,7 @@ unsafe fn horiz_convolution_one_row<const PRECISION: i32>(
let reminder2 = coeffs_by_2.remainder();
for k in coeffs_by_2 {
let mmk = simd_utils::ptr_i16_to_set1_epi32(k, 0);
let mmk = simd_utils::mm_load_and_clone_i16x2(k);
let source = simd_utils::loadl_epi64(src_row, x);
let pix = _mm_shuffle_epi8(source, sh7);
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
+22 -21
View File
@@ -65,6 +65,7 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
let y_start = coeffs_chunk.start;
let coeffs = coeffs_chunk.values();
let max_rows = coeffs.len() as u32;
let y_last = (y_start + max_rows).max(1) - 1;
let initial = _mm_set1_epi32(1 << (PRECISION as u8 - 1));
let initial_256 = _mm256_set1_epi32(1 << (PRECISION as u8 - 1));
@@ -79,14 +80,15 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
let mut sss2 = initial_256;
let mut sss3 = initial_256;
let mut y: u32 = 0;
let coeffs_chunks = coeffs.chunks_exact(2);
let coeffs_reminder = coeffs_chunks.remainder();
for src_rows in src_view.iter_2_rows(y_start, max_rows) {
for (src_rows, two_coeffs) in src_view.iter_2_rows(y_start, max_rows).zip(coeffs_chunks) {
let components1 = T::components(src_rows[0]);
let components2 = T::components(src_rows[1]);
// Load two coefficients at once
let mmk = simd_utils::ptr_i16_to_256set1_epi32(coeffs, y as usize);
let mmk = simd_utils::mm256_load_and_clone_i16x2(two_coeffs);
let source1 = simd_utils::loadu_si256(components1, src_x); // top line
let source2 = simd_utils::loadu_si256(components2, src_x); // bottom line
@@ -102,12 +104,10 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
sss2 = _mm256_add_epi32(sss2, _mm256_madd_epi16(pix, mmk));
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
sss3 = _mm256_add_epi32(sss3, _mm256_madd_epi16(pix, mmk));
y += 2;
}
if let Some(&k) = coeffs.get(y as usize) {
if let Some(s_row) = src_view.iter_rows(y_start + y).next() {
if let Some(&k) = coeffs_reminder.first() {
if let Some(s_row) = src_view.iter_rows(y_last).next() {
let components = T::components(s_row);
let mmk = _mm256_set1_epi32(k as i32);
@@ -149,13 +149,15 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
for dst_chunk in &mut dst_chunks_8 {
let mut sss0 = initial; // left row
let mut sss1 = initial; // right row
let mut y: u32 = 0;
for src_rows in src_view.iter_2_rows(y_start, max_rows) {
let coeffs_chunks = coeffs.chunks_exact(2);
let coeffs_reminder = coeffs_chunks.remainder();
for (src_rows, two_coeffs) in src_view.iter_2_rows(y_start, max_rows).zip(coeffs_chunks) {
let components1 = T::components(src_rows[0]);
let components2 = T::components(src_rows[1]);
// Load two coefficients at once
let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
let mmk = simd_utils::mm_load_and_clone_i16x2(two_coeffs);
let source1 = simd_utils::loadl_epi64(components1, src_x); // top line
let source2 = simd_utils::loadl_epi64(components2, src_x); // bottom line
@@ -165,12 +167,10 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
y += 2;
}
if let Some(&k) = coeffs.get(y as usize) {
if let Some(s_row) = src_view.iter_rows(y_start + y).next() {
if let Some(&k) = coeffs_reminder.first() {
if let Some(s_row) = src_view.iter_rows(y_last).next() {
let components = T::components(s_row);
let mmk = _mm_set1_epi32(k as i32);
@@ -201,12 +201,15 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
let mut dst_chunks_4 = dst_u8.chunks_exact_mut(4);
if let Some(dst_chunk) = dst_chunks_4.next() {
let mut sss = initial;
let mut y: u32 = 0;
for src_rows in src_view.iter_2_rows(y_start, max_rows) {
let coeffs_chunks = coeffs.chunks_exact(2);
let coeffs_reminder = coeffs_chunks.remainder();
for (src_rows, two_coeffs) in src_view.iter_2_rows(y_start, max_rows).zip(coeffs_chunks) {
let components1 = T::components(src_rows[0]);
let components2 = T::components(src_rows[1]);
// Load two coefficients at once
let two_coeffs = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
let two_coeffs = simd_utils::mm_load_and_clone_i16x2(two_coeffs);
let row1 = simd_utils::mm_cvtsi32_si128_from_u8(components1, src_x); // top line
let row2 = simd_utils::mm_cvtsi32_si128_from_u8(components2, src_x); // bottom line
@@ -214,12 +217,10 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
let pixels_u8 = _mm_unpacklo_epi8(row1, row2);
let pixels_i16 = _mm_unpacklo_epi8(pixels_u8, _mm_setzero_si128());
sss = _mm_add_epi32(sss, _mm_madd_epi16(pixels_i16, two_coeffs));
y += 2;
}
if let Some(&k) = coeffs.get(y as usize) {
if let Some(s_row) = src_view.iter_rows(y_start + y).next() {
if let Some(&k) = coeffs_reminder.first() {
if let Some(s_row) = src_view.iter_rows(y_last).next() {
let components = T::components(s_row);
let pix = simd_utils::mm_cvtepu8_epi32_from_u8(components, src_x);
let mmk = _mm_set1_epi32(k as i32);
+21 -21
View File
@@ -64,6 +64,7 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
let y_start = coeffs_chunk.start;
let coeffs = coeffs_chunk.values();
let max_rows = coeffs.len() as u32;
let y_last = (y_start + max_rows).max(1) - 1;
let mut dst_u8 = T::components_mut(dst_row);
let initial = _mm_set1_epi32(1 << (PRECISION - 1));
@@ -79,14 +80,15 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
let mut sss6 = initial;
let mut sss7 = initial;
let mut y: u32 = 0;
let coeffs_chunks = coeffs.chunks_exact(2);
let coeffs_reminder = coeffs_chunks.remainder();
for src_rows in src_view.iter_2_rows(y_start, max_rows) {
for (src_rows, two_coeffs) in src_view.iter_2_rows(y_start, max_rows).zip(coeffs_chunks) {
let components1 = T::components(src_rows[0]);
let components2 = T::components(src_rows[1]);
// Load two coefficients at once
let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
let mmk = simd_utils::mm_load_and_clone_i16x2(two_coeffs);
let source1 = simd_utils::loadu_si128(components1, src_x); // top line
let source2 = simd_utils::loadu_si128(components2, src_x); // bottom line
@@ -117,12 +119,10 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
sss6 = _mm_add_epi32(sss6, _mm_madd_epi16(pix, mmk));
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
sss7 = _mm_add_epi32(sss7, _mm_madd_epi16(pix, mmk));
y += 2;
}
if let Some(&k) = coeffs.get(y as usize) {
if let Some(s_row) = src_view.iter_rows(y_start + y).next() {
if let Some(&k) = coeffs_reminder.first() {
if let Some(s_row) = src_view.iter_rows(y_last).next() {
let components = T::components(s_row);
let mmk = _mm_set1_epi32(k as i32);
@@ -184,13 +184,15 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
for dst_chunk in &mut dst_chunks_8 {
let mut sss0 = initial; // left row
let mut sss1 = initial; // right row
let mut y: u32 = 0;
for src_rows in src_view.iter_2_rows(y_start, max_rows) {
let coeffs_chunks = coeffs.chunks_exact(2);
let coeffs_reminder = coeffs_chunks.remainder();
for (src_rows, two_coeffs) in src_view.iter_2_rows(y_start, max_rows).zip(coeffs_chunks) {
let components1 = T::components(src_rows[0]);
let components2 = T::components(src_rows[1]);
// Load two coefficients at once
let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
let mmk = simd_utils::mm_load_and_clone_i16x2(two_coeffs);
let source1 = simd_utils::loadl_epi64(components1, src_x); // top line
let source2 = simd_utils::loadl_epi64(components2, src_x); // bottom line
@@ -200,12 +202,10 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
y += 2;
}
if let Some(&k) = coeffs.get(y as usize) {
if let Some(s_row) = src_view.iter_rows(y_start + y).next() {
if let Some(&k) = coeffs_reminder.first() {
if let Some(s_row) = src_view.iter_rows(y_last).next() {
let components = T::components(s_row);
let mmk = _mm_set1_epi32(k as i32);
@@ -234,13 +234,15 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
let mut dst_chunks_4 = dst_u8.chunks_exact_mut(4);
if let Some(dst_chunk) = dst_chunks_4.next() {
let mut sss = initial;
let mut y: u32 = 0;
for src_rows in src_view.iter_2_rows(y_start, max_rows) {
let coeffs_chunks = coeffs.chunks_exact(2);
let coeffs_reminder = coeffs_chunks.remainder();
for (src_rows, two_coeffs) in src_view.iter_2_rows(y_start, max_rows).zip(coeffs_chunks) {
let components1 = T::components(src_rows[0]);
let components2 = T::components(src_rows[1]);
// Load two coefficients at once
let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
let mmk = simd_utils::mm_load_and_clone_i16x2(two_coeffs);
let source1 = simd_utils::mm_cvtsi32_si128_from_u8(components1, src_x); // top line
let source2 = simd_utils::mm_cvtsi32_si128_from_u8(components2, src_x); // bottom line
@@ -248,12 +250,10 @@ unsafe fn vert_convolution_into_one_row<T, const PRECISION: i32>(
let source = _mm_unpacklo_epi8(source1, source2);
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
y += 2;
}
if let Some(&k) = coeffs.get(y as usize) {
if let Some(s_row) = src_view.iter_rows(y_start + y).next() {
if let Some(&k) = coeffs_reminder.first() {
if let Some(s_row) = src_view.iter_rows(y_last).next() {
let components = T::components(s_row);
let pix = simd_utils::mm_cvtepu8_epi32_from_u8(components, src_x);
let mmk = _mm_set1_epi32(k as i32);
+2 -3
View File
@@ -1,11 +1,10 @@
use std::ops::DerefMut;
use bytemuck::cast_slice_mut;
use image::DynamicImage;
use crate::image_view::try_pixel_type;
use crate::images::{TypedImage, TypedImageRef};
use crate::{ImageView, ImageViewMut, IntoImageView, IntoImageViewMut, PixelTrait, PixelType};
use bytemuck::cast_slice_mut;
use image::DynamicImage;
impl IntoImageView for DynamicImage {
fn pixel_type(&self) -> Option<PixelType> {
+6 -4
View File
@@ -76,13 +76,15 @@ pub unsafe fn mm_cvtsi32_si128_from_u8(buf: &[u8], index: usize) -> __m128i {
}
#[inline(always)]
pub unsafe fn ptr_i16_to_set1_epi32(buf: &[i16], index: usize) -> __m128i {
_mm_set1_epi32((buf.get_unchecked(index..).as_ptr() as *const i32).read_unaligned())
pub unsafe fn mm_load_and_clone_i16x2(buf: &[i16]) -> __m128i {
debug_assert!(buf.len() >= 2);
_mm_set1_epi32((buf.as_ptr() as *const i32).read_unaligned())
}
#[inline(always)]
pub unsafe fn ptr_i16_to_256set1_epi32(buf: &[i16], index: usize) -> __m256i {
_mm256_set1_epi32((buf.get_unchecked(index..).as_ptr() as *const i32).read_unaligned())
pub unsafe fn mm256_load_and_clone_i16x2(buf: &[i16]) -> __m256i {
debug_assert!(buf.len() >= 2);
_mm256_set1_epi32((buf.as_ptr() as *const i32).read_unaligned())
}
#[inline(always)]
+8 -15
View File
@@ -1,7 +1,7 @@
use std::cmp::Ordering;
use std::fmt::Debug;
use fast_image_resize::images::{Image, TypedImage, TypedImageRef};
use fast_image_resize::images::Image;
use fast_image_resize::pixels::*;
use fast_image_resize::{
testing as fr_testing, CpuExtensions, CropBoxError, Filter, FilterType, IntoImageView,
@@ -42,11 +42,8 @@ fn resize_to_same_size() {
.collect();
let src_image = Image::from_vec_u8(width, height, buffer, PixelType::U8x4).unwrap();
let mut dst_image = Image::new(width, height, PixelType::U8x4);
let src_view: TypedImageRef<U8x4> = src_image.typed_image().unwrap();
let mut dst_view: TypedImage<U8x4> = dst_image.typed_image_mut().unwrap();
let mut resizer = Resizer::new();
resizer
.resize_typed(&src_view, &mut dst_view, None)
Resizer::new()
.resize(&src_image, &mut dst_image, None)
.unwrap();
assert!(matches!(
src_image.buffer().cmp(dst_image.buffer()),
@@ -65,14 +62,11 @@ fn resize_to_same_size_after_cropping() {
.flat_map(|v| [v; 4])
.collect();
let src_image = Image::from_vec_u8(src_width, src_height, buffer, PixelType::U8x4).unwrap();
let src_view = src_image.typed_image::<U8x4>().unwrap();
let mut dst_image = Image::new(width, height, PixelType::U8x4);
let mut dst_view: TypedImage<U8x4> = dst_image.typed_image_mut().unwrap();
let mut resizer = Resizer::new();
let options = ResizeOptions::new().crop(10., 10., width as _, height as _);
resizer
.resize_typed(&src_view, &mut dst_view, &options)
Resizer::new()
.resize(&src_image, &mut dst_image, &options)
.unwrap();
let cropped_buffer: Vec<u8> = (0..12000u32)
@@ -108,13 +102,12 @@ fn resize_to_same_width<const C: usize>(
.flat_map(|v| create_pixel((v % 120) as u8))
.collect();
let src_image = Image::from_vec_u8(src_width, src_height, buffer, pixel_type).unwrap();
let mut dst_image = Image::new(width, height, pixel_type);
let mut resizer = Resizer::new();
unsafe {
resizer.set_cpu_extensions(cpu_extensions);
}
resizer
.resize(
&src_image,
@@ -161,8 +154,8 @@ fn resize_to_same_height<const C: usize>(
.flat_map(|v| create_pixel((v / 120) as u8))
.collect();
let src_image = Image::from_vec_u8(src_width, src_height, buffer, pixel_type).unwrap();
let mut dst_image = Image::new(width, height, pixel_type);
let mut resizer = Resizer::new();
unsafe {
resizer.set_cpu_extensions(cpu_extensions);