mirror of
https://github.com/Cykooz/fast_image_resize.git
synced 2026-10-08 01:11:09 +00:00
- Added optimisation of convolution U16x3 images with helps of `SSE4.1 and AVX2` instructions.
- Added partial optimisation of convolution U8 images with helps of ``SSE4.1`` instructions.
This commit is contained in:
Kirill Kuzminykh
@@ -1,3 +1,16 @@
|
||||
## [Unreleased] - ReleaseDate
|
||||
|
||||
- Added optimisation of convolution U16x3 images with helps of ``SSE4.1`` and
|
||||
``AVX2`` instructions.
|
||||
- Added partial optimisation of convolution U8 images with helps of ``SSE4.1``
|
||||
instructions.
|
||||
- Allowed to create an instance of `Image`, `ImageVew` and `ImageViewMut`
|
||||
from a buffer larger than necessary
|
||||
([#5](https://github.com/Cykooz/fast_image_resize/issues/5)).
|
||||
- Breaking changes:
|
||||
- Removed methods: `Image::from_vec_u32()`, `Image::from_slice_u32()`.
|
||||
- Removed error `InvalidBufferSizeError`.
|
||||
|
||||
## [0.7.0] - 2022-01-27
|
||||
|
||||
- Added support of new type of pixels `PixelType::U16x3`.
|
||||
|
||||
Generated
+279
-161
@@ -33,9 +33,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.52"
|
||||
version = "1.0.56"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "84450d0b4a8bd1ba4144ce8ce718fbc5d071358b1e5384bace6536b3d1f2d5b3"
|
||||
checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27"
|
||||
|
||||
[[package]]
|
||||
name = "argh"
|
||||
@@ -68,9 +68,9 @@ checksum = "e6f8c380fa28aa1b36107cd97f0196474bb7241bb95a453c5c01a15ac74b2eac"
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.0.1"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
|
||||
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
@@ -78,6 +78,12 @@ version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd"
|
||||
|
||||
[[package]]
|
||||
name = "bit_field"
|
||||
version = "0.10.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dcb6dd1c2376d2e096796e234a70e17e94cc2d5d54ff8ce42b28cef1d0d359a4"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.3.2"
|
||||
@@ -97,10 +103,16 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bytemuck"
|
||||
version = "1.7.3"
|
||||
name = "bumpalo"
|
||||
version = "3.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "439989e6b8c38d1b6570a384ef1e49c8848128f5a97f3914baef02920842712f"
|
||||
checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899"
|
||||
|
||||
[[package]]
|
||||
name = "bytemuck"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e851ca7c24871e7336801608a4797d7376545b6928a10d32d75685687141ead"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
@@ -110,9 +122,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.72"
|
||||
version = "1.0.73"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22a9137b95ea06864e018375b72adfb7db6e6f68cfc8df5a04d00288050485ee"
|
||||
checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
|
||||
dependencies = [
|
||||
"jobserver",
|
||||
]
|
||||
@@ -155,9 +167,9 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.3.0"
|
||||
version = "1.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "738c290dfaea84fc1ca15ad9c168d083b05a714e1efddd8edaab678dc28d2836"
|
||||
checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
@@ -199,9 +211,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.6"
|
||||
version = "0.9.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97242a70df9b89a65d0b6df3c4bf5b9ce03c5b7309019777fbde37e7537f8762"
|
||||
checksum = "c00d6d2ea26e8b151d99093005cb442fb9a37aeaca582a03ec70946f49ab5ed9"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
@@ -212,9 +224,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-queue"
|
||||
version = "0.3.3"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b979d76c9fcb84dffc80a73f7290da0f83e4c95773494674cb44b76d13a7a110"
|
||||
checksum = "4dd435b205a4842da59efd07628f921c096bc1cc0a156835b4fa0bcb9a19bcce"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
@@ -222,9 +234,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.6"
|
||||
version = "0.8.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cfcae03edb34f947e64acdb1c33ec169824e20657e9ecb61cef6c8c74dcb8120"
|
||||
checksum = "b5e5bed1f1c269533fa816a0a5492b3545209a205ca1a54842be180eb63a16a6"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"lazy_static",
|
||||
@@ -300,19 +312,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "deflate"
|
||||
version = "0.8.6"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "73770f8e1fe7d64df17ca66ad28994a0a623ea497fa69486e14984e715c5d174"
|
||||
dependencies = [
|
||||
"adler32",
|
||||
"byteorder",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deflate"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5f95bf05dffba6e6cce8dfbb30def788154949ccd9aed761b472119c21e01c70"
|
||||
checksum = "c86f7e25f518f4b81808a2cf1c50996a61f5c2eb394b2393bd87f2a4780a432f"
|
||||
dependencies = [
|
||||
"adler32",
|
||||
]
|
||||
@@ -345,69 +347,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
|
||||
|
||||
[[package]]
|
||||
name = "encoding"
|
||||
version = "0.2.33"
|
||||
name = "exr"
|
||||
version = "1.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6b0d943856b990d12d3b55b359144ff341533e516d94098b1d3fc1ac666d36ec"
|
||||
checksum = "d4badb9489a465cb2c555af1f00f0bfd8cecd6fc12ac11da9d5b40c5dd5f0200"
|
||||
dependencies = [
|
||||
"encoding-index-japanese",
|
||||
"encoding-index-korean",
|
||||
"encoding-index-simpchinese",
|
||||
"encoding-index-singlebyte",
|
||||
"encoding-index-tradchinese",
|
||||
"bit_field",
|
||||
"deflate",
|
||||
"flume",
|
||||
"half",
|
||||
"inflate",
|
||||
"lebe",
|
||||
"smallvec",
|
||||
"threadpool",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "encoding-index-japanese"
|
||||
version = "1.20141219.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "04e8b2ff42e9a05335dbf8b5c6f7567e5591d0d916ccef4e0b1710d32a0d0c91"
|
||||
dependencies = [
|
||||
"encoding_index_tests",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "encoding-index-korean"
|
||||
version = "1.20141219.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4dc33fb8e6bcba213fe2f14275f0963fd16f0a02c878e3095ecfdf5bee529d81"
|
||||
dependencies = [
|
||||
"encoding_index_tests",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "encoding-index-simpchinese"
|
||||
version = "1.20141219.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d87a7194909b9118fc707194baa434a4e3b0fb6a5a757c73c3adb07aa25031f7"
|
||||
dependencies = [
|
||||
"encoding_index_tests",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "encoding-index-singlebyte"
|
||||
version = "1.20141219.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3351d5acffb224af9ca265f435b859c7c01537c0849754d3db3fdf2bfe2ae84a"
|
||||
dependencies = [
|
||||
"encoding_index_tests",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "encoding-index-tradchinese"
|
||||
version = "1.20141219.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd0e20d5688ce3cab59eb3ef3a2083a5c77bf496cb798dc6fcdb75f323890c18"
|
||||
dependencies = [
|
||||
"encoding_index_tests",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "encoding_index_tests"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a246d82be1c9d791c5dfde9a2bd045fc3cbba3fa2b11ad558f27d01712f00569"
|
||||
|
||||
[[package]]
|
||||
name = "fallible-iterator"
|
||||
version = "0.2.0"
|
||||
@@ -436,7 +390,7 @@ dependencies = [
|
||||
"glassbench",
|
||||
"image",
|
||||
"num-traits",
|
||||
"png 0.17.2",
|
||||
"png",
|
||||
"resize",
|
||||
"rgb",
|
||||
"thiserror",
|
||||
@@ -444,13 +398,38 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "fastrand"
|
||||
version = "1.6.0"
|
||||
version = "1.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "779d043b6a0b90cc4c0ed7ee380a6504394cee7efd7db050e3774eee387324b2"
|
||||
checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf"
|
||||
dependencies = [
|
||||
"instant",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.0.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crc32fast",
|
||||
"libc",
|
||||
"miniz_oxide 0.4.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "flume"
|
||||
version = "0.10.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "843c03199d0c0ca54bc1ea90ac0d507274c28abcc4f691ae8b4eaa375087c76a"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"nanorand",
|
||||
"pin-project",
|
||||
"spin",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "form_urlencoded"
|
||||
version = "1.0.1"
|
||||
@@ -462,14 +441,28 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.3"
|
||||
name = "futures-core"
|
||||
version = "0.3.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753"
|
||||
checksum = "0c09fd04b7e4073ac7156a9539b57a484a8ea920f79c7c675d05d289ab6110d3"
|
||||
|
||||
[[package]]
|
||||
name = "futures-sink"
|
||||
version = "0.3.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "21163e139fa306126e6eedaf49ecdb4588f939600f0b1e770f4205ee4b7fa868"
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d39cd93900197114fa1fcb7ae84ca742095eed9442088988ae74fa744e930e77"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"js-sys",
|
||||
"libc",
|
||||
"wasi",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -518,6 +511,12 @@ dependencies = [
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "half"
|
||||
version = "1.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.9.1"
|
||||
@@ -576,23 +575,33 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "image"
|
||||
version = "0.23.14"
|
||||
version = "0.24.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "24ffcb7e7244a9bf19d35bf2883b9c080c4ced3c07a9895572178cdb8f13f6a1"
|
||||
checksum = "db207d030ae38f1eb6f240d5a1c1c88ff422aa005d10f8c6c6fc5e75286ab30e"
|
||||
dependencies = [
|
||||
"bytemuck",
|
||||
"byteorder",
|
||||
"color_quant",
|
||||
"exr",
|
||||
"gif",
|
||||
"jpeg-decoder",
|
||||
"jpeg-decoder 0.2.2",
|
||||
"num-iter",
|
||||
"num-rational",
|
||||
"num-traits",
|
||||
"png 0.16.8",
|
||||
"png",
|
||||
"scoped_threadpool",
|
||||
"tiff",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "inflate"
|
||||
version = "0.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1cdb29978cc5797bd8dcc8e5bf7de604891df2a8dc576973d71a281e916db2ff"
|
||||
dependencies = [
|
||||
"adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "instant"
|
||||
version = "0.1.12"
|
||||
@@ -628,10 +637,25 @@ name = "jpeg-decoder"
|
||||
version = "0.1.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "229d53d58899083193af11e15917b5640cd40b29ff475a1fe4ef725deb02d0f2"
|
||||
|
||||
[[package]]
|
||||
name = "jpeg-decoder"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "105fb082d64e2100074587f59a74231f771750c664af903f1f9f76c9dedfc6f1"
|
||||
dependencies = [
|
||||
"rayon",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.56"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04"
|
||||
dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.4.0"
|
||||
@@ -639,10 +663,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.112"
|
||||
name = "lebe"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125"
|
||||
checksum = "7efd1d698db0759e6ef11a7cd44407407399a910c774dd804c64c032da7826ff"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.119"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1bf2e165bb3457c8e098ea76f3e3bc9db55f87aa90d52d0e6be741470916aaa4"
|
||||
|
||||
[[package]]
|
||||
name = "libgit2-sys"
|
||||
@@ -669,9 +699,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "libz-sys"
|
||||
version = "1.1.3"
|
||||
version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "de5435b8549c16d423ed0c03dbaafe57cf6c3344744f1242520d59c9d8ecec66"
|
||||
checksum = "6f35facd4a5673cb5a48822be2be1d4236c1c99cb4113cab7061ac720d5bf859"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
@@ -681,9 +711,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.5"
|
||||
version = "0.4.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712a4d093c9976e24e7dbca41db895dabcbac38eb5f4045393d17a95bdfb1109"
|
||||
checksum = "88943dd7ef4a2e5a4bfa2753aaab3013e34ce2533d1996fb18ef591e315e2b3b"
|
||||
dependencies = [
|
||||
"scopeguard",
|
||||
]
|
||||
@@ -727,15 +757,6 @@ dependencies = [
|
||||
"lazy_static",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "791daaae1ed6889560f8c4359194f56648355540573244a5448a83ba1ecc7435"
|
||||
dependencies = [
|
||||
"adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.4.4"
|
||||
@@ -746,6 +767,15 @@ dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2b29bd4bc3f33391105ebee3589c19197c4271e3e5a9ec9bfe8127eeff8f082"
|
||||
dependencies = [
|
||||
"adler",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "0.7.14"
|
||||
@@ -769,10 +799,19 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ntapi"
|
||||
version = "0.3.6"
|
||||
name = "nanorand"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44"
|
||||
checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ntapi"
|
||||
version = "0.3.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
@@ -800,9 +839,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "num-rational"
|
||||
version = "0.3.2"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07"
|
||||
checksum = "d41702bd167c2df5520b384281bc111a4b5efcf7fbc4c9c222c815b07e0a6a6a"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"num-integer",
|
||||
@@ -830,9 +869,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.9.0"
|
||||
version = "1.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5"
|
||||
checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9"
|
||||
|
||||
[[package]]
|
||||
name = "open"
|
||||
@@ -881,6 +920,26 @@ version = "2.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
|
||||
|
||||
[[package]]
|
||||
name = "pin-project"
|
||||
version = "1.0.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e"
|
||||
dependencies = [
|
||||
"pin-project-internal",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-internal"
|
||||
version = "1.0.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.24"
|
||||
@@ -889,27 +948,14 @@ checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe"
|
||||
|
||||
[[package]]
|
||||
name = "png"
|
||||
version = "0.16.8"
|
||||
version = "0.17.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c3287920cb847dee3de33d301c463fba14dda99db24214ddf93f83d3021f4c6"
|
||||
checksum = "dc38c0ad57efb786dd57b9864e5b18bae478c00c824dc55a38bbc9da95dde3ba"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"crc32fast",
|
||||
"deflate 0.8.6",
|
||||
"miniz_oxide 0.3.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "png"
|
||||
version = "0.17.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c845088517daa61e8a57eee40309347cea13f273694d1385c553e7a57127763b"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"crc32fast",
|
||||
"deflate 0.9.1",
|
||||
"encoding",
|
||||
"miniz_oxide 0.4.4",
|
||||
"deflate",
|
||||
"miniz_oxide 0.5.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -923,9 +969,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.14"
|
||||
version = "1.0.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "47aa80447ce4daf1717500037052af176af5d38cc3e571d9ec1c7353fc10c87d"
|
||||
checksum = "864d3e96a899863136fc6e99f3d7cae289dafe43bf2c5ac19b70df7210c0a145"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
@@ -957,9 +1003,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.2.10"
|
||||
version = "0.2.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff"
|
||||
checksum = "8380fe0152551244f0747b1bf41737e0f8a74f97a14ccefd1148187271634f3c"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
@@ -1001,9 +1047,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rgb"
|
||||
version = "0.8.31"
|
||||
version = "0.8.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a374af9a0e5fdcdd98c1c7b64f05004f9ea2555b6c75f211daa81268a3c50f1"
|
||||
checksum = "e74fdc210d8f24a7dbfedc13b04ba5764f5232754ccebfdf5fff1bad791ccbc6"
|
||||
dependencies = [
|
||||
"bytemuck",
|
||||
]
|
||||
@@ -1043,18 +1089,18 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.133"
|
||||
version = "1.0.136"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97565067517b60e2d1ea8b268e59ce036de907ac523ad83a0475da04e818989a"
|
||||
checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.133"
|
||||
version = "1.0.136"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed201699328568d8d08208fdd080e3ff594e6c422e438b6705905da01005d537"
|
||||
checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -1063,9 +1109,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.74"
|
||||
version = "1.0.79"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee2bb9cd061c5865d345bb02ca49fcef1391741b672b54a0bf7b679badec3142"
|
||||
checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95"
|
||||
dependencies = [
|
||||
"itoa 1.0.1",
|
||||
"ryu",
|
||||
@@ -1094,9 +1140,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.7.0"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ecab6c735a6bb4139c0caafd0cc3635748bbb3acf4550e8138122099251f309"
|
||||
checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83"
|
||||
|
||||
[[package]]
|
||||
name = "spin"
|
||||
version = "0.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "511254be0c5bcf062b019a6c89c01a664aa359ded62f78aa72c6fc137c0590e5"
|
||||
dependencies = [
|
||||
"lock_api",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "svg"
|
||||
@@ -1106,9 +1161,9 @@ checksum = "3bdb25a4593d6656239319426f4025f7a658157e25e89f0e0319d7516d46042d"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.85"
|
||||
version = "1.0.86"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a684ac3dcd8913827e18cd09a68384ee66c1de24157e3c556c9ab16d85695fb7"
|
||||
checksum = "8a65b3f4ffa0092e9887669db0eae07941f023991ab58ea44da8fe8e2d511c6b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -1164,13 +1219,22 @@ dependencies = [
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tiff"
|
||||
version = "0.6.1"
|
||||
name = "threadpool"
|
||||
version = "1.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9a53f4706d65497df0c4349241deddf35f84cee19c87ed86ea8ca590f4464437"
|
||||
checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa"
|
||||
dependencies = [
|
||||
"jpeg-decoder",
|
||||
"miniz_oxide 0.4.4",
|
||||
"num_cpus",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tiff"
|
||||
version = "0.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0247608e998cb6ce39dfc8f4a16c50361ce71e5b52e6d24ea1227ea8ea8ee0b2"
|
||||
dependencies = [
|
||||
"flate2",
|
||||
"jpeg-decoder 0.1.22",
|
||||
"weezl",
|
||||
]
|
||||
|
||||
@@ -1216,9 +1280,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "unicode-segmentation"
|
||||
version = "1.8.0"
|
||||
version = "1.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b"
|
||||
checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-width"
|
||||
@@ -1262,6 +1326,60 @@ version = "0.10.2+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.79"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"wasm-bindgen-macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-backend"
|
||||
version = "0.2.79"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.79"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"wasm-bindgen-macro-support",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro-support"
|
||||
version = "0.2.79"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"wasm-bindgen-backend",
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-shared"
|
||||
version = "0.2.79"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2"
|
||||
|
||||
[[package]]
|
||||
name = "weezl"
|
||||
version = "0.1.5"
|
||||
|
||||
+3
-3
@@ -20,10 +20,10 @@ thiserror = "1.0.30"
|
||||
|
||||
[dev-dependencies]
|
||||
glassbench = "0.3.1"
|
||||
image = "0.23.14"
|
||||
image = "0.24.1"
|
||||
resize = "0.7.2"
|
||||
rgb = "0.8.31"
|
||||
png = "0.17.2"
|
||||
rgb = "0.8.32"
|
||||
png = "0.17.5"
|
||||
|
||||
|
||||
[[bench]]
|
||||
|
||||
@@ -13,10 +13,11 @@ about resizing with respect to color space._
|
||||
Supported pixel formats and available optimisations:
|
||||
- `U8` - one `u8` component per pixel:
|
||||
- native Rust-code without forced SIMD
|
||||
- SSE4.1 (partial)
|
||||
- AVX2
|
||||
- `U8x3` - three `u8` components per pixel (e.g. RGB):
|
||||
- native Rust-code without forced SIMD
|
||||
- SSE4.1 (auto-vectorization)
|
||||
- SSE4.1 (partial)
|
||||
- AVX2
|
||||
- `U8x4` - four `u8` components per pixel (RGBA, RGBx, CMYK and other):
|
||||
- native Rust-code without forced SIMD
|
||||
@@ -24,6 +25,8 @@ Supported pixel formats and available optimisations:
|
||||
- AVX2
|
||||
- `U16x3` - three `u16` components per pixel (e.g. RGB):
|
||||
- native Rust-code without forced SIMD
|
||||
- SSE4.1
|
||||
- AVX2
|
||||
- `I32` - one `i32` component per pixel:
|
||||
- native Rust-code without forced SIMD
|
||||
- `F32` - one `f32` component per pixel:
|
||||
@@ -34,14 +37,14 @@ Supported pixel formats and available optimisations:
|
||||
Environment:
|
||||
- CPU: Intel(R) Core(TM) i7-6700K CPU @ 4.00GHz
|
||||
- RAM: DDR4 3000 MHz
|
||||
- Ubuntu 20.04 (linux 5.11)
|
||||
- Rust 1.57.1
|
||||
- fast_image_resize = "0.6.0"
|
||||
- Ubuntu 20.04 (linux 5.13)
|
||||
- Rust 1.59.0
|
||||
- fast_image_resize = "0.8.0"
|
||||
- glassbench = "0.3.1"
|
||||
- `rustflags = ["-C", "llvm-args=-x86-branches-within-32B-boundaries"]`
|
||||
|
||||
Other Rust libraries used to compare of resizing speed:
|
||||
- image = "0.23.14" (<https://crates.io/crates/image>)
|
||||
- image = "0.24.1" (<https://crates.io/crates/image>)
|
||||
- resize = "0.7.2" (<https://crates.io/crates/resize>)
|
||||
|
||||
Resize algorithms:
|
||||
@@ -61,11 +64,11 @@ Pipeline:
|
||||
|
||||
| | Nearest | Bilinear | CatmullRom | Lanczos3 |
|
||||
|------------|:-------:|:--------:|:----------:|:--------:|
|
||||
| image | 96.466 | 186.243 | 268.888 | 358.235 |
|
||||
| resize | 15.812 | 68.793 | 125.291 | 181.471 |
|
||||
| fir rust | 0.495 | 56.372 | 93.182 | 127.870 |
|
||||
| fir sse4.1 | - | 44.775 | 56.014 | 78.759 |
|
||||
| fir avx2 | - | 11.290 | 14.731 | 20.678 |
|
||||
| image | 51.959 | 114.977 | 189.989 | 271.316 |
|
||||
| resize | 15.317 | 65.768 | 119.557 | 173.124 |
|
||||
| fir rust | 0.488 | 54.104 | 90.222 | 131.518 |
|
||||
| fir sse4.1 | - | 38.046 | 43.538 | 60.816 |
|
||||
| fir avx2 | - | 10.726 | 14.077 | 19.809 |
|
||||
|
||||
### Resize RGBA image (U8x4) 4928x3279 => 852x567
|
||||
|
||||
@@ -78,11 +81,11 @@ Pipeline:
|
||||
|
||||
| | Nearest | Bilinear | CatmullRom | Lanczos3 |
|
||||
|------------|:-------:|:--------:|:----------:|:--------:|
|
||||
| image | 102.809 | 185.787 | 266.163 | 356.038 |
|
||||
| resize | 18.723 | 83.072 | 156.063 | 229.400 |
|
||||
| fir rust | 12.518 | 65.821 | 92.371 | 122.981 |
|
||||
| fir sse4.1 | 9.529 | 21.008 | 27.444 | 35.534 |
|
||||
| fir avx2 | 7.622 | 15.755 | 19.369 | 25.184 |
|
||||
| image | 52.430 | 109.720 | 181.220 | 261.242 |
|
||||
| resize | 18.326 | 79.389 | 149.271 | 219.359 |
|
||||
| fir rust | 12.123 | 62.975 | 88.053 | 117.095 |
|
||||
| fir sse4.1 | 9.125 | 20.154 | 26.302 | 34.136 |
|
||||
| fir avx2 | 7.401 | 15.160 | 18.672 | 23.940 |
|
||||
|
||||
### Resize grayscale image (U8) 4928x3279 => 852x567
|
||||
|
||||
@@ -94,12 +97,13 @@ Pipeline:
|
||||
has converted into grayscale image with one byte per pixel.
|
||||
- Numbers in table is mean duration of image resizing in milliseconds.
|
||||
|
||||
| | Nearest | Bilinear | CatmullRom | Lanczos3 |
|
||||
|----------|:-------:|:--------:|:----------:|:--------:|
|
||||
| image | 80.937 | 132.497 | 174.721 | 219.534 |
|
||||
| resize | 10.107 | 25.514 | 49.871 | 84.692 |
|
||||
| fir rust | 0.208 | 23.255 | 26.471 | 37.642 |
|
||||
| fir avx2 | - | 9.927 | 8.054 | 12.298 |
|
||||
| | Nearest | Bilinear | CatmullRom | Lanczos3 |
|
||||
|------------|:-------:|:--------:|:----------:|:--------:|
|
||||
| image | 47.712 | 71.624 | 108.758 | 161.086 |
|
||||
| resize | 9.353 | 24.461 | 47.598 | 80.996 |
|
||||
| fir rust | 0.198 | 20.150 | 22.275 | 32.542 |
|
||||
| fir sse4.1 | - | 18.201 | 18.971 | 27.704 |
|
||||
| fir avx2 | - | 9.608 | 7.776 | 11.746 |
|
||||
|
||||
### Resize RGB16 image (U16x3) 4928x3279 => 852x567
|
||||
|
||||
@@ -108,13 +112,16 @@ Pipeline:
|
||||
`src_image => resize => dst_image`
|
||||
|
||||
- Source image [nasa-4928x3279.png](https://github.com/Cykooz/fast_image_resize/blob/main/data/nasa-4928x3279.png)
|
||||
has converted into RGB16 image.
|
||||
- Numbers in table is mean duration of image resizing in milliseconds.
|
||||
|
||||
| | Nearest | Bilinear | CatmullRom | Lanczos3 |
|
||||
|----------|:-------:|:--------:|:----------:|:--------:|
|
||||
| image | 98.962 | 180.516 | 255.045 | 335.265 |
|
||||
| resize | 16.504 | 66.861 | 120.973 | 174.806 |
|
||||
| fir rust | 0.800 | 53.874 | 89.453 | 123.963 |
|
||||
| | Nearest | Bilinear | CatmullRom | Lanczos3 |
|
||||
|------------|:-------:|:--------:|:----------:|:--------:|
|
||||
| image | 52.450 | 112.628 | 180.484 | 255.605 |
|
||||
| resize | 16.983 | 67.382 | 121.378 | 174.913 |
|
||||
| fir rust | 0.785 | 58.209 | 95.504 | 131.758 |
|
||||
| fir sse4.1 | - | 38.562 | 63.013 | 88.722 |
|
||||
| fir avx2 | - | 32.981 | 49.626 | 58.803 |
|
||||
|
||||
## Examples
|
||||
|
||||
|
||||
@@ -70,11 +70,11 @@ pub fn bench_downscale_rgb16(bench: &mut Bench) {
|
||||
.flat_map(|&c| c.to_le_bytes())
|
||||
.collect();
|
||||
let mut cpu_ext_and_name = vec![(CpuExtensions::None, "rust")];
|
||||
// #[cfg(target_arch = "x86_64")]
|
||||
// {
|
||||
// cpu_ext_and_name.push((CpuExtensions::Sse4_1, "sse4.1"));
|
||||
// cpu_ext_and_name.push((CpuExtensions::Avx2, "avx2"));
|
||||
// }
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
{
|
||||
cpu_ext_and_name.push((CpuExtensions::Sse4_1, "sse4.1"));
|
||||
cpu_ext_and_name.push((CpuExtensions::Avx2, "avx2"));
|
||||
}
|
||||
for (cpu_ext, ext_name) in cpu_ext_and_name {
|
||||
for alg_name in alg_names {
|
||||
let src_image_data = Image::from_vec_u8(
|
||||
|
||||
@@ -67,6 +67,7 @@ pub fn bench_downscale_u8(bench: &mut Bench) {
|
||||
let mut cpu_ext_and_name = vec![(CpuExtensions::None, "rust")];
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
{
|
||||
cpu_ext_and_name.push((CpuExtensions::Sse4_1, "sse4.1"));
|
||||
cpu_ext_and_name.push((CpuExtensions::Avx2, "avx2"));
|
||||
}
|
||||
for (cpu_ext, ext_name) in cpu_ext_and_name {
|
||||
|
||||
@@ -54,14 +54,14 @@ fn get_big_u16x3_source_image() -> Image<'static> {
|
||||
|
||||
fn get_big_i32_image() -> Image<'static> {
|
||||
let img = utils::get_big_luma16_image();
|
||||
let img_data: Vec<u32> = img
|
||||
let img_data: Vec<u8> = img
|
||||
.as_raw()
|
||||
.iter()
|
||||
.map(|&p| p as u32 * (i16::MAX as u32 + 1))
|
||||
.flat_map(|&p| (p as u32 * (i16::MAX as u32 + 1)).to_le_bytes())
|
||||
.collect();
|
||||
let width = img.width();
|
||||
let height = img.height();
|
||||
Image::from_vec_u32(
|
||||
Image::from_vec_u8(
|
||||
NonZeroU32::new(width).unwrap(),
|
||||
NonZeroU32::new(height).unwrap(),
|
||||
img_data,
|
||||
@@ -294,10 +294,12 @@ pub fn main() {
|
||||
native_lanczos3_i32_bench(&mut bench);
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
{
|
||||
u8_lanczos3_bench(&mut bench, CpuExtensions::Sse4_1, "u8 lanczos3 sse4.1");
|
||||
u8_lanczos3_bench(&mut bench, CpuExtensions::Avx2, "u8 lanczos3 avx2");
|
||||
|
||||
u8x3_lanczos3_bench(&mut bench, CpuExtensions::Sse4_1, "u8x3 lanczos3 sse4.1");
|
||||
u8x3_lanczos3_bench(&mut bench, CpuExtensions::Avx2, "u8x3 lanczos3 avx2");
|
||||
u16x3_lanczos3_bench(&mut bench, CpuExtensions::Sse4_1, "u16x3 lanczos3 sse4.1");
|
||||
u16x3_lanczos3_bench(&mut bench, CpuExtensions::Avx2, "u16x3 lanczos3 avx2");
|
||||
|
||||
u8x4_lanczos3_bench(&mut bench, CpuExtensions::Sse4_1, "u8x4 lanczos3 sse4.1");
|
||||
|
||||
@@ -16,10 +16,12 @@ mod u16x3;
|
||||
mod u8x1;
|
||||
mod u8x3;
|
||||
mod u8x4;
|
||||
mod vertical_u16;
|
||||
mod vertical_u8;
|
||||
|
||||
pub(crate) trait Convolution
|
||||
where
|
||||
Self: Pixel + Sized,
|
||||
Self: Pixel,
|
||||
{
|
||||
fn horiz_convolution(
|
||||
src_image: TypedImageView<Self>,
|
||||
|
||||
@@ -0,0 +1,314 @@
|
||||
use std::arch::x86_64::*;
|
||||
|
||||
use crate::convolution::{optimisations, Coefficients};
|
||||
use crate::image_view::{FourRows, FourRowsMut, TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::U16x3;
|
||||
use crate::simd_utils;
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn horiz_convolution(
|
||||
src_image: TypedImageView<U16x3>,
|
||||
mut dst_image: TypedImageViewMut<U16x3>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
let (values, window_size, bounds_per_pixel) =
|
||||
(coeffs.values, coeffs.window_size, coeffs.bounds);
|
||||
|
||||
let normalizer_guard = optimisations::NormalizerGuard32::new(values);
|
||||
let coefficients_chunks = normalizer_guard.normalized_chunks(window_size, &bounds_per_pixel);
|
||||
let dst_height = dst_image.height().get();
|
||||
|
||||
let src_iter = src_image.iter_4_rows(offset, dst_height + offset);
|
||||
let dst_iter = dst_image.iter_4_rows_mut();
|
||||
for (src_rows, dst_rows) in src_iter.zip(dst_iter) {
|
||||
unsafe {
|
||||
horiz_convolution_four_rows(
|
||||
src_rows,
|
||||
dst_rows,
|
||||
&coefficients_chunks,
|
||||
&normalizer_guard,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let mut yy = dst_height - dst_height % 4;
|
||||
while yy < dst_height {
|
||||
unsafe {
|
||||
horiz_convolution_one_row(
|
||||
src_image.get_row(yy + offset).unwrap(),
|
||||
dst_image.get_row_mut(yy).unwrap(),
|
||||
&coefficients_chunks,
|
||||
&normalizer_guard,
|
||||
);
|
||||
}
|
||||
yy += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/// For safety, it is necessary to ensure the following conditions:
|
||||
/// - length of all rows in src_rows must be equal
|
||||
/// - length of all rows in dst_rows must be equal
|
||||
/// - coefficients_chunks.len() == dst_rows.0.len()
|
||||
/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.0.len()
|
||||
#[target_feature(enable = "avx2")]
|
||||
unsafe fn horiz_convolution_four_rows(
|
||||
src_rows: FourRows<U16x3>,
|
||||
dst_rows: FourRowsMut<U16x3>,
|
||||
coefficients_chunks: &[optimisations::CoefficientsI32Chunk],
|
||||
normalizer_guard: &optimisations::NormalizerGuard32,
|
||||
) {
|
||||
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
|
||||
let s_rows = [s_row0, s_row1, s_row2, s_row3];
|
||||
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
|
||||
let d_rows = [d_row0, d_row1, d_row2, d_row3];
|
||||
let precision = normalizer_guard.precision();
|
||||
let half_error = 1i64 << (precision - 1);
|
||||
let mut rg_buf = [0i64; 4];
|
||||
let mut rg_bb_buf = [0i64; 4];
|
||||
let mut bbb_buf = [0i64; 4];
|
||||
|
||||
/*
|
||||
|R G B | |R G B | |R G | - |B | |R G B | |R G B | |R |
|
||||
|0001 0203 0405| |0607 0809 1011| |1213 1415| - |0001| |0203 0405 0607| |0809 1011 1213| |1415|
|
||||
|
||||
Shuffle to extract RG components of pixels 0 and 3 as i64:
|
||||
lo: -1, -1, -1, -1, -1, -1, 3, 2, -1, -1, -1, -1, -1, -1, 1, 0
|
||||
hi: -1, -1, -1, -1, -1, -1, 5, 4, -1, -1, -1, -1, -1, -1, 3, 2
|
||||
|
||||
Shuffle to extract RG components of pixels 1 and 4 as i64:
|
||||
lo: -1, -1, -1, -1, -1, -1, 9, 8, -1, -1, -1, -1, -1, -1, 7, 6
|
||||
hi: -1, -1, -1, -1, -1, -1, 11, 10, -1, -1, -1, -1, -1, -1, 9, 8
|
||||
|
||||
Shuffle to extract RG components of pixel 2 and BB of pixels 2-3 as i64:
|
||||
lo: -1, -1, -1, -1, -1, -1, 15, 14, -1, -1, -1, -1, -1, -1, 13, 12
|
||||
hi: -1, -1, -1, -1, -1, -1, 7, 6, -1, -1, -1, -1, -1, -1, 1, 0
|
||||
|
||||
Shuffle to extract BB components of pixels 0, 1 and 4 as i64:
|
||||
lo: -1, -1, -1, -1, -1, -1, 11, 10, -1, -1, -1, -1, -1, -1, 5, 4
|
||||
hi: -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 13, 12
|
||||
*/
|
||||
|
||||
let rg03_shuffle = _mm256_set_m128i(
|
||||
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 5, 4, -1, -1, -1, -1, -1, -1, 3, 2),
|
||||
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 3, 2, -1, -1, -1, -1, -1, -1, 1, 0),
|
||||
);
|
||||
let rg14_shuffle = _mm256_set_m128i(
|
||||
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 11, 10, -1, -1, -1, -1, -1, -1, 9, 8),
|
||||
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 9, 8, -1, -1, -1, -1, -1, -1, 7, 6),
|
||||
);
|
||||
let rg3_b3b4_shuffle = _mm256_set_m128i(
|
||||
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 7, 6, -1, -1, -1, -1, -1, -1, 1, 0),
|
||||
_mm_set_epi8(
|
||||
-1, -1, -1, -1, -1, -1, 15, 14, -1, -1, -1, -1, -1, -1, 13, 12,
|
||||
),
|
||||
);
|
||||
let b1b2_b5_shuffle = _mm256_set_m128i(
|
||||
_mm_set_epi8(
|
||||
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 13, 12,
|
||||
),
|
||||
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 11, 10, -1, -1, -1, -1, -1, -1, 5, 4),
|
||||
);
|
||||
|
||||
let width = s_row0.len();
|
||||
|
||||
for (dst_x, coeffs_chunk) in coefficients_chunks.iter().enumerate() {
|
||||
let mut x: usize = coeffs_chunk.start as usize;
|
||||
let mut rg_sum = [_mm256_set1_epi8(0); 4];
|
||||
let mut rg_bb_sum = [_mm256_set1_epi8(0); 4];
|
||||
let mut bbb_sum = [_mm256_set1_epi8(0); 4];
|
||||
|
||||
let mut coeffs = coeffs_chunk.values;
|
||||
let end_x = x + coeffs.len();
|
||||
|
||||
if width - end_x >= 1 {
|
||||
let coeffs_by_5 = coeffs.chunks_exact(5);
|
||||
coeffs = coeffs_by_5.remainder();
|
||||
|
||||
for k in coeffs_by_5 {
|
||||
let coeff0033_i64x4 =
|
||||
_mm256_set_epi64x(k[3] as i64, k[3] as i64, k[0] as i64, k[0] as i64);
|
||||
let coeff1144_i64x2 =
|
||||
_mm256_set_epi64x(k[4] as i64, k[4] as i64, k[1] as i64, k[1] as i64);
|
||||
let coeff2223_i64x2 =
|
||||
_mm256_set_epi64x(k[3] as i64, k[2] as i64, k[2] as i64, k[2] as i64);
|
||||
let coeff014_i64x2 = _mm256_set_epi64x(0, k[4] as i64, k[1] as i64, k[0] as i64);
|
||||
|
||||
for i in 0..4 {
|
||||
let source = simd_utils::loadu_si256(s_rows[i], x);
|
||||
|
||||
let rg03_i64x4 = _mm256_shuffle_epi8(source, rg03_shuffle);
|
||||
rg_sum[i] =
|
||||
_mm256_add_epi64(rg_sum[i], _mm256_mul_epi32(rg03_i64x4, coeff0033_i64x4));
|
||||
|
||||
let rg14_i64x4 = _mm256_shuffle_epi8(source, rg14_shuffle);
|
||||
rg_sum[i] =
|
||||
_mm256_add_epi64(rg_sum[i], _mm256_mul_epi32(rg14_i64x4, coeff1144_i64x2));
|
||||
|
||||
let rg_bb_i64x4 = _mm256_shuffle_epi8(source, rg3_b3b4_shuffle);
|
||||
rg_bb_sum[i] = _mm256_add_epi64(
|
||||
rg_bb_sum[i],
|
||||
_mm256_mul_epi32(rg_bb_i64x4, coeff2223_i64x2),
|
||||
);
|
||||
|
||||
let bbb_i64x4 = _mm256_shuffle_epi8(source, b1b2_b5_shuffle);
|
||||
bbb_sum[i] =
|
||||
_mm256_add_epi64(bbb_sum[i], _mm256_mul_epi32(bbb_i64x4, coeff014_i64x2));
|
||||
}
|
||||
x += 5;
|
||||
}
|
||||
}
|
||||
|
||||
for &k in coeffs {
|
||||
let coeff_i64x4 = _mm256_set1_epi64x(k as i64);
|
||||
|
||||
for i in 0..4 {
|
||||
let &pixel = s_rows[i].get_unchecked(x);
|
||||
let rgb_i64x4 =
|
||||
_mm256_set_epi64x(0, pixel.0[2] as i64, pixel.0[1] as i64, pixel.0[0] as i64);
|
||||
rg_bb_sum[i] =
|
||||
_mm256_add_epi64(rg_bb_sum[i], _mm256_mul_epi32(rgb_i64x4, coeff_i64x4));
|
||||
}
|
||||
x += 1;
|
||||
}
|
||||
|
||||
for i in 0..4 {
|
||||
_mm256_storeu_si256((&mut rg_buf).as_mut_ptr() as *mut __m256i, rg_sum[i]);
|
||||
_mm256_storeu_si256((&mut rg_bb_buf).as_mut_ptr() as *mut __m256i, rg_bb_sum[i]);
|
||||
_mm256_storeu_si256((&mut bbb_buf).as_mut_ptr() as *mut __m256i, bbb_sum[i]);
|
||||
let dst_pixel = d_rows[i].get_unchecked_mut(dst_x);
|
||||
dst_pixel.0[0] =
|
||||
normalizer_guard.clip(rg_buf[0] + rg_buf[2] + rg_bb_buf[0] + half_error);
|
||||
dst_pixel.0[1] =
|
||||
normalizer_guard.clip(rg_buf[1] + rg_buf[3] + rg_bb_buf[1] + half_error);
|
||||
dst_pixel.0[2] = normalizer_guard.clip(
|
||||
rg_bb_buf[2] + rg_bb_buf[3] + bbb_buf[0] + bbb_buf[1] + bbb_buf[2] + half_error,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// For safety, it is necessary to ensure the following conditions:
|
||||
/// - bounds.len() == dst_row.len()
|
||||
/// - coefficients_chunks.len() == dst_row.len()
|
||||
/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.len()
|
||||
#[target_feature(enable = "avx2")]
|
||||
unsafe fn horiz_convolution_one_row(
|
||||
src_row: &[U16x3],
|
||||
dst_row: &mut [U16x3],
|
||||
coefficients_chunks: &[optimisations::CoefficientsI32Chunk],
|
||||
normalizer_guard: &optimisations::NormalizerGuard32,
|
||||
) {
|
||||
let precision = normalizer_guard.precision();
|
||||
let half_error = 1i64 << (precision - 1);
|
||||
let mut rg_buf = [0i64; 4];
|
||||
let mut rg_bb_buf = [0i64; 4];
|
||||
let mut bbb_buf = [0i64; 4];
|
||||
|
||||
/*
|
||||
|R G B | |R G B | |R G | - |B | |R G B | |R G B | |R |
|
||||
|0001 0203 0405| |0607 0809 1011| |1213 1415| - |0001| |0203 0405 0607| |0809 1011 1213| |1415|
|
||||
|
||||
Shuffle to extract RG components of pixels 0 and 3 as i64:
|
||||
lo: -1, -1, -1, -1, -1, -1, 3, 2, -1, -1, -1, -1, -1, -1, 1, 0
|
||||
hi: -1, -1, -1, -1, -1, -1, 5, 4, -1, -1, -1, -1, -1, -1, 3, 2
|
||||
|
||||
Shuffle to extract RG components of pixels 1 and 4 as i64:
|
||||
lo: -1, -1, -1, -1, -1, -1, 9, 8, -1, -1, -1, -1, -1, -1, 7, 6
|
||||
hi: -1, -1, -1, -1, -1, -1, 11, 10, -1, -1, -1, -1, -1, -1, 9, 8
|
||||
|
||||
Shuffle to extract RG components of pixel 2 and BB of pixels 2-3 as i64:
|
||||
lo: -1, -1, -1, -1, -1, -1, 15, 14, -1, -1, -1, -1, -1, -1, 13, 12
|
||||
hi: -1, -1, -1, -1, -1, -1, 7, 6, -1, -1, -1, -1, -1, -1, 1, 0
|
||||
|
||||
Shuffle to extract BB components of pixels 0, 1 and 4 as i64:
|
||||
lo: -1, -1, -1, -1, -1, -1, 11, 10, -1, -1, -1, -1, -1, -1, 5, 4
|
||||
hi: -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 13, 12
|
||||
*/
|
||||
|
||||
let rg03_shuffle = _mm256_set_m128i(
|
||||
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 5, 4, -1, -1, -1, -1, -1, -1, 3, 2),
|
||||
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 3, 2, -1, -1, -1, -1, -1, -1, 1, 0),
|
||||
);
|
||||
let rg14_shuffle = _mm256_set_m128i(
|
||||
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 11, 10, -1, -1, -1, -1, -1, -1, 9, 8),
|
||||
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 9, 8, -1, -1, -1, -1, -1, -1, 7, 6),
|
||||
);
|
||||
let rg3_b3b4_shuffle = _mm256_set_m128i(
|
||||
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 7, 6, -1, -1, -1, -1, -1, -1, 1, 0),
|
||||
_mm_set_epi8(
|
||||
-1, -1, -1, -1, -1, -1, 15, 14, -1, -1, -1, -1, -1, -1, 13, 12,
|
||||
),
|
||||
);
|
||||
let b1b2_b5_shuffle = _mm256_set_m128i(
|
||||
_mm_set_epi8(
|
||||
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 13, 12,
|
||||
),
|
||||
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 11, 10, -1, -1, -1, -1, -1, -1, 5, 4),
|
||||
);
|
||||
|
||||
let zero_i64x4 = _mm256_set1_epi8(0);
|
||||
|
||||
let width = src_row.len();
|
||||
|
||||
for (dst_x, &coeffs_chunk) in coefficients_chunks.iter().enumerate() {
|
||||
let mut x: usize = coeffs_chunk.start as usize;
|
||||
let mut rg_sum = zero_i64x4;
|
||||
let mut rg_bb_sum = zero_i64x4;
|
||||
let mut bbb_sum = zero_i64x4;
|
||||
|
||||
let mut coeffs = coeffs_chunk.values;
|
||||
let end_x = x + coeffs.len();
|
||||
|
||||
if width - end_x >= 1 {
|
||||
let coeffs_by_5 = coeffs.chunks_exact(5);
|
||||
coeffs = coeffs_by_5.remainder();
|
||||
|
||||
for k in coeffs_by_5 {
|
||||
let coeff0033_i64x4 =
|
||||
_mm256_set_epi64x(k[3] as i64, k[3] as i64, k[0] as i64, k[0] as i64);
|
||||
let coeff1144_i64x2 =
|
||||
_mm256_set_epi64x(k[4] as i64, k[4] as i64, k[1] as i64, k[1] as i64);
|
||||
let coeff2223_i64x2 =
|
||||
_mm256_set_epi64x(k[3] as i64, k[2] as i64, k[2] as i64, k[2] as i64);
|
||||
let coeff014_i64x2 = _mm256_set_epi64x(0, k[4] as i64, k[1] as i64, k[0] as i64);
|
||||
|
||||
let source = simd_utils::loadu_si256(src_row, x);
|
||||
|
||||
let rg03_i64x4 = _mm256_shuffle_epi8(source, rg03_shuffle);
|
||||
rg_sum = _mm256_add_epi64(rg_sum, _mm256_mul_epi32(rg03_i64x4, coeff0033_i64x4));
|
||||
|
||||
let rg14_i64x4 = _mm256_shuffle_epi8(source, rg14_shuffle);
|
||||
rg_sum = _mm256_add_epi64(rg_sum, _mm256_mul_epi32(rg14_i64x4, coeff1144_i64x2));
|
||||
|
||||
let rg_bb_i64x4 = _mm256_shuffle_epi8(source, rg3_b3b4_shuffle);
|
||||
rg_bb_sum =
|
||||
_mm256_add_epi64(rg_bb_sum, _mm256_mul_epi32(rg_bb_i64x4, coeff2223_i64x2));
|
||||
|
||||
let bbb_i64x4 = _mm256_shuffle_epi8(source, b1b2_b5_shuffle);
|
||||
bbb_sum = _mm256_add_epi64(bbb_sum, _mm256_mul_epi32(bbb_i64x4, coeff014_i64x2));
|
||||
|
||||
x += 5;
|
||||
}
|
||||
}
|
||||
|
||||
for &k in coeffs {
|
||||
let coeff_i64x4 = _mm256_set1_epi64x(k as i64);
|
||||
let &pixel = src_row.get_unchecked(x);
|
||||
let rgb_i64x4 =
|
||||
_mm256_set_epi64x(0, pixel.0[2] as i64, pixel.0[1] as i64, pixel.0[0] as i64);
|
||||
rg_bb_sum = _mm256_add_epi64(rg_bb_sum, _mm256_mul_epi32(rgb_i64x4, coeff_i64x4));
|
||||
|
||||
x += 1;
|
||||
}
|
||||
|
||||
_mm256_storeu_si256((&mut rg_buf).as_mut_ptr() as *mut __m256i, rg_sum);
|
||||
_mm256_storeu_si256((&mut rg_bb_buf).as_mut_ptr() as *mut __m256i, rg_bb_sum);
|
||||
_mm256_storeu_si256((&mut bbb_buf).as_mut_ptr() as *mut __m256i, bbb_sum);
|
||||
let dst_pixel = dst_row.get_unchecked_mut(dst_x);
|
||||
dst_pixel.0[0] = normalizer_guard.clip(rg_buf[0] + rg_buf[2] + rg_bb_buf[0] + half_error);
|
||||
dst_pixel.0[1] = normalizer_guard.clip(rg_buf[1] + rg_buf[3] + rg_bb_buf[1] + half_error);
|
||||
dst_pixel.0[2] = normalizer_guard
|
||||
.clip(rg_bb_buf[2] + rg_bb_buf[3] + bbb_buf[0] + bbb_buf[1] + bbb_buf[2] + half_error);
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,14 @@
|
||||
use super::{Coefficients, Convolution};
|
||||
use crate::convolution::vertical_u16::vert_convolution_u16;
|
||||
use crate::image_view::{TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::U16x3;
|
||||
use crate::CpuExtensions;
|
||||
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
mod avx2;
|
||||
mod native;
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
mod sse4;
|
||||
|
||||
impl Convolution for U16x3 {
|
||||
fn horiz_convolution(
|
||||
@@ -13,7 +18,13 @@ impl Convolution for U16x3 {
|
||||
coeffs: Coefficients,
|
||||
cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
native::horiz_convolution(src_image, dst_image, offset, coeffs);
|
||||
match cpu_extensions {
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Avx2 => avx2::horiz_convolution(src_image, dst_image, offset, coeffs),
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Sse4_1 => sse4::horiz_convolution(src_image, dst_image, offset, coeffs),
|
||||
_ => native::horiz_convolution(src_image, dst_image, offset, coeffs),
|
||||
}
|
||||
}
|
||||
|
||||
fn vert_convolution(
|
||||
@@ -22,6 +33,6 @@ impl Convolution for U16x3 {
|
||||
coeffs: Coefficients,
|
||||
cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
native::vert_convolution(src_image, dst_image, coeffs);
|
||||
vert_convolution_u16(src_image, dst_image, coeffs, cpu_extensions);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,37 +34,3 @@ pub(crate) fn horiz_convolution(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub(crate) fn vert_convolution(
|
||||
src_image: TypedImageView<U16x3>,
|
||||
mut dst_image: TypedImageViewMut<U16x3>,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
let (values, window_size, bounds) = (coeffs.values, coeffs.window_size, coeffs.bounds);
|
||||
|
||||
let normalizer_guard = optimisations::NormalizerGuard32::new(values);
|
||||
let precision = normalizer_guard.precision();
|
||||
let coefficients_chunks = normalizer_guard.normalized_chunks(window_size, &bounds);
|
||||
let initial = 1 << (precision - 1);
|
||||
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
for (&coeffs_chunk, dst_row) in coefficients_chunks.iter().zip(dst_rows) {
|
||||
let first_y_src = coeffs_chunk.start;
|
||||
let ks = coeffs_chunk.values;
|
||||
|
||||
for (x_src, dst_pixel) in dst_row.iter_mut().enumerate() {
|
||||
let mut ss = [initial; 3];
|
||||
let src_rows = src_image.iter_rows(first_y_src);
|
||||
for (&k, src_row) in ks.iter().zip(src_rows) {
|
||||
let src_pixel = unsafe { src_row.get_unchecked(x_src as usize) };
|
||||
for (i, s) in ss.iter_mut().enumerate() {
|
||||
*s += src_pixel.0[i] as i64 * (k as i64);
|
||||
}
|
||||
}
|
||||
for (i, s) in ss.iter().copied().enumerate() {
|
||||
dst_pixel.0[i] = normalizer_guard.clip(s);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
use std::arch::x86_64::*;
|
||||
|
||||
use crate::convolution::optimisations::CoefficientsI32Chunk;
|
||||
use crate::convolution::{optimisations, Coefficients};
|
||||
use crate::image_view::{FourRows, FourRowsMut, TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::U16x3;
|
||||
use crate::simd_utils;
|
||||
|
||||
// This code is based on C-implementation from Pillow-SIMD package for Python
|
||||
// https://github.com/uploadcare/pillow-simd
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn horiz_convolution(
|
||||
src_image: TypedImageView<U16x3>,
|
||||
mut dst_image: TypedImageViewMut<U16x3>,
|
||||
offset: u32,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
let (values, window_size, bounds_per_pixel) =
|
||||
(coeffs.values, coeffs.window_size, coeffs.bounds);
|
||||
|
||||
let normalizer_guard = optimisations::NormalizerGuard32::new(values);
|
||||
let coefficients_chunks = normalizer_guard.normalized_chunks(window_size, &bounds_per_pixel);
|
||||
let dst_height = dst_image.height().get();
|
||||
|
||||
let src_iter = src_image.iter_4_rows(offset, dst_height + offset);
|
||||
let dst_iter = dst_image.iter_4_rows_mut();
|
||||
for (src_rows, dst_rows) in src_iter.zip(dst_iter) {
|
||||
unsafe {
|
||||
horiz_convolution_8u4x(src_rows, dst_rows, &coefficients_chunks, &normalizer_guard);
|
||||
}
|
||||
}
|
||||
|
||||
let mut yy = dst_height - dst_height % 4;
|
||||
while yy < dst_height {
|
||||
unsafe {
|
||||
horiz_convolution_8u(
|
||||
src_image.get_row(yy + offset).unwrap(),
|
||||
dst_image.get_row_mut(yy).unwrap(),
|
||||
&coefficients_chunks,
|
||||
&normalizer_guard,
|
||||
);
|
||||
}
|
||||
yy += 1;
|
||||
}
|
||||
}
|
||||
|
||||
/// For safety, it is necessary to ensure the following conditions:
|
||||
/// - length of all rows in src_rows must be equal
|
||||
/// - length of all rows in dst_rows must be equal
|
||||
/// - coefficients_chunks.len() == dst_rows.0.len()
|
||||
/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.0.len()
|
||||
/// - precision <= MAX_COEFS_PRECISION
|
||||
#[target_feature(enable = "sse4.1")]
|
||||
unsafe fn horiz_convolution_8u4x(
|
||||
src_rows: FourRows<U16x3>,
|
||||
dst_rows: FourRowsMut<U16x3>,
|
||||
coefficients_chunks: &[CoefficientsI32Chunk],
|
||||
normalizer_guard: &optimisations::NormalizerGuard32,
|
||||
) {
|
||||
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
|
||||
let s_rows = [s_row0, s_row1, s_row2, s_row3];
|
||||
let (d_row0, d_row1, d_row2, d_row3) = dst_rows;
|
||||
let d_rows = [d_row0, d_row1, d_row2, d_row3];
|
||||
let precision = normalizer_guard.precision();
|
||||
let half_error = 1i64 << (precision - 1);
|
||||
let mut rg_buf = [0i64; 2];
|
||||
let mut bb_buf = [0i64; 2];
|
||||
|
||||
/*
|
||||
|R G B | |R G B | |R G |
|
||||
|0001 0203 0405| |0607 0809 1011| |1213 1415|
|
||||
|
||||
Shuffle to extract RG components of first pixel as i64:
|
||||
-1, -1, -1, -1, -1, -1, 3, 2, -1, -1, -1, -1, -1, -1, 1, 0
|
||||
|
||||
Shuffle to extract RG components of second pixel as i64:
|
||||
-1, -1, -1, -1, -1, -1, 9, 8, -1, -1, -1, -1, -1, -1, 7, 6
|
||||
|
||||
Shuffle to extract B components of two pixels as i64:
|
||||
-1, -1, -1, -1, -1, -1, 11, 10, -1, -1, -1, -1, -1, -1, 5, 4
|
||||
|
||||
*/
|
||||
|
||||
let rg0_shuffle = _mm_set_epi8(-1, -1, -1, -1, -1, -1, 3, 2, -1, -1, -1, -1, -1, -1, 1, 0);
|
||||
let rg1_shuffle = _mm_set_epi8(-1, -1, -1, -1, -1, -1, 9, 8, -1, -1, -1, -1, -1, -1, 7, 6);
|
||||
let bb_shuffle = _mm_set_epi8(-1, -1, -1, -1, -1, -1, 11, 10, -1, -1, -1, -1, -1, -1, 5, 4);
|
||||
|
||||
let width = s_row0.len();
|
||||
|
||||
for (dst_x, coeffs_chunk) in coefficients_chunks.iter().enumerate() {
|
||||
let mut x: usize = coeffs_chunk.start as usize;
|
||||
let mut rg_sum = [_mm_set1_epi8(0); 4];
|
||||
let mut bb_sum = [_mm_set1_epi8(0); 4];
|
||||
|
||||
let mut coeffs = coeffs_chunk.values;
|
||||
let end_x = x + coeffs.len();
|
||||
|
||||
if width - end_x >= 1 {
|
||||
let coeffs_by_2 = coeffs.chunks_exact(2);
|
||||
coeffs = coeffs_by_2.remainder();
|
||||
|
||||
for k in coeffs_by_2 {
|
||||
let coeff0_i64x2 = _mm_set1_epi64x(k[0] as i64);
|
||||
let coeff1_i64x2 = _mm_set1_epi64x(k[1] as i64);
|
||||
let coeff_i64x2 = _mm_set_epi64x(k[1] as i64, k[0] as i64);
|
||||
|
||||
for i in 0..4 {
|
||||
let source = simd_utils::loadu_si128(s_rows[i], x);
|
||||
|
||||
let rg0_i64x2 = _mm_shuffle_epi8(source, rg0_shuffle);
|
||||
rg_sum[i] = _mm_add_epi64(rg_sum[i], _mm_mul_epi32(rg0_i64x2, coeff0_i64x2));
|
||||
|
||||
let rg1_i64x2 = _mm_shuffle_epi8(source, rg1_shuffle);
|
||||
rg_sum[i] = _mm_add_epi64(rg_sum[i], _mm_mul_epi32(rg1_i64x2, coeff1_i64x2));
|
||||
|
||||
let bb_i64x2 = _mm_shuffle_epi8(source, bb_shuffle);
|
||||
bb_sum[i] = _mm_add_epi64(bb_sum[i], _mm_mul_epi32(bb_i64x2, coeff_i64x2));
|
||||
}
|
||||
x += 2;
|
||||
}
|
||||
}
|
||||
|
||||
for &k in coeffs {
|
||||
let coeff_i64x2 = _mm_set1_epi64x(k as i64);
|
||||
|
||||
for i in 0..4 {
|
||||
let &pixel = s_rows[i].get_unchecked(x);
|
||||
let rg_i64x2 = _mm_set_epi64x(pixel.0[1] as i64, pixel.0[0] as i64);
|
||||
rg_sum[i] = _mm_add_epi64(rg_sum[i], _mm_mul_epi32(rg_i64x2, coeff_i64x2));
|
||||
let bb_i64x2 = _mm_set_epi64x(0, pixel.0[2] as i64);
|
||||
bb_sum[i] = _mm_add_epi64(bb_sum[i], _mm_mul_epi32(bb_i64x2, coeff_i64x2));
|
||||
}
|
||||
x += 1;
|
||||
}
|
||||
|
||||
for i in 0..4 {
|
||||
_mm_storeu_si128((&mut rg_buf).as_mut_ptr() as *mut __m128i, rg_sum[i]);
|
||||
_mm_storeu_si128((&mut bb_buf).as_mut_ptr() as *mut __m128i, bb_sum[i]);
|
||||
let dst_pixel = d_rows[i].get_unchecked_mut(dst_x);
|
||||
dst_pixel.0[0] = normalizer_guard.clip(rg_buf[0] + half_error);
|
||||
dst_pixel.0[1] = normalizer_guard.clip(rg_buf[1] + half_error);
|
||||
dst_pixel.0[2] = normalizer_guard.clip(bb_buf[0] + bb_buf[1] + half_error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// For safety, it is necessary to ensure the following conditions:
|
||||
/// - bounds.len() == dst_row.len()
|
||||
/// - coefficients_chunks.len() == dst_row.len()
|
||||
/// - max(chunk.start + chunk.values.len() for chunk in coefficients_chunks) <= src_row.len()
|
||||
/// - precision <= MAX_COEFS_PRECISION
|
||||
#[target_feature(enable = "sse4.1")]
|
||||
unsafe fn horiz_convolution_8u(
|
||||
src_row: &[U16x3],
|
||||
dst_row: &mut [U16x3],
|
||||
coefficients_chunks: &[CoefficientsI32Chunk],
|
||||
normalizer_guard: &optimisations::NormalizerGuard32,
|
||||
) {
|
||||
let precision = normalizer_guard.precision();
|
||||
let rg_initial = _mm_set1_epi64x(1 << (precision - 1));
|
||||
let bb_initial = _mm_set1_epi64x(1 << (precision - 2));
|
||||
|
||||
/*
|
||||
|R G B | |R G B | |R G |
|
||||
|0001 0203 0405| |0607 0809 1011| |1213 1415|
|
||||
|
||||
Shuffle to extract RG components of first pixel as i64:
|
||||
-1, -1, -1, -1, -1, -1, 3, 2, -1, -1, -1, -1, -1, -1, 1, 0
|
||||
|
||||
Shuffle to extract RG components of second pixel as i64:
|
||||
-1, -1, -1, -1, -1, -1, 9, 8, -1, -1, -1, -1, -1, -1, 7, 6
|
||||
|
||||
Shuffle to extract B components of two pixels as i64:
|
||||
-1, -1, -1, -1, -1, -1, 11, 10, -1, -1, -1, -1, -1, -1, 5, 4
|
||||
|
||||
*/
|
||||
|
||||
let rg0_shuffle = _mm_set_epi8(-1, -1, -1, -1, -1, -1, 3, 2, -1, -1, -1, -1, -1, -1, 1, 0);
|
||||
let rg1_shuffle = _mm_set_epi8(-1, -1, -1, -1, -1, -1, 9, 8, -1, -1, -1, -1, -1, -1, 7, 6);
|
||||
let bb_shuffle = _mm_set_epi8(-1, -1, -1, -1, -1, -1, 11, 10, -1, -1, -1, -1, -1, -1, 5, 4);
|
||||
let mut rg_buf = [0i64; 2];
|
||||
let mut bb_buf = [0i64; 2];
|
||||
|
||||
let width = src_row.len();
|
||||
|
||||
for (dst_x, &coeffs_chunk) in coefficients_chunks.iter().enumerate() {
|
||||
let mut x: usize = coeffs_chunk.start as usize;
|
||||
|
||||
let mut rg_sum = rg_initial;
|
||||
let mut bb_sum = bb_initial;
|
||||
|
||||
let mut coeffs = coeffs_chunk.values;
|
||||
let end_x = x + coeffs.len();
|
||||
|
||||
if width - end_x >= 1 {
|
||||
let coeffs_by_2 = coeffs.chunks_exact(2);
|
||||
coeffs = coeffs_by_2.remainder();
|
||||
|
||||
for k in coeffs_by_2 {
|
||||
let coeff0_i64x2 = _mm_set1_epi64x(k[0] as i64);
|
||||
let coeff1_i64x2 = _mm_set1_epi64x(k[1] as i64);
|
||||
let coeff_i64x2 = _mm_set_epi64x(k[1] as i64, k[0] as i64);
|
||||
|
||||
let source = simd_utils::loadu_si128(src_row, x);
|
||||
|
||||
let rg0_i64x2 = _mm_shuffle_epi8(source, rg0_shuffle);
|
||||
rg_sum = _mm_add_epi64(rg_sum, _mm_mul_epi32(rg0_i64x2, coeff0_i64x2));
|
||||
|
||||
let rg1_i64x2 = _mm_shuffle_epi8(source, rg1_shuffle);
|
||||
rg_sum = _mm_add_epi64(rg_sum, _mm_mul_epi32(rg1_i64x2, coeff1_i64x2));
|
||||
|
||||
let bb_i64x2 = _mm_shuffle_epi8(source, bb_shuffle);
|
||||
bb_sum = _mm_add_epi64(bb_sum, _mm_mul_epi32(bb_i64x2, coeff_i64x2));
|
||||
x += 2;
|
||||
}
|
||||
}
|
||||
|
||||
for &k in coeffs {
|
||||
let coeff_i64x2 = _mm_set1_epi64x(k as i64);
|
||||
|
||||
let &pixel = src_row.get_unchecked(x);
|
||||
let rg_i64x2 = _mm_set_epi64x(pixel.0[1] as i64, pixel.0[0] as i64);
|
||||
rg_sum = _mm_add_epi64(rg_sum, _mm_mul_epi32(rg_i64x2, coeff_i64x2));
|
||||
let bb_i64x2 = _mm_set_epi64x(0, pixel.0[2] as i64);
|
||||
bb_sum = _mm_add_epi64(bb_sum, _mm_mul_epi32(bb_i64x2, coeff_i64x2));
|
||||
|
||||
x += 1;
|
||||
}
|
||||
|
||||
_mm_storeu_si128((&mut rg_buf).as_mut_ptr() as *mut __m128i, rg_sum);
|
||||
_mm_storeu_si128((&mut bb_buf).as_mut_ptr() as *mut __m128i, bb_sum);
|
||||
let dst_pixel = dst_row.get_unchecked_mut(dst_x);
|
||||
dst_pixel.0[0] = normalizer_guard.clip(rg_buf[0]);
|
||||
dst_pixel.0[1] = normalizer_guard.clip(rg_buf[1]);
|
||||
dst_pixel.0[2] = normalizer_guard.clip(bb_buf[0] + bb_buf[1]);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
use std::arch::x86_64::*;
|
||||
|
||||
use crate::convolution::optimisations::{CoefficientsI16Chunk, NormalizerGuard16};
|
||||
use crate::convolution::{optimisations, Coefficients};
|
||||
use crate::image_view::{FourRows, FourRowsMut, TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::U8;
|
||||
@@ -42,26 +41,6 @@ pub(crate) fn horiz_convolution(
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn vert_convolution(
|
||||
src_image: TypedImageView<U8>,
|
||||
mut dst_image: TypedImageViewMut<U8>,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
let (values, window_size, bounds_per_pixel) =
|
||||
(coeffs.values, coeffs.window_size, coeffs.bounds);
|
||||
|
||||
let normalizer_guard = optimisations::NormalizerGuard16::new(values);
|
||||
let coefficients_chunks = normalizer_guard.normalized_chunks(window_size, &bounds_per_pixel);
|
||||
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
for (dst_row, coeffs_chunk) in dst_rows.zip(coefficients_chunks) {
|
||||
unsafe {
|
||||
vert_convolution_8u(&src_image, dst_row, coeffs_chunk, &normalizer_guard);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// For safety, it is necessary to ensure the following conditions:
|
||||
/// - length of all rows in src_rows must be equal
|
||||
/// - length of all rows in dst_rows must be equal
|
||||
@@ -73,8 +52,8 @@ pub(crate) fn vert_convolution(
|
||||
unsafe fn horiz_convolution_8u4x(
|
||||
src_rows: FourRows<U8>,
|
||||
dst_rows: FourRowsMut<U8>,
|
||||
coefficients_chunks: &[CoefficientsI16Chunk],
|
||||
normalizer_guard: &NormalizerGuard16,
|
||||
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
|
||||
normalizer_guard: &optimisations::NormalizerGuard16,
|
||||
) {
|
||||
let s_rows = [src_rows.0, src_rows.1, src_rows.2, src_rows.3];
|
||||
let d_rows = [dst_rows.0, dst_rows.1, dst_rows.2, dst_rows.3];
|
||||
@@ -144,8 +123,8 @@ unsafe fn horiz_convolution_8u4x(
|
||||
unsafe fn horiz_convolution_8u(
|
||||
src_row: &[U8],
|
||||
dst_row: &mut [U8],
|
||||
coefficients_chunks: &[CoefficientsI16Chunk],
|
||||
normalizer_guard: &NormalizerGuard16,
|
||||
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
|
||||
normalizer_guard: &optimisations::NormalizerGuard16,
|
||||
) {
|
||||
let zero = _mm_setzero_si128();
|
||||
// 8 components will be added, use only 1/8 of the error
|
||||
@@ -195,196 +174,6 @@ unsafe fn horiz_convolution_8u(
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[target_feature(enable = "avx2")]
|
||||
unsafe fn vert_convolution_8u(
|
||||
src_img: &TypedImageView<U8>,
|
||||
dst_row: &mut [U8],
|
||||
coeffs_chunk: CoefficientsI16Chunk,
|
||||
normalizer_guard: &NormalizerGuard16,
|
||||
) {
|
||||
let src_width = src_img.width().get() as usize;
|
||||
let y_start = coeffs_chunk.start;
|
||||
let coeffs = coeffs_chunk.values;
|
||||
let max_y = y_start + coeffs.len() as u32;
|
||||
let precision = normalizer_guard.precision();
|
||||
|
||||
let initial = _mm_set1_epi32(1 << (precision - 1));
|
||||
let initial_256 = _mm256_set1_epi32(1 << (precision - 1));
|
||||
let zero_128 = _mm_setzero_si128();
|
||||
let zero_256: __m256i = _mm256_setzero_si256();
|
||||
|
||||
let mut x: usize = 0;
|
||||
while x < src_width.saturating_sub(31) {
|
||||
let mut sss0 = initial_256;
|
||||
let mut sss1 = initial_256;
|
||||
let mut sss2 = initial_256;
|
||||
let mut sss3 = initial_256;
|
||||
|
||||
let mut y: u32 = 0;
|
||||
|
||||
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
|
||||
// Load two coefficients at once
|
||||
let two_coeffs = simd_utils::ptr_i16_to_256set1_epi32(coeffs, y as usize);
|
||||
let row1 = simd_utils::loadu_si256(s_row1, x); // top line
|
||||
let row2 = simd_utils::loadu_si256(s_row2, x); // bottom line
|
||||
|
||||
let lo_pixels = _mm256_unpacklo_epi8(row1, row2);
|
||||
let lo_lo = _mm256_unpacklo_epi8(lo_pixels, zero_256);
|
||||
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(lo_lo, two_coeffs));
|
||||
let hi_lo = _mm256_unpackhi_epi8(lo_pixels, zero_256);
|
||||
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(hi_lo, two_coeffs));
|
||||
|
||||
let hi_pixels = _mm256_unpackhi_epi8(row1, row2);
|
||||
let lo_hi = _mm256_unpacklo_epi8(hi_pixels, zero_256);
|
||||
sss2 = _mm256_add_epi32(sss2, _mm256_madd_epi16(lo_hi, two_coeffs));
|
||||
let hi_hi = _mm256_unpackhi_epi8(hi_pixels, zero_256);
|
||||
sss3 = _mm256_add_epi32(sss3, _mm256_madd_epi16(hi_hi, two_coeffs));
|
||||
|
||||
y += 2;
|
||||
}
|
||||
|
||||
if let Some(&k) = coeffs.get(y as usize) {
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let one_coeff = _mm256_set1_epi32(k as i32);
|
||||
|
||||
let row1 = simd_utils::loadu_si256(s_row, x); // top line
|
||||
let row2 = _mm256_setzero_si256(); // bottom line is empty
|
||||
|
||||
let lo_pixels = _mm256_unpacklo_epi8(row1, row2);
|
||||
let lo_lo = _mm256_unpacklo_epi8(lo_pixels, zero_256);
|
||||
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(lo_lo, one_coeff));
|
||||
let hi_lo = _mm256_unpackhi_epi8(lo_pixels, zero_256);
|
||||
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(hi_lo, one_coeff));
|
||||
|
||||
let hi_pixels = _mm256_unpackhi_epi8(row1, zero_256);
|
||||
let lo_hi = _mm256_unpacklo_epi8(hi_pixels, zero_256);
|
||||
sss2 = _mm256_add_epi32(sss2, _mm256_madd_epi16(lo_hi, one_coeff));
|
||||
let hi_hi = _mm256_unpackhi_epi8(hi_pixels, zero_256);
|
||||
sss3 = _mm256_add_epi32(sss3, _mm256_madd_epi16(hi_hi, one_coeff));
|
||||
}
|
||||
|
||||
macro_rules! call {
|
||||
($imm8:expr) => {{
|
||||
sss0 = _mm256_srai_epi32::<$imm8>(sss0);
|
||||
sss1 = _mm256_srai_epi32::<$imm8>(sss1);
|
||||
sss2 = _mm256_srai_epi32::<$imm8>(sss2);
|
||||
sss3 = _mm256_srai_epi32::<$imm8>(sss3);
|
||||
}};
|
||||
}
|
||||
constify_imm8!(precision, call);
|
||||
|
||||
sss0 = _mm256_packs_epi32(sss0, sss1);
|
||||
sss2 = _mm256_packs_epi32(sss2, sss3);
|
||||
sss0 = _mm256_packus_epi16(sss0, sss2);
|
||||
let dst_ptr = dst_row.get_unchecked_mut(x..).as_mut_ptr() as *mut __m256i;
|
||||
_mm256_storeu_si256(dst_ptr, sss0);
|
||||
|
||||
x += 32;
|
||||
}
|
||||
|
||||
while x < src_width.saturating_sub(7) {
|
||||
let mut sss0 = initial; // left row
|
||||
let mut sss1 = initial; // right row
|
||||
let mut y: u32 = 0;
|
||||
|
||||
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
|
||||
// Load two coefficients at once
|
||||
let two_coeffs = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
|
||||
|
||||
let row1 = simd_utils::loadl_epi64(s_row1, x); // top line
|
||||
let row2 = simd_utils::loadl_epi64(s_row2, x); // bottom line
|
||||
|
||||
let pixels = _mm_unpacklo_epi8(row1, row2);
|
||||
let lo_pixels = _mm_unpacklo_epi8(pixels, zero_128);
|
||||
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(lo_pixels, two_coeffs));
|
||||
let hi_pixels = _mm_unpackhi_epi8(pixels, zero_128);
|
||||
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(hi_pixels, two_coeffs));
|
||||
|
||||
y += 2;
|
||||
}
|
||||
|
||||
if let Some(&k) = coeffs.get(y as usize) {
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let one_coeff = _mm_set1_epi32(k as i32);
|
||||
|
||||
let row1 = simd_utils::loadl_epi64(s_row, x); // top line
|
||||
let row2 = _mm_setzero_si128(); // bottom line is empty
|
||||
|
||||
let pixels = _mm_unpacklo_epi8(row1, row2);
|
||||
let lo_pixels = _mm_unpacklo_epi8(pixels, zero_128);
|
||||
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(lo_pixels, one_coeff));
|
||||
let hi_pixels = _mm_unpackhi_epi8(pixels, zero_128);
|
||||
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(hi_pixels, one_coeff));
|
||||
}
|
||||
|
||||
macro_rules! call {
|
||||
($imm8:expr) => {{
|
||||
sss0 = _mm_srai_epi32::<$imm8>(sss0);
|
||||
sss1 = _mm_srai_epi32::<$imm8>(sss1);
|
||||
}};
|
||||
}
|
||||
constify_imm8!(precision, call);
|
||||
|
||||
sss0 = _mm_packs_epi32(sss0, sss1);
|
||||
sss0 = _mm_packus_epi16(sss0, sss0);
|
||||
let dst_ptr = dst_row.get_unchecked_mut(x..).as_mut_ptr() as *mut __m128i;
|
||||
_mm_storel_epi64(dst_ptr, sss0);
|
||||
|
||||
x += 8;
|
||||
}
|
||||
|
||||
while x < src_width.saturating_sub(3) {
|
||||
let mut sss = initial;
|
||||
let mut y: u32 = 0;
|
||||
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
|
||||
// Load two coefficients at once
|
||||
let two_coeffs = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
|
||||
|
||||
let row1 = simd_utils::mm_cvtsi32_si128_from_u8(s_row1, x); // top line
|
||||
let row2 = simd_utils::mm_cvtsi32_si128_from_u8(s_row2, x); // bottom line
|
||||
|
||||
let pixels_u8 = _mm_unpacklo_epi8(row1, row2);
|
||||
let pixels_i16 = _mm_unpacklo_epi8(pixels_u8, _mm_setzero_si128());
|
||||
sss = _mm_add_epi32(sss, _mm_madd_epi16(pixels_i16, two_coeffs));
|
||||
|
||||
y += 2;
|
||||
}
|
||||
|
||||
if let Some(&k) = coeffs.get(y as usize) {
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let pix = simd_utils::mm_cvtepu8_epi32_from_u8(s_row, x);
|
||||
let mmk = _mm_set1_epi32(k as i32);
|
||||
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
|
||||
}
|
||||
|
||||
macro_rules! call {
|
||||
($imm8:expr) => {{
|
||||
sss = _mm_srai_epi32::<$imm8>(sss);
|
||||
}};
|
||||
}
|
||||
constify_imm8!(precision, call);
|
||||
|
||||
sss = _mm_packs_epi32(sss, sss);
|
||||
let u8x4: [u8; 4] = _mm_cvtsi128_si32(_mm_packus_epi16(sss, sss)).to_le_bytes();
|
||||
dst_row.get_unchecked_mut(x).0 = u8x4[0];
|
||||
dst_row.get_unchecked_mut(x + 1).0 = u8x4[1];
|
||||
dst_row.get_unchecked_mut(x + 2).0 = u8x4[2];
|
||||
dst_row.get_unchecked_mut(x + 3).0 = u8x4[3];
|
||||
x += 4;
|
||||
}
|
||||
|
||||
for dst_pixel in dst_row.iter_mut().skip(x) {
|
||||
let mut ss0 = 1 << (precision - 1);
|
||||
for (dy, &k) in coeffs.iter().enumerate() {
|
||||
let src_pixel = src_img.get_pixel(x as u32, y_start + dy as u32);
|
||||
ss0 += src_pixel.0 as i32 * (k as i32);
|
||||
}
|
||||
dst_pixel.0 = normalizer_guard.clip(ss0);
|
||||
x += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// only needs AVX2
|
||||
#[inline]
|
||||
#[target_feature(enable = "avx2")]
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use super::{Coefficients, Convolution};
|
||||
use crate::convolution::vertical_u8::vert_convolution_u8;
|
||||
use crate::image_view::{TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::U8;
|
||||
use crate::CpuExtensions;
|
||||
@@ -28,10 +29,6 @@ impl Convolution for U8 {
|
||||
coeffs: Coefficients,
|
||||
cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
match cpu_extensions {
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Avx2 => avx2::vert_convolution(src_image, dst_image, coeffs),
|
||||
_ => native::vert_convolution(src_image, dst_image, coeffs),
|
||||
}
|
||||
vert_convolution_u8(src_image, dst_image, coeffs, cpu_extensions);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,33 +32,3 @@ pub(crate) fn horiz_convolution(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub(crate) fn vert_convolution(
|
||||
src_image: TypedImageView<U8>,
|
||||
mut dst_image: TypedImageViewMut<U8>,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
let (values, window_size, bounds) = (coeffs.values, coeffs.window_size, coeffs.bounds);
|
||||
|
||||
let normalizer_guard = optimisations::NormalizerGuard16::new(values);
|
||||
let precision = normalizer_guard.precision();
|
||||
let coefficients_chunks = normalizer_guard.normalized_chunks(window_size, &bounds);
|
||||
let initial = 1 << (precision - 1);
|
||||
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
for (&coeffs_chunk, dst_row) in coefficients_chunks.iter().zip(dst_rows) {
|
||||
let first_y_src = coeffs_chunk.start;
|
||||
let ks = coeffs_chunk.values;
|
||||
|
||||
for (x_src, dst_pixel) in dst_row.iter_mut().enumerate() {
|
||||
let mut ss = initial;
|
||||
let src_rows = src_image.iter_rows(first_y_src);
|
||||
for (&k, src_row) in ks.iter().zip(src_rows) {
|
||||
let src_pixel = unsafe { src_row.get_unchecked(x_src as usize) };
|
||||
ss += src_pixel.0 as i32 * (k as i32);
|
||||
}
|
||||
dst_pixel.0 = unsafe { normalizer_guard.clip(ss) };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
use std::arch::x86_64::*;
|
||||
use std::intrinsics::transmute;
|
||||
|
||||
use crate::convolution::optimisations::{CoefficientsI16Chunk, NormalizerGuard16};
|
||||
use crate::convolution::{optimisations, Coefficients};
|
||||
use crate::image_view::{FourRows, FourRowsMut, TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::{Pixel, U8x3};
|
||||
use crate::pixels::U8x3;
|
||||
use crate::simd_utils;
|
||||
|
||||
#[inline]
|
||||
@@ -44,26 +43,6 @@ pub(crate) fn horiz_convolution(
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn vert_convolution(
|
||||
src_image: TypedImageView<U8x3>,
|
||||
mut dst_image: TypedImageViewMut<U8x3>,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
let (values, window_size, bounds_per_pixel) =
|
||||
(coeffs.values, coeffs.window_size, coeffs.bounds);
|
||||
|
||||
let normalizer_guard = optimisations::NormalizerGuard16::new(values);
|
||||
let coefficients_chunks = normalizer_guard.normalized_chunks(window_size, &bounds_per_pixel);
|
||||
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
for (dst_row, coeffs_chunk) in dst_rows.zip(coefficients_chunks) {
|
||||
unsafe {
|
||||
vert_convolution_8u(&src_image, dst_row, coeffs_chunk, &normalizer_guard);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// For safety, it is necessary to ensure the following conditions:
|
||||
/// - length of all rows in src_rows must be equal
|
||||
/// - length of all rows in dst_rows must be equal
|
||||
@@ -75,7 +54,7 @@ pub(crate) fn vert_convolution(
|
||||
unsafe fn horiz_convolution_8u4x(
|
||||
src_rows: FourRows<U8x3>,
|
||||
dst_rows: FourRowsMut<U8x3>,
|
||||
coefficients_chunks: &[CoefficientsI16Chunk],
|
||||
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
|
||||
precision: u8,
|
||||
) {
|
||||
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
|
||||
@@ -246,7 +225,7 @@ unsafe fn horiz_convolution_8u4x(
|
||||
unsafe fn horiz_convolution_8u(
|
||||
src_row: &[U8x3],
|
||||
dst_row: &mut [U8x3],
|
||||
coefficients_chunks: &[CoefficientsI16Chunk],
|
||||
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
|
||||
precision: u8,
|
||||
) {
|
||||
#[rustfmt::skip]
|
||||
@@ -402,203 +381,3 @@ unsafe fn horiz_convolution_8u(
|
||||
dst_row.get_unchecked_mut(dst_x).0 = [bytes[0], bytes[1], bytes[2]];
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[target_feature(enable = "avx2")]
|
||||
unsafe fn vert_convolution_8u(
|
||||
src_img: &TypedImageView<U8x3>,
|
||||
dst_row: &mut [U8x3],
|
||||
coeffs_chunk: CoefficientsI16Chunk,
|
||||
normalizer_guard: &NormalizerGuard16,
|
||||
) {
|
||||
let src_width = src_img.width().get() as usize;
|
||||
let y_start = coeffs_chunk.start;
|
||||
let coeffs = coeffs_chunk.values;
|
||||
let max_y = y_start + coeffs.len() as u32;
|
||||
let precision = normalizer_guard.precision();
|
||||
|
||||
let initial = _mm_set1_epi32(1 << (precision - 1));
|
||||
let initial_256 = _mm256_set1_epi32(1 << (precision - 1));
|
||||
|
||||
let mut x_in_bytes: usize = 0;
|
||||
let width_in_bytes = src_width * U8x3::size();
|
||||
let dst_ptr_u8 = dst_row.as_mut_ptr() as *mut u8;
|
||||
|
||||
while x_in_bytes < width_in_bytes.saturating_sub(31) {
|
||||
let mut sss0 = initial_256;
|
||||
let mut sss1 = initial_256;
|
||||
let mut sss2 = initial_256;
|
||||
let mut sss3 = initial_256;
|
||||
|
||||
let mut y: u32 = 0;
|
||||
|
||||
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
|
||||
// Load two coefficients at once
|
||||
let mmk = simd_utils::ptr_i16_to_256set1_epi32(coeffs, y as usize);
|
||||
|
||||
let source1 = simd_utils::loadu_si256_raw(s_row1, x_in_bytes); // top line
|
||||
let source2 = simd_utils::loadu_si256_raw(s_row2, x_in_bytes); // bottom line
|
||||
|
||||
let source = _mm256_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
|
||||
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
|
||||
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
|
||||
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
|
||||
|
||||
let source = _mm256_unpackhi_epi8(source1, source2);
|
||||
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
|
||||
sss2 = _mm256_add_epi32(sss2, _mm256_madd_epi16(pix, mmk));
|
||||
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
|
||||
sss3 = _mm256_add_epi32(sss3, _mm256_madd_epi16(pix, mmk));
|
||||
|
||||
y += 2;
|
||||
}
|
||||
|
||||
if let Some(&k) = coeffs.get(y as usize) {
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let mmk = _mm256_set1_epi32(k as i32);
|
||||
|
||||
let source1 = simd_utils::loadu_si256_raw(s_row, x_in_bytes); // top line
|
||||
let source2 = _mm256_setzero_si256(); // bottom line is empty
|
||||
|
||||
let source = _mm256_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
|
||||
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
|
||||
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
|
||||
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
|
||||
|
||||
let source = _mm256_unpackhi_epi8(source1, _mm256_setzero_si256());
|
||||
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
|
||||
sss2 = _mm256_add_epi32(sss2, _mm256_madd_epi16(pix, mmk));
|
||||
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
|
||||
sss3 = _mm256_add_epi32(sss3, _mm256_madd_epi16(pix, mmk));
|
||||
}
|
||||
|
||||
macro_rules! call {
|
||||
($imm8:expr) => {{
|
||||
sss0 = _mm256_srai_epi32::<$imm8>(sss0);
|
||||
sss1 = _mm256_srai_epi32::<$imm8>(sss1);
|
||||
sss2 = _mm256_srai_epi32::<$imm8>(sss2);
|
||||
sss3 = _mm256_srai_epi32::<$imm8>(sss3);
|
||||
}};
|
||||
}
|
||||
constify_imm8!(precision, call);
|
||||
|
||||
sss0 = _mm256_packs_epi32(sss0, sss1);
|
||||
sss2 = _mm256_packs_epi32(sss2, sss3);
|
||||
sss0 = _mm256_packus_epi16(sss0, sss2);
|
||||
|
||||
let dst_ptr = dst_ptr_u8.add(x_in_bytes) as *mut __m256i;
|
||||
_mm256_storeu_si256(dst_ptr, sss0);
|
||||
|
||||
x_in_bytes += 32;
|
||||
}
|
||||
|
||||
while x_in_bytes < width_in_bytes.saturating_sub(7) {
|
||||
let mut sss0 = initial; // left row
|
||||
let mut sss1 = initial; // right row
|
||||
let mut y: u32 = 0;
|
||||
|
||||
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
|
||||
// Load two coefficients at once
|
||||
let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
|
||||
|
||||
let source1 = simd_utils::loadl_epi64_raw(s_row1, x_in_bytes); // top line
|
||||
let source2 = simd_utils::loadl_epi64_raw(s_row2, x_in_bytes); // bottom line
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
|
||||
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
y += 2;
|
||||
}
|
||||
|
||||
if let Some(&k) = coeffs.get(y as usize) {
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let mmk = _mm_set1_epi32(k as i32);
|
||||
|
||||
let source1 = simd_utils::loadl_epi64_raw(s_row, x_in_bytes); // top line
|
||||
let source2 = _mm_setzero_si128(); // bottom line is empty
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
|
||||
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
|
||||
}
|
||||
|
||||
macro_rules! call {
|
||||
($imm8:expr) => {{
|
||||
sss0 = _mm_srai_epi32::<$imm8>(sss0);
|
||||
sss1 = _mm_srai_epi32::<$imm8>(sss1);
|
||||
}};
|
||||
}
|
||||
constify_imm8!(precision, call);
|
||||
|
||||
sss0 = _mm_packs_epi32(sss0, sss1);
|
||||
sss0 = _mm_packus_epi16(sss0, sss0);
|
||||
|
||||
let dst_ptr = dst_ptr_u8.add(x_in_bytes) as *mut __m128i;
|
||||
_mm_storel_epi64(dst_ptr, sss0);
|
||||
|
||||
x_in_bytes += 8;
|
||||
}
|
||||
|
||||
while x_in_bytes < width_in_bytes.saturating_sub(3) {
|
||||
let mut sss = initial;
|
||||
let mut y: u32 = 0;
|
||||
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
|
||||
// Load two coefficients at once
|
||||
let two_coeffs = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
|
||||
|
||||
let row1 = simd_utils::mm_cvtsi32_si128_from_raw(s_row1, x_in_bytes); // top line
|
||||
let row2 = simd_utils::mm_cvtsi32_si128_from_raw(s_row2, x_in_bytes); // bottom line
|
||||
|
||||
let pixels_u8 = _mm_unpacklo_epi8(row1, row2);
|
||||
let pixels_i16 = _mm_unpacklo_epi8(pixels_u8, _mm_setzero_si128());
|
||||
sss = _mm_add_epi32(sss, _mm_madd_epi16(pixels_i16, two_coeffs));
|
||||
|
||||
y += 2;
|
||||
}
|
||||
|
||||
if let Some(&k) = coeffs.get(y as usize) {
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let pix = simd_utils::mm_cvtepu8_epi32_from_raw(s_row, x_in_bytes);
|
||||
let mmk = _mm_set1_epi32(k as i32);
|
||||
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
|
||||
}
|
||||
|
||||
macro_rules! call {
|
||||
($imm8:expr) => {{
|
||||
sss = _mm_srai_epi32::<$imm8>(sss);
|
||||
}};
|
||||
}
|
||||
constify_imm8!(precision, call);
|
||||
|
||||
sss = _mm_packs_epi32(sss, sss);
|
||||
let dst_ptr_i32 = dst_ptr_u8.add(x_in_bytes) as *mut i32;
|
||||
*dst_ptr_i32 = _mm_cvtsi128_si32(_mm_packus_epi16(sss, sss));
|
||||
|
||||
x_in_bytes += 4;
|
||||
}
|
||||
|
||||
if x_in_bytes < width_in_bytes {
|
||||
let dst_u8 =
|
||||
std::slice::from_raw_parts_mut(dst_ptr_u8.add(x_in_bytes), width_in_bytes - x_in_bytes);
|
||||
|
||||
for dst_pixel in dst_u8 {
|
||||
let mut ss0 = 1 << (precision - 1);
|
||||
for (dy, &k) in coeffs.iter().enumerate() {
|
||||
if let Some(src_row) = src_img.get_row(y_start + dy as u32) {
|
||||
let src_ptr = src_row.as_ptr() as *const u8;
|
||||
let src_component = *src_ptr.add(x_in_bytes);
|
||||
ss0 += src_component as i32 * (k as i32);
|
||||
}
|
||||
}
|
||||
*dst_pixel = normalizer_guard.clip(ss0);
|
||||
x_in_bytes += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use super::{Coefficients, Convolution};
|
||||
use crate::convolution::vertical_u8::vert_convolution_u8;
|
||||
use crate::image_view::{TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::U8x3;
|
||||
use crate::CpuExtensions;
|
||||
@@ -34,14 +35,6 @@ impl Convolution for U8x3 {
|
||||
coeffs: Coefficients,
|
||||
cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
match cpu_extensions {
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Avx2 => avx2::vert_convolution(src_image, dst_image, coeffs),
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Sse4_1 => unsafe {
|
||||
sse4::vert_convolution(src_image, dst_image, coeffs)
|
||||
},
|
||||
_ => native::vert_convolution(src_image, dst_image, coeffs),
|
||||
}
|
||||
vert_convolution_u8(src_image, dst_image, coeffs, cpu_extensions);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,37 +34,3 @@ pub(crate) fn horiz_convolution(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub(crate) fn vert_convolution(
|
||||
src_image: TypedImageView<U8x3>,
|
||||
mut dst_image: TypedImageViewMut<U8x3>,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
let (values, window_size, bounds) = (coeffs.values, coeffs.window_size, coeffs.bounds);
|
||||
|
||||
let normalizer_guard = optimisations::NormalizerGuard16::new(values);
|
||||
let precision = normalizer_guard.precision();
|
||||
let coefficients_chunks = normalizer_guard.normalized_chunks(window_size, &bounds);
|
||||
let initial = 1 << (precision - 1);
|
||||
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
for (&coeffs_chunk, dst_row) in coefficients_chunks.iter().zip(dst_rows) {
|
||||
let first_y_src = coeffs_chunk.start;
|
||||
let ks = coeffs_chunk.values;
|
||||
|
||||
for (x_src, dst_pixel) in dst_row.iter_mut().enumerate() {
|
||||
let mut ss = [initial; 3];
|
||||
let src_rows = src_image.iter_rows(first_y_src);
|
||||
for (&k, src_row) in ks.iter().zip(src_rows) {
|
||||
let src_pixel = unsafe { src_row.get_unchecked(x_src as usize) };
|
||||
for (i, s) in ss.iter_mut().enumerate() {
|
||||
*s += src_pixel.0[i] as i32 * (k as i32);
|
||||
}
|
||||
}
|
||||
for (i, s) in ss.iter().copied().enumerate() {
|
||||
dst_pixel.0[i] = unsafe { normalizer_guard.clip(s) };
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,12 +13,3 @@ pub(crate) unsafe fn horiz_convolution(
|
||||
) {
|
||||
native::horiz_convolution(src_image, dst_image, offset, coeffs);
|
||||
}
|
||||
|
||||
#[target_feature(enable = "sse4.1")]
|
||||
pub(crate) unsafe fn vert_convolution(
|
||||
src_image: TypedImageView<U8x3>,
|
||||
dst_image: TypedImageViewMut<U8x3>,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
native::vert_convolution(src_image, dst_image, coeffs);
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
use std::arch::x86_64::*;
|
||||
use std::intrinsics::transmute;
|
||||
|
||||
use crate::convolution::optimisations::{CoefficientsI16Chunk, NormalizerGuard16};
|
||||
use crate::convolution::{optimisations, Coefficients};
|
||||
use crate::image_view::{FourRows, FourRowsMut, TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::U8x4;
|
||||
@@ -47,26 +46,6 @@ pub(crate) fn horiz_convolution(
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn vert_convolution(
|
||||
src_image: TypedImageView<U8x4>,
|
||||
mut dst_image: TypedImageViewMut<U8x4>,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
let (values, window_size, bounds_per_pixel) =
|
||||
(coeffs.values, coeffs.window_size, coeffs.bounds);
|
||||
|
||||
let normalizer_guard = optimisations::NormalizerGuard16::new(values);
|
||||
let coefficients_chunks = normalizer_guard.normalized_chunks(window_size, &bounds_per_pixel);
|
||||
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
for (dst_row, coeffs_chunk) in dst_rows.zip(coefficients_chunks) {
|
||||
unsafe {
|
||||
vert_convolution_8u(&src_image, dst_row, coeffs_chunk, &normalizer_guard);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// For safety, it is necessary to ensure the following conditions:
|
||||
/// - length of all rows in src_rows must be equal
|
||||
/// - length of all rows in dst_rows must be equal
|
||||
@@ -78,7 +57,7 @@ pub(crate) fn vert_convolution(
|
||||
unsafe fn horiz_convolution_8u4x(
|
||||
src_rows: FourRows<U8x4>,
|
||||
dst_rows: FourRowsMut<U8x4>,
|
||||
coefficients_chunks: &[CoefficientsI16Chunk],
|
||||
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
|
||||
precision: u8,
|
||||
) {
|
||||
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
|
||||
@@ -206,7 +185,7 @@ unsafe fn horiz_convolution_8u4x(
|
||||
unsafe fn horiz_convolution_8u(
|
||||
src_row: &[U8x4],
|
||||
dst_row: &mut [U8x4],
|
||||
coefficients_chunks: &[CoefficientsI16Chunk],
|
||||
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
|
||||
precision: u8,
|
||||
) {
|
||||
#[rustfmt::skip]
|
||||
@@ -324,177 +303,3 @@ unsafe fn horiz_convolution_8u(
|
||||
transmute(_mm_cvtsi128_si32(_mm_packus_epi16(sss, sss)));
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[target_feature(enable = "avx2")]
|
||||
unsafe fn vert_convolution_8u(
|
||||
src_img: &TypedImageView<U8x4>,
|
||||
dst_row: &mut [U8x4],
|
||||
coeffs_chunk: CoefficientsI16Chunk,
|
||||
normalizer_guard: &NormalizerGuard16,
|
||||
) {
|
||||
let src_width = src_img.width().get() as usize;
|
||||
let y_start = coeffs_chunk.start;
|
||||
let coeffs = coeffs_chunk.values;
|
||||
let max_y = y_start + coeffs.len() as u32;
|
||||
let precision = normalizer_guard.precision();
|
||||
|
||||
let initial = _mm_set1_epi32(1 << (precision - 1));
|
||||
let initial_256 = _mm256_set1_epi32(1 << (precision - 1));
|
||||
|
||||
let mut x: usize = 0;
|
||||
while x < src_width.saturating_sub(7) {
|
||||
let mut sss0 = initial_256;
|
||||
let mut sss1 = initial_256;
|
||||
let mut sss2 = initial_256;
|
||||
let mut sss3 = initial_256;
|
||||
|
||||
let mut y: u32 = 0;
|
||||
|
||||
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
|
||||
// Load two coefficients at once
|
||||
let mmk = simd_utils::ptr_i16_to_256set1_epi32(coeffs, y as usize);
|
||||
|
||||
let source1 = simd_utils::loadu_si256(s_row1, x); // top line
|
||||
let source2 = simd_utils::loadu_si256(s_row2, x); // bottom line
|
||||
|
||||
let source = _mm256_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
|
||||
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
|
||||
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
|
||||
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
|
||||
|
||||
let source = _mm256_unpackhi_epi8(source1, source2);
|
||||
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
|
||||
sss2 = _mm256_add_epi32(sss2, _mm256_madd_epi16(pix, mmk));
|
||||
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
|
||||
sss3 = _mm256_add_epi32(sss3, _mm256_madd_epi16(pix, mmk));
|
||||
|
||||
y += 2;
|
||||
}
|
||||
|
||||
if let Some(&k) = coeffs.get(y as usize) {
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let mmk = _mm256_set1_epi32(k as i32);
|
||||
|
||||
let source1 = simd_utils::loadu_si256(s_row, x); // top line
|
||||
let source2 = _mm256_setzero_si256(); // bottom line is empty
|
||||
|
||||
let source = _mm256_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
|
||||
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
|
||||
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
|
||||
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
|
||||
|
||||
let source = _mm256_unpackhi_epi8(source1, _mm256_setzero_si256());
|
||||
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
|
||||
sss2 = _mm256_add_epi32(sss2, _mm256_madd_epi16(pix, mmk));
|
||||
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
|
||||
sss3 = _mm256_add_epi32(sss3, _mm256_madd_epi16(pix, mmk));
|
||||
}
|
||||
|
||||
macro_rules! call {
|
||||
($imm8:expr) => {{
|
||||
sss0 = _mm256_srai_epi32::<$imm8>(sss0);
|
||||
sss1 = _mm256_srai_epi32::<$imm8>(sss1);
|
||||
sss2 = _mm256_srai_epi32::<$imm8>(sss2);
|
||||
sss3 = _mm256_srai_epi32::<$imm8>(sss3);
|
||||
}};
|
||||
}
|
||||
constify_imm8!(precision, call);
|
||||
|
||||
sss0 = _mm256_packs_epi32(sss0, sss1);
|
||||
sss2 = _mm256_packs_epi32(sss2, sss3);
|
||||
sss0 = _mm256_packus_epi16(sss0, sss2);
|
||||
let dst_ptr = dst_row.get_unchecked_mut(x..).as_mut_ptr() as *mut __m256i;
|
||||
_mm256_storeu_si256(dst_ptr, sss0);
|
||||
|
||||
x += 8;
|
||||
}
|
||||
|
||||
while x < src_width.saturating_sub(1) {
|
||||
let mut sss0 = initial; // left row
|
||||
let mut sss1 = initial; // right row
|
||||
let mut y: u32 = 0;
|
||||
|
||||
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
|
||||
// Load two coefficients at once
|
||||
let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
|
||||
|
||||
let source1 = simd_utils::loadl_epi64(s_row1, x); // top line
|
||||
let source2 = simd_utils::loadl_epi64(s_row2, x); // bottom line
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
|
||||
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
y += 2;
|
||||
}
|
||||
|
||||
if let Some(&k) = coeffs.get(y as usize) {
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let mmk = _mm_set1_epi32(k as i32);
|
||||
|
||||
let source1 = simd_utils::loadl_epi64(s_row, x); // top line
|
||||
let source2 = _mm_setzero_si128(); // bottom line is empty
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
|
||||
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
|
||||
}
|
||||
|
||||
macro_rules! call {
|
||||
($imm8:expr) => {{
|
||||
sss0 = _mm_srai_epi32::<$imm8>(sss0);
|
||||
sss1 = _mm_srai_epi32::<$imm8>(sss1);
|
||||
}};
|
||||
}
|
||||
constify_imm8!(precision, call);
|
||||
|
||||
sss0 = _mm_packs_epi32(sss0, sss1);
|
||||
sss0 = _mm_packus_epi16(sss0, sss0);
|
||||
let dst_ptr = dst_row.get_unchecked_mut(x..).as_mut_ptr() as *mut __m128i;
|
||||
_mm_storel_epi64(dst_ptr, sss0);
|
||||
|
||||
x += 2;
|
||||
}
|
||||
|
||||
if x < src_width {
|
||||
let mut sss = initial;
|
||||
let mut y: u32 = 0;
|
||||
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
|
||||
// Load two coefficients at once
|
||||
let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
|
||||
|
||||
let source1 = simd_utils::mm_cvtsi32_si128_from_u32(s_row1, x); // top line
|
||||
let source2 = simd_utils::mm_cvtsi32_si128_from_u32(s_row2, x); // bottom line
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
y += 2;
|
||||
}
|
||||
|
||||
if let Some(&k) = coeffs.get(y as usize) {
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let pix = simd_utils::mm_cvtepu8_epi32(s_row, x);
|
||||
let mmk = _mm_set1_epi32(k as i32);
|
||||
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
|
||||
}
|
||||
|
||||
macro_rules! call {
|
||||
($imm8:expr) => {{
|
||||
sss = _mm_srai_epi32::<$imm8>(sss);
|
||||
}};
|
||||
}
|
||||
constify_imm8!(precision, call);
|
||||
|
||||
sss = _mm_packs_epi32(sss, sss);
|
||||
*dst_row.get_unchecked_mut(x) = transmute(_mm_cvtsi128_si32(_mm_packus_epi16(sss, sss)));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use super::{Coefficients, Convolution};
|
||||
use crate::convolution::vertical_u8::vert_convolution_u8;
|
||||
use crate::image_view::{TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::U8x4;
|
||||
use crate::CpuExtensions;
|
||||
@@ -32,12 +33,6 @@ impl Convolution for U8x4 {
|
||||
coeffs: Coefficients,
|
||||
cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
match cpu_extensions {
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Avx2 => avx2::vert_convolution(src_image, dst_image, coeffs),
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Sse4_1 => sse4::vert_convolution(src_image, dst_image, coeffs),
|
||||
_ => native::vert_convolution(src_image, dst_image, coeffs),
|
||||
}
|
||||
vert_convolution_u8(src_image, dst_image, coeffs, cpu_extensions);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,35 +33,3 @@ pub(crate) fn horiz_convolution(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn vert_convolution(
|
||||
src_image: TypedImageView<U8x4>,
|
||||
mut dst_image: TypedImageViewMut<U8x4>,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
let (values, window_size, bounds) = (coeffs.values, coeffs.window_size, coeffs.bounds);
|
||||
|
||||
let normalizer_guard = optimisations::NormalizerGuard16::new(values);
|
||||
let precision = normalizer_guard.precision();
|
||||
let coefficients_chunks = normalizer_guard.normalized_chunks(window_size, &bounds);
|
||||
let initial = 1 << (precision - 1);
|
||||
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
for (&coeffs_chunk, dst_row) in coefficients_chunks.iter().zip(dst_rows) {
|
||||
let first_y_src = coeffs_chunk.start;
|
||||
let ks = coeffs_chunk.values;
|
||||
|
||||
for (x_src, dst_pixel) in dst_row.iter_mut().enumerate() {
|
||||
let mut ss = [initial; 4];
|
||||
let src_rows = src_image.iter_rows(first_y_src);
|
||||
for (&k, src_row) in ks.iter().zip(src_rows) {
|
||||
let src_pixel = unsafe { src_row.get_unchecked(x_src as usize) };
|
||||
let components: [u8; 4] = src_pixel.0.to_le_bytes();
|
||||
for (i, s) in ss.iter_mut().enumerate() {
|
||||
*s += components[i] as i32 * (k as i32);
|
||||
}
|
||||
}
|
||||
dst_pixel.0 = u32::from_le_bytes(ss.map(|v| unsafe { normalizer_guard.clip(v) }));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
use std::arch::x86_64::*;
|
||||
use std::intrinsics::transmute;
|
||||
|
||||
use crate::convolution::optimisations::{CoefficientsI16Chunk, NormalizerGuard16};
|
||||
use crate::convolution::{optimisations, Coefficients};
|
||||
use crate::image_view::{FourRows, FourRowsMut, TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::U8x4;
|
||||
@@ -47,26 +46,6 @@ pub(crate) fn horiz_convolution(
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn vert_convolution(
|
||||
src_image: TypedImageView<U8x4>,
|
||||
mut dst_image: TypedImageViewMut<U8x4>,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
let (values, window_size, bounds_per_pixel) =
|
||||
(coeffs.values, coeffs.window_size, coeffs.bounds);
|
||||
|
||||
let normalizer_guard = optimisations::NormalizerGuard16::new(values);
|
||||
let coefficients_chunks = normalizer_guard.normalized_chunks(window_size, &bounds_per_pixel);
|
||||
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
for (dst_row, coeffs_chunk) in dst_rows.zip(coefficients_chunks) {
|
||||
unsafe {
|
||||
vert_convolution_8u(&src_image, dst_row, coeffs_chunk, &normalizer_guard);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// For safety, it is necessary to ensure the following conditions:
|
||||
/// - length of all rows in src_rows must be equal
|
||||
/// - length of all rows in dst_rows must be equal
|
||||
@@ -77,7 +56,7 @@ pub(crate) fn vert_convolution(
|
||||
unsafe fn horiz_convolution_8u4x(
|
||||
src_rows: FourRows<U8x4>,
|
||||
dst_rows: FourRowsMut<U8x4>,
|
||||
coefficients_chunks: &[CoefficientsI16Chunk],
|
||||
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
|
||||
precision: u8,
|
||||
) {
|
||||
let (s_row0, s_row1, s_row2, s_row3) = src_rows;
|
||||
@@ -211,7 +190,7 @@ unsafe fn horiz_convolution_8u4x(
|
||||
unsafe fn horiz_convolution_8u(
|
||||
src_row: &[U8x4],
|
||||
dst_row: &mut [U8x4],
|
||||
coefficients_chunks: &[CoefficientsI16Chunk],
|
||||
coefficients_chunks: &[optimisations::CoefficientsI16Chunk],
|
||||
precision: u8,
|
||||
) {
|
||||
let initial = _mm_set1_epi32(1 << (precision - 1));
|
||||
@@ -306,216 +285,3 @@ unsafe fn horiz_convolution_8u(
|
||||
transmute(_mm_cvtsi128_si32(_mm_packus_epi16(sss, sss)));
|
||||
}
|
||||
}
|
||||
|
||||
#[target_feature(enable = "sse4.1")]
|
||||
pub(crate) unsafe fn vert_convolution_8u(
|
||||
src_img: &TypedImageView<U8x4>,
|
||||
dst_row: &mut [U8x4],
|
||||
coeffs_chunk: CoefficientsI16Chunk,
|
||||
normalizer_guard: &NormalizerGuard16,
|
||||
) {
|
||||
let mut xx: usize = 0;
|
||||
let src_width = src_img.width().get() as usize;
|
||||
let y_start = coeffs_chunk.start;
|
||||
let coeffs = coeffs_chunk.values;
|
||||
let max_y = y_start + coeffs.len() as u32;
|
||||
let precision = normalizer_guard.precision();
|
||||
|
||||
let initial = _mm_set1_epi32(1 << (precision - 1));
|
||||
|
||||
while xx < src_width.saturating_sub(7) {
|
||||
let mut sss0 = initial;
|
||||
let mut sss1 = initial;
|
||||
let mut sss2 = initial;
|
||||
let mut sss3 = initial;
|
||||
let mut sss4 = initial;
|
||||
let mut sss5 = initial;
|
||||
let mut sss6 = initial;
|
||||
let mut sss7 = initial;
|
||||
|
||||
let mut y: u32 = 0;
|
||||
|
||||
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
|
||||
// Load two coefficients at once
|
||||
let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
|
||||
|
||||
let mut source1 = simd_utils::loadu_si128(s_row1, xx); // top line
|
||||
let mut source2 = simd_utils::loadu_si128(s_row2, xx); // bottom line
|
||||
|
||||
let mut source = _mm_unpacklo_epi8(source1, source2);
|
||||
let mut pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
|
||||
pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
source = _mm_unpackhi_epi8(source1, source2);
|
||||
pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss2 = _mm_add_epi32(sss2, _mm_madd_epi16(pix, mmk));
|
||||
pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss3 = _mm_add_epi32(sss3, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
source1 = simd_utils::loadu_si128(s_row1, xx + 4); // top line
|
||||
source2 = simd_utils::loadu_si128(s_row2, xx + 4); // bottom line
|
||||
|
||||
source = _mm_unpacklo_epi8(source1, source2);
|
||||
pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss4 = _mm_add_epi32(sss4, _mm_madd_epi16(pix, mmk));
|
||||
pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss5 = _mm_add_epi32(sss5, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
source = _mm_unpackhi_epi8(source1, source2);
|
||||
pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss6 = _mm_add_epi32(sss6, _mm_madd_epi16(pix, mmk));
|
||||
pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss7 = _mm_add_epi32(sss7, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
y += 2;
|
||||
}
|
||||
|
||||
if let Some(&k) = coeffs.get(y as usize) {
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let mmk = _mm_set1_epi32(k as i32);
|
||||
|
||||
let mut source1 = simd_utils::loadu_si128(s_row, xx); // top line
|
||||
|
||||
let mut source = _mm_unpacklo_epi8(source1, _mm_setzero_si128());
|
||||
let mut pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
|
||||
pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
source = _mm_unpackhi_epi8(source1, _mm_setzero_si128());
|
||||
pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss2 = _mm_add_epi32(sss2, _mm_madd_epi16(pix, mmk));
|
||||
pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss3 = _mm_add_epi32(sss3, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
source1 = simd_utils::loadu_si128(s_row, xx + 4); // top line
|
||||
|
||||
source = _mm_unpacklo_epi8(source1, _mm_setzero_si128());
|
||||
pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss4 = _mm_add_epi32(sss4, _mm_madd_epi16(pix, mmk));
|
||||
pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss5 = _mm_add_epi32(sss5, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
source = _mm_unpackhi_epi8(source1, _mm_setzero_si128());
|
||||
pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss6 = _mm_add_epi32(sss6, _mm_madd_epi16(pix, mmk));
|
||||
pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss7 = _mm_add_epi32(sss7, _mm_madd_epi16(pix, mmk));
|
||||
}
|
||||
|
||||
macro_rules! call {
|
||||
($imm8:expr) => {{
|
||||
sss0 = _mm_srai_epi32::<$imm8>(sss0);
|
||||
sss1 = _mm_srai_epi32::<$imm8>(sss1);
|
||||
sss2 = _mm_srai_epi32::<$imm8>(sss2);
|
||||
sss3 = _mm_srai_epi32::<$imm8>(sss3);
|
||||
sss4 = _mm_srai_epi32::<$imm8>(sss4);
|
||||
sss5 = _mm_srai_epi32::<$imm8>(sss5);
|
||||
sss6 = _mm_srai_epi32::<$imm8>(sss6);
|
||||
sss7 = _mm_srai_epi32::<$imm8>(sss7);
|
||||
}};
|
||||
}
|
||||
constify_imm8!(precision, call);
|
||||
|
||||
sss0 = _mm_packs_epi32(sss0, sss1);
|
||||
sss2 = _mm_packs_epi32(sss2, sss3);
|
||||
sss0 = _mm_packus_epi16(sss0, sss2);
|
||||
let dst_ptr = dst_row.get_unchecked_mut(xx..).as_mut_ptr() as *mut __m128i;
|
||||
_mm_storeu_si128(dst_ptr, sss0);
|
||||
sss4 = _mm_packs_epi32(sss4, sss5);
|
||||
sss6 = _mm_packs_epi32(sss6, sss7);
|
||||
sss4 = _mm_packus_epi16(sss4, sss6);
|
||||
let dst_ptr = dst_row.get_unchecked_mut(xx + 4..).as_mut_ptr() as *mut __m128i;
|
||||
_mm_storeu_si128(dst_ptr, sss4);
|
||||
|
||||
xx += 8;
|
||||
}
|
||||
|
||||
while xx < src_width.saturating_sub(1) {
|
||||
let mut sss0 = initial; // left row
|
||||
let mut sss1 = initial; // right row
|
||||
let mut y: u32 = 0;
|
||||
|
||||
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
|
||||
// Load two coefficients at once
|
||||
let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
|
||||
|
||||
let source1 = simd_utils::loadl_epi64(s_row1, xx); // top line
|
||||
let source2 = simd_utils::loadl_epi64(s_row2, xx); // bottom line
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, source2);
|
||||
let mut pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
|
||||
pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
y += 2;
|
||||
}
|
||||
|
||||
if let Some(&k) = coeffs.get(y as usize) {
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let mmk = _mm_set1_epi32(k as i32);
|
||||
|
||||
let source1 = simd_utils::loadl_epi64(s_row, xx); // top line
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, _mm_setzero_si128());
|
||||
let mut pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
|
||||
pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
|
||||
}
|
||||
|
||||
macro_rules! call {
|
||||
($imm8:expr) => {{
|
||||
sss0 = _mm_srai_epi32::<$imm8>(sss0);
|
||||
sss1 = _mm_srai_epi32::<$imm8>(sss1);
|
||||
}};
|
||||
}
|
||||
constify_imm8!(precision, call);
|
||||
|
||||
sss0 = _mm_packs_epi32(sss0, sss1);
|
||||
sss0 = _mm_packus_epi16(sss0, sss0);
|
||||
let dst_ptr = dst_row.get_unchecked_mut(xx..).as_mut_ptr() as *mut __m128i;
|
||||
_mm_storel_epi64(dst_ptr, sss0);
|
||||
|
||||
xx += 2;
|
||||
}
|
||||
|
||||
if xx < src_width {
|
||||
let mut sss = initial;
|
||||
let mut y: u32 = 0;
|
||||
|
||||
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
|
||||
// Load two coefficients at once
|
||||
let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
|
||||
|
||||
let source1 = simd_utils::mm_cvtsi32_si128_from_u32(s_row1, xx); // top line
|
||||
let source2 = simd_utils::mm_cvtsi32_si128_from_u32(s_row2, xx); // bottom line
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
y += 2;
|
||||
}
|
||||
|
||||
if let Some(&k) = coeffs.get(y as usize) {
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let pix = simd_utils::mm_cvtepu8_epi32(s_row, xx);
|
||||
let mmk = _mm_set1_epi32(k as i32);
|
||||
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
|
||||
}
|
||||
|
||||
macro_rules! call {
|
||||
($imm8:expr) => {{
|
||||
sss = _mm_srai_epi32::<$imm8>(sss);
|
||||
}};
|
||||
}
|
||||
constify_imm8!(precision, call);
|
||||
|
||||
sss = _mm_packs_epi32(sss, sss);
|
||||
*dst_row.get_unchecked_mut(xx) = transmute(_mm_cvtsi128_si32(_mm_packus_epi16(sss, sss)));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
use std::arch::x86_64::*;
|
||||
|
||||
use crate::convolution::{optimisations, Coefficients};
|
||||
use crate::image_view::{TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::Pixel;
|
||||
use crate::simd_utils;
|
||||
|
||||
pub(crate) fn vert_convolution<T>(
|
||||
src_image: TypedImageView<T>,
|
||||
mut dst_image: TypedImageViewMut<T>,
|
||||
coeffs: Coefficients,
|
||||
) where
|
||||
T: Pixel<Component = u16>,
|
||||
{
|
||||
// native::vert_convolution(src_image, dst_image, coeffs);
|
||||
let (values, window_size, bounds_per_pixel) =
|
||||
(coeffs.values, coeffs.window_size, coeffs.bounds);
|
||||
|
||||
let normalizer_guard = optimisations::NormalizerGuard32::new(values);
|
||||
let coefficients_chunks = normalizer_guard.normalized_chunks(window_size, &bounds_per_pixel);
|
||||
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
for (dst_row, coeffs_chunk) in dst_rows.zip(coefficients_chunks) {
|
||||
unsafe {
|
||||
vert_convolution_into_one_row_u16(&src_image, dst_row, coeffs_chunk, &normalizer_guard);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[target_feature(enable = "avx2")]
|
||||
pub(crate) unsafe fn vert_convolution_into_one_row_u16<T>(
|
||||
src_img: &TypedImageView<T>,
|
||||
dst_row: &mut [T],
|
||||
coeffs_chunk: optimisations::CoefficientsI32Chunk,
|
||||
normalizer_guard: &optimisations::NormalizerGuard32,
|
||||
) where
|
||||
T: Pixel<Component = u16>,
|
||||
{
|
||||
let mut xx: usize = 0;
|
||||
let src_width = src_img.width().get() as usize * T::components_count();
|
||||
let y_start = coeffs_chunk.start;
|
||||
let coeffs = coeffs_chunk.values;
|
||||
let dst_components = T::components_mut(dst_row);
|
||||
|
||||
/*
|
||||
|R G B | |R G B | |R G | - |B | |R G B | |R G B | |R |
|
||||
|0001 0203 0405| |0607 0809 1011| |1213 1415| - |0001| |0203 0405 0607| |0809 1011 1213| |1415|
|
||||
|
||||
Shuffle to extract 0-1 components as i64:
|
||||
lo: -1, -1, -1, -1, -1, -1, 3, 2, -1, -1, -1, -1, -1, -1, 1, 0
|
||||
hi: -1, -1, -1, -1, -1, -1, 3, 2, -1, -1, -1, -1, -1, -1, 1, 0
|
||||
|
||||
Shuffle to extract 2-3 components as i64:
|
||||
lo: -1, -1, -1, -1, -1, -1, 7, 6, -1, -1, -1, -1, -1, -1, 5, 4
|
||||
hi: -1, -1, -1, -1, -1, -1, 7, 6, -1, -1, -1, -1, -1, -1, 5, 4
|
||||
|
||||
Shuffle to extract 4-5 components as i64:
|
||||
lo: -1, -1, -1, -1, -1, -1, 11, 10, -1, -1, -1, -1, -1, -1, 9, 8
|
||||
hi: -1, -1, -1, -1, -1, -1, 11, 10, -1, -1, -1, -1, -1, -1, 9, 8
|
||||
|
||||
Shuffle to extract 6-7 components as i64:
|
||||
lo: -1, -1, -1, -1, -1, -1, 15, 14, -1, -1, -1, -1, -1, -1, 13, 12
|
||||
hi: -1, -1, -1, -1, -1, -1, 15, 14, -1, -1, -1, -1, -1, -1, 13, 12
|
||||
*/
|
||||
|
||||
let shuffles = [
|
||||
_mm256_set_m128i(
|
||||
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 3, 2, -1, -1, -1, -1, -1, -1, 1, 0),
|
||||
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 3, 2, -1, -1, -1, -1, -1, -1, 1, 0),
|
||||
),
|
||||
_mm256_set_m128i(
|
||||
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 7, 6, -1, -1, -1, -1, -1, -1, 5, 4),
|
||||
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 7, 6, -1, -1, -1, -1, -1, -1, 5, 4),
|
||||
),
|
||||
_mm256_set_m128i(
|
||||
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 11, 10, -1, -1, -1, -1, -1, -1, 9, 8),
|
||||
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 11, 10, -1, -1, -1, -1, -1, -1, 9, 8),
|
||||
),
|
||||
_mm256_set_m128i(
|
||||
_mm_set_epi8(
|
||||
-1, -1, -1, -1, -1, -1, 15, 14, -1, -1, -1, -1, -1, -1, 13, 12,
|
||||
),
|
||||
_mm_set_epi8(
|
||||
-1, -1, -1, -1, -1, -1, 15, 14, -1, -1, -1, -1, -1, -1, 13, 12,
|
||||
),
|
||||
),
|
||||
];
|
||||
|
||||
let precision = normalizer_guard.precision();
|
||||
let initial = _mm256_set1_epi64x(1 << (precision - 1));
|
||||
let mut comp_buf = [0i64; 4];
|
||||
|
||||
// 16 components in one register - 1 = 15
|
||||
while xx < src_width.saturating_sub(15) {
|
||||
// 16 components / 4 per register = 4 registers
|
||||
let mut sum = [initial; 4];
|
||||
|
||||
for (s_row, &coeff) in src_img.iter_rows(y_start).zip(coeffs) {
|
||||
let components = T::components(s_row);
|
||||
let coeff_i64x4 = _mm256_set1_epi64x(coeff as i64);
|
||||
let source = simd_utils::loadu_si256(components, xx);
|
||||
for i in 0..4 {
|
||||
let comp_i64x4 = _mm256_shuffle_epi8(source, shuffles[i]);
|
||||
sum[i] = _mm256_add_epi64(sum[i], _mm256_mul_epi32(comp_i64x4, coeff_i64x4));
|
||||
}
|
||||
}
|
||||
|
||||
for i in 0..4 {
|
||||
_mm256_storeu_si256((&mut comp_buf).as_mut_ptr() as *mut __m256i, sum[i]);
|
||||
let component = dst_components.get_unchecked_mut(xx + i * 2);
|
||||
*component = normalizer_guard.clip(comp_buf[0]);
|
||||
let component = dst_components.get_unchecked_mut(xx + i * 2 + 1);
|
||||
*component = normalizer_guard.clip(comp_buf[1]);
|
||||
let component = dst_components.get_unchecked_mut(xx + i * 2 + 8);
|
||||
*component = normalizer_guard.clip(comp_buf[2]);
|
||||
let component = dst_components.get_unchecked_mut(xx + i * 2 + 9);
|
||||
*component = normalizer_guard.clip(comp_buf[3]);
|
||||
}
|
||||
|
||||
xx += 16;
|
||||
}
|
||||
|
||||
if xx < src_width {
|
||||
// 16 components / 4 per register = 4 registers
|
||||
let mut sum = [initial; 4];
|
||||
let mut buf = [0u16; 16];
|
||||
|
||||
for (s_row, &coeff) in src_img.iter_rows(y_start).zip(coeffs) {
|
||||
let components = T::components(s_row);
|
||||
for (i, &v) in components.get_unchecked(xx..).iter().enumerate() {
|
||||
buf[i] = v;
|
||||
}
|
||||
let coeff_i64x4 = _mm256_set1_epi64x(coeff as i64);
|
||||
let source = simd_utils::loadu_si256(&buf, 0);
|
||||
for i in 0..4 {
|
||||
let comp_i64x4 = _mm256_shuffle_epi8(source, shuffles[i]);
|
||||
sum[i] = _mm256_add_epi64(sum[i], _mm256_mul_epi32(comp_i64x4, coeff_i64x4));
|
||||
}
|
||||
}
|
||||
|
||||
for i in 0..4 {
|
||||
_mm256_storeu_si256((&mut comp_buf).as_mut_ptr() as *mut __m256i, sum[i]);
|
||||
let component = buf.get_unchecked_mut(i * 2);
|
||||
*component = normalizer_guard.clip(comp_buf[0]);
|
||||
let component = buf.get_unchecked_mut(i * 2 + 1);
|
||||
*component = normalizer_guard.clip(comp_buf[1]);
|
||||
let component = buf.get_unchecked_mut(i * 2 + 8);
|
||||
*component = normalizer_guard.clip(comp_buf[2]);
|
||||
let component = buf.get_unchecked_mut(i * 2 + 9);
|
||||
*component = normalizer_guard.clip(comp_buf[3]);
|
||||
}
|
||||
for (i, v) in dst_components
|
||||
.get_unchecked_mut(xx..)
|
||||
.iter_mut()
|
||||
.enumerate()
|
||||
{
|
||||
*v = buf[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
use crate::convolution::Coefficients;
|
||||
use crate::image_view::{TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::Pixel;
|
||||
use crate::CpuExtensions;
|
||||
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
pub(crate) mod avx2;
|
||||
pub(crate) mod native;
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
pub(crate) mod sse4;
|
||||
|
||||
pub(crate) fn vert_convolution_u16<T: Pixel<Component = u16>>(
|
||||
src_image: TypedImageView<T>,
|
||||
dst_image: TypedImageViewMut<T>,
|
||||
coeffs: Coefficients,
|
||||
cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
match cpu_extensions {
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Avx2 => avx2::vert_convolution(src_image, dst_image, coeffs),
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Sse4_1 => sse4::vert_convolution(src_image, dst_image, coeffs),
|
||||
_ => native::vert_convolution(src_image, dst_image, coeffs),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
use crate::convolution::{optimisations, Coefficients};
|
||||
use crate::image_view::{TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::Pixel;
|
||||
|
||||
#[inline(always)]
|
||||
pub(crate) fn vert_convolution<T: Pixel<Component = u16>>(
|
||||
src_image: TypedImageView<T>,
|
||||
mut dst_image: TypedImageViewMut<T>,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
// Check safety conditions
|
||||
debug_assert_eq!(src_image.width(), dst_image.width());
|
||||
debug_assert_eq!(coeffs.bounds.len(), dst_image.height().get() as usize);
|
||||
|
||||
let (values, window_size, bounds) = (coeffs.values, coeffs.window_size, coeffs.bounds);
|
||||
let normalizer_guard = optimisations::NormalizerGuard32::new(values);
|
||||
let coefficients_chunks = normalizer_guard.normalized_chunks(window_size, &bounds);
|
||||
let precision = normalizer_guard.precision();
|
||||
let initial: i64 = 1 << (precision - 1);
|
||||
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
let coeffs_chunks_iter = coefficients_chunks.into_iter();
|
||||
for (coeffs_chunk, dst_row) in coeffs_chunks_iter.zip(dst_rows) {
|
||||
let first_y_src = coeffs_chunk.start;
|
||||
let ks = coeffs_chunk.values;
|
||||
let dst_components = T::components_mut(dst_row);
|
||||
|
||||
convolution_by_u16(
|
||||
&src_image,
|
||||
&normalizer_guard,
|
||||
initial,
|
||||
dst_components,
|
||||
0,
|
||||
first_y_src,
|
||||
ks,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub(crate) fn convolution_by_u16<T: Pixel<Component = u16>>(
|
||||
src_image: &TypedImageView<T>,
|
||||
normalizer_guard: &optimisations::NormalizerGuard32,
|
||||
initial: i64,
|
||||
dst_components: &mut [u16],
|
||||
mut x_src: usize,
|
||||
first_y_src: u32,
|
||||
ks: &[i32],
|
||||
) -> usize {
|
||||
for dst_component in dst_components.iter_mut().skip(x_src) {
|
||||
let mut ss = initial;
|
||||
let src_rows = src_image.iter_rows(first_y_src);
|
||||
for (&k, src_row) in ks.iter().zip(src_rows) {
|
||||
let src_ptr = src_row.as_ptr() as *const u16;
|
||||
let src_component = unsafe { *src_ptr.add(x_src as usize) };
|
||||
ss += src_component as i64 * (k as i64);
|
||||
}
|
||||
*dst_component = normalizer_guard.clip(ss);
|
||||
x_src += 1
|
||||
}
|
||||
x_src
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
use std::arch::x86_64::*;
|
||||
|
||||
use crate::convolution::optimisations::CoefficientsI32Chunk;
|
||||
use crate::convolution::vertical_u16::native::convolution_by_u16;
|
||||
use crate::convolution::{optimisations, Coefficients};
|
||||
use crate::image_view::{TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::Pixel;
|
||||
use crate::simd_utils;
|
||||
|
||||
pub(crate) fn vert_convolution<T: Pixel<Component = u16>>(
|
||||
src_image: TypedImageView<T>,
|
||||
mut dst_image: TypedImageViewMut<T>,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
// native::vert_convolution(src_image, dst_image, coeffs);
|
||||
let (values, window_size, bounds_per_pixel) =
|
||||
(coeffs.values, coeffs.window_size, coeffs.bounds);
|
||||
|
||||
let normalizer_guard = optimisations::NormalizerGuard32::new(values);
|
||||
let coefficients_chunks = normalizer_guard.normalized_chunks(window_size, &bounds_per_pixel);
|
||||
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
for (dst_row, coeffs_chunk) in dst_rows.zip(coefficients_chunks) {
|
||||
unsafe {
|
||||
vert_convolution_into_one_row_u16(&src_image, dst_row, coeffs_chunk, &normalizer_guard);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[target_feature(enable = "sse4.1")]
|
||||
unsafe fn vert_convolution_into_one_row_u16<T: Pixel<Component = u16>>(
|
||||
src_img: &TypedImageView<T>,
|
||||
dst_row: &mut [T],
|
||||
coeffs_chunk: CoefficientsI32Chunk,
|
||||
normalizer_guard: &optimisations::NormalizerGuard32,
|
||||
) {
|
||||
let mut xx: usize = 0;
|
||||
let src_width = src_img.width().get() as usize * T::components_count();
|
||||
let y_start = coeffs_chunk.start;
|
||||
let coeffs = coeffs_chunk.values;
|
||||
let max_y = y_start + coeffs.len() as u32;
|
||||
let dst_components = T::components_mut(dst_row);
|
||||
let mut dst_ptr_u16 = dst_components.as_mut_ptr() as *mut u16;
|
||||
|
||||
/*
|
||||
|0 1 2 3 4 5 6 7 |
|
||||
|0001 0203 0405 0607 0809 1011 1213 1415|
|
||||
|
||||
Shuffle to extract 0-1 components as i64:
|
||||
-1, -1, -1, -1, -1, -1, 3, 2, -1, -1, -1, -1, -1, -1, 1, 0
|
||||
|
||||
Shuffle to extract 2-3 components as i64:
|
||||
-1, -1, -1, -1, -1, -1, 7, 6, -1, -1, -1, -1, -1, -1, 5, 4
|
||||
|
||||
Shuffle to extract 4-5 components as i64:
|
||||
-1, -1, -1, -1, -1, -1, 11, 10, -1, -1, -1, -1, -1, -1, 9, 8
|
||||
|
||||
Shuffle to extract 6-7 components as i64:
|
||||
-1, -1, -1, -1, -1, -1, 15, 14, -1, -1, -1, -1, -1, -1, 13, 12
|
||||
|
||||
*/
|
||||
|
||||
let c_shuffles = [
|
||||
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 3, 2, -1, -1, -1, -1, -1, -1, 1, 0),
|
||||
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 7, 6, -1, -1, -1, -1, -1, -1, 5, 4),
|
||||
_mm_set_epi8(-1, -1, -1, -1, -1, -1, 11, 10, -1, -1, -1, -1, -1, -1, 9, 8),
|
||||
_mm_set_epi8(
|
||||
-1, -1, -1, -1, -1, -1, 15, 14, -1, -1, -1, -1, -1, -1, 13, 12,
|
||||
),
|
||||
];
|
||||
|
||||
let precision = normalizer_guard.precision();
|
||||
let initial = _mm_set1_epi64x(1 << (precision - 1));
|
||||
let mut c_buf = [0i64; 2];
|
||||
|
||||
// 16 components - 1 = 15
|
||||
while xx < src_width.saturating_sub(15) {
|
||||
let mut sums = [[initial; 2], [initial; 2], [initial; 2], [initial; 2]];
|
||||
|
||||
let mut y: u32 = 0;
|
||||
let coeffs_2 = coeffs.chunks_exact(2);
|
||||
let coeffs_reminder = coeffs_2.remainder();
|
||||
|
||||
for ((s_row0, s_row1), two_coeffs) in src_img.iter_2_rows(y_start, max_y).zip(coeffs_2) {
|
||||
let s_rows = [T::components(s_row0), T::components(s_row1)];
|
||||
|
||||
for r in 0..2 {
|
||||
let coeff_i64x2 = _mm_set1_epi64x(two_coeffs[r] as i64);
|
||||
for x in 0..2 {
|
||||
let source = simd_utils::loadu_si128(s_rows[r], xx + x * 8);
|
||||
for i in 0..4 {
|
||||
let c_i64x2 = _mm_shuffle_epi8(source, c_shuffles[i]);
|
||||
sums[i][x] = _mm_add_epi64(sums[i][x], _mm_mul_epi32(c_i64x2, coeff_i64x2));
|
||||
}
|
||||
}
|
||||
}
|
||||
y += 2;
|
||||
}
|
||||
|
||||
if let Some(&k) = coeffs_reminder.get(0) {
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let components = T::components(s_row);
|
||||
let coeff_i64x2 = _mm_set1_epi64x(k as i64);
|
||||
|
||||
for x in 0..2 {
|
||||
let source = simd_utils::loadu_si128(components, xx + x * 8);
|
||||
for i in 0..4 {
|
||||
let c_i64x2 = _mm_shuffle_epi8(source, c_shuffles[i]);
|
||||
sums[i][x] = _mm_add_epi64(sums[i][x], _mm_mul_epi32(c_i64x2, coeff_i64x2));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for x in 0..2 {
|
||||
for sum in sums {
|
||||
_mm_storeu_si128((&mut c_buf).as_mut_ptr() as *mut __m128i, sum[x]);
|
||||
*dst_ptr_u16 = normalizer_guard.clip(c_buf[0]);
|
||||
dst_ptr_u16 = dst_ptr_u16.add(1);
|
||||
*dst_ptr_u16 = normalizer_guard.clip(c_buf[1]);
|
||||
dst_ptr_u16 = dst_ptr_u16.add(1);
|
||||
}
|
||||
}
|
||||
|
||||
xx += 16;
|
||||
}
|
||||
|
||||
// 8 components - 1 = 7
|
||||
while xx < src_width.saturating_sub(7) {
|
||||
let mut sums = [initial, initial, initial, initial];
|
||||
|
||||
let mut y: u32 = 0;
|
||||
let coeffs_2 = coeffs.chunks_exact(2);
|
||||
let coeffs_reminder = coeffs_2.remainder();
|
||||
|
||||
for ((s_row0, s_row1), two_coeffs) in src_img.iter_2_rows(y_start, max_y).zip(coeffs_2) {
|
||||
let s_rows = [T::components(s_row0), T::components(s_row1)];
|
||||
let coeffs_i64 = [
|
||||
_mm_set1_epi64x(two_coeffs[0] as i64),
|
||||
_mm_set1_epi64x(two_coeffs[1] as i64),
|
||||
];
|
||||
|
||||
for r in 0..2 {
|
||||
let source = simd_utils::loadu_si128(s_rows[r], xx);
|
||||
for i in 0..4 {
|
||||
let c_i64x2 = _mm_shuffle_epi8(source, c_shuffles[i]);
|
||||
sums[i] = _mm_add_epi64(sums[i], _mm_mul_epi32(c_i64x2, coeffs_i64[r]));
|
||||
}
|
||||
}
|
||||
y += 2;
|
||||
}
|
||||
|
||||
if let Some(&k) = coeffs_reminder.get(0) {
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let components = T::components(s_row);
|
||||
let coeff_i64x2 = _mm_set1_epi64x(k as i64);
|
||||
let source = simd_utils::loadu_si128(components, xx);
|
||||
for i in 0..4 {
|
||||
let c_i64x2 = _mm_shuffle_epi8(source, c_shuffles[i]);
|
||||
sums[i] = _mm_add_epi64(sums[i], _mm_mul_epi32(c_i64x2, coeff_i64x2));
|
||||
}
|
||||
}
|
||||
|
||||
for sum in sums {
|
||||
// let mask = _mm_cmpgt_epi64(sums[i], zero);
|
||||
// sums[i] = _mm_and_si128(sums[i] , mask);
|
||||
// sums[i] = _mm_srl_epi64(sums[i] , precision_i64);
|
||||
// _mm_packus_epi32(sums[i] , sums[i] );
|
||||
_mm_storeu_si128((&mut c_buf).as_mut_ptr() as *mut __m128i, sum);
|
||||
*dst_ptr_u16 = normalizer_guard.clip(c_buf[0]);
|
||||
dst_ptr_u16 = dst_ptr_u16.add(1);
|
||||
*dst_ptr_u16 = normalizer_guard.clip(c_buf[1]);
|
||||
dst_ptr_u16 = dst_ptr_u16.add(1);
|
||||
}
|
||||
|
||||
xx += 8;
|
||||
}
|
||||
|
||||
// 4 components - 1 = 3
|
||||
while xx < src_width.saturating_sub(3) {
|
||||
let mut c01 = initial;
|
||||
let mut c23 = initial;
|
||||
let mut y: u32 = 0;
|
||||
let coeffs_2 = coeffs.chunks_exact(2);
|
||||
let coeffs_reminder = coeffs_2.remainder();
|
||||
|
||||
for ((s_row0, s_row1), two_coeffs) in src_img.iter_2_rows(y_start, max_y).zip(coeffs_2) {
|
||||
let s_rows = [T::components(s_row0), T::components(s_row1)];
|
||||
let coeffs_i64 = [
|
||||
_mm_set1_epi64x(two_coeffs[0] as i64),
|
||||
_mm_set1_epi64x(two_coeffs[1] as i64),
|
||||
];
|
||||
for r in 0..2 {
|
||||
let comp_x4 = s_rows[r].get_unchecked(xx..xx + 4);
|
||||
let c_i64x2 = _mm_set_epi64x(comp_x4[1] as i64, comp_x4[0] as i64);
|
||||
c01 = _mm_add_epi64(c01, _mm_mul_epi32(c_i64x2, coeffs_i64[r]));
|
||||
let c_i64x2 = _mm_set_epi64x(comp_x4[3] as i64, comp_x4[2] as i64);
|
||||
c23 = _mm_add_epi64(c23, _mm_mul_epi32(c_i64x2, coeffs_i64[r]));
|
||||
}
|
||||
y += 2;
|
||||
}
|
||||
|
||||
if let Some(&k) = coeffs_reminder.get(0) {
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let components = T::components(s_row);
|
||||
let coeff_i64x2 = _mm_set1_epi64x(k as i64);
|
||||
|
||||
let comp_x4 = components.get_unchecked(xx..xx + 4);
|
||||
let c_i64x2 = _mm_set_epi64x(comp_x4[1] as i64, comp_x4[0] as i64);
|
||||
c01 = _mm_add_epi64(c01, _mm_mul_epi32(c_i64x2, coeff_i64x2));
|
||||
let c_i64x2 = _mm_set_epi64x(comp_x4[3] as i64, comp_x4[2] as i64);
|
||||
c23 = _mm_add_epi64(c23, _mm_mul_epi32(c_i64x2, coeff_i64x2));
|
||||
}
|
||||
|
||||
_mm_storeu_si128((&mut c_buf).as_mut_ptr() as *mut __m128i, c01);
|
||||
*dst_ptr_u16 = normalizer_guard.clip(c_buf[0]);
|
||||
dst_ptr_u16 = dst_ptr_u16.add(1);
|
||||
*dst_ptr_u16 = normalizer_guard.clip(c_buf[1]);
|
||||
dst_ptr_u16 = dst_ptr_u16.add(1);
|
||||
_mm_storeu_si128((&mut c_buf).as_mut_ptr() as *mut __m128i, c23);
|
||||
*dst_ptr_u16 = normalizer_guard.clip(c_buf[0]);
|
||||
dst_ptr_u16 = dst_ptr_u16.add(1);
|
||||
*dst_ptr_u16 = normalizer_guard.clip(c_buf[1]);
|
||||
dst_ptr_u16 = dst_ptr_u16.add(1);
|
||||
|
||||
xx += 4;
|
||||
}
|
||||
|
||||
if xx < src_width {
|
||||
let initial = 1 << (precision - 1);
|
||||
convolution_by_u16(
|
||||
src_img,
|
||||
normalizer_guard,
|
||||
initial,
|
||||
dst_components,
|
||||
xx,
|
||||
y_start,
|
||||
coeffs,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
use std::arch::x86_64::*;
|
||||
|
||||
use crate::convolution::optimisations::{CoefficientsI16Chunk, NormalizerGuard16};
|
||||
use crate::convolution::{optimisations, Coefficients};
|
||||
use crate::image_view::{TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::Pixel;
|
||||
use crate::simd_utils;
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn vert_convolution<T>(
|
||||
src_image: TypedImageView<T>,
|
||||
mut dst_image: TypedImageViewMut<T>,
|
||||
coeffs: Coefficients,
|
||||
) where
|
||||
T: Pixel<Component = u8>,
|
||||
{
|
||||
let (values, window_size, bounds_per_pixel) =
|
||||
(coeffs.values, coeffs.window_size, coeffs.bounds);
|
||||
|
||||
let normalizer_guard = optimisations::NormalizerGuard16::new(values);
|
||||
let coefficients_chunks = normalizer_guard.normalized_chunks(window_size, &bounds_per_pixel);
|
||||
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
for (dst_row, coeffs_chunk) in dst_rows.zip(coefficients_chunks) {
|
||||
unsafe {
|
||||
vert_convolution_into_one_row_u8(&src_image, dst_row, coeffs_chunk, &normalizer_guard);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[target_feature(enable = "avx2")]
|
||||
unsafe fn vert_convolution_into_one_row_u8<T>(
|
||||
src_img: &TypedImageView<T>,
|
||||
dst_row: &mut [T],
|
||||
coeffs_chunk: CoefficientsI16Chunk,
|
||||
normalizer_guard: &NormalizerGuard16,
|
||||
) where
|
||||
T: Pixel<Component = u8>,
|
||||
{
|
||||
let src_width = src_img.width().get() as usize * T::components_count();
|
||||
let y_start = coeffs_chunk.start;
|
||||
let coeffs = coeffs_chunk.values;
|
||||
let max_y = y_start + coeffs.len() as u32;
|
||||
let precision = normalizer_guard.precision();
|
||||
|
||||
let initial = _mm_set1_epi32(1 << (precision - 1));
|
||||
let initial_256 = _mm256_set1_epi32(1 << (precision - 1));
|
||||
|
||||
let mut x_in_bytes: usize = 0;
|
||||
let dst_ptr_u8 = T::components_mut(dst_row).as_mut_ptr() as *mut u8;
|
||||
|
||||
// 32 components in one register - 1 = 31
|
||||
while x_in_bytes < src_width.saturating_sub(31) {
|
||||
let mut sss0 = initial_256;
|
||||
let mut sss1 = initial_256;
|
||||
let mut sss2 = initial_256;
|
||||
let mut sss3 = initial_256;
|
||||
|
||||
let mut y: u32 = 0;
|
||||
|
||||
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
|
||||
let components1 = T::components(s_row1);
|
||||
let components2 = T::components(s_row2);
|
||||
|
||||
// Load two coefficients at once
|
||||
let mmk = simd_utils::ptr_i16_to_256set1_epi32(coeffs, y as usize);
|
||||
|
||||
let source1 = simd_utils::loadu_si256(components1, x_in_bytes); // top line
|
||||
let source2 = simd_utils::loadu_si256(components2, x_in_bytes); // bottom line
|
||||
|
||||
let source = _mm256_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
|
||||
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
|
||||
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
|
||||
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
|
||||
|
||||
let source = _mm256_unpackhi_epi8(source1, source2);
|
||||
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
|
||||
sss2 = _mm256_add_epi32(sss2, _mm256_madd_epi16(pix, mmk));
|
||||
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
|
||||
sss3 = _mm256_add_epi32(sss3, _mm256_madd_epi16(pix, mmk));
|
||||
|
||||
y += 2;
|
||||
}
|
||||
|
||||
if let Some(&k) = coeffs.get(y as usize) {
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let components = T::components(s_row);
|
||||
let mmk = _mm256_set1_epi32(k as i32);
|
||||
|
||||
let source1 = simd_utils::loadu_si256(components, x_in_bytes); // top line
|
||||
let source2 = _mm256_setzero_si256(); // bottom line is empty
|
||||
|
||||
let source = _mm256_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
|
||||
sss0 = _mm256_add_epi32(sss0, _mm256_madd_epi16(pix, mmk));
|
||||
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
|
||||
sss1 = _mm256_add_epi32(sss1, _mm256_madd_epi16(pix, mmk));
|
||||
|
||||
let source = _mm256_unpackhi_epi8(source1, _mm256_setzero_si256());
|
||||
let pix = _mm256_unpacklo_epi8(source, _mm256_setzero_si256());
|
||||
sss2 = _mm256_add_epi32(sss2, _mm256_madd_epi16(pix, mmk));
|
||||
let pix = _mm256_unpackhi_epi8(source, _mm256_setzero_si256());
|
||||
sss3 = _mm256_add_epi32(sss3, _mm256_madd_epi16(pix, mmk));
|
||||
}
|
||||
|
||||
macro_rules! call {
|
||||
($imm8:expr) => {{
|
||||
sss0 = _mm256_srai_epi32::<$imm8>(sss0);
|
||||
sss1 = _mm256_srai_epi32::<$imm8>(sss1);
|
||||
sss2 = _mm256_srai_epi32::<$imm8>(sss2);
|
||||
sss3 = _mm256_srai_epi32::<$imm8>(sss3);
|
||||
}};
|
||||
}
|
||||
constify_imm8!(precision, call);
|
||||
|
||||
sss0 = _mm256_packs_epi32(sss0, sss1);
|
||||
sss2 = _mm256_packs_epi32(sss2, sss3);
|
||||
sss0 = _mm256_packus_epi16(sss0, sss2);
|
||||
|
||||
let dst_ptr = dst_ptr_u8.add(x_in_bytes) as *mut __m256i;
|
||||
_mm256_storeu_si256(dst_ptr, sss0);
|
||||
|
||||
x_in_bytes += 32;
|
||||
}
|
||||
|
||||
// 8 components in half of SSE register - 1 = 7
|
||||
while x_in_bytes < src_width.saturating_sub(7) {
|
||||
let mut sss0 = initial; // left row
|
||||
let mut sss1 = initial; // right row
|
||||
let mut y: u32 = 0;
|
||||
|
||||
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
|
||||
let components1 = T::components(s_row1);
|
||||
let components2 = T::components(s_row2);
|
||||
// Load two coefficients at once
|
||||
let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
|
||||
|
||||
let source1 = simd_utils::loadl_epi64(components1, x_in_bytes); // top line
|
||||
let source2 = simd_utils::loadl_epi64(components2, x_in_bytes); // bottom line
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
|
||||
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
y += 2;
|
||||
}
|
||||
|
||||
if let Some(&k) = coeffs.get(y as usize) {
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let components = T::components(s_row);
|
||||
let mmk = _mm_set1_epi32(k as i32);
|
||||
|
||||
let source1 = simd_utils::loadl_epi64(components, x_in_bytes); // top line
|
||||
let source2 = _mm_setzero_si128(); // bottom line is empty
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
|
||||
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
|
||||
}
|
||||
|
||||
macro_rules! call {
|
||||
($imm8:expr) => {{
|
||||
sss0 = _mm_srai_epi32::<$imm8>(sss0);
|
||||
sss1 = _mm_srai_epi32::<$imm8>(sss1);
|
||||
}};
|
||||
}
|
||||
constify_imm8!(precision, call);
|
||||
|
||||
sss0 = _mm_packs_epi32(sss0, sss1);
|
||||
sss0 = _mm_packus_epi16(sss0, sss0);
|
||||
|
||||
let dst_ptr = dst_ptr_u8.add(x_in_bytes) as *mut __m128i;
|
||||
_mm_storel_epi64(dst_ptr, sss0);
|
||||
|
||||
x_in_bytes += 8;
|
||||
}
|
||||
|
||||
while x_in_bytes < src_width.saturating_sub(3) {
|
||||
let mut sss = initial;
|
||||
let mut y: u32 = 0;
|
||||
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
|
||||
let components1 = T::components(s_row1);
|
||||
let components2 = T::components(s_row2);
|
||||
// Load two coefficients at once
|
||||
let two_coeffs = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
|
||||
|
||||
let row1 = simd_utils::mm_cvtsi32_si128_from_u8(components1, x_in_bytes); // top line
|
||||
let row2 = simd_utils::mm_cvtsi32_si128_from_u8(components2, x_in_bytes); // bottom line
|
||||
|
||||
let pixels_u8 = _mm_unpacklo_epi8(row1, row2);
|
||||
let pixels_i16 = _mm_unpacklo_epi8(pixels_u8, _mm_setzero_si128());
|
||||
sss = _mm_add_epi32(sss, _mm_madd_epi16(pixels_i16, two_coeffs));
|
||||
|
||||
y += 2;
|
||||
}
|
||||
|
||||
if let Some(&k) = coeffs.get(y as usize) {
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let components = T::components(s_row);
|
||||
let pix = simd_utils::mm_cvtepu8_epi32_from_u8(components, x_in_bytes);
|
||||
let mmk = _mm_set1_epi32(k as i32);
|
||||
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
|
||||
}
|
||||
|
||||
macro_rules! call {
|
||||
($imm8:expr) => {{
|
||||
sss = _mm_srai_epi32::<$imm8>(sss);
|
||||
}};
|
||||
}
|
||||
constify_imm8!(precision, call);
|
||||
|
||||
sss = _mm_packs_epi32(sss, sss);
|
||||
let dst_ptr_i32 = dst_ptr_u8.add(x_in_bytes) as *mut i32;
|
||||
*dst_ptr_i32 = _mm_cvtsi128_si32(_mm_packus_epi16(sss, sss));
|
||||
|
||||
x_in_bytes += 4;
|
||||
}
|
||||
|
||||
if x_in_bytes < src_width {
|
||||
let dst_u8 =
|
||||
std::slice::from_raw_parts_mut(dst_ptr_u8.add(x_in_bytes), src_width - x_in_bytes);
|
||||
|
||||
for dst_pixel in dst_u8 {
|
||||
let mut ss0 = 1 << (precision - 1);
|
||||
for (dy, &k) in coeffs.iter().enumerate() {
|
||||
if let Some(src_row) = src_img.get_row(y_start + dy as u32) {
|
||||
let src_ptr = src_row.as_ptr() as *const u8;
|
||||
let src_component = *src_ptr.add(x_in_bytes);
|
||||
ss0 += src_component as i32 * (k as i32);
|
||||
}
|
||||
}
|
||||
*dst_pixel = normalizer_guard.clip(ss0);
|
||||
x_in_bytes += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
use crate::convolution::Coefficients;
|
||||
use crate::image_view::{TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::Pixel;
|
||||
use crate::CpuExtensions;
|
||||
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
pub(crate) mod avx2;
|
||||
pub(crate) mod native;
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
pub(crate) mod sse4;
|
||||
|
||||
pub(crate) fn vert_convolution_u8<T: Pixel<Component = u8>>(
|
||||
src_image: TypedImageView<T>,
|
||||
dst_image: TypedImageViewMut<T>,
|
||||
coeffs: Coefficients,
|
||||
cpu_extensions: CpuExtensions,
|
||||
) {
|
||||
match cpu_extensions {
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Avx2 => avx2::vert_convolution(src_image, dst_image, coeffs),
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
CpuExtensions::Sse4_1 => sse4::vert_convolution(src_image, dst_image, coeffs),
|
||||
_ => native::vert_convolution(src_image, dst_image, coeffs),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
use crate::convolution::optimisations::NormalizerGuard16;
|
||||
use crate::convolution::{optimisations, Coefficients};
|
||||
use crate::image_view::{TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::Pixel;
|
||||
|
||||
#[inline(always)]
|
||||
pub(crate) fn vert_convolution<T>(
|
||||
src_image: TypedImageView<T>,
|
||||
mut dst_image: TypedImageViewMut<T>,
|
||||
coeffs: Coefficients,
|
||||
) where
|
||||
T: Pixel<Component = u8>,
|
||||
{
|
||||
// Check safety conditions
|
||||
debug_assert_eq!(src_image.width(), dst_image.width());
|
||||
debug_assert_eq!(coeffs.bounds.len(), dst_image.height().get() as usize);
|
||||
|
||||
let (values, window_size, bounds) = (coeffs.values, coeffs.window_size, coeffs.bounds);
|
||||
let normalizer_guard = optimisations::NormalizerGuard16::new(values);
|
||||
let coefficients_chunks = normalizer_guard.normalized_chunks(window_size, &bounds);
|
||||
let precision = normalizer_guard.precision();
|
||||
let initial = 1 << (precision - 1);
|
||||
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
let coeffs_chunks_iter = coefficients_chunks.into_iter();
|
||||
for (coeffs_chunk, dst_row) in coeffs_chunks_iter.zip(dst_rows) {
|
||||
let first_y_src = coeffs_chunk.start;
|
||||
let ks = coeffs_chunk.values;
|
||||
let mut x_src: usize = 0;
|
||||
let dst_components = T::components_mut(dst_row);
|
||||
let (head, dst_chunks, tail) = unsafe { dst_components.align_to_mut::<u32>() };
|
||||
|
||||
if !head.is_empty() {
|
||||
x_src = convolution_by_u8(
|
||||
&src_image,
|
||||
&normalizer_guard,
|
||||
initial,
|
||||
head,
|
||||
x_src,
|
||||
first_y_src,
|
||||
ks,
|
||||
);
|
||||
}
|
||||
|
||||
// Convolution by u8x4
|
||||
for dst_chunk in dst_chunks {
|
||||
let mut ss = [initial; 4];
|
||||
let src_rows = src_image.iter_rows(first_y_src);
|
||||
for (&k, src_row) in ks.iter().zip(src_rows) {
|
||||
let src_ptr = src_row.as_ptr() as *const u8;
|
||||
let src_chunk = unsafe {
|
||||
let ptr = src_ptr.add(x_src) as *const u32;
|
||||
*ptr
|
||||
};
|
||||
|
||||
let components: [u8; 4] = src_chunk.to_le_bytes();
|
||||
for (s, c) in ss.iter_mut().zip(components) {
|
||||
*s += c as i32 * (k as i32);
|
||||
}
|
||||
}
|
||||
*dst_chunk = u32::from_le_bytes(ss.map(|v| unsafe { normalizer_guard.clip(v) }));
|
||||
x_src += 4;
|
||||
}
|
||||
|
||||
if !tail.is_empty() {
|
||||
convolution_by_u8(
|
||||
&src_image,
|
||||
&normalizer_guard,
|
||||
initial,
|
||||
tail,
|
||||
x_src,
|
||||
first_y_src,
|
||||
ks,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
fn convolution_by_u8<T>(
|
||||
src_image: &TypedImageView<T>,
|
||||
normalizer_guard: &NormalizerGuard16,
|
||||
initial: i32,
|
||||
dst_components: &mut [u8],
|
||||
mut x_src: usize,
|
||||
first_y_src: u32,
|
||||
ks: &[i16],
|
||||
) -> usize
|
||||
where
|
||||
T: Pixel<Component = u8>,
|
||||
{
|
||||
for dst_component in dst_components {
|
||||
let mut ss = initial;
|
||||
let src_rows = src_image.iter_rows(first_y_src);
|
||||
for (&k, src_row) in ks.iter().zip(src_rows) {
|
||||
let src_ptr = src_row.as_ptr() as *const u8;
|
||||
let src_component = unsafe { *src_ptr.add(x_src as usize) };
|
||||
ss += src_component as i32 * (k as i32);
|
||||
}
|
||||
*dst_component = unsafe { normalizer_guard.clip(ss) };
|
||||
x_src += 1
|
||||
}
|
||||
x_src
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
use std::arch::x86_64::*;
|
||||
|
||||
use crate::convolution::optimisations::{CoefficientsI16Chunk, NormalizerGuard16};
|
||||
use crate::convolution::{optimisations, Coefficients};
|
||||
use crate::image_view::{TypedImageView, TypedImageViewMut};
|
||||
use crate::pixels::Pixel;
|
||||
use crate::simd_utils;
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn vert_convolution<T: Pixel<Component = u8>>(
|
||||
src_image: TypedImageView<T>,
|
||||
mut dst_image: TypedImageViewMut<T>,
|
||||
coeffs: Coefficients,
|
||||
) {
|
||||
let (values, window_size, bounds_per_pixel) =
|
||||
(coeffs.values, coeffs.window_size, coeffs.bounds);
|
||||
|
||||
let normalizer_guard = optimisations::NormalizerGuard16::new(values);
|
||||
let coefficients_chunks = normalizer_guard.normalized_chunks(window_size, &bounds_per_pixel);
|
||||
|
||||
let dst_rows = dst_image.iter_rows_mut();
|
||||
for (dst_row, coeffs_chunk) in dst_rows.zip(coefficients_chunks) {
|
||||
unsafe {
|
||||
vert_convolution_into_one_row_u8(&src_image, dst_row, coeffs_chunk, &normalizer_guard);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[target_feature(enable = "sse4.1")]
|
||||
pub(crate) unsafe fn vert_convolution_into_one_row_u8<T: Pixel<Component = u8>>(
|
||||
src_img: &TypedImageView<T>,
|
||||
dst_row: &mut [T],
|
||||
coeffs_chunk: CoefficientsI16Chunk,
|
||||
normalizer_guard: &NormalizerGuard16,
|
||||
) {
|
||||
let mut xx: usize = 0;
|
||||
let src_width = src_img.width().get() as usize * T::components_count();
|
||||
let y_start = coeffs_chunk.start;
|
||||
let coeffs = coeffs_chunk.values;
|
||||
let max_y = y_start + coeffs.len() as u32;
|
||||
let precision = normalizer_guard.precision();
|
||||
let dst_ptr_u8 = T::components_mut(dst_row).as_mut_ptr() as *mut u8;
|
||||
|
||||
let initial = _mm_set1_epi32(1 << (precision - 1));
|
||||
|
||||
// 32 components in two registers - 1 = 31
|
||||
while xx < src_width.saturating_sub(31) {
|
||||
let mut sss0 = initial;
|
||||
let mut sss1 = initial;
|
||||
let mut sss2 = initial;
|
||||
let mut sss3 = initial;
|
||||
let mut sss4 = initial;
|
||||
let mut sss5 = initial;
|
||||
let mut sss6 = initial;
|
||||
let mut sss7 = initial;
|
||||
|
||||
let mut y: u32 = 0;
|
||||
|
||||
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
|
||||
let components1 = T::components(s_row1);
|
||||
let components2 = T::components(s_row2);
|
||||
|
||||
// Load two coefficients at once
|
||||
let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
|
||||
|
||||
let source1 = simd_utils::loadu_si128(components1, xx); // top line
|
||||
let source2 = simd_utils::loadu_si128(components2, xx); // bottom line
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
|
||||
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
let source = _mm_unpackhi_epi8(source1, source2);
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss2 = _mm_add_epi32(sss2, _mm_madd_epi16(pix, mmk));
|
||||
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss3 = _mm_add_epi32(sss3, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
let source1 = simd_utils::loadu_si128(components1, xx + 16); // top line
|
||||
let source2 = simd_utils::loadu_si128(components2, xx + 16); // bottom line
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss4 = _mm_add_epi32(sss4, _mm_madd_epi16(pix, mmk));
|
||||
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss5 = _mm_add_epi32(sss5, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
let source = _mm_unpackhi_epi8(source1, source2);
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss6 = _mm_add_epi32(sss6, _mm_madd_epi16(pix, mmk));
|
||||
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss7 = _mm_add_epi32(sss7, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
y += 2;
|
||||
}
|
||||
|
||||
if let Some(&k) = coeffs.get(y as usize) {
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let components = T::components(s_row);
|
||||
let mmk = _mm_set1_epi32(k as i32);
|
||||
|
||||
let source1 = simd_utils::loadu_si128(components, xx); // top line
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, _mm_setzero_si128());
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
|
||||
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
let source = _mm_unpackhi_epi8(source1, _mm_setzero_si128());
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss2 = _mm_add_epi32(sss2, _mm_madd_epi16(pix, mmk));
|
||||
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss3 = _mm_add_epi32(sss3, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
let source1 = simd_utils::loadu_si128(components, xx + 16); // top line
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, _mm_setzero_si128());
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss4 = _mm_add_epi32(sss4, _mm_madd_epi16(pix, mmk));
|
||||
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss5 = _mm_add_epi32(sss5, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
let source = _mm_unpackhi_epi8(source1, _mm_setzero_si128());
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss6 = _mm_add_epi32(sss6, _mm_madd_epi16(pix, mmk));
|
||||
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss7 = _mm_add_epi32(sss7, _mm_madd_epi16(pix, mmk));
|
||||
}
|
||||
|
||||
macro_rules! call {
|
||||
($imm8:expr) => {{
|
||||
sss0 = _mm_srai_epi32::<$imm8>(sss0);
|
||||
sss1 = _mm_srai_epi32::<$imm8>(sss1);
|
||||
sss2 = _mm_srai_epi32::<$imm8>(sss2);
|
||||
sss3 = _mm_srai_epi32::<$imm8>(sss3);
|
||||
sss4 = _mm_srai_epi32::<$imm8>(sss4);
|
||||
sss5 = _mm_srai_epi32::<$imm8>(sss5);
|
||||
sss6 = _mm_srai_epi32::<$imm8>(sss6);
|
||||
sss7 = _mm_srai_epi32::<$imm8>(sss7);
|
||||
}};
|
||||
}
|
||||
constify_imm8!(precision, call);
|
||||
|
||||
sss0 = _mm_packs_epi32(sss0, sss1);
|
||||
sss2 = _mm_packs_epi32(sss2, sss3);
|
||||
sss0 = _mm_packus_epi16(sss0, sss2);
|
||||
let dst_ptr = dst_ptr_u8.add(xx) as *mut __m128i;
|
||||
_mm_storeu_si128(dst_ptr, sss0);
|
||||
sss4 = _mm_packs_epi32(sss4, sss5);
|
||||
sss6 = _mm_packs_epi32(sss6, sss7);
|
||||
sss4 = _mm_packus_epi16(sss4, sss6);
|
||||
let dst_ptr = dst_ptr_u8.add(xx + 16) as *mut __m128i;
|
||||
_mm_storeu_si128(dst_ptr, sss4);
|
||||
|
||||
xx += 32;
|
||||
}
|
||||
|
||||
while xx < src_width.saturating_sub(7) {
|
||||
let mut sss0 = initial; // left row
|
||||
let mut sss1 = initial; // right row
|
||||
let mut y: u32 = 0;
|
||||
|
||||
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
|
||||
let components1 = T::components(s_row1);
|
||||
let components2 = T::components(s_row2);
|
||||
// Load two coefficients at once
|
||||
let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
|
||||
|
||||
let source1 = simd_utils::loadl_epi64(components1, xx); // top line
|
||||
let source2 = simd_utils::loadl_epi64(components2, xx); // bottom line
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
|
||||
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
y += 2;
|
||||
}
|
||||
|
||||
if let Some(&k) = coeffs.get(y as usize) {
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let components = T::components(s_row);
|
||||
let mmk = _mm_set1_epi32(k as i32);
|
||||
|
||||
let source1 = simd_utils::loadl_epi64(components, xx); // top line
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, _mm_setzero_si128());
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss0 = _mm_add_epi32(sss0, _mm_madd_epi16(pix, mmk));
|
||||
let pix = _mm_unpackhi_epi8(source, _mm_setzero_si128());
|
||||
sss1 = _mm_add_epi32(sss1, _mm_madd_epi16(pix, mmk));
|
||||
}
|
||||
|
||||
macro_rules! call {
|
||||
($imm8:expr) => {{
|
||||
sss0 = _mm_srai_epi32::<$imm8>(sss0);
|
||||
sss1 = _mm_srai_epi32::<$imm8>(sss1);
|
||||
}};
|
||||
}
|
||||
constify_imm8!(precision, call);
|
||||
|
||||
sss0 = _mm_packs_epi32(sss0, sss1);
|
||||
sss0 = _mm_packus_epi16(sss0, sss0);
|
||||
let dst_ptr = dst_ptr_u8.add(xx) as *mut __m128i;
|
||||
_mm_storel_epi64(dst_ptr, sss0);
|
||||
|
||||
xx += 8;
|
||||
}
|
||||
|
||||
while xx < src_width.saturating_sub(3) {
|
||||
let mut sss = initial;
|
||||
let mut y: u32 = 0;
|
||||
|
||||
for (s_row1, s_row2) in src_img.iter_2_rows(y_start, max_y) {
|
||||
let components1 = T::components(s_row1);
|
||||
let components2 = T::components(s_row2);
|
||||
// Load two coefficients at once
|
||||
let mmk = simd_utils::ptr_i16_to_set1_epi32(coeffs, y as usize);
|
||||
|
||||
let source1 = simd_utils::mm_cvtsi32_si128_from_u8(components1, xx); // top line
|
||||
let source2 = simd_utils::mm_cvtsi32_si128_from_u8(components2, xx); // bottom line
|
||||
|
||||
let source = _mm_unpacklo_epi8(source1, source2);
|
||||
let pix = _mm_unpacklo_epi8(source, _mm_setzero_si128());
|
||||
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
|
||||
|
||||
y += 2;
|
||||
}
|
||||
|
||||
if let Some(&k) = coeffs.get(y as usize) {
|
||||
let s_row = src_img.get_row(y_start + y).unwrap();
|
||||
let components = T::components(s_row);
|
||||
let pix = simd_utils::mm_cvtepu8_epi32_from_u8(components, xx);
|
||||
let mmk = _mm_set1_epi32(k as i32);
|
||||
sss = _mm_add_epi32(sss, _mm_madd_epi16(pix, mmk));
|
||||
}
|
||||
|
||||
macro_rules! call {
|
||||
($imm8:expr) => {{
|
||||
sss = _mm_srai_epi32::<$imm8>(sss);
|
||||
}};
|
||||
}
|
||||
constify_imm8!(precision, call);
|
||||
|
||||
sss = _mm_packs_epi32(sss, sss);
|
||||
let dst_ptr = dst_ptr_u8.add(xx) as *mut i32;
|
||||
*dst_ptr = _mm_cvtsi128_si32(_mm_packus_epi16(sss, sss));
|
||||
|
||||
xx += 4;
|
||||
}
|
||||
|
||||
if xx < src_width {
|
||||
let dst_u8 = std::slice::from_raw_parts_mut(dst_ptr_u8.add(xx), src_width - xx);
|
||||
|
||||
for dst_pixel in dst_u8 {
|
||||
let mut ss0 = 1 << (precision - 1);
|
||||
for (dy, &k) in coeffs.iter().enumerate() {
|
||||
if let Some(src_row) = src_img.get_row(y_start + dy as u32) {
|
||||
let src_ptr = src_row.as_ptr() as *const u8;
|
||||
let src_component = *src_ptr.add(xx);
|
||||
ss0 += src_component as i32 * (k as i32);
|
||||
}
|
||||
}
|
||||
*dst_pixel = normalizer_guard.clip(ss0);
|
||||
xx += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -397,11 +397,6 @@ where
|
||||
self.crop_box
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(crate) fn get_pixel(&self, x: u32, y: u32) -> P {
|
||||
self.rows[y as usize][x as usize]
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub(crate) fn iter_4_rows<'s>(
|
||||
&'s self,
|
||||
|
||||
+53
-5
@@ -1,7 +1,9 @@
|
||||
//! Contains types of pixels.
|
||||
use std::mem::size_of;
|
||||
use std::slice;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[non_exhaustive]
|
||||
pub enum PixelType {
|
||||
U8x3,
|
||||
U8x4,
|
||||
@@ -21,7 +23,7 @@ impl PixelType {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns `true` is given buffer is aligned by the alignment of pixel.
|
||||
/// Returns `true` if given buffer is aligned by the alignment of pixel.
|
||||
pub(crate) fn is_aligned(&self, buffer: &[u8]) -> bool {
|
||||
match self {
|
||||
Self::U8x3 => unsafe { buffer.align_to::<U8x3>().0.is_empty() },
|
||||
@@ -39,8 +41,14 @@ pub trait Pixel
|
||||
where
|
||||
Self: Copy + Sized,
|
||||
{
|
||||
/// Type of pixel components
|
||||
type Component;
|
||||
|
||||
fn pixel_type() -> PixelType;
|
||||
|
||||
/// Count of pixel's components
|
||||
fn components_count() -> usize;
|
||||
|
||||
/// Size of pixel in bytes
|
||||
///
|
||||
/// Example:
|
||||
@@ -52,41 +60,81 @@ where
|
||||
fn size() -> usize {
|
||||
size_of::<Self>()
|
||||
}
|
||||
|
||||
/// Create slice of components of pixels from slice of pixels
|
||||
fn components(buf: &[Self]) -> &[Self::Component] {
|
||||
let size = buf.len() * Self::components_count();
|
||||
let components_ptr = buf.as_ptr() as *const Self::Component;
|
||||
unsafe { slice::from_raw_parts(components_ptr, size) }
|
||||
}
|
||||
|
||||
/// Create mutable slice of components of pixels from mutable slice of pixels
|
||||
fn components_mut(buf: &mut [Self]) -> &mut [Self::Component] {
|
||||
let size = buf.len() * Self::components_count();
|
||||
let components_ptr = buf.as_mut_ptr() as *mut Self::Component;
|
||||
unsafe { slice::from_raw_parts_mut(components_ptr, size) }
|
||||
}
|
||||
}
|
||||
|
||||
macro_rules! pixel_struct {
|
||||
($name:ident, $type:tt, $pixel_type:expr, $doc:expr) => {
|
||||
($name:ident, $type:tt, $comp_type:tt, $comp_count:expr, $pixel_type:expr, $doc:expr) => {
|
||||
#[doc = $doc]
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
#[repr(C)]
|
||||
pub struct $name(pub $type);
|
||||
|
||||
impl Pixel for $name {
|
||||
type Component = $comp_type;
|
||||
|
||||
fn pixel_type() -> PixelType {
|
||||
$pixel_type
|
||||
}
|
||||
|
||||
fn components_count() -> usize {
|
||||
$comp_count
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
pixel_struct!(U8, u8, PixelType::U8, "One byte per pixel");
|
||||
pixel_struct!(U8, u8, u8, 1, PixelType::U8, "One byte per pixel");
|
||||
pixel_struct!(
|
||||
U8x3,
|
||||
[u8; 3],
|
||||
u8,
|
||||
3,
|
||||
PixelType::U8x3,
|
||||
"Three bytes per pixel (e.g. RGB)"
|
||||
);
|
||||
pixel_struct!(
|
||||
U8x4,
|
||||
u32,
|
||||
u8,
|
||||
4,
|
||||
PixelType::U8x4,
|
||||
"Four bytes per pixel (RGBA, RGBx, CMYK and other)"
|
||||
);
|
||||
pixel_struct!(
|
||||
U16x3,
|
||||
[u16; 3],
|
||||
u16,
|
||||
3,
|
||||
PixelType::U16x3,
|
||||
"Three `u16` components per pixel (e.g. RGB)"
|
||||
);
|
||||
pixel_struct!(I32, i32, PixelType::I32, "One `i32` component per pixel");
|
||||
pixel_struct!(F32, f32, PixelType::F32, "One `f32` component per pixel");
|
||||
pixel_struct!(
|
||||
I32,
|
||||
i32,
|
||||
i32,
|
||||
1,
|
||||
PixelType::I32,
|
||||
"One `i32` component per pixel"
|
||||
);
|
||||
pixel_struct!(
|
||||
F32,
|
||||
f32,
|
||||
f32,
|
||||
1,
|
||||
PixelType::F32,
|
||||
"One `f32` component per pixel"
|
||||
);
|
||||
|
||||
+3
-33
@@ -1,7 +1,7 @@
|
||||
use std::arch::x86_64::*;
|
||||
use std::intrinsics::transmute;
|
||||
|
||||
use crate::pixels::{U8x3, U8x4, U8};
|
||||
use crate::pixels::{U8x3, U8x4};
|
||||
|
||||
#[inline(always)]
|
||||
pub unsafe fn loadu_si128<T>(buf: &[T], index: usize) -> __m128i {
|
||||
@@ -13,23 +13,11 @@ pub unsafe fn loadu_si256<T>(buf: &[T], index: usize) -> __m256i {
|
||||
_mm256_loadu_si256(buf.get_unchecked(index..).as_ptr() as *const __m256i)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub unsafe fn loadu_si256_raw<T>(buf: &[T], offset: usize) -> __m256i {
|
||||
let ptr = buf.as_ptr() as *const u8;
|
||||
_mm256_loadu_si256(ptr.add(offset) as *const __m256i)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub unsafe fn loadl_epi64<T>(buf: &[T], index: usize) -> __m128i {
|
||||
_mm_loadl_epi64(buf.get_unchecked(index..).as_ptr() as *const __m128i)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub unsafe fn loadl_epi64_raw<T>(buf: &[T], offset: usize) -> __m128i {
|
||||
let ptr = buf.as_ptr() as *const u8;
|
||||
_mm_loadl_epi64(ptr.add(offset) as *const __m128i)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub unsafe fn mm_cvtepu8_epi32(buf: &[U8x4], index: usize) -> __m128i {
|
||||
let v: i32 = transmute(buf.get_unchecked(index).0);
|
||||
@@ -44,35 +32,17 @@ pub unsafe fn mm_cvtepu8_epi32_u8x3(buf: &[U8x3], index: usize) -> __m128i {
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub unsafe fn mm_cvtepu8_epi32_from_u8(buf: &[U8], index: usize) -> __m128i {
|
||||
pub unsafe fn mm_cvtepu8_epi32_from_u8(buf: &[u8], index: usize) -> __m128i {
|
||||
let ptr = buf.get_unchecked(index..).as_ptr() as *const i32;
|
||||
_mm_cvtepu8_epi32(_mm_cvtsi32_si128(*ptr))
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub unsafe fn mm_cvtepu8_epi32_from_raw<T>(buf: &[T], offset: usize) -> __m128i {
|
||||
let ptr = (buf.as_ptr() as *const u8).add(offset) as *const i32;
|
||||
_mm_cvtepu8_epi32(_mm_cvtsi32_si128(*ptr))
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub unsafe fn mm_cvtsi32_si128_from_u32(buf: &[U8x4], index: usize) -> __m128i {
|
||||
let v: i32 = transmute(*buf.get_unchecked(index));
|
||||
_mm_cvtsi32_si128(v)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub unsafe fn mm_cvtsi32_si128_from_u8(buf: &[U8], index: usize) -> __m128i {
|
||||
pub unsafe fn mm_cvtsi32_si128_from_u8(buf: &[u8], index: usize) -> __m128i {
|
||||
let ptr = buf.get_unchecked(index..).as_ptr() as *const i32;
|
||||
_mm_cvtsi32_si128(*ptr)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub unsafe fn mm_cvtsi32_si128_from_raw<T>(buf: &[T], offset: usize) -> __m128i {
|
||||
let ptr = (buf.as_ptr() as *const u8).add(offset) as *const i32;
|
||||
_mm_cvtsi32_si128(*ptr)
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub unsafe fn ptr_i16_to_set1_epi32(buf: &[i16], index: usize) -> __m128i {
|
||||
_mm_set1_epi32(*(buf.get_unchecked(index..).as_ptr() as *const i32))
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
use fast_image_resize as fr;
|
||||
use std::num::NonZeroU32;
|
||||
|
||||
#[test]
|
||||
fn create_image_from_small_buffer() {
|
||||
let width = NonZeroU32::new(64).unwrap();
|
||||
let height = NonZeroU32::new(32).unwrap();
|
||||
let mut buffer = vec![0; 64 * 30];
|
||||
|
||||
let res = fr::Image::from_slice_u8(width, height, &mut buffer, fr::PixelType::U8);
|
||||
assert_eq!(res.unwrap_err(), fr::ImageBufferError::InvalidBufferSize);
|
||||
|
||||
let res = fr::Image::from_vec_u8(width, height, buffer, fr::PixelType::U8);
|
||||
assert_eq!(res.unwrap_err(), fr::ImageBufferError::InvalidBufferSize);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_image_view_from_small_buffer() {
|
||||
let width = NonZeroU32::new(64).unwrap();
|
||||
let height = NonZeroU32::new(32).unwrap();
|
||||
let mut buffer = vec![0; 64 * 30];
|
||||
|
||||
let res = fr::ImageViewMut::from_buffer(width, height, &mut buffer, fr::PixelType::U8);
|
||||
assert_eq!(res.unwrap_err(), fr::ImageBufferError::InvalidBufferSize);
|
||||
|
||||
let res = fr::ImageView::from_buffer(width, height, &buffer, fr::PixelType::U8);
|
||||
assert_eq!(res.unwrap_err(), fr::ImageBufferError::InvalidBufferSize);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_image_from_big_buffer() {
|
||||
let width = NonZeroU32::new(64).unwrap();
|
||||
let height = NonZeroU32::new(32).unwrap();
|
||||
let mut buffer = vec![0; 65 * 32];
|
||||
|
||||
let res = fr::Image::from_slice_u8(width, height, &mut buffer, fr::PixelType::U8);
|
||||
assert!(res.is_ok());
|
||||
|
||||
let res = fr::Image::from_vec_u8(width, height, buffer, fr::PixelType::U8);
|
||||
assert!(res.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn create_image_view_from_big_buffer() {
|
||||
let width = NonZeroU32::new(64).unwrap();
|
||||
let height = NonZeroU32::new(32).unwrap();
|
||||
let mut buffer = vec![0; 65 * 32];
|
||||
|
||||
let res = fr::ImageViewMut::from_buffer(width, height, &mut buffer, fr::PixelType::U8);
|
||||
assert!(res.is_ok());
|
||||
|
||||
let res = fr::ImageView::from_buffer(width, height, &buffer, fr::PixelType::U8);
|
||||
assert!(res.is_ok());
|
||||
}
|
||||
+14
-14
@@ -122,7 +122,7 @@ fn upscale_test<P: PixelExt>(resize_alg: ResizeAlg, cpu_extensions: CpuExtension
|
||||
fn downscale_u8() {
|
||||
type P = U8;
|
||||
let buffer = downscale_test::<P>(ResizeAlg::Nearest, CpuExtensions::None);
|
||||
assert_eq!(utils::image_checksum::<1>(&buffer), [2920317]);
|
||||
assert_eq!(utils::image_checksum::<1>(&buffer), [2920348]);
|
||||
|
||||
let mut cpu_extensions_vec = vec![CpuExtensions::None];
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
@@ -132,7 +132,7 @@ fn downscale_u8() {
|
||||
for cpu_extensions in cpu_extensions_vec {
|
||||
let buffer =
|
||||
downscale_test::<P>(ResizeAlg::Convolution(FilterType::Lanczos3), cpu_extensions);
|
||||
assert_eq!(utils::image_checksum::<1>(&buffer), [2923520]);
|
||||
assert_eq!(utils::image_checksum::<1>(&buffer), [2923557]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -140,7 +140,7 @@ fn downscale_u8() {
|
||||
fn upscale_u8() {
|
||||
type P = U8;
|
||||
let buffer = upscale_test::<P>(ResizeAlg::Nearest, CpuExtensions::None);
|
||||
assert_eq!(utils::image_checksum::<1>(&buffer), [1148750539]);
|
||||
assert_eq!(utils::image_checksum::<1>(&buffer), [1148754010]);
|
||||
|
||||
let mut cpu_extensions_vec = vec![CpuExtensions::None];
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
@@ -150,7 +150,7 @@ fn upscale_u8() {
|
||||
for cpu_extensions in cpu_extensions_vec {
|
||||
let buffer =
|
||||
upscale_test::<P>(ResizeAlg::Convolution(FilterType::Lanczos3), cpu_extensions);
|
||||
assert_eq!(utils::image_checksum::<1>(&buffer), [1148808058]);
|
||||
assert_eq!(utils::image_checksum::<1>(&buffer), [1148811406]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -214,11 +214,11 @@ fn downscale_u16x3() {
|
||||
);
|
||||
|
||||
let mut cpu_extensions_vec = vec![CpuExtensions::None];
|
||||
// #[cfg(target_arch = "x86_64")]
|
||||
// {
|
||||
// cpu_extensions_vec.push(CpuExtensions::Sse4_1);
|
||||
// cpu_extensions_vec.push(CpuExtensions::Avx2);
|
||||
// }
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
{
|
||||
cpu_extensions_vec.push(CpuExtensions::Sse4_1);
|
||||
cpu_extensions_vec.push(CpuExtensions::Avx2);
|
||||
}
|
||||
for cpu_extensions in cpu_extensions_vec {
|
||||
let buffer =
|
||||
downscale_test::<P>(ResizeAlg::Convolution(FilterType::Lanczos3), cpu_extensions);
|
||||
@@ -239,11 +239,11 @@ fn upscale_u16x3() {
|
||||
);
|
||||
|
||||
let mut cpu_extensions_vec = vec![CpuExtensions::None];
|
||||
// #[cfg(target_arch = "x86_64")]
|
||||
// {
|
||||
// cpu_extensions_vec.push(CpuExtensions::Sse4_1);
|
||||
// cpu_extensions_vec.push(CpuExtensions::Avx2);
|
||||
// }
|
||||
#[cfg(target_arch = "x86_64")]
|
||||
{
|
||||
cpu_extensions_vec.push(CpuExtensions::Sse4_1);
|
||||
cpu_extensions_vec.push(CpuExtensions::Avx2);
|
||||
}
|
||||
for cpu_extensions in cpu_extensions_vec {
|
||||
let buffer =
|
||||
upscale_test::<P>(ResizeAlg::Convolution(FilterType::Lanczos3), cpu_extensions);
|
||||
|
||||
+2
-1
@@ -1,7 +1,7 @@
|
||||
use std::num::NonZeroU32;
|
||||
|
||||
use image::io::Reader as ImageReader;
|
||||
use image::{ColorType, DynamicImage, GenericImageView};
|
||||
use image::{ColorType, DynamicImage};
|
||||
|
||||
use fast_image_resize::pixels::*;
|
||||
use fast_image_resize::{CpuExtensions, Image, PixelType};
|
||||
@@ -32,6 +32,7 @@ pub trait PixelExt: Pixel {
|
||||
PixelType::U16x3 => "u16x3",
|
||||
PixelType::I32 => "i32",
|
||||
PixelType::F32 => "f32",
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user