diff options
| author | jiyong.min <jiyong.min@samsung.com> | 2022-04-06 10:56:55 +0900 |
|---|---|---|
| committer | jiyong.min <jiyong.min@samsung.com> | 2022-04-06 11:13:36 +0900 |
| commit | b357dac3dd94ebb81105eb2e375b6cd64232a7a8 (patch) | |
| tree | 422cc16e4d6bd479dab551f2b5b59f3988f40b59 | |
| parent | b8aee4491f6498d0cc17d18c80f6b3505e2a62ef (diff) | |
| download | libjxl-b357dac3dd94ebb81105eb2e375b6cd64232a7a8.tar.gz libjxl-b357dac3dd94ebb81105eb2e375b6cd64232a7a8.tar.bz2 libjxl-b357dac3dd94ebb81105eb2e375b6cd64232a7a8.zip | |
add libhwy submodule for checking hardware accel.
Change-Id: I696de47a8f06ce3801a1f7c1c2452472f4dfac7e
82 files changed, 52317 insertions, 0 deletions
diff --git a/third_party/highway/BUILD b/third_party/highway/BUILD new file mode 100644 index 0000000..0bd3acd --- /dev/null +++ b/third_party/highway/BUILD @@ -0,0 +1,295 @@ +load("@bazel_skylib//lib:selects.bzl", "selects") +load("@rules_cc//cc:defs.bzl", "cc_test") + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) + +exports_files(["LICENSE"]) + +# Detect compiler: +config_setting( + name = "compiler_clang", + flag_values = {"@bazel_tools//tools/cpp:compiler": "clang"}, +) + +config_setting( + name = "compiler_msvc", + flag_values = {"@bazel_tools//tools/cpp:compiler": "msvc"}, +) + +# See https://github.com/bazelbuild/bazel/issues/12707 +config_setting( + name = "compiler_gcc_bug", + flag_values = { + "@bazel_tools//tools/cpp:compiler": "compiler", + }, +) + +config_setting( + name = "compiler_gcc_actual", + flag_values = { + "@bazel_tools//tools/cpp:compiler": "gcc", + }, +) + +selects.config_setting_group( + name = "compiler_gcc", + match_any = [ + ":compiler_gcc_bug", + ":compiler_gcc_actual", + ], +) + +config_setting( + name = "emulate_sve", + values = { + "copt": "-DHWY_EMULATE_SVE", + }, +) + +# Additional warnings for Clang OR GCC (skip for MSVC) +CLANG_GCC_COPTS = [ + "-Wunused-parameter", + "-Wunused-variable", + "-Wextra-semi", + "-Wunreachable-code", +] + +# Additional warnings only supported by Clang +CLANG_ONLY_COPTS = [ + "-Wfloat-overflow-conversion", + "-Wfloat-zero-conversion", + "-Wfor-loop-analysis", + "-Wgnu-redeclared-enum", + "-Winfinite-recursion", + "-Wliteral-conversion", + "-Wno-c++98-compat", + "-Wno-unused-command-line-argument", + "-Wprivate-header", + "-Wself-assign", + "-Wstring-conversion", + "-Wtautological-overlap-compare", + "-Wthread-safety-analysis", + "-Wundefined-func-template", + "-Wunused-comparison", +] + +COPTS = select({ + ":compiler_msvc": [], + ":compiler_gcc": CLANG_GCC_COPTS, + # Default to clang because compiler detection only works in Bazel + "//conditions:default": CLANG_GCC_COPTS + CLANG_ONLY_COPTS, +}) + select({ + "@platforms//cpu:riscv64": [ + "-march=rv64gcv0p10", + "-menable-experimental-extensions", + ], + "//conditions:default": [ + ], +}) + +# Unused on Bazel builds, where this is not defined/known; Copybara replaces +# usages with an empty list. +COMPAT = [ + "//buildenv/target:non_prod", # includes mobile/vendor. +] + +# WARNING: changing flags such as HWY_DISABLED_TARGETS may break users without +# failing integration tests, if the machine running tests does not support the +# newly enabled instruction set, or the failure is only caught by sanitizers +# which do not run in CI. + +cc_library( + name = "hwy", + srcs = [ + "hwy/aligned_allocator.cc", + "hwy/targets.cc", + ], + # Normal headers with include guards + hdrs = [ + "hwy/aligned_allocator.h", + "hwy/base.h", + "hwy/cache_control.h", + "hwy/detect_compiler_arch.h", # private + "hwy/detect_targets.h", # private + "hwy/targets.h", + ], + compatible_with = [], + copts = COPTS, + textual_hdrs = [ + "hwy/highway.h", # public + "hwy/foreach_target.h", # public + "hwy/ops/arm_neon-inl.h", + "hwy/ops/arm_sve-inl.h", + "hwy/ops/generic_ops-inl.h", + "hwy/ops/rvv-inl.h", + "hwy/ops/scalar-inl.h", + "hwy/ops/set_macros-inl.h", + "hwy/ops/shared-inl.h", + "hwy/ops/wasm_128-inl.h", + "hwy/ops/x86_128-inl.h", + "hwy/ops/x86_256-inl.h", + "hwy/ops/x86_512-inl.h", + ], + deps = select({ + ":emulate_sve": ["//third_party/farm_sve"], + "//conditions:default": [], + }), +) + +cc_library( + name = "dot", + compatible_with = [], + textual_hdrs = [ + "hwy/contrib/dot/dot-inl.h", + ], + deps = [":hwy"], +) + +cc_library( + name = "image", + srcs = [ + "hwy/contrib/image/image.cc", + ], + hdrs = [ + "hwy/contrib/image/image.h", + ], + compatible_with = [], + deps = [":hwy"], +) + +cc_library( + name = "math", + compatible_with = [], + textual_hdrs = [ + "hwy/contrib/math/math-inl.h", + ], + deps = [":hwy"], +) + +cc_library( + name = "sort", + compatible_with = [], + textual_hdrs = [ + "hwy/contrib/sort/sort-inl.h", + ], + deps = [":hwy"], +) + +# Everything required for tests that use Highway. +cc_library( + name = "hwy_test_util", + srcs = ["hwy/tests/test_util.cc"], + hdrs = ["hwy/tests/test_util.h"], + textual_hdrs = [ + "hwy/tests/test_util-inl.h", + "hwy/tests/hwy_gtest.h", + ], + # Must not depend on a gtest variant, which can conflict with the + # GUNIT_INTERNAL_BUILD_MODE defined by the test. + deps = [":hwy"], +) + +cc_library( + name = "nanobenchmark", + srcs = ["hwy/nanobenchmark.cc"], + hdrs = ["hwy/nanobenchmark.h"], + deps = [":hwy"], +) + +cc_binary( + name = "benchmark", + srcs = ["hwy/examples/benchmark.cc"], + deps = [ + ":hwy", + ":nanobenchmark", + ], +) + +cc_library( + name = "skeleton", + srcs = ["hwy/examples/skeleton.cc"], + hdrs = ["hwy/examples/skeleton.h"], + textual_hdrs = ["hwy/examples/skeleton-inl.h"], + deps = [":hwy"], +) + +cc_binary( + name = "list_targets", + srcs = ["hwy/tests/list_targets.cc"], + deps = [":hwy"], +) + +# path, name +HWY_TESTS = [ + ("hwy/contrib/dot/", "dot_test"), + ("hwy/contrib/image/", "image_test"), + ("hwy/contrib/math/", "math_test"), + ("hwy/contrib/sort/", "sort_test"), + ("hwy/examples/", "skeleton_test"), + ("hwy/", "nanobenchmark_test"), + ("hwy/", "aligned_allocator_test"), + ("hwy/", "base_test"), + ("hwy/", "highway_test"), + ("hwy/", "targets_test"), + ("hwy/tests/", "arithmetic_test"), + ("hwy/tests/", "blockwise_test"), + ("hwy/tests/", "combine_test"), + ("hwy/tests/", "compare_test"), + ("hwy/tests/", "convert_test"), + ("hwy/tests/", "crypto_test"), + ("hwy/tests/", "logical_test"), + ("hwy/tests/", "mask_test"), + ("hwy/tests/", "memory_test"), + ("hwy/tests/", "swizzle_test"), + ("hwy/tests/", "test_util_test"), +] + +[ + [ + cc_test( + name = test, + size = "medium", + timeout = "long", # default moderate is not enough for math_test + srcs = [ + subdir + test + ".cc", + ], + copts = COPTS + [ + # gTest triggers this warning (which is enabled by the + # extra-semi in COPTS), so we need to disable it here, + # but it's still enabled for :hwy. + "-Wno-c++98-compat-extra-semi", + ], + features = select({ + "@platforms//cpu:riscv64": ["fully_static_link"], + "//conditions:default": [], + }), + linkstatic = select({ + "@platforms//cpu:riscv64": True, + "//conditions:default": False, + }), + local_defines = ["HWY_IS_TEST"], + # for test_suite. + tags = ["hwy_ops_test"], + deps = [ + ":dot", + ":hwy", + ":hwy_test_util", + ":image", + ":math", + ":nanobenchmark", + ":skeleton", + ":sort", + "@com_google_googletest//:gtest_main", + ], + ), + ] + for subdir, test in HWY_TESTS +] + +# For manually building the tests we define here (:all does not work in --config=msvc) +test_suite( + name = "hwy_ops_tests", + tags = ["hwy_ops_test"], +) diff --git a/third_party/highway/CMakeLists.txt b/third_party/highway/CMakeLists.txt new file mode 100644 index 0000000..b1d360f --- /dev/null +++ b/third_party/highway/CMakeLists.txt @@ -0,0 +1,405 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.10) + +# Set PIE flags for POSITION_INDEPENDENT_CODE targets, added in 3.14. +if(POLICY CMP0083) + cmake_policy(SET CMP0083 NEW) +endif() + +project(hwy VERSION 0.15.0) # Keep in sync with highway.h version + +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_EXTENSIONS OFF) +set(CMAKE_CXX_STANDARD_REQUIRED YES) + +# Enabled PIE binaries by default if supported. +include(CheckPIESupported OPTIONAL RESULT_VARIABLE CHECK_PIE_SUPPORTED) +if(CHECK_PIE_SUPPORTED) + check_pie_supported(LANGUAGES CXX) + if(CMAKE_CXX_LINK_PIE_SUPPORTED) + set(CMAKE_POSITION_INDEPENDENT_CODE TRUE) + endif() +endif() + +include(GNUInstallDirs) + +if (NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE RelWithDebInfo) +endif() + +set(HWY_CMAKE_ARM7 OFF CACHE BOOL "Set copts for ARMv7 with NEON?") + +# Unconditionally adding -Werror risks breaking the build when new warnings +# arise due to compiler/platform changes. Enable this in CI/tests. +set(HWY_WARNINGS_ARE_ERRORS OFF CACHE BOOL "Add -Werror flag?") + +set(HWY_EXAMPLES_TESTS_INSTALL ON CACHE BOOL "Build examples, tests, install?") + +include(CheckCXXSourceCompiles) +check_cxx_source_compiles( + "int main() { + #if !defined(__EMSCRIPTEN__) + static_assert(false, \"__EMSCRIPTEN__ is not defined\"); + #endif + return 0; + }" + HWY_EMSCRIPTEN +) + +set(HWY_CONTRIB_SOURCES + hwy/contrib/dot/dot-inl.h + hwy/contrib/image/image.cc + hwy/contrib/image/image.h + hwy/contrib/math/math-inl.h + hwy/contrib/sort/sort-inl.h +) + +set(HWY_SOURCES + hwy/aligned_allocator.cc + hwy/aligned_allocator.h + hwy/base.h + hwy/cache_control.h + hwy/detect_compiler_arch.h # private + hwy/detect_targets.h # private + hwy/foreach_target.h + hwy/highway.h + hwy/nanobenchmark.cc + hwy/nanobenchmark.h + hwy/ops/arm_neon-inl.h + hwy/ops/arm_sve-inl.h + hwy/ops/generic_ops-inl.h + hwy/ops/scalar-inl.h + hwy/ops/set_macros-inl.h + hwy/ops/shared-inl.h + hwy/ops/wasm_128-inl.h + hwy/ops/x86_128-inl.h + hwy/ops/x86_256-inl.h + hwy/ops/x86_512-inl.h + hwy/targets.cc + hwy/targets.h +) + +set(HWY_TEST_SOURCES + hwy/tests/hwy_gtest.h + hwy/tests/test_util-inl.h + hwy/tests/test_util.cc + hwy/tests/test_util.h +) + +if (MSVC) + # TODO(janwas): add flags +else() + set(HWY_FLAGS + # Avoid changing binaries based on the current time and date. + -Wno-builtin-macro-redefined + -D__DATE__="redacted" + -D__TIMESTAMP__="redacted" + -D__TIME__="redacted" + + # Optimizations + -fmerge-all-constants + + # Warnings + -Wall + -Wextra + # These are not included in Wall nor Wextra: + -Wconversion + -Wsign-conversion + -Wvla + -Wnon-virtual-dtor + ) + + #if (__TIZEN__) + list(APPEND HWY_FLAGS + -DHWY_COMPILE_ONLY_SCALAR + ) + #endif() + if(${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") + list(APPEND HWY_FLAGS + -Wfloat-overflow-conversion + -Wfloat-zero-conversion + -Wfor-loop-analysis + -Wgnu-redeclared-enum + -Winfinite-recursion + -Wself-assign + -Wstring-conversion + -Wtautological-overlap-compare + -Wthread-safety-analysis + -Wundefined-func-template + + -fno-cxx-exceptions + -fno-slp-vectorize + -fno-vectorize + + # Use color in messages + -fdiagnostics-show-option -fcolor-diagnostics + ) + if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 6.0) + list(APPEND HWY_FLAGS -Wc++2a-extensions) + endif() + endif() + + if (WIN32) + if(${CMAKE_CXX_COMPILER_ID} MATCHES "Clang") + list(APPEND HWY_FLAGS + -Wno-global-constructors + -Wno-language-extension-token + -Wno-used-but-marked-unused + -Wno-shadow-field-in-constructor + -Wno-unused-member-function + -Wno-unused-template + -Wno-c++98-compat-pedantic + -Wno-used-but-marked-unused + -Wno-zero-as-null-pointer-constant + ) + endif() + + list(APPEND HWY_FLAGS + -Wno-cast-align + -Wno-double-promotion + -Wno-float-equal + -Wno-format-nonliteral + -Wno-shadow + -Wno-sign-conversion + ) + else() + list(APPEND HWY_FLAGS + -fmath-errno + -fno-exceptions + ) + endif() # WIN32 + + if (HWY_CMAKE_ARM7) + list(APPEND HWY_FLAGS + -march=armv7-a + -mfpu=neon-vfpv4 + -mfloat-abi=hard # must match the toolchain specified as CXX= + -mfp16-format=ieee # required for vcvt_f32_f16 + ) + endif() # HWY_CMAKE_ARM7 + + if (HWY_WARNINGS_ARE_ERRORS) + list(APPEND HWY_FLAGS -Werror) + endif() + +endif() # !MSVC + +add_library(hwy STATIC ${HWY_SOURCES}) +target_compile_options(hwy PRIVATE ${HWY_FLAGS}) +set_property(TARGET hwy PROPERTY POSITION_INDEPENDENT_CODE ON) +target_include_directories(hwy PUBLIC ${CMAKE_CURRENT_LIST_DIR}) + +add_library(hwy_contrib STATIC ${HWY_CONTRIB_SOURCES}) +target_compile_options(hwy_contrib PRIVATE ${HWY_FLAGS}) +set_property(TARGET hwy_contrib PROPERTY POSITION_INDEPENDENT_CODE ON) +target_include_directories(hwy_contrib PUBLIC ${CMAKE_CURRENT_LIST_DIR}) + +add_library(hwy_test STATIC ${HWY_TEST_SOURCES}) +target_compile_options(hwy_test PRIVATE ${HWY_FLAGS}) +set_property(TARGET hwy_test PROPERTY POSITION_INDEPENDENT_CODE ON) +target_include_directories(hwy_test PUBLIC ${CMAKE_CURRENT_LIST_DIR}) + +# -------------------------------------------------------- hwy_list_targets +# Generate a tool to print the compiled-in targets as defined by the current +# flags. This tool will print to stderr at build time, after building hwy. +add_executable(hwy_list_targets hwy/tests/list_targets.cc) +target_compile_options(hwy_list_targets PRIVATE ${HWY_FLAGS}) +target_link_libraries(hwy_list_targets hwy) +target_include_directories(hwy_list_targets PRIVATE + $<TARGET_PROPERTY:hwy,INCLUDE_DIRECTORIES>) +# TARGET_FILE always returns the path to executable +# Naked target also not always could be run (due to the lack of '.\' prefix) +# Thus effective command to run should contain the full path +# and emulator prefix (if any). +add_custom_command(TARGET hwy_list_targets POST_BUILD + COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:hwy_list_targets> || (exit 0)) + +# -------------------------------------------------------- +# Allow skipping the following sections for projects that do not need them: +# tests, examples, benchmarks and installation. +if (HWY_EXAMPLES_TESTS_INSTALL) + +# -------------------------------------------------------- install library +install(TARGETS hwy + DESTINATION "${CMAKE_INSTALL_LIBDIR}") +# Install all the headers keeping the relative path to the current directory +# when installing them. +foreach (source ${HWY_SOURCES}) + if ("${source}" MATCHES "\.h$") + get_filename_component(dirname "${source}" DIRECTORY) + install(FILES "${source}" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/${dirname}") + endif() +endforeach() + +install(TARGETS hwy_contrib + DESTINATION "${CMAKE_INSTALL_LIBDIR}") +# Install all the headers keeping the relative path to the current directory +# when installing them. +foreach (source ${HWY_CONTRIB_SOURCES}) + if ("${source}" MATCHES "\.h$") + get_filename_component(dirname "${source}" DIRECTORY) + install(FILES "${source}" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/${dirname}") + endif() +endforeach() + +install(TARGETS hwy_test + DESTINATION "${CMAKE_INSTALL_LIBDIR}") +# Install all the headers keeping the relative path to the current directory +# when installing them. +foreach (source ${HWY_TEST_SOURCES}) + if ("${source}" MATCHES "\.h$") + get_filename_component(dirname "${source}" DIRECTORY) + install(FILES "${source}" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/${dirname}") + endif() +endforeach() + +# Add a pkg-config file for libhwy and the contrib/test libraries. +set(HWY_LIBRARY_VERSION "${CMAKE_PROJECT_VERSION}") +if (NOT __TIZEN__) +foreach (pc libhwy.pc libhwy-contrib.pc libhwy-test.pc) + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/${pc}.in" "${pc}" @ONLY) + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/${pc}" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") +endforeach() +endif() + +# -------------------------------------------------------- Examples + +# Avoids mismatch between GTest's static CRT and our dynamic. +set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + +# Programming exercise with integrated benchmark +add_executable(hwy_benchmark hwy/examples/benchmark.cc) +target_sources(hwy_benchmark PRIVATE + hwy/nanobenchmark.cc + hwy/nanobenchmark.h) +# Try adding either -DHWY_COMPILE_ONLY_SCALAR or -DHWY_COMPILE_ONLY_STATIC to +# observe the difference in targets printed. +target_compile_options(hwy_benchmark PRIVATE ${HWY_FLAGS}) +target_link_libraries(hwy_benchmark hwy) +set_target_properties(hwy_benchmark + PROPERTIES RUNTIME_OUTPUT_DIRECTORY "examples/") + +# -------------------------------------------------------- Tests + +include(CTest) + +if(BUILD_TESTING) +enable_testing() +include(GoogleTest) + +set(HWY_SYSTEM_GTEST OFF CACHE BOOL "Use pre-installed googletest?") +if(HWY_SYSTEM_GTEST) +find_package(GTest REQUIRED) +else() +# Download and unpack googletest at configure time +configure_file(CMakeLists.txt.in googletest-download/CMakeLists.txt) +execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" . + RESULT_VARIABLE result + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/googletest-download ) +if(result) + message(FATAL_ERROR "CMake step for googletest failed: ${result}") +endif() +execute_process(COMMAND ${CMAKE_COMMAND} --build . + RESULT_VARIABLE result + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/googletest-download ) +if(result) + message(FATAL_ERROR "Build step for googletest failed: ${result}") +endif() + +# Prevent overriding the parent project's compiler/linker +# settings on Windows +set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + +# Add googletest directly to our build. This defines +# the gtest and gtest_main targets. +add_subdirectory(${CMAKE_CURRENT_BINARY_DIR}/googletest-src + ${CMAKE_CURRENT_BINARY_DIR}/googletest-build + EXCLUDE_FROM_ALL) + +# The gtest/gtest_main targets carry header search path +# dependencies automatically when using CMake 2.8.11 or +# later. Otherwise we have to add them here ourselves. +if (CMAKE_VERSION VERSION_LESS 2.8.11) + include_directories("${gtest_SOURCE_DIR}/include") +endif() +endif() # HWY_SYSTEM_GTEST + +set(HWY_TEST_FILES + hwy/contrib/dot/dot_test.cc + hwy/contrib/image/image_test.cc + # Disabled due to SIGILL in clang7 debug build during gtest discovery phase, + # not reproducible locally. Still tested via bazel build. + # hwy/contrib/math/math_test.cc + hwy/contrib/sort/sort_test.cc + hwy/aligned_allocator_test.cc + hwy/base_test.cc + hwy/highway_test.cc + hwy/targets_test.cc + hwy/examples/skeleton_test.cc + hwy/tests/arithmetic_test.cc + hwy/tests/blockwise_test.cc + hwy/tests/combine_test.cc + hwy/tests/compare_test.cc + hwy/tests/convert_test.cc + hwy/tests/crypto_test.cc + hwy/tests/logical_test.cc + hwy/tests/mask_test.cc + hwy/tests/memory_test.cc + hwy/tests/swizzle_test.cc + hwy/tests/test_util_test.cc +) + +file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tests) +foreach (TESTFILE IN LISTS HWY_TEST_FILES) + # The TESTNAME is the name without the extension or directory. + get_filename_component(TESTNAME ${TESTFILE} NAME_WE) + add_executable(${TESTNAME} ${TESTFILE}) + target_compile_options(${TESTNAME} PRIVATE ${HWY_FLAGS}) + # Test all targets, not just the best/baseline. This changes the default + # policy to all-attainable; note that setting -DHWY_COMPILE_* directly can + # cause compile errors because only one may be set, and other CMakeLists.txt + # that include us may set them. + target_compile_options(${TESTNAME} PRIVATE -DHWY_IS_TEST=1) + + if(HWY_SYSTEM_GTEST) + target_link_libraries(${TESTNAME} hwy hwy_contrib hwy_test GTest::GTest GTest::Main) + else() + target_link_libraries(${TESTNAME} hwy hwy_contrib hwy_test gtest gtest_main) + endif() + # Output test targets in the test directory. + set_target_properties(${TESTNAME} PROPERTIES PREFIX "tests/") + + if (HWY_EMSCRIPTEN) + set_target_properties(${TESTNAME} PROPERTIES LINK_FLAGS "-s SINGLE_FILE=1") + endif() + + if(${CMAKE_VERSION} VERSION_LESS "3.10.3") + gtest_discover_tests(${TESTNAME} TIMEOUT 60) + else () + gtest_discover_tests(${TESTNAME} DISCOVERY_TIMEOUT 60) + endif () +endforeach () + +# The skeleton test uses the skeleton library code. +target_sources(skeleton_test PRIVATE hwy/examples/skeleton.cc) + +endif() # BUILD_TESTING + +endif() # HWY_EXAMPLES_TESTS_INSTALL diff --git a/third_party/highway/CMakeLists.txt.in b/third_party/highway/CMakeLists.txt.in new file mode 100644 index 0000000..df40170 --- /dev/null +++ b/third_party/highway/CMakeLists.txt.in @@ -0,0 +1,15 @@ +cmake_minimum_required(VERSION 2.8.12) + +project(googletest-download NONE) + +include(ExternalProject) +ExternalProject_Add(googletest + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG master + SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/googletest-src" + BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/googletest-build" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +)
\ No newline at end of file diff --git a/third_party/highway/CONTRIBUTING b/third_party/highway/CONTRIBUTING new file mode 100644 index 0000000..8b7d4d2 --- /dev/null +++ b/third_party/highway/CONTRIBUTING @@ -0,0 +1,33 @@ +# How to Contribute + +We'd love to accept your patches and contributions to this project. There are +just a few small guidelines you need to follow. + +## Contributor License Agreement + +Contributions to this project must be accompanied by a Contributor License +Agreement. You (or your employer) retain the copyright to your contribution; +this simply gives us permission to use and redistribute your contributions as +part of the project. Head over to <https://cla.developers.google.com/> to see +your current agreements on file or to sign a new one. + +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again. + +## Code reviews + +All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult +[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more +information on using pull requests. + +## Testing + +This repository is used by JPEG XL, so major API changes will require +coordination. Please get in touch with us beforehand, e.g. by raising an issue. + +## Community Guidelines + +This project follows +[Google's Open Source Community Guidelines](https://opensource.google.com/conduct/). diff --git a/third_party/highway/LICENSE b/third_party/highway/LICENSE new file mode 100644 index 0000000..f49a4e1 --- /dev/null +++ b/third_party/highway/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License.
\ No newline at end of file diff --git a/third_party/highway/README.md b/third_party/highway/README.md new file mode 100644 index 0000000..4747e85 --- /dev/null +++ b/third_party/highway/README.md @@ -0,0 +1,192 @@ +# Efficient and performance-portable SIMD + +Highway is a C++ library for SIMD (Single Instruction, Multiple Data), i.e. +applying the same operation to multiple 'lanes' using a single CPU instruction. + +## Why Highway? + +- more portable (same source code) than platform-specific intrinsics, +- works on a wider range of compilers than compiler-specific vector extensions, +- more dependable than autovectorization, +- easier to write/maintain than assembly language, +- supports **runtime dispatch**, +- supports **variable-length vector** architectures. + +## Current status + +Supported targets: scalar, S-SSE3, SSE4, AVX2, AVX-512, AVX3_DL (~Icelake, +requires opt-in by defining `HWY_WANT_AVX3_DL`), NEON (ARMv7 and v8), SVE, +WASM SIMD. + +SVE is tested using farm_sve (see acknowledgments). SVE2 is implemented but not +yet validated. A subset of RVV is implemented and tested with GCC and QEMU. +Work is underway to compile using LLVM, which has different intrinsics with AVL. + +Version 0.11 is considered stable enough to use in other projects, and is +expected to remain backwards compatible unless serious issues are discovered +while finishing the RVV target. After that, Highway will reach version 1.0. + +Continuous integration tests build with a recent version of Clang (running on +x86 and QEMU for ARM) and MSVC from VS2015 (running on x86). + +Before releases, we also test on x86 with Clang and GCC, and ARMv7/8 via +GCC cross-compile and QEMU. See the +[testing process](g3doc/release_testing_process.md) for details. + +The `contrib` directory contains SIMD-related utilities: an image class with +aligned rows, and a math library (16 functions already implemented, mostly +trigonometry). + +## Installation + +This project uses cmake to generate and build. In a Debian-based system you can +install it via: + +```bash +sudo apt install cmake +``` + +Highway's unit tests use [googletest](https://github.com/google/googletest). +By default, Highway's CMake downloads this dependency at configuration time. +You can disable this by setting the `HWY_SYSTEM_GTEST` CMake variable to ON and +installing gtest separately: + +```bash +sudo apt install libgtest-dev +``` + +To build and test the library the standard cmake workflow can be used: + +```bash +mkdir -p build && cd build +cmake .. +make -j && make test +``` + +Or you can run `run_tests.sh` (`run_tests.bat` on Windows). + +Bazel is also supported for building, but it is not as widely used/tested. + +## Quick start + +You can use the `benchmark` inside examples/ as a starting point. + +A [quick-reference page](g3doc/quick_reference.md) briefly lists all operations +and their parameters, and the [instruction_matrix](g3doc/instruction_matrix.pdf) +indicates the number of instructions per operation. + +We recommend using full SIMD vectors whenever possible for maximum performance +portability. To obtain them, pass a `HWY_FULL(float)` tag to functions such as +`Zero/Set/Load`. There is also the option of a vector of up to `N` (a power of +two <= 16/sizeof(T)) lanes of type `T`: `HWY_CAPPED(T, N)`. If `HWY_TARGET == +HWY_SCALAR`, the vector always has one lane. For all other targets, up to +128-bit vectors are guaranteed to be available. + +Functions using Highway must be inside `namespace HWY_NAMESPACE {` +(possibly nested in one or more other namespaces defined by the project), and +additionally either prefixed with `HWY_ATTR`, or residing between +`HWY_BEFORE_NAMESPACE()` and `HWY_AFTER_NAMESPACE()`. + +* For static dispatch, `HWY_TARGET` will be the best available target among + `HWY_BASELINE_TARGETS`, i.e. those allowed for use by the compiler (see + [quick-reference](g3doc/quick_reference.md)). Functions inside `HWY_NAMESPACE` + can be called using `HWY_STATIC_DISPATCH(func)(args)` within the same module + they are defined in. You can call the function from other modules by + wrapping it in a regular function and declaring the regular function in a + header. + +* For dynamic dispatch, a table of function pointers is generated via the + `HWY_EXPORT` macro that is used by `HWY_DYNAMIC_DISPATCH(func)(args)` to + call the best function pointer for the current CPU's supported targets. A + module is automatically compiled for each target in `HWY_TARGETS` (see + [quick-reference](g3doc/quick_reference.md)) if `HWY_TARGET_INCLUDE` is + defined and foreach_target.h is included. + +## Compiler flags + +Applications should be compiled with optimizations enabled - without inlining, +SIMD code may slow down by factors of 10 to 100. For clang and GCC, `-O2` is +generally sufficient. + +For MSVC, we recommend compiling with `/Gv` to allow non-inlined functions to +pass vector arguments in registers. If intending to use the AVX2 target together +with half-width vectors (e.g. for `PromoteTo`), it is also important to compile +with `/arch:AVX2`. This seems to be the only way to generate VEX-encoded SSE4 +instructions on MSVC. Otherwise, mixing VEX-encoded AVX2 instructions and +non-VEX SSE4 may cause severe performance degradation. Unfortunately, the +resulting binary will then require AVX2. Note that no such flag is needed for +clang and GCC because they support target-specific attributes, which we use to +ensure proper VEX code generation for AVX2 targets. + +## Strip-mining loops + +To vectorize a loop, "strip-mining" transforms it into an outer loop and inner +loop with number of iterations matching the preferred vector width. + +In this section, let `T` denote the element type, `d = HWY_FULL(T)`, `count` the +number of elements to process, and `N = Lanes(d)` the number of lanes in a full +vector. Assume the loop body is given as a function `template<bool partial, +class D> void LoopBody(D d, size_t max_n)`. + +Highway offers several ways to express loops where `N` need not divide `count`: + +* Ensure all inputs/outputs are padded. Then the loop is simply + + ``` + for (size_t i = 0; i < count; i += N) LoopBody<false>(d, 0); + ``` + Here, the template parameter and second function argument are not needed. + + This is the preferred option, unless `N` is in the thousands and vector + operations are pipelined with long latencies. This was the case for + supercomputers in the 90s, but nowadays ALUs are cheap and we see most + implementations split vectors into 1, 2 or 4 parts, so there is little cost + to processing entire vectors even if we do not need all their lanes. Indeed + this avoids the (potentially large) cost of predication or partial + loads/stores on older targets, and does not duplicate code. + +* Process whole vectors as above, followed by a scalar loop: + + ``` + size_t i = 0; + for (; i + N <= count; i += N) LoopBody<false>(d, 0); + for (; i < count; ++i) LoopBody<false>(HWY_CAPPED(T, 1)(), 0); + ``` + The template parameter and second function arguments are again not needed. + + This avoids duplicating code, and is reasonable if `count` is large. + If `count` is small, the second loop may be slower than the next option. + +* Process whole vectors as above, followed by a single call to a modified + `LoopBody` with masking: + + ``` + size_t i = 0; + for (; i + N <= count; i += N) { + LoopBody<false>(d, 0); + } + if (i < count) { + LoopBody<true>(d, count - i); + } + ``` + Now the template parameter and second function argument can be used inside + `LoopBody` to 'blend' the new partial vector with previous memory contents: + `Store(IfThenElse(FirstN(d, N), partial, prev_full), d, aligned_pointer);`. + + This is a good default when it is infeasible to ensure vectors are padded. + In contrast to the scalar loop, only a single final iteration is needed. + +## Additional resources + +* [Highway introduction (slides)](g3doc/highway_intro.pdf) +* [Overview of instructions per operation on different architectures](g3doc/instruction_matrix.pdf) +* [Design philosophy and comparison](g3doc/design_philosophy.md) + +## Acknowledgments + +We have used [farm-sve](https://gitlab.inria.fr/bramas/farm-sve) by Berenger +Bramas; it has proved useful for checking the SVE port on an x86 development +machine. + +This is not an officially supported Google product. +Contact: janwas@google.com diff --git a/third_party/highway/WORKSPACE b/third_party/highway/WORKSPACE new file mode 100644 index 0000000..6df1f62 --- /dev/null +++ b/third_party/highway/WORKSPACE @@ -0,0 +1,24 @@ +workspace(name = "highway") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "com_google_googletest", + urls = ["https://github.com/google/googletest/archive/609281088cfefc76f9d0ce82e1ff6c30cc3591e5.zip"], + sha256 = "5cf189eb6847b4f8fc603a3ffff3b0771c08eec7dd4bd961bfd45477dd13eb73", + strip_prefix = "googletest-609281088cfefc76f9d0ce82e1ff6c30cc3591e5", +) + +# See https://google.github.io/googletest/quickstart-bazel.html +http_archive( + name = "rules_cc", + urls = ["https://github.com/bazelbuild/rules_cc/archive/40548a2974f1aea06215272d9c2b47a14a24e556.zip"], + sha256 = "56ac9633c13d74cb71e0546f103ce1c58810e4a76aa8325da593ca4277908d72", + strip_prefix = "rules_cc-40548a2974f1aea06215272d9c2b47a14a24e556", +) + +# Need recent version for config_setting_group +http_archive( + name = "bazel_skylib", + urls = ["https://github.com/bazelbuild/bazel-skylib/releases/download/0.9.0/bazel_skylib-0.9.0.tar.gz"], +) diff --git a/third_party/highway/debian/changelog b/third_party/highway/debian/changelog new file mode 100644 index 0000000..516a15e --- /dev/null +++ b/third_party/highway/debian/changelog @@ -0,0 +1,84 @@ +highway (0.15.0-1) UNRELEASED; urgency=medium + + * New ops: CompressBlendedStore, ConcatOdd/Even, IndicesFromVec + * New ops: OddEvenBlocks, SwapAdjacentBlocks, Reverse, RotateRight + * Add bf16, unsigned comparisons, more lane types for Reverse/TableLookupLanes + * Contrib: add sort(ing network) and dot(product) + * Targets: update RVV for LLVM, add experimental WASM2 + * Separate library hwy_test for test utils + * Add non-macro Simd<> aliases + * Fixes: const V& for GCC, AVX3 BZHI, POPCNT with AVX on MSVC, avoid %zu + + -- Jan Wassenberg <janwas@google.com> Wed, 10 Nov 2021 10:00:00 +0100 + +highway (0.14.2-1) UNRELEASED; urgency=medium + + * Add MaskedLoad + * Fix non-glibc PPC, Windows GCC, MSVC 19.14 + * Opt-in for -Werror; separate design_philosophy.md + + -- Jan Wassenberg <janwas@google.com> Tue, 24 Aug 2021 15:00:00 +0200 + +highway (0.14.1-1) UNRELEASED; urgency=medium + + * Add LoadMaskBits, CompressBits[Store] + * Fix CPU feature check (AES/F16C) and warnings + * Improved DASSERT - disabled in optimized builds + + -- Jan Wassenberg <janwas@google.com> Tue, 17 Aug 2021 14:00:00 +0200 + +highway (0.14.0-1) UNRELEASED; urgency=medium + + * Add SVE, S-SSE3, AVX3_DL targets + * Support partial vectors in all ops + * Add PopulationCount, FindFirstTrue, Ne, TableLookupBytesOr0 + * Add AESRound, CLMul, MulOdd, HWY_CAP_FLOAT16 + + -- Jan Wassenberg <janwas@google.com> Thu, 29 Jul 2021 15:00:00 +0200 + +highway (0.12.2-1) UNRELEASED; urgency=medium + + * fix scalar-only test and Windows macro conflict with Load/StoreFence + * replace deprecated wasm intrinsics + + -- Jan Wassenberg <janwas@google.com> Mon, 31 May 2021 16:00:00 +0200 + +highway (0.12.1-1) UNRELEASED; urgency=medium + + * doc updates, ARM GCC support, fix s390/ppc, complete partial vectors + * fix warnings, faster ARM div/sqrt, separate hwy_contrib library + * add Abs(i64)/FirstN/Pause, enable AVX2 on MSVC + + -- Jan Wassenberg <janwas@google.com> Wed, 19 May 2021 15:00:00 +0200 + +highway (0.12.0-1) UNRELEASED; urgency=medium + + * Add Shift*8, Compress16, emulated Scatter/Gather, StoreInterleaved3/4 + * Remove deprecated HWY_*_LANES, deprecate HWY_GATHER_LANES + * Proper IEEE rounding, reduce libstdc++ usage, inlined math + + -- Jan Wassenberg <janwas@google.com> Thu, 15 Apr 2021 20:00:00 +0200 + +highway (0.11.1-1) UNRELEASED; urgency=medium + + * Fix clang7 asan error, finish f16 conversions and add test + + -- Jan Wassenberg <janwas@google.com> Thu, 25 Feb 2021 16:00:00 +0200 + +highway (0.11.0-1) UNRELEASED; urgency=medium + + * Add RVV+mask logical ops, allow Shl/ShiftLeftSame on all targets, more math + + -- Jan Wassenberg <janwas@google.com> Thu, 18 Feb 2021 20:00:00 +0200 + +highway (0.7.0-1) UNRELEASED; urgency=medium + + * Added API stability notice, Compress[Store], contrib/, SignBit, CopySign + + -- Jan Wassenberg <janwas@google.com> Tue, 5 Jan 2021 17:00:00 +0200 + +highway (0.1-1) UNRELEASED; urgency=medium + + * Initial debian package. + + -- Alex Deymo <deymo@google.com> Mon, 19 Oct 2020 16:48:07 +0200 diff --git a/third_party/highway/debian/compat b/third_party/highway/debian/compat new file mode 100644 index 0000000..f599e28 --- /dev/null +++ b/third_party/highway/debian/compat @@ -0,0 +1 @@ +10 diff --git a/third_party/highway/debian/control b/third_party/highway/debian/control new file mode 100644 index 0000000..7c60ebc --- /dev/null +++ b/third_party/highway/debian/control @@ -0,0 +1,23 @@ +Source: highway +Maintainer: JPEG XL Maintainers <jpegxl@google.com> +Section: misc +Priority: optional +Standards-Version: 3.9.8 +Build-Depends: cmake, + debhelper (>= 9), + libgtest-dev +Homepage: https://github.com/google/highway + +Package: libhwy-dev +Architecture: any +Section: libdevel +Depends: ${misc:Depends} +Description: Efficient and performance-portable SIMD wrapper (developer files) + This library provides type-safe and source-code portable wrappers over + existing platform-specific intrinsics. Its design aims for simplicity, + reliable efficiency across platforms, and immediate usability with current + compilers. + . + This package installs the development files. There's no runtime library + since most of Highway is implemented in headers and only a very small + static library is needed. diff --git a/third_party/highway/debian/copyright b/third_party/highway/debian/copyright new file mode 100644 index 0000000..53ea57a --- /dev/null +++ b/third_party/highway/debian/copyright @@ -0,0 +1,20 @@ +Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: highway + +Files: * +Copyright: 2020 Google LLC +License: Apache-2.0 + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + . + http://www.apache.org/licenses/LICENSE-2.0 + . + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + . + On Debian systems, the complete text of the Apache License, Version 2 + can be found in "/usr/share/common-licenses/Apache-2.0". diff --git a/third_party/highway/debian/rules b/third_party/highway/debian/rules new file mode 100644 index 0000000..969fc12 --- /dev/null +++ b/third_party/highway/debian/rules @@ -0,0 +1,6 @@ +#!/usr/bin/make -f +%: + dh $@ --buildsystem=cmake + +override_dh_auto_configure: + dh_auto_configure -- -DHWY_SYSTEM_GTEST=ON diff --git a/third_party/highway/debian/source/format b/third_party/highway/debian/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/third_party/highway/debian/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/third_party/highway/g3doc/design_philosophy.md b/third_party/highway/g3doc/design_philosophy.md new file mode 100644 index 0000000..10fff8e --- /dev/null +++ b/third_party/highway/g3doc/design_philosophy.md @@ -0,0 +1,186 @@ +# Design philosophy + +* Performance is important but not the sole consideration. Anyone who goes to + the trouble of using SIMD clearly cares about speed. However, portability, + maintainability and readability also matter, otherwise we would write in + assembly. We aim for performance within 10-20% of a hand-written assembly + implementation on the development platform. There is no performance gap vs. + intrinsics: Highway code can do anything they can. If necessary, you can use + platform-specific instructions inside `#if HWY_TARGET == HWY_NEON` etc. + +* The guiding principles of C++ are "pay only for what you use" and "leave no + room for a lower-level language below C++". We apply these by defining a + SIMD API that ensures operation costs are visible, predictable and minimal. + +* Performance portability is important, i.e. the API should be efficient on + all target platforms. Unfortunately, common idioms for one platform can be + inefficient on others. For example: summing lanes horizontally versus + shuffling. Documenting which operations are expensive does not prevent their + use, as evidenced by widespread use of `HADDPS`. Performance acceptance + tests may detect large regressions, but do not help choose the approach + during initial development. Analysis tools can warn about some potential + inefficiencies, but likely not all. We instead provide [a carefully chosen + set of vector types and operations that are efficient on all target + platforms](instruction_matrix.pdf) (PPC8, SSE4/AVX2+, ARMv8). + +* Future SIMD hardware features are difficult to predict. For example, AVX2 + came with surprising semantics (almost no interaction between 128-bit + blocks) and AVX-512 added two kinds of predicates (writemask and zeromask). + To ensure the API reflects hardware realities, we suggest a flexible + approach that adds new operations as they become commonly available, with + fallback implementations where necessary. + +* Masking/predication differs between platforms, and it is not clear how + important the use cases are beyond the ternary operator `IfThenElse`. + AVX-512/ARM SVE zeromasks are useful, but not supported by P0214R5. We + provide `IfThen[Zero]Else[Zero]` variants. + +* "Width-agnostic" SIMD is more future-proof than user-specified fixed sizes. + For example, valarray-like code can iterate over a 1D array with a + library-specified vector width. This will result in better code when vector + sizes increase, and matches the direction taken by + [ARM SVE](https://alastairreid.github.io/papers/sve-ieee-micro-2017.pdf) and + RiscV V as well as Agner Fog's + [ForwardCom instruction set proposal](https://goo.gl/CFizWu). However, some + applications may require fixed sizes, so we also guarantee support for <= + 128-bit vectors in each instruction set. + +* The API and its implementation should be usable and efficient with commonly + used compilers, including MSVC. For example, we write `ShiftLeft<3>(v)` + instead of `v << 3` because MSVC 2017 (ARM64) does not propagate the literal + (https://godbolt.org/g/rKx5Ga). Highway requires function-specific target + attributes, supported by GCC 4.9 / Clang 3.9 / MSVC 2015. + +* Efficient and safe runtime dispatch is important. Modules such as image or + video codecs are typically embedded into larger applications such as + browsers, so they cannot require separate binaries for each CPU. Libraries + also cannot predict whether the application already uses AVX2 (and pays the + frequency throttling cost), so this decision must be left to the + application. Using only the lowest-common denominator instructions + sacrifices too much performance. Therefore, we provide code paths for + multiple instruction sets and choose the most suitable at runtime. To reduce + overhead, dispatch should be hoisted to higher layers instead of checking + inside every low-level function. Highway supports inlining functions in the + same file or in `*-inl.h` headers. We generate all code paths from the same + source to reduce implementation- and debugging cost. + +* Not every CPU need be supported. For example, pre-SSSE3 CPUs are + increasingly rare and the AVX instruction set is limited to floating-point + operations. To reduce code size and compile time, we provide specializations + for S-SSE3, SSE4, AVX2 and AVX-512 instruction sets on x86, plus a scalar + fallback. + +* Access to platform-specific intrinsics is necessary for acceptance in + performance-critical projects. We provide conversions to and from intrinsics + to allow utilizing specialized platform-specific functionality, and simplify + incremental porting of existing code. + +* The core API should be compact and easy to learn; we provide a [concise + reference](quick_reference.md). + +## Prior API designs + +The author has been writing SIMD code since 2002: first via assembly language, +then intrinsics, later Intel's `F32vec4` wrapper, followed by three generations +of custom vector classes. The first used macros to generate the classes, which +reduces duplication but also readability. The second used templates instead. +The third (used in highwayhash and PIK) added support for AVX2 and runtime +dispatch. The current design (used in JPEG XL) enables code generation for +multiple platforms and/or instruction sets from the same source, and improves +runtime dispatch. + +## Overloaded function API + +Most C++ vector APIs rely on class templates. However, the ARM SVE vector type +is sizeless and cannot be wrapped in a class. We instead rely on overloaded +functions. Overloading based on vector types is also undesirable because SVE +vectors cannot be default-constructed. We instead use a dedicated tag type +`Simd` for overloading, abbreviated to `D` for template arguments and `d` in +lvalues. + +Note that generic function templates are possible (see generic_ops-inl.h). + +## Masks + +AVX-512 introduced a major change to the SIMD interface: special mask registers +(one bit per lane) that serve as predicates. It would be expensive to force +AVX-512 implementations to conform to the prior model of full vectors with lanes +set to all one or all zero bits. We instead provide a Mask type that emulates +a subset of this functionality on other platforms at zero cost. + +Masks are returned by comparisons and `TestBit`; they serve as the input to +`IfThen*`. We provide conversions between masks and vector lanes. For clarity +and safety, we use FF..FF as the definition of true. To also benefit from +x86 instructions that only require the sign bit of floating-point inputs to be +set, we provide a special `ZeroIfNegative` function. + +## Differences vs. [P0214R5](https://goo.gl/zKW4SA) / std::experimental::simd + +1. Allowing the use of built-in vector types by relying on non-member + functions. By contrast, P0214R5 requires a wrapper class, which does not + work for sizeless vector types currently used by ARM SVE and Risc-V. + +1. Adding widely used and portable operations such as `AndNot`, `AverageRound`, + bit-shift by immediates and `IfThenElse`. + +1. Designing the API to avoid or minimize overhead on AVX2/AVX-512 caused by + crossing 128-bit 'block' boundaries. + +1. Avoiding the need for non-native vectors. By contrast, P0214R5's `simd_cast` + returns `fixed_size<>` vectors which are more expensive to access because + they reside on the stack. We can avoid this plus additional overhead on + ARM/AVX2 by defining width-expanding operations as functions of a vector + part, e.g. promoting half a vector of `uint8_t` lanes to one full vector of + `uint16_t`, or demoting full vectors to half vectors with half-width lanes. + +1. Guaranteeing access to the underlying intrinsic vector type. This ensures + all platform-specific capabilities can be used. P0214R5 instead only + 'encourages' implementations to provide access. + +1. Enabling safe runtime dispatch and inlining in the same binary. P0214R5 is + based on the Vc library, which does not provide assistance for linking + multiple instruction sets into the same binary. The Vc documentation + suggests compiling separate executables for each instruction set or using + GCC's ifunc (indirect functions). The latter is compiler-specific and risks + crashes due to ODR violations when compiling the same function with + different compiler flags. We solve this problem via target-specific + namespaces and attributes (see HOWTO section below). We also permit a mix of + static target selection and runtime dispatch for hotspots that may benefit + from newer instruction sets if available. + +1. Omitting inefficient or non-performance-portable operations such as `hmax`, + `operator[]`, and unsupported integer comparisons. Applications can often + replace these operations at lower cost than emulating that exact behavior. + +1. Omitting `long double` types: these are not commonly available in hardware. + +1. Ensuring signed integer overflow has well-defined semantics (wraparound). + +1. Simple header-only implementation and a fraction of the size of the + Vc library from which P0214 was derived (39K, vs. 92K lines in + https://github.com/VcDevel/Vc according to the gloc Chrome extension). + +1. Avoiding hidden performance costs. P0214R5 allows implicit conversions from + integer to float, which costs 3-4 cycles on x86. We make these conversions + explicit to ensure their cost is visible. + +## Other related work + +* [Neat SIMD](http://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=7568423) + adopts a similar approach with interchangeable vector/scalar types and + a compact interface. It allows access to the underlying intrinsics, but + does not appear to be designed for other platforms than x86. + +* UME::SIMD ([code](https://goo.gl/yPeVZx), [paper](https://goo.gl/2xpZrk)) + also adopts an explicit vectorization model with vector classes. + However, it exposes the union of all platform capabilities, which makes the + API harder to learn (209-page spec) and implement (the estimated LOC count + is [500K](https://goo.gl/1THFRi)). The API is less performance-portable + because it allows applications to use operations that are inefficient on + other platforms. + +* Inastemp ([code](https://goo.gl/hg3USM), [paper](https://goo.gl/YcTU7S)) + is a vector library for scientific computing with some innovative features: + automatic FLOPS counting, and "if/else branches" using lambda functions. + It supports IBM Power8, but only provides float and double types and does + not support SVE without assuming the runtime vector size. diff --git a/third_party/highway/g3doc/highway_intro.pdf b/third_party/highway/g3doc/highway_intro.pdf Binary files differnew file mode 100644 index 0000000..e051a2c --- /dev/null +++ b/third_party/highway/g3doc/highway_intro.pdf diff --git a/third_party/highway/g3doc/instruction_matrix.pdf b/third_party/highway/g3doc/instruction_matrix.pdf Binary files differnew file mode 100644 index 0000000..23608f3 --- /dev/null +++ b/third_party/highway/g3doc/instruction_matrix.pdf diff --git a/third_party/highway/g3doc/quick_reference.md b/third_party/highway/g3doc/quick_reference.md new file mode 100644 index 0000000..797fc71 --- /dev/null +++ b/third_party/highway/g3doc/quick_reference.md @@ -0,0 +1,1233 @@ +# API synopsis / quick reference + +[[_TOC_]] + +## Usage modes + +Highway can compile for multiple CPU targets, choosing the best available at +runtime (dynamic dispatch), or compile for a single CPU target without runtime +overhead (static dispatch). Examples of both are provided in examples/. + +Dynamic dispatch uses the same source code as static, plus `#define +HWY_TARGET_INCLUDE`, `#include "hwy/foreach_target.h"` and +`HWY_DYNAMIC_DISPATCH`. + +## Headers + +The public headers are: + +* hwy/highway.h: main header, included from source AND/OR header files that + use vector types. Note that including in headers may increase compile time, + but allows declaring functions implemented out of line. + +* hwy/base.h: included from headers that only need compiler/platform-dependent + definitions (e.g. `PopCount`) without the full highway.h. + +* hwy/foreach_target.h: re-includes the translation unit (specified by + `HWY_TARGET_INCLUDE`) once per enabled target to generate code from the same + source code. highway.h must still be included, either before or after. + +* hwy/aligned_allocator.h: defines functions for allocating memory with + alignment suitable for `Load`/`Store`. + +* hwy/cache_control.h: defines stand-alone functions to control caching (e.g. + prefetching), independent of actual SIMD. + +* hwy/nanobenchmark.h: library for precisely measuring elapsed time (under + varying inputs) for benchmarking small/medium regions of code. + +* hwy/tests/test_util-inl.h: defines macros for invoking tests on all + available targets, plus per-target functions useful in tests (e.g. Print). + +SIMD implementations must be preceded and followed by the following: + +``` +#include "hwy/highway.h" +HWY_BEFORE_NAMESPACE(); // at file scope +namespace project { // optional +namespace HWY_NAMESPACE { + +// implementation + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace project - optional +HWY_AFTER_NAMESPACE(); +``` + +## Notation in this doc + +* `T` denotes the type of a vector lane; +* `N` is a size_t value that governs (but is not necessarily identical to) the + number of lanes; +* `D` is shorthand for `Simd<T, N>`; +* `d` is an lvalue of type `D`, passed as a function argument e.g. to Zero; +* `V` is the type of a vector. + +## Vector and tag types + +Highway vectors consist of one or more 'lanes' of the same built-in type +`uint##_t, int##_t` for `## = 8, 16, 32, 64`, plus `float##_t` for `## = 16, 32, +64` and `bfloat16_t`. + +In Highway, `float16_t` (an IEEE binary16 half-float) and `bfloat16_t` (the +upper 16 bits of an IEEE binary32 float) only support load, store, and +conversion to/from `float32_t`. The behavior of infinity and NaN in `float16_t` +is implementation-defined due to ARMv7. + +On RVV, vectors are sizeless and cannot be wrapped inside a class. The Highway +API allows using built-in types as vectors because operations are expressed as +overloaded functions. Instead of constructors, overloaded initialization +functions such as `Set` take a zero-sized tag argument called `d` of type `D = +Simd<T, N>` and return an actual vector of unspecified type. + +`T` is one of the lane types above, and may be retrieved via `TFromD<D>`. + +`N` is target-dependent and not directly user-specified. The actual lane count +may not be known at compile time, but can be obtained via `Lanes(d)`. Use this +value, which is potentially different from `N`, to increment loop counters etc. +Note that `Lanes(d)` could potentially change at runtime, upon user request via +special CPU instructions. Thus we discourage caching the result; it is typically +used inside a function or basic block. + +The actual lane count is guaranteed to be a power of two, even on SVE hardware +where vectors can be a multiple of 128 bits (there, the extra lanes remain +unused). This simplifies alignment: remainders can be computed as `count & +(Lanes(d) - 1)` instead of an expensive modulo. It also ensures loop trip counts +that are a large power of two (at least `MaxLanes`) are evenly divisible by the +lane count, thus avoiding the need for a second loop to handle remainders. + +`d` lvalues (a tag, NOT actual vector) are typically obtained using two aliases: + +* Most common: `ScalableTag<T[, shift]> d;` or the macro form `HWY_FULL(T[, + LMUL=1]) d;`. With the default value of the second argument, these both + select full vectors which utilize all available lanes. + + Only for targets (e.g. RVV) that support register groups, the shift (-3..3) + and LMUL argument (1, 2, 4, 8) specify `LMUL`, the number of registers in + the group. This effectively multiplies the lane count in each operation by + `LMUL`, or shifts by `shift` (negative values are understood as + right-shifting by the absolute value). These arguments will eventually be + optional hints that may improve performance on 1-2 wide machines (at the + cost of reducing the effective number of registers), but the experimental + GCC support for RVV does not support fractional `LMUL`. Thus, + mixed-precision code (e.g. demoting float to uint8_t) currently requires + `LMUL` to be at least the ratio of the sizes of the largest and smallest + type, and smaller `d` to be obtained via `Half<DLarger>`. + +* Less common: `CappedTag<T, kCap> d` or the macro form `HWY_CAPPED(T, kCap) + d;`. These select vectors or masks where *no more than* the first `kCap` (a + power of two) lanes have observable effects such as loading/storing to + memory, or being counted by `CountTrue`. The number of lanes may also be + less; for the `HWY_SCALAR` target, vectors always have a single lane. + +* For applications that require fixed-size vectors: `FixedTag<T, kCount> d;` + will select vectors where exactly `kCount` lanes have observable effects. + These may be implemented using full vectors plus additional runtime cost for + masking in `Load` etc. All targets except `HWY_SCALAR` allow any power of + two `kCount <= 16/sizeof(T)`. This tag can be used when the `HWY_SCALAR` + target is anyway disabled (superseded by a higher baseline) or unusable (due + to use of ops such as `TableLookupBytes`). + +* The result of `UpperHalf`/`LowerHalf` has half the lanes. To obtain a + corresponding `d`, use `Half<decltype(d)>`; the opposite is `Twice<>`. + +User-specified lane counts or tuples of vectors could cause spills on targets +with fewer or smaller vectors. By contrast, Highway encourages vector-length +agnostic code, which is more performance-portable. + +Given that lane counts are potentially compile-time-unknown, storage for vectors +should be dynamically allocated, e.g. via `AllocateAligned(Lanes(d))`. For +applications that require a compile-time bound, `MaxLanes(d)` uses the `N` from +`Simd<T, N>` to return a (loose) upper bound, NOT necessarily the actual lane +count. Note that some compilers are not able to interpret it as constexpr. + +For mixed-precision code (e.g. `uint8_t` lanes promoted to `float`), tags for +the smaller types must be obtained from those of the larger type (e.g. via +`Rebind<uint8_t, HWY_FULL(float)>`). + +## Using unspecified vector types + +Vector types are unspecified and depend on the target. User code could define +them as `auto`, but it is more readable (due to making the type visible) to use +an alias such as `Vec<D>`, or `decltype(Zero(d))`. + +Vectors are sizeless types on RVV/SVE. Therefore, vectors must not be used in +arrays/STL containers (use the lane type `T` instead), class members, +static/thread_local variables, new-expressions (use `AllocateAligned` instead), +and sizeof/pointer arithmetic (increment `T*` by `Lanes(d)` instead). + +Initializing constants requires a tag type `D`, or an lvalue `d` of that type. +The `D` can be passed as a template argument or obtained from a vector type `V` +via `DFromV<V>`. `TFromV<V>` is equivalent to `TFromD<DFromV<V>>`. + +**Note**: Let `DV = DFromV<V>`. For builtin `V` (currently necessary on +RVV/SVE), `DV` might not be the same as the `D` used to create `V`. In +particular, `DV` must not be passed to `Load/Store` functions because it may +lack the limit on `N` established by the original `D`. However, `Vec<DV>` is the +same as `V`. + +Thus a template argument `V` suffices for generic functions that do not load +from/store to memory: `template<class V> V Mul4(V v) { return v * +Set(DFromV<V>(), 4); }`. + +Example of mixing partial vectors with generic functions: + +``` +CappedTag<int16_t, 2> d2; +auto v = Mul4(Set(d2, 2)); +Store(v, d2, ptr); // Use d2, NOT DFromV<decltype(v)>() +``` + +## Operations + +In the following, the argument or return type `V` denotes a vector with `N` +lanes, and `M` a mask. Operations limited to certain vector types begin with a +constraint of the form `V`: `{prefixes}[{bits}]`. The prefixes `u,i,f` denote +unsigned, signed, and floating-point types, and bits indicates the number of +bits per lane: 8, 16, 32, or 64. Any combination of the specified prefixes and +bits are allowed. Abbreviations of the form `u32 = {u}{32}` may also be used. + +Note that Highway functions reside in `hwy::HWY_NAMESPACE`, whereas user-defined +functions reside in `project::[nested]::HWY_NAMESPACE`. Highway functions +generally take either a `Simd` or vector/mask argument. For targets where +vectors and masks are defined in namespace `hwy`, the functions will be found +via Argument-Dependent Lookup. However, this does not work for function +templates, and RVV and SVE both use builtin vectors. There are three options for +portable code, in descending order of preference: + +- `namespace hn = hwy::HWY_NAMESPACE;` alias used to prefix ops, e.g. + `hn::LoadDup128(..)`; +- `using hwy::HWY_NAMESPACE::LoadDup128;` declarations for each op used; +- `using hwy::HWY_NAMESPACE;` directive. This is generally discouraged, + especially for SIMD code residing in a header. + +Note that overloaded operators are not yet supported on RVV and SVE; code that +wishes to run on all targets until that is resolved can use functions such as +`Eq`, `Lt`, `Add`, `Div` etc. + +### Initialization + +* <code>V **Zero**(D)</code>: returns N-lane vector with all bits set to 0. +* <code>V **Set**(D, T)</code>: returns N-lane vector with all lanes equal to + the given value of type `T`. +* <code>V **Undefined**(D)</code>: returns uninitialized N-lane vector, e.g. + for use as an output parameter. +* <code>V **Iota**(D, T)</code>: returns N-lane vector where the lane with + index `i` has the given value of type `T` plus `i`. The least significant + lane has index 0. This is useful in tests for detecting lane-crossing bugs. +* <code>V **SignBit**(D, T)</code>: returns N-lane vector with all lanes set + to a value whose representation has only the most-significant bit set. + +### Arithmetic + +* <code>V **operator+**(V a, V b)</code>: returns `a[i] + b[i]` (mod 2^bits). +* <code>V **operator-**(V a, V b)</code>: returns `a[i] - b[i]` (mod 2^bits). + +* `V`: `{i,f}` \ + <code>V **Neg**(V a)</code>: returns `-a[i]`. + +* `V`: `{i,f}` \ + <code>V **Abs**(V a)</code> returns the absolute value of `a[i]`; for + integers, `LimitsMin()` maps to `LimitsMax() + 1`. + +* `V`: `f32` \ + <code>V **AbsDiff**(V a, V b)</code>: returns `|a[i] - b[i]|` in each lane. + +* `V`: `{u,i}{8,16}` \ + <code>V **SaturatedAdd**(V a, V b)</code> returns `a[i] + b[i]` saturated to + the minimum/maximum representable value. + +* `V`: `{u,i}{8,16}` \ + <code>V **SaturatedSub**(V a, V b)</code> returns `a[i] - b[i]` saturated to + the minimum/maximum representable value. + +* `V`: `{u}{8,16}` \ + <code>V **AverageRound**(V a, V b)</code> returns `(a[i] + b[i] + 1) / 2`. + +* <code>V **Clamp**(V a, V lo, V hi)</code>: returns `a[i]` clamped to + `[lo[i], hi[i]]`. + +* `V`: `{f}` \ + <code>V **operator/**(V a, V b)</code>: returns `a[i] / b[i]` in each lane. + +* `V`: `{f}` \ + <code>V **Sqrt**(V a)</code>: returns `sqrt(a[i])`. + +* `V`: `f32` \ + <code>V **ApproximateReciprocalSqrt**(V a)</code>: returns an approximation + of `1.0 / sqrt(a[i])`. `sqrt(a) ~= ApproximateReciprocalSqrt(a) * a`. x86 + and PPC provide 12-bit approximations but the error on ARM is closer to 1%. + +* `V`: `f32` \ + <code>V **ApproximateReciprocal**(V a)</code>: returns an approximation of + `1.0 / a[i]`. + +**Note**: Min/Max corner cases are target-specific and may change. If either +argument is qNaN, x86 SIMD returns the second argument, ARMv7 Neon returns NaN, +Wasm is supposed to return NaN but does not always, but other targets actually +uphold IEEE 754-2019 minimumNumber: returning the other argument if exactly one +is qNaN, and NaN if both are. + +* <code>V **Min**(V a, V b)</code>: returns `min(a[i], b[i])`. + +* <code>V **Max**(V a, V b)</code>: returns `max(a[i], b[i])`. + +#### Multiply + +* `V`: `{u,i}{16,32}` \ + <code>V <b>operator*</b>(V a, V b)</code>: returns the lower half of `a[i] * + b[i]` in each lane. + +* `V`: `{f}` \ + <code>V <b>operator*</b>(V a, V b)</code>: returns `a[i] * b[i]` in each + lane. + +* `V`: `i16` \ + <code>V **MulHigh**(V a, V b)</code>: returns the upper half of `a[i] * + b[i]` in each lane. + +* `V`: `{u,i}{32},u64` \ + <code>V2 **MulEven**(V a, V b)</code>: returns double-wide result of `a[i] * + b[i]` for every even `i`, in lanes `i` (lower) and `i + 1` (upper). `V2` is + a vector with double-width lanes, or the same as `V` for 64-bit inputs + (which are only supported if `HWY_TARGET != HWY_SCALAR`). + +* `V`: `u64` \ + <code>V **MulOdd**(V a, V b)</code>: returns double-wide result of `a[i] * + b[i]` for every odd `i`, in lanes `i - 1` (lower) and `i` (upper). Only + supported if `HWY_TARGET != HWY_SCALAR`. + +* `V`: `bf16`; `D`: `f32` \ + <code>Vec<D> **ReorderWidenMulAccumulate**(D, V a, V b, Vec<D> sum0, Vec<D>& + sum1)</code>: widens `a` and `b` to `TFromD<D>`, then adds `a[i] * b[i]` to + either `sum1[j]` or lane `j` of the return value, where `j = P(i)` and `P` + is a permutation. The only guarantee is that `SumOfLanes(Add(return_value, + sum1))` is the sum of all `a[i] * b[i]`. This is useful for computing dot + products and the L2 norm. + +#### Fused multiply-add + +When implemented using special instructions, these functions are more precise +and faster than separate multiplication followed by addition. The `*Sub` +variants are somewhat slower on ARM; it is preferable to replace them with +`MulAdd` using a negated constant. + +* `V`: `{f}` \ + <code>V **MulAdd**(V a, V b, V c)</code>: returns `a[i] * b[i] + c[i]`. + +* `V`: `{f}` \ + <code>V **NegMulAdd**(V a, V b, V c)</code>: returns `-a[i] * b[i] + c[i]`. + +* `V`: `{f}` \ + <code>V **MulSub**(V a, V b, V c)</code>: returns `a[i] * b[i] - c[i]`. + +* `V`: `{f}` \ + <code>V **NegMulSub**(V a, V b, V c)</code>: returns `-a[i] * b[i] - c[i]`. + +#### Shifts + +**Note**: Counts not in `[0, sizeof(T)*8)` yield implementation-defined results. +Left-shifting signed `T` and right-shifting positive signed `T` is the same as +shifting `MakeUnsigned<T>` and casting to `T`. Right-shifting negative signed +`T` is the same as an unsigned shift, except that 1-bits are shifted in. + +Compile-time constant shifts: the amount must be in [0, sizeof(T)*8). Generally +the most efficient variant, but 8-bit shifts are potentially slower than other +lane sizes, and `RotateRight` is often emulated with shifts: + +* `V`: `{u,i}` \ + <code>V **ShiftLeft**<int>(V a)</code> returns `a[i] << int`. + +* `V`: `{u,i}` \ + <code>V **ShiftRight**<int>(V a)</code> returns `a[i] >> int`. + +* `V`: `{u}{32,64}` \ + <code>V **RotateRight**<int>(V a)</code> returns `(a[i] >> int) | + (a[i] << (sizeof(T)*8 - int))`. + +Shift all lanes by the same (not necessarily compile-time constant) amount: + +* `V`: `{u,i}` \ + <code>V **ShiftLeftSame**(V a, int bits)</code> returns `a[i] << bits`. + +* `V`: `{u,i}` \ + <code>V **ShiftRightSame**(V a, int bits)</code> returns `a[i] >> bits`. + +Per-lane variable shifts (slow if SSSE3/SSE4, or 16-bit, or Shr i64 on AVX2): + +* `V`: `{u,i}{16,32,64}` \ + <code>V **operator<<**(V a, V b)</code> returns `a[i] << b[i]`. + +* `V`: `{u,i}{16,32,64}` \ + <code>V **operator>>**(V a, V b)</code> returns `a[i] >> b[i]`. + +#### Floating-point rounding + +* `V`: `{f}` \ + <code>V **Round**(V a)</code>: returns `a[i]` rounded towards the nearest + integer, with ties to even. + +* `V`: `{f}` \ + <code>V **Trunc**(V a)</code>: returns `a[i]` rounded towards zero + (truncate). + +* `V`: `{f}` \ + <code>V **Ceil**(V a)</code>: returns `a[i]` rounded towards positive + infinity (ceiling). + +* `V`: `{f}` \ + <code>V **Floor**(V a)</code>: returns `a[i]` rounded towards negative + infinity. + +### Logical + +* `V`: `{u,i}` \ + <code>V **PopulationCount**(V a)</code>: returns the number of 1-bits in + each lane, i.e. `PopCount(a[i])`. + +The following operate on individual bits within each lane: + +* `V`: `{u,i}` \ + <code>V **operator&**(V a, V b)</code>: returns `a[i] & b[i]`. + +* `V`: `{u,i}` \ + <code>V **operator|**(V a, V b)</code>: returns `a[i] | b[i]`. + +* `V`: `{u,i}` \ + <code>V **operator^**(V a, V b)</code>: returns `a[i] ^ b[i]`. + +* `V`: `{u,i}` \ + <code>V **Not**(V v)</code>: returns `~v[i]`. + +For floating-point types, builtin operators are not always available, so +non-operator functions (also available for integers) must be used: + +* <code>V **And**(V a, V b)</code>: returns `a[i] & b[i]`. + +* <code>V **Or**(V a, V b)</code>: returns `a[i] | b[i]`. + +* <code>V **Xor**(V a, V b)</code>: returns `a[i] ^ b[i]`. + +* <code>V **AndNot**(V a, V b)</code>: returns `~a[i] & b[i]`. + +Special functions for signed types: + +* `V`: `{f}` \ + <code>V **CopySign**(V a, V b)</code>: returns the number with the magnitude + of `a` and sign of `b`. + +* `V`: `{f}` \ + <code>V **CopySignToAbs**(V a, V b)</code>: as above, but potentially + slightly more efficient; requires the first argument to be non-negative. + +* `V`: `i32/64` \ + <code>V **BroadcastSignBit**(V a)</code> returns `a[i] < 0 ? -1 : 0`. + +* <code>V **ZeroIfNegative**(V v)</code>: returns `v[i] < 0 ? 0 : v[i]`. + +### Masks + +Let `M` denote a mask capable of storing true/false for each lane. + +#### Creation + +* <code>M **FirstN**(D, size_t N)</code>: returns mask with the first `N` + lanes (those with index `< N`) true. `N >= Lanes(D())` results in an + all-true mask. `N` must not exceed + `LimitsMax<SignedFromSize<HWY_MIN(sizeof(size_t), sizeof(TFromD<D>))>>()`. + Useful for implementing "masked" stores by loading `prev` followed by + `IfThenElse(FirstN(d, N), what_to_store, prev)`. + +* <code>M **MaskFromVec**(V v)</code>: returns false in lane `i` if `v[i] == + 0`, or true if `v[i]` has all bits set. + +* <code>M **LoadMaskBits**(D, const uint8_t* p)</code>: returns a mask + indicating whether the i-th bit in the array is set. Loads bytes and bits in + ascending order of address and index. At least 8 bytes of `p` must be + readable, but only `(Lanes(D()) + 7) / 8` need be initialized. Any unused + bits (happens if `Lanes(D()) < 8`) are treated as if they were zero. + +#### Conversion + +* <code>M1 **RebindMask**(D, M2 m)</code>: returns same mask bits as `m`, but + reinterpreted as a mask for lanes of type `TFromD<D>`. `M1` and `M2` must + have the same number of lanes. + +* <code>V **VecFromMask**(D, M m)</code>: returns 0 in lane `i` if `m[i] == + false`, otherwise all bits set. + +* <code>size_t **StoreMaskBits**(D, M m, uint8_t* p)</code>: stores a bit + array indicating whether `m[i]` is true, in ascending order of `i`, filling + the bits of each byte from least to most significant, then proceeding to the + next byte. Returns the number of bytes written: `(Lanes(D()) + 7) / 8`. At + least 8 bytes of `p` must be writable. + +#### Testing + +* <code>bool **AllTrue**(D, M m)</code>: returns whether all `m[i]` are true. + +* <code>bool **AllFalse**(D, M m)</code>: returns whether all `m[i]` are + false. + +* <code>size_t **CountTrue**(D, M m)</code>: returns how many of `m[i]` are + true [0, N]. This is typically more expensive than AllTrue/False. + +* <code>intptr_t **FindFirstTrue**(D, M m)</code>: returns the index of the + first (i.e. lowest index) `m[i]` that is true, or -1 if none are. + +#### Ternary operator + +* <code>V **IfThenElse**(M mask, V yes, V no)</code>: returns `mask[i] ? + yes[i] : no[i]`. + +* <code>V **IfThenElseZero**(M mask, V yes)</code>: returns `mask[i] ? + yes[i] : 0`. + +* <code>V **IfThenZeroElse**(M mask, V no)</code>: returns `mask[i] ? 0 : + no[i]`. + +#### Logical + +* <code>M **Not**(M m)</code>: returns mask of elements indicating whether the + input mask element was not set. + +* <code>M **And**(M a, M b)</code>: returns mask of elements indicating + whether both input mask elements were set. + +* <code>M **AndNot**(M not_a, M b)</code>: returns mask of elements indicating + whether not_a is not set and b is set. + +* <code>M **Or**(M a, M b)</code>: returns mask of elements indicating whether + either input mask element was set. + +* <code>M **Xor**(M a, M b)</code>: returns mask of elements indicating + whether exactly one input mask element was set. + +#### Compress + +* `V`: `{u,i,f}{16,32,64}` \ + <code>V **Compress**(V v, M m)</code>: returns `r` such that `r[n]` is + `v[i]`, with `i` the n-th lane index (starting from 0) where `m[i]` is true. + Compacts lanes whose mask is set into the lower lanes; upper lanes are + implementation-defined. Slow with 16-bit lanes. Use this form when the input + is already a mask, e.g. returned by a comparison. + +* `V`: `{u,i,f}{16,32,64}` \ + <code>size_t **CompressStore**(V v, M m, D d, T* p)</code>: writes lanes + whose mask `m` is set into `p`, starting from lane 0. Returns `CountTrue(d, + m)`, the number of valid lanes. May be implemented as `Compress` followed by + `StoreU`; lanes after the valid ones may still be overwritten! Slower for + 16-bit lanes. + +* `V`: `{u,i,f}{16,32,64}` \ + <code>size_t **CompressBlendedStore**(V v, M m, D d, T* p)</code>: writes + only lanes whose mask `m` is set into `p`, starting from lane 0. Returns + `CountTrue(d, m)`, the number of lanes written. Does not modify subsequent + lanes, but there is no guarantee of atomicity because this may be + implemented as `Compress, LoadU, IfThenElse(FirstN), StoreU`. + +* `V`: `{u,i,f}{16,32,64}` \ + <code>V **CompressBits**(V v, const uint8_t* HWY_RESTRICT bits)</code>: + Equivalent to, but often faster than `Compress(v, LoadMaskBits(d, bits))`. + `bits` is as specified for `LoadMaskBits`. If called multiple times, the + `bits` pointer passed to this function must also be marked `HWY_RESTRICT` to + avoid repeated work. Note that if the vector has less than 8 elements, + incrementing `bits` will not work as intended for packed bit arrays. + +* `V`: `{u,i,f}{16,32,64}` \ + <code>size_t **CompressBitsStore**(V v, const uint8_t* HWY_RESTRICT bits, D + d, T* p)</code>: combination of `CompressStore` and `CompressBits`, see + remarks there. + +#### Comparisons + +These return a mask (see above) indicating whether the condition is true. + +* <code>M **operator==**(V a, V b)</code>: returns `a[i] == b[i]`. +* <code>M **operator!=**(V a, V b)</code>: returns `a[i] != b[i]`. + +* <code>M **operator<**(V a, V b)</code>: returns `a[i] < b[i]`. + +* <code>M **operator>**(V a, V b)</code>: returns `a[i] > b[i]`. + +* `V`: `{f}` \ + <code>M **operator<=**(V a, V b)</code>: returns `a[i] <= b[i]`. + +* `V`: `{f}` \ + <code>M **operator>=**(V a, V b)</code>: returns `a[i] >= b[i]`. + +* `V`: `{u,i}` \ + <code>M **TestBit**(V v, V bit)</code>: returns `(v[i] & bit[i]) == bit[i]`. + `bit[i]` must have exactly one bit set. + +### Memory + +Memory operands are little-endian, otherwise their order would depend on the +lane configuration. Pointers are the addresses of `N` consecutive `T` values, +either naturally-aligned (`aligned`) or possibly unaligned (`p`). + +**Note**: computations with low arithmetic intensity (FLOP/s per memory traffic +bytes), e.g. dot product, can be *1.5 times as fast* when the memory operands +are naturally aligned. An unaligned access may require two load ports. + +#### Load + +Requires naturally-aligned vectors (e.g. from aligned_allocator.h): + +* <code>Vec<D> **Load**(D, const T* aligned)</code>: returns + `aligned[i]`. May fault if the pointer is not aligned to the vector size. + Using this whenever possible improves codegen on SSSE3/SSE4: unlike `LoadU`, + `Load` can be fused into a memory operand, which reduces register pressure. + +* <code>Vec<D> **MaskedLoad**(M mask, D, const T* aligned)</code>: + returns `aligned[i]` or zero if the `mask` governing element `i` is false. + May fault if the pointer is not aligned to the vector size. The alignment + requirement prevents differing behavior for "masked off" elements at invalid + addresses. Equivalent to, and potentially more efficient than, + `IfThenElseZero(mask, Load(D(), aligned))`. + +Requires only *element-aligned* vectors (e.g. from malloc/std::vector, or +aligned memory at indices which are not a multiple of the vector length): + +* <code>Vec<D> **LoadU**(D, const T* p)</code>: returns `p[i]`. + +* <code>Vec<D> **LoadDup128**(D, const T* p)</code>: returns one 128-bit + block loaded from `p` and broadcasted into all 128-bit block\[s\]. This may + be faster than broadcasting single values, and is more convenient than + preparing constants for the actual vector length. + +#### Scatter/Gather + +**Note**: Offsets/indices are of type `VI = Vec<RebindToSigned<D>>` and need not +be unique. The results are implementation-defined if any are negative. + +**Note**: Where possible, applications should `Load/Store/TableLookup*` entire +vectors, which is much faster than `Scatter/Gather`. Otherwise, code of the form +`dst[tbl[i]] = F(src[i])` should when possible be transformed to `dst[i] = +F(src[tbl[i]])` because `Scatter` is more expensive than `Gather`. + +* `D`: `{u,i,f}{32,64}` \ + <code>void **ScatterOffset**(Vec<D> v, D, const T* base, VI + offsets)</code>: stores `v[i]` to the base address plus *byte* `offsets[i]`. + +* `D`: `{u,i,f}{32,64}` \ + <code>void **ScatterIndex**(Vec<D> v, D, const T* base, VI + indices)</code>: stores `v[i]` to `base[indices[i]]`. + +* `D`: `{u,i,f}{32,64}` \ + <code>Vec<D> **GatherOffset**(D, const T* base, VI offsets)</code>: + returns elements of base selected by *byte* `offsets[i]`. + +* `D`: `{u,i,f}{32,64}` \ + <code>Vec<D> **GatherIndex**(D, const T* base, VI indices)</code>: + returns vector of `base[indices[i]]`. + +#### Store + +* <code>void **Store**(Vec<D> a, D, T* aligned)</code>: copies `a[i]` + into `aligned[i]`, which must be naturally aligned. Writes exactly N * + sizeof(T) bytes. + +* <code>void **StoreU**(Vec<D> a, D, T* p)</code>: as Store, but without + the alignment requirement. + +* `D`: `u8` \ + <code>void **StoreInterleaved3**(Vec<D> v0, Vec<D> v1, + Vec<D> v2, D, T* p)</code>: equivalent to shuffling `v0, v1, v2` + followed by three `StoreU()`, such that `p[0] == v0[0], p[1] == v1[0], + p[2] == v1[0]`. Useful for RGB samples. + +* `D`: `u8` \ + <code>void **StoreInterleaved4**(Vec<D> v0, Vec<D> v1, + Vec<D> v2, Vec<D> v3, D, T* p)</code>: as above, but for four + vectors (e.g. RGBA samples). + +### Cache control + +All functions except `Stream` are defined in cache_control.h. + +* <code>void **Stream**(Vec<D> a, D d, const T* aligned)</code>: copies + `a[i]` into `aligned[i]` with non-temporal hint if available (useful for + write-only data; avoids cache pollution). May be implemented using a + CPU-internal buffer. To avoid partial flushes and unpredictable interactions + with atomics (for example, see Intel SDM Vol 4, Sec. 8.1.2.2), call this + consecutively for an entire naturally aligned cache line (typically 64 + bytes). Each call may write a multiple of `HWY_STREAM_MULTIPLE` bytes, which + can exceed `Lanes(d) * sizeof(T)`. The new contents of `aligned` may not be + visible until `FlushStream` is called. + +* <code>void **FlushStream**()</code>: ensures values written by previous + `Stream` calls are visible on the current core. This is NOT sufficient for + synchronizing across cores; when `Stream` outputs are to be consumed by + other core(s), the producer must publish availability (e.g. via mutex or + atomic_flag) after `FlushStream`. + +* <code>void **FlushCacheline**(const void* p)</code>: invalidates and flushes + the cache line containing "p", if possible. + +* <code>void **Prefetch**(const T* p)</code>: optionally begins loading the + cache line containing "p" to reduce latency of subsequent actual loads. + +* <code>void **Pause**()</code>: when called inside a spin-loop, may reduce + power consumption. + +### Type conversion + +* <code>Vec<D> **BitCast**(D, V)</code>: returns the bits of `V` + reinterpreted as type `Vec<D>`. + +* `V`,`D`: (`u8,u16`), (`u16,u32`), (`u8,u32`), (`u32,u64`), (`u8,i16`), \ + (`u8,i32`), (`u16,i32`), (`i8,i16`), (`i8,i32`), (`i16,i32`), (`i32,i64`) + <code>Vec<D> **PromoteTo**(D, V part)</code>: returns `part[i]` zero- + or sign-extended to the integer type `MakeWide<T>`. + +* `V`,`D`: (`f16,f32`), (`bf16,f32`), (`f32,f64`) \ + <code>Vec<D> **PromoteTo**(D, V part)</code>: returns `part[i]` + widened to the floating-point type `MakeWide<T>`. + +* `V`,`D`: \ + <code>Vec<D> **PromoteTo**(D, V part)</code>: returns `part[i]` + converted to 64-bit floating point. + +* `V`,`D`: (`bf16,f32`) <code>Vec<D> **PromoteLowerTo**(D, V v)</code>: + returns `v[i]` widened to `MakeWide<T>`, for i in `[0, Lanes(D()))`. Note + that `V` has twice as many lanes as `D` and the return value. + +* `V`,`D`: (`bf16,f32`) <code>Vec<D> **PromoteUpperTo**(D, V v)</code>: + returns `v[i]` widened to `MakeWide<T>`, for i in `[Lanes(D()), 2 * + Lanes(D()))`. Note that `V` has twice as many lanes as `D` and the return + value. + +* `V`,`V8`: (`u32,u8`) \ + <code>V8 **U8FromU32**(V)</code>: special-case `u32` to `u8` conversion when + all lanes of `V` are already clamped to `[0, 256)`. + +`DemoteTo` and float-to-int `ConvertTo` return the closest representable value +if the input exceeds the destination range. + +* `V`,`D`: (`i16,i8`), (`i32,i8`), (`i32,i16`), (`i16,u8`), (`i32,u8`), + (`i32,u16`), (`f64,f32`) \ + <code>Vec<D> **DemoteTo**(D, V a)</code>: returns `a[i]` after packing + with signed/unsigned saturation to `MakeNarrow<T>`. + +* `V`,`D`: `f64,i32` \ + <code>Vec<D> **DemoteTo**(D, V a)</code>: rounds floating point + towards zero and converts the value to 32-bit integers. + +* `V`,`D`: (`f32,f16`), (`f32,bf16`) \ + <code>Vec<D> **DemoteTo**(D, V a)</code>: narrows float to half (for + bf16, it is unspecified whether this truncates or rounds). + +* `V`,`D`: (`f32,bf16`) \ + <code>Vec<D> **ReorderDemote2To**(D, V a, V b)</code>: as above, but + converts two inputs, `D` and the output have twice as many lanes as `V`, and + the output order is some permutation of the inputs. + +* `V`,`D`: (`i32`,`f32`), (`i64`,`f64`) \ + <code>Vec<D> **ConvertTo**(D, V)</code>: converts an integer value to + same-sized floating point. + +* `V`,`D`: (`f32`,`i32`), (`f64`,`i64`) \ + <code>Vec<D> **ConvertTo**(D, V)</code>: rounds floating point towards + zero and converts the value to same-sized integer. + +* `V`: `f32`; `Ret`: `i32` \ + <code>Ret **NearestInt**(V a)</code>: returns the integer nearest to `a[i]`; + results are undefined for NaN. + +### Combine + +* <code>V2 **LowerHalf**([D, ] V)</code>: returns the lower half of the vector + `V`. The optional `D` (provided for consistency with `UpperHalf`) is + `Half<DFromV<V>>`. + +* <code>V2 **UpperHalf**(D, V)</code>: returns upper half of the vector `V`, + where `D` is `Half<DFromV<V>>`. + +* <code>V **ZeroExtendVector**(D, V2)</code>: returns vector whose `UpperHalf` + is zero and whose `LowerHalf` is the argument; `D` is `Twice<DFromV<V2>>`. + +* <code>V **Combine**(D, V2, V2)</code>: returns vector whose `UpperHalf` is + the first argument and whose `LowerHalf` is the second argument. This is + currently only implemented for RVV, AVX2, AVX3*. `D` is `Twice<DFromV<V2>>`. + +**Note**: the following operations cross block boundaries, which is typically +more expensive on AVX2/AVX-512 than per-block operations. + +* <code>V **ConcatLowerLower**(D, V hi, V lo)</code>: returns the + concatenation of the lower halves of `hi` and `lo` without splitting into + blocks. `D` is `DFromV<V>`. + +* <code>V **ConcatUpperUpper**(D, V hi, V lo)</code>: returns the + concatenation of the upper halves of `hi` and `lo` without splitting into + blocks. `D` is `DFromV<V>`. + +* <code>V **ConcatLowerUpper**(D, V hi, V lo)</code>: returns the inner half + of the concatenation of `hi` and `lo` without splitting into blocks. Useful + for swapping the two blocks in 256-bit vectors. `D` is `DFromV<V>`. + +* <code>V **ConcatUpperLower**(D, V hi, V lo)</code>: returns the outer + quarters of the concatenation of `hi` and `lo` without splitting into + blocks. Unlike the other variants, this does not incur a block-crossing + penalty on AVX2/3. `D` is `DFromV<V>`. + +* `V`: `{u,i,f}{32,64}` \ + <code>V **ConcatOdd**(V hi, V lo)</code>: returns the concatenation of the + odd lanes of `hi` and the odd lanes of `lo`. + +* `V`: `{u,i,f}{32,64}` \ + <code>V **ConcatEven**(V hi, V lo)</code>: returns the concatenation of the + even lanes of `hi` and the even lanes of `lo`. + +### Blockwise + +**Note**: if vectors are larger than 128 bits, the following operations split +their operands into independently processed 128-bit *blocks*. + +* `V`: `{u,i}{16,32,64}, {f}` \ + <code>V **Broadcast**<int i>(V)</code>: returns individual *blocks*, + each with lanes set to `input_block[i]`, `i = [0, 16/sizeof(T))`. + +* `V`: `{u,i}` \ + <code>VI **TableLookupBytes**(V bytes, VI indices)</code>: returns + `bytes[indices[i]]`. Uses byte lanes regardless of the actual vector types. + Results are implementation-defined if `indices[i] < 0` or `indices[i] >= + HWY_MIN(Lanes(DFromV<V>()), 16)`. `VI` are integers with the same bit width + as a lane in `V`. The number of lanes in `V` and `VI` may differ, e.g. a + full-length table vector loaded via `LoadDup128`, plus partial vector `VI` + of 4-bit indices. + +* `V`: `{u,i}` \ + <code>VI **TableLookupBytesOr0**(V bytes, VI indices)</code>: returns + `bytes[indices[i]]`, or 0 if `indices[i] & 0x80`. Uses byte lanes regardless + of the actual vector types. Results are implementation-defined for + `indices[i] < 0` or in `[HWY_MIN(Lanes(DFromV<V>()), 16), 0x80)`. The + zeroing behavior has zero cost on x86 and ARM. For vectors of >= 256 bytes + (can happen on SVE and RVV), this will set all lanes after the first 128 + to 0. `VI` are integers with the same bit width as a lane in `V`. The number + of lanes in `V` and `VI` may differ. + +#### Zip/Interleave + +* <code>V **InterleaveLower**([D, ] V a, V b)</code>: returns *blocks* with + alternating lanes from the lower halves of `a` and `b` (`a[0]` in the + least-significant lane). The optional `D` (provided for consistency with + `InterleaveUpper`) is `DFromV<V>`. + +* <code>V **InterleaveUpper**(D, V a, V b)</code>: returns *blocks* with + alternating lanes from the upper halves of `a` and `b` (`a[N/2]` in the + least-significant lane). `D` is `DFromV<V>`. + +* `Ret`: `MakeWide<T>`; `V`: `{u,i}{8,16,32}` \ + <code>Ret **ZipLower**([D, ] V a, V b)</code>: returns the same bits as + `InterleaveLower`, but repartitioned into double-width lanes (required in + order to use this operation with scalars). The optional `D` (provided for + consistency with `ZipUpper`) is `RepartitionToWide<DFromV<V>>`. + +* `Ret`: `MakeWide<T>`; `V`: `{u,i}{8,16,32}` \ + <code>Ret **ZipUpper**(D, V a, V b)</code>: returns the same bits as + `InterleaveUpper`, but repartitioned into double-width lanes (required in + order to use this operation with scalars). `D` is + `RepartitionToWide<DFromV<V>>`. + +#### Shift + +* `V`: `{u,i}` \ + <code>V **ShiftLeftBytes**<int>([D, ] V)</code>: returns the result of + shifting independent *blocks* left by `int` bytes \[1, 15\]. The optional + `D` (provided for consistency with `ShiftRightBytes`) is `DFromV<V>`. + +* <code>V **ShiftLeftLanes**<int>([D, ] V)</code>: returns the result of + shifting independent *blocks* left by `int` lanes. The optional `D` + (provided for consistency with `ShiftRightLanes`) is `DFromV<V>`. + +* `V`: `{u,i}` \ + <code>V **ShiftRightBytes**<int>(D, V)</code>: returns the result of + shifting independent *blocks* right by `int` bytes \[1, 15\], shifting in + zeros even for partial vectors. `D` is `DFromV<V>`. + +* <code>V **ShiftRightLanes**<int>(D, V)</code>: returns the result of + shifting independent *blocks* right by `int` lanes, shifting in zeros even + for partial vectors. `D` is `DFromV<V>`. + +* `V`: `{u,i}` \ + <code>V **CombineShiftRightBytes**<int>(D, V hi, V lo)</code>: returns + a vector of *blocks* each the result of shifting two concatenated *blocks* + `hi[i] || lo[i]` right by `int` bytes \[1, 16). `D` is `DFromV<V>`. + +* <code>V **CombineShiftRightLanes**<int>(D, V hi, V lo)</code>: returns + a vector of *blocks* each the result of shifting two concatenated *blocks* + `hi[i] || lo[i]` right by `int` lanes \[1, 16/sizeof(T)). `D` is + `DFromV<V>`. + +#### Shuffle + +* `V`: `{u,i,f}{32}` \ + <code>V **Shuffle2301**(V)</code>: returns *blocks* with 32-bit halves + swapped inside 64-bit halves. + +* `V`: `{u,i,f}{32}` \ + <code>V **Shuffle1032**(V)</code>: returns *blocks* with 64-bit halves + swapped. + +* `V`: `{u,i,f}{64}` \ + <code>V **Shuffle01**(V)</code>: returns *blocks* with 64-bit halves + swapped. + +* `V`: `{u,i,f}{32}` \ + <code>V **Shuffle0321**(V)</code>: returns *blocks* rotated right (toward + the lower end) by 32 bits. + +* `V`: `{u,i,f}{32}` \ + <code>V **Shuffle2103**(V)</code>: returns *blocks* rotated left (toward the + upper end) by 32 bits. + +* `V`: `{u,i,f}{32}` \ + <code>V **Shuffle0123**(V)</code>: returns *blocks* with lanes in reverse + order. + +### Swizzle + +* <code>T **GetLane**(V)</code>: returns lane 0 within `V`. This is useful for + extracting `SumOfLanes` results. + +* <code>V **OddEven**(V a, V b)</code>: returns a vector whose odd lanes are + taken from `a` and the even lanes from `b`. + +* <code>V **OddEvenBlocks**(V a, V b)</code>: returns a vector whose odd + blocks are taken from `a` and the even blocks from `b`. Returns `b` if the + vector has no more than one block (i.e. is 128 bits or scalar). + +* <code>V **SwapAdjacentBlocks**(V v)</code>: returns a vector where blocks of + index `2*i` and `2*i+1` are swapped. Results are undefined for vectors with + less than two blocks; callers must first check that via `Lanes`. + +* `V`: `{u,i,f}{32,64}` \ + <code>V **TableLookupLanes**(V a, unspecified)</code> returns a vector of + `a[indices[i]]`, where `unspecified` is the return value of + `SetTableIndices(D, &indices[0])` or `IndicesFromVec`. The indices are not + limited to blocks, hence this is slower than `TableLookupBytes*` on + AVX2/AVX-512. Results are implementation-defined unless `0 <= indices[i] < + Lanes(D())`. `indices` are always integers, even if `V` is a floating-point + type. + +* `D`: `{u,i}{32,64}` \ + <code>unspecified **IndicesFromVec**(D d, V idx)</code> prepares for + `TableLookupLanes` with integer indices in `idx`, which must be the same bit + width as `TFromD<D>` and in the range `[0, Lanes(d))`, but need not be + unique. + +* `D`: `{u,i}{32,64}` \ + <code>unspecified **SetTableIndices**(D d, TI* idx)</code> prepares for + `TableLookupLanes` by loading `Lanes(d)` integer indices from `idx`, which + must be in the range `[0, Lanes(d))` but need not be unique. The index type + `TI` must be an integer of the same size as `TFromD<D>`. + +* `V`: `{u,i,f}{16,32,64}` \ + <code>V **Reverse**(D, V a)</code> returns a vector with lanes in reversed + order (`out[i] == a[Lanes(D()) - 1 - i]`). + +### Reductions + +**Note**: these 'reduce' all lanes to a single result (e.g. sum), which is +broadcasted to all lanes. To obtain a scalar, you can call `GetLane`. + +Being a horizontal operation (across lanes of the same vector), these are slower +than normal SIMD operations and are typically used outside critical loops. + +* `V`: `{u,i,f}{32,64}` \ + <code>V **SumOfLanes**(D, V v)</code>: returns the sum of all lanes in each + lane. + +* `V`: `{u,i,f}{32,64},{u,i}{16}` \ + <code>V **MinOfLanes**(D, V v)</code>: returns the minimum-valued lane in + each lane. + +* `V`: `{u,i,f}{32,64},{u,i}{16}` \ + <code>V **MaxOfLanes**(D, V v)</code>: returns the maximum-valued lane in + each lane. + +### Crypto + +* `V`: `u8` \ + <code>V **AESRound**(V state, V round_key)</code>: one round of AES + encrytion: `MixColumns(SubBytes(ShiftRows(state))) ^ round_key`. This + matches x86 AES-NI. The latency is independent of the input values. Only + available if `HWY_TARGET != HWY_SCALAR`. + +* `V`: `u64` \ + <code>V **CLMulLower**(V a, V b)</code>: carryless multiplication of the + lower 64 bits of each 128-bit block into a 128-bit product. The latency is + independent of the input values (assuming that is true of normal integer + multiplication) so this can safely be used in cryto. Applications that wish + to multiply upper with lower halves can `Shuffle01` one of the operands; on + x86 that is expected to be latency-neutral. + +* `V`: `u64` \ + <code>V **CLMulUpper**(V a, V b)</code>: as CLMulLower, but multiplies the + upper 64 bits of each 128-bit block. + +### Deprecated + +* <code>bool **AllTrue**(M m)</code>: returns whether all `m[i]` are true. + DEPRECATED, SVE needs an extra D argument. + +* <code>bool **AllFalse**(M m)</code>: returns whether all `m[i]` are false. + DEPRECATED, SVE needs an extra D argument. + +* <code>size_t **StoreMaskBits**(M m, uint8_t* p)</code>: stores a bit array + indicating whether `m[i]` is true, in ascending order of `i`, filling the + bits of each byte from least to most significant, then proceeding to the + next byte. Returns the number of (partial) bytes written. DEPRECATED, SVE + needs an extra D argument. + +* <code>size_t **CountTrue**(M m)</code>: returns how many of `m[i]` are true + [0, N]. This is typically more expensive than AllTrue/False. DEPRECATED, SVE + needs an extra D argument. + +* <code>void **StoreFence**()</code>: DEPRECATED, calls `FlushStream`. + +* <code>void **LoadFence**()</code>: delays subsequent loads until prior loads + are visible. Also a full fence on Intel CPUs. No effect on non-x86. + DEPRECATED due to differing behavior across architectures AND vendors. + +* <code>V2 **UpperHalf**(V)</code>: returns upper half of the vector `V`. + DEPRECATED, supporting partial vectors requires a D argument. + +* `V`: `{u,i}` \ + <code>V **ShiftRightBytes**<int>(V)</code>: returns the result of + shifting independent *blocks* right by `int` bytes \[1, 15\]. DEPRECATED, + supporting partial vectors requires a D argument. + +* <code>V **ShiftRightLanes**<int>(V)</code>: returns the result of + shifting independent *blocks* right by `int` lanes. DEPRECATED, supporting + partial vectors requires a D argument. + +* <code>V **ZeroExtendVector**(V2)</code>: returns vector whose `UpperHalf` is + zero and whose `LowerHalf` is the argument. DEPRECATED, supporting partial + vectors requires a D argument. + +* <code>V **Combine**(V2, V2)</code>: returns vector whose `UpperHalf` is the + first argument and whose `LowerHalf` is the second argument. This is + currently only implemented for RVV, AVX2, AVX3*. DEPRECATED, supporting + partial vectors requires a D argument. + +* <code>V **ConcatLowerLower**(V hi, V lo)</code>: returns the concatenation + of the lower halves of `hi` and `lo` without splitting into blocks. + DEPRECATED, supporting partial vectors requires a D argument. + +* <code>V **ConcatUpperUpper**(V hi, V lo)</code>: returns the concatenation + of the upper halves of `hi` and `lo` without splitting into blocks. + DEPRECATED, supporting partial vectors requires a D argument. + +* <code>V **ConcatLowerUpper**(V hi, V lo)</code>: returns the inner half of + the concatenation of `hi` and `lo` without splitting into blocks. Useful for + swapping the two blocks in 256-bit vectors. DEPRECATED, supporting partial + vectors requires a D argument. + +* <code>V **ConcatUpperLower**(V hi, V lo)</code>: returns the outer quarters + of the concatenation of `hi` and `lo` without splitting into blocks. Unlike + the other variants, this does not incur a block-crossing penalty on AVX2. + DEPRECATED, supporting partial vectors requires a D argument. + +* <code>V **InterleaveUpper**(V a, V b)</code>: returns *blocks* with + alternating lanes from the upper halves of `a` and `b` (`a[N/2]` in the + least-significant lane). DEPRECATED, supporting partial vectors requires a D + argument. + +* `Ret`: `MakeWide<T>`; `V`: `{u,i}{8,16,32}` \ + <code>Ret **ZipUpper**(V a, V b)</code>: returns the same bits as + `InterleaveUpper`, but repartitioned into double-width lanes (required in + order to use this operation with scalars). DEPRECATED, supporting partial + vectors requires a D argument. + +* `V`: `{u,i}` \ + <code>V **CombineShiftRightBytes**<int>(V hi, V lo)</code>: returns a + vector of *blocks* each the result of shifting two concatenated *blocks* + `hi[i] || lo[i]` right by `int` bytes \[1, 16). DEPRECATED, supporting + partial vectors requires a D argument. + +* <code>V **CombineShiftRightLanes**<int>(V hi, V lo)</code>: returns a + vector of *blocks* each the result of shifting two concatenated *blocks* + `hi[i] || lo[i]` right by `int` lanes \[1, 16/sizeof(T)). DEPRECATED, + supporting partial vectors requires a D argument. + +* `V`: `{u,i,f}{32,64}` \ + <code>V **SumOfLanes**(V v)</code>: returns the sum of all lanes in each + lane. DEPRECATED, SVE/RVV require a D argument to support partial vectors. + +* `V`: `{u,i,f}{32,64}` \ + <code>V **MinOfLanes**(V v)</code>: returns the minimum-valued lane in each + lane. DEPRECATED, SVE/RVV require a D argument to support partial vectors. + +* `V`: `{u,i,f}{32,64}` \ + <code>V **MaxOfLanes**(V v)</code>: returns the maximum-valued lane in each + lane. DEPRECATED, SVE/RVV require a D argument to support partial vectors. + +## Preprocessor macros + +* `HWY_ALIGN`: Prefix for stack-allocated (i.e. automatic storage duration) + arrays to ensure they have suitable alignment for Load()/Store(). This is + specific to `HWY_TARGET` and should only be used inside `HWY_NAMESPACE`. + + Arrays should also only be used for partial (<= 128-bit) vectors, or + `LoadDup128`, because full vectors may be too large for the stack and should + be heap-allocated instead (see aligned_allocator.h). + + Example: `HWY_ALIGN float lanes[4];` + +* `HWY_ALIGN_MAX`: as `HWY_ALIGN`, but independent of `HWY_TARGET` and may be + used outside `HWY_NAMESPACE`. + +## Advanced macros + +Let `Target` denote an instruction set: +`SCALAR/SSSE3/SSE4/AVX2/AVX3/AVX3_DL/PPC8/NEON/WASM/RVV`. Targets are only used +if enabled (i.e. not broken nor disabled). Baseline means the compiler is +allowed to generate such instructions (implying the target CPU would have to +support them). + +* `HWY_Target=##` are powers of two uniquely identifying `Target`. + +* `HWY_STATIC_TARGET` is the best enabled baseline `HWY_Target`, and matches + `HWY_TARGET` in static dispatch mode. This is useful even in dynamic + dispatch mode for deducing and printing the compiler flags. + +* `HWY_TARGETS` indicates which targets to generate for dynamic dispatch, and + which headers to include. It is determined by configuration macros and + always includes `HWY_STATIC_TARGET`. + +* `HWY_SUPPORTED_TARGETS` is the set of targets available at runtime. Expands + to a literal if only a single target is enabled, or SupportedTargets(). + +* `HWY_TARGET`: which `HWY_Target` is currently being compiled. This is + initially identical to `HWY_STATIC_TARGET` and remains so in static dispatch + mode. For dynamic dispatch, this changes before each re-inclusion and + finally reverts to `HWY_STATIC_TARGET`. Can be used in `#if` expressions to + provide an alternative to functions which are not supported by HWY_SCALAR. + +* `HWY_WANT_AVX3_DL`: additional opt-in for HWY_AVX3, which is disabled unless + this is defined by the app before including highway.h, OR all AVX3_DL + compiler flags are specified. + +* `HWY_IDE` is 0 except when parsed by IDEs; adding it to conditions such as + `#if HWY_TARGET != HWY_SCALAR || HWY_IDE` avoids code appearing greyed out. + +The following signal capabilities and expand to 1 or 0. + +* `HWY_CAP_INTEGER64`: support for 64-bit signed/unsigned integer lanes. +* `HWY_CAP_FLOAT16`: support for IEEE half-precision floating-point lanes. +* `HWY_CAP_FLOAT64`: support for double-precision floating-point lanes. + +The following were used to signal the maximum number of lanes for certain +operations, but this is no longer necessary (nor possible on SVE/RVV), so they +are DEPRECATED: + +* `HWY_GATHER_LANES(T)`. +* `HWY_CAP_GE256`: the current target supports vectors of >= 256 bits. +* `HWY_CAP_GE512`: the current target supports vectors of >= 512 bits. + +## Detecting supported targets + +`SupportedTargets()` returns a cached (initialized on-demand) bitfield of the +targets supported on the current CPU, detected using CPUID on x86 or equivalent. +This may include targets that are not in `HWY_TARGETS`, and vice versa. If +there is no overlap the binary will likely crash. This can only happen if: + +* the specified baseline is not supported by the current CPU, which + contradicts the definition of baseline, so the configuration is invalid; or +* the baseline does not include the enabled/attainable target(s), which are + also not supported by the current CPU, and baseline targets (in particular + `HWY_SCALAR`) were explicitly disabled. + +## Advanced configuration macros + +The following macros govern which targets to generate. Unless specified +otherwise, they may be defined per translation unit, e.g. to disable >128 bit +vectors in modules that do not benefit from them (if bandwidth-limited or only +called occasionally). This is safe because `HWY_TARGETS` always includes at +least one baseline target which `HWY_EXPORT` can use. + +* `HWY_DISABLE_CACHE_CONTROL` makes the cache-control functions no-ops. +* `HWY_DISABLE_BMI2_FMA` prevents emitting BMI/BMI2/FMA instructions. This + allows using AVX2 in VMs that do not support the other instructions, but + only if defined for all translation units. + +The following `*_TARGETS` are zero or more `HWY_Target` bits and can be defined +as an expression, e.g. `-DHWY_DISABLED_TARGETS=(HWY_SSE4|HWY_AVX3)`. + +* `HWY_BROKEN_TARGETS` defaults to a blocklist of known compiler bugs. + Defining to 0 disables the blocklist. + +* `HWY_DISABLED_TARGETS` defaults to zero. This allows explicitly disabling + targets without interfering with the blocklist. + +* `HWY_BASELINE_TARGETS` defaults to the set whose predefined macros are + defined (i.e. those for which the corresponding flag, e.g. -mavx2, was + passed to the compiler). If specified, this should be the same for all + translation units, otherwise the safety check in SupportedTargets (that all + enabled baseline targets are supported) may be inaccurate. + +Zero or one of the following macros may be defined to replace the default +policy for selecting `HWY_TARGETS`: + +* `HWY_COMPILE_ONLY_SCALAR` selects only `HWY_SCALAR`, which disables SIMD. +* `HWY_COMPILE_ONLY_STATIC` selects only `HWY_STATIC_TARGET`, which + effectively disables dynamic dispatch. +* `HWY_COMPILE_ALL_ATTAINABLE` selects all attainable targets (i.e. enabled + and permitted by the compiler, independently of autovectorization), which + maximizes coverage in tests. + +If none are defined, but `HWY_IS_TEST` is defined, the default is +`HWY_COMPILE_ALL_ATTAINABLE`. Otherwise, the default is to select all attainable +targets except any non-best baseline (typically `HWY_SCALAR`), which reduces +code size. + +## Compiler support + +Clang and GCC require e.g. -mavx2 flags in order to use SIMD intrinsics. +However, this enables AVX2 instructions in the entire translation unit, which +may violate the one-definition rule and cause crashes. Instead, we use +target-specific attributes introduced via #pragma. Function using SIMD must +reside between `HWY_BEFORE_NAMESPACE` and `HWY_AFTER_NAMESPACE`. Alternatively, +individual functions or lambdas may be prefixed with `HWY_ATTR`. + +Immediates (compile-time constants) are specified as template arguments to avoid +constant-propagation issues with Clang on ARM. + +## Type traits + +* `IsFloat<T>()` returns true if the `T` is a floating-point type. +* `IsSigned<T>()` returns true if the `T` is a signed or floating-point type. +* `LimitsMin/Max<T>()` return the smallest/largest value representable in + integer `T`. +* `SizeTag<N>` is an empty struct, used to select overloaded functions + appropriate for `N` bytes. + +## Memory allocation + +`AllocateAligned<T>(items)` returns a unique pointer to newly allocated memory +for `items` elements of POD type `T`. The start address is aligned as required +by `Load/Store`. Furthermore, successive allocations are not congruent modulo a +platform-specific alignment. This helps prevent false dependencies or cache +conflicts. The memory allocation is analogous to using `malloc()` and `free()` +with a `std::unique_ptr` since the returned items are *not* initialized or +default constructed and it is released using `FreeAlignedBytes()` without +calling `~T()`. + +`MakeUniqueAligned<T>(Args&&... args)` creates a single object in newly +allocated aligned memory as above but constructed passing the `args` argument to +`T`'s constructor and returning a unique pointer to it. This is analogous to +using `std::make_unique` with `new` but for aligned memory since the object is +constructed and later destructed when the unique pointer is deleted. Typically +this type `T` is a struct containing multiple members with `HWY_ALIGN` or +`HWY_ALIGN_MAX`, or arrays whose lengths are known to be a multiple of the +vector size. + +`MakeUniqueAlignedArray<T>(size_t items, Args&&... args)` creates an array of +objects in newly allocated aligned memory as above and constructs every element +of the new array using the passed constructor parameters, returning a unique +pointer to the array. Note that only the first element is guaranteed to be +aligned to the vector size; because there is no padding between elements, +the alignment of the remaining elements depends on the size of `T`. diff --git a/third_party/highway/g3doc/release_testing_process.md b/third_party/highway/g3doc/release_testing_process.md new file mode 100644 index 0000000..589f816 --- /dev/null +++ b/third_party/highway/g3doc/release_testing_process.md @@ -0,0 +1,37 @@ +## Release testing process + +We run the following before a release: + +### Windows x86 + +``` +run_tests.bat +``` + +### Linux x86 + +#### Clang, GCC, ARM cross compile + +``` +./run_tests.sh +``` + +#### JPEG XL clang (debug, asan, msan) + +``` +for VER in 7 8 9 10 11; do + rm -rf build_debug$VER && CC=clang-$VER CXX=clang++-$VER BUILD_DIR=build_debug$VER SKIP_TEST=1 ./ci.sh debug && ./ci.sh test -R PassesTest && rm -rf build_debug$VER + rm -rf build_asan$VER && CC=clang-$VER CXX=clang++-$VER BUILD_DIR=build_asan$VER ./ci.sh asan && rm -rf build_asan$VER + rm -rf build_msan$VER && CC=clang-$VER CXX=clang++-$VER BUILD_DIR=build_msan$VER ./ci.sh msan && rm -rf build_msan$VER +done +``` + +#### JPEG XL tests + +``` +git -C third_party/highway pull -r origin master +git diff +vi deps.sh +git commit -a -m"Highway test" +git push git@github.com:$USER/libjxl.git HEAD:main --force +``` diff --git a/third_party/highway/hwy/aligned_allocator.cc b/third_party/highway/hwy/aligned_allocator.cc new file mode 100644 index 0000000..8211952 --- /dev/null +++ b/third_party/highway/hwy/aligned_allocator.cc @@ -0,0 +1,150 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "hwy/aligned_allocator.h" + +#include <stdarg.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> // malloc + +#include <atomic> +#include <limits> + +#include "hwy/base.h" + +namespace hwy { +namespace { + +#if HWY_ARCH_RVV && defined(__riscv_vector) +// Not actually an upper bound on the size, but this value prevents crossing a +// 4K boundary (relevant on Andes). +constexpr size_t kAlignment = HWY_MAX(HWY_ALIGNMENT, 4096); +#else +constexpr size_t kAlignment = HWY_ALIGNMENT; +#endif + +#if HWY_ARCH_X86 +// On x86, aliasing can only occur at multiples of 2K, but that's too wasteful +// if this is used for single-vector allocations. 256 is more reasonable. +constexpr size_t kAlias = kAlignment * 4; +#else +constexpr size_t kAlias = kAlignment; +#endif + +#pragma pack(push, 1) +struct AllocationHeader { + void* allocated; + size_t payload_size; +}; +#pragma pack(pop) + +// Returns a 'random' (cyclical) offset for AllocateAlignedBytes. +size_t NextAlignedOffset() { + static std::atomic<uint32_t> next{0}; + constexpr uint32_t kGroups = kAlias / kAlignment; + const uint32_t group = next.fetch_add(1, std::memory_order_relaxed) % kGroups; + const size_t offset = kAlignment * group; + HWY_DASSERT((offset % kAlignment == 0) && offset <= kAlias); + return offset; +} + +} // namespace + +void* AllocateAlignedBytes(const size_t payload_size, AllocPtr alloc_ptr, + void* opaque_ptr) { + HWY_ASSERT(payload_size != 0); // likely a bug in caller + if (payload_size >= std::numeric_limits<size_t>::max() / 2) { + HWY_DASSERT(false && "payload_size too large"); + return nullptr; + } + + size_t offset = NextAlignedOffset(); + + // What: | misalign | unused | AllocationHeader |payload + // Size: |<= kAlias | offset |payload_size + // ^allocated.^aligned.^header............^payload + // The header must immediately precede payload, which must remain aligned. + // To avoid wasting space, the header resides at the end of `unused`, + // which therefore cannot be empty (offset == 0). + if (offset == 0) { + offset = kAlignment; // = RoundUpTo(sizeof(AllocationHeader), kAlignment) + static_assert(sizeof(AllocationHeader) <= kAlignment, "Else: round up"); + } + + const size_t allocated_size = kAlias + offset + payload_size; + void* allocated; + if (alloc_ptr == nullptr) { + allocated = malloc(allocated_size); + } else { + allocated = (*alloc_ptr)(opaque_ptr, allocated_size); + } + if (allocated == nullptr) return nullptr; + // Always round up even if already aligned - we already asked for kAlias + // extra bytes and there's no way to give them back. + uintptr_t aligned = reinterpret_cast<uintptr_t>(allocated) + kAlias; + static_assert((kAlias & (kAlias - 1)) == 0, "kAlias must be a power of 2"); + static_assert(kAlias >= kAlignment, "Cannot align to more than kAlias"); + aligned &= ~(kAlias - 1); + + const uintptr_t payload = aligned + offset; // still aligned + + // Stash `allocated` and payload_size inside header for FreeAlignedBytes(). + // The allocated_size can be reconstructed from the payload_size. + AllocationHeader* header = reinterpret_cast<AllocationHeader*>(payload) - 1; + header->allocated = allocated; + header->payload_size = payload_size; + + return HWY_ASSUME_ALIGNED(reinterpret_cast<void*>(payload), kAlignment); +} + +void FreeAlignedBytes(const void* aligned_pointer, FreePtr free_ptr, + void* opaque_ptr) { + if (aligned_pointer == nullptr) return; + + const uintptr_t payload = reinterpret_cast<uintptr_t>(aligned_pointer); + HWY_DASSERT(payload % kAlignment == 0); + const AllocationHeader* header = + reinterpret_cast<const AllocationHeader*>(payload) - 1; + + if (free_ptr == nullptr) { + free(header->allocated); + } else { + (*free_ptr)(opaque_ptr, header->allocated); + } +} + +// static +void AlignedDeleter::DeleteAlignedArray(void* aligned_pointer, FreePtr free_ptr, + void* opaque_ptr, + ArrayDeleter deleter) { + if (aligned_pointer == nullptr) return; + + const uintptr_t payload = reinterpret_cast<uintptr_t>(aligned_pointer); + HWY_DASSERT(payload % kAlignment == 0); + const AllocationHeader* header = + reinterpret_cast<const AllocationHeader*>(payload) - 1; + + if (deleter) { + (*deleter)(aligned_pointer, header->payload_size); + } + + if (free_ptr == nullptr) { + free(header->allocated); + } else { + (*free_ptr)(opaque_ptr, header->allocated); + } +} + +} // namespace hwy diff --git a/third_party/highway/hwy/aligned_allocator.h b/third_party/highway/hwy/aligned_allocator.h new file mode 100644 index 0000000..1e76cef --- /dev/null +++ b/third_party/highway/hwy/aligned_allocator.h @@ -0,0 +1,206 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef HIGHWAY_HWY_ALIGNED_ALLOCATOR_H_ +#define HIGHWAY_HWY_ALIGNED_ALLOCATOR_H_ + +// Memory allocator with support for alignment and offsets. + +#include <stddef.h> +#include <memory> + +namespace hwy { + +// Minimum alignment of allocated memory for use in HWY_ASSUME_ALIGNED, which +// requires a literal. This matches typical L1 cache line sizes, which prevents +// false sharing. +#define HWY_ALIGNMENT 64 + +// Pointers to functions equivalent to malloc/free with an opaque void* passed +// to them. +using AllocPtr = void* (*)(void* opaque, size_t bytes); +using FreePtr = void (*)(void* opaque, void* memory); + +// Returns null or a pointer to at least `payload_size` (which can be zero) +// bytes of newly allocated memory, aligned to the larger of HWY_ALIGNMENT and +// the vector size. Calls `alloc` with the passed `opaque` pointer to obtain +// memory or malloc() if it is null. +void* AllocateAlignedBytes(size_t payload_size, AllocPtr alloc_ptr, + void* opaque_ptr); + +// Frees all memory. No effect if `aligned_pointer` == nullptr, otherwise it +// must have been returned from a previous call to `AllocateAlignedBytes`. +// Calls `free_ptr` with the passed `opaque_ptr` pointer to free the memory; if +// `free_ptr` function is null, uses the default free(). +void FreeAlignedBytes(const void* aligned_pointer, FreePtr free_ptr, + void* opaque_ptr); + +// Class that deletes the aligned pointer passed to operator() calling the +// destructor before freeing the pointer. This is equivalent to the +// std::default_delete but for aligned objects. For a similar deleter equivalent +// to free() for aligned memory see AlignedFreer(). +class AlignedDeleter { + public: + AlignedDeleter() : free_(nullptr), opaque_ptr_(nullptr) {} + AlignedDeleter(FreePtr free_ptr, void* opaque_ptr) + : free_(free_ptr), opaque_ptr_(opaque_ptr) {} + + template <typename T> + void operator()(T* aligned_pointer) const { + return DeleteAlignedArray(aligned_pointer, free_, opaque_ptr_, + TypedArrayDeleter<T>); + } + + private: + template <typename T> + static void TypedArrayDeleter(void* ptr, size_t size_in_bytes) { + size_t elems = size_in_bytes / sizeof(T); + for (size_t i = 0; i < elems; i++) { + // Explicitly call the destructor on each element. + (static_cast<T*>(ptr) + i)->~T(); + } + } + + // Function prototype that calls the destructor for each element in a typed + // array. TypeArrayDeleter<T> would match this prototype. + using ArrayDeleter = void (*)(void* t_ptr, size_t t_size); + + static void DeleteAlignedArray(void* aligned_pointer, FreePtr free_ptr, + void* opaque_ptr, ArrayDeleter deleter); + + FreePtr free_; + void* opaque_ptr_; +}; + +// Unique pointer to T with custom aligned deleter. This can be a single +// element U or an array of element if T is a U[]. The custom aligned deleter +// will call the destructor on U or each element of a U[] in the array case. +template <typename T> +using AlignedUniquePtr = std::unique_ptr<T, AlignedDeleter>; + +// Aligned memory equivalent of make_unique<T> using the custom allocators +// alloc/free with the passed `opaque` pointer. This function calls the +// constructor with the passed Args... and calls the destructor of the object +// when the AlignedUniquePtr is destroyed. +template <typename T, typename... Args> +AlignedUniquePtr<T> MakeUniqueAlignedWithAlloc(AllocPtr alloc, FreePtr free, + void* opaque, Args&&... args) { + T* ptr = static_cast<T*>(AllocateAlignedBytes(sizeof(T), alloc, opaque)); + return AlignedUniquePtr<T>(new (ptr) T(std::forward<Args>(args)...), + AlignedDeleter(free, opaque)); +} + +// Similar to MakeUniqueAlignedWithAlloc but using the default alloc/free +// functions. +template <typename T, typename... Args> +AlignedUniquePtr<T> MakeUniqueAligned(Args&&... args) { + T* ptr = static_cast<T*>(AllocateAlignedBytes( + sizeof(T), /*alloc_ptr=*/nullptr, /*opaque_ptr=*/nullptr)); + return AlignedUniquePtr<T>( + new (ptr) T(std::forward<Args>(args)...), AlignedDeleter()); +} + +// Helpers for array allocators (avoids overflow) +namespace detail { + +// Returns x such that 1u << x == n (if n is a power of two). +static inline constexpr size_t ShiftCount(size_t n) { + return (n <= 1) ? 0 : 1 + ShiftCount(n / 2); +} + +template <typename T> +T* AllocateAlignedItems(size_t items, AllocPtr alloc_ptr, void* opaque_ptr) { + constexpr size_t size = sizeof(T); + + constexpr bool is_pow2 = (size & (size - 1)) == 0; + constexpr size_t bits = ShiftCount(size); + static_assert(!is_pow2 || (1ull << bits) == size, "ShiftCount is incorrect"); + + const size_t bytes = is_pow2 ? items << bits : items * size; + const size_t check = is_pow2 ? bytes >> bits : bytes / size; + if (check != items) { + return nullptr; // overflowed + } + return static_cast<T*>(AllocateAlignedBytes(bytes, alloc_ptr, opaque_ptr)); +} + +} // namespace detail + +// Aligned memory equivalent of make_unique<T[]> for array types using the +// custom allocators alloc/free. This function calls the constructor with the +// passed Args... on every created item. The destructor of each element will be +// called when the AlignedUniquePtr is destroyed. +template <typename T, typename... Args> +AlignedUniquePtr<T[]> MakeUniqueAlignedArrayWithAlloc( + size_t items, AllocPtr alloc, FreePtr free, void* opaque, Args&&... args) { + T* ptr = detail::AllocateAlignedItems<T>(items, alloc, opaque); + if (ptr != nullptr) { + for (size_t i = 0; i < items; i++) { + new (ptr + i) T(std::forward<Args>(args)...); + } + } + return AlignedUniquePtr<T[]>(ptr, AlignedDeleter(free, opaque)); +} + +template <typename T, typename... Args> +AlignedUniquePtr<T[]> MakeUniqueAlignedArray(size_t items, Args&&... args) { + return MakeUniqueAlignedArrayWithAlloc<T, Args...>( + items, nullptr, nullptr, nullptr, std::forward<Args>(args)...); +} + +// Custom deleter for std::unique_ptr equivalent to using free() as a deleter +// but for aligned memory. +class AlignedFreer { + public: + // Pass address of this to ctor to skip deleting externally-owned memory. + static void DoNothing(void* /*opaque*/, void* /*aligned_pointer*/) {} + + AlignedFreer() : free_(nullptr), opaque_ptr_(nullptr) {} + AlignedFreer(FreePtr free_ptr, void* opaque_ptr) + : free_(free_ptr), opaque_ptr_(opaque_ptr) {} + + template <typename T> + void operator()(T* aligned_pointer) const { + // TODO(deymo): assert that we are using a POD type T. + FreeAlignedBytes(aligned_pointer, free_, opaque_ptr_); + } + + private: + FreePtr free_; + void* opaque_ptr_; +}; + +// Unique pointer to single POD, or (if T is U[]) an array of POD. For non POD +// data use AlignedUniquePtr. +template <typename T> +using AlignedFreeUniquePtr = std::unique_ptr<T, AlignedFreer>; + +// Allocate an aligned and uninitialized array of POD values as a unique_ptr. +// Upon destruction of the unique_ptr the aligned array will be freed. +template <typename T> +AlignedFreeUniquePtr<T[]> AllocateAligned(const size_t items, AllocPtr alloc, + FreePtr free, void* opaque) { + return AlignedFreeUniquePtr<T[]>( + detail::AllocateAlignedItems<T>(items, alloc, opaque), + AlignedFreer(free, opaque)); +} + +// Same as previous AllocateAligned(), using default allocate/free functions. +template <typename T> +AlignedFreeUniquePtr<T[]> AllocateAligned(const size_t items) { + return AllocateAligned<T>(items, nullptr, nullptr, nullptr); +} + +} // namespace hwy +#endif // HIGHWAY_HWY_ALIGNED_ALLOCATOR_H_ diff --git a/third_party/highway/hwy/aligned_allocator_test.cc b/third_party/highway/hwy/aligned_allocator_test.cc new file mode 100644 index 0000000..c5c3cb9 --- /dev/null +++ b/third_party/highway/hwy/aligned_allocator_test.cc @@ -0,0 +1,284 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "hwy/aligned_allocator.h" + +#include <stddef.h> + +#include <array> +#include <new> +#include <random> +#include <vector> + +#include "gtest/gtest.h" +#include "hwy/base.h" + +namespace { + +// Sample object that keeps track on an external counter of how many times was +// the explicit constructor and destructor called. +template <size_t N> +class SampleObject { + public: + SampleObject() { data_[0] = 'a'; } + explicit SampleObject(int* counter) : counter_(counter) { + if (counter) (*counter)++; + data_[0] = 'b'; + } + + ~SampleObject() { + if (counter_) (*counter_)--; + } + + static_assert(N > sizeof(int*), "SampleObject size too small."); + int* counter_ = nullptr; + char data_[N - sizeof(int*)]; +}; + +class FakeAllocator { + public: + // static AllocPtr and FreePtr member to be used with the alligned + // allocator. These functions calls the private non-static members. + static void* StaticAlloc(void* opaque, size_t bytes) { + return reinterpret_cast<FakeAllocator*>(opaque)->Alloc(bytes); + } + static void StaticFree(void* opaque, void* memory) { + return reinterpret_cast<FakeAllocator*>(opaque)->Free(memory); + } + + // Returns the number of pending allocations to be freed. + size_t PendingAllocs() { return allocs_.size(); } + + private: + void* Alloc(size_t bytes) { + void* ret = malloc(bytes); + allocs_.insert(ret); + return ret; + } + void Free(void* memory) { + if (!memory) return; + EXPECT_NE(allocs_.end(), allocs_.find(memory)); + free(memory); + allocs_.erase(memory); + } + + std::set<void*> allocs_; +}; + +} // namespace + +namespace hwy { + +class AlignedAllocatorTest : public testing::Test {}; + +TEST(AlignedAllocatorTest, FreeNullptr) { + // Calling free with a nullptr is always ok. + FreeAlignedBytes(/*aligned_pointer=*/nullptr, /*free_ptr=*/nullptr, + /*opaque_ptr=*/nullptr); +} + +TEST(AlignedAllocatorTest, Log2) { + EXPECT_EQ(0u, detail::ShiftCount(1)); + EXPECT_EQ(1u, detail::ShiftCount(2)); + EXPECT_EQ(3u, detail::ShiftCount(8)); +} + +// Allocator returns null when it detects overflow of items * sizeof(T). +TEST(AlignedAllocatorTest, Overflow) { + constexpr size_t max = ~size_t(0); + constexpr size_t msb = (max >> 1) + 1; + using Size5 = std::array<uint8_t, 5>; + using Size10 = std::array<uint8_t, 10>; + EXPECT_EQ(nullptr, + detail::AllocateAlignedItems<uint32_t>(max / 2, nullptr, nullptr)); + EXPECT_EQ(nullptr, + detail::AllocateAlignedItems<uint32_t>(max / 3, nullptr, nullptr)); + EXPECT_EQ(nullptr, + detail::AllocateAlignedItems<Size5>(max / 4, nullptr, nullptr)); + EXPECT_EQ(nullptr, + detail::AllocateAlignedItems<uint16_t>(msb, nullptr, nullptr)); + EXPECT_EQ(nullptr, + detail::AllocateAlignedItems<double>(msb + 1, nullptr, nullptr)); + EXPECT_EQ(nullptr, + detail::AllocateAlignedItems<Size10>(msb / 4, nullptr, nullptr)); +} + +TEST(AlignedAllocatorTest, AllocDefaultPointers) { + const size_t kSize = 7777; + void* ptr = AllocateAlignedBytes(kSize, /*alloc_ptr=*/nullptr, + /*opaque_ptr=*/nullptr); + ASSERT_NE(nullptr, ptr); + // Make sure the pointer is actually aligned. + EXPECT_EQ(0U, reinterpret_cast<uintptr_t>(ptr) % HWY_ALIGNMENT); + char* p = static_cast<char*>(ptr); + size_t ret = 0; + for (size_t i = 0; i < kSize; i++) { + // Performs a computation using p[] to prevent it being optimized away. + p[i] = static_cast<char>(i & 0x7F); + if (i) ret += static_cast<size_t>(p[i] * p[i - 1]); + } + EXPECT_NE(0U, ret); + FreeAlignedBytes(ptr, /*free_ptr=*/nullptr, /*opaque_ptr=*/nullptr); +} + +TEST(AlignedAllocatorTest, EmptyAlignedUniquePtr) { + AlignedUniquePtr<SampleObject<32>> ptr(nullptr, AlignedDeleter()); + AlignedUniquePtr<SampleObject<32>[]> arr(nullptr, AlignedDeleter()); +} + +TEST(AlignedAllocatorTest, EmptyAlignedFreeUniquePtr) { + AlignedFreeUniquePtr<SampleObject<32>> ptr(nullptr, AlignedFreer()); + AlignedFreeUniquePtr<SampleObject<32>[]> arr(nullptr, AlignedFreer()); +} + +TEST(AlignedAllocatorTest, CustomAlloc) { + FakeAllocator fake_alloc; + + const size_t kSize = 7777; + void* ptr = + AllocateAlignedBytes(kSize, &FakeAllocator::StaticAlloc, &fake_alloc); + ASSERT_NE(nullptr, ptr); + // We should have only requested one alloc from the allocator. + EXPECT_EQ(1U, fake_alloc.PendingAllocs()); + // Make sure the pointer is actually aligned. + EXPECT_EQ(0U, reinterpret_cast<uintptr_t>(ptr) % HWY_ALIGNMENT); + FreeAlignedBytes(ptr, &FakeAllocator::StaticFree, &fake_alloc); + EXPECT_EQ(0U, fake_alloc.PendingAllocs()); +} + +TEST(AlignedAllocatorTest, MakeUniqueAlignedDefaultConstructor) { + { + auto ptr = MakeUniqueAligned<SampleObject<24>>(); + // Default constructor sets the data_[0] to 'a'. + EXPECT_EQ('a', ptr->data_[0]); + EXPECT_EQ(nullptr, ptr->counter_); + } +} + +TEST(AlignedAllocatorTest, MakeUniqueAligned) { + int counter = 0; + { + // Creates the object, initializes it with the explicit constructor and + // returns an unique_ptr to it. + auto ptr = MakeUniqueAligned<SampleObject<24>>(&counter); + EXPECT_EQ(1, counter); + // Custom constructor sets the data_[0] to 'b'. + EXPECT_EQ('b', ptr->data_[0]); + } + EXPECT_EQ(0, counter); +} + +TEST(AlignedAllocatorTest, MakeUniqueAlignedArray) { + int counter = 0; + { + // Creates the array of objects and initializes them with the explicit + // constructor. + auto arr = MakeUniqueAlignedArray<SampleObject<24>>(7, &counter); + EXPECT_EQ(7, counter); + for (size_t i = 0; i < 7; i++) { + // Custom constructor sets the data_[0] to 'b'. + EXPECT_EQ('b', arr[i].data_[0]) << "Where i = " << i; + } + } + EXPECT_EQ(0, counter); +} + +TEST(AlignedAllocatorTest, AllocSingleInt) { + auto ptr = AllocateAligned<uint32_t>(1); + ASSERT_NE(nullptr, ptr.get()); + EXPECT_EQ(0U, reinterpret_cast<uintptr_t>(ptr.get()) % HWY_ALIGNMENT); + // Force delete of the unique_ptr now to check that it doesn't crash. + ptr.reset(nullptr); + EXPECT_EQ(nullptr, ptr.get()); +} + +TEST(AlignedAllocatorTest, AllocMultipleInt) { + const size_t kSize = 7777; + auto ptr = AllocateAligned<uint32_t>(kSize); + ASSERT_NE(nullptr, ptr.get()); + EXPECT_EQ(0U, reinterpret_cast<uintptr_t>(ptr.get()) % HWY_ALIGNMENT); + // ptr[i] is actually (*ptr.get())[i] which will use the operator[] of the + // underlying type chosen by AllocateAligned() for the std::unique_ptr. + EXPECT_EQ(&(ptr[0]) + 1, &(ptr[1])); + + size_t ret = 0; + for (size_t i = 0; i < kSize; i++) { + // Performs a computation using ptr[] to prevent it being optimized away. + ptr[i] = static_cast<uint32_t>(i); + if (i) ret += ptr[i] * ptr[i - 1]; + } + EXPECT_NE(0U, ret); +} + +TEST(AlignedAllocatorTest, AllocateAlignedObjectWithoutDestructor) { + int counter = 0; + { + // This doesn't call the constructor. + auto obj = AllocateAligned<SampleObject<24>>(1); + obj[0].counter_ = &counter; + } + // Destroying the unique_ptr shouldn't have called the destructor of the + // SampleObject<24>. + EXPECT_EQ(0, counter); +} + +TEST(AlignedAllocatorTest, MakeUniqueAlignedArrayWithCustomAlloc) { + FakeAllocator fake_alloc; + int counter = 0; + { + // Creates the array of objects and initializes them with the explicit + // constructor. + auto arr = MakeUniqueAlignedArrayWithAlloc<SampleObject<24>>( + 7, FakeAllocator::StaticAlloc, FakeAllocator::StaticFree, &fake_alloc, + &counter); + ASSERT_NE(nullptr, arr.get()); + // An array should still only call a single allocation. + EXPECT_EQ(1u, fake_alloc.PendingAllocs()); + EXPECT_EQ(7, counter); + for (size_t i = 0; i < 7; i++) { + // Custom constructor sets the data_[0] to 'b'. + EXPECT_EQ('b', arr[i].data_[0]) << "Where i = " << i; + } + } + EXPECT_EQ(0, counter); + EXPECT_EQ(0u, fake_alloc.PendingAllocs()); +} + +TEST(AlignedAllocatorTest, DefaultInit) { + // The test is whether this compiles. Default-init is useful for output params + // and per-thread storage. + std::vector<AlignedUniquePtr<int[]>> ptrs; + std::vector<AlignedFreeUniquePtr<double[]>> free_ptrs; + ptrs.resize(128); + free_ptrs.resize(128); + // The following is to prevent elision of the pointers. + std::mt19937 rng(129); // Emscripten lacks random_device. + std::uniform_int_distribution<size_t> dist(0, 127); + ptrs[dist(rng)] = MakeUniqueAlignedArray<int>(123); + free_ptrs[dist(rng)] = AllocateAligned<double>(456); + // "Use" pointer without resorting to printf. 0 == 0. Can't shift by 64. + const auto addr1 = reinterpret_cast<uintptr_t>(ptrs[dist(rng)].get()); + const auto addr2 = reinterpret_cast<uintptr_t>(free_ptrs[dist(rng)].get()); + constexpr size_t kBits = sizeof(uintptr_t) * 8; + EXPECT_EQ((addr1 >> (kBits - 1)) >> (kBits - 1), + (addr2 >> (kBits - 1)) >> (kBits - 1)); +} + +} // namespace hwy + +// Ought not to be necessary, but without this, no tests run on RVV. +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/third_party/highway/hwy/base.h b/third_party/highway/hwy/base.h new file mode 100644 index 0000000..009373f --- /dev/null +++ b/third_party/highway/hwy/base.h @@ -0,0 +1,735 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef HIGHWAY_HWY_BASE_H_ +#define HIGHWAY_HWY_BASE_H_ + +// For SIMD module implementations and their callers, target-independent. + +#include <stddef.h> +#include <stdint.h> + +#include <atomic> +#include <cfloat> + +#include "hwy/detect_compiler_arch.h" + +//------------------------------------------------------------------------------ +// Compiler-specific definitions + +#define HWY_STR_IMPL(macro) #macro +#define HWY_STR(macro) HWY_STR_IMPL(macro) + +#if HWY_COMPILER_MSVC + +#include <intrin.h> + +#define HWY_RESTRICT __restrict +#define HWY_INLINE __forceinline +#define HWY_NOINLINE __declspec(noinline) +#define HWY_FLATTEN +#define HWY_NORETURN __declspec(noreturn) +#define HWY_LIKELY(expr) (expr) +#define HWY_UNLIKELY(expr) (expr) +#define HWY_PRAGMA(tokens) __pragma(tokens) +#define HWY_DIAGNOSTICS(tokens) HWY_PRAGMA(warning(tokens)) +#define HWY_DIAGNOSTICS_OFF(msc, gcc) HWY_DIAGNOSTICS(msc) +#define HWY_MAYBE_UNUSED +#define HWY_HAS_ASSUME_ALIGNED 0 +#if (_MSC_VER >= 1700) +#define HWY_MUST_USE_RESULT _Check_return_ +#else +#define HWY_MUST_USE_RESULT +#endif + +#else + +#define HWY_RESTRICT __restrict__ +#define HWY_INLINE inline __attribute__((always_inline)) +#define HWY_NOINLINE __attribute__((noinline)) +#define HWY_FLATTEN __attribute__((flatten)) +#define HWY_NORETURN __attribute__((noreturn)) +#define HWY_LIKELY(expr) __builtin_expect(!!(expr), 1) +#define HWY_UNLIKELY(expr) __builtin_expect(!!(expr), 0) +#define HWY_PRAGMA(tokens) _Pragma(#tokens) +#define HWY_DIAGNOSTICS(tokens) HWY_PRAGMA(GCC diagnostic tokens) +#define HWY_DIAGNOSTICS_OFF(msc, gcc) HWY_DIAGNOSTICS(gcc) +// Encountered "attribute list cannot appear here" when using the C++17 +// [[maybe_unused]], so only use the old style attribute for now. +#define HWY_MAYBE_UNUSED __attribute__((unused)) +#define HWY_MUST_USE_RESULT __attribute__((warn_unused_result)) + +#endif // !HWY_COMPILER_MSVC + +//------------------------------------------------------------------------------ +// Builtin/attributes + +// Enables error-checking of format strings. +#if HWY_HAS_ATTRIBUTE(__format__) +#define HWY_FORMAT(idx_fmt, idx_arg) \ + __attribute__((__format__(__printf__, idx_fmt, idx_arg))) +#else +#define HWY_FORMAT(idx_fmt, idx_arg) +#endif + +// Returns a void* pointer which the compiler then assumes is N-byte aligned. +// Example: float* HWY_RESTRICT aligned = (float*)HWY_ASSUME_ALIGNED(in, 32); +// +// The assignment semantics are required by GCC/Clang. ICC provides an in-place +// __assume_aligned, whereas MSVC's __assume appears unsuitable. +#if HWY_HAS_BUILTIN(__builtin_assume_aligned) +#define HWY_ASSUME_ALIGNED(ptr, align) __builtin_assume_aligned((ptr), (align)) +#else +#define HWY_ASSUME_ALIGNED(ptr, align) (ptr) /* not supported */ +#endif + +// Clang and GCC require attributes on each function into which SIMD intrinsics +// are inlined. Support both per-function annotation (HWY_ATTR) for lambdas and +// automatic annotation via pragmas. +#if HWY_COMPILER_CLANG +#define HWY_PUSH_ATTRIBUTES(targets_str) \ + HWY_PRAGMA(clang attribute push(__attribute__((target(targets_str))), \ + apply_to = function)) +#define HWY_POP_ATTRIBUTES HWY_PRAGMA(clang attribute pop) +#elif HWY_COMPILER_GCC +#define HWY_PUSH_ATTRIBUTES(targets_str) \ + HWY_PRAGMA(GCC push_options) HWY_PRAGMA(GCC target targets_str) +#define HWY_POP_ATTRIBUTES HWY_PRAGMA(GCC pop_options) +#else +#define HWY_PUSH_ATTRIBUTES(targets_str) +#define HWY_POP_ATTRIBUTES +#endif + +//------------------------------------------------------------------------------ +// Macros + +#define HWY_API static HWY_INLINE HWY_FLATTEN HWY_MAYBE_UNUSED + +#define HWY_CONCAT_IMPL(a, b) a##b +#define HWY_CONCAT(a, b) HWY_CONCAT_IMPL(a, b) + +#define HWY_MIN(a, b) ((a) < (b) ? (a) : (b)) +#define HWY_MAX(a, b) ((a) > (b) ? (a) : (b)) + +// Compile-time fence to prevent undesirable code reordering. On Clang x86, the +// typical asm volatile("" : : : "memory") has no effect, whereas atomic fence +// does, without generating code. +#if HWY_ARCH_X86 +#define HWY_FENCE std::atomic_thread_fence(std::memory_order_acq_rel) +#else +// TODO(janwas): investigate alternatives. On ARM, the above generates barriers. +#define HWY_FENCE +#endif + +// 4 instances of a given literal value, useful as input to LoadDup128. +#define HWY_REP4(literal) literal, literal, literal, literal + +#define HWY_ABORT(format, ...) \ + ::hwy::Abort(__FILE__, __LINE__, format, ##__VA_ARGS__) + +// Always enabled. +#define HWY_ASSERT(condition) \ + do { \ + if (!(condition)) { \ + HWY_ABORT("Assert %s", #condition); \ + } \ + } while (0) + +#if HWY_HAS_FEATURE(memory_sanitizer) || defined(MEMORY_SANITIZER) +#define HWY_IS_MSAN 1 +#else +#define HWY_IS_MSAN 0 +#endif + +#if HWY_HAS_FEATURE(address_sanitizer) || defined(ADDRESS_SANITIZER) +#define HWY_IS_ASAN 1 +#else +#define HWY_IS_ASAN 0 +#endif + +#if HWY_HAS_FEATURE(thread_sanitizer) || defined(THREAD_SANITIZER) +#define HWY_IS_TSAN 1 +#else +#define HWY_IS_TSAN 0 +#endif + +// For enabling HWY_DASSERT and shortening tests in slower debug builds +#if !defined(HWY_IS_DEBUG_BUILD) +// Clang does not define NDEBUG, but it and GCC define __OPTIMIZE__, and recent +// MSVC defines NDEBUG (if not, could instead check _DEBUG). +#if (!defined(__OPTIMIZE__) && !defined(NDEBUG)) || HWY_IS_ASAN || \ + HWY_IS_MSAN || HWY_IS_TSAN || defined(__clang_analyzer__) +#define HWY_IS_DEBUG_BUILD 1 +#else +#define HWY_IS_DEBUG_BUILD 0 +#endif +#endif // HWY_IS_DEBUG_BUILD + +#if HWY_IS_DEBUG_BUILD +#define HWY_DASSERT(condition) HWY_ASSERT(condition) +#else +#define HWY_DASSERT(condition) \ + do { \ + } while (0) +#endif + +#if defined(HWY_EMULATE_SVE) +class FarmFloat16; +#endif + +namespace hwy { + +//------------------------------------------------------------------------------ +// kMaxVectorSize (undocumented, pending removal) + +#if HWY_ARCH_X86 +static constexpr HWY_MAYBE_UNUSED size_t kMaxVectorSize = 64; // AVX-512 +#elif HWY_ARCH_RVV && defined(__riscv_vector) +// Not actually an upper bound on the size. +static constexpr HWY_MAYBE_UNUSED size_t kMaxVectorSize = 4096; +#else +static constexpr HWY_MAYBE_UNUSED size_t kMaxVectorSize = 16; +#endif + +//------------------------------------------------------------------------------ +// Alignment + +// For stack-allocated partial arrays or LoadDup128. +#if HWY_ARCH_X86 +#define HWY_ALIGN_MAX alignas(64) +#elif HWY_ARCH_RVV && defined(__riscv_vector) +#define HWY_ALIGN_MAX alignas(8) // only elements need be aligned +#else +#define HWY_ALIGN_MAX alignas(16) +#endif + +//------------------------------------------------------------------------------ +// Lane types + +// Match [u]int##_t naming scheme so rvv-inl.h macros can obtain the type name +// by concatenating base type and bits. + +#if HWY_ARCH_ARM && (__ARM_FP & 2) +#define HWY_NATIVE_FLOAT16 1 +#else +#define HWY_NATIVE_FLOAT16 0 +#endif + +#pragma pack(push, 1) + +#if defined(HWY_EMULATE_SVE) +using float16_t = FarmFloat16; +#elif HWY_NATIVE_FLOAT16 +using float16_t = __fp16; +// Clang does not allow __fp16 arguments, but scalar.h requires LaneType +// arguments, so use a wrapper. +// TODO(janwas): replace with _Float16 when that is supported? +#else +struct float16_t { + uint16_t bits; +}; +#endif + +struct bfloat16_t { + uint16_t bits; +}; + +#pragma pack(pop) + +using float32_t = float; +using float64_t = double; + +//------------------------------------------------------------------------------ +// Controlling overload resolution (SFINAE) + +template <bool Condition, class T> +struct EnableIfT {}; +template <class T> +struct EnableIfT<true, T> { + using type = T; +}; + +template <bool Condition, class T = void> +using EnableIf = typename EnableIfT<Condition, T>::type; + +template <typename T, typename U> +struct IsSameT { + enum { value = 0 }; +}; + +template <typename T> +struct IsSameT<T, T> { + enum { value = 1 }; +}; + +template <typename T, typename U> +HWY_API constexpr bool IsSame() { + return IsSameT<T, U>::value; +} + +// Insert into template/function arguments to enable this overload only for +// vectors of AT MOST this many bits. +// +// Note that enabling for exactly 128 bits is unnecessary because a function can +// simply be overloaded with Vec128<T> and/or Full128<T> tag. Enabling for other +// sizes (e.g. 64 bit) can be achieved via Simd<T, 8 / sizeof(T)>. +#define HWY_IF_LE128(T, N) hwy::EnableIf<N * sizeof(T) <= 16>* = nullptr +#define HWY_IF_LE64(T, N) hwy::EnableIf<N * sizeof(T) <= 8>* = nullptr +#define HWY_IF_LE32(T, N) hwy::EnableIf<N * sizeof(T) <= 4>* = nullptr +#define HWY_IF_GE32(T, N) hwy::EnableIf<N * sizeof(T) >= 4>* = nullptr +#define HWY_IF_GE64(T, N) hwy::EnableIf<N * sizeof(T) >= 8>* = nullptr +#define HWY_IF_GE128(T, N) hwy::EnableIf<N * sizeof(T) >= 16>* = nullptr +#define HWY_IF_GT128(T, N) hwy::EnableIf<(N * sizeof(T) > 16)>* = nullptr + +#define HWY_IF_UNSIGNED(T) hwy::EnableIf<!IsSigned<T>()>* = nullptr +#define HWY_IF_SIGNED(T) \ + hwy::EnableIf<IsSigned<T>() && !IsFloat<T>()>* = nullptr +#define HWY_IF_FLOAT(T) hwy::EnableIf<hwy::IsFloat<T>()>* = nullptr +#define HWY_IF_NOT_FLOAT(T) hwy::EnableIf<!hwy::IsFloat<T>()>* = nullptr + +#define HWY_IF_LANE_SIZE(T, bytes) \ + hwy::EnableIf<sizeof(T) == (bytes)>* = nullptr +#define HWY_IF_NOT_LANE_SIZE(T, bytes) \ + hwy::EnableIf<sizeof(T) != (bytes)>* = nullptr + +// Empty struct used as a size tag type. +template <size_t N> +struct SizeTag {}; + +template <class T> +struct RemoveConstT { + using type = T; +}; +template <class T> +struct RemoveConstT<const T> { + using type = T; +}; + +template <class T> +using RemoveConst = typename RemoveConstT<T>::type; + +//------------------------------------------------------------------------------ +// Type traits + +template <typename T> +HWY_API constexpr bool IsFloat() { + // Cannot use T(1.25) != T(1) for float16_t, which can only be converted to or + // from a float, not compared. + return IsSame<T, float>() || IsSame<T, double>(); +} + +template <typename T> +HWY_API constexpr bool IsSigned() { + return T(0) > T(-1); +} +template <> +constexpr bool IsSigned<float16_t>() { + return true; +} +template <> +constexpr bool IsSigned<bfloat16_t>() { + return true; +} + +// Largest/smallest representable integer values. +template <typename T> +HWY_API constexpr T LimitsMax() { + static_assert(!IsFloat<T>(), "Only for integer types"); + return IsSigned<T>() ? T((1ULL << (sizeof(T) * 8 - 1)) - 1) + : static_cast<T>(~0ull); +} +template <typename T> +HWY_API constexpr T LimitsMin() { + static_assert(!IsFloat<T>(), "Only for integer types"); + return IsSigned<T>() ? T(-1) - LimitsMax<T>() : T(0); +} + +// Largest/smallest representable value (integer or float). This naming avoids +// confusion with numeric_limits<float>::min() (the smallest positive value). +template <typename T> +HWY_API constexpr T LowestValue() { + return LimitsMin<T>(); +} +template <> +constexpr float LowestValue<float>() { + return -FLT_MAX; +} +template <> +constexpr double LowestValue<double>() { + return -DBL_MAX; +} + +template <typename T> +HWY_API constexpr T HighestValue() { + return LimitsMax<T>(); +} +template <> +constexpr float HighestValue<float>() { + return FLT_MAX; +} +template <> +constexpr double HighestValue<double>() { + return DBL_MAX; +} + +// Returns bitmask of the exponent field in IEEE binary32/64. +template <typename T> +constexpr T ExponentMask() { + static_assert(sizeof(T) == 0, "Only instantiate the specializations"); + return 0; +} +template <> +constexpr uint32_t ExponentMask<uint32_t>() { + return 0x7F800000; +} +template <> +constexpr uint64_t ExponentMask<uint64_t>() { + return 0x7FF0000000000000ULL; +} + +// Returns 1 << mantissa_bits as a floating-point number. All integers whose +// absolute value are less than this can be represented exactly. +template <typename T> +constexpr T MantissaEnd() { + static_assert(sizeof(T) == 0, "Only instantiate the specializations"); + return 0; +} +template <> +constexpr float MantissaEnd<float>() { + return 8388608.0f; // 1 << 23 +} +template <> +constexpr double MantissaEnd<double>() { + // floating point literal with p52 requires C++17. + return 4503599627370496.0; // 1 << 52 +} + +//------------------------------------------------------------------------------ +// Type relations + +namespace detail { + +template <typename T> +struct Relations; +template <> +struct Relations<uint8_t> { + using Unsigned = uint8_t; + using Signed = int8_t; + using Wide = uint16_t; +}; +template <> +struct Relations<int8_t> { + using Unsigned = uint8_t; + using Signed = int8_t; + using Wide = int16_t; +}; +template <> +struct Relations<uint16_t> { + using Unsigned = uint16_t; + using Signed = int16_t; + using Wide = uint32_t; + using Narrow = uint8_t; +}; +template <> +struct Relations<int16_t> { + using Unsigned = uint16_t; + using Signed = int16_t; + using Wide = int32_t; + using Narrow = int8_t; +}; +template <> +struct Relations<uint32_t> { + using Unsigned = uint32_t; + using Signed = int32_t; + using Float = float; + using Wide = uint64_t; + using Narrow = uint16_t; +}; +template <> +struct Relations<int32_t> { + using Unsigned = uint32_t; + using Signed = int32_t; + using Float = float; + using Wide = int64_t; + using Narrow = int16_t; +}; +template <> +struct Relations<uint64_t> { + using Unsigned = uint64_t; + using Signed = int64_t; + using Float = double; + using Narrow = uint32_t; +}; +template <> +struct Relations<int64_t> { + using Unsigned = uint64_t; + using Signed = int64_t; + using Float = double; + using Narrow = int32_t; +}; +template <> +struct Relations<float16_t> { + using Unsigned = uint16_t; + using Signed = int16_t; + using Float = float16_t; + using Wide = float; +}; +template <> +struct Relations<bfloat16_t> { + using Unsigned = uint16_t; + using Signed = int16_t; + using Wide = float; +}; +template <> +struct Relations<float> { + using Unsigned = uint32_t; + using Signed = int32_t; + using Float = float; + using Wide = double; + using Narrow = float16_t; +}; +template <> +struct Relations<double> { + using Unsigned = uint64_t; + using Signed = int64_t; + using Float = double; + using Narrow = float; +}; + +template <size_t N> +struct TypeFromSize; +template <> +struct TypeFromSize<1> { + using Unsigned = uint8_t; + using Signed = int8_t; +}; +template <> +struct TypeFromSize<2> { + using Unsigned = uint16_t; + using Signed = int16_t; +}; +template <> +struct TypeFromSize<4> { + using Unsigned = uint32_t; + using Signed = int32_t; + using Float = float; +}; +template <> +struct TypeFromSize<8> { + using Unsigned = uint64_t; + using Signed = int64_t; + using Float = double; +}; + +} // namespace detail + +// Aliases for types of a different category, but the same size. +template <typename T> +using MakeUnsigned = typename detail::Relations<T>::Unsigned; +template <typename T> +using MakeSigned = typename detail::Relations<T>::Signed; +template <typename T> +using MakeFloat = typename detail::Relations<T>::Float; + +// Aliases for types of the same category, but different size. +template <typename T> +using MakeWide = typename detail::Relations<T>::Wide; +template <typename T> +using MakeNarrow = typename detail::Relations<T>::Narrow; + +// Obtain type from its size [bytes]. +template <size_t N> +using UnsignedFromSize = typename detail::TypeFromSize<N>::Unsigned; +template <size_t N> +using SignedFromSize = typename detail::TypeFromSize<N>::Signed; +template <size_t N> +using FloatFromSize = typename detail::TypeFromSize<N>::Float; + +//------------------------------------------------------------------------------ +// Helper functions + +template <typename T1, typename T2> +constexpr inline T1 DivCeil(T1 a, T2 b) { + return (a + b - 1) / b; +} + +// Works for any `align`; if a power of two, compiler emits ADD+AND. +constexpr inline size_t RoundUpTo(size_t what, size_t align) { + return DivCeil(what, align) * align; +} + +// Undefined results for x == 0. +HWY_API size_t Num0BitsBelowLS1Bit_Nonzero32(const uint32_t x) { +#if HWY_COMPILER_MSVC + unsigned long index; // NOLINT + _BitScanForward(&index, x); + return index; +#else // HWY_COMPILER_MSVC + return static_cast<size_t>(__builtin_ctz(x)); +#endif // HWY_COMPILER_MSVC +} + +HWY_API size_t Num0BitsBelowLS1Bit_Nonzero64(const uint64_t x) { +#if HWY_COMPILER_MSVC +#if HWY_ARCH_X86_64 + unsigned long index; // NOLINT + _BitScanForward64(&index, x); + return index; +#else // HWY_ARCH_X86_64 + // _BitScanForward64 not available + uint32_t lsb = static_cast<uint32_t>(x & 0xFFFFFFFF); + unsigned long index; + if (lsb == 0) { + uint32_t msb = static_cast<uint32_t>(x >> 32u); + _BitScanForward(&index, msb); + return 32 + index; + } else { + _BitScanForward(&index, lsb); + return index; + } +#endif // HWY_ARCH_X86_64 +#else // HWY_COMPILER_MSVC + return static_cast<size_t>(__builtin_ctzll(x)); +#endif // HWY_COMPILER_MSVC +} + +// Undefined results for x == 0. +HWY_API size_t Num0BitsAboveMS1Bit_Nonzero32(const uint32_t x) { +#if HWY_COMPILER_MSVC + unsigned long index; // NOLINT + _BitScanReverse(&index, x); + return 31 - index; +#else // HWY_COMPILER_MSVC + return static_cast<size_t>(__builtin_clz(x)); +#endif // HWY_COMPILER_MSVC +} + +HWY_API size_t Num0BitsAboveMS1Bit_Nonzero64(const uint64_t x) { +#if HWY_COMPILER_MSVC +#if HWY_ARCH_X86_64 + unsigned long index; // NOLINT + _BitScanReverse64(&index, x); + return 63 - index; +#else // HWY_ARCH_X86_64 + // _BitScanReverse64 not available + const uint32_t msb = static_cast<uint32_t>(x >> 32u); + unsigned long index; + if (msb == 0) { + const uint32_t lsb = static_cast<uint32_t>(x & 0xFFFFFFFF); + _BitScanReverse(&index, lsb); + return 63 - index; + } else { + _BitScanReverse(&index, msb); + return 31 - index; + } +#endif // HWY_ARCH_X86_64 +#else // HWY_COMPILER_MSVC + return static_cast<size_t>(__builtin_clzll(x)); +#endif // HWY_COMPILER_MSVC +} + +HWY_API size_t PopCount(uint64_t x) { +#if HWY_COMPILER_CLANG || HWY_COMPILER_GCC + return static_cast<size_t>(__builtin_popcountll(x)); + // This instruction has a separate feature flag, but is often called from + // non-SIMD code, so we don't want to require dynamic dispatch. It was first + // supported by Intel in Nehalem (SSE4.2), but MSVC only predefines a macro + // for AVX, so check for that. +#elif HWY_COMPILER_MSVC && HWY_ARCH_X86_64 && defined(__AVX__) + return _mm_popcnt_u64(x); +#elif HWY_COMPILER_MSVC && HWY_ARCH_X86_32 && defined(__AVX__) + return _mm_popcnt_u32(uint32_t(x)) + _mm_popcnt_u32(uint32_t(x >> 32)); +#else + x -= ((x >> 1) & 0x5555555555555555ULL); + x = (((x >> 2) & 0x3333333333333333ULL) + (x & 0x3333333333333333ULL)); + x = (((x >> 4) + x) & 0x0F0F0F0F0F0F0F0FULL); + x += (x >> 8); + x += (x >> 16); + x += (x >> 32); + return static_cast<size_t>(x & 0x7Fu); +#endif +} + +template <typename TI> +HWY_API constexpr size_t FloorLog2(TI x) { + return x == 1 ? 0 : FloorLog2(x >> 1) + 1; +} + +template <typename TI> +HWY_API constexpr size_t CeilLog2(TI x) { + return x == 1 ? 0 : FloorLog2(x - 1) + 1; +} + +#if HWY_COMPILER_MSVC && HWY_ARCH_X86_64 +#pragma intrinsic(_umul128) +#endif + +// 64 x 64 = 128 bit multiplication +HWY_API uint64_t Mul128(uint64_t a, uint64_t b, uint64_t* HWY_RESTRICT upper) { +#if defined(__SIZEOF_INT128__) + __uint128_t product = (__uint128_t)a * (__uint128_t)b; + *upper = (uint64_t)(product >> 64); + return (uint64_t)(product & 0xFFFFFFFFFFFFFFFFULL); +#elif HWY_COMPILER_MSVC && HWY_ARCH_X86_64 + return _umul128(a, b, upper); +#else + constexpr uint64_t kLo32 = 0xFFFFFFFFU; + const uint64_t lo_lo = (a & kLo32) * (b & kLo32); + const uint64_t hi_lo = (a >> 32) * (b & kLo32); + const uint64_t lo_hi = (a & kLo32) * (b >> 32); + const uint64_t hi_hi = (a >> 32) * (b >> 32); + const uint64_t t = (lo_lo >> 32) + (hi_lo & kLo32) + lo_hi; + *upper = (hi_lo >> 32) + (t >> 32) + hi_hi; + return (t << 32) | (lo_lo & kLo32); +#endif +} + +// The source/destination must not overlap/alias. +template <size_t kBytes, typename From, typename To> +HWY_API void CopyBytes(const From* from, To* to) { +#if HWY_COMPILER_MSVC + const uint8_t* HWY_RESTRICT from_bytes = + reinterpret_cast<const uint8_t*>(from); + uint8_t* HWY_RESTRICT to_bytes = reinterpret_cast<uint8_t*>(to); + for (size_t i = 0; i < kBytes; ++i) { + to_bytes[i] = from_bytes[i]; + } +#else + // Avoids horrible codegen on Clang (series of PINSRB) + __builtin_memcpy(to, from, kBytes); +#endif +} + +HWY_API float F32FromBF16(bfloat16_t bf) { + uint32_t bits = bf.bits; + bits <<= 16; + float f; + CopyBytes<4>(&bits, &f); + return f; +} + +HWY_API bfloat16_t BF16FromF32(float f) { + uint32_t bits; + CopyBytes<4>(&f, &bits); + bfloat16_t bf; + bf.bits = static_cast<uint16_t>(bits >> 16); + return bf; +} + +HWY_NORETURN void HWY_FORMAT(3, 4) + Abort(const char* file, int line, const char* format, ...); + +} // namespace hwy + +#endif // HIGHWAY_HWY_BASE_H_ diff --git a/third_party/highway/hwy/base_test.cc b/third_party/highway/hwy/base_test.cc new file mode 100644 index 0000000..4eb9d7d --- /dev/null +++ b/third_party/highway/hwy/base_test.cc @@ -0,0 +1,178 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <stddef.h> +#include <stdint.h> + +#include <limits> + +#include "hwy/base.h" + +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "base_test.cc" +#include "hwy/foreach_target.h" +#include "hwy/highway.h" +#include "hwy/tests/test_util-inl.h" + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +HWY_NOINLINE void TestAllLimits() { + HWY_ASSERT_EQ(uint8_t(0), LimitsMin<uint8_t>()); + HWY_ASSERT_EQ(uint16_t(0), LimitsMin<uint16_t>()); + HWY_ASSERT_EQ(uint32_t(0), LimitsMin<uint32_t>()); + HWY_ASSERT_EQ(uint64_t(0), LimitsMin<uint64_t>()); + + HWY_ASSERT_EQ(int8_t(-128), LimitsMin<int8_t>()); + HWY_ASSERT_EQ(int16_t(-32768), LimitsMin<int16_t>()); + HWY_ASSERT_EQ(int32_t(0x80000000u), LimitsMin<int32_t>()); + HWY_ASSERT_EQ(int64_t(0x8000000000000000ull), LimitsMin<int64_t>()); + + HWY_ASSERT_EQ(uint8_t(0xFF), LimitsMax<uint8_t>()); + HWY_ASSERT_EQ(uint16_t(0xFFFF), LimitsMax<uint16_t>()); + HWY_ASSERT_EQ(uint32_t(0xFFFFFFFFu), LimitsMax<uint32_t>()); + HWY_ASSERT_EQ(uint64_t(0xFFFFFFFFFFFFFFFFull), LimitsMax<uint64_t>()); + + HWY_ASSERT_EQ(int8_t(0x7F), LimitsMax<int8_t>()); + HWY_ASSERT_EQ(int16_t(0x7FFF), LimitsMax<int16_t>()); + HWY_ASSERT_EQ(int32_t(0x7FFFFFFFu), LimitsMax<int32_t>()); + HWY_ASSERT_EQ(int64_t(0x7FFFFFFFFFFFFFFFull), LimitsMax<int64_t>()); +} + +struct TestLowestHighest { + template <class T> + HWY_NOINLINE void operator()(T /*unused*/) const { + HWY_ASSERT_EQ(std::numeric_limits<T>::lowest(), LowestValue<T>()); + HWY_ASSERT_EQ(std::numeric_limits<T>::max(), HighestValue<T>()); + } +}; + +HWY_NOINLINE void TestAllLowestHighest() { ForAllTypes(TestLowestHighest()); } +struct TestIsUnsigned { + template <class T> + HWY_NOINLINE void operator()(T /*unused*/) const { + static_assert(!IsFloat<T>(), "Expected !IsFloat"); + static_assert(!IsSigned<T>(), "Expected !IsSigned"); + } +}; + +struct TestIsSigned { + template <class T> + HWY_NOINLINE void operator()(T /*unused*/) const { + static_assert(!IsFloat<T>(), "Expected !IsFloat"); + static_assert(IsSigned<T>(), "Expected IsSigned"); + } +}; + +struct TestIsFloat { + template <class T> + HWY_NOINLINE void operator()(T /*unused*/) const { + static_assert(IsFloat<T>(), "Expected IsFloat"); + static_assert(IsSigned<T>(), "Floats are also considered signed"); + } +}; + +HWY_NOINLINE void TestAllType() { + ForUnsignedTypes(TestIsUnsigned()); + ForSignedTypes(TestIsSigned()); + ForFloatTypes(TestIsFloat()); +} + +struct TestIsSame { + template <class T> + HWY_NOINLINE void operator()(T /*unused*/) const { + static_assert(IsSame<T, T>(), "T == T"); + static_assert(!IsSame<MakeSigned<T>, MakeUnsigned<T>>(), "S != U"); + static_assert(!IsSame<MakeUnsigned<T>, MakeSigned<T>>(), "U != S"); + } +}; + +HWY_NOINLINE void TestAllIsSame() { ForAllTypes(TestIsSame()); } + +HWY_NOINLINE void TestAllBitScan() { + HWY_ASSERT_EQ(size_t(0), Num0BitsAboveMS1Bit_Nonzero32(0x80000000u)); + HWY_ASSERT_EQ(size_t(0), Num0BitsAboveMS1Bit_Nonzero32(0xFFFFFFFFu)); + HWY_ASSERT_EQ(size_t(1), Num0BitsAboveMS1Bit_Nonzero32(0x40000000u)); + HWY_ASSERT_EQ(size_t(1), Num0BitsAboveMS1Bit_Nonzero32(0x40108210u)); + HWY_ASSERT_EQ(size_t(30), Num0BitsAboveMS1Bit_Nonzero32(2u)); + HWY_ASSERT_EQ(size_t(30), Num0BitsAboveMS1Bit_Nonzero32(3u)); + HWY_ASSERT_EQ(size_t(31), Num0BitsAboveMS1Bit_Nonzero32(1u)); + + HWY_ASSERT_EQ(size_t(0), + Num0BitsAboveMS1Bit_Nonzero64(0x8000000000000000ull)); + HWY_ASSERT_EQ(size_t(0), + Num0BitsAboveMS1Bit_Nonzero64(0xFFFFFFFFFFFFFFFFull)); + HWY_ASSERT_EQ(size_t(1), + Num0BitsAboveMS1Bit_Nonzero64(0x4000000000000000ull)); + HWY_ASSERT_EQ(size_t(1), + Num0BitsAboveMS1Bit_Nonzero64(0x4010821004200011ull)); + HWY_ASSERT_EQ(size_t(62), Num0BitsAboveMS1Bit_Nonzero64(2ull)); + HWY_ASSERT_EQ(size_t(62), Num0BitsAboveMS1Bit_Nonzero64(3ull)); + HWY_ASSERT_EQ(size_t(63), Num0BitsAboveMS1Bit_Nonzero64(1ull)); + + HWY_ASSERT_EQ(size_t(0), Num0BitsBelowLS1Bit_Nonzero32(1u)); + HWY_ASSERT_EQ(size_t(1), Num0BitsBelowLS1Bit_Nonzero32(2u)); + HWY_ASSERT_EQ(size_t(30), Num0BitsBelowLS1Bit_Nonzero32(0xC0000000u)); + HWY_ASSERT_EQ(size_t(31), Num0BitsBelowLS1Bit_Nonzero32(0x80000000u)); + + HWY_ASSERT_EQ(size_t(0), Num0BitsBelowLS1Bit_Nonzero64(1ull)); + HWY_ASSERT_EQ(size_t(1), Num0BitsBelowLS1Bit_Nonzero64(2ull)); + HWY_ASSERT_EQ(size_t(62), + Num0BitsBelowLS1Bit_Nonzero64(0xC000000000000000ull)); + HWY_ASSERT_EQ(size_t(63), + Num0BitsBelowLS1Bit_Nonzero64(0x8000000000000000ull)); +} + +HWY_NOINLINE void TestAllPopCount() { + HWY_ASSERT_EQ(size_t(0), PopCount(0u)); + HWY_ASSERT_EQ(size_t(1), PopCount(1u)); + HWY_ASSERT_EQ(size_t(1), PopCount(2u)); + HWY_ASSERT_EQ(size_t(2), PopCount(3u)); + HWY_ASSERT_EQ(size_t(1), PopCount(0x80000000u)); + HWY_ASSERT_EQ(size_t(31), PopCount(0x7FFFFFFFu)); + HWY_ASSERT_EQ(size_t(32), PopCount(0xFFFFFFFFu)); + + HWY_ASSERT_EQ(size_t(1), PopCount(0x80000000ull)); + HWY_ASSERT_EQ(size_t(31), PopCount(0x7FFFFFFFull)); + HWY_ASSERT_EQ(size_t(32), PopCount(0xFFFFFFFFull)); + HWY_ASSERT_EQ(size_t(33), PopCount(0x10FFFFFFFFull)); + HWY_ASSERT_EQ(size_t(63), PopCount(0xFFFEFFFFFFFFFFFFull)); + HWY_ASSERT_EQ(size_t(64), PopCount(0xFFFFFFFFFFFFFFFFull)); +} + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE + +namespace hwy { +HWY_BEFORE_TEST(BaseTest); +HWY_EXPORT_AND_TEST_P(BaseTest, TestAllLimits); +HWY_EXPORT_AND_TEST_P(BaseTest, TestAllLowestHighest); +HWY_EXPORT_AND_TEST_P(BaseTest, TestAllType); +HWY_EXPORT_AND_TEST_P(BaseTest, TestAllIsSame); +HWY_EXPORT_AND_TEST_P(BaseTest, TestAllBitScan); +HWY_EXPORT_AND_TEST_P(BaseTest, TestAllPopCount); +} // namespace hwy + +// Ought not to be necessary, but without this, no tests run on RVV. +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#endif diff --git a/third_party/highway/hwy/cache_control.h b/third_party/highway/hwy/cache_control.h new file mode 100644 index 0000000..65f326a --- /dev/null +++ b/third_party/highway/hwy/cache_control.h @@ -0,0 +1,115 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef HIGHWAY_HWY_CACHE_CONTROL_H_ +#define HIGHWAY_HWY_CACHE_CONTROL_H_ + +#include <stddef.h> +#include <stdint.h> + +#include "hwy/base.h" + +// Requires SSE2; fails to compile on 32-bit Clang 7 (see +// https://github.com/gperftools/gperftools/issues/946). +#if !defined(__SSE2__) || (HWY_COMPILER_CLANG && HWY_ARCH_X86_32) +#undef HWY_DISABLE_CACHE_CONTROL +#define HWY_DISABLE_CACHE_CONTROL +#endif + +// intrin.h is sufficient on MSVC and already included by base.h. +#if HWY_ARCH_X86 && !defined(HWY_DISABLE_CACHE_CONTROL) && !HWY_COMPILER_MSVC +#include <emmintrin.h> // SSE2 +#endif + +// Windows.h #defines these, which causes infinite recursion. Temporarily +// undefine them in this header; these functions are anyway deprecated. +// TODO(janwas): remove when these functions are removed. +#pragma push_macro("LoadFence") +#pragma push_macro("StoreFence") +#undef LoadFence +#undef StoreFence + +namespace hwy { + +// Even if N*sizeof(T) is smaller, Stream may write a multiple of this size. +#define HWY_STREAM_MULTIPLE 16 + +// The following functions may also require an attribute. +#if HWY_ARCH_X86 && !defined(HWY_DISABLE_CACHE_CONTROL) && !HWY_COMPILER_MSVC +#define HWY_ATTR_CACHE __attribute__((target("sse2"))) +#else +#define HWY_ATTR_CACHE +#endif + +// Delays subsequent loads until prior loads are visible. On Intel CPUs, also +// serves as a full fence (waits for all prior instructions to complete). +// No effect on non-x86. +// DEPRECATED due to differing behavior across architectures AND vendors. +HWY_INLINE HWY_ATTR_CACHE void LoadFence() { +#if HWY_ARCH_X86 && !defined(HWY_DISABLE_CACHE_CONTROL) + _mm_lfence(); +#endif +} + +// Ensures values written by previous `Stream` calls are visible on the current +// core. This is NOT sufficient for synchronizing across cores; when `Stream` +// outputs are to be consumed by other core(s), the producer must publish +// availability (e.g. via mutex or atomic_flag) after `FlushStream`. +HWY_INLINE HWY_ATTR_CACHE void FlushStream() { +#if HWY_ARCH_X86 && !defined(HWY_DISABLE_CACHE_CONTROL) + _mm_sfence(); +#endif +} + +// DEPRECATED, replace with `FlushStream`. +HWY_INLINE HWY_ATTR_CACHE void StoreFence() { FlushStream(); } + +// Optionally begins loading the cache line containing "p" to reduce latency of +// subsequent actual loads. +template <typename T> +HWY_INLINE HWY_ATTR_CACHE void Prefetch(const T* p) { +#if HWY_ARCH_X86 && !defined(HWY_DISABLE_CACHE_CONTROL) + _mm_prefetch(reinterpret_cast<const char*>(p), _MM_HINT_T0); +#elif HWY_COMPILER_GCC || HWY_COMPILER_CLANG + // Hint=0 (NTA) behavior differs, but skipping outer caches is probably not + // desirable, so use the default 3 (keep in caches). + __builtin_prefetch(p, /*write=*/0, /*hint=*/3); +#else + (void)p; +#endif +} + +// Invalidates and flushes the cache line containing "p", if possible. +HWY_INLINE HWY_ATTR_CACHE void FlushCacheline(const void* p) { +#if HWY_ARCH_X86 && !defined(HWY_DISABLE_CACHE_CONTROL) + _mm_clflush(p); +#else + (void)p; +#endif +} + +// When called inside a spin-loop, may reduce power consumption. +HWY_INLINE HWY_ATTR_CACHE void Pause() { +#if HWY_ARCH_X86 && !defined(HWY_DISABLE_CACHE_CONTROL) + _mm_pause(); +#endif +} + +} // namespace hwy + +// TODO(janwas): remove when these functions are removed. (See above.) +#pragma pop_macro("StoreFence") +#pragma pop_macro("LoadFence") + +#endif // HIGHWAY_HWY_CACHE_CONTROL_H_ diff --git a/third_party/highway/hwy/contrib/dot/dot-inl.h b/third_party/highway/hwy/contrib/dot/dot-inl.h new file mode 100644 index 0000000..8e7c842 --- /dev/null +++ b/third_party/highway/hwy/contrib/dot/dot-inl.h @@ -0,0 +1,258 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Include guard (still compiled once per target) +#include <cmath> + +#if defined(HIGHWAY_HWY_CONTRIB_DOT_DOT_INL_H_) == \ + defined(HWY_TARGET_TOGGLE) +#ifdef HIGHWAY_HWY_CONTRIB_DOT_DOT_INL_H_ +#undef HIGHWAY_HWY_CONTRIB_DOT_DOT_INL_H_ +#else +#define HIGHWAY_HWY_CONTRIB_DOT_DOT_INL_H_ +#endif + +#include "hwy/highway.h" + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +struct Dot { + // Specify zero or more of these, ORed together, as the kAssumptions template + // argument to Compute. Each one may improve performance or reduce code size, + // at the cost of additional requirements on the arguments. + enum Assumptions { + // num_elements is at least N, which may be up to HWY_MAX_LANES(T). + kAtLeastOneVector = 1, + // num_elements is divisible by N (a power of two, so this can be used if + // the problem size is known to be a power of two >= HWY_MAX_LANES(T)). + kMultipleOfVector = 2, + // RoundUpTo(num_elements, N) elements are accessible; their value does not + // matter (will be treated as if they were zero). + kPaddedToVector = 4, + // Pointers pa and pb, respectively, are multiples of N * sizeof(T). + // For example, aligned_allocator.h ensures this. Note that it is still + // beneficial to ensure such alignment even if these flags are not set. + // If not set, the pointers need only be aligned to alignof(T). + kVectorAlignedA = 8, + kVectorAlignedB = 16, + }; + + // Returns sum{pa[i] * pb[i]} for float or double inputs. + template <int kAssumptions, class D, typename T = TFromD<D>, + HWY_IF_NOT_LANE_SIZE_D(D, 2)> + static HWY_INLINE T Compute(const D d, const T* const HWY_RESTRICT pa, + const T* const HWY_RESTRICT pb, + const size_t num_elements) { + static_assert(IsFloat<T>(), "MulAdd requires float type"); + using V = decltype(Zero(d)); + + const size_t N = Lanes(d); + size_t i = 0; + + constexpr bool kIsAtLeastOneVector = + (kAssumptions & kAtLeastOneVector) != 0; + constexpr bool kIsMultipleOfVector = + (kAssumptions & kMultipleOfVector) != 0; + constexpr bool kIsPaddedToVector = (kAssumptions & kPaddedToVector) != 0; + constexpr bool kIsAlignedA = (kAssumptions & kVectorAlignedA) != 0; + constexpr bool kIsAlignedB = (kAssumptions & kVectorAlignedB) != 0; + + // Won't be able to do a full vector load without padding => scalar loop. + if (!kIsAtLeastOneVector && !kIsMultipleOfVector && !kIsPaddedToVector && + HWY_UNLIKELY(num_elements < N)) { + // Only 2x unroll to avoid excessive code size. + T sum0 = T(0); + T sum1 = T(0); + for (; i + 2 <= num_elements; i += 2) { + sum0 += pa[i + 0] * pb[i + 0]; + sum1 += pa[i + 1] * pb[i + 1]; + } + if (i < num_elements) { + sum1 += pa[i] * pb[i]; + } + return sum0 + sum1; + } + + // Compiler doesn't make independent sum* accumulators, so unroll manually. + // 2 FMA ports * 4 cycle latency = up to 8 in-flight, but that is excessive + // for unaligned inputs (each unaligned pointer halves the throughput + // because it occupies both L1 load ports for a cycle). We cannot have + // arrays of vectors on RVV/SVE, so always unroll 4x. + V sum0 = Zero(d); + V sum1 = Zero(d); + V sum2 = Zero(d); + V sum3 = Zero(d); + + // Main loop: unrolled + for (; i + 4 * N <= num_elements; /* i += 4 * N */) { // incr in loop + const auto a0 = kIsAlignedA ? Load(d, pa + i) : LoadU(d, pa + i); + const auto b0 = kIsAlignedB ? Load(d, pb + i) : LoadU(d, pb + i); + i += N; + sum0 = MulAdd(a0, b0, sum0); + const auto a1 = kIsAlignedA ? Load(d, pa + i) : LoadU(d, pa + i); + const auto b1 = kIsAlignedB ? Load(d, pb + i) : LoadU(d, pb + i); + i += N; + sum1 = MulAdd(a1, b1, sum1); + const auto a2 = kIsAlignedA ? Load(d, pa + i) : LoadU(d, pa + i); + const auto b2 = kIsAlignedB ? Load(d, pb + i) : LoadU(d, pb + i); + i += N; + sum2 = MulAdd(a2, b2, sum2); + const auto a3 = kIsAlignedA ? Load(d, pa + i) : LoadU(d, pa + i); + const auto b3 = kIsAlignedB ? Load(d, pb + i) : LoadU(d, pb + i); + i += N; + sum3 = MulAdd(a3, b3, sum3); + } + + // Up to 3 iterations of whole vectors + for (; i + N <= num_elements; i += N) { + const auto a = kIsAlignedA ? Load(d, pa + i) : LoadU(d, pa + i); + const auto b = kIsAlignedB ? Load(d, pb + i) : LoadU(d, pb + i); + sum0 = MulAdd(a, b, sum0); + } + + if (!kIsMultipleOfVector) { + const size_t remaining = num_elements - i; + if (remaining != 0) { + if (kIsPaddedToVector) { + const auto mask = FirstN(d, remaining); + const auto a = kIsAlignedA ? Load(d, pa + i) : LoadU(d, pa + i); + const auto b = kIsAlignedB ? Load(d, pb + i) : LoadU(d, pb + i); + sum1 = MulAdd(IfThenElseZero(mask, a), IfThenElseZero(mask, b), sum1); + } else { + // Unaligned load such that the last element is in the highest lane - + // ensures we do not touch any elements outside the valid range. + // If we get here, then num_elements >= N. + HWY_DASSERT(i >= N); + i += remaining - N; + const auto skip = FirstN(d, N - remaining); + const auto a = LoadU(d, pa + i); // always unaligned + const auto b = LoadU(d, pb + i); + sum1 = MulAdd(IfThenZeroElse(skip, a), IfThenZeroElse(skip, b), sum1); + } + } + } // kMultipleOfVector + + // Reduction tree: sum of all accumulators by pairs, then across lanes. + sum0 = Add(sum0, sum1); + sum2 = Add(sum2, sum3); + sum0 = Add(sum0, sum2); + return GetLane(SumOfLanes(d, sum0)); + } + + // Returns sum{pa[i] * pb[i]} for bfloat16 inputs. + template <int kAssumptions, class D> + static HWY_INLINE float Compute(const D d, + const bfloat16_t* const HWY_RESTRICT pa, + const bfloat16_t* const HWY_RESTRICT pb, + const size_t num_elements) { + const RebindToUnsigned<D> du16; + const Repartition<float, D> df32; + + using V = decltype(Zero(df32)); + const size_t N = Lanes(d); + size_t i = 0; + + constexpr bool kIsAtLeastOneVector = + (kAssumptions & kAtLeastOneVector) != 0; + constexpr bool kIsMultipleOfVector = + (kAssumptions & kMultipleOfVector) != 0; + constexpr bool kIsPaddedToVector = (kAssumptions & kPaddedToVector) != 0; + constexpr bool kIsAlignedA = (kAssumptions & kVectorAlignedA) != 0; + constexpr bool kIsAlignedB = (kAssumptions & kVectorAlignedB) != 0; + + // Won't be able to do a full vector load without padding => scalar loop. + if (!kIsAtLeastOneVector && !kIsMultipleOfVector && !kIsPaddedToVector && + HWY_UNLIKELY(num_elements < N)) { + float sum0 = 0.0f; // Only 2x unroll to avoid excessive code size for.. + float sum1 = 0.0f; // this unlikely(?) case. + for (; i + 2 <= num_elements; i += 2) { + sum0 += F32FromBF16(pa[i + 0]) * F32FromBF16(pb[i + 0]); + sum1 += F32FromBF16(pa[i + 1]) * F32FromBF16(pb[i + 1]); + } + if (i < num_elements) { + sum1 += F32FromBF16(pa[i]) * F32FromBF16(pb[i]); + } + return sum0 + sum1; + } + + // See comment in the other Compute() overload. Unroll 2x, but we need + // twice as many sums for ReorderWidenMulAccumulate. + V sum0 = Zero(df32); + V sum1 = Zero(df32); + V sum2 = Zero(df32); + V sum3 = Zero(df32); + + // Main loop: unrolled + for (; i + 2 * N <= num_elements; /* i += 2 * N */) { // incr in loop + const auto a0 = kIsAlignedA ? Load(d, pa + i) : LoadU(d, pa + i); + const auto b0 = kIsAlignedB ? Load(d, pb + i) : LoadU(d, pb + i); + i += N; + sum0 = ReorderWidenMulAccumulate(df32, a0, b0, sum0, sum1); + const auto a1 = kIsAlignedA ? Load(d, pa + i) : LoadU(d, pa + i); + const auto b1 = kIsAlignedB ? Load(d, pb + i) : LoadU(d, pb + i); + i += N; + sum2 = ReorderWidenMulAccumulate(df32, a1, b1, sum2, sum3); + } + + // Possibly one more iteration of whole vectors + if (i + N <= num_elements) { + const auto a0 = kIsAlignedA ? Load(d, pa + i) : LoadU(d, pa + i); + const auto b0 = kIsAlignedB ? Load(d, pb + i) : LoadU(d, pb + i); + i += N; + sum0 = ReorderWidenMulAccumulate(df32, a0, b0, sum0, sum1); + } + + if (!kIsMultipleOfVector) { + const size_t remaining = num_elements - i; + if (remaining != 0) { + if (kIsPaddedToVector) { + const auto mask = FirstN(du16, remaining); + const auto va = kIsAlignedA ? Load(d, pa + i) : LoadU(d, pa + i); + const auto vb = kIsAlignedB ? Load(d, pb + i) : LoadU(d, pb + i); + const auto a16 = BitCast(d, IfThenElseZero(mask, BitCast(du16, va))); + const auto b16 = BitCast(d, IfThenElseZero(mask, BitCast(du16, vb))); + sum2 = ReorderWidenMulAccumulate(df32, a16, b16, sum2, sum3); + + } else { + // Unaligned load such that the last element is in the highest lane - + // ensures we do not touch any elements outside the valid range. + // If we get here, then num_elements >= N. + HWY_DASSERT(i >= N); + i += remaining - N; + const auto skip = FirstN(du16, N - remaining); + const auto va = LoadU(d, pa + i); // always unaligned + const auto vb = LoadU(d, pb + i); + const auto a16 = BitCast(d, IfThenZeroElse(skip, BitCast(du16, va))); + const auto b16 = BitCast(d, IfThenZeroElse(skip, BitCast(du16, vb))); + sum2 = ReorderWidenMulAccumulate(df32, a16, b16, sum2, sum3); + } + } + } // kMultipleOfVector + + // Reduction tree: sum of all accumulators by pairs, then across lanes. + sum0 = Add(sum0, sum1); + sum2 = Add(sum2, sum3); + sum0 = Add(sum0, sum2); + return GetLane(SumOfLanes(df32, sum0)); + } +}; + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); + +#endif // HIGHWAY_HWY_CONTRIB_DOT_DOT_INL_H_ diff --git a/third_party/highway/hwy/contrib/dot/dot_test.cc b/third_party/highway/hwy/contrib/dot/dot_test.cc new file mode 100644 index 0000000..0a0dbbd --- /dev/null +++ b/third_party/highway/hwy/contrib/dot/dot_test.cc @@ -0,0 +1,193 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> + +#include "hwy/aligned_allocator.h" + +// clang-format off +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "hwy/contrib/dot/dot_test.cc" +#include "hwy/foreach_target.h" + +#include "hwy/contrib/dot/dot-inl.h" +#include "hwy/tests/test_util-inl.h" +// clang-format on + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +template <typename T> +HWY_NOINLINE T SimpleDot(const T* pa, const T* pb, size_t num) { + double sum = 0.0; + for (size_t i = 0; i < num; ++i) { + sum += pa[i] * pb[i]; + } + return static_cast<T>(sum); +} + +HWY_NOINLINE float SimpleDot(const bfloat16_t* pa, const bfloat16_t* pb, + size_t num) { + float sum = 0.0f; + for (size_t i = 0; i < num; ++i) { + sum += F32FromBF16(pa[i]) * F32FromBF16(pb[i]); + } + return sum; +} + +template <typename T> +void SetValue(const float value, T* HWY_RESTRICT ptr) { + *ptr = static_cast<T>(value); +} +void SetValue(const float value, bfloat16_t* HWY_RESTRICT ptr) { + *ptr = BF16FromF32(value); +} + +class TestDot { + // Computes/verifies one dot product. + template <int kAssumptions, class D> + void Test(D d, size_t num, size_t misalign_a, size_t misalign_b, + RandomState& rng) { + using T = TFromD<D>; + const size_t N = Lanes(d); + const auto random_t = [&rng]() { + const int32_t bits = static_cast<int32_t>(Random32(&rng)) & 1023; + return static_cast<float>(bits - 512) * (1.0f / 64); + }; + + const bool kIsAlignedA = (kAssumptions & Dot::kVectorAlignedA) != 0; + const bool kIsAlignedB = (kAssumptions & Dot::kVectorAlignedB) != 0; + + HWY_ASSERT(!kIsAlignedA || misalign_a == 0); + HWY_ASSERT(!kIsAlignedB || misalign_b == 0); + const size_t padded = + (kAssumptions & Dot::kPaddedToVector) ? RoundUpTo(num, N) : num; + AlignedFreeUniquePtr<T[]> pa = AllocateAligned<T>(misalign_a + padded); + AlignedFreeUniquePtr<T[]> pb = AllocateAligned<T>(misalign_b + padded); + T* a = pa.get() + misalign_a; + T* b = pb.get() + misalign_b; + size_t i = 0; + for (; i < num; ++i) { + SetValue(random_t(), a + i); + SetValue(random_t(), b + i); + } + // Fill padding with NaN - the values are not used, but avoids MSAN errors. + for (; i < padded; ++i) { + ScalableTag<float> df1; + SetValue(GetLane(NaN(df1)), a + i); + SetValue(GetLane(NaN(df1)), b + i); + } + + const auto expected = SimpleDot(a, b, num); + const auto actual = Dot::Compute<kAssumptions>(d, a, b, num); + const auto max = static_cast<decltype(actual)>(8 * 8 * num); + HWY_ASSERT(-max <= actual && actual <= max); + HWY_ASSERT(expected - 1E-4 <= actual && actual <= expected + 1E-4); + } + + // Runs tests with various alignments compatible with the given assumptions. + template <int kAssumptions, class D> + void ForeachMisalign(D d, size_t num, RandomState& rng) { + static_assert( + (kAssumptions & (Dot::kVectorAlignedA | Dot::kVectorAlignedB)) == 0, + "Alignment must not be specified by caller"); + + const size_t N = Lanes(d); + const size_t misalignments[3] = {0, N / 4, 3 * N / 5}; + + // Both flags, both aligned + Test<kAssumptions | Dot::kVectorAlignedA | Dot::kVectorAlignedB>(d, num, 0, + 0, rng); + + // One flag and aligned, other aligned/misaligned + for (size_t m : misalignments) { + Test<kAssumptions | Dot::kVectorAlignedA>(d, num, 0, m, rng); + Test<kAssumptions | Dot::kVectorAlignedB>(d, num, m, 0, rng); + } + + // Neither flag, all combinations of aligned/misaligned + for (size_t ma : misalignments) { + for (size_t mb : misalignments) { + Test<kAssumptions>(d, num, ma, mb, rng); + } + } + } + + // Runs tests with various lengths compatible with the given assumptions. + template <int kAssumptions, class D> + void ForeachCount(D d, RandomState& rng) { + const size_t N = Lanes(d); + const size_t counts[] = {1, + 3, + 7, + 16, + HWY_MAX(N / 2, 1), + HWY_MAX(2 * N / 3, 1), + N, + N + 1, + 4 * N / 3, + 3 * N, + 8 * N, + 8 * N + 2}; + for (size_t num : counts) { + if ((kAssumptions & Dot::kAtLeastOneVector) && num < N) continue; + if ((kAssumptions & Dot::kMultipleOfVector) && (num % N) != 0) continue; + ForeachMisalign<kAssumptions>(d, num, rng); + } + } + + public: + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + RandomState rng; + + // All 8 combinations of the three length-related flags: + ForeachCount<0>(d, rng); + ForeachCount<Dot::kAtLeastOneVector>(d, rng); + ForeachCount<Dot::kMultipleOfVector>(d, rng); + ForeachCount<Dot::kMultipleOfVector | Dot::kAtLeastOneVector>(d, rng); + ForeachCount<Dot::kPaddedToVector>(d, rng); + ForeachCount<Dot::kPaddedToVector | Dot::kAtLeastOneVector>(d, rng); + ForeachCount<Dot::kPaddedToVector | Dot::kMultipleOfVector>(d, rng); + ForeachCount<Dot::kPaddedToVector | Dot::kMultipleOfVector | + Dot::kAtLeastOneVector>(d, rng); + } +}; + +void TestAllDot() { ForFloatTypes(ForPartialVectors<TestDot>()); } +void TestAllDotBF16() { ForShrinkableVectors<TestDot>()(bfloat16_t()); } + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE + +namespace hwy { +HWY_BEFORE_TEST(DotTest); +HWY_EXPORT_AND_TEST_P(DotTest, TestAllDot); +HWY_EXPORT_AND_TEST_P(DotTest, TestAllDotBF16); +} // namespace hwy + +// Ought not to be necessary, but without this, no tests run on RVV. +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#endif diff --git a/third_party/highway/hwy/contrib/image/image.cc b/third_party/highway/hwy/contrib/image/image.cc new file mode 100644 index 0000000..4b57cd3 --- /dev/null +++ b/third_party/highway/hwy/contrib/image/image.cc @@ -0,0 +1,145 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "hwy/contrib/image/image.h" + +#include <cstddef> + +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "hwy/contrib/image/image.cc" + +#include <algorithm> // swap + +#include "hwy/foreach_target.h" +#include "hwy/highway.h" +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { +size_t GetVectorSize() { return Lanes(ScalableTag<uint8_t>()); } +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE + +} // namespace hwy +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE +namespace hwy { +namespace { +HWY_EXPORT(GetVectorSize); // Local function. +} // namespace + +size_t ImageBase::VectorSize() { + // Do not cache result - must return the current value, which may be greater + // than the first call if it was subject to DisableTargets! + return HWY_DYNAMIC_DISPATCH(GetVectorSize)(); +} + +size_t ImageBase::BytesPerRow(const size_t xsize, const size_t sizeof_t) { + const size_t vec_size = VectorSize(); + size_t valid_bytes = xsize * sizeof_t; + + // Allow unaligned accesses starting at the last valid value - this may raise + // msan errors unless the user calls InitializePaddingForUnalignedAccesses. + // Skip for the scalar case because no extra lanes will be loaded. + if (vec_size != 1) { + HWY_DASSERT(vec_size >= sizeof_t); + valid_bytes += vec_size - sizeof_t; + } + + // Round up to vector and cache line size. + const size_t align = HWY_MAX(vec_size, HWY_ALIGNMENT); + size_t bytes_per_row = RoundUpTo(valid_bytes, align); + + // During the lengthy window before writes are committed to memory, CPUs + // guard against read after write hazards by checking the address, but + // only the lower 11 bits. We avoid a false dependency between writes to + // consecutive rows by ensuring their sizes are not multiples of 2 KiB. + // Avoid2K prevents the same problem for the planes of an Image3. + if (bytes_per_row % HWY_ALIGNMENT == 0) { + bytes_per_row += align; + } + + HWY_DASSERT(bytes_per_row % align == 0); + return bytes_per_row; +} + +ImageBase::ImageBase(const size_t xsize, const size_t ysize, + const size_t sizeof_t) + : xsize_(static_cast<uint32_t>(xsize)), + ysize_(static_cast<uint32_t>(ysize)), + bytes_(nullptr, AlignedFreer(&AlignedFreer::DoNothing, nullptr)) { + HWY_ASSERT(sizeof_t == 1 || sizeof_t == 2 || sizeof_t == 4 || sizeof_t == 8); + + bytes_per_row_ = 0; + // Dimensions can be zero, e.g. for lazily-allocated images. Only allocate + // if nonzero, because "zero" bytes still have padding/bookkeeping overhead. + if (xsize != 0 && ysize != 0) { + bytes_per_row_ = BytesPerRow(xsize, sizeof_t); + bytes_ = AllocateAligned<uint8_t>(bytes_per_row_ * ysize); + HWY_ASSERT(bytes_.get() != nullptr); + InitializePadding(sizeof_t, Padding::kRoundUp); + } +} + +ImageBase::ImageBase(const size_t xsize, const size_t ysize, + const size_t bytes_per_row, void* const aligned) + : xsize_(static_cast<uint32_t>(xsize)), + ysize_(static_cast<uint32_t>(ysize)), + bytes_per_row_(bytes_per_row), + bytes_(static_cast<uint8_t*>(aligned), + AlignedFreer(&AlignedFreer::DoNothing, nullptr)) { + const size_t vec_size = VectorSize(); + HWY_ASSERT(bytes_per_row % vec_size == 0); + HWY_ASSERT(reinterpret_cast<uintptr_t>(aligned) % vec_size == 0); +} + +void ImageBase::InitializePadding(const size_t sizeof_t, Padding padding) { +#if defined(MEMORY_SANITIZER) || HWY_IDE + if (xsize_ == 0 || ysize_ == 0) return; + + const size_t vec_size = VectorSize(); // Bytes, independent of sizeof_t! + if (vec_size == 1) return; // Scalar mode: no padding needed + + const size_t valid_size = xsize_ * sizeof_t; + const size_t initialize_size = padding == Padding::kRoundUp + ? RoundUpTo(valid_size, vec_size) + : valid_size + vec_size - sizeof_t; + if (valid_size == initialize_size) return; + + for (size_t y = 0; y < ysize_; ++y) { + uint8_t* HWY_RESTRICT row = static_cast<uint8_t*>(VoidRow(y)); +#if defined(__clang__) && (__clang_major__ <= 6) + // There's a bug in msan in clang-6 when handling AVX2 operations. This + // workaround allows tests to pass on msan, although it is slower and + // prevents msan warnings from uninitialized images. + memset(row, 0, initialize_size); +#else + memset(row + valid_size, 0, initialize_size - valid_size); +#endif // clang6 + } +#else + (void)sizeof_t; + (void)padding; +#endif // MEMORY_SANITIZER +} + +void ImageBase::Swap(ImageBase& other) { + std::swap(xsize_, other.xsize_); + std::swap(ysize_, other.ysize_); + std::swap(bytes_per_row_, other.bytes_per_row_); + std::swap(bytes_, other.bytes_); +} + +} // namespace hwy +#endif // HWY_ONCE diff --git a/third_party/highway/hwy/contrib/image/image.h b/third_party/highway/hwy/contrib/image/image.h new file mode 100644 index 0000000..69a9a80 --- /dev/null +++ b/third_party/highway/hwy/contrib/image/image.h @@ -0,0 +1,476 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef HIGHWAY_HWY_CONTRIB_IMAGE_IMAGE_H_ +#define HIGHWAY_HWY_CONTRIB_IMAGE_IMAGE_H_ + +// SIMD/multicore-friendly planar image representation with row accessors. + +#include <inttypes.h> +#include <stddef.h> +#include <stdint.h> +#include <string.h> + +#include <cstddef> +#include <utility> // std::move + +#include "hwy/aligned_allocator.h" +#include "hwy/base.h" + +namespace hwy { + +// Type-independent parts of Image<> - reduces code duplication and facilitates +// moving member function implementations to cc file. +struct ImageBase { + // Returns required alignment in bytes for externally allocated memory. + static size_t VectorSize(); + + // Returns distance [bytes] between the start of two consecutive rows, a + // multiple of VectorSize but NOT kAlias (see implementation). + static size_t BytesPerRow(const size_t xsize, const size_t sizeof_t); + + // No allocation (for output params or unused images) + ImageBase() + : xsize_(0), + ysize_(0), + bytes_per_row_(0), + bytes_(nullptr, AlignedFreer(&AlignedFreer::DoNothing, nullptr)) {} + + // Allocates memory (this is the common case) + ImageBase(size_t xsize, size_t ysize, size_t sizeof_t); + + // References but does not take ownership of external memory. Useful for + // interoperability with other libraries. `aligned` must be aligned to a + // multiple of VectorSize() and `bytes_per_row` must also be a multiple of + // VectorSize() or preferably equal to BytesPerRow(). + ImageBase(size_t xsize, size_t ysize, size_t bytes_per_row, void* aligned); + + // Copy construction/assignment is forbidden to avoid inadvertent copies, + // which can be very expensive. Use CopyImageTo() instead. + ImageBase(const ImageBase& other) = delete; + ImageBase& operator=(const ImageBase& other) = delete; + + // Move constructor (required for returning Image from function) + ImageBase(ImageBase&& other) noexcept = default; + + // Move assignment (required for std::vector) + ImageBase& operator=(ImageBase&& other) noexcept = default; + + void Swap(ImageBase& other); + + // Useful for pre-allocating image with some padding for alignment purposes + // and later reporting the actual valid dimensions. Caller is responsible + // for ensuring xsize/ysize are <= the original dimensions. + void ShrinkTo(const size_t xsize, const size_t ysize) { + xsize_ = static_cast<uint32_t>(xsize); + ysize_ = static_cast<uint32_t>(ysize); + // NOTE: we can't recompute bytes_per_row for more compact storage and + // better locality because that would invalidate the image contents. + } + + // How many pixels. + HWY_INLINE size_t xsize() const { return xsize_; } + HWY_INLINE size_t ysize() const { return ysize_; } + + // NOTE: do not use this for copying rows - the valid xsize may be much less. + HWY_INLINE size_t bytes_per_row() const { return bytes_per_row_; } + + // Raw access to byte contents, for interfacing with other libraries. + // Unsigned char instead of char to avoid surprises (sign extension). + HWY_INLINE uint8_t* bytes() { + void* p = bytes_.get(); + return static_cast<uint8_t * HWY_RESTRICT>(HWY_ASSUME_ALIGNED(p, 64)); + } + HWY_INLINE const uint8_t* bytes() const { + const void* p = bytes_.get(); + return static_cast<const uint8_t * HWY_RESTRICT>(HWY_ASSUME_ALIGNED(p, 64)); + } + + protected: + // Returns pointer to the start of a row. + HWY_INLINE void* VoidRow(const size_t y) const { +#if defined(ADDRESS_SANITIZER) || defined(MEMORY_SANITIZER) || \ + defined(THREAD_SANITIZER) + if (y >= ysize_) { + HWY_ABORT("Row(%" PRIu64 ") >= %u\n", static_cast<uint64_t>(y), ysize_); + } +#endif + + void* row = bytes_.get() + y * bytes_per_row_; + return HWY_ASSUME_ALIGNED(row, 64); + } + + enum class Padding { + // Allow Load(d, row + x) for x = 0; x < xsize(); x += Lanes(d). Default. + kRoundUp, + // Allow LoadU(d, row + x) for x <= xsize() - 1. This requires an extra + // vector to be initialized. If done by default, this would suppress + // legitimate msan warnings. We therefore require users to explicitly call + // InitializePadding before using unaligned loads (e.g. convolution). + kUnaligned + }; + + // Initializes the minimum bytes required to suppress msan warnings from + // legitimate (according to Padding mode) vector loads/stores on the right + // border, where some lanes are uninitialized and assumed to be unused. + void InitializePadding(size_t sizeof_t, Padding padding); + + // (Members are non-const to enable assignment during move-assignment.) + uint32_t xsize_; // In valid pixels, not including any padding. + uint32_t ysize_; + size_t bytes_per_row_; // Includes padding. + AlignedFreeUniquePtr<uint8_t[]> bytes_; +}; + +// Single channel, aligned rows separated by padding. T must be POD. +// +// 'Single channel' (one 2D array per channel) simplifies vectorization +// (repeating the same operation on multiple adjacent components) without the +// complexity of a hybrid layout (8 R, 8 G, 8 B, ...). In particular, clients +// can easily iterate over all components in a row and Image requires no +// knowledge of the pixel format beyond the component type "T". +// +// 'Aligned' means each row is aligned to the L1 cache line size. This prevents +// false sharing between two threads operating on adjacent rows. +// +// 'Padding' is still relevant because vectors could potentially be larger than +// a cache line. By rounding up row sizes to the vector size, we allow +// reading/writing ALIGNED vectors whose first lane is a valid sample. This +// avoids needing a separate loop to handle remaining unaligned lanes. +// +// This image layout could also be achieved with a vector and a row accessor +// function, but a class wrapper with support for "deleter" allows wrapping +// existing memory allocated by clients without copying the pixels. It also +// provides convenient accessors for xsize/ysize, which shortens function +// argument lists. Supports move-construction so it can be stored in containers. +template <typename ComponentType> +class Image : public ImageBase { + public: + using T = ComponentType; + + Image() = default; + Image(const size_t xsize, const size_t ysize) + : ImageBase(xsize, ysize, sizeof(T)) {} + Image(const size_t xsize, const size_t ysize, size_t bytes_per_row, + void* aligned) + : ImageBase(xsize, ysize, bytes_per_row, aligned) {} + + void InitializePaddingForUnalignedAccesses() { + InitializePadding(sizeof(T), Padding::kUnaligned); + } + + HWY_INLINE const T* ConstRow(const size_t y) const { + return static_cast<const T*>(VoidRow(y)); + } + HWY_INLINE const T* ConstRow(const size_t y) { + return static_cast<const T*>(VoidRow(y)); + } + + // Returns pointer to non-const. This allows passing const Image* parameters + // when the callee is only supposed to fill the pixels, as opposed to + // allocating or resizing the image. + HWY_INLINE T* MutableRow(const size_t y) const { + return static_cast<T*>(VoidRow(y)); + } + HWY_INLINE T* MutableRow(const size_t y) { + return static_cast<T*>(VoidRow(y)); + } + + // Returns number of pixels (some of which are padding) per row. Useful for + // computing other rows via pointer arithmetic. WARNING: this must + // NOT be used to determine xsize. + HWY_INLINE intptr_t PixelsPerRow() const { + return static_cast<intptr_t>(bytes_per_row_ / sizeof(T)); + } +}; + +using ImageF = Image<float>; + +// A bundle of 3 same-sized images. To fill an existing Image3 using +// single-channel producers, we also need access to each const Image*. Const +// prevents breaking the same-size invariant, while still allowing pixels to be +// changed via MutableRow. +template <typename ComponentType> +class Image3 { + public: + using T = ComponentType; + using ImageT = Image<T>; + static constexpr size_t kNumPlanes = 3; + + Image3() : planes_{ImageT(), ImageT(), ImageT()} {} + + Image3(const size_t xsize, const size_t ysize) + : planes_{ImageT(xsize, ysize), ImageT(xsize, ysize), + ImageT(xsize, ysize)} {} + + Image3(Image3&& other) noexcept { + for (size_t i = 0; i < kNumPlanes; i++) { + planes_[i] = std::move(other.planes_[i]); + } + } + + Image3(ImageT&& plane0, ImageT&& plane1, ImageT&& plane2) { + if (!SameSize(plane0, plane1) || !SameSize(plane0, plane2)) { + HWY_ABORT("Not same size: %" PRIu64 " x %" PRIu64 ", %" PRIu64 + " x %" PRIu64 ", %" PRIu64 " x %" PRIu64 "\n", + static_cast<uint64_t>(plane0.xsize()), + static_cast<uint64_t>(plane0.ysize()), + static_cast<uint64_t>(plane1.xsize()), + static_cast<uint64_t>(plane1.ysize()), + static_cast<uint64_t>(plane2.xsize()), + static_cast<uint64_t>(plane2.ysize())); + } + planes_[0] = std::move(plane0); + planes_[1] = std::move(plane1); + planes_[2] = std::move(plane2); + } + + // Copy construction/assignment is forbidden to avoid inadvertent copies, + // which can be very expensive. Use CopyImageTo instead. + Image3(const Image3& other) = delete; + Image3& operator=(const Image3& other) = delete; + + Image3& operator=(Image3&& other) noexcept { + for (size_t i = 0; i < kNumPlanes; i++) { + planes_[i] = std::move(other.planes_[i]); + } + return *this; + } + + HWY_INLINE const T* ConstPlaneRow(const size_t c, const size_t y) const { + return static_cast<const T*>(VoidPlaneRow(c, y)); + } + HWY_INLINE const T* ConstPlaneRow(const size_t c, const size_t y) { + return static_cast<const T*>(VoidPlaneRow(c, y)); + } + + HWY_INLINE T* MutablePlaneRow(const size_t c, const size_t y) const { + return static_cast<T*>(VoidPlaneRow(c, y)); + } + HWY_INLINE T* MutablePlaneRow(const size_t c, const size_t y) { + return static_cast<T*>(VoidPlaneRow(c, y)); + } + + HWY_INLINE const ImageT& Plane(size_t idx) const { return planes_[idx]; } + + void Swap(Image3& other) { + for (size_t c = 0; c < 3; ++c) { + other.planes_[c].Swap(planes_[c]); + } + } + + void ShrinkTo(const size_t xsize, const size_t ysize) { + for (ImageT& plane : planes_) { + plane.ShrinkTo(xsize, ysize); + } + } + + // Sizes of all three images are guaranteed to be equal. + HWY_INLINE size_t xsize() const { return planes_[0].xsize(); } + HWY_INLINE size_t ysize() const { return planes_[0].ysize(); } + // Returns offset [bytes] from one row to the next row of the same plane. + // WARNING: this must NOT be used to determine xsize, nor for copying rows - + // the valid xsize may be much less. + HWY_INLINE size_t bytes_per_row() const { return planes_[0].bytes_per_row(); } + // Returns number of pixels (some of which are padding) per row. Useful for + // computing other rows via pointer arithmetic. WARNING: this must NOT be used + // to determine xsize. + HWY_INLINE intptr_t PixelsPerRow() const { return planes_[0].PixelsPerRow(); } + + private: + // Returns pointer to the start of a row. + HWY_INLINE void* VoidPlaneRow(const size_t c, const size_t y) const { +#if defined(ADDRESS_SANITIZER) || defined(MEMORY_SANITIZER) || \ + defined(THREAD_SANITIZER) + if (c >= kNumPlanes || y >= ysize()) { + HWY_ABORT("PlaneRow(%" PRIu64 ", %" PRIu64 ") >= %" PRIu64 "\n", + static_cast<uint64_t>(c), static_cast<uint64_t>(y), + static_cast<uint64_t>(ysize())); + } +#endif + // Use the first plane's stride because the compiler might not realize they + // are all equal. Thus we only need a single multiplication for all planes. + const size_t row_offset = y * planes_[0].bytes_per_row(); + const void* row = planes_[c].bytes() + row_offset; + return static_cast<const T * HWY_RESTRICT>( + HWY_ASSUME_ALIGNED(row, HWY_ALIGNMENT)); + } + + private: + ImageT planes_[kNumPlanes]; +}; + +using Image3F = Image3<float>; + +// Rectangular region in image(s). Factoring this out of Image instead of +// shifting the pointer by x0/y0 allows this to apply to multiple images with +// different resolutions. Can compare size via SameSize(rect1, rect2). +class Rect { + public: + // Most windows are xsize_max * ysize_max, except those on the borders where + // begin + size_max > end. + constexpr Rect(size_t xbegin, size_t ybegin, size_t xsize_max, + size_t ysize_max, size_t xend, size_t yend) + : x0_(xbegin), + y0_(ybegin), + xsize_(ClampedSize(xbegin, xsize_max, xend)), + ysize_(ClampedSize(ybegin, ysize_max, yend)) {} + + // Construct with origin and known size (typically from another Rect). + constexpr Rect(size_t xbegin, size_t ybegin, size_t xsize, size_t ysize) + : x0_(xbegin), y0_(ybegin), xsize_(xsize), ysize_(ysize) {} + + // Construct a rect that covers a whole image. + template <typename Image> + explicit Rect(const Image& image) + : Rect(0, 0, image.xsize(), image.ysize()) {} + + Rect() : Rect(0, 0, 0, 0) {} + + Rect(const Rect&) = default; + Rect& operator=(const Rect&) = default; + + Rect Subrect(size_t xbegin, size_t ybegin, size_t xsize_max, + size_t ysize_max) { + return Rect(x0_ + xbegin, y0_ + ybegin, xsize_max, ysize_max, x0_ + xsize_, + y0_ + ysize_); + } + + template <typename T> + const T* ConstRow(const Image<T>* image, size_t y) const { + return image->ConstRow(y + y0_) + x0_; + } + + template <typename T> + T* MutableRow(const Image<T>* image, size_t y) const { + return image->MutableRow(y + y0_) + x0_; + } + + template <typename T> + const T* ConstPlaneRow(const Image3<T>& image, size_t c, size_t y) const { + return image.ConstPlaneRow(c, y + y0_) + x0_; + } + + template <typename T> + T* MutablePlaneRow(Image3<T>* image, const size_t c, size_t y) const { + return image->MutablePlaneRow(c, y + y0_) + x0_; + } + + // Returns true if this Rect fully resides in the given image. ImageT could be + // Image<T> or Image3<T>; however if ImageT is Rect, results are nonsensical. + template <class ImageT> + bool IsInside(const ImageT& image) const { + return (x0_ + xsize_ <= image.xsize()) && (y0_ + ysize_ <= image.ysize()); + } + + size_t x0() const { return x0_; } + size_t y0() const { return y0_; } + size_t xsize() const { return xsize_; } + size_t ysize() const { return ysize_; } + + private: + // Returns size_max, or whatever is left in [begin, end). + static constexpr size_t ClampedSize(size_t begin, size_t size_max, + size_t end) { + return (begin + size_max <= end) ? size_max + : (end > begin ? end - begin : 0); + } + + size_t x0_; + size_t y0_; + + size_t xsize_; + size_t ysize_; +}; + +// Works for any image-like input type(s). +template <class Image1, class Image2> +HWY_MAYBE_UNUSED bool SameSize(const Image1& image1, const Image2& image2) { + return image1.xsize() == image2.xsize() && image1.ysize() == image2.ysize(); +} + +// Mirrors out of bounds coordinates and returns valid coordinates unchanged. +// We assume the radius (distance outside the image) is small compared to the +// image size, otherwise this might not terminate. +// The mirror is outside the last column (border pixel is also replicated). +static HWY_INLINE HWY_MAYBE_UNUSED size_t Mirror(int64_t x, + const int64_t xsize) { + HWY_DASSERT(xsize != 0); + + // TODO(janwas): replace with branchless version + while (x < 0 || x >= xsize) { + if (x < 0) { + x = -x - 1; + } else { + x = 2 * xsize - 1 - x; + } + } + return static_cast<size_t>(x); +} + +// Wrap modes for ensuring X/Y coordinates are in the valid range [0, size): + +// Mirrors (repeating the edge pixel once). Useful for convolutions. +struct WrapMirror { + HWY_INLINE size_t operator()(const int64_t coord, const size_t size) const { + return Mirror(coord, static_cast<int64_t>(size)); + } +}; + +// Returns the same coordinate, for when we know "coord" is already valid (e.g. +// interior of an image). +struct WrapUnchanged { + HWY_INLINE size_t operator()(const int64_t coord, size_t /*size*/) const { + return static_cast<size_t>(coord); + } +}; + +// Similar to Wrap* but for row pointers (reduces Row() multiplications). + +class WrapRowMirror { + public: + template <class View> + WrapRowMirror(const View& image, size_t ysize) + : first_row_(image.ConstRow(0)), last_row_(image.ConstRow(ysize - 1)) {} + + const float* operator()(const float* const HWY_RESTRICT row, + const int64_t stride) const { + if (row < first_row_) { + const int64_t num_before = first_row_ - row; + // Mirrored; one row before => row 0, two before = row 1, ... + return first_row_ + num_before - stride; + } + if (row > last_row_) { + const int64_t num_after = row - last_row_; + // Mirrored; one row after => last row, two after = last - 1, ... + return last_row_ - num_after + stride; + } + return row; + } + + private: + const float* const HWY_RESTRICT first_row_; + const float* const HWY_RESTRICT last_row_; +}; + +struct WrapRowUnchanged { + HWY_INLINE const float* operator()(const float* const HWY_RESTRICT row, + int64_t /*stride*/) const { + return row; + } +}; + +} // namespace hwy + +#endif // HIGHWAY_HWY_CONTRIB_IMAGE_IMAGE_H_ diff --git a/third_party/highway/hwy/contrib/image/image_test.cc b/third_party/highway/hwy/contrib/image/image_test.cc new file mode 100644 index 0000000..d5d64a3 --- /dev/null +++ b/third_party/highway/hwy/contrib/image/image_test.cc @@ -0,0 +1,159 @@ +// Copyright (c) the JPEG XL Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "hwy/contrib/image/image.h" + +#include <cstddef> + +#include "hwy/base.h" + +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "hwy/contrib/image/image_test.cc" +#include "hwy/foreach_target.h" + +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> + +#include <random> +#include <utility> + +#include "hwy/highway.h" +#include "hwy/tests/test_util-inl.h" + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +// Ensure we can always write full aligned vectors. +struct TestAlignedT { + template <typename T> + void operator()(T /*unused*/) const { + std::mt19937 rng(129); + std::uniform_int_distribution<int> dist(0, 16); + const ScalableTag<T> d; + + for (size_t ysize = 1; ysize < 4; ++ysize) { + for (size_t xsize = 1; xsize < 64; ++xsize) { + Image<T> img(xsize, ysize); + + for (size_t y = 0; y < ysize; ++y) { + T* HWY_RESTRICT row = img.MutableRow(y); + for (size_t x = 0; x < xsize; x += Lanes(d)) { + const auto values = Iota(d, dist(rng)); + Store(values, d, row + x); + } + } + + // Sanity check to prevent optimizing out the writes + const auto x = std::uniform_int_distribution<size_t>(0, xsize - 1)(rng); + const auto y = std::uniform_int_distribution<size_t>(0, ysize - 1)(rng); + HWY_ASSERT(img.ConstRow(y)[x] < 16 + Lanes(d)); + } + } + } +}; + +void TestAligned() { ForUnsignedTypes(TestAlignedT()); } + +// Ensure we can write an unaligned vector starting at the last valid value. +struct TestUnalignedT { + template <typename T> + void operator()(T /*unused*/) const { + std::mt19937 rng(129); + std::uniform_int_distribution<int> dist(0, 3); + const ScalableTag<T> d; + + for (size_t ysize = 1; ysize < 4; ++ysize) { + for (size_t xsize = 1; xsize < 128; ++xsize) { + Image<T> img(xsize, ysize); + img.InitializePaddingForUnalignedAccesses(); + +// This test reads padding, which only works if it was initialized, +// which only happens in MSAN builds. +#if defined(MEMORY_SANITIZER) || HWY_IDE + // Initialize only the valid samples + for (size_t y = 0; y < ysize; ++y) { + T* HWY_RESTRICT row = img.MutableRow(y); + for (size_t x = 0; x < xsize; ++x) { + row[x] = static_cast<T>(1u << dist(rng)); + } + } + + // Read padding bits + auto accum = Zero(d); + for (size_t y = 0; y < ysize; ++y) { + T* HWY_RESTRICT row = img.MutableRow(y); + for (size_t x = 0; x < xsize; ++x) { + accum = Or(accum, LoadU(d, row + x)); + } + } + + // Ensure padding was zero + const size_t N = Lanes(d); + auto lanes = AllocateAligned<T>(N); + Store(accum, d, lanes.get()); + for (size_t i = 0; i < N; ++i) { + HWY_ASSERT(lanes[i] < 16); + } +#else // Check that writing padding does not overwrite valid samples + // Initialize only the valid samples + for (size_t y = 0; y < ysize; ++y) { + T* HWY_RESTRICT row = img.MutableRow(y); + for (size_t x = 0; x < xsize; ++x) { + row[x] = static_cast<T>(x); + } + } + + // Zero padding and rightmost sample + for (size_t y = 0; y < ysize; ++y) { + T* HWY_RESTRICT row = img.MutableRow(y); + StoreU(Zero(d), d, row + xsize - 1); + } + + // Ensure no samples except the rightmost were overwritten + for (size_t y = 0; y < ysize; ++y) { + T* HWY_RESTRICT row = img.MutableRow(y); + for (size_t x = 0; x < xsize - 1; ++x) { + HWY_ASSERT_EQ(static_cast<T>(x), row[x]); + } + } +#endif + } + } + } +}; + +void TestUnaligned() { ForUnsignedTypes(TestUnalignedT()); } + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE + +namespace hwy { +HWY_BEFORE_TEST(ImageTest); +HWY_EXPORT_AND_TEST_P(ImageTest, TestAligned); +HWY_EXPORT_AND_TEST_P(ImageTest, TestUnaligned); +} // namespace hwy + +// Ought not to be necessary, but without this, no tests run on RVV. +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#endif diff --git a/third_party/highway/hwy/contrib/math/math-inl.h b/third_party/highway/hwy/contrib/math/math-inl.h new file mode 100644 index 0000000..77f686d --- /dev/null +++ b/third_party/highway/hwy/contrib/math/math-inl.h @@ -0,0 +1,1241 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Include guard (still compiled once per target) +#if defined(HIGHWAY_HWY_CONTRIB_MATH_MATH_INL_H_) == \ + defined(HWY_TARGET_TOGGLE) +#ifdef HIGHWAY_HWY_CONTRIB_MATH_MATH_INL_H_ +#undef HIGHWAY_HWY_CONTRIB_MATH_MATH_INL_H_ +#else +#define HIGHWAY_HWY_CONTRIB_MATH_MATH_INL_H_ +#endif + +#include "hwy/highway.h" + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +/** + * Highway SIMD version of std::acos(x). + * + * Valid Lane Types: float32, float64 + * Max Error: ULP = 2 + * Valid Range: [-1, +1] + * @return arc cosine of 'x' + */ +template <class D, class V> +HWY_INLINE V Acos(const D d, V x); +template <class D, class V> +HWY_NOINLINE V CallAcos(const D d, VecArg<V> x) { + return Acos(d, x); +} + +/** + * Highway SIMD version of std::acosh(x). + * + * Valid Lane Types: float32, float64 + * Max Error: ULP = 3 + * Valid Range: float32[1, +FLT_MAX], float64[1, +DBL_MAX] + * @return hyperbolic arc cosine of 'x' + */ +template <class D, class V> +HWY_INLINE V Acosh(const D d, V x); +template <class D, class V> +HWY_NOINLINE V CallAcosh(const D d, VecArg<V> x) { + return Acosh(d, x); +} + +/** + * Highway SIMD version of std::asin(x). + * + * Valid Lane Types: float32, float64 + * Max Error: ULP = 2 + * Valid Range: [-1, +1] + * @return arc sine of 'x' + */ +template <class D, class V> +HWY_INLINE V Asin(const D d, V x); +template <class D, class V> +HWY_NOINLINE V CallAsin(const D d, VecArg<V> x) { + return Asin(d, x); +} + +/** + * Highway SIMD version of std::asinh(x). + * + * Valid Lane Types: float32, float64 + * Max Error: ULP = 3 + * Valid Range: float32[-FLT_MAX, +FLT_MAX], float64[-DBL_MAX, +DBL_MAX] + * @return hyperbolic arc sine of 'x' + */ +template <class D, class V> +HWY_INLINE V Asinh(const D d, V x); +template <class D, class V> +HWY_NOINLINE V CallAsinh(const D d, VecArg<V> x) { + return Asinh(d, x); +} + +/** + * Highway SIMD version of std::atan(x). + * + * Valid Lane Types: float32, float64 + * Max Error: ULP = 3 + * Valid Range: float32[-FLT_MAX, +FLT_MAX], float64[-DBL_MAX, +DBL_MAX] + * @return arc tangent of 'x' + */ +template <class D, class V> +HWY_INLINE V Atan(const D d, V x); +template <class D, class V> +HWY_NOINLINE V CallAtan(const D d, VecArg<V> x) { + return Atan(d, x); +} + +/** + * Highway SIMD version of std::atanh(x). + * + * Valid Lane Types: float32, float64 + * Max Error: ULP = 3 + * Valid Range: (-1, +1) + * @return hyperbolic arc tangent of 'x' + */ +template <class D, class V> +HWY_INLINE V Atanh(const D d, V x); +template <class D, class V> +HWY_NOINLINE V CallAtanh(const D d, VecArg<V> x) { + return Atanh(d, x); +} + +/** + * Highway SIMD version of std::cos(x). + * + * Valid Lane Types: float32, float64 + * Max Error: ULP = 3 + * Valid Range: [-39000, +39000] + * @return cosine of 'x' + */ +template <class D, class V> +HWY_INLINE V Cos(const D d, V x); +template <class D, class V> +HWY_NOINLINE V CallCos(const D d, VecArg<V> x) { + return Cos(d, x); +} + +/** + * Highway SIMD version of std::exp(x). + * + * Valid Lane Types: float32, float64 + * Max Error: ULP = 1 + * Valid Range: float32[-FLT_MAX, +104], float64[-DBL_MAX, +706] + * @return e^x + */ +template <class D, class V> +HWY_INLINE V Exp(const D d, V x); +template <class D, class V> +HWY_NOINLINE V CallExp(const D d, VecArg<V> x) { + return Exp(d, x); +} + +/** + * Highway SIMD version of std::expm1(x). + * + * Valid Lane Types: float32, float64 + * Max Error: ULP = 4 + * Valid Range: float32[-FLT_MAX, +104], float64[-DBL_MAX, +706] + * @return e^x - 1 + */ +template <class D, class V> +HWY_INLINE V Expm1(const D d, V x); +template <class D, class V> +HWY_NOINLINE V CallExpm1(const D d, VecArg<V> x) { + return Expm1(d, x); +} + +/** + * Highway SIMD version of std::log(x). + * + * Valid Lane Types: float32, float64 + * Max Error: ULP = 4 + * Valid Range: float32(0, +FLT_MAX], float64(0, +DBL_MAX] + * @return natural logarithm of 'x' + */ +template <class D, class V> +HWY_INLINE V Log(const D d, V x); +template <class D, class V> +HWY_NOINLINE V CallLog(const D d, VecArg<V> x) { + return Log(d, x); +} + +/** + * Highway SIMD version of std::log10(x). + * + * Valid Lane Types: float32, float64 + * Max Error: ULP = 2 + * Valid Range: float32(0, +FLT_MAX], float64(0, +DBL_MAX] + * @return base 10 logarithm of 'x' + */ +template <class D, class V> +HWY_INLINE V Log10(const D d, V x); +template <class D, class V> +HWY_NOINLINE V CallLog10(const D d, VecArg<V> x) { + return Log10(d, x); +} + +/** + * Highway SIMD version of std::log1p(x). + * + * Valid Lane Types: float32, float64 + * Max Error: ULP = 2 + * Valid Range: float32[0, +FLT_MAX], float64[0, +DBL_MAX] + * @return log(1 + x) + */ +template <class D, class V> +HWY_INLINE V Log1p(const D d, V x); +template <class D, class V> +HWY_NOINLINE V CallLog1p(const D d, VecArg<V> x) { + return Log1p(d, x); +} + +/** + * Highway SIMD version of std::log2(x). + * + * Valid Lane Types: float32, float64 + * Max Error: ULP = 2 + * Valid Range: float32(0, +FLT_MAX], float64(0, +DBL_MAX] + * @return base 2 logarithm of 'x' + */ +template <class D, class V> +HWY_INLINE V Log2(const D d, V x); +template <class D, class V> +HWY_NOINLINE V CallLog2(const D d, VecArg<V> x) { + return Log2(d, x); +} + +/** + * Highway SIMD version of std::sin(x). + * + * Valid Lane Types: float32, float64 + * Max Error: ULP = 3 + * Valid Range: [-39000, +39000] + * @return sine of 'x' + */ +template <class D, class V> +HWY_INLINE V Sin(const D d, V x); +template <class D, class V> +HWY_NOINLINE V CallSin(const D d, VecArg<V> x) { + return Sin(d, x); +} + +/** + * Highway SIMD version of std::sinh(x). + * + * Valid Lane Types: float32, float64 + * Max Error: ULP = 4 + * Valid Range: float32[-88.7228, +88.7228], float64[-709, +709] + * @return hyperbolic sine of 'x' + */ +template <class D, class V> +HWY_INLINE V Sinh(const D d, V x); +template <class D, class V> +HWY_NOINLINE V CallSinh(const D d, VecArg<V> x) { + return Sinh(d, x); +} + +/** + * Highway SIMD version of std::tanh(x). + * + * Valid Lane Types: float32, float64 + * Max Error: ULP = 4 + * Valid Range: float32[-FLT_MAX, +FLT_MAX], float64[-DBL_MAX, +DBL_MAX] + * @return hyperbolic tangent of 'x' + */ +template <class D, class V> +HWY_INLINE V Tanh(const D d, V x); +template <class D, class V> +HWY_NOINLINE V CallTanh(const D d, VecArg<V> x) { + return Tanh(d, x); +} + +//////////////////////////////////////////////////////////////////////////////// +// Implementation +//////////////////////////////////////////////////////////////////////////////// +namespace impl { + +// Estrin's Scheme is a faster method for evaluating large polynomials on +// super scalar architectures. It works by factoring the Horner's Method +// polynomial into power of two sub-trees that can be evaluated in parallel. +// Wikipedia Link: https://en.wikipedia.org/wiki/Estrin%27s_scheme +template <class T> +HWY_INLINE HWY_MAYBE_UNUSED T Estrin(T x, T c0, T c1) { + return MulAdd(c1, x, c0); +} +template <class T> +HWY_INLINE HWY_MAYBE_UNUSED T Estrin(T x, T c0, T c1, T c2) { + T x2 = Mul(x, x); + return MulAdd(x2, c2, MulAdd(c1, x, c0)); +} +template <class T> +HWY_INLINE HWY_MAYBE_UNUSED T Estrin(T x, T c0, T c1, T c2, T c3) { + T x2 = Mul(x, x); + return MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)); +} +template <class T> +HWY_INLINE HWY_MAYBE_UNUSED T Estrin(T x, T c0, T c1, T c2, T c3, T c4) { + T x2 = Mul(x, x); + T x4 = Mul(x2, x2); + return MulAdd(x4, c4, MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))); +} +template <class T> +HWY_INLINE HWY_MAYBE_UNUSED T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5) { + T x2 = Mul(x, x); + T x4 = Mul(x2, x2); + return MulAdd(x4, MulAdd(c5, x, c4), + MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))); +} +template <class T> +HWY_INLINE HWY_MAYBE_UNUSED T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5, + T c6) { + T x2 = Mul(x, x); + T x4 = Mul(x2, x2); + return MulAdd(x4, MulAdd(x2, c6, MulAdd(c5, x, c4)), + MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))); +} +template <class T> +HWY_INLINE HWY_MAYBE_UNUSED T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5, + T c6, T c7) { + T x2 = Mul(x, x); + T x4 = Mul(x2, x2); + return MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)), + MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))); +} +template <class T> +HWY_INLINE HWY_MAYBE_UNUSED T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5, + T c6, T c7, T c8) { + T x2 = Mul(x, x); + T x4 = Mul(x2, x2); + T x8 = Mul(x4, x4); + return MulAdd(x8, c8, + MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)), + MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)))); +} +template <class T> +HWY_INLINE HWY_MAYBE_UNUSED T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5, + T c6, T c7, T c8, T c9) { + T x2 = Mul(x, x); + T x4 = Mul(x2, x2); + T x8 = Mul(x4, x4); + return MulAdd(x8, MulAdd(c9, x, c8), + MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)), + MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)))); +} +template <class T> +HWY_INLINE HWY_MAYBE_UNUSED T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5, + T c6, T c7, T c8, T c9, T c10) { + T x2 = Mul(x, x); + T x4 = Mul(x2, x2); + T x8 = Mul(x4, x4); + return MulAdd(x8, MulAdd(x2, c10, MulAdd(c9, x, c8)), + MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)), + MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)))); +} +template <class T> +HWY_INLINE HWY_MAYBE_UNUSED T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5, + T c6, T c7, T c8, T c9, T c10, T c11) { + T x2 = Mul(x, x); + T x4 = Mul(x2, x2); + T x8 = Mul(x4, x4); + return MulAdd(x8, MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8)), + MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)), + MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)))); +} +template <class T> +HWY_INLINE HWY_MAYBE_UNUSED T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5, + T c6, T c7, T c8, T c9, T c10, T c11, + T c12) { + T x2 = Mul(x, x); + T x4 = Mul(x2, x2); + T x8 = Mul(x4, x4); + return MulAdd( + x8, MulAdd(x4, c12, MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))), + MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)), + MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)))); +} +template <class T> +HWY_INLINE HWY_MAYBE_UNUSED T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5, + T c6, T c7, T c8, T c9, T c10, T c11, + T c12, T c13) { + T x2 = Mul(x, x); + T x4 = Mul(x2, x2); + T x8 = Mul(x4, x4); + return MulAdd(x8, + MulAdd(x4, MulAdd(c13, x, c12), + MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))), + MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)), + MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)))); +} +template <class T> +HWY_INLINE HWY_MAYBE_UNUSED T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5, + T c6, T c7, T c8, T c9, T c10, T c11, + T c12, T c13, T c14) { + T x2 = Mul(x, x); + T x4 = Mul(x2, x2); + T x8 = Mul(x4, x4); + return MulAdd(x8, + MulAdd(x4, MulAdd(x2, c14, MulAdd(c13, x, c12)), + MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))), + MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)), + MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)))); +} +template <class T> +HWY_INLINE HWY_MAYBE_UNUSED T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5, + T c6, T c7, T c8, T c9, T c10, T c11, + T c12, T c13, T c14, T c15) { + T x2 = Mul(x, x); + T x4 = Mul(x2, x2); + T x8 = Mul(x4, x4); + return MulAdd(x8, + MulAdd(x4, MulAdd(x2, MulAdd(c15, x, c14), MulAdd(c13, x, c12)), + MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))), + MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)), + MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0)))); +} +template <class T> +HWY_INLINE HWY_MAYBE_UNUSED T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5, + T c6, T c7, T c8, T c9, T c10, T c11, + T c12, T c13, T c14, T c15, T c16) { + T x2 = Mul(x, x); + T x4 = Mul(x2, x2); + T x8 = Mul(x4, x4); + T x16 = Mul(x8, x8); + return MulAdd( + x16, c16, + MulAdd(x8, + MulAdd(x4, MulAdd(x2, MulAdd(c15, x, c14), MulAdd(c13, x, c12)), + MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))), + MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)), + MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))))); +} +template <class T> +HWY_INLINE HWY_MAYBE_UNUSED T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5, + T c6, T c7, T c8, T c9, T c10, T c11, + T c12, T c13, T c14, T c15, T c16, T c17) { + T x2 = Mul(x, x); + T x4 = Mul(x2, x2); + T x8 = Mul(x4, x4); + T x16 = Mul(x8, x8); + return MulAdd( + x16, MulAdd(c17, x, c16), + MulAdd(x8, + MulAdd(x4, MulAdd(x2, MulAdd(c15, x, c14), MulAdd(c13, x, c12)), + MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))), + MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)), + MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))))); +} +template <class T> +HWY_INLINE HWY_MAYBE_UNUSED T Estrin(T x, T c0, T c1, T c2, T c3, T c4, T c5, + T c6, T c7, T c8, T c9, T c10, T c11, + T c12, T c13, T c14, T c15, T c16, T c17, + T c18) { + T x2 = Mul(x, x); + T x4 = Mul(x2, x2); + T x8 = Mul(x4, x4); + T x16 = Mul(x8, x8); + return MulAdd( + x16, MulAdd(x2, c18, MulAdd(c17, x, c16)), + MulAdd(x8, + MulAdd(x4, MulAdd(x2, MulAdd(c15, x, c14), MulAdd(c13, x, c12)), + MulAdd(x2, MulAdd(c11, x, c10), MulAdd(c9, x, c8))), + MulAdd(x4, MulAdd(x2, MulAdd(c7, x, c6), MulAdd(c5, x, c4)), + MulAdd(x2, MulAdd(c3, x, c2), MulAdd(c1, x, c0))))); +} + +template <class FloatOrDouble> +struct AsinImpl {}; +template <class FloatOrDouble> +struct AtanImpl {}; +template <class FloatOrDouble> +struct CosSinImpl {}; +template <class FloatOrDouble> +struct ExpImpl {}; +template <class FloatOrDouble> +struct LogImpl {}; + +template <> +struct AsinImpl<float> { + // Polynomial approximation for asin(x) over the range [0, 0.5). + template <class D, class V> + HWY_INLINE V AsinPoly(D d, V x2, V /*x*/) { + const auto k0 = Set(d, +0.1666677296f); + const auto k1 = Set(d, +0.07495029271f); + const auto k2 = Set(d, +0.04547423869f); + const auto k3 = Set(d, +0.02424046025f); + const auto k4 = Set(d, +0.04197454825f); + + return Estrin(x2, k0, k1, k2, k3, k4); + } +}; + +#if HWY_CAP_FLOAT64 && HWY_CAP_INTEGER64 + +template <> +struct AsinImpl<double> { + // Polynomial approximation for asin(x) over the range [0, 0.5). + template <class D, class V> + HWY_INLINE V AsinPoly(D d, V x2, V /*x*/) { + const auto k0 = Set(d, +0.1666666666666497543); + const auto k1 = Set(d, +0.07500000000378581611); + const auto k2 = Set(d, +0.04464285681377102438); + const auto k3 = Set(d, +0.03038195928038132237); + const auto k4 = Set(d, +0.02237176181932048341); + const auto k5 = Set(d, +0.01735956991223614604); + const auto k6 = Set(d, +0.01388715184501609218); + const auto k7 = Set(d, +0.01215360525577377331); + const auto k8 = Set(d, +0.006606077476277170610); + const auto k9 = Set(d, +0.01929045477267910674); + const auto k10 = Set(d, -0.01581918243329996643); + const auto k11 = Set(d, +0.03161587650653934628); + + return Estrin(x2, k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11); + } +}; + +#endif + +template <> +struct AtanImpl<float> { + // Polynomial approximation for atan(x) over the range [0, 1.0). + template <class D, class V> + HWY_INLINE V AtanPoly(D d, V x) { + const auto k0 = Set(d, -0.333331018686294555664062f); + const auto k1 = Set(d, +0.199926957488059997558594f); + const auto k2 = Set(d, -0.142027363181114196777344f); + const auto k3 = Set(d, +0.106347933411598205566406f); + const auto k4 = Set(d, -0.0748900920152664184570312f); + const auto k5 = Set(d, +0.0425049886107444763183594f); + const auto k6 = Set(d, -0.0159569028764963150024414f); + const auto k7 = Set(d, +0.00282363896258175373077393f); + + const auto y = Mul(x, x); + return MulAdd(Estrin(y, k0, k1, k2, k3, k4, k5, k6, k7), Mul(y, x), x); + } +}; + +#if HWY_CAP_FLOAT64 && HWY_CAP_INTEGER64 + +template <> +struct AtanImpl<double> { + // Polynomial approximation for atan(x) over the range [0, 1.0). + template <class D, class V> + HWY_INLINE V AtanPoly(D d, V x) { + const auto k0 = Set(d, -0.333333333333311110369124); + const auto k1 = Set(d, +0.199999999996591265594148); + const auto k2 = Set(d, -0.14285714266771329383765); + const auto k3 = Set(d, +0.111111105648261418443745); + const auto k4 = Set(d, -0.090908995008245008229153); + const auto k5 = Set(d, +0.0769219538311769618355029); + const auto k6 = Set(d, -0.0666573579361080525984562); + const auto k7 = Set(d, +0.0587666392926673580854313); + const auto k8 = Set(d, -0.0523674852303482457616113); + const auto k9 = Set(d, +0.0466667150077840625632675); + const auto k10 = Set(d, -0.0407629191276836500001934); + const auto k11 = Set(d, +0.0337852580001353069993897); + const auto k12 = Set(d, -0.0254517624932312641616861); + const auto k13 = Set(d, +0.016599329773529201970117); + const auto k14 = Set(d, -0.00889896195887655491740809); + const auto k15 = Set(d, +0.00370026744188713119232403); + const auto k16 = Set(d, -0.00110611831486672482563471); + const auto k17 = Set(d, +0.000209850076645816976906797); + const auto k18 = Set(d, -1.88796008463073496563746e-5); + + const auto y = Mul(x, x); + return MulAdd(Estrin(y, k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11, + k12, k13, k14, k15, k16, k17, k18), + Mul(y, x), x); + } +}; + +#endif + +template <> +struct CosSinImpl<float> { + // Rounds float toward zero and returns as int32_t. + template <class D, class V> + HWY_INLINE Vec<Rebind<int32_t, D>> ToInt32(D /*unused*/, V x) { + return ConvertTo(Rebind<int32_t, D>(), x); + } + + template <class D, class V> + HWY_INLINE V Poly(D d, V x) { + const auto k0 = Set(d, -1.66666597127914428710938e-1f); + const auto k1 = Set(d, +8.33307858556509017944336e-3f); + const auto k2 = Set(d, -1.981069071916863322258e-4f); + const auto k3 = Set(d, +2.6083159809786593541503e-6f); + + const auto y = Mul(x, x); + return MulAdd(Estrin(y, k0, k1, k2, k3), Mul(y, x), x); + } + + template <class D, class V, class VI32> + HWY_INLINE V CosReduce(D d, V x, VI32 q) { + // kHalfPiPart0f + kHalfPiPart1f + kHalfPiPart2f + kHalfPiPart3f ~= -pi/2 + const V kHalfPiPart0f = Set(d, -0.5f * 3.140625f); + const V kHalfPiPart1f = Set(d, -0.5f * 0.0009670257568359375f); + const V kHalfPiPart2f = Set(d, -0.5f * 6.2771141529083251953e-7f); + const V kHalfPiPart3f = Set(d, -0.5f * 1.2154201256553420762e-10f); + + // Extended precision modular arithmetic. + const V qf = ConvertTo(d, q); + x = MulAdd(qf, kHalfPiPart0f, x); + x = MulAdd(qf, kHalfPiPart1f, x); + x = MulAdd(qf, kHalfPiPart2f, x); + x = MulAdd(qf, kHalfPiPart3f, x); + return x; + } + + template <class D, class V, class VI32> + HWY_INLINE V SinReduce(D d, V x, VI32 q) { + // kPiPart0f + kPiPart1f + kPiPart2f + kPiPart3f ~= -pi + const V kPiPart0f = Set(d, -3.140625f); + const V kPiPart1f = Set(d, -0.0009670257568359375f); + const V kPiPart2f = Set(d, -6.2771141529083251953e-7f); + const V kPiPart3f = Set(d, -1.2154201256553420762e-10f); + + // Extended precision modular arithmetic. + const V qf = ConvertTo(d, q); + x = MulAdd(qf, kPiPart0f, x); + x = MulAdd(qf, kPiPart1f, x); + x = MulAdd(qf, kPiPart2f, x); + x = MulAdd(qf, kPiPart3f, x); + return x; + } + + // (q & 2) == 0 ? -0.0 : +0.0 + template <class D, class VI32> + HWY_INLINE Vec<Rebind<float, D>> CosSignFromQuadrant(D d, VI32 q) { + const VI32 kTwo = Set(Rebind<int32_t, D>(), 2); + return BitCast(d, ShiftLeft<30>(AndNot(q, kTwo))); + } + + // ((q & 1) ? -0.0 : +0.0) + template <class D, class VI32> + HWY_INLINE Vec<Rebind<float, D>> SinSignFromQuadrant(D d, VI32 q) { + const VI32 kOne = Set(Rebind<int32_t, D>(), 1); + return BitCast(d, ShiftLeft<31>(And(q, kOne))); + } +}; + +#if HWY_CAP_FLOAT64 && HWY_CAP_INTEGER64 + +template <> +struct CosSinImpl<double> { + // Rounds double toward zero and returns as int32_t. + template <class D, class V> + HWY_INLINE Vec<Rebind<int32_t, D>> ToInt32(D /*unused*/, V x) { + return DemoteTo(Rebind<int32_t, D>(), x); + } + + template <class D, class V> + HWY_INLINE V Poly(D d, V x) { + const auto k0 = Set(d, -0.166666666666666657414808); + const auto k1 = Set(d, +0.00833333333333332974823815); + const auto k2 = Set(d, -0.000198412698412696162806809); + const auto k3 = Set(d, +2.75573192239198747630416e-6); + const auto k4 = Set(d, -2.50521083763502045810755e-8); + const auto k5 = Set(d, +1.60590430605664501629054e-10); + const auto k6 = Set(d, -7.64712219118158833288484e-13); + const auto k7 = Set(d, +2.81009972710863200091251e-15); + const auto k8 = Set(d, -7.97255955009037868891952e-18); + + const auto y = Mul(x, x); + return MulAdd(Estrin(y, k0, k1, k2, k3, k4, k5, k6, k7, k8), Mul(y, x), x); + } + + template <class D, class V, class VI32> + HWY_INLINE V CosReduce(D d, V x, VI32 q) { + // kHalfPiPart0d + kHalfPiPart1d + kHalfPiPart2d + kHalfPiPart3d ~= -pi/2 + const V kHalfPiPart0d = Set(d, -0.5 * 3.1415926218032836914); + const V kHalfPiPart1d = Set(d, -0.5 * 3.1786509424591713469e-8); + const V kHalfPiPart2d = Set(d, -0.5 * 1.2246467864107188502e-16); + const V kHalfPiPart3d = Set(d, -0.5 * 1.2736634327021899816e-24); + + // Extended precision modular arithmetic. + const V qf = PromoteTo(d, q); + x = MulAdd(qf, kHalfPiPart0d, x); + x = MulAdd(qf, kHalfPiPart1d, x); + x = MulAdd(qf, kHalfPiPart2d, x); + x = MulAdd(qf, kHalfPiPart3d, x); + return x; + } + + template <class D, class V, class VI32> + HWY_INLINE V SinReduce(D d, V x, VI32 q) { + // kPiPart0d + kPiPart1d + kPiPart2d + kPiPart3d ~= -pi + const V kPiPart0d = Set(d, -3.1415926218032836914); + const V kPiPart1d = Set(d, -3.1786509424591713469e-8); + const V kPiPart2d = Set(d, -1.2246467864107188502e-16); + const V kPiPart3d = Set(d, -1.2736634327021899816e-24); + + // Extended precision modular arithmetic. + const V qf = PromoteTo(d, q); + x = MulAdd(qf, kPiPart0d, x); + x = MulAdd(qf, kPiPart1d, x); + x = MulAdd(qf, kPiPart2d, x); + x = MulAdd(qf, kPiPart3d, x); + return x; + } + + // (q & 2) == 0 ? -0.0 : +0.0 + template <class D, class VI32> + HWY_INLINE Vec<Rebind<double, D>> CosSignFromQuadrant(D d, VI32 q) { + const VI32 kTwo = Set(Rebind<int32_t, D>(), 2); + return BitCast( + d, ShiftLeft<62>(PromoteTo(Rebind<int64_t, D>(), AndNot(q, kTwo)))); + } + + // ((q & 1) ? -0.0 : +0.0) + template <class D, class VI32> + HWY_INLINE Vec<Rebind<double, D>> SinSignFromQuadrant(D d, VI32 q) { + const VI32 kOne = Set(Rebind<int32_t, D>(), 1); + return BitCast( + d, ShiftLeft<63>(PromoteTo(Rebind<int64_t, D>(), And(q, kOne)))); + } +}; + +#endif + +template <> +struct ExpImpl<float> { + // Rounds float toward zero and returns as int32_t. + template <class D, class V> + HWY_INLINE Vec<Rebind<int32_t, D>> ToInt32(D /*unused*/, V x) { + return ConvertTo(Rebind<int32_t, D>(), x); + } + + template <class D, class V> + HWY_INLINE V ExpPoly(D d, V x) { + const auto k0 = Set(d, +0.5f); + const auto k1 = Set(d, +0.166666671633720397949219f); + const auto k2 = Set(d, +0.0416664853692054748535156f); + const auto k3 = Set(d, +0.00833336077630519866943359f); + const auto k4 = Set(d, +0.00139304355252534151077271f); + const auto k5 = Set(d, +0.000198527617612853646278381f); + + return MulAdd(Estrin(x, k0, k1, k2, k3, k4, k5), Mul(x, x), x); + } + + // Computes 2^x, where x is an integer. + template <class D, class VI32> + HWY_INLINE Vec<D> Pow2I(D d, VI32 x) { + const Rebind<int32_t, D> di32; + const VI32 kOffset = Set(di32, 0x7F); + return BitCast(d, ShiftLeft<23>(Add(x, kOffset))); + } + + // Sets the exponent of 'x' to 2^e. + template <class D, class V, class VI32> + HWY_INLINE V LoadExpShortRange(D d, V x, VI32 e) { + const VI32 y = ShiftRight<1>(e); + return Mul(Mul(x, Pow2I(d, y)), Pow2I(d, Sub(e, y))); + } + + template <class D, class V, class VI32> + HWY_INLINE V ExpReduce(D d, V x, VI32 q) { + // kLn2Part0f + kLn2Part1f ~= -ln(2) + const V kLn2Part0f = Set(d, -0.693145751953125f); + const V kLn2Part1f = Set(d, -1.428606765330187045e-6f); + + // Extended precision modular arithmetic. + const V qf = ConvertTo(d, q); + x = MulAdd(qf, kLn2Part0f, x); + x = MulAdd(qf, kLn2Part1f, x); + return x; + } +}; + +template <> +struct LogImpl<float> { + template <class D, class V> + HWY_INLINE Vec<Rebind<int32_t, D>> Log2p1NoSubnormal(D /*d*/, V x) { + const Rebind<int32_t, D> di32; + const Rebind<uint32_t, D> du32; + const auto kBias = Set(di32, 0x7F); + return Sub(BitCast(di32, ShiftRight<23>(BitCast(du32, x))), kBias); + } + + // Approximates Log(x) over the range [sqrt(2) / 2, sqrt(2)]. + template <class D, class V> + HWY_INLINE V LogPoly(D d, V x) { + const V k0 = Set(d, 0.66666662693f); + const V k1 = Set(d, 0.40000972152f); + const V k2 = Set(d, 0.28498786688f); + const V k3 = Set(d, 0.24279078841f); + + const V x2 = Mul(x, x); + const V x4 = Mul(x2, x2); + return MulAdd(MulAdd(k2, x4, k0), x2, Mul(MulAdd(k3, x4, k1), x4)); + } +}; + +#if HWY_CAP_FLOAT64 && HWY_CAP_INTEGER64 +template <> +struct ExpImpl<double> { + // Rounds double toward zero and returns as int32_t. + template <class D, class V> + HWY_INLINE Vec<Rebind<int32_t, D>> ToInt32(D /*unused*/, V x) { + return DemoteTo(Rebind<int32_t, D>(), x); + } + + template <class D, class V> + HWY_INLINE V ExpPoly(D d, V x) { + const auto k0 = Set(d, +0.5); + const auto k1 = Set(d, +0.166666666666666851703837); + const auto k2 = Set(d, +0.0416666666666665047591422); + const auto k3 = Set(d, +0.00833333333331652721664984); + const auto k4 = Set(d, +0.00138888888889774492207962); + const auto k5 = Set(d, +0.000198412698960509205564975); + const auto k6 = Set(d, +2.4801587159235472998791e-5); + const auto k7 = Set(d, +2.75572362911928827629423e-6); + const auto k8 = Set(d, +2.75573911234900471893338e-7); + const auto k9 = Set(d, +2.51112930892876518610661e-8); + const auto k10 = Set(d, +2.08860621107283687536341e-9); + + return MulAdd(Estrin(x, k0, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10), + Mul(x, x), x); + } + + // Computes 2^x, where x is an integer. + template <class D, class VI32> + HWY_INLINE Vec<D> Pow2I(D d, VI32 x) { + const Rebind<int32_t, D> di32; + const Rebind<int64_t, D> di64; + const VI32 kOffset = Set(di32, 0x3FF); + return BitCast(d, ShiftLeft<52>(PromoteTo(di64, Add(x, kOffset)))); + } + + // Sets the exponent of 'x' to 2^e. + template <class D, class V, class VI32> + HWY_INLINE V LoadExpShortRange(D d, V x, VI32 e) { + const VI32 y = ShiftRight<1>(e); + return Mul(Mul(x, Pow2I(d, y)), Pow2I(d, Sub(e, y))); + } + + template <class D, class V, class VI32> + HWY_INLINE V ExpReduce(D d, V x, VI32 q) { + // kLn2Part0d + kLn2Part1d ~= -ln(2) + const V kLn2Part0d = Set(d, -0.6931471805596629565116018); + const V kLn2Part1d = Set(d, -0.28235290563031577122588448175e-12); + + // Extended precision modular arithmetic. + const V qf = PromoteTo(d, q); + x = MulAdd(qf, kLn2Part0d, x); + x = MulAdd(qf, kLn2Part1d, x); + return x; + } +}; + +template <> +struct LogImpl<double> { + template <class D, class V> + HWY_INLINE Vec<Rebind<int64_t, D>> Log2p1NoSubnormal(D /*d*/, V x) { + const Rebind<int64_t, D> di64; + const Rebind<uint64_t, D> du64; + return Sub(BitCast(di64, ShiftRight<52>(BitCast(du64, x))), + Set(di64, 0x3FF)); + } + + // Approximates Log(x) over the range [sqrt(2) / 2, sqrt(2)]. + template <class D, class V> + HWY_INLINE V LogPoly(D d, V x) { + const V k0 = Set(d, 0.6666666666666735130); + const V k1 = Set(d, 0.3999999999940941908); + const V k2 = Set(d, 0.2857142874366239149); + const V k3 = Set(d, 0.2222219843214978396); + const V k4 = Set(d, 0.1818357216161805012); + const V k5 = Set(d, 0.1531383769920937332); + const V k6 = Set(d, 0.1479819860511658591); + + const V x2 = Mul(x, x); + const V x4 = Mul(x2, x2); + return MulAdd(MulAdd(MulAdd(MulAdd(k6, x4, k4), x4, k2), x4, k0), x2, + (Mul(MulAdd(MulAdd(k5, x4, k3), x4, k1), x4))); + } +}; + +#endif + +template <class D, class V, bool kAllowSubnormals = true> +HWY_INLINE V Log(const D d, V x) { + // http://git.musl-libc.org/cgit/musl/tree/src/math/log.c for more info. + using T = TFromD<D>; + impl::LogImpl<T> impl; + + constexpr bool kIsF32 = (sizeof(T) == 4); + + // Float Constants + const V kLn2Hi = Set(d, kIsF32 ? static_cast<T>(0.69313812256f) + : static_cast<T>(0.693147180369123816490)); + const V kLn2Lo = Set(d, kIsF32 ? static_cast<T>(9.0580006145e-6f) + : static_cast<T>(1.90821492927058770002e-10)); + const V kOne = Set(d, static_cast<T>(+1.0)); + const V kMinNormal = Set(d, kIsF32 ? static_cast<T>(1.175494351e-38f) + : static_cast<T>(2.2250738585072014e-308)); + const V kScale = Set(d, kIsF32 ? static_cast<T>(3.355443200e+7f) + : static_cast<T>(1.8014398509481984e+16)); + + // Integer Constants + using TI = MakeSigned<T>; + const Rebind<TI, D> di; + using VI = decltype(Zero(di)); + const VI kLowerBits = Set(di, kIsF32 ? static_cast<TI>(0x00000000L) + : static_cast<TI>(0xFFFFFFFFLL)); + const VI kMagic = Set(di, kIsF32 ? static_cast<TI>(0x3F3504F3L) + : static_cast<TI>(0x3FE6A09E00000000LL)); + const VI kExpMask = Set(di, kIsF32 ? static_cast<TI>(0x3F800000L) + : static_cast<TI>(0x3FF0000000000000LL)); + const VI kExpScale = + Set(di, kIsF32 ? static_cast<TI>(-25) : static_cast<TI>(-54)); + const VI kManMask = Set(di, kIsF32 ? static_cast<TI>(0x7FFFFFL) + : static_cast<TI>(0xFFFFF00000000LL)); + + // Scale up 'x' so that it is no longer denormalized. + VI exp_bits; + V exp; + if (kAllowSubnormals == true) { + const auto is_denormal = Lt(x, kMinNormal); + x = IfThenElse(is_denormal, Mul(x, kScale), x); + + // Compute the new exponent. + exp_bits = Add(BitCast(di, x), Sub(kExpMask, kMagic)); + const VI exp_scale = + BitCast(di, IfThenElseZero(is_denormal, BitCast(d, kExpScale))); + exp = ConvertTo( + d, Add(exp_scale, impl.Log2p1NoSubnormal(d, BitCast(d, exp_bits)))); + } else { + // Compute the new exponent. + exp_bits = Add(BitCast(di, x), Sub(kExpMask, kMagic)); + exp = ConvertTo(d, impl.Log2p1NoSubnormal(d, BitCast(d, exp_bits))); + } + + // Renormalize. + const V y = Or(And(x, BitCast(d, kLowerBits)), + BitCast(d, Add(And(exp_bits, kManMask), kMagic))); + + // Approximate and reconstruct. + const V ym1 = Sub(y, kOne); + const V z = Div(ym1, Add(y, kOne)); + + return MulSub( + exp, kLn2Hi, + Sub(MulSub(z, Sub(ym1, impl.LogPoly(d, z)), Mul(exp, kLn2Lo)), ym1)); +} + +} // namespace impl + +template <class D, class V> +HWY_INLINE V Acos(const D d, V x) { + using T = TFromD<D>; + + const V kZero = Zero(d); + const V kHalf = Set(d, static_cast<T>(+0.5)); + const V kPi = Set(d, static_cast<T>(+3.14159265358979323846264)); + const V kPiOverTwo = Set(d, static_cast<T>(+1.57079632679489661923132169)); + + const V sign_x = And(SignBit(d), x); + const V abs_x = Xor(x, sign_x); + const auto mask = Lt(abs_x, kHalf); + const V yy = + IfThenElse(mask, Mul(abs_x, abs_x), NegMulAdd(abs_x, kHalf, kHalf)); + const V y = IfThenElse(mask, abs_x, Sqrt(yy)); + + impl::AsinImpl<T> impl; + const V t = Mul(impl.AsinPoly(d, yy, y), Mul(y, yy)); + + const V t_plus_y = Add(t, y); + const V z = + IfThenElse(mask, Sub(kPiOverTwo, Add(Xor(y, sign_x), Xor(t, sign_x))), + Add(t_plus_y, t_plus_y)); + return IfThenElse(Or(mask, Ge(x, kZero)), z, Sub(kPi, z)); +} + +template <class D, class V> +HWY_INLINE V Acosh(const D d, V x) { + using T = TFromD<D>; + + const V kLarge = Set(d, static_cast<T>(268435456.0)); + const V kLog2 = Set(d, static_cast<T>(0.693147180559945286227)); + const V kOne = Set(d, static_cast<T>(+1.0)); + const V kTwo = Set(d, static_cast<T>(+2.0)); + + const auto is_x_large = Gt(x, kLarge); + const auto is_x_gt_2 = Gt(x, kTwo); + + const V x_minus_1 = Sub(x, kOne); + const V y0 = MulSub(kTwo, x, Div(kOne, Add(Sqrt(MulSub(x, x, kOne)), x))); + const V y1 = + Add(Sqrt(MulAdd(x_minus_1, kTwo, Mul(x_minus_1, x_minus_1))), x_minus_1); + const V y2 = + IfThenElse(is_x_gt_2, IfThenElse(is_x_large, x, y0), Add(y1, kOne)); + const V z = impl::Log<D, V, /*kAllowSubnormals=*/false>(d, y2); + + const auto is_pole = Eq(y2, kOne); + const auto divisor = Sub(IfThenZeroElse(is_pole, y2), kOne); + return Add(IfThenElse(is_x_gt_2, z, + IfThenElse(is_pole, y1, Div(Mul(z, y1), divisor))), + IfThenElseZero(is_x_large, kLog2)); +} + +template <class D, class V> +HWY_INLINE V Asin(const D d, V x) { + using T = TFromD<D>; + + const V kHalf = Set(d, static_cast<T>(+0.5)); + const V kTwo = Set(d, static_cast<T>(+2.0)); + const V kPiOverTwo = Set(d, static_cast<T>(+1.57079632679489661923132169)); + + const V sign_x = And(SignBit(d), x); + const V abs_x = Xor(x, sign_x); + const auto mask = Lt(abs_x, kHalf); + const V yy = + IfThenElse(mask, Mul(abs_x, abs_x), NegMulAdd(abs_x, kHalf, kHalf)); + const V y = IfThenElse(mask, abs_x, Sqrt(yy)); + + impl::AsinImpl<T> impl; + const V z0 = MulAdd(impl.AsinPoly(d, yy, y), Mul(yy, y), y); + const V z1 = NegMulAdd(z0, kTwo, kPiOverTwo); + return Or(IfThenElse(mask, z0, z1), sign_x); +} + +template <class D, class V> +HWY_INLINE V Asinh(const D d, V x) { + using T = TFromD<D>; + + const V kSmall = Set(d, static_cast<T>(1.0 / 268435456.0)); + const V kLarge = Set(d, static_cast<T>(268435456.0)); + const V kLog2 = Set(d, static_cast<T>(0.693147180559945286227)); + const V kOne = Set(d, static_cast<T>(+1.0)); + const V kTwo = Set(d, static_cast<T>(+2.0)); + + const V sign_x = And(SignBit(d), x); // Extract the sign bit + const V abs_x = Xor(x, sign_x); + + const auto is_x_large = Gt(abs_x, kLarge); + const auto is_x_lt_2 = Lt(abs_x, kTwo); + + const V x2 = Mul(x, x); + const V sqrt_x2_plus_1 = Sqrt(Add(x2, kOne)); + + const V y0 = MulAdd(abs_x, kTwo, Div(kOne, Add(sqrt_x2_plus_1, abs_x))); + const V y1 = Add(Div(x2, Add(sqrt_x2_plus_1, kOne)), abs_x); + const V y2 = + IfThenElse(is_x_lt_2, Add(y1, kOne), IfThenElse(is_x_large, abs_x, y0)); + const V z = impl::Log<D, V, /*kAllowSubnormals=*/false>(d, y2); + + const auto is_pole = Eq(y2, kOne); + const auto divisor = Sub(IfThenZeroElse(is_pole, y2), kOne); + const auto large = IfThenElse(is_pole, y1, Div(Mul(z, y1), divisor)); + const V y = IfThenElse(Lt(abs_x, kSmall), x, large); + return Or(Add(IfThenElse(is_x_lt_2, y, z), IfThenElseZero(is_x_large, kLog2)), + sign_x); +} + +template <class D, class V> +HWY_INLINE V Atan(const D d, V x) { + using T = TFromD<D>; + + const V kOne = Set(d, static_cast<T>(+1.0)); + const V kPiOverTwo = Set(d, static_cast<T>(+1.57079632679489661923132169)); + + const V sign = And(SignBit(d), x); + const V abs_x = Xor(x, sign); + const auto mask = Gt(abs_x, kOne); + + impl::AtanImpl<T> impl; + const auto divisor = IfThenElse(mask, abs_x, kOne); + const V y = impl.AtanPoly(d, IfThenElse(mask, Div(kOne, divisor), abs_x)); + return Or(IfThenElse(mask, Sub(kPiOverTwo, y), y), sign); +} + +template <class D, class V> +HWY_INLINE V Atanh(const D d, V x) { + using T = TFromD<D>; + + const V kHalf = Set(d, static_cast<T>(+0.5)); + const V kOne = Set(d, static_cast<T>(+1.0)); + + const V sign = And(SignBit(d), x); // Extract the sign bit + const V abs_x = Xor(x, sign); + return Mul(Log1p(d, Div(Add(abs_x, abs_x), Sub(kOne, abs_x))), + Xor(kHalf, sign)); +} + +template <class D, class V> +HWY_INLINE V Cos(const D d, V x) { + using T = TFromD<D>; + impl::CosSinImpl<T> impl; + + // Float Constants + const V kOneOverPi = Set(d, static_cast<T>(0.31830988618379067153)); + + // Integer Constants + const Rebind<int32_t, D> di32; + using VI32 = decltype(Zero(di32)); + const VI32 kOne = Set(di32, 1); + + const V y = Abs(x); // cos(x) == cos(|x|) + + // Compute the quadrant, q = int(|x| / pi) * 2 + 1 + const VI32 q = Add(ShiftLeft<1>(impl.ToInt32(d, Mul(y, kOneOverPi))), kOne); + + // Reduce range, apply sign, and approximate. + return impl.Poly( + d, Xor(impl.CosReduce(d, y, q), impl.CosSignFromQuadrant(d, q))); +} + +template <class D, class V> +HWY_INLINE V Exp(const D d, V x) { + using T = TFromD<D>; + + const V kHalf = Set(d, static_cast<T>(+0.5)); + const V kLowerBound = + Set(d, static_cast<T>((sizeof(T) == 4 ? -104.0 : -1000.0))); + const V kNegZero = Set(d, static_cast<T>(-0.0)); + const V kOne = Set(d, static_cast<T>(+1.0)); + const V kOneOverLog2 = Set(d, static_cast<T>(+1.442695040888963407359924681)); + + impl::ExpImpl<T> impl; + + // q = static_cast<int32>((x / log(2)) + ((x < 0) ? -0.5 : +0.5)) + const auto q = + impl.ToInt32(d, MulAdd(x, kOneOverLog2, Or(kHalf, And(x, kNegZero)))); + + // Reduce, approximate, and then reconstruct. + const V y = impl.LoadExpShortRange( + d, Add(impl.ExpPoly(d, impl.ExpReduce(d, x, q)), kOne), q); + return IfThenElseZero(Ge(x, kLowerBound), y); +} + +template <class D, class V> +HWY_INLINE V Expm1(const D d, V x) { + using T = TFromD<D>; + + const V kHalf = Set(d, static_cast<T>(+0.5)); + const V kLowerBound = + Set(d, static_cast<T>((sizeof(T) == 4 ? -104.0 : -1000.0))); + const V kLn2Over2 = Set(d, static_cast<T>(+0.346573590279972654708616)); + const V kNegOne = Set(d, static_cast<T>(-1.0)); + const V kNegZero = Set(d, static_cast<T>(-0.0)); + const V kOne = Set(d, static_cast<T>(+1.0)); + const V kOneOverLog2 = Set(d, static_cast<T>(+1.442695040888963407359924681)); + + impl::ExpImpl<T> impl; + + // q = static_cast<int32>((x / log(2)) + ((x < 0) ? -0.5 : +0.5)) + const auto q = + impl.ToInt32(d, MulAdd(x, kOneOverLog2, Or(kHalf, And(x, kNegZero)))); + + // Reduce, approximate, and then reconstruct. + const V y = impl.ExpPoly(d, impl.ExpReduce(d, x, q)); + const V z = IfThenElse(Lt(Abs(x), kLn2Over2), y, + Sub(impl.LoadExpShortRange(d, Add(y, kOne), q), kOne)); + return IfThenElse(Lt(x, kLowerBound), kNegOne, z); +} + +template <class D, class V> +HWY_INLINE V Log(const D d, V x) { + return impl::Log<D, V, /*kAllowSubnormals=*/true>(d, x); +} + +template <class D, class V> +HWY_INLINE V Log10(const D d, V x) { + using T = TFromD<D>; + return Mul(Log(d, x), Set(d, static_cast<T>(0.4342944819032518276511))); +} + +template <class D, class V> +HWY_INLINE V Log1p(const D d, V x) { + using T = TFromD<D>; + const V kOne = Set(d, static_cast<T>(+1.0)); + + const V y = Add(x, kOne); + const auto is_pole = Eq(y, kOne); + const auto divisor = Sub(IfThenZeroElse(is_pole, y), kOne); + const auto non_pole = + Mul(impl::Log<D, V, /*kAllowSubnormals=*/false>(d, y), Div(x, divisor)); + return IfThenElse(is_pole, x, non_pole); +} + +template <class D, class V> +HWY_INLINE V Log2(const D d, V x) { + using T = TFromD<D>; + return Mul(Log(d, x), Set(d, static_cast<T>(1.44269504088896340735992))); +} + +template <class D, class V> +HWY_INLINE V Sin(const D d, V x) { + using T = TFromD<D>; + impl::CosSinImpl<T> impl; + + // Float Constants + const V kOneOverPi = Set(d, static_cast<T>(0.31830988618379067153)); + const V kHalf = Set(d, static_cast<T>(0.5)); + + // Integer Constants + const Rebind<int32_t, D> di32; + using VI32 = decltype(Zero(di32)); + + const V abs_x = Abs(x); + const V sign_x = Xor(abs_x, x); + + // Compute the quadrant, q = int((|x| / pi) + 0.5) + const VI32 q = impl.ToInt32(d, MulAdd(abs_x, kOneOverPi, kHalf)); + + // Reduce range, apply sign, and approximate. + return impl.Poly(d, Xor(impl.SinReduce(d, abs_x, q), + Xor(impl.SinSignFromQuadrant(d, q), sign_x))); +} + +template <class D, class V> +HWY_INLINE V Sinh(const D d, V x) { + using T = TFromD<D>; + const V kHalf = Set(d, static_cast<T>(+0.5)); + const V kOne = Set(d, static_cast<T>(+1.0)); + const V kTwo = Set(d, static_cast<T>(+2.0)); + + const V sign = And(SignBit(d), x); // Extract the sign bit + const V abs_x = Xor(x, sign); + const V y = Expm1(d, abs_x); + const V z = Mul(Div(Add(y, kTwo), Add(y, kOne)), Mul(y, kHalf)); + return Xor(z, sign); // Reapply the sign bit +} + +template <class D, class V> +HWY_INLINE V Tanh(const D d, V x) { + using T = TFromD<D>; + const V kLimit = Set(d, static_cast<T>(18.714973875)); + const V kOne = Set(d, static_cast<T>(+1.0)); + const V kTwo = Set(d, static_cast<T>(+2.0)); + + const V sign = And(SignBit(d), x); // Extract the sign bit + const V abs_x = Xor(x, sign); + const V y = Expm1(d, Mul(abs_x, kTwo)); + const V z = IfThenElse(Gt(abs_x, kLimit), kOne, Div(y, Add(y, kTwo))); + return Xor(z, sign); // Reapply the sign bit +} + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); + +#endif // HIGHWAY_HWY_CONTRIB_MATH_MATH_INL_H_ diff --git a/third_party/highway/hwy/contrib/math/math_test.cc b/third_party/highway/hwy/contrib/math/math_test.cc new file mode 100644 index 0000000..cf9f8e3 --- /dev/null +++ b/third_party/highway/hwy/contrib/math/math_test.cc @@ -0,0 +1,241 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <stdio.h> + +#include <cfloat> // FLT_MAX +#include <type_traits> + +// clang-format off +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "hwy/contrib/math/math_test.cc" +#include "hwy/foreach_target.h" + +#include "hwy/contrib/math/math-inl.h" +#include "hwy/tests/test_util-inl.h" +// clang-format on + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +template <class Out, class In> +inline Out BitCast(const In& in) { + static_assert(sizeof(Out) == sizeof(In), ""); + Out out; + CopyBytes<sizeof(out)>(&in, &out); + return out; +} + +template <class T, class D> +HWY_NOINLINE void TestMath(const std::string name, T (*fx1)(T), + Vec<D> (*fxN)(D, VecArg<Vec<D>>), D d, T min, T max, + uint64_t max_error_ulp) { + using UintT = MakeUnsigned<T>; + + const UintT min_bits = BitCast<UintT>(min); + const UintT max_bits = BitCast<UintT>(max); + + // If min is negative and max is positive, the range needs to be broken into + // two pieces, [+0, max] and [-0, min], otherwise [min, max]. + int range_count = 1; + UintT ranges[2][2] = {{min_bits, max_bits}, {0, 0}}; + if ((min < 0.0) && (max > 0.0)) { + ranges[0][0] = BitCast<UintT>(static_cast<T>(+0.0)); + ranges[0][1] = max_bits; + ranges[1][0] = BitCast<UintT>(static_cast<T>(-0.0)); + ranges[1][1] = min_bits; + range_count = 2; + } + + uint64_t max_ulp = 0; + // Emulation is slower, so cannot afford as many. + constexpr UintT kSamplesPerRange = static_cast<UintT>(AdjustedReps(10000)); + for (int range_index = 0; range_index < range_count; ++range_index) { + const UintT start = ranges[range_index][0]; + const UintT stop = ranges[range_index][1]; + const UintT step = HWY_MAX(1, ((stop - start) / kSamplesPerRange)); + for (UintT value_bits = start; value_bits <= stop; value_bits += step) { + // For reasons unknown, the HWY_MAX is necessary on RVV, otherwise + // value_bits can be less than start, and thus possibly NaN. + const T value = BitCast<T>(HWY_MIN(HWY_MAX(start, value_bits), stop)); + const T actual = GetLane(fxN(d, Set(d, value))); + const T expected = fx1(value); + + // Skip small inputs and outputs on armv7, it flushes subnormals to zero. +#if HWY_TARGET == HWY_NEON && HWY_ARCH_ARM_V7 + if ((std::abs(value) < 1e-37f) || (std::abs(expected) < 1e-37f)) { + continue; + } +#endif + + const auto ulp = hwy::detail::ComputeUlpDelta(actual, expected); + max_ulp = HWY_MAX(max_ulp, ulp); + if (ulp > max_error_ulp) { + fprintf(stderr, + "%s: %s(%f) expected %f actual %f ulp %" PRIu64 " max ulp %u\n", + hwy::TypeName(T(), Lanes(d)).c_str(), name.c_str(), value, + expected, actual, static_cast<uint64_t>(ulp), + static_cast<uint32_t>(max_error_ulp)); + } + } + } + fprintf(stderr, "%s: %s max_ulp %" PRIu64 "\n", + hwy::TypeName(T(), Lanes(d)).c_str(), name.c_str(), max_ulp); + HWY_ASSERT(max_ulp <= max_error_ulp); +} + +// TODO(janwas): remove once RVV supports fractional LMUL +#undef DEFINE_MATH_TEST_FUNC +#if HWY_TARGET == HWY_RVV + +#define DEFINE_MATH_TEST_FUNC(NAME) \ + HWY_NOINLINE void TestAll##NAME() { \ + ForFloatTypes(ForShrinkableVectors<Test##NAME>()); \ + } + +#else + +#define DEFINE_MATH_TEST_FUNC(NAME) \ + HWY_NOINLINE void TestAll##NAME() { \ + ForFloatTypes(ForPartialVectors<Test##NAME>()); \ + } + +#endif + +#undef DEFINE_MATH_TEST +#define DEFINE_MATH_TEST(NAME, F32x1, F32xN, F32_MIN, F32_MAX, F32_ERROR, \ + F64x1, F64xN, F64_MIN, F64_MAX, F64_ERROR) \ + struct Test##NAME { \ + template <class T, class D> \ + HWY_NOINLINE void operator()(T, D d) { \ + if (sizeof(T) == 4) { \ + TestMath<T, D>(HWY_STR(NAME), F32x1, F32xN, d, F32_MIN, F32_MAX, \ + F32_ERROR); \ + } else { \ + TestMath<T, D>(HWY_STR(NAME), F64x1, F64xN, d, \ + static_cast<T>(F64_MIN), static_cast<T>(F64_MAX), \ + F64_ERROR); \ + } \ + } \ + }; \ + DEFINE_MATH_TEST_FUNC(NAME) + +// Floating point values closest to but less than 1.0 +const float kNearOneF = BitCast<float>(0x3F7FFFFF); +const double kNearOneD = BitCast<double>(0x3FEFFFFFFFFFFFFFULL); + +// The discrepancy is unacceptably large for MSYS2 (less accurate libm?), so +// only increase the error tolerance there. +constexpr uint64_t Cos64ULP() { +#if defined(__MINGW32__) + return 23; +#else + return 3; +#endif +} + +constexpr uint64_t ACosh32ULP() { +#if defined(__MINGW32__) + return 8; +#else + return 3; +#endif +} + +// clang-format off +DEFINE_MATH_TEST(Acos, + std::acos, CallAcos, -1.0f, +1.0f, 3, // NEON is 3 instead of 2 + std::acos, CallAcos, -1.0, +1.0, 2) +DEFINE_MATH_TEST(Acosh, + std::acosh, CallAcosh, +1.0f, +FLT_MAX, ACosh32ULP(), + std::acosh, CallAcosh, +1.0, +DBL_MAX, 3) +DEFINE_MATH_TEST(Asin, + std::asin, CallAsin, -1.0f, +1.0f, 4, // ARMv7 is 4 instead of 2 + std::asin, CallAsin, -1.0, +1.0, 2) +DEFINE_MATH_TEST(Asinh, + std::asinh, CallAsinh, -FLT_MAX, +FLT_MAX, 3, + std::asinh, CallAsinh, -DBL_MAX, +DBL_MAX, 3) +DEFINE_MATH_TEST(Atan, + std::atan, CallAtan, -FLT_MAX, +FLT_MAX, 3, + std::atan, CallAtan, -DBL_MAX, +DBL_MAX, 3) +DEFINE_MATH_TEST(Atanh, + std::atanh, CallAtanh, -kNearOneF, +kNearOneF, 4, // NEON is 4 instead of 3 + std::atanh, CallAtanh, -kNearOneD, +kNearOneD, 3) +DEFINE_MATH_TEST(Cos, + std::cos, CallCos, -39000.0f, +39000.0f, 3, + std::cos, CallCos, -39000.0, +39000.0, Cos64ULP()) +DEFINE_MATH_TEST(Exp, + std::exp, CallExp, -FLT_MAX, +104.0f, 1, + std::exp, CallExp, -DBL_MAX, +104.0, 1) +DEFINE_MATH_TEST(Expm1, + std::expm1, CallExpm1, -FLT_MAX, +104.0f, 4, + std::expm1, CallExpm1, -DBL_MAX, +104.0, 4) +DEFINE_MATH_TEST(Log, + std::log, CallLog, +FLT_MIN, +FLT_MAX, 1, + std::log, CallLog, +DBL_MIN, +DBL_MAX, 1) +DEFINE_MATH_TEST(Log10, + std::log10, CallLog10, +FLT_MIN, +FLT_MAX, 2, + std::log10, CallLog10, +DBL_MIN, +DBL_MAX, 2) +DEFINE_MATH_TEST(Log1p, + std::log1p, CallLog1p, +0.0f, +1e37f, 3, // NEON is 3 instead of 2 + std::log1p, CallLog1p, +0.0, +DBL_MAX, 2) +DEFINE_MATH_TEST(Log2, + std::log2, CallLog2, +FLT_MIN, +FLT_MAX, 2, + std::log2, CallLog2, +DBL_MIN, +DBL_MAX, 2) +DEFINE_MATH_TEST(Sin, + std::sin, CallSin, -39000.0f, +39000.0f, 3, + std::sin, CallSin, -39000.0, +39000.0, 4) // MSYS is 4 instead of 3 +DEFINE_MATH_TEST(Sinh, + std::sinh, CallSinh, -80.0f, +80.0f, 4, + std::sinh, CallSinh, -709.0, +709.0, 4) +DEFINE_MATH_TEST(Tanh, + std::tanh, CallTanh, -FLT_MAX, +FLT_MAX, 4, + std::tanh, CallTanh, -DBL_MAX, +DBL_MAX, 4) +// clang-format on + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE + +namespace hwy { +HWY_BEFORE_TEST(HwyMathTest); +HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllAcos); +HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllAcosh); +HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllAsin); +HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllAsinh); +HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllAtan); +HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllAtanh); +HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllCos); +HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllExp); +HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllExpm1); +HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllLog); +HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllLog10); +HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllLog1p); +HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllLog2); +HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllSin); +HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllSinh); +HWY_EXPORT_AND_TEST_P(HwyMathTest, TestAllTanh); +} // namespace hwy + +// Ought not to be necessary, but without this, no tests run on RVV. +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#endif diff --git a/third_party/highway/hwy/contrib/sort/sort-inl.h b/third_party/highway/hwy/contrib/sort/sort-inl.h new file mode 100644 index 0000000..75ccd39 --- /dev/null +++ b/third_party/highway/hwy/contrib/sort/sort-inl.h @@ -0,0 +1,909 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Per-target include guard + +#if defined(HIGHWAY_HWY_CONTRIB_SORT_SORT_INL_H_) == \ + defined(HWY_TARGET_TOGGLE) +#ifdef HIGHWAY_HWY_CONTRIB_SORT_SORT_INL_H_ +#undef HIGHWAY_HWY_CONTRIB_SORT_SORT_INL_H_ +#else +#define HIGHWAY_HWY_CONTRIB_SORT_SORT_INL_H_ +#endif + +#include <inttypes.h> + +#include "hwy/aligned_allocator.h" +#include "hwy/highway.h" + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +enum class SortOrder { kAscending, kDescending }; + +#if HWY_TARGET != HWY_SCALAR && HWY_ARCH_X86 + +#define HWY_SORT_VERIFY 1 + +constexpr inline SortOrder Reverse(SortOrder order) { + return (order == SortOrder::kAscending) ? SortOrder::kDescending + : SortOrder::kAscending; +} + +namespace verify { + +template <typename T> +bool Compare(T a, T b, SortOrder kOrder) { + if (kOrder == SortOrder::kAscending) return a <= b; + return a >= b; +} + +#if HWY_SORT_VERIFY + +template <class D> +class Runs { + using T = TFromD<D>; + + public: + Runs(D d, size_t num_regs, size_t run_length = 0, bool alternating = false) { + const size_t N = Lanes(d); + + buf_ = AllocateAligned<T>(N); + consecutive_ = AllocateAligned<T>(num_regs * N); + + num_regs_ = num_regs; + if (run_length) { + run_length_ = run_length; + num_runs_ = num_regs * N / run_length; + is_vector_ = true; + alternating_ = alternating; + } else { + run_length_ = num_regs * 4; + num_runs_ = N / 4; + is_vector_ = false; + alternating_ = false; + } + } + + void ScatterQuartets(D d, const size_t idx_reg, Vec<D> v) { + HWY_ASSERT(idx_reg < num_regs_); + const size_t N = Lanes(d); + for (size_t i = 0; i < N; i += 4) { + Store(v, d, buf_.get()); + const size_t idx_q = (i / 4) * num_regs_ + idx_reg; + CopyBytes<16>(buf_.get() + i, consecutive_.get() + idx_q * 4); + } + } + + void StoreVector(D d, const size_t idx_reg, Vec<D> v) { + HWY_ASSERT(idx_reg < num_regs_); + Store(v, d, &consecutive_[idx_reg * Lanes(d)]); + } + + bool IsBitonic() const { + HWY_ASSERT(!alternating_); + for (size_t ir = 0; ir < num_runs_; ++ir) { + const T* p = &consecutive_[ir * run_length_]; + bool is_asc = true; + bool is_desc = true; + bool is_zero = true; + + for (size_t i = 0; i < run_length_ / 2 - 1; ++i) { + is_asc &= (p[i] <= p[i + 1]); + is_desc &= (p[i] >= p[i + 1]); + } + for (size_t i = 0; i < run_length_; ++i) { + is_zero &= (p[i] == 0); + } + + bool is_asc2 = true; + bool is_desc2 = true; + for (size_t i = run_length_ / 2; i < run_length_ - 1; ++i) { + is_asc2 &= (p[i] <= p[i + 1]); + is_desc2 &= (p[i] >= p[i + 1]); + } + + if (is_zero) continue; + if (is_asc && is_desc2) continue; + if (is_desc && is_asc2) continue; + return false; + } + return true; + } + + void CheckBitonic(int line, int caller) const { + if (IsBitonic()) return; + for (size_t ir = 0; ir < num_runs_; ++ir) { + const T* p = &consecutive_[ir * run_length_]; + printf("run %" PRIu64 " (len %" PRIu64 ")\n", static_cast<uint64_t>(ir), + static_cast<uint64_t>(run_length_)); + for (size_t i = 0; i < run_length_; ++i) { + printf("%.0f\n", static_cast<float>(p[i])); + } + } + printf("caller %d\n", caller); + hwy::Abort("", line, "not bitonic"); + } + + void CheckSorted(SortOrder kOrder, int line, int caller) const { + for (size_t ir = 0; ir < num_runs_; ++ir) { + const SortOrder order = + (alternating_ && (ir & 1)) ? Reverse(kOrder) : kOrder; + const T* p = &consecutive_[ir * run_length_]; + + for (size_t i = 0; i < run_length_ - 1; ++i) { + if (!Compare(p[i], p[i + 1], order)) { + printf("ir%" PRIu64 " run_length=%" PRIu64 + " alt=%d original order=%d this order=%d\n", + static_cast<uint64_t>(ir), static_cast<uint64_t>(run_length_), + alternating_, static_cast<int>(kOrder), + static_cast<int>(order)); + for (size_t i = 0; i < run_length_; ++i) { + printf(" %.0f\n", static_cast<float>(p[i])); + } + printf("caller %d\n", caller); + hwy::Abort("", line, "not sorted"); + } + } + } + } + + private: + AlignedFreeUniquePtr<T[]> buf_; + AlignedFreeUniquePtr<T[]> consecutive_; + size_t num_regs_; + size_t run_length_; + size_t num_runs_; + bool is_vector_; + bool alternating_; +}; + +template <class D> +Runs<D> StoreDeinterleavedQuartets(D d, Vec<D> v0) { + Runs<D> runs(d, 1); + runs.ScatterQuartets(d, 0, v0); + return runs; +} + +template <class D> +Runs<D> StoreDeinterleavedQuartets(D d, Vec<D> v0, Vec<D> v1) { + Runs<D> runs(d, 2); + runs.ScatterQuartets(d, 0, v0); + runs.ScatterQuartets(d, 1, v1); + return runs; +} + +template <class D> +Runs<D> StoreDeinterleavedQuartets(D d, Vec<D> v0, Vec<D> v1, Vec<D> v2, + Vec<D> v3) { + Runs<D> runs(d, 4); + runs.ScatterQuartets(d, 0, v0); + runs.ScatterQuartets(d, 1, v1); + runs.ScatterQuartets(d, 2, v2); + runs.ScatterQuartets(d, 3, v3); + return runs; +} + +template <class D> +Runs<D> StoreDeinterleavedQuartets(D d, Vec<D> v0, Vec<D> v1, Vec<D> v2, + Vec<D> v3, Vec<D> v4, Vec<D> v5, Vec<D> v6, + Vec<D> v7) { + Runs<D> runs(d, 8); + runs.ScatterQuartets(d, 0, v0); + runs.ScatterQuartets(d, 1, v1); + runs.ScatterQuartets(d, 2, v2); + runs.ScatterQuartets(d, 3, v3); + runs.ScatterQuartets(d, 4, v4); + runs.ScatterQuartets(d, 5, v5); + runs.ScatterQuartets(d, 6, v6); + runs.ScatterQuartets(d, 7, v7); + return runs; +} + +template <class D> +Runs<D> StoreDeinterleavedQuartets(D d, Vec<D> v0, Vec<D> v1, Vec<D> v2, + Vec<D> v3, Vec<D> v4, Vec<D> v5, Vec<D> v6, + Vec<D> v7, Vec<D> v8, Vec<D> v9, Vec<D> vA, + Vec<D> vB, Vec<D> vC, Vec<D> vD, Vec<D> vE, + Vec<D> vF) { + Runs<D> runs(d, 16); + runs.ScatterQuartets(d, 0x0, v0); + runs.ScatterQuartets(d, 0x1, v1); + runs.ScatterQuartets(d, 0x2, v2); + runs.ScatterQuartets(d, 0x3, v3); + runs.ScatterQuartets(d, 0x4, v4); + runs.ScatterQuartets(d, 0x5, v5); + runs.ScatterQuartets(d, 0x6, v6); + runs.ScatterQuartets(d, 0x7, v7); + runs.ScatterQuartets(d, 0x8, v8); + runs.ScatterQuartets(d, 0x9, v9); + runs.ScatterQuartets(d, 0xA, vA); + runs.ScatterQuartets(d, 0xB, vB); + runs.ScatterQuartets(d, 0xC, vC); + runs.ScatterQuartets(d, 0xD, vD); + runs.ScatterQuartets(d, 0xE, vE); + runs.ScatterQuartets(d, 0xF, vF); + return runs; +} + +template <class D> +Runs<D> StoreDeinterleavedQuartets( + D d, const Vec<D>& v00, const Vec<D>& v01, const Vec<D>& v02, + const Vec<D>& v03, const Vec<D>& v04, const Vec<D>& v05, const Vec<D>& v06, + const Vec<D>& v07, const Vec<D>& v08, const Vec<D>& v09, const Vec<D>& v0A, + const Vec<D>& v0B, const Vec<D>& v0C, const Vec<D>& v0D, const Vec<D>& v0E, + const Vec<D>& v0F, const Vec<D>& v10, const Vec<D>& v11, const Vec<D>& v12, + const Vec<D>& v13, const Vec<D>& v14, const Vec<D>& v15, const Vec<D>& v16, + const Vec<D>& v17, const Vec<D>& v18, const Vec<D>& v19, const Vec<D>& v1A, + const Vec<D>& v1B, const Vec<D>& v1C, const Vec<D>& v1D, const Vec<D>& v1E, + const Vec<D>& v1F) { + Runs<D> runs(d, 32); + runs.ScatterQuartets(d, 0x00, v00); + runs.ScatterQuartets(d, 0x01, v01); + runs.ScatterQuartets(d, 0x02, v02); + runs.ScatterQuartets(d, 0x03, v03); + runs.ScatterQuartets(d, 0x04, v04); + runs.ScatterQuartets(d, 0x05, v05); + runs.ScatterQuartets(d, 0x06, v06); + runs.ScatterQuartets(d, 0x07, v07); + runs.ScatterQuartets(d, 0x08, v08); + runs.ScatterQuartets(d, 0x09, v09); + runs.ScatterQuartets(d, 0x0A, v0A); + runs.ScatterQuartets(d, 0x0B, v0B); + runs.ScatterQuartets(d, 0x0C, v0C); + runs.ScatterQuartets(d, 0x0D, v0D); + runs.ScatterQuartets(d, 0x0E, v0E); + runs.ScatterQuartets(d, 0x0F, v0F); + runs.ScatterQuartets(d, 0x10, v10); + runs.ScatterQuartets(d, 0x11, v11); + runs.ScatterQuartets(d, 0x12, v12); + runs.ScatterQuartets(d, 0x13, v13); + runs.ScatterQuartets(d, 0x14, v14); + runs.ScatterQuartets(d, 0x15, v15); + runs.ScatterQuartets(d, 0x16, v16); + runs.ScatterQuartets(d, 0x17, v17); + runs.ScatterQuartets(d, 0x18, v18); + runs.ScatterQuartets(d, 0x19, v19); + runs.ScatterQuartets(d, 0x1A, v1A); + runs.ScatterQuartets(d, 0x1B, v1B); + runs.ScatterQuartets(d, 0x1C, v1C); + runs.ScatterQuartets(d, 0x1D, v1D); + runs.ScatterQuartets(d, 0x1E, v1E); + runs.ScatterQuartets(d, 0x1F, v1F); + return runs; +} + +template <class D> +Runs<D> StoreVectors(D d, Vec<D> v0, size_t run_length, bool alternating) { + Runs<D> runs(d, 1, run_length, alternating); + runs.StoreVector(d, 0, v0); + return runs; +} + +template <class D> +Runs<D> StoreVectors(D d, Vec<D> v0, Vec<D> v1) { + constexpr size_t kRegs = 2; + Runs<D> runs(d, kRegs, /*run_length=*/kRegs * Lanes(d), /*alternating=*/false); + runs.StoreVector(d, 0, v0); + runs.StoreVector(d, 1, v1); + return runs; +} + +template <class D> +Runs<D> StoreVectors(D d, Vec<D> v0, Vec<D> v1, Vec<D> v2, Vec<D> v3) { + constexpr size_t kRegs = 4; + Runs<D> runs(d, kRegs, /*run_length=*/kRegs * Lanes(d), /*alternating=*/false); + runs.StoreVector(d, 0, v0); + runs.StoreVector(d, 1, v1); + runs.StoreVector(d, 2, v2); + runs.StoreVector(d, 3, v3); + return runs; +} + +template <class D> +Runs<D> StoreVectors(D d, Vec<D> v0, Vec<D> v1, Vec<D> v2, Vec<D> v3, Vec<D> v4, + Vec<D> v5, Vec<D> v6, Vec<D> v7) { + constexpr size_t kRegs = 8; + Runs<D> runs(d, kRegs, /*run_length=*/kRegs * Lanes(d), /*alternating=*/false); + runs.StoreVector(d, 0, v0); + runs.StoreVector(d, 1, v1); + runs.StoreVector(d, 2, v2); + runs.StoreVector(d, 3, v3); + runs.StoreVector(d, 4, v4); + runs.StoreVector(d, 5, v5); + runs.StoreVector(d, 6, v6); + runs.StoreVector(d, 7, v7); + return runs; +} + +#endif // HWY_SORT_VERIFY +} // namespace verify + +namespace detail { + +// ------------------------------ Vector-length agnostic (quartets) + +// For each lane i: replaces a[i] with the first and b[i] with the second +// according to kOrder. +// Corresponds to a conditional swap, which is one "node" of a sorting network. +// Min/Max are cheaper than compare + blend at least for integers. +template <SortOrder kOrder, class V> +HWY_INLINE void SortLanesIn2Vectors(V& a, V& b) { + V temp = a; + a = (kOrder == SortOrder::kAscending) ? Min(a, b) : Max(a, b); + b = (kOrder == SortOrder::kAscending) ? Max(temp, b) : Min(temp, b); +} + +// For each lane: sorts the four values in the that lane of the four vectors. +template <SortOrder kOrder, class D, class V = Vec<D>> +HWY_INLINE void SortLanesIn4Vectors(D d, const TFromD<D>* in, V& v0, V& v1, + V& v2, V& v3) { + const size_t N = Lanes(d); + + // Bitonic and odd-even sorters both have 5 nodes. This one is from + // http://users.telenet.be/bertdobbelaere/SorterHunter/sorting_networks.html + + // layer 1 + v0 = Load(d, in + 0 * N); + v2 = Load(d, in + 2 * N); + SortLanesIn2Vectors<kOrder>(v0, v2); + v1 = Load(d, in + 1 * N); + v3 = Load(d, in + 3 * N); + SortLanesIn2Vectors<kOrder>(v1, v3); + + // layer 2 + SortLanesIn2Vectors<kOrder>(v0, v1); + SortLanesIn2Vectors<kOrder>(v2, v3); + + // layer 3 + SortLanesIn2Vectors<kOrder>(v1, v2); +} + +// Inputs are vectors with columns in sorted order (from SortLanesIn4Vectors). +// Transposes so that output vectors are sorted quartets (128-bit blocks), +// and a quartet in v0 comes before its counterpart in v1, etc. +template <class D, class V = Vec<D>> +HWY_INLINE void Transpose4x4(D d, V& v0, V& v1, V& v2, V& v3) { + const RepartitionToWide<decltype(d)> dw; + + // Input: first number is reg, second is lane (0 is lowest) + // 03 02 01 00 | + // 13 12 11 10 | columns are sorted + // 23 22 21 20 | (in this order) + // 33 32 31 30 V + const V t0 = InterleaveLower(d, v0, v1); // 11 01 10 00 + const V t1 = InterleaveLower(d, v2, v3); // 31 21 30 20 + const V t2 = InterleaveUpper(d, v0, v1); // 13 03 12 02 + const V t3 = InterleaveUpper(d, v2, v3); // 33 23 32 22 + + // 30 20 10 00 + v0 = BitCast(d, InterleaveLower(BitCast(dw, t0), BitCast(dw, t1))); + // 31 21 11 01 + v1 = BitCast(d, InterleaveUpper(BitCast(dw, t0), BitCast(dw, t1))); + // 32 22 12 02 + v2 = BitCast(d, InterleaveLower(BitCast(dw, t2), BitCast(dw, t3))); + // 33 23 13 03 --> sorted in descending order (03=smallest in lane 0). + v3 = BitCast(d, InterleaveUpper(BitCast(dw, t2), BitCast(dw, t3))); +} + +// 12 ops (including 4 swizzle) +// Precondition: v0 and v1 are already sorted according to kOrder. +// Postcondition: concatenate(v0, v1) is sorted and v0 is the lower half. +template <SortOrder kOrder, class D, class V = Vec<D>> +HWY_INLINE void Merge2SortedQuartets(D d, V& v0, V& v1, int caller) { +#if HWY_SORT_VERIFY + const verify::Runs<D> input0 = verify::StoreDeinterleavedQuartets(d, v0); + const verify::Runs<D> input1 = verify::StoreDeinterleavedQuartets(d, v1); + input0.CheckSorted(kOrder, __LINE__, caller); + input1.CheckSorted(kOrder, __LINE__, caller); +#endif + + // See figure 5 from https://www.vldb.org/pvldb/vol8/p1274-inoue.pdf. + // This requires 8 min/max vs 6 for bitonic merge (see Figure 2 in + // http://www.vldb.org/pvldb/vol1/1454171.pdf), but is faster overall because + // it needs less shuffling, and does not need a bitonic input. + SortLanesIn2Vectors<kOrder>(v0, v1); + v0 = Shuffle0321(v0); + SortLanesIn2Vectors<kOrder>(v0, v1); + v0 = Shuffle0321(v0); + SortLanesIn2Vectors<kOrder>(v0, v1); + v0 = Shuffle0321(v0); + SortLanesIn2Vectors<kOrder>(v0, v1); + v0 = Shuffle0321(v0); + +#if HWY_SORT_VERIFY + auto output = verify::StoreDeinterleavedQuartets(d, v0, v1); + output.CheckSorted(kOrder, __LINE__, caller); +#endif +} + +// ------------------------------ Bitonic merge (quartets) + +// For the last layer of bitonic merge. Conditionally swaps even-numbered lanes +// with their odd-numbered neighbor. Works for both quartets and vectors. +template <SortOrder kOrder, class D> +HWY_INLINE void SortAdjacentLanesQV(D d, Vec<D>& q_or_v) { + (void)d; + // Optimization for 32-bit integers: swap via Shuffle and 64-bit Min/Max. + // (not worthwhile on SSE4/AVX2 because they lack 64-bit Min/Max) +#if !HWY_ARCH_X86 || HWY_TARGET <= HWY_AVX3 + if (sizeof(TFromD<D>) == 4 && !IsFloat<TFromD<D>>()) { + const RepartitionToWide<decltype(d)> dw; + const auto wide = BitCast(dw, q_or_v); + const auto swap = BitCast(dw, Shuffle2301(q_or_v)); + if (kOrder == SortOrder::kAscending) { + q_or_v = BitCast(d, Max(wide, swap)); + } else { + q_or_v = BitCast(d, Min(wide, swap)); + } + } else +#endif + { + Vec<D> swapped = Shuffle2301(q_or_v); + SortLanesIn2Vectors<kOrder>(q_or_v, swapped); + q_or_v = OddEven(swapped, q_or_v); + } +} + +// Lane 0 with 2, 1 with 3 etc. Works for both quartets and vectors. +template <SortOrder kOrder, class D> +HWY_INLINE void SortDistance2LanesQV(D d, Vec<D>& q_or_v) { + const RepartitionToWide<decltype(d)> dw; + Vec<D> swapped = Shuffle1032(q_or_v); + SortLanesIn2Vectors<kOrder>(q_or_v, swapped); + q_or_v = BitCast(d, OddEven(BitCast(dw, swapped), BitCast(dw, q_or_v))); +} + +// For all BitonicMerge*, and each block, the concatenation of those blocks from +// the first half and second half of the input vectors must be sorted in +// opposite orders. + +// 14 ops (including 4 swizzle) +template <SortOrder kOrder, class D, class V = Vec<D>> +HWY_INLINE void BitonicMerge2Quartets(D d, V& q0, V& q1, int caller) { +#if HWY_SORT_VERIFY + const verify::Runs<D> input = verify::StoreDeinterleavedQuartets(d, q0, q1); + if (caller == -1) input.CheckBitonic(__LINE__, __LINE__); +#endif + + // Layer 1: lane stride 4 (2 ops) + SortLanesIn2Vectors<kOrder>(q0, q1); + + // Layer 2: lane stride 2 (6 ops) + SortDistance2LanesQV<kOrder>(d, q0); + SortDistance2LanesQV<kOrder>(d, q1); + + // Layer 3: lane stride 1 (4 ops) + SortAdjacentLanesQV<kOrder>(d, q0); + SortAdjacentLanesQV<kOrder>(d, q1); + +#if HWY_SORT_VERIFY + const verify::Runs<D> output = verify::StoreDeinterleavedQuartets(d, q0, q1); + output.CheckSorted(kOrder, __LINE__, caller); +#endif +} + +// 32 ops, more efficient than three 4+4 merges (36 ops). +template <SortOrder kOrder, class D, class V = Vec<D>> +HWY_INLINE void BitonicMerge4Quartets(D d, V& q0, V& q1, V& q2, V& q3, + int caller) { +#if HWY_SORT_VERIFY + const verify::Runs<D> input = + verify::StoreDeinterleavedQuartets(d, q0, q1, q2, q3); + if (caller == -1) input.CheckBitonic(__LINE__, __LINE__); +#endif + + // Layer 1: lane stride 8 + SortLanesIn2Vectors<kOrder>(q0, q2); + SortLanesIn2Vectors<kOrder>(q1, q3); + + // Layers 2 to 4 + // Inputs are not fully sorted, so cannot use Merge2SortedQuartets. + BitonicMerge2Quartets<kOrder>(d, q0, q1, __LINE__); + BitonicMerge2Quartets<kOrder>(d, q2, q3, __LINE__); + +#if HWY_SORT_VERIFY + const verify::Runs<D> output = + verify::StoreDeinterleavedQuartets(d, q0, q1, q2, q3); + output.CheckSorted(kOrder, __LINE__, caller); +#endif +} + +// 72 ops. +template <SortOrder kOrder, class D, class V = Vec<D>> +HWY_INLINE void BitonicMerge8Quartets(D d, V& q0, V& q1, V& q2, V& q3, V& q4, + V& q5, V& q6, V& q7, int caller) { +#if HWY_SORT_VERIFY + const verify::Runs<D> input = + verify::StoreDeinterleavedQuartets(d, q0, q1, q2, q3, q4, q5, q6, q7); + if (caller == -1) input.CheckBitonic(__LINE__, __LINE__); +#endif + + // Layer 1: lane stride 16 + SortLanesIn2Vectors<kOrder>(q0, q4); + SortLanesIn2Vectors<kOrder>(q1, q5); + SortLanesIn2Vectors<kOrder>(q2, q6); + SortLanesIn2Vectors<kOrder>(q3, q7); + + // Layers 2 to 5 + BitonicMerge4Quartets<kOrder>(d, q0, q1, q2, q3, __LINE__); + BitonicMerge4Quartets<kOrder>(d, q4, q5, q6, q7, __LINE__); + +#if HWY_SORT_VERIFY + const verify::Runs<D> output = + verify::StoreDeinterleavedQuartets(d, q0, q1, q2, q3, q4, q5, q6, q7); + output.CheckSorted(kOrder, __LINE__, caller); +#endif +} + +// ------------------------------ Bitonic merge (vectors) + +// Lane 0 with 4, 1 with 5 etc. Only used for vectors with at least 8 lanes. +#if HWY_TARGET <= HWY_AVX3 + +// TODO(janwas): move to op +template <typename T> +Vec512<T> Shuffle128_2020(Vec512<T> a, Vec512<T> b) { + return Vec512<T>{_mm512_shuffle_i32x4(a.raw, b.raw, _MM_SHUFFLE(2, 0, 2, 0))}; +} + +template <typename T> +Vec512<T> Shuffle128_3131(Vec512<T> a, Vec512<T> b) { + return Vec512<T>{_mm512_shuffle_i32x4(a.raw, b.raw, _MM_SHUFFLE(3, 1, 3, 1))}; +} + +template <typename T> +Vec512<T> Shuffle128_2301(Vec512<T> a, Vec512<T> b) { + return Vec512<T>{_mm512_shuffle_i32x4(a.raw, b.raw, _MM_SHUFFLE(2, 3, 0, 1))}; +} + +template <typename T> +Vec512<T> OddEven128(Vec512<T> odd, Vec512<T> even) { + return Vec512<T>{_mm512_mask_blend_epi64(__mmask8{0x33u}, odd.raw, even.raw)}; +} + +template <SortOrder kOrder, class T> +HWY_INLINE void SortDistance4LanesV(Simd<T, 16> d, Vec<decltype(d)>& v) { + // In: FEDCBA98 76543210 + // Swap 128-bit halves of each 256 bits => BA98FEDC 32107654 + Vec512<T> swapped = Shuffle128_2301(v, v); + SortLanesIn2Vectors<kOrder>(v, swapped); + v = OddEven128(swapped, v); +} + +#endif + +template <SortOrder kOrder, typename T> +HWY_INLINE void SortDistance4LanesV(Simd<T, 8> d, Vec<decltype(d)>& v) { + Vec<decltype(d)> swapped = ConcatLowerUpper(d, v, v); + SortLanesIn2Vectors<kOrder>(v, swapped); + v = ConcatUpperLower(swapped, v); +} + +template <SortOrder kOrder, typename T> +HWY_INLINE void SortDistance4LanesV(Simd<T, 4> /* tag */, ...) {} + +// Only used for vectors with at least 16 lanes. +template <SortOrder kOrder, class D> +HWY_INLINE void SortDistance8LanesV(D d, Vec<D>& v) { + Vec<D> swapped = ConcatLowerUpper(d, v, v); + SortLanesIn2Vectors<kOrder>(v, swapped); + v = ConcatUpperLower(swapped, v); +} + +// 120 ops. Only used if vectors are at least 8 lanes. +template <SortOrder kOrder, class D, class V = Vec<D>> +HWY_INLINE void BitonicMergeTo64(D d, V& v0, V& v1, V& v2, V& v3, V& v4, V& v5, + V& v6, V& v7, int caller) { +#if HWY_SORT_VERIFY + const verify::Runs<D> input = + verify::StoreVectors(d, v0, v1, v2, v3, v4, v5, v6, v7); + if (caller == -1) input.CheckBitonic(__LINE__, __LINE__); +#endif + + // Layer 1: lane stride 32 + SortLanesIn2Vectors<kOrder>(v0, v4); + SortLanesIn2Vectors<kOrder>(v1, v5); + SortLanesIn2Vectors<kOrder>(v2, v6); + SortLanesIn2Vectors<kOrder>(v3, v7); + + // Layer 2: lane stride 16 + SortLanesIn2Vectors<kOrder>(v0, v2); + SortLanesIn2Vectors<kOrder>(v1, v3); + SortLanesIn2Vectors<kOrder>(v4, v6); + SortLanesIn2Vectors<kOrder>(v5, v7); + + // Layer 3: lane stride 8 + SortLanesIn2Vectors<kOrder>(v0, v1); + SortLanesIn2Vectors<kOrder>(v2, v3); + SortLanesIn2Vectors<kOrder>(v4, v5); + SortLanesIn2Vectors<kOrder>(v6, v7); + + // Layer 4: lane stride 4 + SortDistance4LanesV<kOrder>(d, v0); + SortDistance4LanesV<kOrder>(d, v1); + SortDistance4LanesV<kOrder>(d, v2); + SortDistance4LanesV<kOrder>(d, v3); + SortDistance4LanesV<kOrder>(d, v4); + SortDistance4LanesV<kOrder>(d, v5); + SortDistance4LanesV<kOrder>(d, v6); + SortDistance4LanesV<kOrder>(d, v7); + + // Layer 5: lane stride 2 + SortDistance2LanesQV<kOrder>(d, v0); + SortDistance2LanesQV<kOrder>(d, v1); + SortDistance2LanesQV<kOrder>(d, v2); + SortDistance2LanesQV<kOrder>(d, v3); + SortDistance2LanesQV<kOrder>(d, v4); + SortDistance2LanesQV<kOrder>(d, v5); + SortDistance2LanesQV<kOrder>(d, v6); + SortDistance2LanesQV<kOrder>(d, v7); + + // Layer 6: lane stride 1 + SortAdjacentLanesQV<kOrder>(d, v0); + SortAdjacentLanesQV<kOrder>(d, v1); + SortAdjacentLanesQV<kOrder>(d, v2); + SortAdjacentLanesQV<kOrder>(d, v3); + SortAdjacentLanesQV<kOrder>(d, v4); + SortAdjacentLanesQV<kOrder>(d, v5); + SortAdjacentLanesQV<kOrder>(d, v6); + SortAdjacentLanesQV<kOrder>(d, v7); + +#if HWY_SORT_VERIFY + const verify::Runs<D> output = + verify::StoreVectors(d, v0, v1, v2, v3, v4, v5, v6, v7); + output.CheckSorted(kOrder, __LINE__, caller); +#endif +} + +// 60 ops. Only used if vectors are at least 16 lanes. +template <SortOrder kOrder, class D, class V = Vec<D>> +HWY_INLINE void BitonicMergeTo64(D d, V& v0, V& v1, V& v2, V& v3, int caller) { +#if HWY_SORT_VERIFY + const verify::Runs<D> input = verify::StoreVectors(d, v0, v1, v2, v3); + if (caller == -1) input.CheckBitonic(__LINE__, __LINE__); +#endif + + // Layer 1: lane stride 32 + SortLanesIn2Vectors<kOrder>(v0, v2); + SortLanesIn2Vectors<kOrder>(v1, v3); + + // Layer 2: lane stride 16 + SortLanesIn2Vectors<kOrder>(v0, v1); + SortLanesIn2Vectors<kOrder>(v2, v3); + + // Layer 3: lane stride 8 + SortDistance8LanesV<kOrder>(d, v0); + SortDistance8LanesV<kOrder>(d, v1); + SortDistance8LanesV<kOrder>(d, v2); + SortDistance8LanesV<kOrder>(d, v3); + + // Layer 4: lane stride 4 + SortDistance4LanesV<kOrder>(d, v0); + SortDistance4LanesV<kOrder>(d, v1); + SortDistance4LanesV<kOrder>(d, v2); + SortDistance4LanesV<kOrder>(d, v3); + + // Layer 5: lane stride 2 + SortDistance2LanesQV<kOrder>(d, v0); + SortDistance2LanesQV<kOrder>(d, v1); + SortDistance2LanesQV<kOrder>(d, v2); + SortDistance2LanesQV<kOrder>(d, v3); + + // Layer 6: lane stride 1 + SortAdjacentLanesQV<kOrder>(d, v0); + SortAdjacentLanesQV<kOrder>(d, v1); + SortAdjacentLanesQV<kOrder>(d, v2); + SortAdjacentLanesQV<kOrder>(d, v3); + +#if HWY_SORT_VERIFY + const verify::Runs<D> output = verify::StoreVectors(d, v0, v1, v2, v3); + output.CheckSorted(kOrder, __LINE__, caller); +#endif +} + +// 128 ops. Only used if vectors are at least 16 lanes. +template <SortOrder kOrder, class D, class V = Vec<D>> +HWY_INLINE void BitonicMergeTo128(D d, V& v0, V& v1, V& v2, V& v3, V& v4, V& v5, + V& v6, V& v7, int caller) { +#if HWY_SORT_VERIFY + const verify::Runs<D> input = + verify::StoreVectors(d, v0, v1, v2, v3, v4, v5, v6, v7); + if (caller == -1) input.CheckBitonic(__LINE__, __LINE__); +#endif + + // Layer 1: lane stride 64 + SortLanesIn2Vectors<kOrder>(v0, v4); + SortLanesIn2Vectors<kOrder>(v1, v5); + SortLanesIn2Vectors<kOrder>(v2, v6); + SortLanesIn2Vectors<kOrder>(v3, v7); + + BitonicMergeTo64<kOrder>(d, v0, v1, v2, v3, __LINE__); + BitonicMergeTo64<kOrder>(d, v4, v5, v6, v7, __LINE__); + +#if HWY_SORT_VERIFY + const verify::Runs<D> output = + verify::StoreVectors(d, v0, v1, v2, v3, v4, v5, v6, v7); + output.CheckSorted(kOrder, __LINE__, caller); +#endif +} + +// ------------------------------ Vector-length dependent + +// Only called when N=4 (single block, so quartets can just be stored). +template <SortOrder kOrder, class D, class V> +HWY_API size_t SingleQuartetPerVector(D d, V& q0, V& q1, V& q2, V& q3, V& q4, + V& q5, V& q6, V& q7, TFromD<D>* inout) { + Store(q0, d, inout + 0 * 4); + Store(q1, d, inout + 1 * 4); + Store(q2, d, inout + 2 * 4); + Store(q3, d, inout + 3 * 4); + Store(q4, d, inout + 4 * 4); + Store(q5, d, inout + 5 * 4); + Store(q6, d, inout + 6 * 4); + Store(q7, d, inout + 7 * 4); + return 8 * 4; +} + +// Only called when N=8. +template <SortOrder kOrder, class D, class V> +HWY_API size_t TwoQuartetsPerVector(D d, V& q0, V& q1, V& q2, V& q3, V& q4, + V& q5, V& q6, V& q7, TFromD<D>* inout) { + V v0 = ConcatLowerLower(d, q1, q0); + V v1 = ConcatLowerLower(d, q3, q2); + V v2 = ConcatLowerLower(d, q5, q4); + V v3 = ConcatLowerLower(d, q7, q6); + // TODO(janwas): merge into single table + V v4 = Reverse(d, ConcatUpperUpper(d, q7, q6)); + V v5 = Reverse(d, ConcatUpperUpper(d, q5, q4)); + V v6 = Reverse(d, ConcatUpperUpper(d, q3, q2)); + V v7 = Reverse(d, ConcatUpperUpper(d, q1, q0)); + detail::BitonicMergeTo64<kOrder>(d, v0, v1, v2, v3, v4, v5, v6, v7, -1); + + Store(v0, d, inout + 0 * 8); + Store(v1, d, inout + 1 * 8); + Store(v2, d, inout + 2 * 8); + Store(v3, d, inout + 3 * 8); + Store(v4, d, inout + 4 * 8); + Store(v5, d, inout + 5 * 8); + Store(v6, d, inout + 6 * 8); + Store(v7, d, inout + 7 * 8); + return 8 * 8; +} + +// Only called when N=16. +template <SortOrder kOrder, typename T, class V> +HWY_API size_t FourQuartetsPerVector(Simd<T, 16> d, V& q0, V& q1, V& q2, V& q3, + V& q4, V& q5, V& q6, V& q7, T* inout) { + const V q11_01_10_00 = Shuffle128_2020(q0, q1); + const V q13_03_12_02 = Shuffle128_2020(q2, q3); + V v0 = Shuffle128_2020(q11_01_10_00, q13_03_12_02); // 3..0 + + const V q15_05_14_04 = Shuffle128_2020(q4, q5); + const V q17_07_16_06 = Shuffle128_2020(q6, q7); + V v1 = Shuffle128_2020(q15_05_14_04, q17_07_16_06); // 7..4 + + const V q19_09_18_08 = Shuffle128_3131(q0, q1); + const V q1b_0b_1a_0a = Shuffle128_3131(q2, q3); + V v3 = Reverse(d, Shuffle128_2020(q19_09_18_08, q1b_0b_1a_0a)); // b..8 + + const V q1d_0d_1c_0c = Shuffle128_3131(q4, q5); + const V q1f_0f_1e_0e = Shuffle128_3131(q6, q7); + V v2 = Reverse(d, Shuffle128_2020(q1d_0d_1c_0c, q1f_0f_1e_0e)); // f..c + + detail::BitonicMergeTo64<kOrder>(d, v0, v1, v2, v3, -1); + + // TODO(janwas): merge into single table + V v4 = Shuffle128_3131(q11_01_10_00, q13_03_12_02); // 13..10 + V v5 = Shuffle128_3131(q15_05_14_04, q17_07_16_06); // 17..14 + V v7 = Reverse(d, Shuffle128_3131(q19_09_18_08, q1b_0b_1a_0a)); // 1b..18 + V v6 = Reverse(d, Shuffle128_3131(q1d_0d_1c_0c, q1f_0f_1e_0e)); // 1f..1c + + detail::BitonicMergeTo64<Reverse(kOrder)>(d, v4, v5, v6, v7, -1); + + detail::BitonicMergeTo128<kOrder>(d, v0, v1, v2, v3, v4, v5, v6, v7, -1); + + Store(v0, d, inout + 0 * 16); + Store(v1, d, inout + 1 * 16); + Store(v2, d, inout + 2 * 16); + Store(v3, d, inout + 3 * 16); + Store(v4, d, inout + 4 * 16); + Store(v5, d, inout + 5 * 16); + Store(v6, d, inout + 6 * 16); + Store(v7, d, inout + 7 * 16); + return 8 * 16; +} + +// Avoid needing #if at the call sites. +template <SortOrder kOrder, typename T> +HWY_API size_t TwoQuartetsPerVector(Simd<T, 4> /* tag */, ...) { + return 0; +} + +template <SortOrder kOrder, typename T> +HWY_API size_t FourQuartetsPerVector(Simd<T, 4> /* tag */, ...) { + return 0; +} +template <SortOrder kOrder, typename T> +HWY_API size_t FourQuartetsPerVector(Simd<T, 8> /* tag */, ...) { + return 0; +} + +} // namespace detail + +template <class D> +HWY_API size_t SortBatchSize(D d) { + const size_t N = Lanes(d); + if (N == 4) return 32; + if (N == 8) return 64; + if (N == 16) return 128; + return 0; +} + +template <SortOrder kOrder, class D> +HWY_API size_t SortBatch(D d, TFromD<D>* inout) { + const size_t N = Lanes(d); + + Vec<D> q0, q1, q2, q3; + detail::SortLanesIn4Vectors<kOrder>(d, inout, q0, q1, q2, q3); + detail::Transpose4x4(d, q0, q1, q2, q3); + detail::Merge2SortedQuartets<kOrder>(d, q0, q1, -1); + detail::Merge2SortedQuartets<kOrder>(d, q2, q3, -1); + + // Bitonic merges require one input to be in reverse order. + constexpr SortOrder kReverse = Reverse(kOrder); + + Vec<D> q4, q5, q6, q7; + detail::SortLanesIn4Vectors<kReverse>(d, inout + 4 * N, q4, q5, q6, q7); + detail::Transpose4x4(d, q4, q5, q6, q7); + detail::Merge2SortedQuartets<kReverse>(d, q4, q5, -1); + detail::Merge2SortedQuartets<kReverse>(d, q6, q7, -1); + + detail::BitonicMerge4Quartets<kOrder>(d, q0, q1, q4, q5, -1); + detail::BitonicMerge4Quartets<kReverse>(d, q2, q3, q6, q7, -1); + + detail::BitonicMerge8Quartets<kOrder>(d, q0, q1, q4, q5, q2, q3, q6, q7, + __LINE__); + + if (N == 4) { + return detail::SingleQuartetPerVector<kOrder>(d, q0, q1, q4, q5, q2, q3, q6, + q7, inout); + } + + if (N == 8) { + return detail::TwoQuartetsPerVector<kOrder>(d, q0, q1, q4, q5, q2, q3, q6, + q7, inout); + } + + return detail::FourQuartetsPerVector<kOrder>(d, q0, q1, q4, q5, q2, q3, q6, + q7, inout); +} + +#else + +// Avoids unused attribute warning +template <SortOrder kOrder, class D> +HWY_API size_t SortBatch(D /* tag */, TFromD<D>* /* inout */) { + return 0; +} + +#endif // HWY_TARGET != HWY_SCALAR && HWY_ARCH_X86 + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); + +#endif // HIGHWAY_HWY_CONTRIB_SORT_SORT_INL_H_ diff --git a/third_party/highway/hwy/contrib/sort/sort_test.cc b/third_party/highway/hwy/contrib/sort/sort_test.cc new file mode 100644 index 0000000..c3b421f --- /dev/null +++ b/third_party/highway/hwy/contrib/sort/sort_test.cc @@ -0,0 +1,188 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <inttypes.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> + +// clang-format off +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "hwy/contrib/sort/sort_test.cc" +#include "hwy/foreach_target.h" + +#include "hwy/contrib/sort/sort-inl.h" +#include "hwy/tests/test_util-inl.h" +// clang-format on + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +#if HWY_TARGET != HWY_SCALAR && HWY_ARCH_X86 + +template <class D> +size_t K(D d) { + return SortBatchSize(d); +} + +template <SortOrder kOrder, class D> +void Validate(D d, const TFromD<D>* in, const TFromD<D>* out) { + const size_t N = Lanes(d); + // Ensure it matches the sort order + for (size_t i = 0; i < K(d) - 1; ++i) { + if (!verify::Compare(out[i], out[i + 1], kOrder)) { + printf("range=%" PRIu64 " lane=%" PRIu64 " N=%" PRIu64 " %.0f %.0f\n\n", + static_cast<uint64_t>(i), static_cast<uint64_t>(i), + static_cast<uint64_t>(N), static_cast<float>(out[i + 0]), + static_cast<float>(out[i + 1])); + for (size_t i = 0; i < K(d); ++i) { + printf("%.0f\n", static_cast<float>(out[i])); + } + + printf("\n\nin was:\n"); + for (size_t i = 0; i < K(d); ++i) { + printf("%.0f\n", static_cast<float>(in[i])); + } + fflush(stdout); + HWY_ABORT("Sort is incorrect"); + } + } + + // Also verify sums match (detects duplicated/lost values) + double expected_sum = 0.0; + double actual_sum = 0.0; + for (size_t i = 0; i < K(d); ++i) { + expected_sum += in[i]; + actual_sum += out[i]; + } + if (expected_sum != actual_sum) { + for (size_t i = 0; i < K(d); ++i) { + printf("%.0f %.0f\n", static_cast<float>(in[i]), + static_cast<float>(out[i])); + } + HWY_ABORT("Mismatch"); + } +} + +class TestReverse { + template <SortOrder kOrder, class D> + void TestOrder(D d, RandomState& /* rng */) { + using T = TFromD<D>; + const size_t N = Lanes(d); + HWY_ASSERT((N % 4) == 0); + auto in = AllocateAligned<T>(K(d)); + auto inout = AllocateAligned<T>(K(d)); + + const size_t expected_size = SortBatchSize(d); + + for (size_t i = 0; i < K(d); ++i) { + in[i] = static_cast<T>(K(d) - i); + inout[i] = in[i]; + } + + const size_t actual_size = SortBatch<kOrder>(d, inout.get()); + HWY_ASSERT_EQ(expected_size, actual_size); + Validate<kOrder>(d, in.get(), inout.get()); + } + + public: + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + RandomState rng; + TestOrder<SortOrder::kAscending>(d, rng); + TestOrder<SortOrder::kDescending>(d, rng); + } +}; + +void TestAllReverse() { + TestReverse test; + test(int32_t(), CappedTag<int32_t, 16>()); + test(uint32_t(), CappedTag<uint32_t, 16>()); +} + +class TestRanges { + template <SortOrder kOrder, class D> + void TestOrder(D d, RandomState& rng) { + using T = TFromD<D>; + const size_t N = Lanes(d); + HWY_ASSERT((N % 4) == 0); + auto in = AllocateAligned<T>(K(d)); + auto inout = AllocateAligned<T>(K(d)); + + const size_t expected_size = SortBatchSize(d); + + // For each range, try all 0/1 combinations and set any other lanes to + // random inputs. + constexpr size_t kRange = 8; + for (size_t range = 0; range < K(d); range += kRange) { + for (size_t bits = 0; bits < (1ull << kRange); ++bits) { + // First set all to random, will later overwrite those for `range` + for (size_t i = 0; i < K(d); ++i) { + in[i] = inout[i] = static_cast<T>(Random32(&rng) & 0xFF); + } + // Now set the current combination of {0,1} for elements in the range. + // This is sufficient to establish correctness (arbitrary inputs could + // be mapped to 0/1 with a comparison predicate). + for (size_t i = 0; i < kRange; ++i) { + in[range + i] = inout[range + i] = (bits >> i) & 1; + } + + const size_t actual_size = SortBatch<kOrder>(d, inout.get()); + HWY_ASSERT_EQ(expected_size, actual_size); + Validate<kOrder>(d, in.get(), inout.get()); + } + } + } + + public: + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + RandomState rng; + TestOrder<SortOrder::kAscending>(d, rng); + TestOrder<SortOrder::kDescending>(d, rng); + } +}; + +void TestAllRanges() { + TestRanges test; + test(int32_t(), CappedTag<int32_t, 16>()); + test(uint32_t(), CappedTag<uint32_t, 16>()); +} + +#else +void TestAllReverse() {} +void TestAllRanges() {} +#endif // HWY_TARGET != HWY_SCALAR && HWY_ARCH_X86 + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE + +namespace hwy { +HWY_BEFORE_TEST(SortTest); +HWY_EXPORT_AND_TEST_P(SortTest, TestAllReverse); +HWY_EXPORT_AND_TEST_P(SortTest, TestAllRanges); +} // namespace hwy + +// Ought not to be necessary, but without this, no tests run on RVV. +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#endif diff --git a/third_party/highway/hwy/detect_compiler_arch.h b/third_party/highway/hwy/detect_compiler_arch.h new file mode 100644 index 0000000..27f573c --- /dev/null +++ b/third_party/highway/hwy/detect_compiler_arch.h @@ -0,0 +1,194 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef HIGHWAY_HWY_DETECT_COMPILER_ARCH_H_ +#define HIGHWAY_HWY_DETECT_COMPILER_ARCH_H_ + +// Detects compiler and arch from predefined macros. Zero dependencies for +// inclusion by foreach_target.h. + +// Add to #if conditions to prevent IDE from graying out code. +#if (defined __CDT_PARSER__) || (defined __INTELLISENSE__) || \ + (defined Q_CREATOR_RUN) || (defined(__CLANGD__)) +#define HWY_IDE 1 +#else +#define HWY_IDE 0 +#endif + +//------------------------------------------------------------------------------ +// Compiler + +// clang-cl defines _MSC_VER but doesn't behave like MSVC in other aspects like +// used in HWY_DIAGNOSTICS(). We include a check that we are not clang for that +// purpose. +#if defined(_MSC_VER) && !defined(__clang__) +#define HWY_COMPILER_MSVC _MSC_VER +#else +#define HWY_COMPILER_MSVC 0 +#endif + +#ifdef __INTEL_COMPILER +#define HWY_COMPILER_ICC __INTEL_COMPILER +#else +#define HWY_COMPILER_ICC 0 +#endif + +#ifdef __GNUC__ +#define HWY_COMPILER_GCC (__GNUC__ * 100 + __GNUC_MINOR__) +#else +#define HWY_COMPILER_GCC 0 +#endif + +// Clang can masquerade as MSVC/GCC, in which case both are set. +#ifdef __clang__ +#ifdef __APPLE__ +// Apple LLVM version is unrelated to the actual Clang version, which we need +// for enabling workarounds. Use the presence of warning flags to deduce it. +// Adapted from https://github.com/simd-everywhere/simde/ simde-detect-clang.h. +#if __has_warning("-Wformat-insufficient-args") +#define HWY_COMPILER_CLANG 1200 +#elif __has_warning("-Wimplicit-const-int-float-conversion") +#define HWY_COMPILER_CLANG 1100 +#elif __has_warning("-Wmisleading-indentation") +#define HWY_COMPILER_CLANG 1000 +#elif defined(__FILE_NAME__) +#define HWY_COMPILER_CLANG 900 +#elif __has_warning("-Wextra-semi-stmt") || \ + __has_builtin(__builtin_rotateleft32) +#define HWY_COMPILER_CLANG 800 +#elif __has_warning("-Wc++98-compat-extra-semi") +#define HWY_COMPILER_CLANG 700 +#else // Anything older than 7.0 is not recommended for Highway. +#define HWY_COMPILER_CLANG 600 +#endif // __has_warning chain +#else // Non-Apple: normal version +#define HWY_COMPILER_CLANG (__clang_major__ * 100 + __clang_minor__) +#endif +#else // Not clang +#define HWY_COMPILER_CLANG 0 +#endif + +// More than one may be nonzero, but we want at least one. +#if !HWY_COMPILER_MSVC && !HWY_COMPILER_ICC && !HWY_COMPILER_GCC && \ + !HWY_COMPILER_CLANG +#error "Unsupported compiler" +#endif + +#ifdef __has_builtin +#define HWY_HAS_BUILTIN(name) __has_builtin(name) +#else +#define HWY_HAS_BUILTIN(name) 0 +#endif + +#ifdef __has_attribute +#define HWY_HAS_ATTRIBUTE(name) __has_attribute(name) +#else +#define HWY_HAS_ATTRIBUTE(name) 0 +#endif + +#ifdef __has_feature +#define HWY_HAS_FEATURE(name) __has_feature(name) +#else +#define HWY_HAS_FEATURE(name) 0 +#endif + +//------------------------------------------------------------------------------ +// Architecture + +#if defined(HWY_EMULATE_SVE) + +#define HWY_ARCH_X86_32 0 +#define HWY_ARCH_X86_64 0 +#define HWY_ARCH_X86 0 +#define HWY_ARCH_PPC 0 +#define HWY_ARCH_ARM_A64 1 +#define HWY_ARCH_ARM_V7 0 +#define HWY_ARCH_ARM 1 +#define HWY_ARCH_WASM 0 +#define HWY_ARCH_RVV 0 + +#else + +#if defined(__i386__) || defined(_M_IX86) +#define HWY_ARCH_X86_32 1 +#else +#define HWY_ARCH_X86_32 0 +#endif + +#if defined(__x86_64__) || defined(_M_X64) +#define HWY_ARCH_X86_64 1 +#else +#define HWY_ARCH_X86_64 0 +#endif + +#if HWY_ARCH_X86_32 && HWY_ARCH_X86_64 +#error "Cannot have both x86-32 and x86-64" +#endif + +#if HWY_ARCH_X86_32 || HWY_ARCH_X86_64 +#define HWY_ARCH_X86 1 +#else +#define HWY_ARCH_X86 0 +#endif + +#if defined(__powerpc64__) || defined(_M_PPC) +#define HWY_ARCH_PPC 1 +#else +#define HWY_ARCH_PPC 0 +#endif + +#if defined(__ARM_ARCH_ISA_A64) || defined(__aarch64__) || defined(_M_ARM64) +#define HWY_ARCH_ARM_A64 1 +#else +#define HWY_ARCH_ARM_A64 0 +#endif + +#if defined(__arm__) || defined(_M_ARM) +#define HWY_ARCH_ARM_V7 1 +#else +#define HWY_ARCH_ARM_V7 0 +#endif + +#if HWY_ARCH_ARM_A64 && HWY_ARCH_ARM_V7 +#error "Cannot have both A64 and V7" +#endif + +#if HWY_ARCH_ARM_A64 || HWY_ARCH_ARM_V7 +#define HWY_ARCH_ARM 1 +#else +#define HWY_ARCH_ARM 0 +#endif + +#if defined(__EMSCRIPTEN__) || defined(__wasm__) || defined(__WASM__) +#define HWY_ARCH_WASM 1 +#else +#define HWY_ARCH_WASM 0 +#endif + +#ifdef __riscv +#define HWY_ARCH_RVV 1 +#else +#define HWY_ARCH_RVV 0 +#endif + +#endif // defined(HWY_EMULATE_SVE) + +// It is an error to detect multiple architectures at the same time, but OK to +// detect none of the above. +#if (HWY_ARCH_X86 + HWY_ARCH_PPC + HWY_ARCH_ARM + HWY_ARCH_WASM + \ + HWY_ARCH_RVV) > 1 +#error "Must not detect more than one architecture" +#endif + +#endif // HIGHWAY_HWY_DETECT_COMPILER_ARCH_H_ diff --git a/third_party/highway/hwy/detect_targets.h b/third_party/highway/hwy/detect_targets.h new file mode 100644 index 0000000..e1e46b2 --- /dev/null +++ b/third_party/highway/hwy/detect_targets.h @@ -0,0 +1,392 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef HIGHWAY_HWY_DETECT_TARGETS_H_ +#define HIGHWAY_HWY_DETECT_TARGETS_H_ + +// Defines targets and chooses which to enable. + +#include "hwy/detect_compiler_arch.h" + +//------------------------------------------------------------------------------ +// Optional configuration + +// See ../quick_reference.md for documentation of these macros. + +// Uncomment to override the default baseline determined from predefined macros: +// #define HWY_BASELINE_TARGETS (HWY_SSE4 | HWY_SCALAR) + +// Uncomment to override the default blocklist: +// #define HWY_BROKEN_TARGETS HWY_AVX3 + +// Uncomment to definitely avoid generating those target(s): +// #define HWY_DISABLED_TARGETS HWY_SSE4 + +// Uncomment to avoid emitting BMI/BMI2/FMA instructions (allows generating +// AVX2 target for VMs which support AVX2 but not the other instruction sets) +// #define HWY_DISABLE_BMI2_FMA + +//------------------------------------------------------------------------------ +// Targets + +// Unique bit value for each target. A lower value is "better" (e.g. more lanes) +// than a higher value within the same group/platform - see HWY_STATIC_TARGET. +// +// All values are unconditionally defined so we can test HWY_TARGETS without +// first checking the HWY_ARCH_*. +// +// The C99 preprocessor evaluates #if expressions using intmax_t types, so we +// can use 32-bit literals. + +// 1,2: reserved + +// Currently satisfiable by Ice Lake (VNNI, VPCLMULQDQ, VBMI2, VAES). Later to +// be added: BF16 (Cooper Lake). VP2INTERSECT is only in Tiger Lake? We do not +// yet have uses for VBMI, VPOPCNTDQ, BITALG, GFNI. +#define HWY_AVX3_DL 4 // see HWY_WANT_AVX3_DL below +#define HWY_AVX3 8 +#define HWY_AVX2 16 +// 32: reserved for AVX +#define HWY_SSE4 64 +#define HWY_SSSE3 128 +// 0x100, 0x200: reserved for SSE3, SSE2 + +// The highest bit in the HWY_TARGETS mask that a x86 target can have. Used for +// dynamic dispatch. All x86 target bits must be lower or equal to +// (1 << HWY_HIGHEST_TARGET_BIT_X86) and they can only use +// HWY_MAX_DYNAMIC_TARGETS in total. +#define HWY_HIGHEST_TARGET_BIT_X86 9 + +#define HWY_SVE2 0x400 +#define HWY_SVE 0x800 +// 0x1000 reserved for Helium +#define HWY_NEON 0x2000 + +#define HWY_HIGHEST_TARGET_BIT_ARM 13 + +// 0x4000, 0x8000 reserved +#define HWY_PPC8 0x10000 // v2.07 or 3 +// 0x20000, 0x40000 reserved for prior VSX/AltiVec + +#define HWY_HIGHEST_TARGET_BIT_PPC 18 + +#define HWY_WASM2 0x80000 // Experimental +#define HWY_WASM 0x100000 + +#define HWY_HIGHEST_TARGET_BIT_WASM 20 + +// 0x200000, 0x400000, 0x800000 reserved + +#define HWY_RVV 0x1000000 + +#define HWY_HIGHEST_TARGET_BIT_RVV 24 + +// 0x2000000, 0x4000000, 0x8000000, 0x10000000 reserved + +#define HWY_SCALAR 0x20000000 + +#define HWY_HIGHEST_TARGET_BIT_SCALAR 29 + +// Cannot use higher values, otherwise HWY_TARGETS computation might overflow. + +//------------------------------------------------------------------------------ +// Set default blocklists + +// Disabled means excluded from enabled at user's request. A separate config +// macro allows disabling without deactivating the blocklist below. +#ifndef HWY_DISABLED_TARGETS +#define HWY_DISABLED_TARGETS 0 +#endif + +// Broken means excluded from enabled due to known compiler issues. Allow the +// user to override this blocklist without any guarantee of success. +#ifndef HWY_BROKEN_TARGETS + +// x86 clang-6: we saw multiple AVX2/3 compile errors and in one case invalid +// SSE4 codegen (possibly only for msan), so disable all those targets. +#if HWY_ARCH_X86 && (HWY_COMPILER_CLANG != 0 && HWY_COMPILER_CLANG < 700) +#define HWY_BROKEN_TARGETS (HWY_SSE4 | HWY_AVX2 | HWY_AVX3 | HWY_AVX3_DL) +// This entails a major speed reduction, so warn unless the user explicitly +// opts in to scalar-only. +#if !defined(HWY_COMPILE_ONLY_SCALAR) +#pragma message("x86 Clang <= 6: define HWY_COMPILE_ONLY_SCALAR or upgrade.") +#endif + +// 32-bit may fail to compile AVX2/3. +#elif HWY_ARCH_X86_32 +#define HWY_BROKEN_TARGETS (HWY_AVX2 | HWY_AVX3 | HWY_AVX3_DL) + +// MSVC AVX3 support is buggy: https://github.com/Mysticial/Flops/issues/16 +#elif HWY_COMPILER_MSVC != 0 +#define HWY_BROKEN_TARGETS (HWY_AVX3 | HWY_AVX3_DL) + +// armv7be has not been tested and is not yet supported. +#elif HWY_ARCH_ARM_V7 && \ + (defined(__ARM_BIG_ENDIAN) || \ + (defined(__BYTE_ORDER) && __BYTE_ORDER == __BIG_ENDIAN)) +#define HWY_BROKEN_TARGETS (HWY_NEON) + +// SVE[2] require recent clang or gcc versions. +#elif (HWY_COMPILER_CLANG && HWY_COMPILER_CLANG < 1100) ||\ +(!HWY_COMPILER_CLANG && HWY_COMPILER_GCC && HWY_COMPILER_GCC < 1000) +#define HWY_BROKEN_TARGETS (HWY_SVE | HWY_SVE2) + +#else +#define HWY_BROKEN_TARGETS 0 +#endif + +#endif // HWY_BROKEN_TARGETS + +// Enabled means not disabled nor blocklisted. +#define HWY_ENABLED(targets) \ + ((targets) & ~((HWY_DISABLED_TARGETS) | (HWY_BROKEN_TARGETS))) + +//------------------------------------------------------------------------------ +// Detect baseline targets using predefined macros + +// Baseline means the targets for which the compiler is allowed to generate +// instructions, implying the target CPU would have to support them. Do not use +// this directly because it does not take the blocklist into account. Allow the +// user to override this without any guarantee of success. +#ifndef HWY_BASELINE_TARGETS + +#if defined(HWY_EMULATE_SVE) +#define HWY_BASELINE_TARGETS HWY_SVE // does not support SVE2 +#define HWY_BASELINE_AVX3_DL 0 +#else + +// Also check HWY_ARCH to ensure that simulating unknown platforms ends up with +// HWY_TARGET == HWY_SCALAR. + +#if HWY_ARCH_WASM && defined(__wasm_simd128__) +#if defined(HWY_WANT_WASM2) +#define HWY_BASELINE_WASM HWY_WASM2 +#else +#define HWY_BASELINE_WASM HWY_WASM +#endif // HWY_WANT_WASM2 +#else +#define HWY_BASELINE_WASM 0 +#endif + +// Avoid choosing the PPC target until we have an implementation. +#if HWY_ARCH_PPC && defined(__VSX__) && 0 +#define HWY_BASELINE_PPC8 HWY_PPC8 +#else +#define HWY_BASELINE_PPC8 0 +#endif + +// SVE compiles, but is not yet tested. +#if HWY_ARCH_ARM && defined(__ARM_FEATURE_SVE2) +#define HWY_BASELINE_SVE2 HWY_SVE2 +#else +#define HWY_BASELINE_SVE2 0 +#endif + +#if HWY_ARCH_ARM && defined(__ARM_FEATURE_SVE) +#define HWY_BASELINE_SVE HWY_SVE +#else +#define HWY_BASELINE_SVE 0 +#endif + +// GCC 4.5.4 only defines __ARM_NEON__; 5.4 defines both. +#if HWY_ARCH_ARM && (defined(__ARM_NEON__) || defined(__ARM_NEON)) +#define HWY_BASELINE_NEON HWY_NEON +#else +#define HWY_BASELINE_NEON 0 +#endif + +// Special handling for MSVC because it has fewer predefined macros +#if HWY_COMPILER_MSVC && !HWY_COMPILER_CLANG + +// We can only be sure SSSE3/SSE4 are enabled if AVX is +// (https://stackoverflow.com/questions/18563978/) +#if defined(__AVX__) +#define HWY_CHECK_SSSE3 1 +#define HWY_CHECK_SSE4 1 +#else +#define HWY_CHECK_SSSE3 0 +#define HWY_CHECK_SSE4 0 +#endif + +// Cannot check for PCLMUL/AES and BMI2/FMA/F16C individually; we assume +// PCLMUL/AES are available if SSE4 is, and BMI2/FMA/F16C if AVX2 is. +#define HWY_CHECK_PCLMUL_AES 1 +#define HWY_CHECK_BMI2_FMA 1 +#define HWY_CHECK_F16C 1 + +#else // non-MSVC + +#if defined(__SSSE3__) +#define HWY_CHECK_SSSE3 1 +#else +#define HWY_CHECK_SSSE3 0 +#endif + +#if defined(__SSE4_1__) && defined(__SSE4_2__) +#define HWY_CHECK_SSE4 1 +#else +#define HWY_CHECK_SSE4 0 +#endif + +// If these are disabled, they should not gate the availability of SSE4/AVX2. +#if defined(HWY_DISABLE_PCLMUL_AES) || (defined(__PCLMUL__) && defined(__AES__)) +#define HWY_CHECK_PCLMUL_AES 1 +#else +#define HWY_CHECK_PCLMUL_AES 0 +#endif + +#if defined(HWY_DISABLE_BMI2_FMA) || (defined(__BMI2__) && defined(__FMA__)) +#define HWY_CHECK_BMI2_FMA 1 +#else +#define HWY_CHECK_BMI2_FMA 0 +#endif + +#if defined(HWY_DISABLE_F16C) || defined(__F16C__) +#define HWY_CHECK_F16C 1 +#else +#define HWY_CHECK_F16C 0 +#endif + +#endif // non-MSVC + +#if HWY_ARCH_X86 && HWY_CHECK_SSSE3 +#define HWY_BASELINE_SSSE3 HWY_SSSE3 +#else +#define HWY_BASELINE_SSSE3 0 +#endif + +#if HWY_ARCH_X86 && HWY_CHECK_SSE4 && HWY_CHECK_PCLMUL_AES +#define HWY_BASELINE_SSE4 HWY_SSE4 +#else +#define HWY_BASELINE_SSE4 0 +#endif + +#if HWY_BASELINE_SSE4 != 0 && HWY_CHECK_BMI2_FMA && HWY_CHECK_F16C && \ + defined(__AVX2__) +#define HWY_BASELINE_AVX2 HWY_AVX2 +#else +#define HWY_BASELINE_AVX2 0 +#endif + +// Require everything in AVX2 plus AVX-512 flags (also set by MSVC) +#if HWY_BASELINE_AVX2 != 0 && defined(__AVX512F__) && defined(__AVX512BW__) && \ + defined(__AVX512DQ__) && defined(__AVX512VL__) +#define HWY_BASELINE_AVX3 HWY_AVX3 +#else +#define HWY_BASELINE_AVX3 0 +#endif + +// TODO(janwas): not yet known whether these will be set by MSVC +#if HWY_BASELINE_AVX3 != 0 && defined(__AVXVNNI__) && defined(__VAES__) && \ + defined(__VPCLMULQDQ__) +#define HWY_BASELINE_AVX3_DL HWY_AVX3_DL +#else +#define HWY_BASELINE_AVX3_DL 0 +#endif + +#if HWY_ARCH_RVV && defined(__riscv_vector) +#define HWY_BASELINE_RVV HWY_RVV +#else +#define HWY_BASELINE_RVV 0 +#endif + +#define HWY_BASELINE_TARGETS \ + (HWY_SCALAR | HWY_BASELINE_WASM | HWY_BASELINE_PPC8 | HWY_BASELINE_SVE2 | \ + HWY_BASELINE_SVE | HWY_BASELINE_NEON | HWY_BASELINE_SSSE3 | \ + HWY_BASELINE_SSE4 | HWY_BASELINE_AVX2 | HWY_BASELINE_AVX3 | \ + HWY_BASELINE_AVX3_DL | HWY_BASELINE_RVV) + +#endif // HWY_EMULATE_SVE + +#else +// User already defined HWY_BASELINE_TARGETS, but we still need to define +// HWY_BASELINE_AVX3 (matching user's definition) for HWY_CHECK_AVX3_DL. +#define HWY_BASELINE_AVX3_DL (HWY_BASELINE_TARGETS & HWY_AVX3_DL) +#endif // HWY_BASELINE_TARGETS + +//------------------------------------------------------------------------------ +// Choose target for static dispatch + +#define HWY_ENABLED_BASELINE HWY_ENABLED(HWY_BASELINE_TARGETS) +#if HWY_ENABLED_BASELINE == 0 +#error "At least one baseline target must be defined and enabled" +#endif + +// Best baseline, used for static dispatch. This is the least-significant 1-bit +// within HWY_ENABLED_BASELINE and lower bit values imply "better". +#define HWY_STATIC_TARGET (HWY_ENABLED_BASELINE & -HWY_ENABLED_BASELINE) + +// Start by assuming static dispatch. If we later use dynamic dispatch, this +// will be defined to other targets during the multiple-inclusion, and finally +// return to the initial value. Defining this outside begin/end_target ensures +// inl headers successfully compile by themselves (required by Bazel). +#define HWY_TARGET HWY_STATIC_TARGET + +//------------------------------------------------------------------------------ +// Choose targets for dynamic dispatch according to one of four policies + +#if (defined(HWY_COMPILE_ONLY_SCALAR) + defined(HWY_COMPILE_ONLY_STATIC) + \ + defined(HWY_COMPILE_ALL_ATTAINABLE)) > 1 +#error "Invalid config: can only define a single policy for targets" +#endif + +// Further to checking for disabled/broken targets, we only use AVX3_DL after +// explicit opt-in (via this macro OR baseline compiler flags) to avoid +// generating a codepath which is only helpful if the app uses AVX3_DL features. +#if defined(HWY_WANT_AVX3_DL) +#define HWY_CHECK_AVX3_DL HWY_AVX3_DL +#else +#define HWY_CHECK_AVX3_DL HWY_BASELINE_AVX3_DL +#endif + +// Attainable means enabled and the compiler allows intrinsics (even when not +// allowed to autovectorize). Used in 3 and 4. +#if HWY_ARCH_X86 +#define HWY_ATTAINABLE_TARGETS \ + HWY_ENABLED(HWY_SCALAR | HWY_SSSE3 | HWY_SSE4 | HWY_AVX2 | HWY_AVX3 | \ + HWY_CHECK_AVX3_DL) +#else +#define HWY_ATTAINABLE_TARGETS HWY_ENABLED_BASELINE +#endif + +// 1) For older compilers: disable all SIMD (could also set HWY_DISABLED_TARGETS +// to ~HWY_SCALAR, but this is more explicit). +#if defined(HWY_COMPILE_ONLY_SCALAR) +#undef HWY_STATIC_TARGET +#define HWY_STATIC_TARGET HWY_SCALAR // override baseline +#define HWY_TARGETS HWY_SCALAR + +// 2) For forcing static dispatch without code changes (removing HWY_EXPORT) +#elif defined(HWY_COMPILE_ONLY_STATIC) +#define HWY_TARGETS HWY_STATIC_TARGET + +// 3) For tests: include all attainable targets (in particular: scalar) +#elif defined(HWY_COMPILE_ALL_ATTAINABLE) || defined(HWY_IS_TEST) +#define HWY_TARGETS HWY_ATTAINABLE_TARGETS + +// 4) Default: attainable WITHOUT non-best baseline. This reduces code size by +// excluding superseded targets, in particular scalar. +#else +#define HWY_TARGETS (HWY_ATTAINABLE_TARGETS & (2 * HWY_STATIC_TARGET - 1)) + +#endif // target policy + +// HWY_ONCE and the multiple-inclusion mechanism rely on HWY_STATIC_TARGET being +// one of the dynamic targets. This also implies HWY_TARGETS != 0 and +// (HWY_TARGETS & HWY_ENABLED_BASELINE) != 0. +#if (HWY_TARGETS & HWY_STATIC_TARGET) == 0 +#error "Logic error: best baseline should be included in dynamic targets" +#endif + +#endif // HIGHWAY_HWY_DETECT_TARGETS_H_ diff --git a/third_party/highway/hwy/examples/benchmark.cc b/third_party/highway/hwy/examples/benchmark.cc new file mode 100644 index 0000000..159e4c7 --- /dev/null +++ b/third_party/highway/hwy/examples/benchmark.cc @@ -0,0 +1,245 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "hwy/examples/benchmark.cc" +#include "hwy/foreach_target.h" + +#include <inttypes.h> +#include <stddef.h> +#include <stdint.h> +#include <stdio.h> + +#include <memory> +#include <numeric> // iota + +#include "hwy/aligned_allocator.h" +#include "hwy/highway.h" +#include "hwy/nanobenchmark.h" +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +// These templates are not found via ADL. +#if HWY_TARGET != HWY_SCALAR +using hwy::HWY_NAMESPACE::CombineShiftRightBytes; +#endif + +class TwoArray { + public: + // Must be a multiple of the vector lane count * 8. + static size_t NumItems() { return 3456; } + + TwoArray() + : a_(AllocateAligned<float>(NumItems() * 2)), b_(a_.get() + NumItems()) { + // = 1, but compiler doesn't know + const float init = static_cast<float>(Unpredictable1()); + std::iota(a_.get(), a_.get() + NumItems(), init); + std::iota(b_, b_ + NumItems(), init); + } + + protected: + AlignedFreeUniquePtr<float[]> a_; + float* b_; +}; + +// Measures durations, verifies results, prints timings. +template <class Benchmark> +void RunBenchmark(const char* caption) { + printf("%10s: ", caption); + const size_t kNumInputs = 1; + const size_t num_items = Benchmark::NumItems() * size_t(Unpredictable1()); + const FuncInput inputs[kNumInputs] = {num_items}; + Result results[kNumInputs]; + + Benchmark benchmark; + + Params p; + p.verbose = false; + p.max_evals = 7; + p.target_rel_mad = 0.002; + const size_t num_results = MeasureClosure( + [&benchmark](const FuncInput input) { return benchmark(input); }, inputs, + kNumInputs, results, p); + if (num_results != kNumInputs) { + fprintf(stderr, "MeasureClosure failed.\n"); + } + + benchmark.Verify(num_items); + + for (size_t i = 0; i < num_results; ++i) { + const double cycles_per_item = results[i].ticks / double(results[i].input); + const double mad = results[i].variability * cycles_per_item; + printf("%6" PRIu64 ": %6.3f (+/- %5.3f)\n", + static_cast<uint64_t>(results[i].input), cycles_per_item, mad); + } +} + +void Intro() { + HWY_ALIGN const float in[16] = {1, 2, 3, 4, 5, 6}; + HWY_ALIGN float out[16]; + const ScalableTag<float> d; // largest possible vector + for (size_t i = 0; i < 16; i += Lanes(d)) { + const auto vec = Load(d, in + i); // aligned! + auto result = vec * vec; + result += result; // can update if not const + Store(result, d, out + i); + } + printf("\nF(x)->2*x^2, F(%.0f) = %.1f\n", in[2], out[2]); +} + +// BEGINNER: dot product +// 0.4 cyc/float = bronze, 0.25 = silver, 0.15 = gold! +class BenchmarkDot : public TwoArray { + public: + BenchmarkDot() : dot_{-1.0f} {} + + FuncOutput operator()(const size_t num_items) { + const ScalableTag<float> d; + const size_t N = Lanes(d); + using V = decltype(Zero(d)); + constexpr size_t unroll = 8; + // Compiler doesn't make independent sum* accumulators, so unroll manually. + // Some older compilers might not be able to fit the 8 arrays in registers, + // so manual unrolling can be helpfull if you run into this issue. + // 2 FMA ports * 4 cycle latency = 8x unrolled. + V sum[unroll]; + for (size_t i = 0; i < unroll; ++i) { + sum[i] = Zero(d); + } + const float* const HWY_RESTRICT pa = &a_[0]; + const float* const HWY_RESTRICT pb = b_; + for (size_t i = 0; i < num_items; i += unroll * N) { + for (size_t j = 0; j < unroll; ++j) { + const auto a = Load(d, pa + i + j * N); + const auto b = Load(d, pb + i + j * N); + sum[j] = MulAdd(a, b, sum[j]); + } + } + // Reduction tree: sum of all accumulators by pairs into sum[0], then the + // lanes. + for (size_t power = 1; power < unroll; power *= 2) { + for (size_t i = 0; i < unroll; i += 2 * power) { + sum[i] += sum[i + power]; + } + } + dot_ = GetLane(SumOfLanes(d, sum[0])); + return static_cast<FuncOutput>(dot_); + } + void Verify(size_t num_items) { + if (dot_ == -1.0f) { + fprintf(stderr, "Dot: must call Verify after benchmark"); + abort(); + } + + const float expected = + std::inner_product(a_.get(), a_.get() + num_items, b_, 0.0f); + const float rel_err = std::abs(expected - dot_) / expected; + if (rel_err > 1.1E-6f) { + fprintf(stderr, "Dot: expected %e actual %e (%e)\n", expected, dot_, + rel_err); + abort(); + } + } + + private: + float dot_; // for Verify +}; + +// INTERMEDIATE: delta coding +// 1.0 cycles/float = bronze, 0.7 = silver, 0.4 = gold! +struct BenchmarkDelta : public TwoArray { + FuncOutput operator()(const size_t num_items) const { +#if HWY_TARGET == HWY_SCALAR + b_[0] = a_[0]; + for (size_t i = 1; i < num_items; ++i) { + b_[i] = a_[i] - a_[i - 1]; + } +#elif HWY_CAP_GE256 + // Larger vectors are split into 128-bit blocks, easiest to use the + // unaligned load support to shift between them. + const ScalableTag<float> df; + const size_t N = Lanes(df); + size_t i; + b_[0] = a_[0]; + for (i = 1; i < N; ++i) { + b_[i] = a_[i] - a_[i - 1]; + } + for (; i < num_items; i += N) { + const auto a = Load(df, &a_[i]); + const auto shifted = LoadU(df, &a_[i - 1]); + Store(a - shifted, df, &b_[i]); + } +#else // 128-bit + // Slightly better than unaligned loads + const HWY_CAPPED(float, 4) df; + const size_t N = Lanes(df); + size_t i; + b_[0] = a_[0]; + for (i = 1; i < N; ++i) { + b_[i] = a_[i] - a_[i - 1]; + } + auto prev = Load(df, &a_[0]); + for (; i < num_items; i += Lanes(df)) { + const auto a = Load(df, &a_[i]); + const auto shifted = CombineShiftRightLanes<3>(a, prev); + prev = a; + Store(a - shifted, df, &b_[i]); + } +#endif + return static_cast<FuncOutput>(b_[num_items - 1]); + } + + void Verify(size_t num_items) { + for (size_t i = 0; i < num_items; ++i) { + const float expected = (i == 0) ? a_[0] : a_[i] - a_[i - 1]; + const float err = std::abs(expected - b_[i]); + if (err > 1E-6f) { + fprintf(stderr, "Delta: expected %e, actual %e\n", expected, b_[i]); + } + } + } +}; + +void RunBenchmarks() { + Intro(); + printf("------------------------ %s\n", TargetName(HWY_TARGET)); + RunBenchmark<BenchmarkDot>("dot"); + RunBenchmark<BenchmarkDelta>("delta"); +} + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE +namespace hwy { +HWY_EXPORT(RunBenchmarks); + +void Run() { + for (uint32_t target : SupportedAndGeneratedTargets()) { + SetSupportedTargetsForTest(target); + HWY_DYNAMIC_DISPATCH(RunBenchmarks)(); + } + SetSupportedTargetsForTest(0); // Reset the mask afterwards. +} + +} // namespace hwy + +int main(int /*argc*/, char** /*argv*/) { + hwy::Run(); + return 0; +} +#endif // HWY_ONCE diff --git a/third_party/highway/hwy/examples/skeleton-inl.h b/third_party/highway/hwy/examples/skeleton-inl.h new file mode 100644 index 0000000..d8136be --- /dev/null +++ b/third_party/highway/hwy/examples/skeleton-inl.h @@ -0,0 +1,62 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Demo of functions that might be called from multiple SIMD modules (either +// other -inl.h files, or a .cc file between begin/end_target-inl). This is +// optional - all SIMD code can reside in .cc files. However, this allows +// splitting code into different files while still inlining instead of requiring +// calling through function pointers. + +// Include guard (still compiled once per target) +#if defined(HIGHWAY_HWY_EXAMPLES_SKELETON_INL_H_) == defined(HWY_TARGET_TOGGLE) +#ifdef HIGHWAY_HWY_EXAMPLES_SKELETON_INL_H_ +#undef HIGHWAY_HWY_EXAMPLES_SKELETON_INL_H_ +#else +#define HIGHWAY_HWY_EXAMPLES_SKELETON_INL_H_ +#endif + +// It is fine to #include normal or *-inl headers. +#include <stddef.h> + +#include "hwy/highway.h" + +HWY_BEFORE_NAMESPACE(); +namespace skeleton { +namespace HWY_NAMESPACE { + +using namespace hwy::HWY_NAMESPACE; + +// Example of a type-agnostic (caller-specified lane type) and width-agnostic +// (uses best available instruction set) function in a header. +// +// Computes x[i] = mul_array[i] * x_array[i] + add_array[i] for i < size. +template <class D, typename T> +HWY_MAYBE_UNUSED void MulAddLoop(const D d, const T* HWY_RESTRICT mul_array, + const T* HWY_RESTRICT add_array, + const size_t size, T* HWY_RESTRICT x_array) { + for (size_t i = 0; i < size; i += Lanes(d)) { + const auto mul = Load(d, mul_array + i); + const auto add = Load(d, add_array + i); + auto x = Load(d, x_array + i); + x = MulAdd(mul, x, add); + Store(x, d, x_array + i); + } +} + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace skeleton +HWY_AFTER_NAMESPACE(); + +#endif // include guard diff --git a/third_party/highway/hwy/examples/skeleton.cc b/third_party/highway/hwy/examples/skeleton.cc new file mode 100644 index 0000000..590d8be --- /dev/null +++ b/third_party/highway/hwy/examples/skeleton.cc @@ -0,0 +1,115 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "hwy/examples/skeleton.h" + +#include <stdio.h> + +// First undef to prevent error when re-included. +#undef HWY_TARGET_INCLUDE +// For runtime dispatch, specify the name of the current file (unfortunately +// __FILE__ is not reliable) so that foreach_target.h can re-include it. +#define HWY_TARGET_INCLUDE "hwy/examples/skeleton.cc" +// Generates code for each enabled target by re-including this source file. +#include "hwy/foreach_target.h" + +#include "hwy/highway.h" + +// Optional, can instead add HWY_ATTR to all functions. +HWY_BEFORE_NAMESPACE(); +namespace skeleton { +namespace HWY_NAMESPACE { + +// Highway ops reside here; ADL does not find templates nor builtins. +using namespace hwy::HWY_NAMESPACE; + +// For reasons unknown, optimized msan builds encounter long build times here; +// work around it until a cause is found. +#if HWY_COMPILER_CLANG && defined(MEMORY_SANITIZER) && defined(__OPTIMIZE__) +#define ATTR_MSAN __attribute__((optnone)) +#else +#define ATTR_MSAN +#endif + +// Computes log2 by converting to a vector of floats. Compiled once per target. +template <class DF> +ATTR_MSAN void OneFloorLog2(const DF df, const uint8_t* HWY_RESTRICT values, + uint8_t* HWY_RESTRICT log2) { + // Type tags for converting to other element types (Rebind = same count). + const Rebind<int32_t, DF> d32; + const Rebind<uint8_t, DF> d8; + + const auto u8 = Load(d8, values); + const auto bits = BitCast(d32, ConvertTo(df, PromoteTo(d32, u8))); + const auto exponent = Sub(ShiftRight<23>(bits), Set(d32, 127)); + Store(DemoteTo(d8, exponent), d8, log2); +} + +void CodepathDemo() { + // Highway defaults to portability, but per-target codepaths may be selected + // via #if HWY_TARGET == HWY_SSE4 or by testing capability macros: +#if HWY_CAP_INTEGER64 + const char* gather = "Has int64"; +#else + const char* gather = "No int64"; +#endif + printf("Target %s: %s\n", hwy::TargetName(HWY_TARGET), gather); +} + +void FloorLog2(const uint8_t* HWY_RESTRICT values, size_t count, + uint8_t* HWY_RESTRICT log2) { + CodepathDemo(); + + // Second argument is necessary on RVV until it supports fractional lengths. + const ScalableTag<float, 2> df; + + const size_t N = Lanes(df); + size_t i = 0; + for (; i + N <= count; i += N) { + OneFloorLog2(df, values + i, log2 + i); + } + // TODO(janwas): implement +#if HWY_TARGET != HWY_RVV + for (; i < count; ++i) { + OneFloorLog2(HWY_CAPPED(float, 1)(), values + i, log2 + i); + } +#endif +} + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace skeleton +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE + +namespace skeleton { + +// This macro declares a static array used for dynamic dispatch; it resides in +// the same outer namespace that contains FloorLog2. +HWY_EXPORT(FloorLog2); + +// This function is optional and only needed in the case of exposing it in the +// header file. Otherwise using HWY_DYNAMIC_DISPATCH(FloorLog2) in this module +// is equivalent to inlining this function. +void CallFloorLog2(const uint8_t* HWY_RESTRICT in, const size_t count, + uint8_t* HWY_RESTRICT out) { + return HWY_DYNAMIC_DISPATCH(FloorLog2)(in, count, out); +} + +// Optional: anything to compile only once, e.g. non-SIMD implementations of +// public functions provided by this module, can go inside #if HWY_ONCE. + +} // namespace skeleton +#endif // HWY_ONCE diff --git a/third_party/highway/hwy/examples/skeleton.h b/third_party/highway/hwy/examples/skeleton.h new file mode 100644 index 0000000..4935b88 --- /dev/null +++ b/third_party/highway/hwy/examples/skeleton.h @@ -0,0 +1,35 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Demo interface to target-specific code in skeleton.cc + +// Normal header with include guard and namespace. +#ifndef HIGHWAY_HWY_EXAMPLES_SKELETON_H_ +#define HIGHWAY_HWY_EXAMPLES_SKELETON_H_ + +#include <stddef.h> + +// Platform-specific definitions used for declaring an interface, independent of +// the SIMD instruction set. +#include "hwy/base.h" // HWY_RESTRICT + +namespace skeleton { + +// Computes base-2 logarithm by converting to float. Supports dynamic dispatch. +void CallFloorLog2(const uint8_t* HWY_RESTRICT in, const size_t count, + uint8_t* HWY_RESTRICT out); + +} // namespace skeleton + +#endif // HIGHWAY_HWY_EXAMPLES_SKELETON_H_ diff --git a/third_party/highway/hwy/examples/skeleton_test.cc b/third_party/highway/hwy/examples/skeleton_test.cc new file mode 100644 index 0000000..7f79b18 --- /dev/null +++ b/third_party/highway/hwy/examples/skeleton_test.cc @@ -0,0 +1,115 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Example of unit test for the "skeleton" library. + +#include "hwy/examples/skeleton.h" + +#include <stdio.h> + +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "examples/skeleton_test.cc" +#include "hwy/foreach_target.h" +#include "hwy/highway.h" +#include "hwy/tests/test_util-inl.h" + +// Optional: factor out parts of the implementation into *-inl.h +#include "hwy/examples/skeleton-inl.h" + +HWY_BEFORE_NAMESPACE(); +namespace skeleton { +namespace HWY_NAMESPACE { + +using namespace hwy::HWY_NAMESPACE; + +// Calls function defined in skeleton.cc. +struct TestFloorLog2 { + template <class T, class DF> + HWY_NOINLINE void operator()(T /*unused*/, DF df) { + const size_t count = 5 * Lanes(df); + auto in = hwy::AllocateAligned<uint8_t>(count); + auto expected = hwy::AllocateAligned<uint8_t>(count); + + hwy::RandomState rng; + for (size_t i = 0; i < count; ++i) { + expected[i] = Random32(&rng) & 7; + in[i] = static_cast<uint8_t>(1u << expected[i]); + } + auto out = hwy::AllocateAligned<uint8_t>(count); + CallFloorLog2(in.get(), count, out.get()); + int sum = 0; + for (size_t i = 0; i < count; ++i) { + // TODO(janwas): implement +#if HWY_TARGET != HWY_RVV + HWY_ASSERT_EQ(expected[i], out[i]); +#endif + sum += out[i]; + } + hwy::PreventElision(sum); + } +}; + +HWY_NOINLINE void TestAllFloorLog2() { + ForPartialVectors<TestFloorLog2>()(float()); +} + +// Calls function defined in skeleton-inl.h. +struct TestSumMulAdd { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + hwy::RandomState rng; + const size_t count = 4096; + EXPECT_TRUE(count % Lanes(d) == 0); + auto mul = hwy::AllocateAligned<T>(count); + auto x = hwy::AllocateAligned<T>(count); + auto add = hwy::AllocateAligned<T>(count); + for (size_t i = 0; i < count; ++i) { + mul[i] = static_cast<T>(Random32(&rng) & 0xF); + x[i] = static_cast<T>(Random32(&rng) & 0xFF); + add[i] = static_cast<T>(Random32(&rng) & 0xFF); + } + double expected_sum = 0.0; + for (size_t i = 0; i < count; ++i) { + expected_sum += mul[i] * x[i] + add[i]; + } + + MulAddLoop(d, mul.get(), add.get(), count, x.get()); + HWY_ASSERT_EQ(4344240.0, expected_sum); + } +}; + +HWY_NOINLINE void TestAllSumMulAdd() { + ForFloatTypes(ForPartialVectors<TestSumMulAdd>()); +} + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace skeleton +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE + +namespace skeleton { +HWY_BEFORE_TEST(SkeletonTest); +HWY_EXPORT_AND_TEST_P(SkeletonTest, TestAllFloorLog2); +HWY_EXPORT_AND_TEST_P(SkeletonTest, TestAllSumMulAdd); +} // namespace skeleton + +// Ought not to be necessary, but without this, no tests run on RVV. +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#endif diff --git a/third_party/highway/hwy/foreach_target.h b/third_party/highway/hwy/foreach_target.h new file mode 100644 index 0000000..8ce0560 --- /dev/null +++ b/third_party/highway/hwy/foreach_target.h @@ -0,0 +1,194 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef HIGHWAY_HWY_FOREACH_TARGET_H_ +#define HIGHWAY_HWY_FOREACH_TARGET_H_ + +// Re-includes the translation unit zero or more times to compile for any +// targets except HWY_STATIC_TARGET. Defines unique HWY_TARGET each time so that +// highway.h defines the corresponding macro/namespace. + +#include "hwy/detect_targets.h" + +// *_inl.h may include other headers, which requires include guards to prevent +// repeated inclusion. The guards must be reset after compiling each target, so +// the header is again visible. This is done by flipping HWY_TARGET_TOGGLE, +// defining it if undefined and vice versa. This macro is initially undefined +// so that IDEs don't gray out the contents of each header. +#ifdef HWY_TARGET_TOGGLE +#error "This macro must not be defined outside foreach_target.h" +#endif + +#ifdef HWY_HIGHWAY_INCLUDED // highway.h include guard +// Trigger fixup at the bottom of this header. +#define HWY_ALREADY_INCLUDED + +// The next highway.h must re-include set_macros-inl.h because the first +// highway.h chose the static target instead of what we will set below. +#undef HWY_SET_MACROS_PER_TARGET +#endif + +// Disable HWY_EXPORT in user code until we have generated all targets. Note +// that a subsequent highway.h will not override this definition. +#undef HWY_ONCE +#define HWY_ONCE (0 || HWY_IDE) + +// Avoid warnings on #include HWY_TARGET_INCLUDE by hiding them from the IDE; +// also skip if only 1 target defined (no re-inclusion will be necessary). +#if !HWY_IDE && (HWY_TARGETS != HWY_STATIC_TARGET) + +#if !defined(HWY_TARGET_INCLUDE) +#error ">1 target enabled => define HWY_TARGET_INCLUDE before foreach_target.h" +#endif + +#if (HWY_TARGETS & HWY_SCALAR) && (HWY_STATIC_TARGET != HWY_SCALAR) +#undef HWY_TARGET +#define HWY_TARGET HWY_SCALAR +#include HWY_TARGET_INCLUDE +#ifdef HWY_TARGET_TOGGLE +#undef HWY_TARGET_TOGGLE +#else +#define HWY_TARGET_TOGGLE +#endif +#endif + +#if (HWY_TARGETS & HWY_NEON) && (HWY_STATIC_TARGET != HWY_NEON) +#undef HWY_TARGET +#define HWY_TARGET HWY_NEON +#include HWY_TARGET_INCLUDE +#ifdef HWY_TARGET_TOGGLE +#undef HWY_TARGET_TOGGLE +#else +#define HWY_TARGET_TOGGLE +#endif +#endif + +#if (HWY_TARGETS & HWY_SSSE3) && (HWY_STATIC_TARGET != HWY_SSSE3) +#undef HWY_TARGET +#define HWY_TARGET HWY_SSSE3 +#include HWY_TARGET_INCLUDE +#ifdef HWY_TARGET_TOGGLE +#undef HWY_TARGET_TOGGLE +#else +#define HWY_TARGET_TOGGLE +#endif +#endif + +#if (HWY_TARGETS & HWY_SSE4) && (HWY_STATIC_TARGET != HWY_SSE4) +#undef HWY_TARGET +#define HWY_TARGET HWY_SSE4 +#include HWY_TARGET_INCLUDE +#ifdef HWY_TARGET_TOGGLE +#undef HWY_TARGET_TOGGLE +#else +#define HWY_TARGET_TOGGLE +#endif +#endif + +#if (HWY_TARGETS & HWY_AVX2) && (HWY_STATIC_TARGET != HWY_AVX2) +#undef HWY_TARGET +#define HWY_TARGET HWY_AVX2 +#include HWY_TARGET_INCLUDE +#ifdef HWY_TARGET_TOGGLE +#undef HWY_TARGET_TOGGLE +#else +#define HWY_TARGET_TOGGLE +#endif +#endif + +#if (HWY_TARGETS & HWY_AVX3) && (HWY_STATIC_TARGET != HWY_AVX3) +#undef HWY_TARGET +#define HWY_TARGET HWY_AVX3 +#include HWY_TARGET_INCLUDE +#ifdef HWY_TARGET_TOGGLE +#undef HWY_TARGET_TOGGLE +#else +#define HWY_TARGET_TOGGLE +#endif +#endif + +#if (HWY_TARGETS & HWY_AVX3_DL) && (HWY_STATIC_TARGET != HWY_AVX3_DL) +#undef HWY_TARGET +#define HWY_TARGET HWY_AVX3_DL +#include HWY_TARGET_INCLUDE +#ifdef HWY_TARGET_TOGGLE +#undef HWY_TARGET_TOGGLE +#else +#define HWY_TARGET_TOGGLE +#endif +#endif + +#if (HWY_TARGETS & HWY_WASM2) && (HWY_STATIC_TARGET != HWY_WASM2) +#undef HWY_TARGET +#define HWY_TARGET HWY_WASM2 +#include HWY_TARGET_INCLUDE +#ifdef HWY_TARGET_TOGGLE +#undef HWY_TARGET_TOGGLE +#else +#define HWY_TARGET_TOGGLE +#endif +#endif + +#if (HWY_TARGETS & HWY_WASM) && (HWY_STATIC_TARGET != HWY_WASM) +#undef HWY_TARGET +#define HWY_TARGET HWY_WASM +#include HWY_TARGET_INCLUDE +#ifdef HWY_TARGET_TOGGLE +#undef HWY_TARGET_TOGGLE +#else +#define HWY_TARGET_TOGGLE +#endif +#endif + +#if (HWY_TARGETS & HWY_PPC8) && (HWY_STATIC_TARGET != HWY_PPC8) +#undef HWY_TARGET +#define HWY_TARGET HWY_PPC8 +#include HWY_TARGET_INCLUDE +#ifdef HWY_TARGET_TOGGLE +#undef HWY_TARGET_TOGGLE +#else +#define HWY_TARGET_TOGGLE +#endif +#endif + +#endif // !HWY_IDE && (HWY_TARGETS != HWY_STATIC_TARGET) + +// Now that all but the static target have been generated, re-enable HWY_EXPORT. +#undef HWY_ONCE +#define HWY_ONCE 1 + +// If we re-include once per enabled target, the translation unit's +// implementation would have to be skipped via #if to avoid redefining symbols. +// We instead skip the re-include for HWY_STATIC_TARGET, and generate its +// implementation when resuming compilation of the translation unit. +#undef HWY_TARGET +#define HWY_TARGET HWY_STATIC_TARGET + +#ifdef HWY_ALREADY_INCLUDED +// Revert the previous toggle to prevent redefinitions for the static target. +#ifdef HWY_TARGET_TOGGLE +#undef HWY_TARGET_TOGGLE +#else +#define HWY_TARGET_TOGGLE +#endif + +// Force re-inclusion of set_macros-inl.h now that HWY_TARGET is restored. +#ifdef HWY_SET_MACROS_PER_TARGET +#undef HWY_SET_MACROS_PER_TARGET +#else +#define HWY_SET_MACROS_PER_TARGET +#endif +#endif + +#endif // HIGHWAY_HWY_FOREACH_TARGET_H_ diff --git a/third_party/highway/hwy/highway.h b/third_party/highway/hwy/highway.h new file mode 100644 index 0000000..174e171 --- /dev/null +++ b/third_party/highway/hwy/highway.h @@ -0,0 +1,320 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This include guard is checked by foreach_target, so avoid the usual _H_ +// suffix to prevent copybara from renaming it. NOTE: ops/*-inl.h are included +// after/outside this include guard. +#ifndef HWY_HIGHWAY_INCLUDED +#define HWY_HIGHWAY_INCLUDED + +// Main header required before using vector types. + +#include "hwy/base.h" +#include "hwy/targets.h" + +namespace hwy { + +// API version (https://semver.org/); keep in sync with CMakeLists.txt. +#define HWY_MAJOR 0 +#define HWY_MINOR 15 +#define HWY_PATCH 0 + +//------------------------------------------------------------------------------ +// Shorthand for tags (defined in shared-inl.h) used to select overloads. +// Note that ScalableTag<T> is preferred over HWY_FULL, and CappedTag<T, N> over +// HWY_CAPPED(T, N). + +// HWY_FULL(T[,LMUL=1]) is a native vector/group. LMUL is the number of +// registers in the group, and is ignored on targets that do not support groups. +#define HWY_FULL1(T) hwy::HWY_NAMESPACE::Simd<T, HWY_LANES(T)> +#define HWY_3TH_ARG(arg1, arg2, arg3, ...) arg3 +// Workaround for MSVC grouping __VA_ARGS__ into a single argument +#define HWY_FULL_RECOMPOSER(args_with_paren) HWY_3TH_ARG args_with_paren +// Trailing comma avoids -pedantic false alarm +#define HWY_CHOOSE_FULL(...) \ + HWY_FULL_RECOMPOSER((__VA_ARGS__, HWY_FULL2, HWY_FULL1, )) +#define HWY_FULL(...) HWY_CHOOSE_FULL(__VA_ARGS__())(__VA_ARGS__) + +// Vector of up to MAX_N lanes. Discouraged, when possible, use Half<> instead. +#define HWY_CAPPED(T, MAX_N) \ + hwy::HWY_NAMESPACE::Simd<T, HWY_MIN(MAX_N, HWY_LANES(T))> + +//------------------------------------------------------------------------------ +// Export user functions for static/dynamic dispatch + +// Evaluates to 0 inside a translation unit if it is generating anything but the +// static target (the last one if multiple targets are enabled). Used to prevent +// redefinitions of HWY_EXPORT. Unless foreach_target.h is included, we only +// compile once anyway, so this is 1 unless it is or has been included. +#ifndef HWY_ONCE +#define HWY_ONCE 1 +#endif + +// HWY_STATIC_DISPATCH(FUNC_NAME) is the namespace-qualified FUNC_NAME for +// HWY_STATIC_TARGET (the only defined namespace unless HWY_TARGET_INCLUDE is +// defined), and can be used to deduce the return type of Choose*. +#if HWY_STATIC_TARGET == HWY_SCALAR +#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SCALAR::FUNC_NAME +#elif HWY_STATIC_TARGET == HWY_RVV +#define HWY_STATIC_DISPATCH(FUNC_NAME) N_RVV::FUNC_NAME +#elif HWY_STATIC_TARGET == HWY_WASM2 +#define HWY_STATIC_DISPATCH(FUNC_NAME) N_WASM2::FUNC_NAME +#elif HWY_STATIC_TARGET == HWY_WASM +#define HWY_STATIC_DISPATCH(FUNC_NAME) N_WASM::FUNC_NAME +#elif HWY_STATIC_TARGET == HWY_NEON +#define HWY_STATIC_DISPATCH(FUNC_NAME) N_NEON::FUNC_NAME +#elif HWY_STATIC_TARGET == HWY_SVE +#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE::FUNC_NAME +#elif HWY_STATIC_TARGET == HWY_SVE2 +#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SVE2::FUNC_NAME +#elif HWY_STATIC_TARGET == HWY_PPC8 +#define HWY_STATIC_DISPATCH(FUNC_NAME) N_PPC8::FUNC_NAME +#elif HWY_STATIC_TARGET == HWY_SSSE3 +#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SSSE3::FUNC_NAME +#elif HWY_STATIC_TARGET == HWY_SSE4 +#define HWY_STATIC_DISPATCH(FUNC_NAME) N_SSE4::FUNC_NAME +#elif HWY_STATIC_TARGET == HWY_AVX2 +#define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX2::FUNC_NAME +#elif HWY_STATIC_TARGET == HWY_AVX3 +#define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX3::FUNC_NAME +#elif HWY_STATIC_TARGET == HWY_AVX3_DL +#define HWY_STATIC_DISPATCH(FUNC_NAME) N_AVX3_DL::FUNC_NAME +#endif + +// Dynamic dispatch declarations. + +template <typename RetType, typename... Args> +struct FunctionCache { + public: + typedef RetType(FunctionType)(Args...); + + // A template function that when instantiated has the same signature as the + // function being called. This function initializes the global cache of the + // current supported targets mask used for dynamic dispatch and calls the + // appropriate function. Since this mask used for dynamic dispatch is a + // global cache, all the highway exported functions, even those exposed by + // different modules, will be initialized after this function runs for any one + // of those exported functions. + template <FunctionType* const table[]> + static RetType ChooseAndCall(Args... args) { + // If we are running here it means we need to update the chosen target. + chosen_target.Update(); + return (table[chosen_target.GetIndex()])(args...); + } +}; + +// Factory function only used to infer the template parameters RetType and Args +// from a function passed to the factory. +template <typename RetType, typename... Args> +FunctionCache<RetType, Args...> FunctionCacheFactory(RetType (*)(Args...)) { + return FunctionCache<RetType, Args...>(); +} + +// HWY_CHOOSE_*(FUNC_NAME) expands to the function pointer for that target or +// nullptr is that target was not compiled. +#if HWY_TARGETS & HWY_SCALAR +#define HWY_CHOOSE_SCALAR(FUNC_NAME) &N_SCALAR::FUNC_NAME +#else +// When scalar is not present and we try to use scalar because other targets +// were disabled at runtime we fall back to the baseline with +// HWY_STATIC_DISPATCH() +#define HWY_CHOOSE_SCALAR(FUNC_NAME) &HWY_STATIC_DISPATCH(FUNC_NAME) +#endif + +#if HWY_TARGETS & HWY_WASM2 +#define HWY_CHOOSE_WASM2(FUNC_NAME) &N_WASM2::FUNC_NAME +#else +#define HWY_CHOOSE_WASM2(FUNC_NAME) nullptr +#endif + +#if HWY_TARGETS & HWY_WASM +#define HWY_CHOOSE_WASM(FUNC_NAME) &N_WASM::FUNC_NAME +#else +#define HWY_CHOOSE_WASM(FUNC_NAME) nullptr +#endif + +#if HWY_TARGETS & HWY_RVV +#define HWY_CHOOSE_RVV(FUNC_NAME) &N_RVV::FUNC_NAME +#else +#define HWY_CHOOSE_RVV(FUNC_NAME) nullptr +#endif + +#if HWY_TARGETS & HWY_NEON +#define HWY_CHOOSE_NEON(FUNC_NAME) &N_NEON::FUNC_NAME +#else +#define HWY_CHOOSE_NEON(FUNC_NAME) nullptr +#endif + +#if HWY_TARGETS & HWY_SVE +#define HWY_CHOOSE_SVE(FUNC_NAME) &N_SVE::FUNC_NAME +#else +#define HWY_CHOOSE_SVE(FUNC_NAME) nullptr +#endif + +#if HWY_TARGETS & HWY_SVE2 +#define HWY_CHOOSE_SVE2(FUNC_NAME) &N_SVE2::FUNC_NAME +#else +#define HWY_CHOOSE_SVE2(FUNC_NAME) nullptr +#endif + +#if HWY_TARGETS & HWY_PPC8 +#define HWY_CHOOSE_PCC8(FUNC_NAME) &N_PPC8::FUNC_NAME +#else +#define HWY_CHOOSE_PPC8(FUNC_NAME) nullptr +#endif + +#if HWY_TARGETS & HWY_SSSE3 +#define HWY_CHOOSE_SSSE3(FUNC_NAME) &N_SSSE3::FUNC_NAME +#else +#define HWY_CHOOSE_SSSE3(FUNC_NAME) nullptr +#endif + +#if HWY_TARGETS & HWY_SSE4 +#define HWY_CHOOSE_SSE4(FUNC_NAME) &N_SSE4::FUNC_NAME +#else +#define HWY_CHOOSE_SSE4(FUNC_NAME) nullptr +#endif + +#if HWY_TARGETS & HWY_AVX2 +#define HWY_CHOOSE_AVX2(FUNC_NAME) &N_AVX2::FUNC_NAME +#else +#define HWY_CHOOSE_AVX2(FUNC_NAME) nullptr +#endif + +#if HWY_TARGETS & HWY_AVX3 +#define HWY_CHOOSE_AVX3(FUNC_NAME) &N_AVX3::FUNC_NAME +#else +#define HWY_CHOOSE_AVX3(FUNC_NAME) nullptr +#endif + +#if HWY_TARGETS & HWY_AVX3_DL +#define HWY_CHOOSE_AVX3_DL(FUNC_NAME) &N_AVX3_DL::FUNC_NAME +#else +#define HWY_CHOOSE_AVX3_DL(FUNC_NAME) nullptr +#endif + +#define HWY_DISPATCH_TABLE(FUNC_NAME) \ + HWY_CONCAT(FUNC_NAME, HighwayDispatchTable) + +// HWY_EXPORT(FUNC_NAME); expands to a static array that is used by +// HWY_DYNAMIC_DISPATCH() to call the appropriate function at runtime. This +// static array must be defined at the same namespace level as the function +// it is exporting. +// After being exported, it can be called from other parts of the same source +// file using HWY_DYNAMIC_DISTPATCH(), in particular from a function wrapper +// like in the following example: +// +// #include "hwy/highway.h" +// HWY_BEFORE_NAMESPACE(); +// namespace skeleton { +// namespace HWY_NAMESPACE { +// +// void MyFunction(int a, char b, const char* c) { ... } +// +// // NOLINTNEXTLINE(google-readability-namespace-comments) +// } // namespace HWY_NAMESPACE +// } // namespace skeleton +// HWY_AFTER_NAMESPACE(); +// +// namespace skeleton { +// HWY_EXPORT(MyFunction); // Defines the dispatch table in this scope. +// +// void MyFunction(int a, char b, const char* c) { +// return HWY_DYNAMIC_DISPATCH(MyFunction)(a, b, c); +// } +// } // namespace skeleton +// + +#if HWY_IDE || ((HWY_TARGETS & (HWY_TARGETS - 1)) == 0) + +// Simplified version for IDE or the dynamic dispatch case with only one target. +// This case still uses a table, although of a single element, to provide the +// same compile error conditions as with the dynamic dispatch case when multiple +// targets are being compiled. +#define HWY_EXPORT(FUNC_NAME) \ + HWY_MAYBE_UNUSED static decltype(&HWY_STATIC_DISPATCH(FUNC_NAME)) \ + const HWY_DISPATCH_TABLE(FUNC_NAME)[1] = { \ + &HWY_STATIC_DISPATCH(FUNC_NAME)} +#define HWY_DYNAMIC_DISPATCH(FUNC_NAME) HWY_STATIC_DISPATCH(FUNC_NAME) + +#else + +// Dynamic dispatch case with one entry per dynamic target plus the scalar +// mode and the initialization wrapper. +#define HWY_EXPORT(FUNC_NAME) \ + static decltype(&HWY_STATIC_DISPATCH(FUNC_NAME)) \ + const HWY_DISPATCH_TABLE(FUNC_NAME)[HWY_MAX_DYNAMIC_TARGETS + 2] = { \ + /* The first entry in the table initializes the global cache and \ + * calls the appropriate function. */ \ + &decltype(hwy::FunctionCacheFactory(&HWY_STATIC_DISPATCH( \ + FUNC_NAME)))::ChooseAndCall<HWY_DISPATCH_TABLE(FUNC_NAME)>, \ + HWY_CHOOSE_TARGET_LIST(FUNC_NAME), \ + HWY_CHOOSE_SCALAR(FUNC_NAME), \ + } +#define HWY_DYNAMIC_DISPATCH(FUNC_NAME) \ + (*(HWY_DISPATCH_TABLE(FUNC_NAME)[hwy::chosen_target.GetIndex()])) + +#endif // HWY_IDE || ((HWY_TARGETS & (HWY_TARGETS - 1)) == 0) + +} // namespace hwy + +#endif // HWY_HIGHWAY_INCLUDED + +//------------------------------------------------------------------------------ + +// NOTE: the following definitions and ops/*.h depend on HWY_TARGET, so we want +// to include them once per target, which is ensured by the toggle check. +// Because ops/*.h are included under it, they do not need their own guard. +#if defined(HWY_HIGHWAY_PER_TARGET) == defined(HWY_TARGET_TOGGLE) +#ifdef HWY_HIGHWAY_PER_TARGET +#undef HWY_HIGHWAY_PER_TARGET +#else +#define HWY_HIGHWAY_PER_TARGET +#endif + +#undef HWY_FULL2 +#if HWY_TARGET == HWY_RVV +#define HWY_FULL2(T, LMUL) hwy::HWY_NAMESPACE::Simd<T, HWY_LANES(T) * (LMUL)> +#else +#define HWY_FULL2(T, LMUL) hwy::HWY_NAMESPACE::Simd<T, HWY_LANES(T)> +#endif + +// These define ops inside namespace hwy::HWY_NAMESPACE. +#if HWY_TARGET == HWY_SSSE3 || HWY_TARGET == HWY_SSE4 +#include "hwy/ops/x86_128-inl.h" +#elif HWY_TARGET == HWY_AVX2 +#include "hwy/ops/x86_256-inl.h" +#elif HWY_TARGET == HWY_AVX3 || HWY_TARGET == HWY_AVX3_DL +#include "hwy/ops/x86_512-inl.h" +#elif HWY_TARGET == HWY_PPC8 +#error "PPC is not yet supported" +#elif HWY_TARGET == HWY_NEON +#include "hwy/ops/arm_neon-inl.h" +#elif HWY_TARGET == HWY_SVE || HWY_TARGET == HWY_SVE2 +#include "hwy/ops/arm_sve-inl.h" +#elif HWY_TARGET == HWY_WASM2 +#include "hwy/ops/wasm_256-inl.h" +#elif HWY_TARGET == HWY_WASM +#include "hwy/ops/wasm_128-inl.h" +#elif HWY_TARGET == HWY_RVV +#include "hwy/ops/rvv-inl.h" +#elif HWY_TARGET == HWY_SCALAR +#include "hwy/ops/scalar-inl.h" +#else +#pragma message("HWY_TARGET does not match any known target") +#endif // HWY_TARGET + +#include "hwy/ops/generic_ops-inl.h" + +#endif // HWY_HIGHWAY_PER_TARGET diff --git a/third_party/highway/hwy/highway_test.cc b/third_party/highway/hwy/highway_test.cc new file mode 100644 index 0000000..d71f419 --- /dev/null +++ b/third_party/highway/hwy/highway_test.cc @@ -0,0 +1,341 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <stddef.h> +#include <stdint.h> + +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "highway_test.cc" +#include "hwy/foreach_target.h" +#include "hwy/highway.h" +#include "hwy/nanobenchmark.h" // Unpredictable1 +#include "hwy/tests/test_util-inl.h" + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +struct TestSet { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + // Zero + const auto v0 = Zero(d); + const size_t N = Lanes(d); + auto expected = AllocateAligned<T>(N); + std::fill(expected.get(), expected.get() + N, T(0)); + HWY_ASSERT_VEC_EQ(d, expected.get(), v0); + + // Set + const auto v2 = Set(d, T(2)); + for (size_t i = 0; i < N; ++i) { + expected[i] = 2; + } + HWY_ASSERT_VEC_EQ(d, expected.get(), v2); + + // Iota + const auto vi = Iota(d, T(5)); + for (size_t i = 0; i < N; ++i) { + expected[i] = T(5 + i); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), vi); + + // Undefined + const auto vu = Undefined(d); + Store(vu, d, expected.get()); + } +}; + +HWY_NOINLINE void TestAllSet() { ForAllTypes(ForPartialVectors<TestSet>()); } + +// Ensures wraparound (mod 2^bits) +struct TestOverflow { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v1 = Set(d, T(1)); + const auto vmax = Set(d, LimitsMax<T>()); + const auto vmin = Set(d, LimitsMin<T>()); + // Unsigned underflow / negative -> positive + HWY_ASSERT_VEC_EQ(d, vmax, Sub(vmin, v1)); + // Unsigned overflow / positive -> negative + HWY_ASSERT_VEC_EQ(d, vmin, Add(vmax, v1)); + } +}; + +HWY_NOINLINE void TestAllOverflow() { + ForIntegerTypes(ForPartialVectors<TestOverflow>()); +} + +struct TestClamp { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v0 = Zero(d); + const auto v1 = Set(d, 1); + const auto v2 = Set(d, 2); + + HWY_ASSERT_VEC_EQ(d, v1, Clamp(v2, v0, v1)); + HWY_ASSERT_VEC_EQ(d, v1, Clamp(v0, v1, v2)); + } +}; + +HWY_NOINLINE void TestAllClamp() { + ForAllTypes(ForPartialVectors<TestClamp>()); +} + +struct TestSignBitInteger { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v0 = Zero(d); + const auto all = VecFromMask(d, Eq(v0, v0)); + const auto vs = SignBit(d); + const auto other = Sub(vs, Set(d, 1)); + + // Shifting left by one => overflow, equal zero + HWY_ASSERT_VEC_EQ(d, v0, Add(vs, vs)); + // Verify the lower bits are zero (only +/- and logical ops are available + // for all types) + HWY_ASSERT_VEC_EQ(d, all, Add(vs, other)); + } +}; + +struct TestSignBitFloat { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v0 = Zero(d); + const auto vs = SignBit(d); + const auto vp = Set(d, 2.25); + const auto vn = Set(d, -2.25); + HWY_ASSERT_VEC_EQ(d, Or(vp, vs), vn); + HWY_ASSERT_VEC_EQ(d, AndNot(vs, vn), vp); + HWY_ASSERT_VEC_EQ(d, v0, vs); + } +}; + +HWY_NOINLINE void TestAllSignBit() { + ForIntegerTypes(ForPartialVectors<TestSignBitInteger>()); + ForFloatTypes(ForPartialVectors<TestSignBitFloat>()); +} + +// std::isnan returns false for 0x7F..FF in clang AVX3 builds, so DIY. +template <typename TF> +bool IsNaN(TF f) { + MakeUnsigned<TF> bits; + memcpy(&bits, &f, sizeof(TF)); + bits += bits; + bits >>= 1; // clear sign bit + // NaN if all exponent bits are set and the mantissa is not zero. + return bits > ExponentMask<decltype(bits)>(); +} + +template <class D, class V> +HWY_NOINLINE void AssertNaN(D d, VecArg<V> v, const char* file, int line) { + using T = TFromD<D>; + const T lane = GetLane(v); + if (!IsNaN(lane)) { + const std::string type_name = TypeName(T(), Lanes(d)); + // RVV lacks PRIu64 and MSYS still has problems with %zu, so print bytes to + // avoid truncating doubles. + uint8_t bytes[HWY_MAX(sizeof(T), 8)] = {0}; + memcpy(bytes, &lane, sizeof(T)); + Abort(file, line, + "Expected %s NaN, got %E (bytes %02x %02x %02x %02x %02x %02x %02x " + "%02x)", + type_name.c_str(), lane, bytes[0], bytes[1], bytes[2], bytes[3], + bytes[4], bytes[5], bytes[6], bytes[7]); + } +} + +#define HWY_ASSERT_NAN(d, v) AssertNaN(d, v, __FILE__, __LINE__) + +struct TestNaN { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v1 = Set(d, T(Unpredictable1())); + const auto nan = IfThenElse(Eq(v1, Set(d, T(1))), NaN(d), v1); + HWY_ASSERT_NAN(d, nan); + + // Arithmetic + HWY_ASSERT_NAN(d, Add(nan, v1)); + HWY_ASSERT_NAN(d, Add(v1, nan)); + HWY_ASSERT_NAN(d, Sub(nan, v1)); + HWY_ASSERT_NAN(d, Sub(v1, nan)); + HWY_ASSERT_NAN(d, Mul(nan, v1)); + HWY_ASSERT_NAN(d, Mul(v1, nan)); + HWY_ASSERT_NAN(d, Div(nan, v1)); + HWY_ASSERT_NAN(d, Div(v1, nan)); + + // FMA + HWY_ASSERT_NAN(d, MulAdd(nan, v1, v1)); + HWY_ASSERT_NAN(d, MulAdd(v1, nan, v1)); + HWY_ASSERT_NAN(d, MulAdd(v1, v1, nan)); + HWY_ASSERT_NAN(d, MulSub(nan, v1, v1)); + HWY_ASSERT_NAN(d, MulSub(v1, nan, v1)); + HWY_ASSERT_NAN(d, MulSub(v1, v1, nan)); + HWY_ASSERT_NAN(d, NegMulAdd(nan, v1, v1)); + HWY_ASSERT_NAN(d, NegMulAdd(v1, nan, v1)); + HWY_ASSERT_NAN(d, NegMulAdd(v1, v1, nan)); + HWY_ASSERT_NAN(d, NegMulSub(nan, v1, v1)); + HWY_ASSERT_NAN(d, NegMulSub(v1, nan, v1)); + HWY_ASSERT_NAN(d, NegMulSub(v1, v1, nan)); + + // Rcp/Sqrt + HWY_ASSERT_NAN(d, Sqrt(nan)); + + // Sign manipulation + HWY_ASSERT_NAN(d, Abs(nan)); + HWY_ASSERT_NAN(d, Neg(nan)); + HWY_ASSERT_NAN(d, CopySign(nan, v1)); + HWY_ASSERT_NAN(d, CopySignToAbs(nan, v1)); + + // Rounding + HWY_ASSERT_NAN(d, Ceil(nan)); + HWY_ASSERT_NAN(d, Floor(nan)); + HWY_ASSERT_NAN(d, Round(nan)); + HWY_ASSERT_NAN(d, Trunc(nan)); + + // Logical (And/AndNot/Xor will clear NaN!) + HWY_ASSERT_NAN(d, Or(nan, v1)); + + // Comparison + HWY_ASSERT(AllFalse(d, Eq(nan, v1))); + HWY_ASSERT(AllFalse(d, Gt(nan, v1))); + HWY_ASSERT(AllFalse(d, Lt(nan, v1))); + HWY_ASSERT(AllFalse(d, Ge(nan, v1))); + HWY_ASSERT(AllFalse(d, Le(nan, v1))); + + // Reduction + HWY_ASSERT_NAN(d, SumOfLanes(d, nan)); +// TODO(janwas): re-enable after QEMU is fixed +#if HWY_TARGET != HWY_RVV + HWY_ASSERT_NAN(d, MinOfLanes(d, nan)); + HWY_ASSERT_NAN(d, MaxOfLanes(d, nan)); +#endif + + // Min +#if HWY_ARCH_X86 && HWY_TARGET != HWY_SCALAR + // x86 SIMD returns the second operand if any input is NaN. + HWY_ASSERT_VEC_EQ(d, v1, Min(nan, v1)); + HWY_ASSERT_VEC_EQ(d, v1, Max(nan, v1)); + HWY_ASSERT_NAN(d, Min(v1, nan)); + HWY_ASSERT_NAN(d, Max(v1, nan)); +#elif HWY_ARCH_WASM + // Should return NaN if any input is NaN, but does not for scalar. + // TODO(janwas): remove once this is fixed. +#elif HWY_TARGET == HWY_NEON && HWY_ARCH_ARM_V7 + // ARMv7 NEON returns NaN if any input is NaN. + HWY_ASSERT_NAN(d, Min(v1, nan)); + HWY_ASSERT_NAN(d, Max(v1, nan)); + HWY_ASSERT_NAN(d, Min(nan, v1)); + HWY_ASSERT_NAN(d, Max(nan, v1)); +#else + // IEEE 754-2019 minimumNumber is defined as the other argument if exactly + // one is NaN, and qNaN if both are. + HWY_ASSERT_VEC_EQ(d, v1, Min(nan, v1)); + HWY_ASSERT_VEC_EQ(d, v1, Max(nan, v1)); + HWY_ASSERT_VEC_EQ(d, v1, Min(v1, nan)); + HWY_ASSERT_VEC_EQ(d, v1, Max(v1, nan)); +#endif + HWY_ASSERT_NAN(d, Min(nan, nan)); + HWY_ASSERT_NAN(d, Max(nan, nan)); + } +}; + +// For functions only available for float32 +struct TestF32NaN { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v1 = Set(d, T(Unpredictable1())); + const auto nan = IfThenElse(Eq(v1, Set(d, T(1))), NaN(d), v1); + HWY_ASSERT_NAN(d, ApproximateReciprocal(nan)); + HWY_ASSERT_NAN(d, ApproximateReciprocalSqrt(nan)); + HWY_ASSERT_NAN(d, AbsDiff(nan, v1)); + HWY_ASSERT_NAN(d, AbsDiff(v1, nan)); + } +}; + +HWY_NOINLINE void TestAllNaN() { + ForFloatTypes(ForPartialVectors<TestNaN>()); + ForPartialVectors<TestF32NaN>()(float()); +} + +struct TestCopyAndAssign { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + // copy V + const auto v3 = Iota(d, 3); + auto v3b(v3); + HWY_ASSERT_VEC_EQ(d, v3, v3b); + + // assign V + auto v3c = Undefined(d); + v3c = v3; + HWY_ASSERT_VEC_EQ(d, v3, v3c); + } +}; + +HWY_NOINLINE void TestAllCopyAndAssign() { + ForAllTypes(ForPartialVectors<TestCopyAndAssign>()); +} + +struct TestGetLane { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + HWY_ASSERT_EQ(T(0), GetLane(Zero(d))); + HWY_ASSERT_EQ(T(1), GetLane(Set(d, 1))); + } +}; + +HWY_NOINLINE void TestAllGetLane() { + ForAllTypes(ForPartialVectors<TestGetLane>()); +} + +struct TestDFromV { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v0 = Zero(d); + using D0 = DFromV<decltype(v0)>; // not necessarily same as D + const auto v0b = And(v0, Set(D0(), 1)); // but vectors can interoperate + HWY_ASSERT_VEC_EQ(d, v0, v0b); + } +}; + +HWY_NOINLINE void TestAllDFromV() { + ForAllTypes(ForPartialVectors<TestDFromV>()); +} + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE + +namespace hwy { +HWY_BEFORE_TEST(HighwayTest); +HWY_EXPORT_AND_TEST_P(HighwayTest, TestAllSet); +HWY_EXPORT_AND_TEST_P(HighwayTest, TestAllOverflow); +HWY_EXPORT_AND_TEST_P(HighwayTest, TestAllClamp); +HWY_EXPORT_AND_TEST_P(HighwayTest, TestAllSignBit); +HWY_EXPORT_AND_TEST_P(HighwayTest, TestAllNaN); +HWY_EXPORT_AND_TEST_P(HighwayTest, TestAllCopyAndAssign); +HWY_EXPORT_AND_TEST_P(HighwayTest, TestAllGetLane); +HWY_EXPORT_AND_TEST_P(HighwayTest, TestAllDFromV); +} // namespace hwy + +// Ought not to be necessary, but without this, no tests run on RVV. +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#endif diff --git a/third_party/highway/hwy/nanobenchmark.cc b/third_party/highway/hwy/nanobenchmark.cc new file mode 100644 index 0000000..9998c7e --- /dev/null +++ b/third_party/highway/hwy/nanobenchmark.cc @@ -0,0 +1,726 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "hwy/nanobenchmark.h" + +#include <inttypes.h> +#include <stddef.h> +#include <stdio.h> +#include <stdlib.h> // abort +#include <string.h> // memcpy +#include <time.h> // clock_gettime + +#include <algorithm> // sort +#include <array> +#include <atomic> +#include <limits> +#include <numeric> // iota +#include <random> +#include <string> +#include <vector> + +#if defined(_WIN32) || defined(_WIN64) +#ifndef NOMINMAX +#define NOMINMAX +#endif // NOMINMAX +#include <windows.h> +#endif + +#if defined(__MACH__) +#include <mach/mach.h> +#include <mach/mach_time.h> +#endif + +#if defined(__HAIKU__) +#include <OS.h> +#endif + +#include "hwy/base.h" +#if HWY_ARCH_PPC && defined(__GLIBC__) +#include <sys/platform/ppc.h> // NOLINT __ppc_get_timebase_freq +#elif HWY_ARCH_X86 + +#if HWY_COMPILER_MSVC +#include <intrin.h> +#else +#include <cpuid.h> // NOLINT +#endif // HWY_COMPILER_MSVC + +#endif // HWY_ARCH_X86 + +namespace hwy { +namespace { +namespace timer { + +// Ticks := platform-specific timer values (CPU cycles on x86). Must be +// unsigned to guarantee wraparound on overflow. +using Ticks = uint64_t; + +// Start/Stop return absolute timestamps and must be placed immediately before +// and after the region to measure. We provide separate Start/Stop functions +// because they use different fences. +// +// Background: RDTSC is not 'serializing'; earlier instructions may complete +// after it, and/or later instructions may complete before it. 'Fences' ensure +// regions' elapsed times are independent of such reordering. The only +// documented unprivileged serializing instruction is CPUID, which acts as a +// full fence (no reordering across it in either direction). Unfortunately +// the latency of CPUID varies wildly (perhaps made worse by not initializing +// its EAX input). Because it cannot reliably be deducted from the region's +// elapsed time, it must not be included in the region to measure (i.e. +// between the two RDTSC). +// +// The newer RDTSCP is sometimes described as serializing, but it actually +// only serves as a half-fence with release semantics. Although all +// instructions in the region will complete before the final timestamp is +// captured, subsequent instructions may leak into the region and increase the +// elapsed time. Inserting another fence after the final RDTSCP would prevent +// such reordering without affecting the measured region. +// +// Fortunately, such a fence exists. The LFENCE instruction is only documented +// to delay later loads until earlier loads are visible. However, Intel's +// reference manual says it acts as a full fence (waiting until all earlier +// instructions have completed, and delaying later instructions until it +// completes). AMD assigns the same behavior to MFENCE. +// +// We need a fence before the initial RDTSC to prevent earlier instructions +// from leaking into the region, and arguably another after RDTSC to avoid +// region instructions from completing before the timestamp is recorded. +// When surrounded by fences, the additional RDTSCP half-fence provides no +// benefit, so the initial timestamp can be recorded via RDTSC, which has +// lower overhead than RDTSCP because it does not read TSC_AUX. In summary, +// we define Start = LFENCE/RDTSC/LFENCE; Stop = RDTSCP/LFENCE. +// +// Using Start+Start leads to higher variance and overhead than Stop+Stop. +// However, Stop+Stop includes an LFENCE in the region measurements, which +// adds a delay dependent on earlier loads. The combination of Start+Stop +// is faster than Start+Start and more consistent than Stop+Stop because +// the first LFENCE already delayed subsequent loads before the measured +// region. This combination seems not to have been considered in prior work: +// http://akaros.cs.berkeley.edu/lxr/akaros/kern/arch/x86/rdtsc_test.c +// +// Note: performance counters can measure 'exact' instructions-retired or +// (unhalted) cycle counts. The RDPMC instruction is not serializing and also +// requires fences. Unfortunately, it is not accessible on all OSes and we +// prefer to avoid kernel-mode drivers. Performance counters are also affected +// by several under/over-count errata, so we use the TSC instead. + +// Returns a 64-bit timestamp in unit of 'ticks'; to convert to seconds, +// divide by InvariantTicksPerSecond. +inline Ticks Start() { + Ticks t; +#if HWY_ARCH_PPC && defined(__GLIBC__) + asm volatile("mfspr %0, %1" : "=r"(t) : "i"(268)); +#elif HWY_ARCH_X86 && HWY_COMPILER_MSVC + _ReadWriteBarrier(); + _mm_lfence(); + _ReadWriteBarrier(); + t = __rdtsc(); + _ReadWriteBarrier(); + _mm_lfence(); + _ReadWriteBarrier(); +#elif HWY_ARCH_X86_64 + asm volatile( + "lfence\n\t" + "rdtsc\n\t" + "shl $32, %%rdx\n\t" + "or %%rdx, %0\n\t" + "lfence" + : "=a"(t) + : + // "memory" avoids reordering. rdx = TSC >> 32. + // "cc" = flags modified by SHL. + : "rdx", "memory", "cc"); +#elif HWY_ARCH_RVV + asm volatile("rdcycle %0" : "=r"(t)); +#elif defined(_WIN32) || defined(_WIN64) + LARGE_INTEGER counter; + (void)QueryPerformanceCounter(&counter); + t = counter.QuadPart; +#elif defined(__MACH__) + t = mach_absolute_time(); +#elif defined(__HAIKU__) + t = system_time_nsecs(); // since boot +#else // POSIX + timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + t = static_cast<Ticks>(ts.tv_sec * 1000000000LL + ts.tv_nsec); +#endif + return t; +} + +inline Ticks Stop() { + uint64_t t; +#if HWY_ARCH_PPC && defined(__GLIBC__) + asm volatile("mfspr %0, %1" : "=r"(t) : "i"(268)); +#elif HWY_ARCH_X86 && HWY_COMPILER_MSVC + _ReadWriteBarrier(); + unsigned aux; + t = __rdtscp(&aux); + _ReadWriteBarrier(); + _mm_lfence(); + _ReadWriteBarrier(); +#elif HWY_ARCH_X86_64 + // Use inline asm because __rdtscp generates code to store TSC_AUX (ecx). + asm volatile( + "rdtscp\n\t" + "shl $32, %%rdx\n\t" + "or %%rdx, %0\n\t" + "lfence" + : "=a"(t) + : + // "memory" avoids reordering. rcx = TSC_AUX. rdx = TSC >> 32. + // "cc" = flags modified by SHL. + : "rcx", "rdx", "memory", "cc"); +#else + t = Start(); +#endif + return t; +} + +} // namespace timer + +namespace robust_statistics { + +// Sorts integral values in ascending order (e.g. for Mode). About 3x faster +// than std::sort for input distributions with very few unique values. +template <class T> +void CountingSort(T* values, size_t num_values) { + // Unique values and their frequency (similar to flat_map). + using Unique = std::pair<T, int>; + std::vector<Unique> unique; + for (size_t i = 0; i < num_values; ++i) { + const T value = values[i]; + const auto pos = + std::find_if(unique.begin(), unique.end(), + [value](const Unique u) { return u.first == value; }); + if (pos == unique.end()) { + unique.push_back(std::make_pair(value, 1)); + } else { + ++pos->second; + } + } + + // Sort in ascending order of value (pair.first). + std::sort(unique.begin(), unique.end()); + + // Write that many copies of each unique value to the array. + T* HWY_RESTRICT p = values; + for (const auto& value_count : unique) { + std::fill(p, p + value_count.second, value_count.first); + p += value_count.second; + } + NANOBENCHMARK_CHECK(p == values + num_values); +} + +// @return i in [idx_begin, idx_begin + half_count) that minimizes +// sorted[i + half_count] - sorted[i]. +template <typename T> +size_t MinRange(const T* const HWY_RESTRICT sorted, const size_t idx_begin, + const size_t half_count) { + T min_range = std::numeric_limits<T>::max(); + size_t min_idx = 0; + + for (size_t idx = idx_begin; idx < idx_begin + half_count; ++idx) { + NANOBENCHMARK_CHECK(sorted[idx] <= sorted[idx + half_count]); + const T range = sorted[idx + half_count] - sorted[idx]; + if (range < min_range) { + min_range = range; + min_idx = idx; + } + } + + return min_idx; +} + +// Returns an estimate of the mode by calling MinRange on successively +// halved intervals. "sorted" must be in ascending order. This is the +// Half Sample Mode estimator proposed by Bickel in "On a fast, robust +// estimator of the mode", with complexity O(N log N). The mode is less +// affected by outliers in highly-skewed distributions than the median. +// The averaging operation below assumes "T" is an unsigned integer type. +template <typename T> +T ModeOfSorted(const T* const HWY_RESTRICT sorted, const size_t num_values) { + size_t idx_begin = 0; + size_t half_count = num_values / 2; + while (half_count > 1) { + idx_begin = MinRange(sorted, idx_begin, half_count); + half_count >>= 1; + } + + const T x = sorted[idx_begin + 0]; + if (half_count == 0) { + return x; + } + NANOBENCHMARK_CHECK(half_count == 1); + const T average = (x + sorted[idx_begin + 1] + 1) / 2; + return average; +} + +// Returns the mode. Side effect: sorts "values". +template <typename T> +T Mode(T* values, const size_t num_values) { + CountingSort(values, num_values); + return ModeOfSorted(values, num_values); +} + +template <typename T, size_t N> +T Mode(T (&values)[N]) { + return Mode(&values[0], N); +} + +// Returns the median value. Side effect: sorts "values". +template <typename T> +T Median(T* values, const size_t num_values) { + NANOBENCHMARK_CHECK(!values->empty()); + std::sort(values, values + num_values); + const size_t half = num_values / 2; + // Odd count: return middle + if (num_values % 2) { + return values[half]; + } + // Even count: return average of middle two. + return (values[half] + values[half - 1] + 1) / 2; +} + +// Returns a robust measure of variability. +template <typename T> +T MedianAbsoluteDeviation(const T* values, const size_t num_values, + const T median) { + NANOBENCHMARK_CHECK(num_values != 0); + std::vector<T> abs_deviations; + abs_deviations.reserve(num_values); + for (size_t i = 0; i < num_values; ++i) { + const int64_t abs = std::abs(int64_t(values[i]) - int64_t(median)); + abs_deviations.push_back(static_cast<T>(abs)); + } + return Median(abs_deviations.data(), num_values); +} + +} // namespace robust_statistics +} // namespace +namespace platform { +namespace { + +// Prevents the compiler from eliding the computations that led to "output". +template <class T> +inline void PreventElision(T&& output) { +#if HWY_COMPILER_MSVC == 0 + // Works by indicating to the compiler that "output" is being read and + // modified. The +r constraint avoids unnecessary writes to memory, but only + // works for built-in types (typically FuncOutput). + asm volatile("" : "+r"(output) : : "memory"); +#else + // MSVC does not support inline assembly anymore (and never supported GCC's + // RTL constraints). Self-assignment with #pragma optimize("off") might be + // expected to prevent elision, but it does not with MSVC 2015. Type-punning + // with volatile pointers generates inefficient code on MSVC 2017. + static std::atomic<T> dummy(T{}); + dummy.store(output, std::memory_order_relaxed); +#endif +} + +#if HWY_ARCH_X86 + +void Cpuid(const uint32_t level, const uint32_t count, + uint32_t* HWY_RESTRICT abcd) { +#if HWY_COMPILER_MSVC + int regs[4]; + __cpuidex(regs, level, count); + for (int i = 0; i < 4; ++i) { + abcd[i] = regs[i]; + } +#else + uint32_t a; + uint32_t b; + uint32_t c; + uint32_t d; + __cpuid_count(level, count, a, b, c, d); + abcd[0] = a; + abcd[1] = b; + abcd[2] = c; + abcd[3] = d; +#endif +} + +bool HasRDTSCP() { + uint32_t abcd[4]; + Cpuid(0x80000001U, 0, abcd); // Extended feature flags + return (abcd[3] & (1u << 27)) != 0; // RDTSCP +} + +std::string BrandString() { + char brand_string[49]; + std::array<uint32_t, 4> abcd; + + // Check if brand string is supported (it is on all reasonable Intel/AMD) + Cpuid(0x80000000U, 0, abcd.data()); + if (abcd[0] < 0x80000004U) { + return std::string(); + } + + for (size_t i = 0; i < 3; ++i) { + Cpuid(static_cast<uint32_t>(0x80000002U + i), 0, abcd.data()); + memcpy(brand_string + i * 16, abcd.data(), sizeof(abcd)); + } + brand_string[48] = 0; + return brand_string; +} + +// Returns the frequency quoted inside the brand string. This does not +// account for throttling nor Turbo Boost. +double NominalClockRate() { + const std::string& brand_string = BrandString(); + // Brand strings include the maximum configured frequency. These prefixes are + // defined by Intel CPUID documentation. + const char* prefixes[3] = {"MHz", "GHz", "THz"}; + const double multipliers[3] = {1E6, 1E9, 1E12}; + for (size_t i = 0; i < 3; ++i) { + const size_t pos_prefix = brand_string.find(prefixes[i]); + if (pos_prefix != std::string::npos) { + const size_t pos_space = brand_string.rfind(' ', pos_prefix - 1); + if (pos_space != std::string::npos) { + const std::string digits = + brand_string.substr(pos_space + 1, pos_prefix - pos_space - 1); + return std::stod(digits) * multipliers[i]; + } + } + } + + return 0.0; +} + +#endif // HWY_ARCH_X86 + +} // namespace + +double InvariantTicksPerSecond() { +#if HWY_ARCH_PPC && defined(__GLIBC__) + return double(__ppc_get_timebase_freq()); +#elif HWY_ARCH_X86 + // We assume the TSC is invariant; it is on all recent Intel/AMD CPUs. + return NominalClockRate(); +#elif defined(_WIN32) || defined(_WIN64) + LARGE_INTEGER freq; + (void)QueryPerformanceFrequency(&freq); + return double(freq.QuadPart); +#elif defined(__MACH__) + // https://developer.apple.com/library/mac/qa/qa1398/_index.html + mach_timebase_info_data_t timebase; + (void)mach_timebase_info(&timebase); + return double(timebase.denom) / timebase.numer * 1E9; +#else + // TODO(janwas): ARM? Unclear how to reliably query cntvct_el0 frequency. + return 1E9; // Haiku and clock_gettime return nanoseconds. +#endif +} + +double Now() { + static const double mul = 1.0 / InvariantTicksPerSecond(); + return static_cast<double>(timer::Start()) * mul; +} + +uint64_t TimerResolution() { + // Nested loop avoids exceeding stack/L1 capacity. + timer::Ticks repetitions[Params::kTimerSamples]; + for (size_t rep = 0; rep < Params::kTimerSamples; ++rep) { + timer::Ticks samples[Params::kTimerSamples]; + for (size_t i = 0; i < Params::kTimerSamples; ++i) { + const timer::Ticks t0 = timer::Start(); + const timer::Ticks t1 = timer::Stop(); + samples[i] = t1 - t0; + } + repetitions[rep] = robust_statistics::Mode(samples); + } + return robust_statistics::Mode(repetitions); +} + +} // namespace platform +namespace { + +static const timer::Ticks timer_resolution = platform::TimerResolution(); + +// Estimates the expected value of "lambda" values with a variable number of +// samples until the variability "rel_mad" is less than "max_rel_mad". +template <class Lambda> +timer::Ticks SampleUntilStable(const double max_rel_mad, double* rel_mad, + const Params& p, const Lambda& lambda) { + // Choose initial samples_per_eval based on a single estimated duration. + timer::Ticks t0 = timer::Start(); + lambda(); + timer::Ticks t1 = timer::Stop(); + timer::Ticks est = t1 - t0; + static const double ticks_per_second = platform::InvariantTicksPerSecond(); + const size_t ticks_per_eval = + static_cast<size_t>(ticks_per_second * p.seconds_per_eval); + size_t samples_per_eval = est == 0 + ? p.min_samples_per_eval + : static_cast<size_t>(ticks_per_eval / est); + samples_per_eval = HWY_MAX(samples_per_eval, p.min_samples_per_eval); + + std::vector<timer::Ticks> samples; + samples.reserve(1 + samples_per_eval); + samples.push_back(est); + + // Percentage is too strict for tiny differences, so also allow a small + // absolute "median absolute deviation". + const timer::Ticks max_abs_mad = (timer_resolution + 99) / 100; + *rel_mad = 0.0; // ensure initialized + + for (size_t eval = 0; eval < p.max_evals; ++eval, samples_per_eval *= 2) { + samples.reserve(samples.size() + samples_per_eval); + for (size_t i = 0; i < samples_per_eval; ++i) { + t0 = timer::Start(); + lambda(); + t1 = timer::Stop(); + samples.push_back(t1 - t0); + } + + if (samples.size() >= p.min_mode_samples) { + est = robust_statistics::Mode(samples.data(), samples.size()); + } else { + // For "few" (depends also on the variance) samples, Median is safer. + est = robust_statistics::Median(samples.data(), samples.size()); + } + NANOBENCHMARK_CHECK(est != 0); + + // Median absolute deviation (mad) is a robust measure of 'variability'. + const timer::Ticks abs_mad = robust_statistics::MedianAbsoluteDeviation( + samples.data(), samples.size(), est); + *rel_mad = static_cast<double>(abs_mad) / static_cast<double>(est); + + if (*rel_mad <= max_rel_mad || abs_mad <= max_abs_mad) { + if (p.verbose) { + printf("%6" PRIu64 " samples => %5" PRIu64 " (abs_mad=%4" PRIu64 + ", rel_mad=%4.2f%%)\n", + static_cast<uint64_t>(samples.size()), + static_cast<uint64_t>(est), static_cast<uint64_t>(abs_mad), + *rel_mad * 100.0); + } + return est; + } + } + + if (p.verbose) { + printf("WARNING: rel_mad=%4.2f%% still exceeds %4.2f%% after %6" PRIu64 + " samples.\n", + *rel_mad * 100.0, max_rel_mad * 100.0, + static_cast<uint64_t>(samples.size())); + } + return est; +} + +using InputVec = std::vector<FuncInput>; + +// Returns vector of unique input values. +InputVec UniqueInputs(const FuncInput* inputs, const size_t num_inputs) { + InputVec unique(inputs, inputs + num_inputs); + std::sort(unique.begin(), unique.end()); + unique.erase(std::unique(unique.begin(), unique.end()), unique.end()); + return unique; +} + +// Returns how often we need to call func for sufficient precision. +size_t NumSkip(const Func func, const uint8_t* arg, const InputVec& unique, + const Params& p) { + // Min elapsed ticks for any input. + timer::Ticks min_duration = ~timer::Ticks(0); + + for (const FuncInput input : unique) { + double rel_mad; + const timer::Ticks total = SampleUntilStable( + p.target_rel_mad, &rel_mad, p, + [func, arg, input]() { platform::PreventElision(func(arg, input)); }); + min_duration = HWY_MIN(min_duration, total - timer_resolution); + } + + // Number of repetitions required to reach the target resolution. + const size_t max_skip = p.precision_divisor; + // Number of repetitions given the estimated duration. + const size_t num_skip = + min_duration == 0 + ? 0 + : static_cast<size_t>((max_skip + min_duration - 1) / min_duration); + if (p.verbose) { + printf("res=%" PRIu64 " max_skip=%" PRIu64 " min_dur=%" PRIu64 + " num_skip=%" PRIu64 "\n", + static_cast<uint64_t>(timer_resolution), + static_cast<uint64_t>(max_skip), static_cast<uint64_t>(min_duration), + static_cast<uint64_t>(num_skip)); + } + return num_skip; +} + +// Replicates inputs until we can omit "num_skip" occurrences of an input. +InputVec ReplicateInputs(const FuncInput* inputs, const size_t num_inputs, + const size_t num_unique, const size_t num_skip, + const Params& p) { + InputVec full; + if (num_unique == 1) { + full.assign(p.subset_ratio * num_skip, inputs[0]); + return full; + } + + full.reserve(p.subset_ratio * num_skip * num_inputs); + for (size_t i = 0; i < p.subset_ratio * num_skip; ++i) { + full.insert(full.end(), inputs, inputs + num_inputs); + } + std::mt19937 rng; + std::shuffle(full.begin(), full.end(), rng); + return full; +} + +// Copies the "full" to "subset" in the same order, but with "num_skip" +// randomly selected occurrences of "input_to_skip" removed. +void FillSubset(const InputVec& full, const FuncInput input_to_skip, + const size_t num_skip, InputVec* subset) { + const size_t count = + static_cast<size_t>(std::count(full.begin(), full.end(), input_to_skip)); + // Generate num_skip random indices: which occurrence to skip. + std::vector<uint32_t> omit(count); + std::iota(omit.begin(), omit.end(), 0); + // omit[] is the same on every call, but that's OK because they identify the + // Nth instance of input_to_skip, so the position within full[] differs. + std::mt19937 rng; + std::shuffle(omit.begin(), omit.end(), rng); + omit.resize(num_skip); + std::sort(omit.begin(), omit.end()); + + uint32_t occurrence = ~0u; // 0 after preincrement + size_t idx_omit = 0; // cursor within omit[] + size_t idx_subset = 0; // cursor within *subset + for (const FuncInput next : full) { + if (next == input_to_skip) { + ++occurrence; + // Haven't removed enough already + if (idx_omit < num_skip) { + // This one is up for removal + if (occurrence == omit[idx_omit]) { + ++idx_omit; + continue; + } + } + } + if (idx_subset < subset->size()) { + (*subset)[idx_subset++] = next; + } + } + NANOBENCHMARK_CHECK(idx_subset == subset->size()); + NANOBENCHMARK_CHECK(idx_omit == omit.size()); + NANOBENCHMARK_CHECK(occurrence == count - 1); +} + +// Returns total ticks elapsed for all inputs. +timer::Ticks TotalDuration(const Func func, const uint8_t* arg, + const InputVec* inputs, const Params& p, + double* max_rel_mad) { + double rel_mad; + const timer::Ticks duration = + SampleUntilStable(p.target_rel_mad, &rel_mad, p, [func, arg, inputs]() { + for (const FuncInput input : *inputs) { + platform::PreventElision(func(arg, input)); + } + }); + *max_rel_mad = HWY_MAX(*max_rel_mad, rel_mad); + return duration; +} + +// (Nearly) empty Func for measuring timer overhead/resolution. +HWY_NOINLINE FuncOutput EmptyFunc(const void* /*arg*/, const FuncInput input) { + return input; +} + +// Returns overhead of accessing inputs[] and calling a function; this will +// be deducted from future TotalDuration return values. +timer::Ticks Overhead(const uint8_t* arg, const InputVec* inputs, + const Params& p) { + double rel_mad; + // Zero tolerance because repeatability is crucial and EmptyFunc is fast. + return SampleUntilStable(0.0, &rel_mad, p, [arg, inputs]() { + for (const FuncInput input : *inputs) { + platform::PreventElision(EmptyFunc(arg, input)); + } + }); +} + +} // namespace + +int Unpredictable1() { return timer::Start() != ~0ULL; } + +size_t Measure(const Func func, const uint8_t* arg, const FuncInput* inputs, + const size_t num_inputs, Result* results, const Params& p) { + NANOBENCHMARK_CHECK(num_inputs != 0); + +#if HWY_ARCH_X86 + if (!platform::HasRDTSCP()) { + fprintf(stderr, "CPU '%s' does not support RDTSCP, skipping benchmark.\n", + platform::BrandString().c_str()); + return 0; + } +#endif + + const InputVec& unique = UniqueInputs(inputs, num_inputs); + + const size_t num_skip = NumSkip(func, arg, unique, p); // never 0 + if (num_skip == 0) return 0; // NumSkip already printed error message + // (slightly less work on x86 to cast from signed integer) + const float mul = 1.0f / static_cast<float>(static_cast<int>(num_skip)); + + const InputVec& full = + ReplicateInputs(inputs, num_inputs, unique.size(), num_skip, p); + InputVec subset(full.size() - num_skip); + + const timer::Ticks overhead = Overhead(arg, &full, p); + const timer::Ticks overhead_skip = Overhead(arg, &subset, p); + if (overhead < overhead_skip) { + fprintf(stderr, "Measurement failed: overhead %" PRIu64 " < %" PRIu64 "\n", + static_cast<uint64_t>(overhead), + static_cast<uint64_t>(overhead_skip)); + return 0; + } + + if (p.verbose) { + printf("#inputs=%5" PRIu64 ",%5" PRIu64 " overhead=%5" PRIu64 ",%5" PRIu64 + "\n", + static_cast<uint64_t>(full.size()), + static_cast<uint64_t>(subset.size()), + static_cast<uint64_t>(overhead), + static_cast<uint64_t>(overhead_skip)); + } + + double max_rel_mad = 0.0; + const timer::Ticks total = TotalDuration(func, arg, &full, p, &max_rel_mad); + + for (size_t i = 0; i < unique.size(); ++i) { + FillSubset(full, unique[i], num_skip, &subset); + const timer::Ticks total_skip = + TotalDuration(func, arg, &subset, p, &max_rel_mad); + + if (total < total_skip) { + fprintf(stderr, "Measurement failed: total %" PRIu64 " < %" PRIu64 "\n", + static_cast<uint64_t>(total), static_cast<uint64_t>(total_skip)); + return 0; + } + + const timer::Ticks duration = + (total - overhead) - (total_skip - overhead_skip); + results[i].input = unique[i]; + results[i].ticks = static_cast<float>(duration) * mul; + results[i].variability = static_cast<float>(max_rel_mad); + } + + return unique.size(); +} + +} // namespace hwy diff --git a/third_party/highway/hwy/nanobenchmark.h b/third_party/highway/hwy/nanobenchmark.h new file mode 100644 index 0000000..18065f8 --- /dev/null +++ b/third_party/highway/hwy/nanobenchmark.h @@ -0,0 +1,191 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef HIGHWAY_HWY_NANOBENCHMARK_H_ +#define HIGHWAY_HWY_NANOBENCHMARK_H_ + +// Benchmarks functions of a single integer argument with realistic branch +// prediction hit rates. Uses a robust estimator to summarize the measurements. +// The precision is about 0.2%. +// +// Examples: see nanobenchmark_test.cc. +// +// Background: Microbenchmarks such as http://github.com/google/benchmark +// can measure elapsed times on the order of a microsecond. Shorter functions +// are typically measured by repeating them thousands of times and dividing +// the total elapsed time by this count. Unfortunately, repetition (especially +// with the same input parameter!) influences the runtime. In time-critical +// code, it is reasonable to expect warm instruction/data caches and TLBs, +// but a perfect record of which branches will be taken is unrealistic. +// Unless the application also repeatedly invokes the measured function with +// the same parameter, the benchmark is measuring something very different - +// a best-case result, almost as if the parameter were made a compile-time +// constant. This may lead to erroneous conclusions about branch-heavy +// algorithms outperforming branch-free alternatives. +// +// Our approach differs in three ways. Adding fences to the timer functions +// reduces variability due to instruction reordering, improving the timer +// resolution to about 40 CPU cycles. However, shorter functions must still +// be invoked repeatedly. For more realistic branch prediction performance, +// we vary the input parameter according to a user-specified distribution. +// Thus, instead of VaryInputs(Measure(Repeat(func))), we change the +// loop nesting to Measure(Repeat(VaryInputs(func))). We also estimate the +// central tendency of the measurement samples with the "half sample mode", +// which is more robust to outliers and skewed data than the mean or median. + +#include <stddef.h> +#include <stdint.h> + +// Enables sanity checks that verify correct operation at the cost of +// longer benchmark runs. +#ifndef NANOBENCHMARK_ENABLE_CHECKS +#define NANOBENCHMARK_ENABLE_CHECKS 0 +#endif + +#define NANOBENCHMARK_CHECK_ALWAYS(condition) \ + while (!(condition)) { \ + fprintf(stderr, "Nanobenchmark check failed at line %d\n", __LINE__); \ + abort(); \ + } + +#if NANOBENCHMARK_ENABLE_CHECKS +#define NANOBENCHMARK_CHECK(condition) NANOBENCHMARK_CHECK_ALWAYS(condition) +#else +#define NANOBENCHMARK_CHECK(condition) +#endif + +namespace hwy { + +namespace platform { + +// Returns tick rate, useful for converting measurements to seconds. Invariant +// means the tick counter frequency is independent of CPU throttling or sleep. +// This call may be expensive, callers should cache the result. +double InvariantTicksPerSecond(); + +// Returns current timestamp [in seconds] relative to an unspecified origin. +// Features: monotonic (no negative elapsed time), steady (unaffected by system +// time changes), high-resolution (on the order of microseconds). +double Now(); + +// Returns ticks elapsed in back to back timer calls, i.e. a function of the +// timer resolution (minimum measurable difference) and overhead. +// This call is expensive, callers should cache the result. +uint64_t TimerResolution(); + +} // namespace platform + +// Returns 1, but without the compiler knowing what the value is. This prevents +// optimizing out code. +int Unpredictable1(); + +// Input influencing the function being measured (e.g. number of bytes to copy). +using FuncInput = size_t; + +// "Proof of work" returned by Func to ensure the compiler does not elide it. +using FuncOutput = uint64_t; + +// Function to measure: either 1) a captureless lambda or function with two +// arguments or 2) a lambda with capture, in which case the first argument +// is reserved for use by MeasureClosure. +using Func = FuncOutput (*)(const void*, FuncInput); + +// Internal parameters that determine precision/resolution/measuring time. +struct Params { + // For measuring timer overhead/resolution. Used in a nested loop => + // quadratic time, acceptable because we know timer overhead is "low". + // constexpr because this is used to define array bounds. + static constexpr size_t kTimerSamples = 256; + + // Best-case precision, expressed as a divisor of the timer resolution. + // Larger => more calls to Func and higher precision. + size_t precision_divisor = 1024; + + // Ratio between full and subset input distribution sizes. Cannot be less + // than 2; larger values increase measurement time but more faithfully + // model the given input distribution. + size_t subset_ratio = 2; + + // Together with the estimated Func duration, determines how many times to + // call Func before checking the sample variability. Larger values increase + // measurement time, memory/cache use and precision. + double seconds_per_eval = 4E-3; + + // The minimum number of samples before estimating the central tendency. + size_t min_samples_per_eval = 7; + + // The mode is better than median for estimating the central tendency of + // skewed/fat-tailed distributions, but it requires sufficient samples + // relative to the width of half-ranges. + size_t min_mode_samples = 64; + + // Maximum permissible variability (= median absolute deviation / center). + double target_rel_mad = 0.002; + + // Abort after this many evals without reaching target_rel_mad. This + // prevents infinite loops. + size_t max_evals = 9; + + // Whether to print additional statistics to stdout. + bool verbose = true; +}; + +// Measurement result for each unique input. +struct Result { + FuncInput input; + + // Robust estimate (mode or median) of duration. + float ticks; + + // Measure of variability (median absolute deviation relative to "ticks"). + float variability; +}; + +// Precisely measures the number of ticks elapsed when calling "func" with the +// given inputs, shuffled to ensure realistic branch prediction hit rates. +// +// "func" returns a 'proof of work' to ensure its computations are not elided. +// "arg" is passed to Func, or reserved for internal use by MeasureClosure. +// "inputs" is an array of "num_inputs" (not necessarily unique) arguments to +// "func". The values should be chosen to maximize coverage of "func". This +// represents a distribution, so a value's frequency should reflect its +// probability in the real application. Order does not matter; for example, a +// uniform distribution over [0, 4) could be represented as {3,0,2,1}. +// Returns how many Result were written to "results": one per unique input, or +// zero if the measurement failed (an error message goes to stderr). +size_t Measure(const Func func, const uint8_t* arg, const FuncInput* inputs, + const size_t num_inputs, Result* results, + const Params& p = Params()); + +// Calls operator() of the given closure (lambda function). +template <class Closure> +static FuncOutput CallClosure(const Closure* f, const FuncInput input) { + return (*f)(input); +} + +// Same as Measure, except "closure" is typically a lambda function of +// FuncInput -> FuncOutput with a capture list. +template <class Closure> +static inline size_t MeasureClosure(const Closure& closure, + const FuncInput* inputs, + const size_t num_inputs, Result* results, + const Params& p = Params()) { + return Measure(reinterpret_cast<Func>(&CallClosure<Closure>), + reinterpret_cast<const uint8_t*>(&closure), inputs, num_inputs, + results, p); +} + +} // namespace hwy + +#endif // HIGHWAY_HWY_NANOBENCHMARK_H_ diff --git a/third_party/highway/hwy/nanobenchmark_test.cc b/third_party/highway/hwy/nanobenchmark_test.cc new file mode 100644 index 0000000..e09f40c --- /dev/null +++ b/third_party/highway/hwy/nanobenchmark_test.cc @@ -0,0 +1,96 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "hwy/nanobenchmark.h" + +#include <inttypes.h> +#include <stdint.h> +#include <stdio.h> + +#include <random> + +#include "hwy/tests/test_util-inl.h" + +namespace hwy { +namespace { + +// Governs duration of test; avoid timeout in debug builds. +#if HWY_IS_DEBUG_BUILD +constexpr size_t kMaxEvals = 3; +#else +constexpr size_t kMaxEvals = 4; +#endif + +FuncOutput Div(const void*, FuncInput in) { + // Here we're measuring the throughput because benchmark invocations are + // independent. Any dividend will do; the divisor is nonzero. + return 0xFFFFF / in; +} + +template <size_t N> +void MeasureDiv(const FuncInput (&inputs)[N]) { + printf("Measuring integer division (output on final two lines)\n"); + Result results[N]; + Params params; + params.max_evals = kMaxEvals; + const size_t num_results = Measure(&Div, nullptr, inputs, N, results, params); + for (size_t i = 0; i < num_results; ++i) { + printf("%5" PRIu64 ": %6.2f ticks; MAD=%4.2f%%\n", + static_cast<uint64_t>(results[i].input), results[i].ticks, + results[i].variability * 100.0); + } +} + +std::mt19937 rng; + +// A function whose runtime depends on rng. +FuncOutput Random(const void* /*arg*/, FuncInput in) { + const size_t r = rng() & 0xF; + uint32_t ret = in; + for (size_t i = 0; i < r; ++i) { + ret /= ((rng() & 1) + 2); + } + return ret; +} + +// Ensure the measured variability is high. +template <size_t N> +void MeasureRandom(const FuncInput (&inputs)[N]) { + Result results[N]; + Params p; + p.max_evals = kMaxEvals; + p.verbose = false; + const size_t num_results = Measure(&Random, nullptr, inputs, N, results, p); + for (size_t i = 0; i < num_results; ++i) { + NANOBENCHMARK_CHECK(results[i].variability > 1E-3); + } +} + +TEST(NanobenchmarkTest, RunAll) { + const int unpredictable = Unpredictable1(); // == 1, unknown to compiler. + static const FuncInput inputs[] = {static_cast<FuncInput>(unpredictable) + 2, + static_cast<FuncInput>(unpredictable + 9)}; + + MeasureDiv(inputs); + MeasureRandom(inputs); +} + +} // namespace +} // namespace hwy + +// Ought not to be necessary, but without this, no tests run on RVV. +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/third_party/highway/hwy/ops/arm_neon-inl.h b/third_party/highway/hwy/ops/arm_neon-inl.h new file mode 100644 index 0000000..774ca5d --- /dev/null +++ b/third_party/highway/hwy/ops/arm_neon-inl.h @@ -0,0 +1,5244 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// 128-bit ARM64 NEON vectors and operations. +// External include guard in highway.h - see comment there. + +#include <arm_neon.h> +#include <stddef.h> +#include <stdint.h> + +#include "hwy/base.h" +#include "hwy/ops/shared-inl.h" + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +template <typename T> +using Full128 = Simd<T, 16 / sizeof(T)>; + +namespace detail { // for code folding and Raw128 + +// Macros used to define single and double function calls for multiple types +// for full and half vectors. These macros are undefined at the end of the file. + +// HWY_NEON_BUILD_TPL_* is the template<...> prefix to the function. +#define HWY_NEON_BUILD_TPL_1 +#define HWY_NEON_BUILD_TPL_2 +#define HWY_NEON_BUILD_TPL_3 + +// HWY_NEON_BUILD_RET_* is return type. +#define HWY_NEON_BUILD_RET_1(type, size) Vec128<type, size> +#define HWY_NEON_BUILD_RET_2(type, size) Vec128<type, size> +#define HWY_NEON_BUILD_RET_3(type, size) Vec128<type, size> + +// HWY_NEON_BUILD_PARAM_* is the list of parameters the function receives. +#define HWY_NEON_BUILD_PARAM_1(type, size) const Vec128<type, size> a +#define HWY_NEON_BUILD_PARAM_2(type, size) \ + const Vec128<type, size> a, const Vec128<type, size> b +#define HWY_NEON_BUILD_PARAM_3(type, size) \ + const Vec128<type, size> a, const Vec128<type, size> b, \ + const Vec128<type, size> c + +// HWY_NEON_BUILD_ARG_* is the list of arguments passed to the underlying +// function. +#define HWY_NEON_BUILD_ARG_1 a.raw +#define HWY_NEON_BUILD_ARG_2 a.raw, b.raw +#define HWY_NEON_BUILD_ARG_3 a.raw, b.raw, c.raw + +// We use HWY_NEON_EVAL(func, ...) to delay the evaluation of func until after +// the __VA_ARGS__ have been expanded. This allows "func" to be a macro on +// itself like with some of the library "functions" such as vshlq_u8. For +// example, HWY_NEON_EVAL(vshlq_u8, MY_PARAMS) where MY_PARAMS is defined as +// "a, b" (without the quotes) will end up expanding "vshlq_u8(a, b)" if needed. +// Directly writing vshlq_u8(MY_PARAMS) would fail since vshlq_u8() macro +// expects two arguments. +#define HWY_NEON_EVAL(func, ...) func(__VA_ARGS__) + +// Main macro definition that defines a single function for the given type and +// size of vector, using the underlying (prefix##infix##suffix) function and +// the template, return type, parameters and arguments defined by the "args" +// parameters passed here (see HWY_NEON_BUILD_* macros defined before). +#define HWY_NEON_DEF_FUNCTION(type, size, name, prefix, infix, suffix, args) \ + HWY_CONCAT(HWY_NEON_BUILD_TPL_, args) \ + HWY_API HWY_CONCAT(HWY_NEON_BUILD_RET_, args)(type, size) \ + name(HWY_CONCAT(HWY_NEON_BUILD_PARAM_, args)(type, size)) { \ + return HWY_CONCAT(HWY_NEON_BUILD_RET_, args)(type, size)( \ + HWY_NEON_EVAL(prefix##infix##suffix, HWY_NEON_BUILD_ARG_##args)); \ + } + +// The HWY_NEON_DEF_FUNCTION_* macros define all the variants of a function +// called "name" using the set of neon functions starting with the given +// "prefix" for all the variants of certain types, as specified next to each +// macro. For example, the prefix "vsub" can be used to define the operator- +// using args=2. + +// uint8_t +#define HWY_NEON_DEF_FUNCTION_UINT_8(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION(uint8_t, 16, name, prefix##q, infix, u8, args) \ + HWY_NEON_DEF_FUNCTION(uint8_t, 8, name, prefix, infix, u8, args) \ + HWY_NEON_DEF_FUNCTION(uint8_t, 4, name, prefix, infix, u8, args) \ + HWY_NEON_DEF_FUNCTION(uint8_t, 2, name, prefix, infix, u8, args) \ + HWY_NEON_DEF_FUNCTION(uint8_t, 1, name, prefix, infix, u8, args) + +// int8_t +#define HWY_NEON_DEF_FUNCTION_INT_8(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION(int8_t, 16, name, prefix##q, infix, s8, args) \ + HWY_NEON_DEF_FUNCTION(int8_t, 8, name, prefix, infix, s8, args) \ + HWY_NEON_DEF_FUNCTION(int8_t, 4, name, prefix, infix, s8, args) \ + HWY_NEON_DEF_FUNCTION(int8_t, 2, name, prefix, infix, s8, args) \ + HWY_NEON_DEF_FUNCTION(int8_t, 1, name, prefix, infix, s8, args) + +// uint16_t +#define HWY_NEON_DEF_FUNCTION_UINT_16(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION(uint16_t, 8, name, prefix##q, infix, u16, args) \ + HWY_NEON_DEF_FUNCTION(uint16_t, 4, name, prefix, infix, u16, args) \ + HWY_NEON_DEF_FUNCTION(uint16_t, 2, name, prefix, infix, u16, args) \ + HWY_NEON_DEF_FUNCTION(uint16_t, 1, name, prefix, infix, u16, args) + +// int16_t +#define HWY_NEON_DEF_FUNCTION_INT_16(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION(int16_t, 8, name, prefix##q, infix, s16, args) \ + HWY_NEON_DEF_FUNCTION(int16_t, 4, name, prefix, infix, s16, args) \ + HWY_NEON_DEF_FUNCTION(int16_t, 2, name, prefix, infix, s16, args) \ + HWY_NEON_DEF_FUNCTION(int16_t, 1, name, prefix, infix, s16, args) + +// uint32_t +#define HWY_NEON_DEF_FUNCTION_UINT_32(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION(uint32_t, 4, name, prefix##q, infix, u32, args) \ + HWY_NEON_DEF_FUNCTION(uint32_t, 2, name, prefix, infix, u32, args) \ + HWY_NEON_DEF_FUNCTION(uint32_t, 1, name, prefix, infix, u32, args) + +// int32_t +#define HWY_NEON_DEF_FUNCTION_INT_32(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION(int32_t, 4, name, prefix##q, infix, s32, args) \ + HWY_NEON_DEF_FUNCTION(int32_t, 2, name, prefix, infix, s32, args) \ + HWY_NEON_DEF_FUNCTION(int32_t, 1, name, prefix, infix, s32, args) + +// uint64_t +#define HWY_NEON_DEF_FUNCTION_UINT_64(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION(uint64_t, 2, name, prefix##q, infix, u64, args) \ + HWY_NEON_DEF_FUNCTION(uint64_t, 1, name, prefix, infix, u64, args) + +// int64_t +#define HWY_NEON_DEF_FUNCTION_INT_64(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION(int64_t, 2, name, prefix##q, infix, s64, args) \ + HWY_NEON_DEF_FUNCTION(int64_t, 1, name, prefix, infix, s64, args) + +// float and double +#if HWY_ARCH_ARM_A64 +#define HWY_NEON_DEF_FUNCTION_ALL_FLOATS(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION(float, 4, name, prefix##q, infix, f32, args) \ + HWY_NEON_DEF_FUNCTION(float, 2, name, prefix, infix, f32, args) \ + HWY_NEON_DEF_FUNCTION(float, 1, name, prefix, infix, f32, args) \ + HWY_NEON_DEF_FUNCTION(double, 2, name, prefix##q, infix, f64, args) \ + HWY_NEON_DEF_FUNCTION(double, 1, name, prefix, infix, f64, args) +#else +#define HWY_NEON_DEF_FUNCTION_ALL_FLOATS(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION(float, 4, name, prefix##q, infix, f32, args) \ + HWY_NEON_DEF_FUNCTION(float, 2, name, prefix, infix, f32, args) \ + HWY_NEON_DEF_FUNCTION(float, 1, name, prefix, infix, f32, args) +#endif + +// Helper macros to define for more than one type. +// uint8_t, uint16_t and uint32_t +#define HWY_NEON_DEF_FUNCTION_UINT_8_16_32(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION_UINT_8(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION_UINT_16(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION_UINT_32(name, prefix, infix, args) + +// int8_t, int16_t and int32_t +#define HWY_NEON_DEF_FUNCTION_INT_8_16_32(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION_INT_8(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION_INT_16(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION_INT_32(name, prefix, infix, args) + +// uint8_t, uint16_t, uint32_t and uint64_t +#define HWY_NEON_DEF_FUNCTION_UINTS(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION_UINT_8_16_32(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION_UINT_64(name, prefix, infix, args) + +// int8_t, int16_t, int32_t and int64_t +#define HWY_NEON_DEF_FUNCTION_INTS(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION_INT_8_16_32(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION_INT_64(name, prefix, infix, args) + +// All int*_t and uint*_t up to 64 +#define HWY_NEON_DEF_FUNCTION_INTS_UINTS(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION_INTS(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION_UINTS(name, prefix, infix, args) + +// All previous types. +#define HWY_NEON_DEF_FUNCTION_ALL_TYPES(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION_INTS_UINTS(name, prefix, infix, args) \ + HWY_NEON_DEF_FUNCTION_ALL_FLOATS(name, prefix, infix, args) + +// Emulation of some intrinsics on armv7. +#if HWY_ARCH_ARM_V7 +#define vuzp1_s8(x, y) vuzp_s8(x, y).val[0] +#define vuzp1_u8(x, y) vuzp_u8(x, y).val[0] +#define vuzp1_s16(x, y) vuzp_s16(x, y).val[0] +#define vuzp1_u16(x, y) vuzp_u16(x, y).val[0] +#define vuzp1_s32(x, y) vuzp_s32(x, y).val[0] +#define vuzp1_u32(x, y) vuzp_u32(x, y).val[0] +#define vuzp1_f32(x, y) vuzp_f32(x, y).val[0] +#define vuzp1q_s8(x, y) vuzpq_s8(x, y).val[0] +#define vuzp1q_u8(x, y) vuzpq_u8(x, y).val[0] +#define vuzp1q_s16(x, y) vuzpq_s16(x, y).val[0] +#define vuzp1q_u16(x, y) vuzpq_u16(x, y).val[0] +#define vuzp1q_s32(x, y) vuzpq_s32(x, y).val[0] +#define vuzp1q_u32(x, y) vuzpq_u32(x, y).val[0] +#define vuzp1q_f32(x, y) vuzpq_f32(x, y).val[0] +#define vuzp2_s8(x, y) vuzp_s8(x, y).val[1] +#define vuzp2_u8(x, y) vuzp_u8(x, y).val[1] +#define vuzp2_s16(x, y) vuzp_s16(x, y).val[1] +#define vuzp2_u16(x, y) vuzp_u16(x, y).val[1] +#define vuzp2_s32(x, y) vuzp_s32(x, y).val[1] +#define vuzp2_u32(x, y) vuzp_u32(x, y).val[1] +#define vuzp2_f32(x, y) vuzp_f32(x, y).val[1] +#define vuzp2q_s8(x, y) vuzpq_s8(x, y).val[1] +#define vuzp2q_u8(x, y) vuzpq_u8(x, y).val[1] +#define vuzp2q_s16(x, y) vuzpq_s16(x, y).val[1] +#define vuzp2q_u16(x, y) vuzpq_u16(x, y).val[1] +#define vuzp2q_s32(x, y) vuzpq_s32(x, y).val[1] +#define vuzp2q_u32(x, y) vuzpq_u32(x, y).val[1] +#define vuzp2q_f32(x, y) vuzpq_f32(x, y).val[1] +#define vzip1_s8(x, y) vzip_s8(x, y).val[0] +#define vzip1_u8(x, y) vzip_u8(x, y).val[0] +#define vzip1_s16(x, y) vzip_s16(x, y).val[0] +#define vzip1_u16(x, y) vzip_u16(x, y).val[0] +#define vzip1_f32(x, y) vzip_f32(x, y).val[0] +#define vzip1_u32(x, y) vzip_u32(x, y).val[0] +#define vzip1_s32(x, y) vzip_s32(x, y).val[0] +#define vzip1q_s8(x, y) vzipq_s8(x, y).val[0] +#define vzip1q_u8(x, y) vzipq_u8(x, y).val[0] +#define vzip1q_s16(x, y) vzipq_s16(x, y).val[0] +#define vzip1q_u16(x, y) vzipq_u16(x, y).val[0] +#define vzip1q_s32(x, y) vzipq_s32(x, y).val[0] +#define vzip1q_u32(x, y) vzipq_u32(x, y).val[0] +#define vzip1q_f32(x, y) vzipq_f32(x, y).val[0] +#define vzip2_s8(x, y) vzip_s8(x, y).val[1] +#define vzip2_u8(x, y) vzip_u8(x, y).val[1] +#define vzip2_s16(x, y) vzip_s16(x, y).val[1] +#define vzip2_u16(x, y) vzip_u16(x, y).val[1] +#define vzip2_s32(x, y) vzip_s32(x, y).val[1] +#define vzip2_u32(x, y) vzip_u32(x, y).val[1] +#define vzip2_f32(x, y) vzip_f32(x, y).val[1] +#define vzip2q_s8(x, y) vzipq_s8(x, y).val[1] +#define vzip2q_u8(x, y) vzipq_u8(x, y).val[1] +#define vzip2q_s16(x, y) vzipq_s16(x, y).val[1] +#define vzip2q_u16(x, y) vzipq_u16(x, y).val[1] +#define vzip2q_s32(x, y) vzipq_s32(x, y).val[1] +#define vzip2q_u32(x, y) vzipq_u32(x, y).val[1] +#define vzip2q_f32(x, y) vzipq_f32(x, y).val[1] +#endif + +template <typename T, size_t N> +struct Raw128; + +// 128 +template <> +struct Raw128<uint8_t, 16> { + using type = uint8x16_t; +}; + +template <> +struct Raw128<uint16_t, 8> { + using type = uint16x8_t; +}; + +template <> +struct Raw128<uint32_t, 4> { + using type = uint32x4_t; +}; + +template <> +struct Raw128<uint64_t, 2> { + using type = uint64x2_t; +}; + +template <> +struct Raw128<int8_t, 16> { + using type = int8x16_t; +}; + +template <> +struct Raw128<int16_t, 8> { + using type = int16x8_t; +}; + +template <> +struct Raw128<int32_t, 4> { + using type = int32x4_t; +}; + +template <> +struct Raw128<int64_t, 2> { + using type = int64x2_t; +}; + +template <> +struct Raw128<float16_t, 8> { + using type = uint16x8_t; +}; + +template <> +struct Raw128<bfloat16_t, 8> { + using type = uint16x8_t; +}; + +template <> +struct Raw128<float, 4> { + using type = float32x4_t; +}; + +#if HWY_ARCH_ARM_A64 +template <> +struct Raw128<double, 2> { + using type = float64x2_t; +}; +#endif + +// 64 +template <> +struct Raw128<uint8_t, 8> { + using type = uint8x8_t; +}; + +template <> +struct Raw128<uint16_t, 4> { + using type = uint16x4_t; +}; + +template <> +struct Raw128<uint32_t, 2> { + using type = uint32x2_t; +}; + +template <> +struct Raw128<uint64_t, 1> { + using type = uint64x1_t; +}; + +template <> +struct Raw128<int8_t, 8> { + using type = int8x8_t; +}; + +template <> +struct Raw128<int16_t, 4> { + using type = int16x4_t; +}; + +template <> +struct Raw128<int32_t, 2> { + using type = int32x2_t; +}; + +template <> +struct Raw128<int64_t, 1> { + using type = int64x1_t; +}; + +template <> +struct Raw128<float16_t, 4> { + using type = uint16x4_t; +}; + +template <> +struct Raw128<bfloat16_t, 4> { + using type = uint16x4_t; +}; + +template <> +struct Raw128<float, 2> { + using type = float32x2_t; +}; + +#if HWY_ARCH_ARM_A64 +template <> +struct Raw128<double, 1> { + using type = float64x1_t; +}; +#endif + +// 32 (same as 64) +template <> +struct Raw128<uint8_t, 4> { + using type = uint8x8_t; +}; + +template <> +struct Raw128<uint16_t, 2> { + using type = uint16x4_t; +}; + +template <> +struct Raw128<uint32_t, 1> { + using type = uint32x2_t; +}; + +template <> +struct Raw128<int8_t, 4> { + using type = int8x8_t; +}; + +template <> +struct Raw128<int16_t, 2> { + using type = int16x4_t; +}; + +template <> +struct Raw128<int32_t, 1> { + using type = int32x2_t; +}; + +template <> +struct Raw128<float16_t, 2> { + using type = uint16x4_t; +}; + +template <> +struct Raw128<bfloat16_t, 2> { + using type = uint16x4_t; +}; + +template <> +struct Raw128<float, 1> { + using type = float32x2_t; +}; + +// 16 (same as 64) +template <> +struct Raw128<uint8_t, 2> { + using type = uint8x8_t; +}; + +template <> +struct Raw128<uint16_t, 1> { + using type = uint16x4_t; +}; + +template <> +struct Raw128<int8_t, 2> { + using type = int8x8_t; +}; + +template <> +struct Raw128<int16_t, 1> { + using type = int16x4_t; +}; + +template <> +struct Raw128<float16_t, 1> { + using type = uint16x4_t; +}; + +template <> +struct Raw128<bfloat16_t, 1> { + using type = uint16x4_t; +}; + +// 8 (same as 64) +template <> +struct Raw128<uint8_t, 1> { + using type = uint8x8_t; +}; + +template <> +struct Raw128<int8_t, 1> { + using type = int8x8_t; +}; + +} // namespace detail + +template <typename T, size_t N = 16 / sizeof(T)> +class Vec128 { + using Raw = typename detail::Raw128<T, N>::type; + + public: + HWY_INLINE Vec128() {} + Vec128(const Vec128&) = default; + Vec128& operator=(const Vec128&) = default; + HWY_INLINE explicit Vec128(const Raw raw) : raw(raw) {} + + // Compound assignment. Only usable if there is a corresponding non-member + // binary operator overload. For example, only f32 and f64 support division. + HWY_INLINE Vec128& operator*=(const Vec128 other) { + return *this = (*this * other); + } + HWY_INLINE Vec128& operator/=(const Vec128 other) { + return *this = (*this / other); + } + HWY_INLINE Vec128& operator+=(const Vec128 other) { + return *this = (*this + other); + } + HWY_INLINE Vec128& operator-=(const Vec128 other) { + return *this = (*this - other); + } + HWY_INLINE Vec128& operator&=(const Vec128 other) { + return *this = (*this & other); + } + HWY_INLINE Vec128& operator|=(const Vec128 other) { + return *this = (*this | other); + } + HWY_INLINE Vec128& operator^=(const Vec128 other) { + return *this = (*this ^ other); + } + + Raw raw; +}; + +// FF..FF or 0. +template <typename T, size_t N = 16 / sizeof(T)> +class Mask128 { + // ARM C Language Extensions return and expect unsigned type. + using Raw = typename detail::Raw128<MakeUnsigned<T>, N>::type; + + public: + HWY_INLINE Mask128() {} + Mask128(const Mask128&) = default; + Mask128& operator=(const Mask128&) = default; + HWY_INLINE explicit Mask128(const Raw raw) : raw(raw) {} + + Raw raw; +}; + +namespace detail { + +// Deduce Simd<T, N> from Vec128<T, N> +struct DeduceD { + template <typename T, size_t N> + Simd<T, N> operator()(Vec128<T, N>) const { + return Simd<T, N>(); + } +}; + +} // namespace detail + +template <class V> +using DFromV = decltype(detail::DeduceD()(V())); + +template <class V> +using TFromV = TFromD<DFromV<V>>; + +// ------------------------------ BitCast + +namespace detail { + +// Converts from Vec128<T, N> to Vec128<uint8_t, N * sizeof(T)> using the +// vreinterpret*_u8_*() set of functions. +#define HWY_NEON_BUILD_TPL_HWY_CAST_TO_U8 +#define HWY_NEON_BUILD_RET_HWY_CAST_TO_U8(type, size) \ + Vec128<uint8_t, size * sizeof(type)> +#define HWY_NEON_BUILD_PARAM_HWY_CAST_TO_U8(type, size) Vec128<type, size> v +#define HWY_NEON_BUILD_ARG_HWY_CAST_TO_U8 v.raw + +// Special case of u8 to u8 since vreinterpret*_u8_u8 is obviously not defined. +template <size_t N> +HWY_INLINE Vec128<uint8_t, N> BitCastToByte(Vec128<uint8_t, N> v) { + return v; +} + +HWY_NEON_DEF_FUNCTION_ALL_FLOATS(BitCastToByte, vreinterpret, _u8_, + HWY_CAST_TO_U8) +HWY_NEON_DEF_FUNCTION_INTS(BitCastToByte, vreinterpret, _u8_, HWY_CAST_TO_U8) +HWY_NEON_DEF_FUNCTION_UINT_16(BitCastToByte, vreinterpret, _u8_, HWY_CAST_TO_U8) +HWY_NEON_DEF_FUNCTION_UINT_32(BitCastToByte, vreinterpret, _u8_, HWY_CAST_TO_U8) +HWY_NEON_DEF_FUNCTION_UINT_64(BitCastToByte, vreinterpret, _u8_, HWY_CAST_TO_U8) + +// Special cases for [b]float16_t, which have the same Raw as uint16_t. +template <size_t N> +HWY_INLINE Vec128<uint8_t, N * 2> BitCastToByte(Vec128<float16_t, N> v) { + return BitCastToByte(Vec128<uint16_t, N>(v.raw)); +} +template <size_t N> +HWY_INLINE Vec128<uint8_t, N * 2> BitCastToByte(Vec128<bfloat16_t, N> v) { + return BitCastToByte(Vec128<uint16_t, N>(v.raw)); +} + +#undef HWY_NEON_BUILD_TPL_HWY_CAST_TO_U8 +#undef HWY_NEON_BUILD_RET_HWY_CAST_TO_U8 +#undef HWY_NEON_BUILD_PARAM_HWY_CAST_TO_U8 +#undef HWY_NEON_BUILD_ARG_HWY_CAST_TO_U8 + +template <size_t N> +HWY_INLINE Vec128<uint8_t, N> BitCastFromByte(Simd<uint8_t, N> /* tag */, + Vec128<uint8_t, N> v) { + return v; +} + +// 64-bit or less: + +template <size_t N, HWY_IF_LE64(int8_t, N)> +HWY_INLINE Vec128<int8_t, N> BitCastFromByte(Simd<int8_t, N> /* tag */, + Vec128<uint8_t, N> v) { + return Vec128<int8_t, N>(vreinterpret_s8_u8(v.raw)); +} +template <size_t N, HWY_IF_LE64(uint16_t, N)> +HWY_INLINE Vec128<uint16_t, N> BitCastFromByte(Simd<uint16_t, N> /* tag */, + Vec128<uint8_t, N * 2> v) { + return Vec128<uint16_t, N>(vreinterpret_u16_u8(v.raw)); +} +template <size_t N, HWY_IF_LE64(int16_t, N)> +HWY_INLINE Vec128<int16_t, N> BitCastFromByte(Simd<int16_t, N> /* tag */, + Vec128<uint8_t, N * 2> v) { + return Vec128<int16_t, N>(vreinterpret_s16_u8(v.raw)); +} +template <size_t N, HWY_IF_LE64(uint32_t, N)> +HWY_INLINE Vec128<uint32_t, N> BitCastFromByte(Simd<uint32_t, N> /* tag */, + Vec128<uint8_t, N * 4> v) { + return Vec128<uint32_t, N>(vreinterpret_u32_u8(v.raw)); +} +template <size_t N, HWY_IF_LE64(int32_t, N)> +HWY_INLINE Vec128<int32_t, N> BitCastFromByte(Simd<int32_t, N> /* tag */, + Vec128<uint8_t, N * 4> v) { + return Vec128<int32_t, N>(vreinterpret_s32_u8(v.raw)); +} +template <size_t N, HWY_IF_LE64(float, N)> +HWY_INLINE Vec128<float, N> BitCastFromByte(Simd<float, N> /* tag */, + Vec128<uint8_t, N * 4> v) { + return Vec128<float, N>(vreinterpret_f32_u8(v.raw)); +} +HWY_INLINE Vec128<uint64_t, 1> BitCastFromByte(Simd<uint64_t, 1> /* tag */, + Vec128<uint8_t, 1 * 8> v) { + return Vec128<uint64_t, 1>(vreinterpret_u64_u8(v.raw)); +} +HWY_INLINE Vec128<int64_t, 1> BitCastFromByte(Simd<int64_t, 1> /* tag */, + Vec128<uint8_t, 1 * 8> v) { + return Vec128<int64_t, 1>(vreinterpret_s64_u8(v.raw)); +} +#if HWY_ARCH_ARM_A64 +HWY_INLINE Vec128<double, 1> BitCastFromByte(Simd<double, 1> /* tag */, + Vec128<uint8_t, 1 * 8> v) { + return Vec128<double, 1>(vreinterpret_f64_u8(v.raw)); +} +#endif + +// 128-bit full: + +HWY_INLINE Vec128<int8_t> BitCastFromByte(Full128<int8_t> /* tag */, + Vec128<uint8_t> v) { + return Vec128<int8_t>(vreinterpretq_s8_u8(v.raw)); +} +HWY_INLINE Vec128<uint16_t> BitCastFromByte(Full128<uint16_t> /* tag */, + Vec128<uint8_t> v) { + return Vec128<uint16_t>(vreinterpretq_u16_u8(v.raw)); +} +HWY_INLINE Vec128<int16_t> BitCastFromByte(Full128<int16_t> /* tag */, + Vec128<uint8_t> v) { + return Vec128<int16_t>(vreinterpretq_s16_u8(v.raw)); +} +HWY_INLINE Vec128<uint32_t> BitCastFromByte(Full128<uint32_t> /* tag */, + Vec128<uint8_t> v) { + return Vec128<uint32_t>(vreinterpretq_u32_u8(v.raw)); +} +HWY_INLINE Vec128<int32_t> BitCastFromByte(Full128<int32_t> /* tag */, + Vec128<uint8_t> v) { + return Vec128<int32_t>(vreinterpretq_s32_u8(v.raw)); +} +HWY_INLINE Vec128<float> BitCastFromByte(Full128<float> /* tag */, + Vec128<uint8_t> v) { + return Vec128<float>(vreinterpretq_f32_u8(v.raw)); +} +HWY_INLINE Vec128<uint64_t> BitCastFromByte(Full128<uint64_t> /* tag */, + Vec128<uint8_t> v) { + return Vec128<uint64_t>(vreinterpretq_u64_u8(v.raw)); +} +HWY_INLINE Vec128<int64_t> BitCastFromByte(Full128<int64_t> /* tag */, + Vec128<uint8_t> v) { + return Vec128<int64_t>(vreinterpretq_s64_u8(v.raw)); +} + +#if HWY_ARCH_ARM_A64 +HWY_INLINE Vec128<double> BitCastFromByte(Full128<double> /* tag */, + Vec128<uint8_t> v) { + return Vec128<double>(vreinterpretq_f64_u8(v.raw)); +} +#endif + +// Special cases for [b]float16_t, which have the same Raw as uint16_t. +template <size_t N> +HWY_INLINE Vec128<float16_t, N> BitCastFromByte(Simd<float16_t, N> /* tag */, + Vec128<uint8_t, N * 2> v) { + return Vec128<float16_t, N>(BitCastFromByte(Simd<uint16_t, N>(), v).raw); +} +template <size_t N> +HWY_INLINE Vec128<bfloat16_t, N> BitCastFromByte(Simd<bfloat16_t, N> /* tag */, + Vec128<uint8_t, N * 2> v) { + return Vec128<bfloat16_t, N>(BitCastFromByte(Simd<uint16_t, N>(), v).raw); +} + +} // namespace detail + +template <typename T, size_t N, typename FromT> +HWY_API Vec128<T, N> BitCast(Simd<T, N> d, + Vec128<FromT, N * sizeof(T) / sizeof(FromT)> v) { + return detail::BitCastFromByte(d, detail::BitCastToByte(v)); +} + +// ------------------------------ Set + +// Returns a vector with all lanes set to "t". +#define HWY_NEON_BUILD_TPL_HWY_SET1 +#define HWY_NEON_BUILD_RET_HWY_SET1(type, size) Vec128<type, size> +#define HWY_NEON_BUILD_PARAM_HWY_SET1(type, size) \ + Simd<type, size> /* tag */, const type t +#define HWY_NEON_BUILD_ARG_HWY_SET1 t + +HWY_NEON_DEF_FUNCTION_ALL_TYPES(Set, vdup, _n_, HWY_SET1) + +#undef HWY_NEON_BUILD_TPL_HWY_SET1 +#undef HWY_NEON_BUILD_RET_HWY_SET1 +#undef HWY_NEON_BUILD_PARAM_HWY_SET1 +#undef HWY_NEON_BUILD_ARG_HWY_SET1 + +// Returns an all-zero vector. +template <typename T, size_t N> +HWY_API Vec128<T, N> Zero(Simd<T, N> d) { + return Set(d, 0); +} + +template <size_t N> +HWY_API Vec128<bfloat16_t, N> Zero(Simd<bfloat16_t, N> /* tag */) { + return Vec128<bfloat16_t, N>(Zero(Simd<uint16_t, N>()).raw); +} + +template <class D> +using VFromD = decltype(Zero(D())); + +// Returns a vector with uninitialized elements. +template <typename T, size_t N> +HWY_API Vec128<T, N> Undefined(Simd<T, N> /*d*/) { + HWY_DIAGNOSTICS(push) + HWY_DIAGNOSTICS_OFF(disable : 4701, ignored "-Wuninitialized") + typename detail::Raw128<T, N>::type a; + return Vec128<T, N>(a); + HWY_DIAGNOSTICS(pop) +} + +// Returns a vector with lane i=[0, N) set to "first" + i. +template <typename T, size_t N, typename T2> +Vec128<T, N> Iota(const Simd<T, N> d, const T2 first) { + HWY_ALIGN T lanes[16 / sizeof(T)]; + for (size_t i = 0; i < 16 / sizeof(T); ++i) { + lanes[i] = static_cast<T>(first + static_cast<T2>(i)); + } + return Load(d, lanes); +} + +// ------------------------------ GetLane + +HWY_API uint8_t GetLane(const Vec128<uint8_t, 16> v) { + return vgetq_lane_u8(v.raw, 0); +} +template <size_t N> +HWY_API uint8_t GetLane(const Vec128<uint8_t, N> v) { + return vget_lane_u8(v.raw, 0); +} + +HWY_API int8_t GetLane(const Vec128<int8_t, 16> v) { + return vgetq_lane_s8(v.raw, 0); +} +template <size_t N> +HWY_API int8_t GetLane(const Vec128<int8_t, N> v) { + return vget_lane_s8(v.raw, 0); +} + +HWY_API uint16_t GetLane(const Vec128<uint16_t, 8> v) { + return vgetq_lane_u16(v.raw, 0); +} +template <size_t N> +HWY_API uint16_t GetLane(const Vec128<uint16_t, N> v) { + return vget_lane_u16(v.raw, 0); +} + +HWY_API int16_t GetLane(const Vec128<int16_t, 8> v) { + return vgetq_lane_s16(v.raw, 0); +} +template <size_t N> +HWY_API int16_t GetLane(const Vec128<int16_t, N> v) { + return vget_lane_s16(v.raw, 0); +} + +HWY_API uint32_t GetLane(const Vec128<uint32_t, 4> v) { + return vgetq_lane_u32(v.raw, 0); +} +template <size_t N> +HWY_API uint32_t GetLane(const Vec128<uint32_t, N> v) { + return vget_lane_u32(v.raw, 0); +} + +HWY_API int32_t GetLane(const Vec128<int32_t, 4> v) { + return vgetq_lane_s32(v.raw, 0); +} +template <size_t N> +HWY_API int32_t GetLane(const Vec128<int32_t, N> v) { + return vget_lane_s32(v.raw, 0); +} + +HWY_API uint64_t GetLane(const Vec128<uint64_t, 2> v) { + return vgetq_lane_u64(v.raw, 0); +} +HWY_API uint64_t GetLane(const Vec128<uint64_t, 1> v) { + return vget_lane_u64(v.raw, 0); +} +HWY_API int64_t GetLane(const Vec128<int64_t, 2> v) { + return vgetq_lane_s64(v.raw, 0); +} +HWY_API int64_t GetLane(const Vec128<int64_t, 1> v) { + return vget_lane_s64(v.raw, 0); +} + +HWY_API float GetLane(const Vec128<float, 4> v) { + return vgetq_lane_f32(v.raw, 0); +} +HWY_API float GetLane(const Vec128<float, 2> v) { + return vget_lane_f32(v.raw, 0); +} +HWY_API float GetLane(const Vec128<float, 1> v) { + return vget_lane_f32(v.raw, 0); +} +#if HWY_ARCH_ARM_A64 +HWY_API double GetLane(const Vec128<double, 2> v) { + return vgetq_lane_f64(v.raw, 0); +} +HWY_API double GetLane(const Vec128<double, 1> v) { + return vget_lane_f64(v.raw, 0); +} +#endif + +// ================================================== ARITHMETIC + +// ------------------------------ Addition +HWY_NEON_DEF_FUNCTION_ALL_TYPES(operator+, vadd, _, 2) + +// ------------------------------ Subtraction +HWY_NEON_DEF_FUNCTION_ALL_TYPES(operator-, vsub, _, 2) + +// ------------------------------ Saturating addition and subtraction +// Only defined for uint8_t, uint16_t and their signed versions, as in other +// architectures. + +// Returns a + b clamped to the destination range. +HWY_NEON_DEF_FUNCTION_INT_8(SaturatedAdd, vqadd, _, 2) +HWY_NEON_DEF_FUNCTION_INT_16(SaturatedAdd, vqadd, _, 2) +HWY_NEON_DEF_FUNCTION_UINT_8(SaturatedAdd, vqadd, _, 2) +HWY_NEON_DEF_FUNCTION_UINT_16(SaturatedAdd, vqadd, _, 2) + +// Returns a - b clamped to the destination range. +HWY_NEON_DEF_FUNCTION_INT_8(SaturatedSub, vqsub, _, 2) +HWY_NEON_DEF_FUNCTION_INT_16(SaturatedSub, vqsub, _, 2) +HWY_NEON_DEF_FUNCTION_UINT_8(SaturatedSub, vqsub, _, 2) +HWY_NEON_DEF_FUNCTION_UINT_16(SaturatedSub, vqsub, _, 2) + +// Not part of API, used in implementation. +namespace detail { +HWY_NEON_DEF_FUNCTION_UINT_32(SaturatedSub, vqsub, _, 2) +HWY_NEON_DEF_FUNCTION_UINT_64(SaturatedSub, vqsub, _, 2) +HWY_NEON_DEF_FUNCTION_INT_32(SaturatedSub, vqsub, _, 2) +HWY_NEON_DEF_FUNCTION_INT_64(SaturatedSub, vqsub, _, 2) +} // namespace detail + +// ------------------------------ Average + +// Returns (a + b + 1) / 2 +HWY_NEON_DEF_FUNCTION_UINT_8(AverageRound, vrhadd, _, 2) +HWY_NEON_DEF_FUNCTION_UINT_16(AverageRound, vrhadd, _, 2) + +// ------------------------------ Neg + +HWY_NEON_DEF_FUNCTION_ALL_FLOATS(Neg, vneg, _, 1) +HWY_NEON_DEF_FUNCTION_INT_8_16_32(Neg, vneg, _, 1) // i64 implemented below + +HWY_API Vec128<int64_t, 1> Neg(const Vec128<int64_t, 1> v) { +#if HWY_ARCH_ARM_A64 + return Vec128<int64_t, 1>(vneg_s64(v.raw)); +#else + return Zero(Simd<int64_t, 1>()) - v; +#endif +} + +HWY_API Vec128<int64_t> Neg(const Vec128<int64_t> v) { +#if HWY_ARCH_ARM_A64 + return Vec128<int64_t>(vnegq_s64(v.raw)); +#else + return Zero(Full128<int64_t>()) - v; +#endif +} + +// ------------------------------ ShiftLeft + +// Customize HWY_NEON_DEF_FUNCTION to special-case count=0 (not supported). +#pragma push_macro("HWY_NEON_DEF_FUNCTION") +#undef HWY_NEON_DEF_FUNCTION +#define HWY_NEON_DEF_FUNCTION(type, size, name, prefix, infix, suffix, args) \ + template <int kBits> \ + HWY_API Vec128<type, size> name(const Vec128<type, size> v) { \ + return kBits == 0 ? v \ + : Vec128<type, size>(HWY_NEON_EVAL( \ + prefix##infix##suffix, v.raw, HWY_MAX(1, kBits))); \ + } + +HWY_NEON_DEF_FUNCTION_INTS_UINTS(ShiftLeft, vshl, _n_, HWY_SHIFT) + +HWY_NEON_DEF_FUNCTION_UINTS(ShiftRight, vshr, _n_, HWY_SHIFT) +HWY_NEON_DEF_FUNCTION_INTS(ShiftRight, vshr, _n_, HWY_SHIFT) + +#pragma pop_macro("HWY_NEON_DEF_FUNCTION") + +// ------------------------------ RotateRight (ShiftRight, Or) + +template <int kBits, size_t N> +HWY_API Vec128<uint32_t, N> RotateRight(const Vec128<uint32_t, N> v) { + static_assert(0 <= kBits && kBits < 32, "Invalid shift count"); + if (kBits == 0) return v; + return Or(ShiftRight<kBits>(v), ShiftLeft<HWY_MIN(31, 32 - kBits)>(v)); +} + +template <int kBits, size_t N> +HWY_API Vec128<uint64_t, N> RotateRight(const Vec128<uint64_t, N> v) { + static_assert(0 <= kBits && kBits < 64, "Invalid shift count"); + if (kBits == 0) return v; + return Or(ShiftRight<kBits>(v), ShiftLeft<HWY_MIN(63, 64 - kBits)>(v)); +} + +// NOTE: vxarq_u64 can be applied to uint64_t, but we do not yet have a +// mechanism for checking for extensions to ARMv8. + +// ------------------------------ Shl + +HWY_API Vec128<uint8_t> operator<<(const Vec128<uint8_t> v, + const Vec128<uint8_t> bits) { + return Vec128<uint8_t>(vshlq_u8(v.raw, vreinterpretq_s8_u8(bits.raw))); +} +template <size_t N, HWY_IF_LE64(uint8_t, N)> +HWY_API Vec128<uint8_t, N> operator<<(const Vec128<uint8_t, N> v, + const Vec128<uint8_t, N> bits) { + return Vec128<uint8_t, N>(vshl_u8(v.raw, vreinterpret_s8_u8(bits.raw))); +} + +HWY_API Vec128<uint16_t> operator<<(const Vec128<uint16_t> v, + const Vec128<uint16_t> bits) { + return Vec128<uint16_t>(vshlq_u16(v.raw, vreinterpretq_s16_u16(bits.raw))); +} +template <size_t N, HWY_IF_LE64(uint16_t, N)> +HWY_API Vec128<uint16_t, N> operator<<(const Vec128<uint16_t, N> v, + const Vec128<uint16_t, N> bits) { + return Vec128<uint16_t, N>(vshl_u16(v.raw, vreinterpret_s16_u16(bits.raw))); +} + +HWY_API Vec128<uint32_t> operator<<(const Vec128<uint32_t> v, + const Vec128<uint32_t> bits) { + return Vec128<uint32_t>(vshlq_u32(v.raw, vreinterpretq_s32_u32(bits.raw))); +} +template <size_t N, HWY_IF_LE64(uint32_t, N)> +HWY_API Vec128<uint32_t, N> operator<<(const Vec128<uint32_t, N> v, + const Vec128<uint32_t, N> bits) { + return Vec128<uint32_t, N>(vshl_u32(v.raw, vreinterpret_s32_u32(bits.raw))); +} + +HWY_API Vec128<uint64_t> operator<<(const Vec128<uint64_t> v, + const Vec128<uint64_t> bits) { + return Vec128<uint64_t>(vshlq_u64(v.raw, vreinterpretq_s64_u64(bits.raw))); +} +HWY_API Vec128<uint64_t, 1> operator<<(const Vec128<uint64_t, 1> v, + const Vec128<uint64_t, 1> bits) { + return Vec128<uint64_t, 1>(vshl_u64(v.raw, vreinterpret_s64_u64(bits.raw))); +} + +HWY_API Vec128<int8_t> operator<<(const Vec128<int8_t> v, + const Vec128<int8_t> bits) { + return Vec128<int8_t>(vshlq_s8(v.raw, bits.raw)); +} +template <size_t N, HWY_IF_LE64(int8_t, N)> +HWY_API Vec128<int8_t, N> operator<<(const Vec128<int8_t, N> v, + const Vec128<int8_t, N> bits) { + return Vec128<int8_t, N>(vshl_s8(v.raw, bits.raw)); +} + +HWY_API Vec128<int16_t> operator<<(const Vec128<int16_t> v, + const Vec128<int16_t> bits) { + return Vec128<int16_t>(vshlq_s16(v.raw, bits.raw)); +} +template <size_t N, HWY_IF_LE64(int16_t, N)> +HWY_API Vec128<int16_t, N> operator<<(const Vec128<int16_t, N> v, + const Vec128<int16_t, N> bits) { + return Vec128<int16_t, N>(vshl_s16(v.raw, bits.raw)); +} + +HWY_API Vec128<int32_t> operator<<(const Vec128<int32_t> v, + const Vec128<int32_t> bits) { + return Vec128<int32_t>(vshlq_s32(v.raw, bits.raw)); +} +template <size_t N, HWY_IF_LE64(int32_t, N)> +HWY_API Vec128<int32_t, N> operator<<(const Vec128<int32_t, N> v, + const Vec128<int32_t, N> bits) { + return Vec128<int32_t, N>(vshl_s32(v.raw, bits.raw)); +} + +HWY_API Vec128<int64_t> operator<<(const Vec128<int64_t> v, + const Vec128<int64_t> bits) { + return Vec128<int64_t>(vshlq_s64(v.raw, bits.raw)); +} +HWY_API Vec128<int64_t, 1> operator<<(const Vec128<int64_t, 1> v, + const Vec128<int64_t, 1> bits) { + return Vec128<int64_t, 1>(vshl_s64(v.raw, bits.raw)); +} + +// ------------------------------ Shr (Neg) + +HWY_API Vec128<uint8_t> operator>>(const Vec128<uint8_t> v, + const Vec128<uint8_t> bits) { + const int8x16_t neg_bits = Neg(BitCast(Full128<int8_t>(), bits)).raw; + return Vec128<uint8_t>(vshlq_u8(v.raw, neg_bits)); +} +template <size_t N, HWY_IF_LE64(uint8_t, N)> +HWY_API Vec128<uint8_t, N> operator>>(const Vec128<uint8_t, N> v, + const Vec128<uint8_t, N> bits) { + const int8x8_t neg_bits = Neg(BitCast(Simd<int8_t, N>(), bits)).raw; + return Vec128<uint8_t, N>(vshl_u8(v.raw, neg_bits)); +} + +HWY_API Vec128<uint16_t> operator>>(const Vec128<uint16_t> v, + const Vec128<uint16_t> bits) { + const int16x8_t neg_bits = Neg(BitCast(Full128<int16_t>(), bits)).raw; + return Vec128<uint16_t>(vshlq_u16(v.raw, neg_bits)); +} +template <size_t N, HWY_IF_LE64(uint16_t, N)> +HWY_API Vec128<uint16_t, N> operator>>(const Vec128<uint16_t, N> v, + const Vec128<uint16_t, N> bits) { + const int16x4_t neg_bits = Neg(BitCast(Simd<int16_t, N>(), bits)).raw; + return Vec128<uint16_t, N>(vshl_u16(v.raw, neg_bits)); +} + +HWY_API Vec128<uint32_t> operator>>(const Vec128<uint32_t> v, + const Vec128<uint32_t> bits) { + const int32x4_t neg_bits = Neg(BitCast(Full128<int32_t>(), bits)).raw; + return Vec128<uint32_t>(vshlq_u32(v.raw, neg_bits)); +} +template <size_t N, HWY_IF_LE64(uint32_t, N)> +HWY_API Vec128<uint32_t, N> operator>>(const Vec128<uint32_t, N> v, + const Vec128<uint32_t, N> bits) { + const int32x2_t neg_bits = Neg(BitCast(Simd<int32_t, N>(), bits)).raw; + return Vec128<uint32_t, N>(vshl_u32(v.raw, neg_bits)); +} + +HWY_API Vec128<uint64_t> operator>>(const Vec128<uint64_t> v, + const Vec128<uint64_t> bits) { + const int64x2_t neg_bits = Neg(BitCast(Full128<int64_t>(), bits)).raw; + return Vec128<uint64_t>(vshlq_u64(v.raw, neg_bits)); +} +HWY_API Vec128<uint64_t, 1> operator>>(const Vec128<uint64_t, 1> v, + const Vec128<uint64_t, 1> bits) { + const int64x1_t neg_bits = Neg(BitCast(Simd<int64_t, 1>(), bits)).raw; + return Vec128<uint64_t, 1>(vshl_u64(v.raw, neg_bits)); +} + +HWY_API Vec128<int8_t> operator>>(const Vec128<int8_t> v, + const Vec128<int8_t> bits) { + return Vec128<int8_t>(vshlq_s8(v.raw, Neg(bits).raw)); +} +template <size_t N, HWY_IF_LE64(int8_t, N)> +HWY_API Vec128<int8_t, N> operator>>(const Vec128<int8_t, N> v, + const Vec128<int8_t, N> bits) { + return Vec128<int8_t, N>(vshl_s8(v.raw, Neg(bits).raw)); +} + +HWY_API Vec128<int16_t> operator>>(const Vec128<int16_t> v, + const Vec128<int16_t> bits) { + return Vec128<int16_t>(vshlq_s16(v.raw, Neg(bits).raw)); +} +template <size_t N, HWY_IF_LE64(int16_t, N)> +HWY_API Vec128<int16_t, N> operator>>(const Vec128<int16_t, N> v, + const Vec128<int16_t, N> bits) { + return Vec128<int16_t, N>(vshl_s16(v.raw, Neg(bits).raw)); +} + +HWY_API Vec128<int32_t> operator>>(const Vec128<int32_t> v, + const Vec128<int32_t> bits) { + return Vec128<int32_t>(vshlq_s32(v.raw, Neg(bits).raw)); +} +template <size_t N, HWY_IF_LE64(int32_t, N)> +HWY_API Vec128<int32_t, N> operator>>(const Vec128<int32_t, N> v, + const Vec128<int32_t, N> bits) { + return Vec128<int32_t, N>(vshl_s32(v.raw, Neg(bits).raw)); +} + +HWY_API Vec128<int64_t> operator>>(const Vec128<int64_t> v, + const Vec128<int64_t> bits) { + return Vec128<int64_t>(vshlq_s64(v.raw, Neg(bits).raw)); +} +HWY_API Vec128<int64_t, 1> operator>>(const Vec128<int64_t, 1> v, + const Vec128<int64_t, 1> bits) { + return Vec128<int64_t, 1>(vshl_s64(v.raw, Neg(bits).raw)); +} + +// ------------------------------ ShiftLeftSame (Shl) + +template <typename T, size_t N> +HWY_API Vec128<T, N> ShiftLeftSame(const Vec128<T, N> v, int bits) { + return v << Set(Simd<T, N>(), static_cast<T>(bits)); +} +template <typename T, size_t N> +HWY_API Vec128<T, N> ShiftRightSame(const Vec128<T, N> v, int bits) { + return v >> Set(Simd<T, N>(), static_cast<T>(bits)); +} + +// ------------------------------ Integer multiplication + +// Unsigned +HWY_API Vec128<uint16_t> operator*(const Vec128<uint16_t> a, + const Vec128<uint16_t> b) { + return Vec128<uint16_t>(vmulq_u16(a.raw, b.raw)); +} +HWY_API Vec128<uint32_t> operator*(const Vec128<uint32_t> a, + const Vec128<uint32_t> b) { + return Vec128<uint32_t>(vmulq_u32(a.raw, b.raw)); +} + +template <size_t N, HWY_IF_LE64(uint16_t, N)> +HWY_API Vec128<uint16_t, N> operator*(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>(vmul_u16(a.raw, b.raw)); +} +template <size_t N, HWY_IF_LE64(uint32_t, N)> +HWY_API Vec128<uint32_t, N> operator*(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { + return Vec128<uint32_t, N>(vmul_u32(a.raw, b.raw)); +} + +// Signed +HWY_API Vec128<int16_t> operator*(const Vec128<int16_t> a, + const Vec128<int16_t> b) { + return Vec128<int16_t>(vmulq_s16(a.raw, b.raw)); +} +HWY_API Vec128<int32_t> operator*(const Vec128<int32_t> a, + const Vec128<int32_t> b) { + return Vec128<int32_t>(vmulq_s32(a.raw, b.raw)); +} + +template <size_t N, HWY_IF_LE64(uint16_t, N)> +HWY_API Vec128<int16_t, N> operator*(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>(vmul_s16(a.raw, b.raw)); +} +template <size_t N, HWY_IF_LE64(int32_t, N)> +HWY_API Vec128<int32_t, N> operator*(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + return Vec128<int32_t, N>(vmul_s32(a.raw, b.raw)); +} + +// Returns the upper 16 bits of a * b in each lane. +HWY_API Vec128<int16_t> MulHigh(const Vec128<int16_t> a, + const Vec128<int16_t> b) { + int32x4_t rlo = vmull_s16(vget_low_s16(a.raw), vget_low_s16(b.raw)); +#if HWY_ARCH_ARM_A64 + int32x4_t rhi = vmull_high_s16(a.raw, b.raw); +#else + int32x4_t rhi = vmull_s16(vget_high_s16(a.raw), vget_high_s16(b.raw)); +#endif + return Vec128<int16_t>( + vuzp2q_s16(vreinterpretq_s16_s32(rlo), vreinterpretq_s16_s32(rhi))); +} +HWY_API Vec128<uint16_t> MulHigh(const Vec128<uint16_t> a, + const Vec128<uint16_t> b) { + uint32x4_t rlo = vmull_u16(vget_low_u16(a.raw), vget_low_u16(b.raw)); +#if HWY_ARCH_ARM_A64 + uint32x4_t rhi = vmull_high_u16(a.raw, b.raw); +#else + uint32x4_t rhi = vmull_u16(vget_high_u16(a.raw), vget_high_u16(b.raw)); +#endif + return Vec128<uint16_t>( + vuzp2q_u16(vreinterpretq_u16_u32(rlo), vreinterpretq_u16_u32(rhi))); +} + +template <size_t N, HWY_IF_LE64(int16_t, N)> +HWY_API Vec128<int16_t, N> MulHigh(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + int16x8_t hi_lo = vreinterpretq_s16_s32(vmull_s16(a.raw, b.raw)); + return Vec128<int16_t, N>(vget_low_s16(vuzp2q_s16(hi_lo, hi_lo))); +} +template <size_t N, HWY_IF_LE64(uint16_t, N)> +HWY_API Vec128<uint16_t, N> MulHigh(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + uint16x8_t hi_lo = vreinterpretq_u16_u32(vmull_u16(a.raw, b.raw)); + return Vec128<uint16_t, N>(vget_low_u16(vuzp2q_u16(hi_lo, hi_lo))); +} + +// ------------------------------ Floating-point mul / div + +HWY_NEON_DEF_FUNCTION_ALL_FLOATS(operator*, vmul, _, 2) + +// Approximate reciprocal +HWY_API Vec128<float> ApproximateReciprocal(const Vec128<float> v) { + return Vec128<float>(vrecpeq_f32(v.raw)); +} +template <size_t N> +HWY_API Vec128<float, N> ApproximateReciprocal(const Vec128<float, N> v) { + return Vec128<float, N>(vrecpe_f32(v.raw)); +} + +#if HWY_ARCH_ARM_A64 +HWY_NEON_DEF_FUNCTION_ALL_FLOATS(operator/, vdiv, _, 2) +#else +// Not defined on armv7: approximate +namespace detail { + +HWY_INLINE Vec128<float> ReciprocalNewtonRaphsonStep( + const Vec128<float> recip, const Vec128<float> divisor) { + return Vec128<float>(vrecpsq_f32(recip.raw, divisor.raw)); +} +template <size_t N> +HWY_INLINE Vec128<float, N> ReciprocalNewtonRaphsonStep( + const Vec128<float, N> recip, Vec128<float, N> divisor) { + return Vec128<float, N>(vrecps_f32(recip.raw, divisor.raw)); +} + +} // namespace detail + +template <size_t N> +HWY_API Vec128<float, N> operator/(const Vec128<float, N> a, + const Vec128<float, N> b) { + auto x = ApproximateReciprocal(b); + x *= detail::ReciprocalNewtonRaphsonStep(x, b); + x *= detail::ReciprocalNewtonRaphsonStep(x, b); + x *= detail::ReciprocalNewtonRaphsonStep(x, b); + return a * x; +} +#endif + +// ------------------------------ Absolute value of difference. + +HWY_API Vec128<float> AbsDiff(const Vec128<float> a, const Vec128<float> b) { + return Vec128<float>(vabdq_f32(a.raw, b.raw)); +} +template <size_t N, HWY_IF_LE64(float, N)> +HWY_API Vec128<float, N> AbsDiff(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Vec128<float, N>(vabd_f32(a.raw, b.raw)); +} + +// ------------------------------ Floating-point multiply-add variants + +// Returns add + mul * x +#if defined(__ARM_VFPV4__) || HWY_ARCH_ARM_A64 +template <size_t N, HWY_IF_LE64(float, N)> +HWY_API Vec128<float, N> MulAdd(const Vec128<float, N> mul, + const Vec128<float, N> x, + const Vec128<float, N> add) { + return Vec128<float, N>(vfma_f32(add.raw, mul.raw, x.raw)); +} +HWY_API Vec128<float> MulAdd(const Vec128<float> mul, const Vec128<float> x, + const Vec128<float> add) { + return Vec128<float>(vfmaq_f32(add.raw, mul.raw, x.raw)); +} +#else +// Emulate FMA for floats. +template <size_t N> +HWY_API Vec128<float, N> MulAdd(const Vec128<float, N> mul, + const Vec128<float, N> x, + const Vec128<float, N> add) { + return mul * x + add; +} +#endif + +#if HWY_ARCH_ARM_A64 +HWY_API Vec128<double, 1> MulAdd(const Vec128<double, 1> mul, + const Vec128<double, 1> x, + const Vec128<double, 1> add) { + return Vec128<double, 1>(vfma_f64(add.raw, mul.raw, x.raw)); +} +HWY_API Vec128<double> MulAdd(const Vec128<double> mul, const Vec128<double> x, + const Vec128<double> add) { + return Vec128<double>(vfmaq_f64(add.raw, mul.raw, x.raw)); +} +#endif + +// Returns add - mul * x +#if defined(__ARM_VFPV4__) || HWY_ARCH_ARM_A64 +template <size_t N, HWY_IF_LE64(float, N)> +HWY_API Vec128<float, N> NegMulAdd(const Vec128<float, N> mul, + const Vec128<float, N> x, + const Vec128<float, N> add) { + return Vec128<float, N>(vfms_f32(add.raw, mul.raw, x.raw)); +} +HWY_API Vec128<float> NegMulAdd(const Vec128<float> mul, const Vec128<float> x, + const Vec128<float> add) { + return Vec128<float>(vfmsq_f32(add.raw, mul.raw, x.raw)); +} +#else +// Emulate FMA for floats. +template <size_t N> +HWY_API Vec128<float, N> NegMulAdd(const Vec128<float, N> mul, + const Vec128<float, N> x, + const Vec128<float, N> add) { + return add - mul * x; +} +#endif + +#if HWY_ARCH_ARM_A64 +HWY_API Vec128<double, 1> NegMulAdd(const Vec128<double, 1> mul, + const Vec128<double, 1> x, + const Vec128<double, 1> add) { + return Vec128<double, 1>(vfms_f64(add.raw, mul.raw, x.raw)); +} +HWY_API Vec128<double> NegMulAdd(const Vec128<double> mul, + const Vec128<double> x, + const Vec128<double> add) { + return Vec128<double>(vfmsq_f64(add.raw, mul.raw, x.raw)); +} +#endif + +// Returns mul * x - sub +template <size_t N> +HWY_API Vec128<float, N> MulSub(const Vec128<float, N> mul, + const Vec128<float, N> x, + const Vec128<float, N> sub) { + return MulAdd(mul, x, Neg(sub)); +} + +// Returns -mul * x - sub +template <size_t N> +HWY_API Vec128<float, N> NegMulSub(const Vec128<float, N> mul, + const Vec128<float, N> x, + const Vec128<float, N> sub) { + return Neg(MulAdd(mul, x, sub)); +} + +#if HWY_ARCH_ARM_A64 +template <size_t N> +HWY_API Vec128<double, N> MulSub(const Vec128<double, N> mul, + const Vec128<double, N> x, + const Vec128<double, N> sub) { + return MulAdd(mul, x, Neg(sub)); +} +template <size_t N> +HWY_API Vec128<double, N> NegMulSub(const Vec128<double, N> mul, + const Vec128<double, N> x, + const Vec128<double, N> sub) { + return Neg(MulAdd(mul, x, sub)); +} +#endif + +// ------------------------------ Floating-point square root (IfThenZeroElse) + +// Approximate reciprocal square root +HWY_API Vec128<float> ApproximateReciprocalSqrt(const Vec128<float> v) { + return Vec128<float>(vrsqrteq_f32(v.raw)); +} +template <size_t N> +HWY_API Vec128<float, N> ApproximateReciprocalSqrt(const Vec128<float, N> v) { + return Vec128<float, N>(vrsqrte_f32(v.raw)); +} + +// Full precision square root +#if HWY_ARCH_ARM_A64 +HWY_NEON_DEF_FUNCTION_ALL_FLOATS(Sqrt, vsqrt, _, 1) +#else +namespace detail { + +HWY_INLINE Vec128<float> ReciprocalSqrtStep(const Vec128<float> root, + const Vec128<float> recip) { + return Vec128<float>(vrsqrtsq_f32(root.raw, recip.raw)); +} +template <size_t N> +HWY_INLINE Vec128<float, N> ReciprocalSqrtStep(const Vec128<float, N> root, + Vec128<float, N> recip) { + return Vec128<float, N>(vrsqrts_f32(root.raw, recip.raw)); +} + +} // namespace detail + +// Not defined on armv7: approximate +template <size_t N> +HWY_API Vec128<float, N> Sqrt(const Vec128<float, N> v) { + auto recip = ApproximateReciprocalSqrt(v); + + recip *= detail::ReciprocalSqrtStep(v * recip, recip); + recip *= detail::ReciprocalSqrtStep(v * recip, recip); + recip *= detail::ReciprocalSqrtStep(v * recip, recip); + + const auto root = v * recip; + return IfThenZeroElse(v == Zero(Simd<float, N>()), root); +} +#endif + +// ================================================== LOGICAL + +// ------------------------------ Not + +// There is no 64-bit vmvn, so cast instead of using HWY_NEON_DEF_FUNCTION. +template <typename T> +HWY_API Vec128<T> Not(const Vec128<T> v) { + const Full128<T> d; + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, Vec128<uint8_t>(vmvnq_u8(BitCast(d8, v).raw))); +} +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_API Vec128<T, N> Not(const Vec128<T, N> v) { + const Simd<T, N> d; + const Repartition<uint8_t, decltype(d)> d8; + using V8 = decltype(Zero(d8)); + return BitCast(d, V8(vmvn_u8(BitCast(d8, v).raw))); +} + +// ------------------------------ And +HWY_NEON_DEF_FUNCTION_INTS_UINTS(And, vand, _, 2) + +// Uses the u32/64 defined above. +template <typename T, size_t N, HWY_IF_FLOAT(T)> +HWY_API Vec128<T, N> And(const Vec128<T, N> a, const Vec128<T, N> b) { + const Simd<MakeUnsigned<T>, N> d; + return BitCast(Simd<T, N>(), BitCast(d, a) & BitCast(d, b)); +} + +// ------------------------------ AndNot + +namespace internal { +// reversed_andnot returns a & ~b. +HWY_NEON_DEF_FUNCTION_INTS_UINTS(reversed_andnot, vbic, _, 2) +} // namespace internal + +// Returns ~not_mask & mask. +template <typename T, size_t N, HWY_IF_NOT_FLOAT(T)> +HWY_API Vec128<T, N> AndNot(const Vec128<T, N> not_mask, + const Vec128<T, N> mask) { + return internal::reversed_andnot(mask, not_mask); +} + +// Uses the u32/64 defined above. +template <typename T, size_t N, HWY_IF_FLOAT(T)> +HWY_API Vec128<T, N> AndNot(const Vec128<T, N> not_mask, + const Vec128<T, N> mask) { + const Simd<MakeUnsigned<T>, N> du; + Vec128<MakeUnsigned<T>, N> ret = + internal::reversed_andnot(BitCast(du, mask), BitCast(du, not_mask)); + return BitCast(Simd<T, N>(), ret); +} + +// ------------------------------ Or + +HWY_NEON_DEF_FUNCTION_INTS_UINTS(Or, vorr, _, 2) + +// Uses the u32/64 defined above. +template <typename T, size_t N, HWY_IF_FLOAT(T)> +HWY_API Vec128<T, N> Or(const Vec128<T, N> a, const Vec128<T, N> b) { + const Simd<MakeUnsigned<T>, N> d; + return BitCast(Simd<T, N>(), BitCast(d, a) | BitCast(d, b)); +} + +// ------------------------------ Xor + +HWY_NEON_DEF_FUNCTION_INTS_UINTS(Xor, veor, _, 2) + +// Uses the u32/64 defined above. +template <typename T, size_t N, HWY_IF_FLOAT(T)> +HWY_API Vec128<T, N> Xor(const Vec128<T, N> a, const Vec128<T, N> b) { + const Simd<MakeUnsigned<T>, N> d; + return BitCast(Simd<T, N>(), BitCast(d, a) ^ BitCast(d, b)); +} + +// ------------------------------ Operator overloads (internal-only if float) + +template <typename T, size_t N> +HWY_API Vec128<T, N> operator&(const Vec128<T, N> a, const Vec128<T, N> b) { + return And(a, b); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> operator|(const Vec128<T, N> a, const Vec128<T, N> b) { + return Or(a, b); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> operator^(const Vec128<T, N> a, const Vec128<T, N> b) { + return Xor(a, b); +} + +// ------------------------------ PopulationCount + +#ifdef HWY_NATIVE_POPCNT +#undef HWY_NATIVE_POPCNT +#else +#define HWY_NATIVE_POPCNT +#endif + +namespace detail { + +template <typename T> +HWY_INLINE Vec128<T> PopulationCount(hwy::SizeTag<1> /* tag */, Vec128<T> v) { + const Full128<uint8_t> d8; + return Vec128<T>(vcntq_u8(BitCast(d8, v).raw)); +} +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_INLINE Vec128<T, N> PopulationCount(hwy::SizeTag<1> /* tag */, + Vec128<T, N> v) { + const Simd<uint8_t, N> d8; + return Vec128<T, N>(vcnt_u8(BitCast(d8, v).raw)); +} + +// ARM lacks popcount for lane sizes > 1, so take pairwise sums of the bytes. +template <typename T> +HWY_INLINE Vec128<T> PopulationCount(hwy::SizeTag<2> /* tag */, Vec128<T> v) { + const Full128<uint8_t> d8; + const uint8x16_t bytes = vcntq_u8(BitCast(d8, v).raw); + return Vec128<T>(vpaddlq_u8(bytes)); +} +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_INLINE Vec128<T, N> PopulationCount(hwy::SizeTag<2> /* tag */, + Vec128<T, N> v) { + const Repartition<uint8_t, Simd<T, N>> d8; + const uint8x8_t bytes = vcnt_u8(BitCast(d8, v).raw); + return Vec128<T, N>(vpaddl_u8(bytes)); +} + +template <typename T> +HWY_INLINE Vec128<T> PopulationCount(hwy::SizeTag<4> /* tag */, Vec128<T> v) { + const Full128<uint8_t> d8; + const uint8x16_t bytes = vcntq_u8(BitCast(d8, v).raw); + return Vec128<T>(vpaddlq_u16(vpaddlq_u8(bytes))); +} +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_INLINE Vec128<T, N> PopulationCount(hwy::SizeTag<4> /* tag */, + Vec128<T, N> v) { + const Repartition<uint8_t, Simd<T, N>> d8; + const uint8x8_t bytes = vcnt_u8(BitCast(d8, v).raw); + return Vec128<T, N>(vpaddl_u16(vpaddl_u8(bytes))); +} + +template <typename T> +HWY_INLINE Vec128<T> PopulationCount(hwy::SizeTag<8> /* tag */, Vec128<T> v) { + const Full128<uint8_t> d8; + const uint8x16_t bytes = vcntq_u8(BitCast(d8, v).raw); + return Vec128<T>(vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(bytes)))); +} +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_INLINE Vec128<T, N> PopulationCount(hwy::SizeTag<8> /* tag */, + Vec128<T, N> v) { + const Repartition<uint8_t, Simd<T, N>> d8; + const uint8x8_t bytes = vcnt_u8(BitCast(d8, v).raw); + return Vec128<T, N>(vpaddl_u32(vpaddl_u16(vpaddl_u8(bytes)))); +} + +} // namespace detail + +template <typename T, size_t N, HWY_IF_NOT_FLOAT(T)> +HWY_API Vec128<T, N> PopulationCount(Vec128<T, N> v) { + return detail::PopulationCount(hwy::SizeTag<sizeof(T)>(), v); +} + +// ================================================== SIGN + +// ------------------------------ Abs + +// Returns absolute value, except that LimitsMin() maps to LimitsMax() + 1. +HWY_API Vec128<int8_t> Abs(const Vec128<int8_t> v) { + return Vec128<int8_t>(vabsq_s8(v.raw)); +} +HWY_API Vec128<int16_t> Abs(const Vec128<int16_t> v) { + return Vec128<int16_t>(vabsq_s16(v.raw)); +} +HWY_API Vec128<int32_t> Abs(const Vec128<int32_t> v) { + return Vec128<int32_t>(vabsq_s32(v.raw)); +} +// i64 is implemented after BroadcastSignBit. +HWY_API Vec128<float> Abs(const Vec128<float> v) { + return Vec128<float>(vabsq_f32(v.raw)); +} + +template <size_t N, HWY_IF_LE64(int8_t, N)> +HWY_API Vec128<int8_t, N> Abs(const Vec128<int8_t, N> v) { + return Vec128<int8_t, N>(vabs_s8(v.raw)); +} +template <size_t N, HWY_IF_LE64(int16_t, N)> +HWY_API Vec128<int16_t, N> Abs(const Vec128<int16_t, N> v) { + return Vec128<int16_t, N>(vabs_s16(v.raw)); +} +template <size_t N, HWY_IF_LE64(int32_t, N)> +HWY_API Vec128<int32_t, N> Abs(const Vec128<int32_t, N> v) { + return Vec128<int32_t, N>(vabs_s32(v.raw)); +} +template <size_t N, HWY_IF_LE64(float, N)> +HWY_API Vec128<float, N> Abs(const Vec128<float, N> v) { + return Vec128<float, N>(vabs_f32(v.raw)); +} + +#if HWY_ARCH_ARM_A64 +HWY_API Vec128<double> Abs(const Vec128<double> v) { + return Vec128<double>(vabsq_f64(v.raw)); +} + +HWY_API Vec128<double, 1> Abs(const Vec128<double, 1> v) { + return Vec128<double, 1>(vabs_f64(v.raw)); +} +#endif + +// ------------------------------ CopySign + +template <typename T, size_t N> +HWY_API Vec128<T, N> CopySign(const Vec128<T, N> magn, + const Vec128<T, N> sign) { + static_assert(IsFloat<T>(), "Only makes sense for floating-point"); + const auto msb = SignBit(Simd<T, N>()); + return Or(AndNot(msb, magn), And(msb, sign)); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> CopySignToAbs(const Vec128<T, N> abs, + const Vec128<T, N> sign) { + static_assert(IsFloat<T>(), "Only makes sense for floating-point"); + return Or(abs, And(SignBit(Simd<T, N>()), sign)); +} + +// ------------------------------ BroadcastSignBit + +template <typename T, size_t N, HWY_IF_SIGNED(T)> +HWY_API Vec128<T, N> BroadcastSignBit(const Vec128<T, N> v) { + return ShiftRight<sizeof(T) * 8 - 1>(v); +} + +// ================================================== MASK + +// ------------------------------ To/from vector + +// Mask and Vec have the same representation (true = FF..FF). +template <typename T, size_t N> +HWY_API Mask128<T, N> MaskFromVec(const Vec128<T, N> v) { + const Simd<MakeUnsigned<T>, N> du; + return Mask128<T, N>(BitCast(du, v).raw); +} + +// DEPRECATED +template <typename T, size_t N> +HWY_API Vec128<T, N> VecFromMask(const Mask128<T, N> v) { + return BitCast(Simd<T, N>(), Vec128<MakeUnsigned<T>, N>(v.raw)); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> VecFromMask(Simd<T, N> d, const Mask128<T, N> v) { + return BitCast(d, Vec128<MakeUnsigned<T>, N>(v.raw)); +} + +// ------------------------------ RebindMask + +template <typename TFrom, typename TTo, size_t N> +HWY_API Mask128<TTo, N> RebindMask(Simd<TTo, N> dto, Mask128<TFrom, N> m) { + static_assert(sizeof(TFrom) == sizeof(TTo), "Must have same size"); + return MaskFromVec(BitCast(dto, VecFromMask(Simd<TFrom, N>(), m))); +} + +// ------------------------------ IfThenElse(mask, yes, no) = mask ? b : a. + +#define HWY_NEON_BUILD_TPL_HWY_IF +#define HWY_NEON_BUILD_RET_HWY_IF(type, size) Vec128<type, size> +#define HWY_NEON_BUILD_PARAM_HWY_IF(type, size) \ + const Mask128<type, size> mask, const Vec128<type, size> yes, \ + const Vec128<type, size> no +#define HWY_NEON_BUILD_ARG_HWY_IF mask.raw, yes.raw, no.raw + +HWY_NEON_DEF_FUNCTION_ALL_TYPES(IfThenElse, vbsl, _, HWY_IF) + +#undef HWY_NEON_BUILD_TPL_HWY_IF +#undef HWY_NEON_BUILD_RET_HWY_IF +#undef HWY_NEON_BUILD_PARAM_HWY_IF +#undef HWY_NEON_BUILD_ARG_HWY_IF + +// mask ? yes : 0 +template <typename T, size_t N> +HWY_API Vec128<T, N> IfThenElseZero(const Mask128<T, N> mask, + const Vec128<T, N> yes) { + return yes & VecFromMask(Simd<T, N>(), mask); +} + +// mask ? 0 : no +template <typename T, size_t N> +HWY_API Vec128<T, N> IfThenZeroElse(const Mask128<T, N> mask, + const Vec128<T, N> no) { + return AndNot(VecFromMask(Simd<T, N>(), mask), no); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> ZeroIfNegative(Vec128<T, N> v) { + const auto zero = Zero(Simd<T, N>()); + return Max(zero, v); +} + +// ------------------------------ Mask logical + +template <typename T, size_t N> +HWY_API Mask128<T, N> Not(const Mask128<T, N> m) { + return MaskFromVec(Not(VecFromMask(Simd<T, N>(), m))); +} + +template <typename T, size_t N> +HWY_API Mask128<T, N> And(const Mask128<T, N> a, Mask128<T, N> b) { + const Simd<T, N> d; + return MaskFromVec(And(VecFromMask(d, a), VecFromMask(d, b))); +} + +template <typename T, size_t N> +HWY_API Mask128<T, N> AndNot(const Mask128<T, N> a, Mask128<T, N> b) { + const Simd<T, N> d; + return MaskFromVec(AndNot(VecFromMask(d, a), VecFromMask(d, b))); +} + +template <typename T, size_t N> +HWY_API Mask128<T, N> Or(const Mask128<T, N> a, Mask128<T, N> b) { + const Simd<T, N> d; + return MaskFromVec(Or(VecFromMask(d, a), VecFromMask(d, b))); +} + +template <typename T, size_t N> +HWY_API Mask128<T, N> Xor(const Mask128<T, N> a, Mask128<T, N> b) { + const Simd<T, N> d; + return MaskFromVec(Xor(VecFromMask(d, a), VecFromMask(d, b))); +} + +// ================================================== COMPARE + +// Comparisons fill a lane with 1-bits if the condition is true, else 0. + +// ------------------------------ Shuffle2301 (for i64 compares) + +// Swap 32-bit halves in 64-bits +HWY_API Vec128<uint32_t, 2> Shuffle2301(const Vec128<uint32_t, 2> v) { + return Vec128<uint32_t, 2>(vrev64_u32(v.raw)); +} +HWY_API Vec128<int32_t, 2> Shuffle2301(const Vec128<int32_t, 2> v) { + return Vec128<int32_t, 2>(vrev64_s32(v.raw)); +} +HWY_API Vec128<float, 2> Shuffle2301(const Vec128<float, 2> v) { + return Vec128<float, 2>(vrev64_f32(v.raw)); +} +HWY_API Vec128<uint32_t> Shuffle2301(const Vec128<uint32_t> v) { + return Vec128<uint32_t>(vrev64q_u32(v.raw)); +} +HWY_API Vec128<int32_t> Shuffle2301(const Vec128<int32_t> v) { + return Vec128<int32_t>(vrev64q_s32(v.raw)); +} +HWY_API Vec128<float> Shuffle2301(const Vec128<float> v) { + return Vec128<float>(vrev64q_f32(v.raw)); +} + +#define HWY_NEON_BUILD_TPL_HWY_COMPARE +#define HWY_NEON_BUILD_RET_HWY_COMPARE(type, size) Mask128<type, size> +#define HWY_NEON_BUILD_PARAM_HWY_COMPARE(type, size) \ + const Vec128<type, size> a, const Vec128<type, size> b +#define HWY_NEON_BUILD_ARG_HWY_COMPARE a.raw, b.raw + +// ------------------------------ Equality +HWY_NEON_DEF_FUNCTION_ALL_FLOATS(operator==, vceq, _, HWY_COMPARE) +#if HWY_ARCH_ARM_A64 +HWY_NEON_DEF_FUNCTION_INTS_UINTS(operator==, vceq, _, HWY_COMPARE) +#else +// No 64-bit comparisons on armv7: emulate them below, after Shuffle2301. +HWY_NEON_DEF_FUNCTION_INT_8_16_32(operator==, vceq, _, HWY_COMPARE) +HWY_NEON_DEF_FUNCTION_UINT_8_16_32(operator==, vceq, _, HWY_COMPARE) +#endif + +// ------------------------------ Inequality +template <typename T, size_t N> +HWY_API Mask128<T, N> operator!=(const Vec128<T, N> a, const Vec128<T, N> b) { + return Not(a == b); +} + +// ------------------------------ Strict inequality (signed, float) +#if HWY_ARCH_ARM_A64 +HWY_NEON_DEF_FUNCTION_INTS_UINTS(operator<, vclt, _, HWY_COMPARE) +#else +HWY_NEON_DEF_FUNCTION_UINT_8_16_32(operator<, vclt, _, HWY_COMPARE) +HWY_NEON_DEF_FUNCTION_INT_8_16_32(operator<, vclt, _, HWY_COMPARE) +#endif +HWY_NEON_DEF_FUNCTION_ALL_FLOATS(operator<, vclt, _, HWY_COMPARE) + +// ------------------------------ Weak inequality (float) +HWY_NEON_DEF_FUNCTION_ALL_FLOATS(operator<=, vcle, _, HWY_COMPARE) + +#undef HWY_NEON_BUILD_TPL_HWY_COMPARE +#undef HWY_NEON_BUILD_RET_HWY_COMPARE +#undef HWY_NEON_BUILD_PARAM_HWY_COMPARE +#undef HWY_NEON_BUILD_ARG_HWY_COMPARE + +// ------------------------------ ARMv7 i64 compare (Shuffle2301, Eq) + +#if HWY_ARCH_ARM_V7 + +template <size_t N> +HWY_API Mask128<int64_t, N> operator==(const Vec128<int64_t, N> a, + const Vec128<int64_t, N> b) { + const Simd<int32_t, N * 2> d32; + const Simd<int64_t, N> d64; + const auto cmp32 = VecFromMask(d32, Eq(BitCast(d32, a), BitCast(d32, b))); + const auto cmp64 = cmp32 & Shuffle2301(cmp32); + return MaskFromVec(BitCast(d64, cmp64)); +} + +template <size_t N> +HWY_API Mask128<uint64_t, N> operator==(const Vec128<uint64_t, N> a, + const Vec128<uint64_t, N> b) { + const Simd<uint32_t, N * 2> d32; + const Simd<uint64_t, N> d64; + const auto cmp32 = VecFromMask(d32, Eq(BitCast(d32, a), BitCast(d32, b))); + const auto cmp64 = cmp32 & Shuffle2301(cmp32); + return MaskFromVec(BitCast(d64, cmp64)); +} + +HWY_API Mask128<int64_t> operator<(const Vec128<int64_t> a, + const Vec128<int64_t> b) { + const int64x2_t sub = vqsubq_s64(a.raw, b.raw); + return MaskFromVec(BroadcastSignBit(Vec128<int64_t>(sub))); +} +HWY_API Mask128<int64_t, 1> operator<(const Vec128<int64_t, 1> a, + const Vec128<int64_t, 1> b) { + const int64x1_t sub = vqsub_s64(a.raw, b.raw); + return MaskFromVec(BroadcastSignBit(Vec128<int64_t, 1>(sub))); +} + +template <size_t N> +HWY_API Mask128<uint64_t, N> operator<(const Vec128<uint64_t, N> a, + const Vec128<uint64_t, N> b) { + const Simd<int64_t, N> di; + const Simd<uint64_t, N> du; + const Vec128<uint64_t, N> msb = AndNot(a, b) | AndNot(a ^ b, a - b); + return MaskFromVec(BitCast(du, BroadcastSignBit(BitCast(di, msb)))); +} + +#endif + +// ------------------------------ Reversed comparisons + +template <typename T, size_t N> +HWY_API Mask128<T, N> operator>(Vec128<T, N> a, Vec128<T, N> b) { + return operator<(b, a); +} +template <typename T, size_t N> +HWY_API Mask128<T, N> operator>=(Vec128<T, N> a, Vec128<T, N> b) { + return operator<=(b, a); +} + +// ------------------------------ FirstN (Iota, Lt) + +template <typename T, size_t N> +HWY_API Mask128<T, N> FirstN(const Simd<T, N> d, size_t num) { + const RebindToSigned<decltype(d)> di; // Signed comparisons are cheaper. + return RebindMask(d, Iota(di, 0) < Set(di, static_cast<MakeSigned<T>>(num))); +} + +// ------------------------------ TestBit (Eq) + +#define HWY_NEON_BUILD_TPL_HWY_TESTBIT +#define HWY_NEON_BUILD_RET_HWY_TESTBIT(type, size) Mask128<type, size> +#define HWY_NEON_BUILD_PARAM_HWY_TESTBIT(type, size) \ + Vec128<type, size> v, Vec128<type, size> bit +#define HWY_NEON_BUILD_ARG_HWY_TESTBIT v.raw, bit.raw + +#if HWY_ARCH_ARM_A64 +HWY_NEON_DEF_FUNCTION_INTS_UINTS(TestBit, vtst, _, HWY_TESTBIT) +#else +// No 64-bit versions on armv7 +HWY_NEON_DEF_FUNCTION_UINT_8_16_32(TestBit, vtst, _, HWY_TESTBIT) +HWY_NEON_DEF_FUNCTION_INT_8_16_32(TestBit, vtst, _, HWY_TESTBIT) + +template <size_t N> +HWY_API Mask128<uint64_t, N> TestBit(Vec128<uint64_t, N> v, + Vec128<uint64_t, N> bit) { + return (v & bit) == bit; +} +template <size_t N> +HWY_API Mask128<int64_t, N> TestBit(Vec128<int64_t, N> v, + Vec128<int64_t, N> bit) { + return (v & bit) == bit; +} + +#endif +#undef HWY_NEON_BUILD_TPL_HWY_TESTBIT +#undef HWY_NEON_BUILD_RET_HWY_TESTBIT +#undef HWY_NEON_BUILD_PARAM_HWY_TESTBIT +#undef HWY_NEON_BUILD_ARG_HWY_TESTBIT + +// ------------------------------ Abs i64 (IfThenElse, BroadcastSignBit) +HWY_API Vec128<int64_t> Abs(const Vec128<int64_t> v) { +#if HWY_ARCH_ARM_A64 + return Vec128<int64_t>(vabsq_s64(v.raw)); +#else + const auto zero = Zero(Full128<int64_t>()); + return IfThenElse(MaskFromVec(BroadcastSignBit(v)), zero - v, v); +#endif +} +HWY_API Vec128<int64_t, 1> Abs(const Vec128<int64_t, 1> v) { +#if HWY_ARCH_ARM_A64 + return Vec128<int64_t, 1>(vabs_s64(v.raw)); +#else + const auto zero = Zero(Simd<int64_t, 1>()); + return IfThenElse(MaskFromVec(BroadcastSignBit(v)), zero - v, v); +#endif +} + +// ------------------------------ Min (IfThenElse, BroadcastSignBit) + +// Unsigned +HWY_NEON_DEF_FUNCTION_UINT_8_16_32(Min, vmin, _, 2) + +template <size_t N> +HWY_API Vec128<uint64_t, N> Min(const Vec128<uint64_t, N> a, + const Vec128<uint64_t, N> b) { +#if HWY_ARCH_ARM_A64 + return IfThenElse(b < a, b, a); +#else + const Simd<uint64_t, N> du; + const Simd<int64_t, N> di; + return BitCast(du, BitCast(di, a) - BitCast(di, detail::SaturatedSub(a, b))); +#endif +} + +// Signed +HWY_NEON_DEF_FUNCTION_INT_8_16_32(Min, vmin, _, 2) + +template <size_t N> +HWY_API Vec128<int64_t, N> Min(const Vec128<int64_t, N> a, + const Vec128<int64_t, N> b) { +#if HWY_ARCH_ARM_A64 + return IfThenElse(b < a, b, a); +#else + const Vec128<int64_t, N> sign = detail::SaturatedSub(a, b); + return IfThenElse(MaskFromVec(BroadcastSignBit(sign)), a, b); +#endif +} + +// Float: IEEE minimumNumber on v8, otherwise NaN if any is NaN. +#if HWY_ARCH_ARM_A64 +HWY_NEON_DEF_FUNCTION_ALL_FLOATS(Min, vminnm, _, 2) +#else +HWY_NEON_DEF_FUNCTION_ALL_FLOATS(Min, vmin, _, 2) +#endif + +// ------------------------------ Max (IfThenElse, BroadcastSignBit) + +// Unsigned (no u64) +HWY_NEON_DEF_FUNCTION_UINT_8_16_32(Max, vmax, _, 2) + +template <size_t N> +HWY_API Vec128<uint64_t, N> Max(const Vec128<uint64_t, N> a, + const Vec128<uint64_t, N> b) { +#if HWY_ARCH_ARM_A64 + return IfThenElse(b < a, a, b); +#else + const Simd<uint64_t, N> du; + const Simd<int64_t, N> di; + return BitCast(du, BitCast(di, b) + BitCast(di, detail::SaturatedSub(a, b))); +#endif +} + +// Signed (no i64) +HWY_NEON_DEF_FUNCTION_INT_8_16_32(Max, vmax, _, 2) + +template <size_t N> +HWY_API Vec128<int64_t, N> Max(const Vec128<int64_t, N> a, + const Vec128<int64_t, N> b) { +#if HWY_ARCH_ARM_A64 + return IfThenElse(b < a, a, b); +#else + const Vec128<int64_t, N> sign = detail::SaturatedSub(a, b); + return IfThenElse(MaskFromVec(BroadcastSignBit(sign)), b, a); +#endif +} + +// Float: IEEE maximumNumber on v8, otherwise NaN if any is NaN. +#if HWY_ARCH_ARM_A64 +HWY_NEON_DEF_FUNCTION_ALL_FLOATS(Max, vmaxnm, _, 2) +#else +HWY_NEON_DEF_FUNCTION_ALL_FLOATS(Max, vmax, _, 2) +#endif + +// ================================================== MEMORY + +// ------------------------------ Load 128 + +HWY_API Vec128<uint8_t> LoadU(Full128<uint8_t> /* tag */, + const uint8_t* HWY_RESTRICT unaligned) { + return Vec128<uint8_t>(vld1q_u8(unaligned)); +} +HWY_API Vec128<uint16_t> LoadU(Full128<uint16_t> /* tag */, + const uint16_t* HWY_RESTRICT unaligned) { + return Vec128<uint16_t>(vld1q_u16(unaligned)); +} +HWY_API Vec128<uint32_t> LoadU(Full128<uint32_t> /* tag */, + const uint32_t* HWY_RESTRICT unaligned) { + return Vec128<uint32_t>(vld1q_u32(unaligned)); +} +HWY_API Vec128<uint64_t> LoadU(Full128<uint64_t> /* tag */, + const uint64_t* HWY_RESTRICT unaligned) { + return Vec128<uint64_t>(vld1q_u64(unaligned)); +} +HWY_API Vec128<int8_t> LoadU(Full128<int8_t> /* tag */, + const int8_t* HWY_RESTRICT unaligned) { + return Vec128<int8_t>(vld1q_s8(unaligned)); +} +HWY_API Vec128<int16_t> LoadU(Full128<int16_t> /* tag */, + const int16_t* HWY_RESTRICT unaligned) { + return Vec128<int16_t>(vld1q_s16(unaligned)); +} +HWY_API Vec128<int32_t> LoadU(Full128<int32_t> /* tag */, + const int32_t* HWY_RESTRICT unaligned) { + return Vec128<int32_t>(vld1q_s32(unaligned)); +} +HWY_API Vec128<int64_t> LoadU(Full128<int64_t> /* tag */, + const int64_t* HWY_RESTRICT unaligned) { + return Vec128<int64_t>(vld1q_s64(unaligned)); +} +HWY_API Vec128<float> LoadU(Full128<float> /* tag */, + const float* HWY_RESTRICT unaligned) { + return Vec128<float>(vld1q_f32(unaligned)); +} +#if HWY_ARCH_ARM_A64 +HWY_API Vec128<double> LoadU(Full128<double> /* tag */, + const double* HWY_RESTRICT unaligned) { + return Vec128<double>(vld1q_f64(unaligned)); +} +#endif + +// ------------------------------ Load 64 + +HWY_API Vec128<uint8_t, 8> LoadU(Simd<uint8_t, 8> /* tag */, + const uint8_t* HWY_RESTRICT p) { + return Vec128<uint8_t, 8>(vld1_u8(p)); +} +HWY_API Vec128<uint16_t, 4> LoadU(Simd<uint16_t, 4> /* tag */, + const uint16_t* HWY_RESTRICT p) { + return Vec128<uint16_t, 4>(vld1_u16(p)); +} +HWY_API Vec128<uint32_t, 2> LoadU(Simd<uint32_t, 2> /* tag */, + const uint32_t* HWY_RESTRICT p) { + return Vec128<uint32_t, 2>(vld1_u32(p)); +} +HWY_API Vec128<uint64_t, 1> LoadU(Simd<uint64_t, 1> /* tag */, + const uint64_t* HWY_RESTRICT p) { + return Vec128<uint64_t, 1>(vld1_u64(p)); +} +HWY_API Vec128<int8_t, 8> LoadU(Simd<int8_t, 8> /* tag */, + const int8_t* HWY_RESTRICT p) { + return Vec128<int8_t, 8>(vld1_s8(p)); +} +HWY_API Vec128<int16_t, 4> LoadU(Simd<int16_t, 4> /* tag */, + const int16_t* HWY_RESTRICT p) { + return Vec128<int16_t, 4>(vld1_s16(p)); +} +HWY_API Vec128<int32_t, 2> LoadU(Simd<int32_t, 2> /* tag */, + const int32_t* HWY_RESTRICT p) { + return Vec128<int32_t, 2>(vld1_s32(p)); +} +HWY_API Vec128<int64_t, 1> LoadU(Simd<int64_t, 1> /* tag */, + const int64_t* HWY_RESTRICT p) { + return Vec128<int64_t, 1>(vld1_s64(p)); +} +HWY_API Vec128<float, 2> LoadU(Simd<float, 2> /* tag */, + const float* HWY_RESTRICT p) { + return Vec128<float, 2>(vld1_f32(p)); +} +#if HWY_ARCH_ARM_A64 +HWY_API Vec128<double, 1> LoadU(Simd<double, 1> /* tag */, + const double* HWY_RESTRICT p) { + return Vec128<double, 1>(vld1_f64(p)); +} +#endif + +// ------------------------------ Load 32 + +// In the following load functions, |a| is purposely undefined. +// It is a required parameter to the intrinsic, however +// we don't actually care what is in it, and we don't want +// to introduce extra overhead by initializing it to something. + +HWY_API Vec128<uint8_t, 4> LoadU(Simd<uint8_t, 4> /*tag*/, + const uint8_t* HWY_RESTRICT p) { + uint32x2_t a = Undefined(Simd<uint32_t, 2>()).raw; + uint32x2_t b = vld1_lane_u32(reinterpret_cast<const uint32_t*>(p), a, 0); + return Vec128<uint8_t, 4>(vreinterpret_u8_u32(b)); +} +HWY_API Vec128<uint16_t, 2> LoadU(Simd<uint16_t, 2> /*tag*/, + const uint16_t* HWY_RESTRICT p) { + uint32x2_t a = Undefined(Simd<uint32_t, 2>()).raw; + uint32x2_t b = vld1_lane_u32(reinterpret_cast<const uint32_t*>(p), a, 0); + return Vec128<uint16_t, 2>(vreinterpret_u16_u32(b)); +} +HWY_API Vec128<uint32_t, 1> LoadU(Simd<uint32_t, 1> /*tag*/, + const uint32_t* HWY_RESTRICT p) { + uint32x2_t a = Undefined(Simd<uint32_t, 2>()).raw; + uint32x2_t b = vld1_lane_u32(p, a, 0); + return Vec128<uint32_t, 1>(b); +} +HWY_API Vec128<int8_t, 4> LoadU(Simd<int8_t, 4> /*tag*/, + const int8_t* HWY_RESTRICT p) { + int32x2_t a = Undefined(Simd<int32_t, 2>()).raw; + int32x2_t b = vld1_lane_s32(reinterpret_cast<const int32_t*>(p), a, 0); + return Vec128<int8_t, 4>(vreinterpret_s8_s32(b)); +} +HWY_API Vec128<int16_t, 2> LoadU(Simd<int16_t, 2> /*tag*/, + const int16_t* HWY_RESTRICT p) { + int32x2_t a = Undefined(Simd<int32_t, 2>()).raw; + int32x2_t b = vld1_lane_s32(reinterpret_cast<const int32_t*>(p), a, 0); + return Vec128<int16_t, 2>(vreinterpret_s16_s32(b)); +} +HWY_API Vec128<int32_t, 1> LoadU(Simd<int32_t, 1> /*tag*/, + const int32_t* HWY_RESTRICT p) { + int32x2_t a = Undefined(Simd<int32_t, 2>()).raw; + int32x2_t b = vld1_lane_s32(p, a, 0); + return Vec128<int32_t, 1>(b); +} +HWY_API Vec128<float, 1> LoadU(Simd<float, 1> /*tag*/, + const float* HWY_RESTRICT p) { + float32x2_t a = Undefined(Simd<float, 2>()).raw; + float32x2_t b = vld1_lane_f32(p, a, 0); + return Vec128<float, 1>(b); +} + +// ------------------------------ Load 16 + +HWY_API Vec128<uint8_t, 2> LoadU(Simd<uint8_t, 2> /*tag*/, + const uint8_t* HWY_RESTRICT p) { + uint16x4_t a = Undefined(Simd<uint16_t, 4>()).raw; + uint16x4_t b = vld1_lane_u16(reinterpret_cast<const uint16_t*>(p), a, 0); + return Vec128<uint8_t, 2>(vreinterpret_u8_u16(b)); +} +HWY_API Vec128<uint16_t, 1> LoadU(Simd<uint16_t, 1> /*tag*/, + const uint16_t* HWY_RESTRICT p) { + uint16x4_t a = Undefined(Simd<uint16_t, 4>()).raw; + uint16x4_t b = vld1_lane_u16(p, a, 0); + return Vec128<uint16_t, 1>(b); +} +HWY_API Vec128<int8_t, 2> LoadU(Simd<int8_t, 2> /*tag*/, + const int8_t* HWY_RESTRICT p) { + int16x4_t a = Undefined(Simd<int16_t, 4>()).raw; + int16x4_t b = vld1_lane_s16(reinterpret_cast<const int16_t*>(p), a, 0); + return Vec128<int8_t, 2>(vreinterpret_s8_s16(b)); +} +HWY_API Vec128<int16_t, 1> LoadU(Simd<int16_t, 1> /*tag*/, + const int16_t* HWY_RESTRICT p) { + int16x4_t a = Undefined(Simd<int16_t, 4>()).raw; + int16x4_t b = vld1_lane_s16(p, a, 0); + return Vec128<int16_t, 1>(b); +} + +// ------------------------------ Load 8 + +HWY_API Vec128<uint8_t, 1> LoadU(Simd<uint8_t, 1> d, + const uint8_t* HWY_RESTRICT p) { + uint8x8_t a = Undefined(d).raw; + uint8x8_t b = vld1_lane_u8(p, a, 0); + return Vec128<uint8_t, 1>(b); +} + +HWY_API Vec128<int8_t, 1> LoadU(Simd<int8_t, 1> d, + const int8_t* HWY_RESTRICT p) { + int8x8_t a = Undefined(d).raw; + int8x8_t b = vld1_lane_s8(p, a, 0); + return Vec128<int8_t, 1>(b); +} + +// [b]float16_t use the same Raw as uint16_t, so forward to that. +template <size_t N> +HWY_API Vec128<float16_t, N> LoadU(Simd<float16_t, N> /*d*/, + const float16_t* HWY_RESTRICT p) { + const Simd<uint16_t, N> du16; + const auto pu16 = reinterpret_cast<const uint16_t*>(p); + return Vec128<float16_t, N>(LoadU(du16, pu16).raw); +} +template <size_t N> +HWY_API Vec128<bfloat16_t, N> LoadU(Simd<bfloat16_t, N> /*d*/, + const bfloat16_t* HWY_RESTRICT p) { + const Simd<uint16_t, N> du16; + const auto pu16 = reinterpret_cast<const uint16_t*>(p); + return Vec128<bfloat16_t, N>(LoadU(du16, pu16).raw); +} + +// On ARM, Load is the same as LoadU. +template <typename T, size_t N> +HWY_API Vec128<T, N> Load(Simd<T, N> d, const T* HWY_RESTRICT p) { + return LoadU(d, p); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> MaskedLoad(Mask128<T, N> m, Simd<T, N> d, + const T* HWY_RESTRICT aligned) { + return IfThenElseZero(m, Load(d, aligned)); +} + +// 128-bit SIMD => nothing to duplicate, same as an unaligned load. +template <typename T, size_t N, HWY_IF_LE128(T, N)> +HWY_API Vec128<T, N> LoadDup128(Simd<T, N> d, const T* const HWY_RESTRICT p) { + return LoadU(d, p); +} + +// ------------------------------ Store 128 + +HWY_API void StoreU(const Vec128<uint8_t> v, Full128<uint8_t> /* tag */, + uint8_t* HWY_RESTRICT unaligned) { + vst1q_u8(unaligned, v.raw); +} +HWY_API void StoreU(const Vec128<uint16_t> v, Full128<uint16_t> /* tag */, + uint16_t* HWY_RESTRICT unaligned) { + vst1q_u16(unaligned, v.raw); +} +HWY_API void StoreU(const Vec128<uint32_t> v, Full128<uint32_t> /* tag */, + uint32_t* HWY_RESTRICT unaligned) { + vst1q_u32(unaligned, v.raw); +} +HWY_API void StoreU(const Vec128<uint64_t> v, Full128<uint64_t> /* tag */, + uint64_t* HWY_RESTRICT unaligned) { + vst1q_u64(unaligned, v.raw); +} +HWY_API void StoreU(const Vec128<int8_t> v, Full128<int8_t> /* tag */, + int8_t* HWY_RESTRICT unaligned) { + vst1q_s8(unaligned, v.raw); +} +HWY_API void StoreU(const Vec128<int16_t> v, Full128<int16_t> /* tag */, + int16_t* HWY_RESTRICT unaligned) { + vst1q_s16(unaligned, v.raw); +} +HWY_API void StoreU(const Vec128<int32_t> v, Full128<int32_t> /* tag */, + int32_t* HWY_RESTRICT unaligned) { + vst1q_s32(unaligned, v.raw); +} +HWY_API void StoreU(const Vec128<int64_t> v, Full128<int64_t> /* tag */, + int64_t* HWY_RESTRICT unaligned) { + vst1q_s64(unaligned, v.raw); +} +HWY_API void StoreU(const Vec128<float> v, Full128<float> /* tag */, + float* HWY_RESTRICT unaligned) { + vst1q_f32(unaligned, v.raw); +} +#if HWY_ARCH_ARM_A64 +HWY_API void StoreU(const Vec128<double> v, Full128<double> /* tag */, + double* HWY_RESTRICT unaligned) { + vst1q_f64(unaligned, v.raw); +} +#endif + +// ------------------------------ Store 64 + +HWY_API void StoreU(const Vec128<uint8_t, 8> v, Simd<uint8_t, 8> /* tag */, + uint8_t* HWY_RESTRICT p) { + vst1_u8(p, v.raw); +} +HWY_API void StoreU(const Vec128<uint16_t, 4> v, Simd<uint16_t, 4> /* tag */, + uint16_t* HWY_RESTRICT p) { + vst1_u16(p, v.raw); +} +HWY_API void StoreU(const Vec128<uint32_t, 2> v, Simd<uint32_t, 2> /* tag */, + uint32_t* HWY_RESTRICT p) { + vst1_u32(p, v.raw); +} +HWY_API void StoreU(const Vec128<uint64_t, 1> v, Simd<uint64_t, 1> /* tag */, + uint64_t* HWY_RESTRICT p) { + vst1_u64(p, v.raw); +} +HWY_API void StoreU(const Vec128<int8_t, 8> v, Simd<int8_t, 8> /* tag */, + int8_t* HWY_RESTRICT p) { + vst1_s8(p, v.raw); +} +HWY_API void StoreU(const Vec128<int16_t, 4> v, Simd<int16_t, 4> /* tag */, + int16_t* HWY_RESTRICT p) { + vst1_s16(p, v.raw); +} +HWY_API void StoreU(const Vec128<int32_t, 2> v, Simd<int32_t, 2> /* tag */, + int32_t* HWY_RESTRICT p) { + vst1_s32(p, v.raw); +} +HWY_API void StoreU(const Vec128<int64_t, 1> v, Simd<int64_t, 1> /* tag */, + int64_t* HWY_RESTRICT p) { + vst1_s64(p, v.raw); +} +HWY_API void StoreU(const Vec128<float, 2> v, Simd<float, 2> /* tag */, + float* HWY_RESTRICT p) { + vst1_f32(p, v.raw); +} +#if HWY_ARCH_ARM_A64 +HWY_API void StoreU(const Vec128<double, 1> v, Simd<double, 1> /* tag */, + double* HWY_RESTRICT p) { + vst1_f64(p, v.raw); +} +#endif + +// ------------------------------ Store 32 + +HWY_API void StoreU(const Vec128<uint8_t, 4> v, Simd<uint8_t, 4>, + uint8_t* HWY_RESTRICT p) { + uint32x2_t a = vreinterpret_u32_u8(v.raw); + vst1_lane_u32(reinterpret_cast<uint32_t*>(p), a, 0); +} +HWY_API void StoreU(const Vec128<uint16_t, 2> v, Simd<uint16_t, 2>, + uint16_t* HWY_RESTRICT p) { + uint32x2_t a = vreinterpret_u32_u16(v.raw); + vst1_lane_u32(reinterpret_cast<uint32_t*>(p), a, 0); +} +HWY_API void StoreU(const Vec128<uint32_t, 1> v, Simd<uint32_t, 1>, + uint32_t* HWY_RESTRICT p) { + vst1_lane_u32(p, v.raw, 0); +} +HWY_API void StoreU(const Vec128<int8_t, 4> v, Simd<int8_t, 4>, + int8_t* HWY_RESTRICT p) { + int32x2_t a = vreinterpret_s32_s8(v.raw); + vst1_lane_s32(reinterpret_cast<int32_t*>(p), a, 0); +} +HWY_API void StoreU(const Vec128<int16_t, 2> v, Simd<int16_t, 2>, + int16_t* HWY_RESTRICT p) { + int32x2_t a = vreinterpret_s32_s16(v.raw); + vst1_lane_s32(reinterpret_cast<int32_t*>(p), a, 0); +} +HWY_API void StoreU(const Vec128<int32_t, 1> v, Simd<int32_t, 1>, + int32_t* HWY_RESTRICT p) { + vst1_lane_s32(p, v.raw, 0); +} +HWY_API void StoreU(const Vec128<float, 1> v, Simd<float, 1>, + float* HWY_RESTRICT p) { + vst1_lane_f32(p, v.raw, 0); +} + +// ------------------------------ Store 16 + +HWY_API void StoreU(const Vec128<uint8_t, 2> v, Simd<uint8_t, 2>, + uint8_t* HWY_RESTRICT p) { + uint16x4_t a = vreinterpret_u16_u8(v.raw); + vst1_lane_u16(reinterpret_cast<uint16_t*>(p), a, 0); +} +HWY_API void StoreU(const Vec128<uint16_t, 1> v, Simd<uint16_t, 1>, + uint16_t* HWY_RESTRICT p) { + vst1_lane_u16(p, v.raw, 0); +} +HWY_API void StoreU(const Vec128<int8_t, 2> v, Simd<int8_t, 2>, + int8_t* HWY_RESTRICT p) { + int16x4_t a = vreinterpret_s16_s8(v.raw); + vst1_lane_s16(reinterpret_cast<int16_t*>(p), a, 0); +} +HWY_API void StoreU(const Vec128<int16_t, 1> v, Simd<int16_t, 1>, + int16_t* HWY_RESTRICT p) { + vst1_lane_s16(p, v.raw, 0); +} + +// ------------------------------ Store 8 + +HWY_API void StoreU(const Vec128<uint8_t, 1> v, Simd<uint8_t, 1>, + uint8_t* HWY_RESTRICT p) { + vst1_lane_u8(p, v.raw, 0); +} +HWY_API void StoreU(const Vec128<int8_t, 1> v, Simd<int8_t, 1>, + int8_t* HWY_RESTRICT p) { + vst1_lane_s8(p, v.raw, 0); +} + +// [b]float16_t use the same Raw as uint16_t, so forward to that. +template <size_t N> +HWY_API void StoreU(Vec128<float16_t, N> v, Simd<float16_t, N> /* tag */, + float16_t* HWY_RESTRICT p) { + const Simd<uint16_t, N> du16; + const auto pu16 = reinterpret_cast<uint16_t*>(p); + return StoreU(Vec128<uint16_t, N>(v.raw), du16, pu16); +} +template <size_t N> +HWY_API void StoreU(Vec128<bfloat16_t, N> v, Simd<bfloat16_t, N> /* tag */, + bfloat16_t* HWY_RESTRICT p) { + const Simd<uint16_t, N> du16; + const auto pu16 = reinterpret_cast<uint16_t*>(p); + return StoreU(Vec128<uint16_t, N>(v.raw), du16, pu16); +} + +// On ARM, Store is the same as StoreU. +template <typename T, size_t N> +HWY_API void Store(Vec128<T, N> v, Simd<T, N> d, T* HWY_RESTRICT aligned) { + StoreU(v, d, aligned); +} + +// ------------------------------ Non-temporal stores + +// Same as aligned stores on non-x86. + +template <typename T, size_t N> +HWY_API void Stream(const Vec128<T, N> v, Simd<T, N> d, + T* HWY_RESTRICT aligned) { + Store(v, d, aligned); +} + +// ================================================== CONVERT + +// ------------------------------ Promotions (part w/ narrow lanes -> full) + +// Unsigned: zero-extend to full vector. +HWY_API Vec128<uint16_t> PromoteTo(Full128<uint16_t> /* tag */, + const Vec128<uint8_t, 8> v) { + return Vec128<uint16_t>(vmovl_u8(v.raw)); +} +HWY_API Vec128<uint32_t> PromoteTo(Full128<uint32_t> /* tag */, + const Vec128<uint8_t, 4> v) { + uint16x8_t a = vmovl_u8(v.raw); + return Vec128<uint32_t>(vmovl_u16(vget_low_u16(a))); +} +HWY_API Vec128<uint32_t> PromoteTo(Full128<uint32_t> /* tag */, + const Vec128<uint16_t, 4> v) { + return Vec128<uint32_t>(vmovl_u16(v.raw)); +} +HWY_API Vec128<uint64_t> PromoteTo(Full128<uint64_t> /* tag */, + const Vec128<uint32_t, 2> v) { + return Vec128<uint64_t>(vmovl_u32(v.raw)); +} +HWY_API Vec128<int16_t> PromoteTo(Full128<int16_t> d, + const Vec128<uint8_t, 8> v) { + return BitCast(d, Vec128<uint16_t>(vmovl_u8(v.raw))); +} +HWY_API Vec128<int32_t> PromoteTo(Full128<int32_t> d, + const Vec128<uint8_t, 4> v) { + uint16x8_t a = vmovl_u8(v.raw); + return BitCast(d, Vec128<uint32_t>(vmovl_u16(vget_low_u16(a)))); +} +HWY_API Vec128<int32_t> PromoteTo(Full128<int32_t> d, + const Vec128<uint16_t, 4> v) { + return BitCast(d, Vec128<uint32_t>(vmovl_u16(v.raw))); +} + +// Unsigned: zero-extend to half vector. +template <size_t N, HWY_IF_LE64(uint16_t, N)> +HWY_API Vec128<uint16_t, N> PromoteTo(Simd<uint16_t, N> /* tag */, + const Vec128<uint8_t, N> v) { + return Vec128<uint16_t, N>(vget_low_u16(vmovl_u8(v.raw))); +} +template <size_t N, HWY_IF_LE64(uint32_t, N)> +HWY_API Vec128<uint32_t, N> PromoteTo(Simd<uint32_t, N> /* tag */, + const Vec128<uint8_t, N> v) { + uint16x8_t a = vmovl_u8(v.raw); + return Vec128<uint32_t, N>(vget_low_u32(vmovl_u16(vget_low_u16(a)))); +} +template <size_t N> +HWY_API Vec128<uint32_t, N> PromoteTo(Simd<uint32_t, N> /* tag */, + const Vec128<uint16_t, N> v) { + return Vec128<uint32_t, N>(vget_low_u32(vmovl_u16(v.raw))); +} +template <size_t N, HWY_IF_LE64(uint64_t, N)> +HWY_API Vec128<uint64_t, N> PromoteTo(Simd<uint64_t, N> /* tag */, + const Vec128<uint32_t, N> v) { + return Vec128<uint64_t, N>(vget_low_u64(vmovl_u32(v.raw))); +} +template <size_t N, HWY_IF_LE64(int16_t, N)> +HWY_API Vec128<int16_t, N> PromoteTo(Simd<int16_t, N> d, + const Vec128<uint8_t, N> v) { + return BitCast(d, Vec128<uint16_t, N>(vget_low_u16(vmovl_u8(v.raw)))); +} +template <size_t N, HWY_IF_LE64(int32_t, N)> +HWY_API Vec128<int32_t, N> PromoteTo(Simd<int32_t, N> /* tag */, + const Vec128<uint8_t, N> v) { + uint16x8_t a = vmovl_u8(v.raw); + uint32x4_t b = vmovl_u16(vget_low_u16(a)); + return Vec128<int32_t, N>(vget_low_s32(vreinterpretq_s32_u32(b))); +} +template <size_t N, HWY_IF_LE64(int32_t, N)> +HWY_API Vec128<int32_t, N> PromoteTo(Simd<int32_t, N> /* tag */, + const Vec128<uint16_t, N> v) { + uint32x4_t a = vmovl_u16(v.raw); + return Vec128<int32_t, N>(vget_low_s32(vreinterpretq_s32_u32(a))); +} + +// Signed: replicate sign bit to full vector. +HWY_API Vec128<int16_t> PromoteTo(Full128<int16_t> /* tag */, + const Vec128<int8_t, 8> v) { + return Vec128<int16_t>(vmovl_s8(v.raw)); +} +HWY_API Vec128<int32_t> PromoteTo(Full128<int32_t> /* tag */, + const Vec128<int8_t, 4> v) { + int16x8_t a = vmovl_s8(v.raw); + return Vec128<int32_t>(vmovl_s16(vget_low_s16(a))); +} +HWY_API Vec128<int32_t> PromoteTo(Full128<int32_t> /* tag */, + const Vec128<int16_t, 4> v) { + return Vec128<int32_t>(vmovl_s16(v.raw)); +} +HWY_API Vec128<int64_t> PromoteTo(Full128<int64_t> /* tag */, + const Vec128<int32_t, 2> v) { + return Vec128<int64_t>(vmovl_s32(v.raw)); +} + +// Signed: replicate sign bit to half vector. +template <size_t N> +HWY_API Vec128<int16_t, N> PromoteTo(Simd<int16_t, N> /* tag */, + const Vec128<int8_t, N> v) { + return Vec128<int16_t, N>(vget_low_s16(vmovl_s8(v.raw))); +} +template <size_t N> +HWY_API Vec128<int32_t, N> PromoteTo(Simd<int32_t, N> /* tag */, + const Vec128<int8_t, N> v) { + int16x8_t a = vmovl_s8(v.raw); + int32x4_t b = vmovl_s16(vget_low_s16(a)); + return Vec128<int32_t, N>(vget_low_s32(b)); +} +template <size_t N> +HWY_API Vec128<int32_t, N> PromoteTo(Simd<int32_t, N> /* tag */, + const Vec128<int16_t, N> v) { + return Vec128<int32_t, N>(vget_low_s32(vmovl_s16(v.raw))); +} +template <size_t N> +HWY_API Vec128<int64_t, N> PromoteTo(Simd<int64_t, N> /* tag */, + const Vec128<int32_t, N> v) { + return Vec128<int64_t, N>(vget_low_s64(vmovl_s32(v.raw))); +} + +#if __ARM_FP & 2 + +HWY_API Vec128<float> PromoteTo(Full128<float> /* tag */, + const Vec128<float16_t, 4> v) { + const float32x4_t f32 = vcvt_f32_f16(vreinterpret_f16_u16(v.raw)); + return Vec128<float>(f32); +} +template <size_t N> +HWY_API Vec128<float, N> PromoteTo(Simd<float, N> /* tag */, + const Vec128<float16_t, N> v) { + const float32x4_t f32 = vcvt_f32_f16(vreinterpret_f16_u16(v.raw)); + return Vec128<float, N>(vget_low_f32(f32)); +} + +#else + +template <size_t N> +HWY_API Vec128<float, N> PromoteTo(Simd<float, N> /* tag */, + const Vec128<float16_t, N> v) { + const Simd<int32_t, N> di32; + const Simd<uint32_t, N> du32; + const Simd<float, N> df32; + // Expand to u32 so we can shift. + const auto bits16 = PromoteTo(du32, Vec128<uint16_t, N>{v.raw}); + const auto sign = ShiftRight<15>(bits16); + const auto biased_exp = ShiftRight<10>(bits16) & Set(du32, 0x1F); + const auto mantissa = bits16 & Set(du32, 0x3FF); + const auto subnormal = + BitCast(du32, ConvertTo(df32, BitCast(di32, mantissa)) * + Set(df32, 1.0f / 16384 / 1024)); + + const auto biased_exp32 = biased_exp + Set(du32, 127 - 15); + const auto mantissa32 = ShiftLeft<23 - 10>(mantissa); + const auto normal = ShiftLeft<23>(biased_exp32) | mantissa32; + const auto bits32 = IfThenElse(biased_exp == Zero(du32), subnormal, normal); + return BitCast(df32, ShiftLeft<31>(sign) | bits32); +} + +#endif + +#if HWY_ARCH_ARM_A64 + +HWY_API Vec128<double> PromoteTo(Full128<double> /* tag */, + const Vec128<float, 2> v) { + return Vec128<double>(vcvt_f64_f32(v.raw)); +} + +HWY_API Vec128<double, 1> PromoteTo(Simd<double, 1> /* tag */, + const Vec128<float, 1> v) { + return Vec128<double, 1>(vget_low_f64(vcvt_f64_f32(v.raw))); +} + +HWY_API Vec128<double> PromoteTo(Full128<double> /* tag */, + const Vec128<int32_t, 2> v) { + const int64x2_t i64 = vmovl_s32(v.raw); + return Vec128<double>(vcvtq_f64_s64(i64)); +} + +HWY_API Vec128<double, 1> PromoteTo(Simd<double, 1> /* tag */, + const Vec128<int32_t, 1> v) { + const int64x1_t i64 = vget_low_s64(vmovl_s32(v.raw)); + return Vec128<double, 1>(vcvt_f64_s64(i64)); +} + +#endif + +// ------------------------------ Demotions (full -> part w/ narrow lanes) + +// From full vector to half or quarter +HWY_API Vec128<uint16_t, 4> DemoteTo(Simd<uint16_t, 4> /* tag */, + const Vec128<int32_t> v) { + return Vec128<uint16_t, 4>(vqmovun_s32(v.raw)); +} +HWY_API Vec128<int16_t, 4> DemoteTo(Simd<int16_t, 4> /* tag */, + const Vec128<int32_t> v) { + return Vec128<int16_t, 4>(vqmovn_s32(v.raw)); +} +HWY_API Vec128<uint8_t, 4> DemoteTo(Simd<uint8_t, 4> /* tag */, + const Vec128<int32_t> v) { + const uint16x4_t a = vqmovun_s32(v.raw); + return Vec128<uint8_t, 4>(vqmovn_u16(vcombine_u16(a, a))); +} +HWY_API Vec128<uint8_t, 8> DemoteTo(Simd<uint8_t, 8> /* tag */, + const Vec128<int16_t> v) { + return Vec128<uint8_t, 8>(vqmovun_s16(v.raw)); +} +HWY_API Vec128<int8_t, 4> DemoteTo(Simd<int8_t, 4> /* tag */, + const Vec128<int32_t> v) { + const int16x4_t a = vqmovn_s32(v.raw); + return Vec128<int8_t, 4>(vqmovn_s16(vcombine_s16(a, a))); +} +HWY_API Vec128<int8_t, 8> DemoteTo(Simd<int8_t, 8> /* tag */, + const Vec128<int16_t> v) { + return Vec128<int8_t, 8>(vqmovn_s16(v.raw)); +} + +// From half vector to partial half +template <size_t N, HWY_IF_LE64(int32_t, N)> +HWY_API Vec128<uint16_t, N> DemoteTo(Simd<uint16_t, N> /* tag */, + const Vec128<int32_t, N> v) { + return Vec128<uint16_t, N>(vqmovun_s32(vcombine_s32(v.raw, v.raw))); +} +template <size_t N, HWY_IF_LE64(int32_t, N)> +HWY_API Vec128<int16_t, N> DemoteTo(Simd<int16_t, N> /* tag */, + const Vec128<int32_t, N> v) { + return Vec128<int16_t, N>(vqmovn_s32(vcombine_s32(v.raw, v.raw))); +} +template <size_t N, HWY_IF_LE64(int32_t, N)> +HWY_API Vec128<uint8_t, N> DemoteTo(Simd<uint8_t, N> /* tag */, + const Vec128<int32_t, N> v) { + const uint16x4_t a = vqmovun_s32(vcombine_s32(v.raw, v.raw)); + return Vec128<uint8_t, N>(vqmovn_u16(vcombine_u16(a, a))); +} +template <size_t N, HWY_IF_LE64(int16_t, N)> +HWY_API Vec128<uint8_t, N> DemoteTo(Simd<uint8_t, N> /* tag */, + const Vec128<int16_t, N> v) { + return Vec128<uint8_t, N>(vqmovun_s16(vcombine_s16(v.raw, v.raw))); +} +template <size_t N, HWY_IF_LE64(int32_t, N)> +HWY_API Vec128<int8_t, N> DemoteTo(Simd<int8_t, N> /* tag */, + const Vec128<int32_t, N> v) { + const int16x4_t a = vqmovn_s32(vcombine_s32(v.raw, v.raw)); + return Vec128<int8_t, N>(vqmovn_s16(vcombine_s16(a, a))); +} +template <size_t N, HWY_IF_LE64(int16_t, N)> +HWY_API Vec128<int8_t, N> DemoteTo(Simd<int8_t, N> /* tag */, + const Vec128<int16_t, N> v) { + return Vec128<int8_t, N>(vqmovn_s16(vcombine_s16(v.raw, v.raw))); +} + +#if __ARM_FP & 2 + +HWY_API Vec128<float16_t, 4> DemoteTo(Simd<float16_t, 4> /* tag */, + const Vec128<float> v) { + return Vec128<float16_t, 4>{vreinterpret_u16_f16(vcvt_f16_f32(v.raw))}; +} +template <size_t N> +HWY_API Vec128<float16_t, N> DemoteTo(Simd<float16_t, N> /* tag */, + const Vec128<float, N> v) { + const float16x4_t f16 = vcvt_f16_f32(vcombine_f32(v.raw, v.raw)); + return Vec128<float16_t, N>(vreinterpret_u16_f16(f16)); +} + +#else + +template <size_t N> +HWY_API Vec128<float16_t, N> DemoteTo(Simd<float16_t, N> /* tag */, + const Vec128<float, N> v) { + const Simd<int32_t, N> di; + const Simd<uint32_t, N> du; + const Simd<uint16_t, N> du16; + const auto bits32 = BitCast(du, v); + const auto sign = ShiftRight<31>(bits32); + const auto biased_exp32 = ShiftRight<23>(bits32) & Set(du, 0xFF); + const auto mantissa32 = bits32 & Set(du, 0x7FFFFF); + + const auto k15 = Set(di, 15); + const auto exp = Min(BitCast(di, biased_exp32) - Set(di, 127), k15); + const auto is_tiny = exp < Set(di, -24); + + const auto is_subnormal = exp < Set(di, -14); + const auto biased_exp16 = + BitCast(du, IfThenZeroElse(is_subnormal, exp + k15)); + const auto sub_exp = BitCast(du, Set(di, -14) - exp); // [1, 11) + const auto sub_m = (Set(du, 1) << (Set(du, 10) - sub_exp)) + + (mantissa32 >> (Set(du, 13) + sub_exp)); + const auto mantissa16 = IfThenElse(RebindMask(du, is_subnormal), sub_m, + ShiftRight<13>(mantissa32)); // <1024 + + const auto sign16 = ShiftLeft<15>(sign); + const auto normal16 = sign16 | ShiftLeft<10>(biased_exp16) | mantissa16; + const auto bits16 = IfThenZeroElse(is_tiny, BitCast(di, normal16)); + return Vec128<float16_t, N>(DemoteTo(du16, bits16).raw); +} + +#endif + +template <size_t N> +HWY_API Vec128<bfloat16_t, N> DemoteTo(Simd<bfloat16_t, N> dbf16, + const Vec128<float, N> v) { + const Rebind<int32_t, decltype(dbf16)> di32; + const Rebind<uint32_t, decltype(dbf16)> du32; // for logical shift right + const Rebind<uint16_t, decltype(dbf16)> du16; + const auto bits_in_32 = BitCast(di32, ShiftRight<16>(BitCast(du32, v))); + return BitCast(dbf16, DemoteTo(du16, bits_in_32)); +} + +#if HWY_ARCH_ARM_A64 + +HWY_API Vec128<float, 2> DemoteTo(Simd<float, 2> /* tag */, + const Vec128<double> v) { + return Vec128<float, 2>(vcvt_f32_f64(v.raw)); +} +HWY_API Vec128<float, 1> DemoteTo(Simd<float, 1> /* tag */, + const Vec128<double, 1> v) { + return Vec128<float, 1>(vcvt_f32_f64(vcombine_f64(v.raw, v.raw))); +} + +HWY_API Vec128<int32_t, 2> DemoteTo(Simd<int32_t, 2> /* tag */, + const Vec128<double> v) { + const int64x2_t i64 = vcvtq_s64_f64(v.raw); + return Vec128<int32_t, 2>(vqmovn_s64(i64)); +} +HWY_API Vec128<int32_t, 1> DemoteTo(Simd<int32_t, 1> /* tag */, + const Vec128<double, 1> v) { + const int64x1_t i64 = vcvt_s64_f64(v.raw); + // There is no i64x1 -> i32x1 narrow, so expand to int64x2_t first. + const int64x2_t i64x2 = vcombine_s64(i64, i64); + return Vec128<int32_t, 1>(vqmovn_s64(i64x2)); +} + +#endif + +HWY_API Vec128<uint8_t, 4> U8FromU32(const Vec128<uint32_t> v) { + const uint8x16_t org_v = detail::BitCastToByte(v).raw; + const uint8x16_t w = vuzp1q_u8(org_v, org_v); + return Vec128<uint8_t, 4>(vget_low_u8(vuzp1q_u8(w, w))); +} +template <size_t N, HWY_IF_LE64(uint32_t, N)> +HWY_API Vec128<uint8_t, N> U8FromU32(const Vec128<uint32_t, N> v) { + const uint8x8_t org_v = detail::BitCastToByte(v).raw; + const uint8x8_t w = vuzp1_u8(org_v, org_v); + return Vec128<uint8_t, N>(vuzp1_u8(w, w)); +} + +// In the following DemoteTo functions, |b| is purposely undefined. +// The value a needs to be extended to 128 bits so that vqmovn can be +// used and |b| is undefined so that no extra overhead is introduced. +HWY_DIAGNOSTICS(push) +HWY_DIAGNOSTICS_OFF(disable : 4701, ignored "-Wuninitialized") + +template <size_t N> +HWY_API Vec128<uint8_t, N> DemoteTo(Simd<uint8_t, N> /* tag */, + const Vec128<int32_t> v) { + Vec128<uint16_t, N> a = DemoteTo(Simd<uint16_t, N>(), v); + Vec128<uint16_t, N> b; + uint16x8_t c = vcombine_u16(a.raw, b.raw); + return Vec128<uint8_t, N>(vqmovn_u16(c)); +} + +template <size_t N> +HWY_API Vec128<int8_t, N> DemoteTo(Simd<int8_t, N> /* tag */, + const Vec128<int32_t> v) { + Vec128<int16_t, N> a = DemoteTo(Simd<int16_t, N>(), v); + Vec128<int16_t, N> b; + int16x8_t c = vcombine_s16(a.raw, b.raw); + return Vec128<int8_t, N>(vqmovn_s16(c)); +} + +HWY_DIAGNOSTICS(pop) + +// ------------------------------ Convert integer <=> floating-point + +HWY_API Vec128<float> ConvertTo(Full128<float> /* tag */, + const Vec128<int32_t> v) { + return Vec128<float>(vcvtq_f32_s32(v.raw)); +} +template <size_t N, HWY_IF_LE64(int32_t, N)> +HWY_API Vec128<float, N> ConvertTo(Simd<float, N> /* tag */, + const Vec128<int32_t, N> v) { + return Vec128<float, N>(vcvt_f32_s32(v.raw)); +} + +// Truncates (rounds toward zero). +HWY_API Vec128<int32_t> ConvertTo(Full128<int32_t> /* tag */, + const Vec128<float> v) { + return Vec128<int32_t>(vcvtq_s32_f32(v.raw)); +} +template <size_t N, HWY_IF_LE64(float, N)> +HWY_API Vec128<int32_t, N> ConvertTo(Simd<int32_t, N> /* tag */, + const Vec128<float, N> v) { + return Vec128<int32_t, N>(vcvt_s32_f32(v.raw)); +} + +#if HWY_ARCH_ARM_A64 + +HWY_API Vec128<double> ConvertTo(Full128<double> /* tag */, + const Vec128<int64_t> v) { + return Vec128<double>(vcvtq_f64_s64(v.raw)); +} +HWY_API Vec128<double, 1> ConvertTo(Simd<double, 1> /* tag */, + const Vec128<int64_t, 1> v) { + return Vec128<double, 1>(vcvt_f64_s64(v.raw)); +} + +// Truncates (rounds toward zero). +HWY_API Vec128<int64_t> ConvertTo(Full128<int64_t> /* tag */, + const Vec128<double> v) { + return Vec128<int64_t>(vcvtq_s64_f64(v.raw)); +} +HWY_API Vec128<int64_t, 1> ConvertTo(Simd<int64_t, 1> /* tag */, + const Vec128<double, 1> v) { + return Vec128<int64_t, 1>(vcvt_s64_f64(v.raw)); +} + +#endif + +// ------------------------------ Round (IfThenElse, mask, logical) + +#if HWY_ARCH_ARM_A64 +// Toward nearest integer +HWY_NEON_DEF_FUNCTION_ALL_FLOATS(Round, vrndn, _, 1) + +// Toward zero, aka truncate +HWY_NEON_DEF_FUNCTION_ALL_FLOATS(Trunc, vrnd, _, 1) + +// Toward +infinity, aka ceiling +HWY_NEON_DEF_FUNCTION_ALL_FLOATS(Ceil, vrndp, _, 1) + +// Toward -infinity, aka floor +HWY_NEON_DEF_FUNCTION_ALL_FLOATS(Floor, vrndm, _, 1) +#else + +// ------------------------------ Trunc + +// ARMv7 only supports truncation to integer. We can either convert back to +// float (3 floating-point and 2 logic operations) or manipulate the binary32 +// representation, clearing the lowest 23-exp mantissa bits. This requires 9 +// integer operations and 3 constants, which is likely more expensive. + +namespace detail { + +// The original value is already the desired result if NaN or the magnitude is +// large (i.e. the value is already an integer). +template <size_t N> +HWY_INLINE Mask128<float, N> UseInt(const Vec128<float, N> v) { + return Abs(v) < Set(Simd<float, N>(), MantissaEnd<float>()); +} + +} // namespace detail + +template <size_t N> +HWY_API Vec128<float, N> Trunc(const Vec128<float, N> v) { + const Simd<float, N> df; + const RebindToSigned<decltype(df)> di; + + const auto integer = ConvertTo(di, v); // round toward 0 + const auto int_f = ConvertTo(df, integer); + + return IfThenElse(detail::UseInt(v), int_f, v); +} + +template <size_t N> +HWY_API Vec128<float, N> Round(const Vec128<float, N> v) { + const Simd<float, N> df; + + // ARMv7 also lacks a native NearestInt, but we can instead rely on rounding + // (we assume the current mode is nearest-even) after addition with a large + // value such that no mantissa bits remain. We may need a compiler flag for + // precise floating-point to prevent this from being "optimized" out. + const auto max = Set(df, MantissaEnd<float>()); + const auto large = CopySignToAbs(max, v); + const auto added = large + v; + const auto rounded = added - large; + + // Keep original if NaN or the magnitude is large (already an int). + return IfThenElse(Abs(v) < max, rounded, v); +} + +template <size_t N> +HWY_API Vec128<float, N> Ceil(const Vec128<float, N> v) { + const Simd<float, N> df; + const RebindToSigned<decltype(df)> di; + + const auto integer = ConvertTo(di, v); // round toward 0 + const auto int_f = ConvertTo(df, integer); + + // Truncating a positive non-integer ends up smaller; if so, add 1. + const auto neg1 = ConvertTo(df, VecFromMask(di, RebindMask(di, int_f < v))); + + return IfThenElse(detail::UseInt(v), int_f - neg1, v); +} + +template <size_t N> +HWY_API Vec128<float, N> Floor(const Vec128<float, N> v) { + const Simd<float, N> df; + const Simd<int32_t, N> di; + + const auto integer = ConvertTo(di, v); // round toward 0 + const auto int_f = ConvertTo(df, integer); + + // Truncating a negative non-integer ends up larger; if so, subtract 1. + const auto neg1 = ConvertTo(df, VecFromMask(di, RebindMask(di, int_f > v))); + + return IfThenElse(detail::UseInt(v), int_f + neg1, v); +} + +#endif + +// ------------------------------ NearestInt (Round) + +#if HWY_ARCH_ARM_A64 + +HWY_API Vec128<int32_t> NearestInt(const Vec128<float> v) { + return Vec128<int32_t>(vcvtnq_s32_f32(v.raw)); +} +template <size_t N, HWY_IF_LE64(float, N)> +HWY_API Vec128<int32_t, N> NearestInt(const Vec128<float, N> v) { + return Vec128<int32_t, N>(vcvtn_s32_f32(v.raw)); +} + +#else + +template <size_t N> +HWY_API Vec128<int32_t, N> NearestInt(const Vec128<float, N> v) { + const Simd<int32_t, N> di; + return ConvertTo(di, Round(v)); +} + +#endif + +// ================================================== SWIZZLE + +// ------------------------------ LowerHalf + +// <= 64 bit: just return different type +template <typename T, size_t N, HWY_IF_LE64(uint8_t, N)> +HWY_API Vec128<T, N / 2> LowerHalf(const Vec128<T, N> v) { + return Vec128<T, N / 2>(v.raw); +} + +HWY_API Vec128<uint8_t, 8> LowerHalf(const Vec128<uint8_t> v) { + return Vec128<uint8_t, 8>(vget_low_u8(v.raw)); +} +HWY_API Vec128<uint16_t, 4> LowerHalf(const Vec128<uint16_t> v) { + return Vec128<uint16_t, 4>(vget_low_u16(v.raw)); +} +HWY_API Vec128<uint32_t, 2> LowerHalf(const Vec128<uint32_t> v) { + return Vec128<uint32_t, 2>(vget_low_u32(v.raw)); +} +HWY_API Vec128<uint64_t, 1> LowerHalf(const Vec128<uint64_t> v) { + return Vec128<uint64_t, 1>(vget_low_u64(v.raw)); +} +HWY_API Vec128<int8_t, 8> LowerHalf(const Vec128<int8_t> v) { + return Vec128<int8_t, 8>(vget_low_s8(v.raw)); +} +HWY_API Vec128<int16_t, 4> LowerHalf(const Vec128<int16_t> v) { + return Vec128<int16_t, 4>(vget_low_s16(v.raw)); +} +HWY_API Vec128<int32_t, 2> LowerHalf(const Vec128<int32_t> v) { + return Vec128<int32_t, 2>(vget_low_s32(v.raw)); +} +HWY_API Vec128<int64_t, 1> LowerHalf(const Vec128<int64_t> v) { + return Vec128<int64_t, 1>(vget_low_s64(v.raw)); +} +HWY_API Vec128<float, 2> LowerHalf(const Vec128<float> v) { + return Vec128<float, 2>(vget_low_f32(v.raw)); +} +#if HWY_ARCH_ARM_A64 +HWY_API Vec128<double, 1> LowerHalf(const Vec128<double> v) { + return Vec128<double, 1>(vget_low_f64(v.raw)); +} +#endif + +template <typename T, size_t N> +HWY_API Vec128<T, N / 2> LowerHalf(Simd<T, N / 2> /* tag */, Vec128<T, N> v) { + return LowerHalf(v); +} + +// ------------------------------ CombineShiftRightBytes + +// 128-bit +template <int kBytes, typename T, class V128 = Vec128<T>> +HWY_API V128 CombineShiftRightBytes(Full128<T> d, V128 hi, V128 lo) { + static_assert(0 < kBytes && kBytes < 16, "kBytes must be in [1, 15]"); + const Repartition<uint8_t, decltype(d)> d8; + uint8x16_t v8 = vextq_u8(BitCast(d8, lo).raw, BitCast(d8, hi).raw, kBytes); + return BitCast(d, Vec128<uint8_t>(v8)); +} + +// 64-bit +template <int kBytes, typename T, class V64 = Vec128<T, 8 / sizeof(T)>> +HWY_API V64 CombineShiftRightBytes(Simd<T, 8 / sizeof(T)> d, V64 hi, V64 lo) { + static_assert(0 < kBytes && kBytes < 8, "kBytes must be in [1, 7]"); + const Repartition<uint8_t, decltype(d)> d8; + uint8x8_t v8 = vext_u8(BitCast(d8, lo).raw, BitCast(d8, hi).raw, kBytes); + return BitCast(d, VFromD<decltype(d8)>(v8)); +} + +// <= 32-bit defined after ShiftLeftBytes. + +// ------------------------------ Shift vector by constant #bytes + +namespace detail { + +// Partially specialize because kBytes = 0 and >= size are compile errors; +// callers replace the latter with 0xFF for easier specialization. +template <int kBytes> +struct ShiftLeftBytesT { + // Full + template <class T> + HWY_INLINE Vec128<T> operator()(const Vec128<T> v) { + const Full128<T> d; + return CombineShiftRightBytes<16 - kBytes>(d, v, Zero(d)); + } + + // Partial + template <class T, size_t N, HWY_IF_LE64(T, N)> + HWY_INLINE Vec128<T, N> operator()(const Vec128<T, N> v) { + // Expand to 64-bit so we only use the native EXT instruction. + const Simd<T, 8 / sizeof(T)> d64; + const auto zero64 = Zero(d64); + const decltype(zero64) v64(v.raw); + return Vec128<T, N>( + CombineShiftRightBytes<8 - kBytes>(d64, v64, zero64).raw); + } +}; +template <> +struct ShiftLeftBytesT<0> { + template <class T, size_t N> + HWY_INLINE Vec128<T, N> operator()(const Vec128<T, N> v) { + return v; + } +}; +template <> +struct ShiftLeftBytesT<0xFF> { + template <class T, size_t N> + HWY_INLINE Vec128<T, N> operator()(const Vec128<T, N> /* v */) { + return Zero(Simd<T, N>()); + } +}; + +template <int kBytes> +struct ShiftRightBytesT { + template <class T, size_t N> + HWY_INLINE Vec128<T, N> operator()(Vec128<T, N> v) { + const Simd<T, N> d; + // For < 64-bit vectors, zero undefined lanes so we shift in zeros. + if (N * sizeof(T) < 8) { + constexpr size_t kReg = N * sizeof(T) == 16 ? 16 : 8; + const Simd<T, kReg / sizeof(T)> dreg; + v = Vec128<T, N>( + IfThenElseZero(FirstN(dreg, N), VFromD<decltype(dreg)>(v.raw)).raw); + } + return CombineShiftRightBytes<kBytes>(d, Zero(d), v); + } +}; +template <> +struct ShiftRightBytesT<0> { + template <class T, size_t N> + HWY_INLINE Vec128<T, N> operator()(const Vec128<T, N> v) { + return v; + } +}; +template <> +struct ShiftRightBytesT<0xFF> { + template <class T, size_t N> + HWY_INLINE Vec128<T, N> operator()(const Vec128<T, N> /* v */) { + return Zero(Simd<T, N>()); + } +}; + +} // namespace detail + +template <int kBytes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftLeftBytes(Simd<T, N> /* tag */, Vec128<T, N> v) { + return detail::ShiftLeftBytesT < kBytes >= N * sizeof(T) ? 0xFF + : kBytes > ()(v); +} + +template <int kBytes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftLeftBytes(const Vec128<T, N> v) { + return ShiftLeftBytes<kBytes>(Simd<T, N>(), v); +} + +template <int kLanes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftLeftLanes(Simd<T, N> d, const Vec128<T, N> v) { + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, ShiftLeftBytes<kLanes * sizeof(T)>(BitCast(d8, v))); +} + +template <int kLanes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftLeftLanes(const Vec128<T, N> v) { + return ShiftLeftLanes<kLanes>(Simd<T, N>(), v); +} + +// 0x01..0F, kBytes = 1 => 0x0001..0E +template <int kBytes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftRightBytes(Simd<T, N> /* tag */, Vec128<T, N> v) { + return detail::ShiftRightBytesT < kBytes >= N * sizeof(T) ? 0xFF + : kBytes > ()(v); +} + +template <int kLanes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftRightLanes(Simd<T, N> d, const Vec128<T, N> v) { + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, ShiftRightBytes<kLanes * sizeof(T)>(BitCast(d8, v))); +} + +// Calls ShiftLeftBytes +template <int kBytes, typename T, size_t N, HWY_IF_LE32(T, N)> +HWY_API Vec128<T, N> CombineShiftRightBytes(Simd<T, N> d, Vec128<T, N> hi, + Vec128<T, N> lo) { + constexpr size_t kSize = N * sizeof(T); + static_assert(0 < kBytes && kBytes < kSize, "kBytes invalid"); + const Repartition<uint8_t, decltype(d)> d8; + const Simd<uint8_t, 8> d_full8; + const Repartition<T, decltype(d_full8)> d_full; + using V64 = VFromD<decltype(d_full8)>; + const V64 hi64(BitCast(d8, hi).raw); + // Move into most-significant bytes + const V64 lo64 = ShiftLeftBytes<8 - kSize>(V64(BitCast(d8, lo).raw)); + const V64 r = CombineShiftRightBytes<8 - kSize + kBytes>(d_full8, hi64, lo64); + // After casting to full 64-bit vector of correct type, shrink to 32-bit + return Vec128<T, N>(BitCast(d_full, r).raw); +} + +// ------------------------------ UpperHalf (ShiftRightBytes) + +// Full input +HWY_API Vec128<uint8_t, 8> UpperHalf(Simd<uint8_t, 8> /* tag */, + const Vec128<uint8_t> v) { + return Vec128<uint8_t, 8>(vget_high_u8(v.raw)); +} +HWY_API Vec128<uint16_t, 4> UpperHalf(Simd<uint16_t, 4> /* tag */, + const Vec128<uint16_t> v) { + return Vec128<uint16_t, 4>(vget_high_u16(v.raw)); +} +HWY_API Vec128<uint32_t, 2> UpperHalf(Simd<uint32_t, 2> /* tag */, + const Vec128<uint32_t> v) { + return Vec128<uint32_t, 2>(vget_high_u32(v.raw)); +} +HWY_API Vec128<uint64_t, 1> UpperHalf(Simd<uint64_t, 1> /* tag */, + const Vec128<uint64_t> v) { + return Vec128<uint64_t, 1>(vget_high_u64(v.raw)); +} +HWY_API Vec128<int8_t, 8> UpperHalf(Simd<int8_t, 8> /* tag */, + const Vec128<int8_t> v) { + return Vec128<int8_t, 8>(vget_high_s8(v.raw)); +} +HWY_API Vec128<int16_t, 4> UpperHalf(Simd<int16_t, 4> /* tag */, + const Vec128<int16_t> v) { + return Vec128<int16_t, 4>(vget_high_s16(v.raw)); +} +HWY_API Vec128<int32_t, 2> UpperHalf(Simd<int32_t, 2> /* tag */, + const Vec128<int32_t> v) { + return Vec128<int32_t, 2>(vget_high_s32(v.raw)); +} +HWY_API Vec128<int64_t, 1> UpperHalf(Simd<int64_t, 1> /* tag */, + const Vec128<int64_t> v) { + return Vec128<int64_t, 1>(vget_high_s64(v.raw)); +} +HWY_API Vec128<float, 2> UpperHalf(Simd<float, 2> /* tag */, + const Vec128<float> v) { + return Vec128<float, 2>(vget_high_f32(v.raw)); +} +#if HWY_ARCH_ARM_A64 +HWY_API Vec128<double, 1> UpperHalf(Simd<double, 1> /* tag */, + const Vec128<double> v) { + return Vec128<double, 1>(vget_high_f64(v.raw)); +} +#endif + +// Partial +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_API Vec128<T, (N + 1) / 2> UpperHalf(Half<Simd<T, N>> /* tag */, + Vec128<T, N> v) { + const Simd<T, N> d; + const auto vu = BitCast(RebindToUnsigned<decltype(d)>(), v); + const auto upper = BitCast(d, ShiftRightBytes<N * sizeof(T) / 2>(vu)); + return Vec128<T, (N + 1) / 2>(upper.raw); +} + +// ------------------------------ Broadcast/splat any lane + +#if HWY_ARCH_ARM_A64 +// Unsigned +template <int kLane> +HWY_API Vec128<uint16_t> Broadcast(const Vec128<uint16_t> v) { + static_assert(0 <= kLane && kLane < 8, "Invalid lane"); + return Vec128<uint16_t>(vdupq_laneq_u16(v.raw, kLane)); +} +template <int kLane, size_t N, HWY_IF_LE64(uint16_t, N)> +HWY_API Vec128<uint16_t, N> Broadcast(const Vec128<uint16_t, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<uint16_t, N>(vdup_lane_u16(v.raw, kLane)); +} +template <int kLane> +HWY_API Vec128<uint32_t> Broadcast(const Vec128<uint32_t> v) { + static_assert(0 <= kLane && kLane < 4, "Invalid lane"); + return Vec128<uint32_t>(vdupq_laneq_u32(v.raw, kLane)); +} +template <int kLane, size_t N, HWY_IF_LE64(uint32_t, N)> +HWY_API Vec128<uint32_t, N> Broadcast(const Vec128<uint32_t, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<uint32_t, N>(vdup_lane_u32(v.raw, kLane)); +} +template <int kLane> +HWY_API Vec128<uint64_t> Broadcast(const Vec128<uint64_t> v) { + static_assert(0 <= kLane && kLane < 2, "Invalid lane"); + return Vec128<uint64_t>(vdupq_laneq_u64(v.raw, kLane)); +} +// Vec128<uint64_t, 1> is defined below. + +// Signed +template <int kLane> +HWY_API Vec128<int16_t> Broadcast(const Vec128<int16_t> v) { + static_assert(0 <= kLane && kLane < 8, "Invalid lane"); + return Vec128<int16_t>(vdupq_laneq_s16(v.raw, kLane)); +} +template <int kLane, size_t N, HWY_IF_LE64(int16_t, N)> +HWY_API Vec128<int16_t, N> Broadcast(const Vec128<int16_t, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<int16_t, N>(vdup_lane_s16(v.raw, kLane)); +} +template <int kLane> +HWY_API Vec128<int32_t> Broadcast(const Vec128<int32_t> v) { + static_assert(0 <= kLane && kLane < 4, "Invalid lane"); + return Vec128<int32_t>(vdupq_laneq_s32(v.raw, kLane)); +} +template <int kLane, size_t N, HWY_IF_LE64(int32_t, N)> +HWY_API Vec128<int32_t, N> Broadcast(const Vec128<int32_t, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<int32_t, N>(vdup_lane_s32(v.raw, kLane)); +} +template <int kLane> +HWY_API Vec128<int64_t> Broadcast(const Vec128<int64_t> v) { + static_assert(0 <= kLane && kLane < 2, "Invalid lane"); + return Vec128<int64_t>(vdupq_laneq_s64(v.raw, kLane)); +} +// Vec128<int64_t, 1> is defined below. + +// Float +template <int kLane> +HWY_API Vec128<float> Broadcast(const Vec128<float> v) { + static_assert(0 <= kLane && kLane < 4, "Invalid lane"); + return Vec128<float>(vdupq_laneq_f32(v.raw, kLane)); +} +template <int kLane, size_t N, HWY_IF_LE64(float, N)> +HWY_API Vec128<float, N> Broadcast(const Vec128<float, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<float, N>(vdup_lane_f32(v.raw, kLane)); +} +template <int kLane> +HWY_API Vec128<double> Broadcast(const Vec128<double> v) { + static_assert(0 <= kLane && kLane < 2, "Invalid lane"); + return Vec128<double>(vdupq_laneq_f64(v.raw, kLane)); +} +template <int kLane> +HWY_API Vec128<double, 1> Broadcast(const Vec128<double, 1> v) { + static_assert(0 <= kLane && kLane < 1, "Invalid lane"); + return v; +} + +#else +// No vdupq_laneq_* on armv7: use vgetq_lane_* + vdupq_n_*. + +// Unsigned +template <int kLane> +HWY_API Vec128<uint16_t> Broadcast(const Vec128<uint16_t> v) { + static_assert(0 <= kLane && kLane < 8, "Invalid lane"); + return Vec128<uint16_t>(vdupq_n_u16(vgetq_lane_u16(v.raw, kLane))); +} +template <int kLane, size_t N, HWY_IF_LE64(uint16_t, N)> +HWY_API Vec128<uint16_t, N> Broadcast(const Vec128<uint16_t, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<uint16_t, N>(vdup_lane_u16(v.raw, kLane)); +} +template <int kLane> +HWY_API Vec128<uint32_t> Broadcast(const Vec128<uint32_t> v) { + static_assert(0 <= kLane && kLane < 4, "Invalid lane"); + return Vec128<uint32_t>(vdupq_n_u32(vgetq_lane_u32(v.raw, kLane))); +} +template <int kLane, size_t N, HWY_IF_LE64(uint32_t, N)> +HWY_API Vec128<uint32_t, N> Broadcast(const Vec128<uint32_t, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<uint32_t, N>(vdup_lane_u32(v.raw, kLane)); +} +template <int kLane> +HWY_API Vec128<uint64_t> Broadcast(const Vec128<uint64_t> v) { + static_assert(0 <= kLane && kLane < 2, "Invalid lane"); + return Vec128<uint64_t>(vdupq_n_u64(vgetq_lane_u64(v.raw, kLane))); +} +// Vec128<uint64_t, 1> is defined below. + +// Signed +template <int kLane> +HWY_API Vec128<int16_t> Broadcast(const Vec128<int16_t> v) { + static_assert(0 <= kLane && kLane < 8, "Invalid lane"); + return Vec128<int16_t>(vdupq_n_s16(vgetq_lane_s16(v.raw, kLane))); +} +template <int kLane, size_t N, HWY_IF_LE64(int16_t, N)> +HWY_API Vec128<int16_t, N> Broadcast(const Vec128<int16_t, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<int16_t, N>(vdup_lane_s16(v.raw, kLane)); +} +template <int kLane> +HWY_API Vec128<int32_t> Broadcast(const Vec128<int32_t> v) { + static_assert(0 <= kLane && kLane < 4, "Invalid lane"); + return Vec128<int32_t>(vdupq_n_s32(vgetq_lane_s32(v.raw, kLane))); +} +template <int kLane, size_t N, HWY_IF_LE64(int32_t, N)> +HWY_API Vec128<int32_t, N> Broadcast(const Vec128<int32_t, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<int32_t, N>(vdup_lane_s32(v.raw, kLane)); +} +template <int kLane> +HWY_API Vec128<int64_t> Broadcast(const Vec128<int64_t> v) { + static_assert(0 <= kLane && kLane < 2, "Invalid lane"); + return Vec128<int64_t>(vdupq_n_s64(vgetq_lane_s64(v.raw, kLane))); +} +// Vec128<int64_t, 1> is defined below. + +// Float +template <int kLane> +HWY_API Vec128<float> Broadcast(const Vec128<float> v) { + static_assert(0 <= kLane && kLane < 4, "Invalid lane"); + return Vec128<float>(vdupq_n_f32(vgetq_lane_f32(v.raw, kLane))); +} +template <int kLane, size_t N, HWY_IF_LE64(float, N)> +HWY_API Vec128<float, N> Broadcast(const Vec128<float, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<float, N>(vdup_lane_f32(v.raw, kLane)); +} + +#endif + +template <int kLane> +HWY_API Vec128<uint64_t, 1> Broadcast(const Vec128<uint64_t, 1> v) { + static_assert(0 <= kLane && kLane < 1, "Invalid lane"); + return v; +} +template <int kLane> +HWY_API Vec128<int64_t, 1> Broadcast(const Vec128<int64_t, 1> v) { + static_assert(0 <= kLane && kLane < 1, "Invalid lane"); + return v; +} + +// ------------------------------ TableLookupLanes + +// Returned by SetTableIndices for use by TableLookupLanes. +template <typename T, size_t N> +struct Indices128 { + typename detail::Raw128<T, N>::type raw; +}; + +template <typename T, size_t N, typename TI, HWY_IF_LE128(T, N)> +HWY_API Indices128<T, N> IndicesFromVec(Simd<T, N> d, Vec128<TI, N> vec) { + static_assert(sizeof(T) == sizeof(TI), "Index size must match lane"); +#if HWY_IS_DEBUG_BUILD + const Simd<TI, N> di; + HWY_DASSERT(AllFalse(di, Lt(vec, Zero(di))) && + AllTrue(di, Lt(vec, Set(di, static_cast<TI>(N))))); +#endif + + const Repartition<uint8_t, decltype(d)> d8; + using V8 = VFromD<decltype(d8)>; + const Repartition<uint16_t, decltype(d)> d16; + + // Broadcast each lane index to all bytes of T and shift to bytes + static_assert(sizeof(T) == 4 || sizeof(T) == 8, ""); + if (sizeof(T) == 4) { + alignas(16) constexpr uint8_t kBroadcastLaneBytes[16] = { + 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12}; + const V8 lane_indices = + TableLookupBytes(BitCast(d8, vec), Load(d8, kBroadcastLaneBytes)); + const V8 byte_indices = + BitCast(d8, ShiftLeft<2>(BitCast(d16, lane_indices))); + alignas(16) constexpr uint8_t kByteOffsets[16] = {0, 1, 2, 3, 0, 1, 2, 3, + 0, 1, 2, 3, 0, 1, 2, 3}; + const V8 sum = Add(byte_indices, Load(d8, kByteOffsets)); + return Indices128<T, N>{BitCast(d, sum).raw}; + } else { + alignas(16) constexpr uint8_t kBroadcastLaneBytes[16] = { + 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 8, 8, 8, 8}; + const V8 lane_indices = + TableLookupBytes(BitCast(d8, vec), Load(d8, kBroadcastLaneBytes)); + const V8 byte_indices = + BitCast(d8, ShiftLeft<3>(BitCast(d16, lane_indices))); + alignas(16) constexpr uint8_t kByteOffsets[16] = {0, 1, 2, 3, 4, 5, 6, 7, + 0, 1, 2, 3, 4, 5, 6, 7}; + const V8 sum = Add(byte_indices, Load(d8, kByteOffsets)); + return Indices128<T, N>{BitCast(d, sum).raw}; + } +} + +template <typename T, size_t N, typename TI, HWY_IF_LE128(T, N)> +HWY_API Indices128<T, N> SetTableIndices(Simd<T, N> d, const TI* idx) { + const Rebind<TI, decltype(d)> di; + return IndicesFromVec(d, LoadU(di, idx)); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> TableLookupLanes(Vec128<T, N> v, Indices128<T, N> idx) { + const Simd<T, N> d; + const RebindToSigned<decltype(d)> di; + return BitCast( + d, TableLookupBytes(BitCast(di, v), BitCast(di, Vec128<T, N>{idx.raw}))); +} + +// ------------------------------ Reverse (Shuffle0123, Shuffle2301, Shuffle01) + +// Single lane: no change +template <typename T> +HWY_API Vec128<T, 1> Reverse(Simd<T, 1> /* tag */, const Vec128<T, 1> v) { + return v; +} + +// Two lanes: shuffle +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T, 2> Reverse(Simd<T, 2> /* tag */, const Vec128<T, 2> v) { + return Vec128<T, 2>(Shuffle2301(v)); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec128<T> Reverse(Full128<T> /* tag */, const Vec128<T> v) { + return Shuffle01(v); +} + +// Four lanes: shuffle +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T> Reverse(Full128<T> /* tag */, const Vec128<T> v) { + return Shuffle0123(v); +} + +// 16-bit +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Vec128<T, N> Reverse(Simd<T, N> d, const Vec128<T, N> v) { + const RepartitionToWide<RebindToUnsigned<decltype(d)>> du32; + return BitCast(d, RotateRight<16>(Reverse(du32, BitCast(du32, v)))); +} + +// ------------------------------ Other shuffles (TableLookupBytes) + +// Notation: let Vec128<int32_t> have lanes 3,2,1,0 (0 is least-significant). +// Shuffle0321 rotates one lane to the right (the previous least-significant +// lane is now most-significant). These could also be implemented via +// CombineShiftRightBytes but the shuffle_abcd notation is more convenient. + +// Swap 64-bit halves +template <typename T> +HWY_API Vec128<T> Shuffle1032(const Vec128<T> v) { + return CombineShiftRightBytes<8>(Full128<T>(), v, v); +} +template <typename T> +HWY_API Vec128<T> Shuffle01(const Vec128<T> v) { + return CombineShiftRightBytes<8>(Full128<T>(), v, v); +} + +// Rotate right 32 bits +template <typename T> +HWY_API Vec128<T> Shuffle0321(const Vec128<T> v) { + return CombineShiftRightBytes<4>(Full128<T>(), v, v); +} + +// Rotate left 32 bits +template <typename T> +HWY_API Vec128<T> Shuffle2103(const Vec128<T> v) { + return CombineShiftRightBytes<12>(Full128<T>(), v, v); +} + +// Reverse +template <typename T> +HWY_API Vec128<T> Shuffle0123(const Vec128<T> v) { + return Shuffle2301(Shuffle1032(v)); +} + +// ------------------------------ InterleaveLower + +// Interleaves lanes from halves of the 128-bit blocks of "a" (which provides +// the least-significant lane) and "b". To concatenate two half-width integers +// into one, use ZipLower/Upper instead (also works with scalar). +HWY_NEON_DEF_FUNCTION_INT_8_16_32(InterleaveLower, vzip1, _, 2) +HWY_NEON_DEF_FUNCTION_UINT_8_16_32(InterleaveLower, vzip1, _, 2) + +#if HWY_ARCH_ARM_A64 +// N=1 makes no sense (in that case, there would be no upper/lower). +HWY_API Vec128<uint64_t> InterleaveLower(const Vec128<uint64_t> a, + const Vec128<uint64_t> b) { + return Vec128<uint64_t>(vzip1q_u64(a.raw, b.raw)); +} +HWY_API Vec128<int64_t> InterleaveLower(const Vec128<int64_t> a, + const Vec128<int64_t> b) { + return Vec128<int64_t>(vzip1q_s64(a.raw, b.raw)); +} +HWY_API Vec128<double> InterleaveLower(const Vec128<double> a, + const Vec128<double> b) { + return Vec128<double>(vzip1q_f64(a.raw, b.raw)); +} +#else +// ARMv7 emulation. +HWY_API Vec128<uint64_t> InterleaveLower(const Vec128<uint64_t> a, + const Vec128<uint64_t> b) { + return CombineShiftRightBytes<8>(Full128<uint64_t>(), b, Shuffle01(a)); +} +HWY_API Vec128<int64_t> InterleaveLower(const Vec128<int64_t> a, + const Vec128<int64_t> b) { + return CombineShiftRightBytes<8>(Full128<int64_t>(), b, Shuffle01(a)); +} +#endif + +// Floats +HWY_API Vec128<float> InterleaveLower(const Vec128<float> a, + const Vec128<float> b) { + return Vec128<float>(vzip1q_f32(a.raw, b.raw)); +} +template <size_t N, HWY_IF_LE64(float, N)> +HWY_API Vec128<float, N> InterleaveLower(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Vec128<float, N>(vzip1_f32(a.raw, b.raw)); +} + +// < 64 bit parts +template <typename T, size_t N, HWY_IF_LE32(T, N)> +HWY_API Vec128<T, N> InterleaveLower(Vec128<T, N> a, Vec128<T, N> b) { + using V64 = Vec128<T, 8 / sizeof(T)>; + return Vec128<T, N>(InterleaveLower(V64(a.raw), V64(b.raw)).raw); +} + +// Additional overload for the optional Simd<> tag. +template <typename T, size_t N, class V = Vec128<T, N>> +HWY_API V InterleaveLower(Simd<T, N> /* tag */, V a, V b) { + return InterleaveLower(a, b); +} + +// ------------------------------ InterleaveUpper (UpperHalf) + +// All functions inside detail lack the required D parameter. +namespace detail { +HWY_NEON_DEF_FUNCTION_INT_8_16_32(InterleaveUpper, vzip2, _, 2) +HWY_NEON_DEF_FUNCTION_UINT_8_16_32(InterleaveUpper, vzip2, _, 2) + +#if HWY_ARCH_ARM_A64 +// N=1 makes no sense (in that case, there would be no upper/lower). +HWY_API Vec128<uint64_t> InterleaveUpper(const Vec128<uint64_t> a, + const Vec128<uint64_t> b) { + return Vec128<uint64_t>(vzip2q_u64(a.raw, b.raw)); +} +HWY_API Vec128<int64_t> InterleaveUpper(Vec128<int64_t> a, Vec128<int64_t> b) { + return Vec128<int64_t>(vzip2q_s64(a.raw, b.raw)); +} +HWY_API Vec128<double> InterleaveUpper(Vec128<double> a, Vec128<double> b) { + return Vec128<double>(vzip2q_f64(a.raw, b.raw)); +} +#else +// ARMv7 emulation. +HWY_API Vec128<uint64_t> InterleaveUpper(const Vec128<uint64_t> a, + const Vec128<uint64_t> b) { + return CombineShiftRightBytes<8>(Full128<uint64_t>(), Shuffle01(b), a); +} +HWY_API Vec128<int64_t> InterleaveUpper(Vec128<int64_t> a, Vec128<int64_t> b) { + return CombineShiftRightBytes<8>(Full128<int64_t>(), Shuffle01(b), a); +} +#endif + +HWY_API Vec128<float> InterleaveUpper(Vec128<float> a, Vec128<float> b) { + return Vec128<float>(vzip2q_f32(a.raw, b.raw)); +} +HWY_API Vec128<float, 2> InterleaveUpper(const Vec128<float, 2> a, + const Vec128<float, 2> b) { + return Vec128<float, 2>(vzip2_f32(a.raw, b.raw)); +} + +} // namespace detail + +// Full register +template <typename T, size_t N, HWY_IF_GE64(T, N), class V = Vec128<T, N>> +HWY_API V InterleaveUpper(Simd<T, N> /* tag */, V a, V b) { + return detail::InterleaveUpper(a, b); +} + +// Partial +template <typename T, size_t N, HWY_IF_LE32(T, N), class V = Vec128<T, N>> +HWY_API V InterleaveUpper(Simd<T, N> d, V a, V b) { + const Half<decltype(d)> d2; + return InterleaveLower(d, V(UpperHalf(d2, a).raw), V(UpperHalf(d2, b).raw)); +} + +// ------------------------------ ZipLower/ZipUpper (InterleaveLower) + +// Same as Interleave*, except that the return lanes are double-width integers; +// this is necessary because the single-lane scalar cannot return two values. +template <typename T, size_t N, class DW = RepartitionToWide<Simd<T, N>>> +HWY_API VFromD<DW> ZipLower(Vec128<T, N> a, Vec128<T, N> b) { + return BitCast(DW(), InterleaveLower(a, b)); +} +template <typename T, size_t N, class D = Simd<T, N>, + class DW = RepartitionToWide<D>> +HWY_API VFromD<DW> ZipLower(DW dw, Vec128<T, N> a, Vec128<T, N> b) { + return BitCast(dw, InterleaveLower(D(), a, b)); +} + +template <typename T, size_t N, class D = Simd<T, N>, + class DW = RepartitionToWide<D>> +HWY_API VFromD<DW> ZipUpper(DW dw, Vec128<T, N> a, Vec128<T, N> b) { + return BitCast(dw, InterleaveUpper(D(), a, b)); +} + +// ------------------------------ ReorderWidenMulAccumulate (MulAdd, ZipLower) + +template <size_t N> +HWY_API Vec128<float, N> ReorderWidenMulAccumulate(Simd<float, N> df32, + Vec128<bfloat16_t, 2 * N> a, + Vec128<bfloat16_t, 2 * N> b, + const Vec128<float, N> sum0, + Vec128<float, N>& sum1) { + const Repartition<uint16_t, decltype(df32)> du16; + const RebindToUnsigned<decltype(df32)> du32; + const Vec128<uint16_t, 2 * N> zero = Zero(du16); + const Vec128<uint32_t, N> a0 = ZipLower(du32, zero, BitCast(du16, a)); + const Vec128<uint32_t, N> a1 = ZipUpper(du32, zero, BitCast(du16, a)); + const Vec128<uint32_t, N> b0 = ZipLower(du32, zero, BitCast(du16, b)); + const Vec128<uint32_t, N> b1 = ZipUpper(du32, zero, BitCast(du16, b)); + sum1 = MulAdd(BitCast(df32, a1), BitCast(df32, b1), sum1); + return MulAdd(BitCast(df32, a0), BitCast(df32, b0), sum0); +} + +// ================================================== COMBINE + +// ------------------------------ Combine (InterleaveLower) + +// Full result +HWY_API Vec128<uint8_t> Combine(Full128<uint8_t> /* tag */, + Vec128<uint8_t, 8> hi, Vec128<uint8_t, 8> lo) { + return Vec128<uint8_t>(vcombine_u8(lo.raw, hi.raw)); +} +HWY_API Vec128<uint16_t> Combine(Full128<uint16_t> /* tag */, + Vec128<uint16_t, 4> hi, + Vec128<uint16_t, 4> lo) { + return Vec128<uint16_t>(vcombine_u16(lo.raw, hi.raw)); +} +HWY_API Vec128<uint32_t> Combine(Full128<uint32_t> /* tag */, + Vec128<uint32_t, 2> hi, + Vec128<uint32_t, 2> lo) { + return Vec128<uint32_t>(vcombine_u32(lo.raw, hi.raw)); +} +HWY_API Vec128<uint64_t> Combine(Full128<uint64_t> /* tag */, + Vec128<uint64_t, 1> hi, + Vec128<uint64_t, 1> lo) { + return Vec128<uint64_t>(vcombine_u64(lo.raw, hi.raw)); +} + +HWY_API Vec128<int8_t> Combine(Full128<int8_t> /* tag */, Vec128<int8_t, 8> hi, + Vec128<int8_t, 8> lo) { + return Vec128<int8_t>(vcombine_s8(lo.raw, hi.raw)); +} +HWY_API Vec128<int16_t> Combine(Full128<int16_t> /* tag */, + Vec128<int16_t, 4> hi, Vec128<int16_t, 4> lo) { + return Vec128<int16_t>(vcombine_s16(lo.raw, hi.raw)); +} +HWY_API Vec128<int32_t> Combine(Full128<int32_t> /* tag */, + Vec128<int32_t, 2> hi, Vec128<int32_t, 2> lo) { + return Vec128<int32_t>(vcombine_s32(lo.raw, hi.raw)); +} +HWY_API Vec128<int64_t> Combine(Full128<int64_t> /* tag */, + Vec128<int64_t, 1> hi, Vec128<int64_t, 1> lo) { + return Vec128<int64_t>(vcombine_s64(lo.raw, hi.raw)); +} + +HWY_API Vec128<float> Combine(Full128<float> /* tag */, Vec128<float, 2> hi, + Vec128<float, 2> lo) { + return Vec128<float>(vcombine_f32(lo.raw, hi.raw)); +} +#if HWY_ARCH_ARM_A64 +HWY_API Vec128<double> Combine(Full128<double> /* tag */, Vec128<double, 1> hi, + Vec128<double, 1> lo) { + return Vec128<double>(vcombine_f64(lo.raw, hi.raw)); +} +#endif + +// < 64bit input, <= 64 bit result +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_API Vec128<T, N> Combine(Simd<T, N> d, Vec128<T, N / 2> hi, + Vec128<T, N / 2> lo) { + // First double N (only lower halves will be used). + const Vec128<T, N> hi2(hi.raw); + const Vec128<T, N> lo2(lo.raw); + // Repartition to two unsigned lanes (each the size of the valid input). + const Simd<UnsignedFromSize<N * sizeof(T) / 2>, 2> du; + return BitCast(d, InterleaveLower(BitCast(du, lo2), BitCast(du, hi2))); +} + +// ------------------------------ ZeroExtendVector (Combine) + +template <typename T, size_t N> +HWY_API Vec128<T, N> ZeroExtendVector(Simd<T, N> d, Vec128<T, N / 2> lo) { + return Combine(d, Zero(Half<decltype(d)>()), lo); +} + +// ------------------------------ ConcatLowerLower + +// 64 or 128-bit input: just interleave +template <typename T, size_t N, HWY_IF_GE64(T, N)> +HWY_API Vec128<T, N> ConcatLowerLower(const Simd<T, N> d, Vec128<T, N> hi, + Vec128<T, N> lo) { + // Treat half-width input as a single lane and interleave them. + const Repartition<UnsignedFromSize<N * sizeof(T) / 2>, decltype(d)> du; + return BitCast(d, InterleaveLower(BitCast(du, lo), BitCast(du, hi))); +} + +#if HWY_ARCH_ARM_A64 +namespace detail { + +HWY_INLINE Vec128<uint8_t, 2> ConcatEven(Vec128<uint8_t, 2> hi, + Vec128<uint8_t, 2> lo) { + return Vec128<uint8_t, 2>(vtrn1_u8(lo.raw, hi.raw)); +} +HWY_INLINE Vec128<uint16_t, 2> ConcatEven(Vec128<uint16_t, 2> hi, + Vec128<uint16_t, 2> lo) { + return Vec128<uint16_t, 2>(vtrn1_u16(lo.raw, hi.raw)); +} + +} // namespace detail + +// <= 32-bit input/output +template <typename T, size_t N, HWY_IF_LE32(T, N)> +HWY_API Vec128<T, N> ConcatLowerLower(const Simd<T, N> d, Vec128<T, N> hi, + Vec128<T, N> lo) { + // Treat half-width input as two lanes and take every second one. + const Repartition<UnsignedFromSize<N * sizeof(T) / 2>, decltype(d)> du; + return BitCast(d, detail::ConcatEven(BitCast(du, hi), BitCast(du, lo))); +} + +#else + +template <typename T, size_t N, HWY_IF_LE32(T, N)> +HWY_API Vec128<T, N> ConcatLowerLower(const Simd<T, N> d, Vec128<T, N> hi, + Vec128<T, N> lo) { + const Half<decltype(d)> d2; + return Combine(LowerHalf(d2, hi), LowerHalf(d2, lo)); +} +#endif // HWY_ARCH_ARM_A64 + +// ------------------------------ ConcatUpperUpper + +// 64 or 128-bit input: just interleave +template <typename T, size_t N, HWY_IF_GE64(T, N)> +HWY_API Vec128<T, N> ConcatUpperUpper(const Simd<T, N> d, Vec128<T, N> hi, + Vec128<T, N> lo) { + // Treat half-width input as a single lane and interleave them. + const Repartition<UnsignedFromSize<N * sizeof(T) / 2>, decltype(d)> du; + return BitCast(d, InterleaveUpper(du, BitCast(du, lo), BitCast(du, hi))); +} + +#if HWY_ARCH_ARM_A64 +namespace detail { + +HWY_INLINE Vec128<uint8_t, 2> ConcatOdd(Vec128<uint8_t, 2> hi, + Vec128<uint8_t, 2> lo) { + return Vec128<uint8_t, 2>(vtrn2_u8(lo.raw, hi.raw)); +} +HWY_INLINE Vec128<uint16_t, 2> ConcatOdd(Vec128<uint16_t, 2> hi, + Vec128<uint16_t, 2> lo) { + return Vec128<uint16_t, 2>(vtrn2_u16(lo.raw, hi.raw)); +} + +} // namespace detail + +// <= 32-bit input/output +template <typename T, size_t N, HWY_IF_LE32(T, N)> +HWY_API Vec128<T, N> ConcatUpperUpper(const Simd<T, N> d, Vec128<T, N> hi, + Vec128<T, N> lo) { + // Treat half-width input as two lanes and take every second one. + const Repartition<UnsignedFromSize<N * sizeof(T) / 2>, decltype(d)> du; + return BitCast(d, detail::ConcatOdd(BitCast(du, hi), BitCast(du, lo))); +} + +#else + +template <typename T, size_t N, HWY_IF_LE32(T, N)> +HWY_API Vec128<T, N> ConcatUpperUpper(const Simd<T, N> d, Vec128<T, N> hi, + Vec128<T, N> lo) { + const Half<decltype(d)> d2; + return Combine(UpperHalf(d2, hi), UpperHalf(d2, lo)); +} + +#endif // HWY_ARCH_ARM_A64 + +// ------------------------------ ConcatLowerUpper (ShiftLeftBytes) + +// 64 or 128-bit input: extract from concatenated +template <typename T, size_t N, HWY_IF_GE64(T, N)> +HWY_API Vec128<T, N> ConcatLowerUpper(const Simd<T, N> d, Vec128<T, N> hi, + Vec128<T, N> lo) { + return CombineShiftRightBytes<N * sizeof(T) / 2>(d, hi, lo); +} + +// <= 32-bit input/output +template <typename T, size_t N, HWY_IF_LE32(T, N)> +HWY_API Vec128<T, N> ConcatLowerUpper(const Simd<T, N> d, Vec128<T, N> hi, + Vec128<T, N> lo) { + constexpr size_t kSize = N * sizeof(T); + const Repartition<uint8_t, decltype(d)> d8; + const Simd<uint8_t, 8> d8x8; + const Simd<T, 8 / sizeof(T)> d64; + using V8x8 = VFromD<decltype(d8x8)>; + const V8x8 hi8x8(BitCast(d8, hi).raw); + // Move into most-significant bytes + const V8x8 lo8x8 = ShiftLeftBytes<8 - kSize>(V8x8(BitCast(d8, lo).raw)); + const V8x8 r = CombineShiftRightBytes<8 - kSize / 2>(d8x8, hi8x8, lo8x8); + // Back to original lane type, then shrink N. + return Vec128<T, N>(BitCast(d64, r).raw); +} + +// ------------------------------ ConcatUpperLower + +// Works for all N. +template <typename T, size_t N> +HWY_API Vec128<T, N> ConcatUpperLower(Simd<T, N> d, Vec128<T, N> hi, + Vec128<T, N> lo) { + return IfThenElse(FirstN(d, Lanes(d) / 2), lo, hi); +} + +// ------------------------------ ConcatOdd (InterleaveUpper) + +// 32-bit full +HWY_API Vec128<uint32_t> ConcatOdd(Full128<uint32_t> /* tag */, + Vec128<uint32_t> hi, Vec128<uint32_t> lo) { + return Vec128<uint32_t>(vuzp2q_u32(lo.raw, hi.raw)); +} +HWY_API Vec128<int32_t> ConcatOdd(Full128<int32_t> /* tag */, + Vec128<int32_t> hi, Vec128<int32_t> lo) { + return Vec128<int32_t>(vuzp2q_s32(lo.raw, hi.raw)); +} +HWY_API Vec128<float> ConcatOdd(Full128<float> /* tag */, Vec128<float> hi, + Vec128<float> lo) { + return Vec128<float>(vuzp2q_f32(lo.raw, hi.raw)); +} + +// 32-bit partial +template <size_t N, HWY_IF_LE64(uint32_t, N)> +HWY_API Vec128<uint32_t, N> ConcatOdd(Simd<uint32_t, N> /* tag */, + Vec128<uint32_t, N> hi, + Vec128<uint32_t, N> lo) { + return Vec128<uint32_t, N>(vuzp2_u32(lo.raw, hi.raw)); +} +template <size_t N, HWY_IF_LE64(int32_t, N)> +HWY_API Vec128<int32_t, N> ConcatOdd(Simd<int32_t, N> /* tag */, + Vec128<int32_t, N> hi, + Vec128<int32_t, N> lo) { + return Vec128<int32_t, N>(vuzp2_s32(lo.raw, hi.raw)); +} +template <size_t N, HWY_IF_LE64(float, N)> +HWY_API Vec128<float, N> ConcatOdd(Simd<float, N> /* tag */, + Vec128<float, N> hi, Vec128<float, N> lo) { + return Vec128<float, N>(vuzp2_f32(lo.raw, hi.raw)); +} + +// 64-bit full - no partial because we need at least two inputs to have +// even/odd. ARMv7 lacks vuzpq_u64, and it's anyway the same as InterleaveUpper. +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec128<T> ConcatOdd(Full128<T> d, Vec128<T> hi, Vec128<T> lo) { + return InterleaveUpper(d, lo, hi); +} + +// ------------------------------ ConcatEven (InterleaveLower) + +// 32-bit full +HWY_API Vec128<uint32_t> ConcatEven(Full128<uint32_t> /* tag */, + Vec128<uint32_t> hi, Vec128<uint32_t> lo) { + return Vec128<uint32_t>(vuzp1q_u32(lo.raw, hi.raw)); +} +HWY_API Vec128<int32_t> ConcatEven(Full128<int32_t> /* tag */, + Vec128<int32_t> hi, Vec128<int32_t> lo) { + return Vec128<int32_t>(vuzp1q_s32(lo.raw, hi.raw)); +} +HWY_API Vec128<float> ConcatEven(Full128<float> /* tag */, Vec128<float> hi, + Vec128<float> lo) { + return Vec128<float>(vuzp1q_f32(lo.raw, hi.raw)); +} + +// 32-bit partial +template <size_t N, HWY_IF_LE64(uint32_t, N)> +HWY_API Vec128<uint32_t, N> ConcatEven(Simd<uint32_t, N> /* tag */, + Vec128<uint32_t, N> hi, + Vec128<uint32_t, N> lo) { + return Vec128<uint32_t, N>(vuzp1_u32(lo.raw, hi.raw)); +} +template <size_t N, HWY_IF_LE64(int32_t, N)> +HWY_API Vec128<int32_t, N> ConcatEven(Simd<int32_t, N> /* tag */, + Vec128<int32_t, N> hi, + Vec128<int32_t, N> lo) { + return Vec128<int32_t, N>(vuzp1_s32(lo.raw, hi.raw)); +} +template <size_t N, HWY_IF_LE64(float, N)> +HWY_API Vec128<float, N> ConcatEven(Simd<float, N> /* tag */, + Vec128<float, N> hi, Vec128<float, N> lo) { + return Vec128<float, N>(vuzp1_f32(lo.raw, hi.raw)); +} + +// 64-bit full - no partial because we need at least two inputs to have +// even/odd. ARMv7 lacks vuzpq_u64, and it's anyway the same as InterleaveUpper. +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec128<T> ConcatEven(Full128<T> d, Vec128<T> hi, Vec128<T> lo) { + return InterleaveLower(d, lo, hi); +} + +// ------------------------------ OddEven (IfThenElse) + +template <typename T, size_t N> +HWY_API Vec128<T, N> OddEven(const Vec128<T, N> a, const Vec128<T, N> b) { + const Simd<T, N> d; + const Repartition<uint8_t, decltype(d)> d8; + alignas(16) constexpr uint8_t kBytes[16] = { + ((0 / sizeof(T)) & 1) ? 0 : 0xFF, ((1 / sizeof(T)) & 1) ? 0 : 0xFF, + ((2 / sizeof(T)) & 1) ? 0 : 0xFF, ((3 / sizeof(T)) & 1) ? 0 : 0xFF, + ((4 / sizeof(T)) & 1) ? 0 : 0xFF, ((5 / sizeof(T)) & 1) ? 0 : 0xFF, + ((6 / sizeof(T)) & 1) ? 0 : 0xFF, ((7 / sizeof(T)) & 1) ? 0 : 0xFF, + ((8 / sizeof(T)) & 1) ? 0 : 0xFF, ((9 / sizeof(T)) & 1) ? 0 : 0xFF, + ((10 / sizeof(T)) & 1) ? 0 : 0xFF, ((11 / sizeof(T)) & 1) ? 0 : 0xFF, + ((12 / sizeof(T)) & 1) ? 0 : 0xFF, ((13 / sizeof(T)) & 1) ? 0 : 0xFF, + ((14 / sizeof(T)) & 1) ? 0 : 0xFF, ((15 / sizeof(T)) & 1) ? 0 : 0xFF, + }; + const auto vec = BitCast(d, Load(d8, kBytes)); + return IfThenElse(MaskFromVec(vec), b, a); +} + +// ------------------------------ OddEvenBlocks +template <typename T, size_t N> +HWY_API Vec128<T, N> OddEvenBlocks(Vec128<T, N> /* odd */, Vec128<T, N> even) { + return even; +} + +// ------------------------------ SwapAdjacentBlocks + +template <typename T, size_t N> +HWY_API Vec128<T, N> SwapAdjacentBlocks(Vec128<T, N> v) { + return v; +} + +// ------------------------------ ReorderDemote2To (OddEven) + +template <size_t N> +HWY_API Vec128<bfloat16_t, 2 * N> ReorderDemote2To( + Simd<bfloat16_t, 2 * N> dbf16, Vec128<float, N> a, Vec128<float, N> b) { + const RebindToUnsigned<decltype(dbf16)> du16; + const Repartition<uint32_t, decltype(dbf16)> du32; + const Vec128<uint32_t, N> b_in_even = ShiftRight<16>(BitCast(du32, b)); + return BitCast(dbf16, OddEven(BitCast(du16, a), BitCast(du16, b_in_even))); +} + +// ================================================== CRYPTO + +#if defined(__ARM_FEATURE_AES) + +// Per-target flag to prevent generic_ops-inl.h from defining AESRound. +#ifdef HWY_NATIVE_AES +#undef HWY_NATIVE_AES +#else +#define HWY_NATIVE_AES +#endif + +HWY_API Vec128<uint8_t> AESRound(Vec128<uint8_t> state, + Vec128<uint8_t> round_key) { + // NOTE: it is important that AESE and AESMC be consecutive instructions so + // they can be fused. AESE includes AddRoundKey, which is a different ordering + // than the AES-NI semantics we adopted, so XOR by 0 and later with the actual + // round key (the compiler will hopefully optimize this for multiple rounds). + return Vec128<uint8_t>(vaesmcq_u8(vaeseq_u8(state.raw, vdupq_n_u8(0)))) ^ + round_key; +} + +HWY_API Vec128<uint64_t> CLMulLower(Vec128<uint64_t> a, Vec128<uint64_t> b) { + return Vec128<uint64_t>((uint64x2_t)vmull_p64(GetLane(a), GetLane(b))); +} + +HWY_API Vec128<uint64_t> CLMulUpper(Vec128<uint64_t> a, Vec128<uint64_t> b) { + return Vec128<uint64_t>( + (uint64x2_t)vmull_high_p64((poly64x2_t)a.raw, (poly64x2_t)b.raw)); +} + +#endif // __ARM_FEATURE_AES + +// ================================================== MISC + +template <size_t N> +HWY_API Vec128<float, N> PromoteTo(Simd<float, N> df32, + const Vec128<bfloat16_t, N> v) { + const Rebind<uint16_t, decltype(df32)> du16; + const RebindToSigned<decltype(df32)> di32; + return BitCast(df32, ShiftLeft<16>(PromoteTo(di32, BitCast(du16, v)))); +} + +// ------------------------------ MulEven (ConcatEven) + +// Multiplies even lanes (0, 2 ..) and places the double-wide result into +// even and the upper half into its odd neighbor lane. +HWY_API Vec128<int64_t> MulEven(Vec128<int32_t> a, Vec128<int32_t> b) { + const Full128<int32_t> d; + int32x4_t a_packed = ConcatEven(d, a, a).raw; + int32x4_t b_packed = ConcatEven(d, b, b).raw; + return Vec128<int64_t>( + vmull_s32(vget_low_s32(a_packed), vget_low_s32(b_packed))); +} +HWY_API Vec128<uint64_t> MulEven(Vec128<uint32_t> a, Vec128<uint32_t> b) { + const Full128<uint32_t> d; + uint32x4_t a_packed = ConcatEven(d, a, a).raw; + uint32x4_t b_packed = ConcatEven(d, b, b).raw; + return Vec128<uint64_t>( + vmull_u32(vget_low_u32(a_packed), vget_low_u32(b_packed))); +} + +template <size_t N> +HWY_API Vec128<int64_t, (N + 1) / 2> MulEven(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + const Simd<int32_t, N> d; + int32x2_t a_packed = ConcatEven(d, a, a).raw; + int32x2_t b_packed = ConcatEven(d, b, b).raw; + return Vec128<int64_t, (N + 1) / 2>( + vget_low_s64(vmull_s32(a_packed, b_packed))); +} +template <size_t N> +HWY_API Vec128<uint64_t, (N + 1) / 2> MulEven(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { + const Simd<uint32_t, N> d; + uint32x2_t a_packed = ConcatEven(d, a, a).raw; + uint32x2_t b_packed = ConcatEven(d, b, b).raw; + return Vec128<uint64_t, (N + 1) / 2>( + vget_low_u64(vmull_u32(a_packed, b_packed))); +} + +HWY_INLINE Vec128<uint64_t> MulEven(Vec128<uint64_t> a, Vec128<uint64_t> b) { + uint64_t hi; + uint64_t lo = Mul128(vgetq_lane_u64(a.raw, 0), vgetq_lane_u64(b.raw, 0), &hi); + return Vec128<uint64_t>(vsetq_lane_u64(hi, vdupq_n_u64(lo), 1)); +} + +HWY_INLINE Vec128<uint64_t> MulOdd(Vec128<uint64_t> a, Vec128<uint64_t> b) { + uint64_t hi; + uint64_t lo = Mul128(vgetq_lane_u64(a.raw, 1), vgetq_lane_u64(b.raw, 1), &hi); + return Vec128<uint64_t>(vsetq_lane_u64(hi, vdupq_n_u64(lo), 1)); +} + +// ------------------------------ TableLookupBytes (Combine, LowerHalf) + +// Both full +template <typename T, typename TI> +HWY_API Vec128<TI> TableLookupBytes(const Vec128<T> bytes, + const Vec128<TI> from) { + const Full128<TI> d; + const Repartition<uint8_t, decltype(d)> d8; +#if HWY_ARCH_ARM_A64 + return BitCast(d, Vec128<uint8_t>(vqtbl1q_u8(BitCast(d8, bytes).raw, + BitCast(d8, from).raw))); +#else + uint8x16_t table0 = BitCast(d8, bytes).raw; + uint8x8x2_t table; + table.val[0] = vget_low_u8(table0); + table.val[1] = vget_high_u8(table0); + uint8x16_t idx = BitCast(d8, from).raw; + uint8x8_t low = vtbl2_u8(table, vget_low_u8(idx)); + uint8x8_t hi = vtbl2_u8(table, vget_high_u8(idx)); + return BitCast(d, Vec128<uint8_t>(vcombine_u8(low, hi))); +#endif +} + +// Partial index vector +template <typename T, typename TI, size_t NI, HWY_IF_LE64(TI, NI)> +HWY_API Vec128<TI, NI> TableLookupBytes(const Vec128<T> bytes, + const Vec128<TI, NI> from) { + const Full128<TI> d_full; + const Vec128<TI, 8 / sizeof(T)> from64(from.raw); + const auto idx_full = Combine(d_full, from64, from64); + const auto out_full = TableLookupBytes(bytes, idx_full); + return Vec128<TI, NI>(LowerHalf(Half<decltype(d_full)>(), out_full).raw); +} + +// Partial table vector +template <typename T, size_t N, typename TI, HWY_IF_LE64(T, N)> +HWY_API Vec128<TI> TableLookupBytes(const Vec128<T, N> bytes, + const Vec128<TI> from) { + const Full128<T> d_full; + return TableLookupBytes(Combine(d_full, bytes, bytes), from); +} + +// Partial both +template <typename T, size_t N, typename TI, size_t NI, HWY_IF_LE64(T, N), + HWY_IF_LE64(TI, NI)> +HWY_API VFromD<Repartition<T, Simd<TI, NI>>> TableLookupBytes( + Vec128<T, N> bytes, Vec128<TI, NI> from) { + const Simd<T, N> d; + const Simd<TI, NI> d_idx; + const Repartition<uint8_t, decltype(d_idx)> d_idx8; + // uint8x8 + const auto bytes8 = BitCast(Repartition<uint8_t, decltype(d)>(), bytes); + const auto from8 = BitCast(d_idx8, from); + const VFromD<decltype(d_idx8)> v8(vtbl1_u8(bytes8.raw, from8.raw)); + return BitCast(d_idx, v8); +} + +// For all vector widths; ARM anyway zeroes if >= 0x10. +template <class V, class VI> +HWY_API VI TableLookupBytesOr0(const V bytes, const VI from) { + return TableLookupBytes(bytes, from); +} + +// ------------------------------ Scatter (Store) + +template <typename T, size_t N, typename Offset, HWY_IF_LE128(T, N)> +HWY_API void ScatterOffset(Vec128<T, N> v, Simd<T, N> d, T* HWY_RESTRICT base, + const Vec128<Offset, N> offset) { + static_assert(sizeof(T) == sizeof(Offset), "Must match for portability"); + + alignas(16) T lanes[N]; + Store(v, d, lanes); + + alignas(16) Offset offset_lanes[N]; + Store(offset, Simd<Offset, N>(), offset_lanes); + + uint8_t* base_bytes = reinterpret_cast<uint8_t*>(base); + for (size_t i = 0; i < N; ++i) { + CopyBytes<sizeof(T)>(&lanes[i], base_bytes + offset_lanes[i]); + } +} + +template <typename T, size_t N, typename Index, HWY_IF_LE128(T, N)> +HWY_API void ScatterIndex(Vec128<T, N> v, Simd<T, N> d, T* HWY_RESTRICT base, + const Vec128<Index, N> index) { + static_assert(sizeof(T) == sizeof(Index), "Must match for portability"); + + alignas(16) T lanes[N]; + Store(v, d, lanes); + + alignas(16) Index index_lanes[N]; + Store(index, Simd<Index, N>(), index_lanes); + + for (size_t i = 0; i < N; ++i) { + base[index_lanes[i]] = lanes[i]; + } +} + +// ------------------------------ Gather (Load/Store) + +template <typename T, size_t N, typename Offset> +HWY_API Vec128<T, N> GatherOffset(const Simd<T, N> d, + const T* HWY_RESTRICT base, + const Vec128<Offset, N> offset) { + static_assert(sizeof(T) == sizeof(Offset), "Must match for portability"); + + alignas(16) Offset offset_lanes[N]; + Store(offset, Simd<Offset, N>(), offset_lanes); + + alignas(16) T lanes[N]; + const uint8_t* base_bytes = reinterpret_cast<const uint8_t*>(base); + for (size_t i = 0; i < N; ++i) { + CopyBytes<sizeof(T)>(base_bytes + offset_lanes[i], &lanes[i]); + } + return Load(d, lanes); +} + +template <typename T, size_t N, typename Index> +HWY_API Vec128<T, N> GatherIndex(const Simd<T, N> d, const T* HWY_RESTRICT base, + const Vec128<Index, N> index) { + static_assert(sizeof(T) == sizeof(Index), "Must match for portability"); + + alignas(16) Index index_lanes[N]; + Store(index, Simd<Index, N>(), index_lanes); + + alignas(16) T lanes[N]; + for (size_t i = 0; i < N; ++i) { + lanes[i] = base[index_lanes[i]]; + } + return Load(d, lanes); +} + +// ------------------------------ Reductions + +namespace detail { + +// N=1 for any T: no-op +template <typename T> +HWY_INLINE Vec128<T, 1> SumOfLanes(const Vec128<T, 1> v) { + return v; +} +template <typename T> +HWY_INLINE Vec128<T, 1> MinOfLanes(hwy::SizeTag<sizeof(T)> /* tag */, + const Vec128<T, 1> v) { + return v; +} +template <typename T> +HWY_INLINE Vec128<T, 1> MaxOfLanes(hwy::SizeTag<sizeof(T)> /* tag */, + const Vec128<T, 1> v) { + return v; +} + +// u32/i32/f32: N=2 +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_INLINE Vec128<T, 2> SumOfLanes(const Vec128<T, 2> v10) { + return v10 + Shuffle2301(v10); +} +template <typename T> +HWY_INLINE Vec128<T, 2> MinOfLanes(hwy::SizeTag<4> /* tag */, + const Vec128<T, 2> v10) { + return Min(v10, Shuffle2301(v10)); +} +template <typename T> +HWY_INLINE Vec128<T, 2> MaxOfLanes(hwy::SizeTag<4> /* tag */, + const Vec128<T, 2> v10) { + return Max(v10, Shuffle2301(v10)); +} + +// full vectors +#if HWY_ARCH_ARM_A64 +HWY_INLINE Vec128<uint32_t> SumOfLanes(const Vec128<uint32_t> v) { + return Vec128<uint32_t>(vdupq_n_u32(vaddvq_u32(v.raw))); +} +HWY_INLINE Vec128<int32_t> SumOfLanes(const Vec128<int32_t> v) { + return Vec128<int32_t>(vdupq_n_s32(vaddvq_s32(v.raw))); +} +HWY_INLINE Vec128<float> SumOfLanes(const Vec128<float> v) { + return Vec128<float>(vdupq_n_f32(vaddvq_f32(v.raw))); +} +HWY_INLINE Vec128<uint64_t> SumOfLanes(const Vec128<uint64_t> v) { + return Vec128<uint64_t>(vdupq_n_u64(vaddvq_u64(v.raw))); +} +HWY_INLINE Vec128<int64_t> SumOfLanes(const Vec128<int64_t> v) { + return Vec128<int64_t>(vdupq_n_s64(vaddvq_s64(v.raw))); +} +HWY_INLINE Vec128<double> SumOfLanes(const Vec128<double> v) { + return Vec128<double>(vdupq_n_f64(vaddvq_f64(v.raw))); +} +#else +// ARMv7 version for everything except doubles. +HWY_INLINE Vec128<uint32_t> SumOfLanes(const Vec128<uint32_t> v) { + uint32x4x2_t v0 = vuzpq_u32(v.raw, v.raw); + uint32x4_t c0 = vaddq_u32(v0.val[0], v0.val[1]); + uint32x4x2_t v1 = vuzpq_u32(c0, c0); + return Vec128<uint32_t>(vaddq_u32(v1.val[0], v1.val[1])); +} +HWY_INLINE Vec128<int32_t> SumOfLanes(const Vec128<int32_t> v) { + int32x4x2_t v0 = vuzpq_s32(v.raw, v.raw); + int32x4_t c0 = vaddq_s32(v0.val[0], v0.val[1]); + int32x4x2_t v1 = vuzpq_s32(c0, c0); + return Vec128<int32_t>(vaddq_s32(v1.val[0], v1.val[1])); +} +HWY_INLINE Vec128<float> SumOfLanes(const Vec128<float> v) { + float32x4x2_t v0 = vuzpq_f32(v.raw, v.raw); + float32x4_t c0 = vaddq_f32(v0.val[0], v0.val[1]); + float32x4x2_t v1 = vuzpq_f32(c0, c0); + return Vec128<float>(vaddq_f32(v1.val[0], v1.val[1])); +} +HWY_INLINE Vec128<uint64_t> SumOfLanes(const Vec128<uint64_t> v) { + return v + Shuffle01(v); +} +HWY_INLINE Vec128<int64_t> SumOfLanes(const Vec128<int64_t> v) { + return v + Shuffle01(v); +} +#endif + +template <typename T> +HWY_INLINE Vec128<T> MinOfLanes(hwy::SizeTag<4> /* tag */, + const Vec128<T> v3210) { + const Vec128<T> v1032 = Shuffle1032(v3210); + const Vec128<T> v31_20_31_20 = Min(v3210, v1032); + const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20); + return Min(v20_31_20_31, v31_20_31_20); +} +template <typename T> +HWY_INLINE Vec128<T> MaxOfLanes(hwy::SizeTag<4> /* tag */, + const Vec128<T> v3210) { + const Vec128<T> v1032 = Shuffle1032(v3210); + const Vec128<T> v31_20_31_20 = Max(v3210, v1032); + const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20); + return Max(v20_31_20_31, v31_20_31_20); +} + +// For u64/i64[/f64]. +template <typename T> +HWY_INLINE Vec128<T> MinOfLanes(hwy::SizeTag<8> /* tag */, + const Vec128<T> v10) { + const Vec128<T> v01 = Shuffle01(v10); + return Min(v10, v01); +} +template <typename T> +HWY_INLINE Vec128<T> MaxOfLanes(hwy::SizeTag<8> /* tag */, + const Vec128<T> v10) { + const Vec128<T> v01 = Shuffle01(v10); + return Max(v10, v01); +} + +// u16/i16 +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 2), HWY_IF_GE32(T, N)> +HWY_API Vec128<T, N> MinOfLanes(hwy::SizeTag<2> /* tag */, Vec128<T, N> v) { + const Repartition<int32_t, Simd<T, N>> d32; + const auto even = And(BitCast(d32, v), Set(d32, 0xFFFF)); + const auto odd = ShiftRight<16>(BitCast(d32, v)); + const auto min = MinOfLanes(d32, Min(even, odd)); + // Also broadcast into odd lanes. + return BitCast(Simd<T, N>(), Or(min, ShiftLeft<16>(min))); +} +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 2), HWY_IF_GE32(T, N)> +HWY_API Vec128<T, N> MaxOfLanes(hwy::SizeTag<2> /* tag */, Vec128<T, N> v) { + const Repartition<int32_t, Simd<T, N>> d32; + const auto even = And(BitCast(d32, v), Set(d32, 0xFFFF)); + const auto odd = ShiftRight<16>(BitCast(d32, v)); + const auto min = MaxOfLanes(d32, Max(even, odd)); + // Also broadcast into odd lanes. + return BitCast(Simd<T, N>(), Or(min, ShiftLeft<16>(min))); +} + +} // namespace detail + +template <typename T, size_t N> +HWY_API Vec128<T, N> SumOfLanes(Simd<T, N> /* tag */, const Vec128<T, N> v) { + return detail::SumOfLanes(v); +} +template <typename T, size_t N> +HWY_API Vec128<T, N> MinOfLanes(Simd<T, N> /* tag */, const Vec128<T, N> v) { + return detail::MinOfLanes(hwy::SizeTag<sizeof(T)>(), v); +} +template <typename T, size_t N> +HWY_API Vec128<T, N> MaxOfLanes(Simd<T, N> /* tag */, const Vec128<T, N> v) { + return detail::MaxOfLanes(hwy::SizeTag<sizeof(T)>(), v); +} + +// ------------------------------ LoadMaskBits (TestBit) + +namespace detail { + +// Helper function to set 64 bits and potentially return a smaller vector. The +// overload is required to call the q vs non-q intrinsics. Note that 8-bit +// LoadMaskBits only requires 16 bits, but 64 avoids casting. +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_INLINE Vec128<T, N> Set64(Simd<T, N> /* tag */, uint64_t mask_bits) { + const auto v64 = Vec128<uint64_t, 1>(vdup_n_u64(mask_bits)); + return Vec128<T, N>(BitCast(Simd<T, 8 / sizeof(T)>(), v64).raw); +} +template <typename T> +HWY_INLINE Vec128<T> Set64(Full128<T> d, uint64_t mask_bits) { + return BitCast(d, Vec128<uint64_t>(vdupq_n_u64(mask_bits))); +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 1)> +HWY_INLINE Mask128<T, N> LoadMaskBits(Simd<T, N> d, uint64_t mask_bits) { + const RebindToUnsigned<decltype(d)> du; + // Easier than Set(), which would require an >8-bit type, which would not + // compile for T=uint8_t, N=1. + const auto vmask_bits = Set64(du, mask_bits); + + // Replicate bytes 8x such that each byte contains the bit that governs it. + alignas(16) constexpr uint8_t kRep8[16] = {0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1}; + const auto rep8 = TableLookupBytes(vmask_bits, Load(du, kRep8)); + + alignas(16) constexpr uint8_t kBit[16] = {1, 2, 4, 8, 16, 32, 64, 128, + 1, 2, 4, 8, 16, 32, 64, 128}; + return RebindMask(d, TestBit(rep8, LoadDup128(du, kBit))); +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 2)> +HWY_INLINE Mask128<T, N> LoadMaskBits(Simd<T, N> d, uint64_t mask_bits) { + const RebindToUnsigned<decltype(d)> du; + alignas(16) constexpr uint16_t kBit[8] = {1, 2, 4, 8, 16, 32, 64, 128}; + const auto vmask_bits = Set(du, static_cast<uint16_t>(mask_bits)); + return RebindMask(d, TestBit(vmask_bits, Load(du, kBit))); +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 4)> +HWY_INLINE Mask128<T, N> LoadMaskBits(Simd<T, N> d, uint64_t mask_bits) { + const RebindToUnsigned<decltype(d)> du; + alignas(16) constexpr uint32_t kBit[8] = {1, 2, 4, 8}; + const auto vmask_bits = Set(du, static_cast<uint32_t>(mask_bits)); + return RebindMask(d, TestBit(vmask_bits, Load(du, kBit))); +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 8)> +HWY_INLINE Mask128<T, N> LoadMaskBits(Simd<T, N> d, uint64_t mask_bits) { + const RebindToUnsigned<decltype(d)> du; + alignas(16) constexpr uint64_t kBit[8] = {1, 2}; + return RebindMask(d, TestBit(Set(du, mask_bits), Load(du, kBit))); +} + +} // namespace detail + +// `p` points to at least 8 readable bytes, not all of which need be valid. +template <typename T, size_t N, HWY_IF_LE128(T, N)> +HWY_API Mask128<T, N> LoadMaskBits(Simd<T, N> d, + const uint8_t* HWY_RESTRICT bits) { + uint64_t mask_bits = 0; + CopyBytes<(N + 7) / 8>(bits, &mask_bits); + return detail::LoadMaskBits(d, mask_bits); +} + +// ------------------------------ Mask + +namespace detail { + +template <typename T> +HWY_INLINE uint64_t BitsFromMask(hwy::SizeTag<1> /*tag*/, + const Mask128<T> mask) { + alignas(16) constexpr uint8_t kSliceLanes[16] = { + 1, 2, 4, 8, 0x10, 0x20, 0x40, 0x80, 1, 2, 4, 8, 0x10, 0x20, 0x40, 0x80, + }; + const Full128<uint8_t> du; + const Vec128<uint8_t> values = + BitCast(du, VecFromMask(Full128<T>(), mask)) & Load(du, kSliceLanes); + +#if HWY_ARCH_ARM_A64 + // Can't vaddv - we need two separate bytes (16 bits). + const uint8x8_t x2 = vget_low_u8(vpaddq_u8(values.raw, values.raw)); + const uint8x8_t x4 = vpadd_u8(x2, x2); + const uint8x8_t x8 = vpadd_u8(x4, x4); + return vget_lane_u64(vreinterpret_u64_u8(x8), 0); +#else + // Don't have vpaddq, so keep doubling lane size. + const uint16x8_t x2 = vpaddlq_u8(values.raw); + const uint32x4_t x4 = vpaddlq_u16(x2); + const uint64x2_t x8 = vpaddlq_u32(x4); + return (vgetq_lane_u64(x8, 1) << 8) | vgetq_lane_u64(x8, 0); +#endif +} + +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_INLINE uint64_t BitsFromMask(hwy::SizeTag<1> /*tag*/, + const Mask128<T, N> mask) { + // Upper lanes of partial loads are undefined. OnlyActive will fix this if + // we load all kSliceLanes so the upper lanes do not pollute the valid bits. + alignas(8) constexpr uint8_t kSliceLanes[8] = {1, 2, 4, 8, + 0x10, 0x20, 0x40, 0x80}; + const Simd<T, N> d; + const Simd<uint8_t, N> du; + const Vec128<uint8_t, N> slice(Load(Simd<uint8_t, 8>(), kSliceLanes).raw); + const Vec128<uint8_t, N> values = BitCast(du, VecFromMask(d, mask)) & slice; + +#if HWY_ARCH_ARM_A64 + return vaddv_u8(values.raw); +#else + const uint16x4_t x2 = vpaddl_u8(values.raw); + const uint32x2_t x4 = vpaddl_u16(x2); + const uint64x1_t x8 = vpaddl_u32(x4); + return vget_lane_u64(x8, 0); +#endif +} + +template <typename T> +HWY_INLINE uint64_t BitsFromMask(hwy::SizeTag<2> /*tag*/, + const Mask128<T> mask) { + alignas(16) constexpr uint16_t kSliceLanes[8] = {1, 2, 4, 8, + 0x10, 0x20, 0x40, 0x80}; + const Full128<T> d; + const Full128<uint16_t> du; + const Vec128<uint16_t> values = + BitCast(du, VecFromMask(d, mask)) & Load(du, kSliceLanes); +#if HWY_ARCH_ARM_A64 + return vaddvq_u16(values.raw); +#else + const uint32x4_t x2 = vpaddlq_u16(values.raw); + const uint64x2_t x4 = vpaddlq_u32(x2); + return vgetq_lane_u64(x4, 0) + vgetq_lane_u64(x4, 1); +#endif +} + +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_INLINE uint64_t BitsFromMask(hwy::SizeTag<2> /*tag*/, + const Mask128<T, N> mask) { + // Upper lanes of partial loads are undefined. OnlyActive will fix this if + // we load all kSliceLanes so the upper lanes do not pollute the valid bits. + alignas(8) constexpr uint16_t kSliceLanes[4] = {1, 2, 4, 8}; + const Simd<T, N> d; + const Simd<uint16_t, N> du; + const Vec128<uint16_t, N> slice(Load(Simd<uint16_t, 4>(), kSliceLanes).raw); + const Vec128<uint16_t, N> values = BitCast(du, VecFromMask(d, mask)) & slice; +#if HWY_ARCH_ARM_A64 + return vaddv_u16(values.raw); +#else + const uint32x2_t x2 = vpaddl_u16(values.raw); + const uint64x1_t x4 = vpaddl_u32(x2); + return vget_lane_u64(x4, 0); +#endif +} + +template <typename T> +HWY_INLINE uint64_t BitsFromMask(hwy::SizeTag<4> /*tag*/, + const Mask128<T> mask) { + alignas(16) constexpr uint32_t kSliceLanes[4] = {1, 2, 4, 8}; + const Full128<T> d; + const Full128<uint32_t> du; + const Vec128<uint32_t> values = + BitCast(du, VecFromMask(d, mask)) & Load(du, kSliceLanes); +#if HWY_ARCH_ARM_A64 + return vaddvq_u32(values.raw); +#else + const uint64x2_t x2 = vpaddlq_u32(values.raw); + return vgetq_lane_u64(x2, 0) + vgetq_lane_u64(x2, 1); +#endif +} + +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_INLINE uint64_t BitsFromMask(hwy::SizeTag<4> /*tag*/, + const Mask128<T, N> mask) { + // Upper lanes of partial loads are undefined. OnlyActive will fix this if + // we load all kSliceLanes so the upper lanes do not pollute the valid bits. + alignas(8) constexpr uint32_t kSliceLanes[2] = {1, 2}; + const Simd<T, N> d; + const Simd<uint32_t, N> du; + const Vec128<uint32_t, N> slice(Load(Simd<uint32_t, 2>(), kSliceLanes).raw); + const Vec128<uint32_t, N> values = BitCast(du, VecFromMask(d, mask)) & slice; +#if HWY_ARCH_ARM_A64 + return vaddv_u32(values.raw); +#else + const uint64x1_t x2 = vpaddl_u32(values.raw); + return vget_lane_u64(x2, 0); +#endif +} + +template <typename T> +HWY_INLINE uint64_t BitsFromMask(hwy::SizeTag<8> /*tag*/, const Mask128<T> m) { + alignas(16) constexpr uint64_t kSliceLanes[2] = {1, 2}; + const Full128<T> d; + const Full128<uint64_t> du; + const Vec128<uint64_t> values = + BitCast(du, VecFromMask(d, m)) & Load(du, kSliceLanes); +#if HWY_ARCH_ARM_A64 + return vaddvq_u64(values.raw); +#else + return vgetq_lane_u64(values.raw, 0) + vgetq_lane_u64(values.raw, 1); +#endif +} + +template <typename T> +HWY_INLINE uint64_t BitsFromMask(hwy::SizeTag<8> /*tag*/, + const Mask128<T, 1> m) { + const Simd<T, 1> d; + const Simd<uint64_t, 1> du; + const Vec128<uint64_t, 1> values = + BitCast(du, VecFromMask(d, m)) & Set(du, 1); + return vget_lane_u64(values.raw, 0); +} + +// Returns the lowest N for the BitsFromMask result. +template <typename T, size_t N> +constexpr uint64_t OnlyActive(uint64_t bits) { + return ((N * sizeof(T)) >= 8) ? bits : (bits & ((1ull << N) - 1)); +} + +template <typename T, size_t N> +HWY_INLINE uint64_t BitsFromMask(const Mask128<T, N> mask) { + return OnlyActive<T, N>(BitsFromMask(hwy::SizeTag<sizeof(T)>(), mask)); +} + +// Returns number of lanes whose mask is set. +// +// Masks are either FF..FF or 0. Unfortunately there is no reduce-sub op +// ("vsubv"). ANDing with 1 would work but requires a constant. Negating also +// changes each lane to 1 (if mask set) or 0. + +template <typename T> +HWY_INLINE size_t CountTrue(hwy::SizeTag<1> /*tag*/, const Mask128<T> mask) { + const Full128<int8_t> di; + const int8x16_t ones = + vnegq_s8(BitCast(di, VecFromMask(Full128<T>(), mask)).raw); + +#if HWY_ARCH_ARM_A64 + return static_cast<size_t>(vaddvq_s8(ones)); +#else + const int16x8_t x2 = vpaddlq_s8(ones); + const int32x4_t x4 = vpaddlq_s16(x2); + const int64x2_t x8 = vpaddlq_s32(x4); + return static_cast<size_t>(vgetq_lane_s64(x8, 0) + vgetq_lane_s64(x8, 1)); +#endif +} +template <typename T> +HWY_INLINE size_t CountTrue(hwy::SizeTag<2> /*tag*/, const Mask128<T> mask) { + const Full128<int16_t> di; + const int16x8_t ones = + vnegq_s16(BitCast(di, VecFromMask(Full128<T>(), mask)).raw); + +#if HWY_ARCH_ARM_A64 + return static_cast<size_t>(vaddvq_s16(ones)); +#else + const int32x4_t x2 = vpaddlq_s16(ones); + const int64x2_t x4 = vpaddlq_s32(x2); + return static_cast<size_t>(vgetq_lane_s64(x4, 0) + vgetq_lane_s64(x4, 1)); +#endif +} + +template <typename T> +HWY_INLINE size_t CountTrue(hwy::SizeTag<4> /*tag*/, const Mask128<T> mask) { + const Full128<int32_t> di; + const int32x4_t ones = + vnegq_s32(BitCast(di, VecFromMask(Full128<T>(), mask)).raw); + +#if HWY_ARCH_ARM_A64 + return static_cast<size_t>(vaddvq_s32(ones)); +#else + const int64x2_t x2 = vpaddlq_s32(ones); + return static_cast<size_t>(vgetq_lane_s64(x2, 0) + vgetq_lane_s64(x2, 1)); +#endif +} + +template <typename T> +HWY_INLINE size_t CountTrue(hwy::SizeTag<8> /*tag*/, const Mask128<T> mask) { +#if HWY_ARCH_ARM_A64 + const Full128<int64_t> di; + const int64x2_t ones = + vnegq_s64(BitCast(di, VecFromMask(Full128<T>(), mask)).raw); + return static_cast<size_t>(vaddvq_s64(ones)); +#else + const Full128<uint64_t> du; + const auto mask_u = VecFromMask(du, RebindMask(du, mask)); + const uint64x2_t ones = vshrq_n_u64(mask_u.raw, 63); + return static_cast<size_t>(vgetq_lane_u64(ones, 0) + vgetq_lane_u64(ones, 1)); +#endif +} + +} // namespace detail + +// Full +template <typename T> +HWY_API size_t CountTrue(Full128<T> /* tag */, const Mask128<T> mask) { + return detail::CountTrue(hwy::SizeTag<sizeof(T)>(), mask); +} + +// Partial +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_API size_t CountTrue(Simd<T, N> /* tag */, const Mask128<T, N> mask) { + return PopCount(detail::BitsFromMask(mask)); +} + +template <typename T, size_t N> +HWY_API intptr_t FindFirstTrue(const Simd<T, N> /* tag */, + const Mask128<T, N> mask) { + const uint64_t bits = detail::BitsFromMask(mask); + return bits ? static_cast<intptr_t>(Num0BitsBelowLS1Bit_Nonzero64(bits)) : -1; +} + +// `p` points to at least 8 writable bytes. +template <typename T, size_t N> +HWY_API size_t StoreMaskBits(Simd<T, N> /* tag */, const Mask128<T, N> mask, + uint8_t* bits) { + const uint64_t mask_bits = detail::BitsFromMask(mask); + const size_t kNumBytes = (N + 7) / 8; + CopyBytes<kNumBytes>(&mask_bits, bits); + return kNumBytes; +} + +// Full +template <typename T> +HWY_API bool AllFalse(const Full128<T> d, const Mask128<T> m) { +#if HWY_ARCH_ARM_A64 + const Full128<uint32_t> d32; + const auto m32 = MaskFromVec(BitCast(d32, VecFromMask(d, m))); + return (vmaxvq_u32(m32.raw) == 0); +#else + const auto v64 = BitCast(Full128<uint64_t>(), VecFromMask(d, m)); + uint32x2_t a = vqmovn_u64(v64.raw); + return vget_lane_u64(vreinterpret_u64_u32(a), 0) == 0; +#endif +} + +// Partial +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_API bool AllFalse(const Simd<T, N> /* tag */, const Mask128<T, N> m) { + return detail::BitsFromMask(m) == 0; +} + +template <typename T, size_t N> +HWY_API bool AllTrue(const Simd<T, N> d, const Mask128<T, N> m) { + return AllFalse(VecFromMask(d, m) == Zero(d)); +} + +// ------------------------------ Compress + +namespace detail { + +// Load 8 bytes, replicate into upper half so ZipLower can use the lower half. +HWY_INLINE Vec128<uint8_t> Load8Bytes(Full128<uint8_t> /*d*/, + const uint8_t* bytes) { + return Vec128<uint8_t>(vreinterpretq_u8_u64( + vld1q_dup_u64(reinterpret_cast<const uint64_t*>(bytes)))); +} + +// Load 8 bytes and return half-reg with N <= 8 bytes. +template <size_t N, HWY_IF_LE64(uint8_t, N)> +HWY_INLINE Vec128<uint8_t, N> Load8Bytes(Simd<uint8_t, N> d, + const uint8_t* bytes) { + return Load(d, bytes); +} + +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> IdxFromBits(hwy::SizeTag<2> /*tag*/, + const uint64_t mask_bits) { + HWY_DASSERT(mask_bits < 256); + const Simd<T, N> d; + const Repartition<uint8_t, decltype(d)> d8; + const Simd<uint16_t, N> du; + + // ARM does not provide an equivalent of AVX2 permutevar, so we need byte + // indices for VTBL (one vector's worth for each of 256 combinations of + // 8 mask bits). Loading them directly would require 4 KiB. We can instead + // store lane indices and convert to byte indices (2*lane + 0..1), with the + // doubling baked into the table. AVX2 Compress32 stores eight 4-bit lane + // indices (total 1 KiB), broadcasts them into each 32-bit lane and shifts. + // Here, 16-bit lanes are too narrow to hold all bits, and unpacking nibbles + // is likely more costly than the higher cache footprint from storing bytes. + alignas(16) constexpr uint8_t table[256 * 8] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, + 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, + 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, + 0, 6, 0, 0, 0, 0, 0, 0, 2, 6, 0, 0, 0, 0, 0, 0, 0, 2, + 6, 0, 0, 0, 0, 0, 4, 6, 0, 0, 0, 0, 0, 0, 0, 4, 6, 0, + 0, 0, 0, 0, 2, 4, 6, 0, 0, 0, 0, 0, 0, 2, 4, 6, 0, 0, + 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, + 2, 8, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 0, 0, 4, 8, + 0, 0, 0, 0, 0, 0, 0, 4, 8, 0, 0, 0, 0, 0, 2, 4, 8, 0, + 0, 0, 0, 0, 0, 2, 4, 8, 0, 0, 0, 0, 6, 8, 0, 0, 0, 0, + 0, 0, 0, 6, 8, 0, 0, 0, 0, 0, 2, 6, 8, 0, 0, 0, 0, 0, + 0, 2, 6, 8, 0, 0, 0, 0, 4, 6, 8, 0, 0, 0, 0, 0, 0, 4, + 6, 8, 0, 0, 0, 0, 2, 4, 6, 8, 0, 0, 0, 0, 0, 2, 4, 6, + 8, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, + 0, 0, 2, 10, 0, 0, 0, 0, 0, 0, 0, 2, 10, 0, 0, 0, 0, 0, + 4, 10, 0, 0, 0, 0, 0, 0, 0, 4, 10, 0, 0, 0, 0, 0, 2, 4, + 10, 0, 0, 0, 0, 0, 0, 2, 4, 10, 0, 0, 0, 0, 6, 10, 0, 0, + 0, 0, 0, 0, 0, 6, 10, 0, 0, 0, 0, 0, 2, 6, 10, 0, 0, 0, + 0, 0, 0, 2, 6, 10, 0, 0, 0, 0, 4, 6, 10, 0, 0, 0, 0, 0, + 0, 4, 6, 10, 0, 0, 0, 0, 2, 4, 6, 10, 0, 0, 0, 0, 0, 2, + 4, 6, 10, 0, 0, 0, 8, 10, 0, 0, 0, 0, 0, 0, 0, 8, 10, 0, + 0, 0, 0, 0, 2, 8, 10, 0, 0, 0, 0, 0, 0, 2, 8, 10, 0, 0, + 0, 0, 4, 8, 10, 0, 0, 0, 0, 0, 0, 4, 8, 10, 0, 0, 0, 0, + 2, 4, 8, 10, 0, 0, 0, 0, 0, 2, 4, 8, 10, 0, 0, 0, 6, 8, + 10, 0, 0, 0, 0, 0, 0, 6, 8, 10, 0, 0, 0, 0, 2, 6, 8, 10, + 0, 0, 0, 0, 0, 2, 6, 8, 10, 0, 0, 0, 4, 6, 8, 10, 0, 0, + 0, 0, 0, 4, 6, 8, 10, 0, 0, 0, 2, 4, 6, 8, 10, 0, 0, 0, + 0, 2, 4, 6, 8, 10, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 12, + 0, 0, 0, 0, 0, 0, 2, 12, 0, 0, 0, 0, 0, 0, 0, 2, 12, 0, + 0, 0, 0, 0, 4, 12, 0, 0, 0, 0, 0, 0, 0, 4, 12, 0, 0, 0, + 0, 0, 2, 4, 12, 0, 0, 0, 0, 0, 0, 2, 4, 12, 0, 0, 0, 0, + 6, 12, 0, 0, 0, 0, 0, 0, 0, 6, 12, 0, 0, 0, 0, 0, 2, 6, + 12, 0, 0, 0, 0, 0, 0, 2, 6, 12, 0, 0, 0, 0, 4, 6, 12, 0, + 0, 0, 0, 0, 0, 4, 6, 12, 0, 0, 0, 0, 2, 4, 6, 12, 0, 0, + 0, 0, 0, 2, 4, 6, 12, 0, 0, 0, 8, 12, 0, 0, 0, 0, 0, 0, + 0, 8, 12, 0, 0, 0, 0, 0, 2, 8, 12, 0, 0, 0, 0, 0, 0, 2, + 8, 12, 0, 0, 0, 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 4, 8, 12, + 0, 0, 0, 0, 2, 4, 8, 12, 0, 0, 0, 0, 0, 2, 4, 8, 12, 0, + 0, 0, 6, 8, 12, 0, 0, 0, 0, 0, 0, 6, 8, 12, 0, 0, 0, 0, + 2, 6, 8, 12, 0, 0, 0, 0, 0, 2, 6, 8, 12, 0, 0, 0, 4, 6, + 8, 12, 0, 0, 0, 0, 0, 4, 6, 8, 12, 0, 0, 0, 2, 4, 6, 8, + 12, 0, 0, 0, 0, 2, 4, 6, 8, 12, 0, 0, 10, 12, 0, 0, 0, 0, + 0, 0, 0, 10, 12, 0, 0, 0, 0, 0, 2, 10, 12, 0, 0, 0, 0, 0, + 0, 2, 10, 12, 0, 0, 0, 0, 4, 10, 12, 0, 0, 0, 0, 0, 0, 4, + 10, 12, 0, 0, 0, 0, 2, 4, 10, 12, 0, 0, 0, 0, 0, 2, 4, 10, + 12, 0, 0, 0, 6, 10, 12, 0, 0, 0, 0, 0, 0, 6, 10, 12, 0, 0, + 0, 0, 2, 6, 10, 12, 0, 0, 0, 0, 0, 2, 6, 10, 12, 0, 0, 0, + 4, 6, 10, 12, 0, 0, 0, 0, 0, 4, 6, 10, 12, 0, 0, 0, 2, 4, + 6, 10, 12, 0, 0, 0, 0, 2, 4, 6, 10, 12, 0, 0, 8, 10, 12, 0, + 0, 0, 0, 0, 0, 8, 10, 12, 0, 0, 0, 0, 2, 8, 10, 12, 0, 0, + 0, 0, 0, 2, 8, 10, 12, 0, 0, 0, 4, 8, 10, 12, 0, 0, 0, 0, + 0, 4, 8, 10, 12, 0, 0, 0, 2, 4, 8, 10, 12, 0, 0, 0, 0, 2, + 4, 8, 10, 12, 0, 0, 6, 8, 10, 12, 0, 0, 0, 0, 0, 6, 8, 10, + 12, 0, 0, 0, 2, 6, 8, 10, 12, 0, 0, 0, 0, 2, 6, 8, 10, 12, + 0, 0, 4, 6, 8, 10, 12, 0, 0, 0, 0, 4, 6, 8, 10, 12, 0, 0, + 2, 4, 6, 8, 10, 12, 0, 0, 0, 2, 4, 6, 8, 10, 12, 0, 14, 0, + 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 2, 14, 0, 0, + 0, 0, 0, 0, 0, 2, 14, 0, 0, 0, 0, 0, 4, 14, 0, 0, 0, 0, + 0, 0, 0, 4, 14, 0, 0, 0, 0, 0, 2, 4, 14, 0, 0, 0, 0, 0, + 0, 2, 4, 14, 0, 0, 0, 0, 6, 14, 0, 0, 0, 0, 0, 0, 0, 6, + 14, 0, 0, 0, 0, 0, 2, 6, 14, 0, 0, 0, 0, 0, 0, 2, 6, 14, + 0, 0, 0, 0, 4, 6, 14, 0, 0, 0, 0, 0, 0, 4, 6, 14, 0, 0, + 0, 0, 2, 4, 6, 14, 0, 0, 0, 0, 0, 2, 4, 6, 14, 0, 0, 0, + 8, 14, 0, 0, 0, 0, 0, 0, 0, 8, 14, 0, 0, 0, 0, 0, 2, 8, + 14, 0, 0, 0, 0, 0, 0, 2, 8, 14, 0, 0, 0, 0, 4, 8, 14, 0, + 0, 0, 0, 0, 0, 4, 8, 14, 0, 0, 0, 0, 2, 4, 8, 14, 0, 0, + 0, 0, 0, 2, 4, 8, 14, 0, 0, 0, 6, 8, 14, 0, 0, 0, 0, 0, + 0, 6, 8, 14, 0, 0, 0, 0, 2, 6, 8, 14, 0, 0, 0, 0, 0, 2, + 6, 8, 14, 0, 0, 0, 4, 6, 8, 14, 0, 0, 0, 0, 0, 4, 6, 8, + 14, 0, 0, 0, 2, 4, 6, 8, 14, 0, 0, 0, 0, 2, 4, 6, 8, 14, + 0, 0, 10, 14, 0, 0, 0, 0, 0, 0, 0, 10, 14, 0, 0, 0, 0, 0, + 2, 10, 14, 0, 0, 0, 0, 0, 0, 2, 10, 14, 0, 0, 0, 0, 4, 10, + 14, 0, 0, 0, 0, 0, 0, 4, 10, 14, 0, 0, 0, 0, 2, 4, 10, 14, + 0, 0, 0, 0, 0, 2, 4, 10, 14, 0, 0, 0, 6, 10, 14, 0, 0, 0, + 0, 0, 0, 6, 10, 14, 0, 0, 0, 0, 2, 6, 10, 14, 0, 0, 0, 0, + 0, 2, 6, 10, 14, 0, 0, 0, 4, 6, 10, 14, 0, 0, 0, 0, 0, 4, + 6, 10, 14, 0, 0, 0, 2, 4, 6, 10, 14, 0, 0, 0, 0, 2, 4, 6, + 10, 14, 0, 0, 8, 10, 14, 0, 0, 0, 0, 0, 0, 8, 10, 14, 0, 0, + 0, 0, 2, 8, 10, 14, 0, 0, 0, 0, 0, 2, 8, 10, 14, 0, 0, 0, + 4, 8, 10, 14, 0, 0, 0, 0, 0, 4, 8, 10, 14, 0, 0, 0, 2, 4, + 8, 10, 14, 0, 0, 0, 0, 2, 4, 8, 10, 14, 0, 0, 6, 8, 10, 14, + 0, 0, 0, 0, 0, 6, 8, 10, 14, 0, 0, 0, 2, 6, 8, 10, 14, 0, + 0, 0, 0, 2, 6, 8, 10, 14, 0, 0, 4, 6, 8, 10, 14, 0, 0, 0, + 0, 4, 6, 8, 10, 14, 0, 0, 2, 4, 6, 8, 10, 14, 0, 0, 0, 2, + 4, 6, 8, 10, 14, 0, 12, 14, 0, 0, 0, 0, 0, 0, 0, 12, 14, 0, + 0, 0, 0, 0, 2, 12, 14, 0, 0, 0, 0, 0, 0, 2, 12, 14, 0, 0, + 0, 0, 4, 12, 14, 0, 0, 0, 0, 0, 0, 4, 12, 14, 0, 0, 0, 0, + 2, 4, 12, 14, 0, 0, 0, 0, 0, 2, 4, 12, 14, 0, 0, 0, 6, 12, + 14, 0, 0, 0, 0, 0, 0, 6, 12, 14, 0, 0, 0, 0, 2, 6, 12, 14, + 0, 0, 0, 0, 0, 2, 6, 12, 14, 0, 0, 0, 4, 6, 12, 14, 0, 0, + 0, 0, 0, 4, 6, 12, 14, 0, 0, 0, 2, 4, 6, 12, 14, 0, 0, 0, + 0, 2, 4, 6, 12, 14, 0, 0, 8, 12, 14, 0, 0, 0, 0, 0, 0, 8, + 12, 14, 0, 0, 0, 0, 2, 8, 12, 14, 0, 0, 0, 0, 0, 2, 8, 12, + 14, 0, 0, 0, 4, 8, 12, 14, 0, 0, 0, 0, 0, 4, 8, 12, 14, 0, + 0, 0, 2, 4, 8, 12, 14, 0, 0, 0, 0, 2, 4, 8, 12, 14, 0, 0, + 6, 8, 12, 14, 0, 0, 0, 0, 0, 6, 8, 12, 14, 0, 0, 0, 2, 6, + 8, 12, 14, 0, 0, 0, 0, 2, 6, 8, 12, 14, 0, 0, 4, 6, 8, 12, + 14, 0, 0, 0, 0, 4, 6, 8, 12, 14, 0, 0, 2, 4, 6, 8, 12, 14, + 0, 0, 0, 2, 4, 6, 8, 12, 14, 0, 10, 12, 14, 0, 0, 0, 0, 0, + 0, 10, 12, 14, 0, 0, 0, 0, 2, 10, 12, 14, 0, 0, 0, 0, 0, 2, + 10, 12, 14, 0, 0, 0, 4, 10, 12, 14, 0, 0, 0, 0, 0, 4, 10, 12, + 14, 0, 0, 0, 2, 4, 10, 12, 14, 0, 0, 0, 0, 2, 4, 10, 12, 14, + 0, 0, 6, 10, 12, 14, 0, 0, 0, 0, 0, 6, 10, 12, 14, 0, 0, 0, + 2, 6, 10, 12, 14, 0, 0, 0, 0, 2, 6, 10, 12, 14, 0, 0, 4, 6, + 10, 12, 14, 0, 0, 0, 0, 4, 6, 10, 12, 14, 0, 0, 2, 4, 6, 10, + 12, 14, 0, 0, 0, 2, 4, 6, 10, 12, 14, 0, 8, 10, 12, 14, 0, 0, + 0, 0, 0, 8, 10, 12, 14, 0, 0, 0, 2, 8, 10, 12, 14, 0, 0, 0, + 0, 2, 8, 10, 12, 14, 0, 0, 4, 8, 10, 12, 14, 0, 0, 0, 0, 4, + 8, 10, 12, 14, 0, 0, 2, 4, 8, 10, 12, 14, 0, 0, 0, 2, 4, 8, + 10, 12, 14, 0, 6, 8, 10, 12, 14, 0, 0, 0, 0, 6, 8, 10, 12, 14, + 0, 0, 2, 6, 8, 10, 12, 14, 0, 0, 0, 2, 6, 8, 10, 12, 14, 0, + 4, 6, 8, 10, 12, 14, 0, 0, 0, 4, 6, 8, 10, 12, 14, 0, 2, 4, + 6, 8, 10, 12, 14, 0, 0, 2, 4, 6, 8, 10, 12, 14}; + + const Vec128<uint8_t, 2 * N> byte_idx = Load8Bytes(d8, table + mask_bits * 8); + const Vec128<uint16_t, N> pairs = ZipLower(byte_idx, byte_idx); + return BitCast(d, pairs + Set(du, 0x0100)); +} + +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> IdxFromBits(hwy::SizeTag<4> /*tag*/, + const uint64_t mask_bits) { + HWY_DASSERT(mask_bits < 16); + + // There are only 4 lanes, so we can afford to load the index vector directly. + alignas(16) constexpr uint8_t packed_array[16 * 16] = { + 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, // + 4, 5, 6, 7, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 0, 1, 2, 3, // + 8, 9, 10, 11, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 8, 9, 10, 11, 0, 1, 2, 3, 0, 1, 2, 3, // + 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, // + 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3, // + 4, 5, 6, 7, 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 15, 0, 1, 2, 3, // + 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, // + 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, // + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + + const Simd<T, N> d; + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, Load(d8, packed_array + 16 * mask_bits)); +} + +#if HWY_CAP_INTEGER64 || HWY_CAP_FLOAT64 + +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> IdxFromBits(hwy::SizeTag<8> /*tag*/, + const uint64_t mask_bits) { + HWY_DASSERT(mask_bits < 4); + + // There are only 2 lanes, so we can afford to load the index vector directly. + alignas(16) constexpr uint8_t packed_array[4 * 16] = { + 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, // + 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, // + 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, // + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + + const Simd<T, N> d; + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, Load(d8, packed_array + 16 * mask_bits)); +} + +#endif + +// Helper function called by both Compress and CompressStore - avoids a +// redundant BitsFromMask in the latter. +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> Compress(Vec128<T, N> v, const uint64_t mask_bits) { + const auto idx = + detail::IdxFromBits<T, N>(hwy::SizeTag<sizeof(T)>(), mask_bits); + using D = Simd<T, N>; + const RebindToSigned<D> di; + return BitCast(D(), TableLookupBytes(BitCast(di, v), BitCast(di, idx))); +} + +} // namespace detail + +template <typename T, size_t N> +HWY_API Vec128<T, N> Compress(Vec128<T, N> v, const Mask128<T, N> mask) { + return detail::Compress(v, detail::BitsFromMask(mask)); +} + +// ------------------------------ CompressBits + +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> CompressBits(Vec128<T, N> v, + const uint8_t* HWY_RESTRICT bits) { + uint64_t mask_bits = 0; + constexpr size_t kNumBytes = (N + 7) / 8; + CopyBytes<kNumBytes>(bits, &mask_bits); + if (N < 8) { + mask_bits &= (1ull << N) - 1; + } + + return detail::Compress(v, mask_bits); +} + +// ------------------------------ CompressStore +template <typename T, size_t N> +HWY_API size_t CompressStore(Vec128<T, N> v, const Mask128<T, N> mask, + Simd<T, N> d, T* HWY_RESTRICT unaligned) { + const uint64_t mask_bits = detail::BitsFromMask(mask); + StoreU(detail::Compress(v, mask_bits), d, unaligned); + return PopCount(mask_bits); +} + +// ------------------------------ CompressBlendedStore +template <typename T, size_t N> +HWY_API size_t CompressBlendedStore(Vec128<T, N> v, Mask128<T, N> m, + Simd<T, N> d, T* HWY_RESTRICT unaligned) { + const RebindToUnsigned<decltype(d)> du; // so we can support fp16/bf16 + using TU = TFromD<decltype(du)>; + const uint64_t mask_bits = detail::BitsFromMask(m); + const size_t count = PopCount(mask_bits); + const Mask128<TU, N> store_mask = FirstN(du, count); + const Vec128<TU, N> compressed = detail::Compress(BitCast(du, v), mask_bits); + const Vec128<TU, N> prev = BitCast(du, LoadU(d, unaligned)); + StoreU(BitCast(d, IfThenElse(store_mask, compressed, prev)), d, unaligned); + return count; +} + +// ------------------------------ CompressBitsStore + +template <typename T, size_t N> +HWY_API size_t CompressBitsStore(Vec128<T, N> v, + const uint8_t* HWY_RESTRICT bits, Simd<T, N> d, + T* HWY_RESTRICT unaligned) { + uint64_t mask_bits = 0; + constexpr size_t kNumBytes = (N + 7) / 8; + CopyBytes<kNumBytes>(bits, &mask_bits); + if (N < 8) { + mask_bits &= (1ull << N) - 1; + } + + StoreU(detail::Compress(v, mask_bits), d, unaligned); + return PopCount(mask_bits); +} + +// ------------------------------ StoreInterleaved3 + +// 128 bits +HWY_API void StoreInterleaved3(const Vec128<uint8_t> v0, + const Vec128<uint8_t> v1, + const Vec128<uint8_t> v2, + Full128<uint8_t> /*tag*/, + uint8_t* HWY_RESTRICT unaligned) { + const uint8x16x3_t triple = {v0.raw, v1.raw, v2.raw}; + vst3q_u8(unaligned, triple); +} + +// 64 bits +HWY_API void StoreInterleaved3(const Vec128<uint8_t, 8> v0, + const Vec128<uint8_t, 8> v1, + const Vec128<uint8_t, 8> v2, + Simd<uint8_t, 8> /*tag*/, + uint8_t* HWY_RESTRICT unaligned) { + const uint8x8x3_t triple = {v0.raw, v1.raw, v2.raw}; + vst3_u8(unaligned, triple); +} + +// <= 32 bits: avoid writing more than N bytes by copying to buffer +template <size_t N, HWY_IF_LE32(uint8_t, N)> +HWY_API void StoreInterleaved3(const Vec128<uint8_t, N> v0, + const Vec128<uint8_t, N> v1, + const Vec128<uint8_t, N> v2, + Simd<uint8_t, N> /*tag*/, + uint8_t* HWY_RESTRICT unaligned) { + alignas(16) uint8_t buf[24]; + const uint8x8x3_t triple = {v0.raw, v1.raw, v2.raw}; + vst3_u8(buf, triple); + CopyBytes<N * 3>(buf, unaligned); +} + +// ------------------------------ StoreInterleaved4 + +// 128 bits +HWY_API void StoreInterleaved4(const Vec128<uint8_t> v0, + const Vec128<uint8_t> v1, + const Vec128<uint8_t> v2, + const Vec128<uint8_t> v3, + Full128<uint8_t> /*tag*/, + uint8_t* HWY_RESTRICT unaligned) { + const uint8x16x4_t quad = {v0.raw, v1.raw, v2.raw, v3.raw}; + vst4q_u8(unaligned, quad); +} + +// 64 bits +HWY_API void StoreInterleaved4(const Vec128<uint8_t, 8> v0, + const Vec128<uint8_t, 8> v1, + const Vec128<uint8_t, 8> v2, + const Vec128<uint8_t, 8> v3, + Simd<uint8_t, 8> /*tag*/, + uint8_t* HWY_RESTRICT unaligned) { + const uint8x8x4_t quad = {v0.raw, v1.raw, v2.raw, v3.raw}; + vst4_u8(unaligned, quad); +} + +// <= 32 bits: avoid writing more than N bytes by copying to buffer +template <size_t N, HWY_IF_LE32(uint8_t, N)> +HWY_API void StoreInterleaved4(const Vec128<uint8_t, N> v0, + const Vec128<uint8_t, N> v1, + const Vec128<uint8_t, N> v2, + const Vec128<uint8_t, N> v3, + Simd<uint8_t, N> /*tag*/, + uint8_t* HWY_RESTRICT unaligned) { + alignas(16) uint8_t buf[32]; + const uint8x8x4_t quad = {v0.raw, v1.raw, v2.raw, v3.raw}; + vst4_u8(buf, quad); + CopyBytes<N * 4>(buf, unaligned); +} + +// ================================================== DEPRECATED + +template <typename T, size_t N> +HWY_API size_t StoreMaskBits(const Mask128<T, N> mask, uint8_t* bits) { + return StoreMaskBits(Simd<T, N>(), mask, bits); +} + +template <typename T, size_t N> +HWY_API bool AllTrue(const Mask128<T, N> mask) { + return AllTrue(Simd<T, N>(), mask); +} + +template <typename T, size_t N> +HWY_API bool AllFalse(const Mask128<T, N> mask) { + return AllFalse(Simd<T, N>(), mask); +} + +template <typename T, size_t N> +HWY_API size_t CountTrue(const Mask128<T, N> mask) { + return CountTrue(Simd<T, N>(), mask); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> SumOfLanes(const Vec128<T, N> v) { + return SumOfLanes(Simd<T, N>(), v); +} +template <typename T, size_t N> +HWY_API Vec128<T, N> MinOfLanes(const Vec128<T, N> v) { + return MinOfLanes(Simd<T, N>(), v); +} +template <typename T, size_t N> +HWY_API Vec128<T, N> MaxOfLanes(const Vec128<T, N> v) { + return MaxOfLanes(Simd<T, N>(), v); +} + +template <typename T, size_t N> +HWY_API Vec128<T, (N + 1) / 2> UpperHalf(Vec128<T, N> v) { + return UpperHalf(Half<Simd<T, N>>(), v); +} + +template <int kBytes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftRightBytes(const Vec128<T, N> v) { + return ShiftRightBytes<kBytes>(Simd<T, N>(), v); +} + +template <int kLanes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftRightLanes(const Vec128<T, N> v) { + return ShiftRightLanes<kLanes>(Simd<T, N>(), v); +} + +template <size_t kBytes, typename T, size_t N> +HWY_API Vec128<T, N> CombineShiftRightBytes(Vec128<T, N> hi, Vec128<T, N> lo) { + return CombineShiftRightBytes<kBytes>(Simd<T, N>(), hi, lo); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> InterleaveUpper(Vec128<T, N> a, Vec128<T, N> b) { + return InterleaveUpper(Simd<T, N>(), a, b); +} + +template <typename T, size_t N, class D = Simd<T, N>> +HWY_API VFromD<RepartitionToWide<D>> ZipUpper(Vec128<T, N> a, Vec128<T, N> b) { + return InterleaveUpper(RepartitionToWide<D>(), a, b); +} + +template <typename T, size_t N2> +HWY_API Vec128<T, N2 * 2> Combine(Vec128<T, N2> hi2, Vec128<T, N2> lo2) { + return Combine(Simd<T, N2 * 2>(), hi2, lo2); +} + +template <typename T, size_t N2, HWY_IF_LE64(T, N2)> +HWY_API Vec128<T, N2 * 2> ZeroExtendVector(Vec128<T, N2> lo) { + return ZeroExtendVector(Simd<T, N2 * 2>(), lo); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> ConcatLowerLower(Vec128<T, N> hi, Vec128<T, N> lo) { + return ConcatLowerLower(Simd<T, N>(), hi, lo); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> ConcatUpperUpper(Vec128<T, N> hi, Vec128<T, N> lo) { + return ConcatUpperUpper(Simd<T, N>(), hi, lo); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> ConcatLowerUpper(const Vec128<T, N> hi, + const Vec128<T, N> lo) { + return ConcatLowerUpper(Simd<T, N>(), hi, lo); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> ConcatUpperLower(Vec128<T, N> hi, Vec128<T, N> lo) { + return ConcatUpperLower(Simd<T, N>(), hi, lo); +} + +// ================================================== Operator wrapper + +// These apply to all x86_*-inl.h because there are no restrictions on V. + +template <class V> +HWY_API V Add(V a, V b) { + return a + b; +} +template <class V> +HWY_API V Sub(V a, V b) { + return a - b; +} + +template <class V> +HWY_API V Mul(V a, V b) { + return a * b; +} +template <class V> +HWY_API V Div(V a, V b) { + return a / b; +} + +template <class V> +V Shl(V a, V b) { + return a << b; +} +template <class V> +V Shr(V a, V b) { + return a >> b; +} + +template <class V> +HWY_API auto Eq(V a, V b) -> decltype(a == b) { + return a == b; +} +template <class V> +HWY_API auto Ne(V a, V b) -> decltype(a == b) { + return a != b; +} +template <class V> +HWY_API auto Lt(V a, V b) -> decltype(a == b) { + return a < b; +} + +template <class V> +HWY_API auto Gt(V a, V b) -> decltype(a == b) { + return a > b; +} +template <class V> +HWY_API auto Ge(V a, V b) -> decltype(a == b) { + return a >= b; +} + +template <class V> +HWY_API auto Le(V a, V b) -> decltype(a == b) { + return a <= b; +} + +namespace detail { // for code folding +#if HWY_ARCH_ARM_V7 +#undef vuzp1_s8 +#undef vuzp1_u8 +#undef vuzp1_s16 +#undef vuzp1_u16 +#undef vuzp1_s32 +#undef vuzp1_u32 +#undef vuzp1_f32 +#undef vuzp1q_s8 +#undef vuzp1q_u8 +#undef vuzp1q_s16 +#undef vuzp1q_u16 +#undef vuzp1q_s32 +#undef vuzp1q_u32 +#undef vuzp1q_f32 +#undef vuzp2_s8 +#undef vuzp2_u8 +#undef vuzp2_s16 +#undef vuzp2_u16 +#undef vuzp2_s32 +#undef vuzp2_u32 +#undef vuzp2_f32 +#undef vuzp2q_s8 +#undef vuzp2q_u8 +#undef vuzp2q_s16 +#undef vuzp2q_u16 +#undef vuzp2q_s32 +#undef vuzp2q_u32 +#undef vuzp2q_f32 +#undef vzip1_s8 +#undef vzip1_u8 +#undef vzip1_s16 +#undef vzip1_u16 +#undef vzip1_s32 +#undef vzip1_u32 +#undef vzip1_f32 +#undef vzip1q_s8 +#undef vzip1q_u8 +#undef vzip1q_s16 +#undef vzip1q_u16 +#undef vzip1q_s32 +#undef vzip1q_u32 +#undef vzip1q_f32 +#undef vzip2_s8 +#undef vzip2_u8 +#undef vzip2_s16 +#undef vzip2_u16 +#undef vzip2_s32 +#undef vzip2_u32 +#undef vzip2_f32 +#undef vzip2q_s8 +#undef vzip2q_u8 +#undef vzip2q_s16 +#undef vzip2q_u16 +#undef vzip2q_s32 +#undef vzip2q_u32 +#undef vzip2q_f32 +#endif + +#undef HWY_NEON_BUILD_ARG_1 +#undef HWY_NEON_BUILD_ARG_2 +#undef HWY_NEON_BUILD_ARG_3 +#undef HWY_NEON_BUILD_PARAM_1 +#undef HWY_NEON_BUILD_PARAM_2 +#undef HWY_NEON_BUILD_PARAM_3 +#undef HWY_NEON_BUILD_RET_1 +#undef HWY_NEON_BUILD_RET_2 +#undef HWY_NEON_BUILD_RET_3 +#undef HWY_NEON_BUILD_TPL_1 +#undef HWY_NEON_BUILD_TPL_2 +#undef HWY_NEON_BUILD_TPL_3 +#undef HWY_NEON_DEF_FUNCTION +#undef HWY_NEON_DEF_FUNCTION_ALL_FLOATS +#undef HWY_NEON_DEF_FUNCTION_ALL_TYPES +#undef HWY_NEON_DEF_FUNCTION_INT_8 +#undef HWY_NEON_DEF_FUNCTION_INT_16 +#undef HWY_NEON_DEF_FUNCTION_INT_32 +#undef HWY_NEON_DEF_FUNCTION_INT_8_16_32 +#undef HWY_NEON_DEF_FUNCTION_INTS +#undef HWY_NEON_DEF_FUNCTION_INTS_UINTS +#undef HWY_NEON_DEF_FUNCTION_TPL +#undef HWY_NEON_DEF_FUNCTION_UINT_8 +#undef HWY_NEON_DEF_FUNCTION_UINT_16 +#undef HWY_NEON_DEF_FUNCTION_UINT_32 +#undef HWY_NEON_DEF_FUNCTION_UINT_8_16_32 +#undef HWY_NEON_DEF_FUNCTION_UINTS +#undef HWY_NEON_EVAL +} // namespace detail + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); diff --git a/third_party/highway/hwy/ops/arm_sve-inl.h b/third_party/highway/hwy/ops/arm_sve-inl.h new file mode 100644 index 0000000..85b4e34 --- /dev/null +++ b/third_party/highway/hwy/ops/arm_sve-inl.h @@ -0,0 +1,2138 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ARM SVE[2] vectors (length not known at compile time). +// External include guard in highway.h - see comment there. + +#include <stddef.h> +#include <stdint.h> + +#if defined(HWY_EMULATE_SVE) +#include "third_party/farm_sve/farm_sve.h" +#else +#include <arm_sve.h> +#endif + +#include "hwy/base.h" +#include "hwy/ops/shared-inl.h" + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +// SVE only supports fractions, not LMUL > 1. +template <typename T, int kShift = 0> +using Full = Simd<T, (kShift <= 0) ? (HWY_LANES(T) >> (-kShift)) : 0>; + +template <class V> +struct DFromV_t {}; // specialized in macros +template <class V> +using DFromV = typename DFromV_t<RemoveConst<V>>::type; + +template <class V> +using TFromV = TFromD<DFromV<V>>; + +#define HWY_IF_UNSIGNED_V(V) HWY_IF_UNSIGNED(TFromV<V>) +#define HWY_IF_SIGNED_V(V) HWY_IF_SIGNED(TFromV<V>) +#define HWY_IF_FLOAT_V(V) HWY_IF_FLOAT(TFromV<V>) +#define HWY_IF_LANE_SIZE_V(V, bytes) HWY_IF_LANE_SIZE(TFromV<V>, bytes) + +// ================================================== MACROS + +// Generate specializations and function definitions using X macros. Although +// harder to read and debug, writing everything manually is too bulky. + +namespace detail { // for code folding + +// Unsigned: +#define HWY_SVE_FOREACH_U08(X_MACRO, NAME, OP) X_MACRO(uint, u, 8, NAME, OP) +#define HWY_SVE_FOREACH_U16(X_MACRO, NAME, OP) X_MACRO(uint, u, 16, NAME, OP) +#define HWY_SVE_FOREACH_U32(X_MACRO, NAME, OP) X_MACRO(uint, u, 32, NAME, OP) +#define HWY_SVE_FOREACH_U64(X_MACRO, NAME, OP) X_MACRO(uint, u, 64, NAME, OP) + +// Signed: +#define HWY_SVE_FOREACH_I08(X_MACRO, NAME, OP) X_MACRO(int, s, 8, NAME, OP) +#define HWY_SVE_FOREACH_I16(X_MACRO, NAME, OP) X_MACRO(int, s, 16, NAME, OP) +#define HWY_SVE_FOREACH_I32(X_MACRO, NAME, OP) X_MACRO(int, s, 32, NAME, OP) +#define HWY_SVE_FOREACH_I64(X_MACRO, NAME, OP) X_MACRO(int, s, 64, NAME, OP) + +// Float: +#define HWY_SVE_FOREACH_F16(X_MACRO, NAME, OP) X_MACRO(float, f, 16, NAME, OP) +#define HWY_SVE_FOREACH_F32(X_MACRO, NAME, OP) X_MACRO(float, f, 32, NAME, OP) +#define HWY_SVE_FOREACH_F64(X_MACRO, NAME, OP) X_MACRO(float, f, 64, NAME, OP) + +// For all element sizes: +#define HWY_SVE_FOREACH_U(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_U08(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_U16(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_U32(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_U64(X_MACRO, NAME, OP) + +#define HWY_SVE_FOREACH_I(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_I08(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_I16(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_I32(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_I64(X_MACRO, NAME, OP) + +#define HWY_SVE_FOREACH_F(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_F16(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_F32(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_F64(X_MACRO, NAME, OP) + +// Commonly used type categories for a given element size: +#define HWY_SVE_FOREACH_UI08(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_U08(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_I08(X_MACRO, NAME, OP) + +#define HWY_SVE_FOREACH_UI16(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_U16(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_I16(X_MACRO, NAME, OP) + +#define HWY_SVE_FOREACH_UI32(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_U32(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_I32(X_MACRO, NAME, OP) + +#define HWY_SVE_FOREACH_UI64(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_U64(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_I64(X_MACRO, NAME, OP) + +#define HWY_SVE_FOREACH_UIF3264(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_UI32(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_UI64(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_F32(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_F64(X_MACRO, NAME, OP) + +// Commonly used type categories: +#define HWY_SVE_FOREACH_UI(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_U(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_I(X_MACRO, NAME, OP) + +#define HWY_SVE_FOREACH_IF(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_I(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_F(X_MACRO, NAME, OP) + +#define HWY_SVE_FOREACH(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_U(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_I(X_MACRO, NAME, OP) \ + HWY_SVE_FOREACH_F(X_MACRO, NAME, OP) + +// Assemble types for use in x-macros +#define HWY_SVE_T(BASE, BITS) BASE##BITS##_t +#define HWY_SVE_D(BASE, BITS, N) Simd<HWY_SVE_T(BASE, BITS), N> +#define HWY_SVE_V(BASE, BITS) sv##BASE##BITS##_t + +} // namespace detail + +#define HWY_SPECIALIZE(BASE, CHAR, BITS, NAME, OP) \ + template <> \ + struct DFromV_t<HWY_SVE_V(BASE, BITS)> { \ + using type = HWY_SVE_D(BASE, BITS, HWY_LANES(HWY_SVE_T(BASE, BITS))); \ + }; + +HWY_SVE_FOREACH(HWY_SPECIALIZE, _, _) +#undef HWY_SPECIALIZE + +// vector = f(d), e.g. Undefined +#define HWY_SVE_RETV_ARGD(BASE, CHAR, BITS, NAME, OP) \ + template <size_t N> \ + HWY_API HWY_SVE_V(BASE, BITS) NAME(HWY_SVE_D(BASE, BITS, N) d) { \ + return sv##OP##_##CHAR##BITS(); \ + } + +// Note: _x (don't-care value for inactive lanes) avoids additional MOVPRFX +// instructions, and we anyway only use it when the predicate is ptrue. + +// vector = f(vector), e.g. Not +#define HWY_SVE_RETV_ARGPV(BASE, CHAR, BITS, NAME, OP) \ + HWY_API HWY_SVE_V(BASE, BITS) NAME(HWY_SVE_V(BASE, BITS) v) { \ + return sv##OP##_##CHAR##BITS##_x(HWY_SVE_PTRUE(BITS), v); \ + } +#define HWY_SVE_RETV_ARGV(BASE, CHAR, BITS, NAME, OP) \ + HWY_API HWY_SVE_V(BASE, BITS) NAME(HWY_SVE_V(BASE, BITS) v) { \ + return sv##OP##_##CHAR##BITS(v); \ + } + +// vector = f(vector, scalar), e.g. detail::AddK +#define HWY_SVE_RETV_ARGPVN(BASE, CHAR, BITS, NAME, OP) \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME(HWY_SVE_V(BASE, BITS) a, HWY_SVE_T(BASE, BITS) b) { \ + return sv##OP##_##CHAR##BITS##_x(HWY_SVE_PTRUE(BITS), a, b); \ + } +#define HWY_SVE_RETV_ARGVN(BASE, CHAR, BITS, NAME, OP) \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME(HWY_SVE_V(BASE, BITS) a, HWY_SVE_T(BASE, BITS) b) { \ + return sv##OP##_##CHAR##BITS(a, b); \ + } + +// vector = f(vector, vector), e.g. Add +#define HWY_SVE_RETV_ARGPVV(BASE, CHAR, BITS, NAME, OP) \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME(HWY_SVE_V(BASE, BITS) a, HWY_SVE_V(BASE, BITS) b) { \ + return sv##OP##_##CHAR##BITS##_x(HWY_SVE_PTRUE(BITS), a, b); \ + } +#define HWY_SVE_RETV_ARGVV(BASE, CHAR, BITS, NAME, OP) \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME(HWY_SVE_V(BASE, BITS) a, HWY_SVE_V(BASE, BITS) b) { \ + return sv##OP##_##CHAR##BITS(a, b); \ + } + +// ------------------------------ Lanes + +namespace detail { + +// Returns actual lanes of a hardware vector without rounding to a power of two. +HWY_INLINE size_t AllHardwareLanes(hwy::SizeTag<1> /* tag */) { + return svcntb_pat(SV_ALL); +} +HWY_INLINE size_t AllHardwareLanes(hwy::SizeTag<2> /* tag */) { + return svcnth_pat(SV_ALL); +} +HWY_INLINE size_t AllHardwareLanes(hwy::SizeTag<4> /* tag */) { + return svcntw_pat(SV_ALL); +} +HWY_INLINE size_t AllHardwareLanes(hwy::SizeTag<8> /* tag */) { + return svcntd_pat(SV_ALL); +} + +// Returns actual lanes of a hardware vector, rounded down to a power of two. +HWY_INLINE size_t HardwareLanes(hwy::SizeTag<1> /* tag */) { + return svcntb_pat(SV_POW2); +} +HWY_INLINE size_t HardwareLanes(hwy::SizeTag<2> /* tag */) { + return svcnth_pat(SV_POW2); +} +HWY_INLINE size_t HardwareLanes(hwy::SizeTag<4> /* tag */) { + return svcntw_pat(SV_POW2); +} +HWY_INLINE size_t HardwareLanes(hwy::SizeTag<8> /* tag */) { + return svcntd_pat(SV_POW2); +} + +} // namespace detail + +// Capped to <= 128-bit: SVE is at least that large, so no need to query actual. +template <typename T, size_t N, HWY_IF_LE128(T, N)> +HWY_API constexpr size_t Lanes(Simd<T, N> /* tag */) { + return N; +} + +// Returns actual number of lanes after dividing by div={1,2,4,8}. +// May return 0 if div > 16/sizeof(T): there is no "1/8th" of a u32x4, but it +// would be valid for u32x8 (i.e. hardware vectors >= 256 bits). +template <typename T, size_t N, HWY_IF_GT128(T, N)> +HWY_API size_t Lanes(Simd<T, N> /* tag */) { + static_assert(N <= HWY_LANES(T), "N cannot exceed a full vector"); + + const size_t actual = detail::HardwareLanes(hwy::SizeTag<sizeof(T)>()); + const size_t div = HWY_LANES(T) / N; + return (div <= 8) ? actual / div : HWY_MIN(actual, N); +} + +// ================================================== MASK INIT + +// One mask bit per byte; only the one belonging to the lowest byte is valid. + +// ------------------------------ FirstN +#define HWY_SVE_FIRSTN(BASE, CHAR, BITS, NAME, OP) \ + template <size_t KN> \ + HWY_API svbool_t NAME(HWY_SVE_D(BASE, BITS, KN) /* d */, size_t N) { \ + return sv##OP##_b##BITS##_u32(uint32_t{0}, static_cast<uint32_t>(N)); \ + } +HWY_SVE_FOREACH(HWY_SVE_FIRSTN, FirstN, whilelt) +#undef HWY_SVE_FIRSTN + +namespace detail { + +// All-true mask from a macro +#define HWY_SVE_PTRUE(BITS) svptrue_pat_b##BITS(SV_POW2) + +#define HWY_SVE_WRAP_PTRUE(BASE, CHAR, BITS, NAME, OP) \ + template <size_t N> \ + HWY_API svbool_t NAME(HWY_SVE_D(BASE, BITS, N) d) { \ + return HWY_SVE_PTRUE(BITS); \ + } + +HWY_SVE_FOREACH(HWY_SVE_WRAP_PTRUE, PTrue, ptrue) // return all-true +#undef HWY_SVE_WRAP_PTRUE + +HWY_API svbool_t PFalse() { return svpfalse_b(); } + +// Returns all-true if d is HWY_FULL or FirstN(N) after capping N. +// +// This is used in functions that load/store memory; other functions (e.g. +// arithmetic on partial vectors) can ignore d and use PTrue instead. +template <typename T, size_t N> +svbool_t Mask(Simd<T, N> d) { + return N == HWY_LANES(T) ? PTrue(d) : FirstN(d, Lanes(d)); +} + +} // namespace detail + +// ================================================== INIT + +// ------------------------------ Set +// vector = f(d, scalar), e.g. Set +#define HWY_SVE_SET(BASE, CHAR, BITS, NAME, OP) \ + template <size_t N> \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME(HWY_SVE_D(BASE, BITS, N) d, HWY_SVE_T(BASE, BITS) arg) { \ + return sv##OP##_##CHAR##BITS(arg); \ + } + +HWY_SVE_FOREACH(HWY_SVE_SET, Set, dup_n) +#undef HWY_SVE_SET + +// Required for Zero and VFromD +template <size_t N> +svuint16_t Set(Simd<bfloat16_t, N> d, bfloat16_t arg) { + return Set(RebindToUnsigned<decltype(d)>(), arg.bits); +} + +template <class D> +using VFromD = decltype(Set(D(), TFromD<D>())); + +// ------------------------------ Zero + +template <class D> +VFromD<D> Zero(D d) { + return Set(d, 0); +} + +// ------------------------------ Undefined + +#if defined(HWY_EMULATE_SVE) +template <class D> +VFromD<D> Undefined(D d) { + return Zero(d); +} +#else +HWY_SVE_FOREACH(HWY_SVE_RETV_ARGD, Undefined, undef) +#endif + +// ------------------------------ BitCast + +namespace detail { + +// u8: no change +#define HWY_SVE_CAST_NOP(BASE, CHAR, BITS, NAME, OP) \ + HWY_API HWY_SVE_V(BASE, BITS) BitCastToByte(HWY_SVE_V(BASE, BITS) v) { \ + return v; \ + } \ + template <size_t N> \ + HWY_API HWY_SVE_V(BASE, BITS) BitCastFromByte( \ + HWY_SVE_D(BASE, BITS, N) /* d */, HWY_SVE_V(BASE, BITS) v) { \ + return v; \ + } + +// All other types +#define HWY_SVE_CAST(BASE, CHAR, BITS, NAME, OP) \ + HWY_INLINE svuint8_t BitCastToByte(HWY_SVE_V(BASE, BITS) v) { \ + return sv##OP##_u8_##CHAR##BITS(v); \ + } \ + template <size_t N> \ + HWY_INLINE HWY_SVE_V(BASE, BITS) \ + BitCastFromByte(HWY_SVE_D(BASE, BITS, N) /* d */, svuint8_t v) { \ + return sv##OP##_##CHAR##BITS##_u8(v); \ + } + +HWY_SVE_FOREACH_U08(HWY_SVE_CAST_NOP, _, _) +HWY_SVE_FOREACH_I08(HWY_SVE_CAST, _, reinterpret) +HWY_SVE_FOREACH_UI16(HWY_SVE_CAST, _, reinterpret) +HWY_SVE_FOREACH_UI32(HWY_SVE_CAST, _, reinterpret) +HWY_SVE_FOREACH_UI64(HWY_SVE_CAST, _, reinterpret) +HWY_SVE_FOREACH_F(HWY_SVE_CAST, _, reinterpret) + +#undef HWY_SVE_CAST_NOP +#undef HWY_SVE_CAST + +template <size_t N> +HWY_INLINE svuint16_t BitCastFromByte(Simd<bfloat16_t, N> /* d */, + svuint8_t v) { + return BitCastFromByte(Simd<uint16_t, N>(), v); +} + +} // namespace detail + +template <class D, class FromV> +HWY_API VFromD<D> BitCast(D d, FromV v) { + return detail::BitCastFromByte(d, detail::BitCastToByte(v)); +} + +// ================================================== LOGICAL + +// detail::*N() functions accept a scalar argument to avoid extra Set(). + +// ------------------------------ Not + +HWY_SVE_FOREACH_UI(HWY_SVE_RETV_ARGPV, Not, not ) + +// ------------------------------ And + +namespace detail { +HWY_SVE_FOREACH_UI(HWY_SVE_RETV_ARGPVN, AndN, and_n) +} // namespace detail + +HWY_SVE_FOREACH_UI(HWY_SVE_RETV_ARGPVV, And, and) + +template <class V, HWY_IF_FLOAT_V(V)> +HWY_API V And(const V a, const V b) { + const DFromV<V> df; + const RebindToUnsigned<decltype(df)> du; + return BitCast(df, And(BitCast(du, a), BitCast(du, b))); +} + +// ------------------------------ Or + +HWY_SVE_FOREACH_UI(HWY_SVE_RETV_ARGPVV, Or, orr) + +template <class V, HWY_IF_FLOAT_V(V)> +HWY_API V Or(const V a, const V b) { + const DFromV<V> df; + const RebindToUnsigned<decltype(df)> du; + return BitCast(df, Or(BitCast(du, a), BitCast(du, b))); +} + +// ------------------------------ Xor + +namespace detail { +HWY_SVE_FOREACH_UI(HWY_SVE_RETV_ARGPVN, XorN, eor_n) +} // namespace detail + +HWY_SVE_FOREACH_UI(HWY_SVE_RETV_ARGPVV, Xor, eor) + +template <class V, HWY_IF_FLOAT_V(V)> +HWY_API V Xor(const V a, const V b) { + const DFromV<V> df; + const RebindToUnsigned<decltype(df)> du; + return BitCast(df, Xor(BitCast(du, a), BitCast(du, b))); +} + +// ------------------------------ AndNot + +namespace detail { +#define HWY_SVE_RETV_ARGPVN_SWAP(BASE, CHAR, BITS, NAME, OP) \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME(HWY_SVE_T(BASE, BITS) a, HWY_SVE_V(BASE, BITS) b) { \ + return sv##OP##_##CHAR##BITS##_x(HWY_SVE_PTRUE(BITS), b, a); \ + } + +HWY_SVE_FOREACH_UI(HWY_SVE_RETV_ARGPVN_SWAP, AndNotN, bic_n) +#undef HWY_SVE_RETV_ARGPVN_SWAP +} // namespace detail + +#define HWY_SVE_RETV_ARGPVV_SWAP(BASE, CHAR, BITS, NAME, OP) \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME(HWY_SVE_V(BASE, BITS) a, HWY_SVE_V(BASE, BITS) b) { \ + return sv##OP##_##CHAR##BITS##_x(HWY_SVE_PTRUE(BITS), b, a); \ + } +HWY_SVE_FOREACH_UI(HWY_SVE_RETV_ARGPVV_SWAP, AndNot, bic) +#undef HWY_SVE_RETV_ARGPVV_SWAP + +template <class V, HWY_IF_FLOAT_V(V)> +HWY_API V AndNot(const V a, const V b) { + const DFromV<V> df; + const RebindToUnsigned<decltype(df)> du; + return BitCast(df, AndNot(BitCast(du, a), BitCast(du, b))); +} + +// ------------------------------ PopulationCount + +#ifdef HWY_NATIVE_POPCNT +#undef HWY_NATIVE_POPCNT +#else +#define HWY_NATIVE_POPCNT +#endif + +// Need to return original type instead of unsigned. +#define HWY_SVE_POPCNT(BASE, CHAR, BITS, NAME, OP) \ + HWY_API HWY_SVE_V(BASE, BITS) NAME(HWY_SVE_V(BASE, BITS) v) { \ + return BitCast(DFromV<decltype(v)>(), \ + sv##OP##_##CHAR##BITS##_x(HWY_SVE_PTRUE(BITS), v)); \ + } +HWY_SVE_FOREACH_UI(HWY_SVE_POPCNT, PopulationCount, cnt) +#undef HWY_SVE_POPCNT + +// ================================================== SIGN + +// ------------------------------ Neg +HWY_SVE_FOREACH_IF(HWY_SVE_RETV_ARGPV, Neg, neg) + +// ------------------------------ Abs +HWY_SVE_FOREACH_IF(HWY_SVE_RETV_ARGPV, Abs, abs) + +// ------------------------------ CopySign[ToAbs] + +template <class V> +HWY_API V CopySign(const V magn, const V sign) { + const auto msb = SignBit(DFromV<V>()); + return Or(AndNot(msb, magn), And(msb, sign)); +} + +template <class V> +HWY_API V CopySignToAbs(const V abs, const V sign) { + const auto msb = SignBit(DFromV<V>()); + return Or(abs, And(msb, sign)); +} + +// ================================================== ARITHMETIC + +// ------------------------------ Add + +namespace detail { +HWY_SVE_FOREACH(HWY_SVE_RETV_ARGPVN, AddN, add_n) +} // namespace detail + +HWY_SVE_FOREACH(HWY_SVE_RETV_ARGPVV, Add, add) + +// ------------------------------ Sub + +namespace detail { +// Can't use HWY_SVE_RETV_ARGPVN because caller wants to specify pg. +#define HWY_SVE_RETV_ARGPVN_MASK(BASE, CHAR, BITS, NAME, OP) \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME(svbool_t pg, HWY_SVE_V(BASE, BITS) a, HWY_SVE_T(BASE, BITS) b) { \ + return sv##OP##_##CHAR##BITS##_z(pg, a, b); \ + } + +HWY_SVE_FOREACH(HWY_SVE_RETV_ARGPVN_MASK, SubN, sub_n) +#undef HWY_SVE_RETV_ARGPVN_MASK +} // namespace detail + +HWY_SVE_FOREACH(HWY_SVE_RETV_ARGPVV, Sub, sub) + +// ------------------------------ SaturatedAdd + +HWY_SVE_FOREACH_UI08(HWY_SVE_RETV_ARGVV, SaturatedAdd, qadd) +HWY_SVE_FOREACH_UI16(HWY_SVE_RETV_ARGVV, SaturatedAdd, qadd) + +// ------------------------------ SaturatedSub + +HWY_SVE_FOREACH_UI08(HWY_SVE_RETV_ARGVV, SaturatedSub, qsub) +HWY_SVE_FOREACH_UI16(HWY_SVE_RETV_ARGVV, SaturatedSub, qsub) + +// ------------------------------ AbsDiff +HWY_SVE_FOREACH_IF(HWY_SVE_RETV_ARGPVV, AbsDiff, abd) + +// ------------------------------ ShiftLeft[Same] + +#define HWY_SVE_SHIFT_N(BASE, CHAR, BITS, NAME, OP) \ + template <int kBits> \ + HWY_API HWY_SVE_V(BASE, BITS) NAME(HWY_SVE_V(BASE, BITS) v) { \ + return sv##OP##_##CHAR##BITS##_x(HWY_SVE_PTRUE(BITS), v, kBits); \ + } \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME##Same(HWY_SVE_V(BASE, BITS) v, HWY_SVE_T(uint, BITS) bits) { \ + return sv##OP##_##CHAR##BITS##_x(HWY_SVE_PTRUE(BITS), v, bits); \ + } + +HWY_SVE_FOREACH_UI(HWY_SVE_SHIFT_N, ShiftLeft, lsl_n) + +// ------------------------------ ShiftRight[Same] + +HWY_SVE_FOREACH_U(HWY_SVE_SHIFT_N, ShiftRight, lsr_n) +HWY_SVE_FOREACH_I(HWY_SVE_SHIFT_N, ShiftRight, asr_n) + +#undef HWY_SVE_SHIFT_N + +// ------------------------------ RotateRight + +// TODO(janwas): svxar on SVE2 +template <int kBits, class V> +HWY_API V RotateRight(const V v) { + constexpr size_t kSizeInBits = sizeof(TFromV<V>) * 8; + static_assert(0 <= kBits && kBits < kSizeInBits, "Invalid shift count"); + if (kBits == 0) return v; + return Or(ShiftRight<kBits>(v), ShiftLeft<kSizeInBits - kBits>(v)); +} + +// ------------------------------ Shl/r + +#define HWY_SVE_SHIFT(BASE, CHAR, BITS, NAME, OP) \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME(HWY_SVE_V(BASE, BITS) v, HWY_SVE_V(BASE, BITS) bits) { \ + using TU = HWY_SVE_T(uint, BITS); \ + return sv##OP##_##CHAR##BITS##_x( \ + HWY_SVE_PTRUE(BITS), v, BitCast(Simd<TU, HWY_LANES(TU)>(), bits)); \ + } + +HWY_SVE_FOREACH_UI(HWY_SVE_SHIFT, Shl, lsl) + +HWY_SVE_FOREACH_U(HWY_SVE_SHIFT, Shr, lsr) +HWY_SVE_FOREACH_I(HWY_SVE_SHIFT, Shr, asr) + +#undef HWY_SVE_SHIFT + +// ------------------------------ Min/Max + +HWY_SVE_FOREACH_UI(HWY_SVE_RETV_ARGPVV, Min, min) +HWY_SVE_FOREACH_UI(HWY_SVE_RETV_ARGPVV, Max, max) +HWY_SVE_FOREACH_F(HWY_SVE_RETV_ARGPVV, Min, minnm) +HWY_SVE_FOREACH_F(HWY_SVE_RETV_ARGPVV, Max, maxnm) + +namespace detail { +HWY_SVE_FOREACH_UI(HWY_SVE_RETV_ARGPVN, MinN, min_n) +HWY_SVE_FOREACH_UI(HWY_SVE_RETV_ARGPVN, MaxN, max_n) +} // namespace detail + +// ------------------------------ Mul +HWY_SVE_FOREACH_UI16(HWY_SVE_RETV_ARGPVV, Mul, mul) +HWY_SVE_FOREACH_UIF3264(HWY_SVE_RETV_ARGPVV, Mul, mul) + +// ------------------------------ MulHigh +HWY_SVE_FOREACH_UI16(HWY_SVE_RETV_ARGPVV, MulHigh, mulh) +namespace detail { +HWY_SVE_FOREACH_UI32(HWY_SVE_RETV_ARGPVV, MulHigh, mulh) +HWY_SVE_FOREACH_U64(HWY_SVE_RETV_ARGPVV, MulHigh, mulh) +} // namespace detail + +// ------------------------------ Div +HWY_SVE_FOREACH_F(HWY_SVE_RETV_ARGPVV, Div, div) + +// ------------------------------ ApproximateReciprocal +HWY_SVE_FOREACH_F32(HWY_SVE_RETV_ARGV, ApproximateReciprocal, recpe) + +// ------------------------------ Sqrt +HWY_SVE_FOREACH_F(HWY_SVE_RETV_ARGPV, Sqrt, sqrt) + +// ------------------------------ ApproximateReciprocalSqrt +HWY_SVE_FOREACH_F32(HWY_SVE_RETV_ARGV, ApproximateReciprocalSqrt, rsqrte) + +// ------------------------------ MulAdd +#define HWY_SVE_FMA(BASE, CHAR, BITS, NAME, OP) \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME(HWY_SVE_V(BASE, BITS) mul, HWY_SVE_V(BASE, BITS) x, \ + HWY_SVE_V(BASE, BITS) add) { \ + return sv##OP##_##CHAR##BITS##_x(HWY_SVE_PTRUE(BITS), x, mul, add); \ + } + +HWY_SVE_FOREACH_F(HWY_SVE_FMA, MulAdd, mad) + +// ------------------------------ NegMulAdd +HWY_SVE_FOREACH_F(HWY_SVE_FMA, NegMulAdd, msb) + +// ------------------------------ MulSub +HWY_SVE_FOREACH_F(HWY_SVE_FMA, MulSub, nmsb) + +// ------------------------------ NegMulSub +HWY_SVE_FOREACH_F(HWY_SVE_FMA, NegMulSub, nmad) + +#undef HWY_SVE_FMA + +// ------------------------------ Round etc. + +HWY_SVE_FOREACH_F(HWY_SVE_RETV_ARGPV, Round, rintn) +HWY_SVE_FOREACH_F(HWY_SVE_RETV_ARGPV, Floor, rintm) +HWY_SVE_FOREACH_F(HWY_SVE_RETV_ARGPV, Ceil, rintp) +HWY_SVE_FOREACH_F(HWY_SVE_RETV_ARGPV, Trunc, rintz) + +// ================================================== MASK + +// ------------------------------ RebindMask +template <class D, typename MFrom> +HWY_API svbool_t RebindMask(const D /*d*/, const MFrom mask) { + return mask; +} + +// ------------------------------ Mask logical + +HWY_API svbool_t Not(svbool_t m) { + // We don't know the lane type, so assume 8-bit. For larger types, this will + // de-canonicalize the predicate, i.e. set bits to 1 even though they do not + // correspond to the lowest byte in the lane. Per ARM, such bits are ignored. + return svnot_b_z(HWY_SVE_PTRUE(8), m); +} +HWY_API svbool_t And(svbool_t a, svbool_t b) { + return svand_b_z(b, b, a); // same order as AndNot for consistency +} +HWY_API svbool_t AndNot(svbool_t a, svbool_t b) { + return svbic_b_z(b, b, a); // reversed order like NEON +} +HWY_API svbool_t Or(svbool_t a, svbool_t b) { + return svsel_b(a, a, b); // a ? true : b +} +HWY_API svbool_t Xor(svbool_t a, svbool_t b) { + return svsel_b(a, svnand_b_z(a, a, b), b); // a ? !(a & b) : b. +} + +// ------------------------------ CountTrue + +#define HWY_SVE_COUNT_TRUE(BASE, CHAR, BITS, NAME, OP) \ + template <size_t N> \ + HWY_API size_t NAME(HWY_SVE_D(BASE, BITS, N) d, svbool_t m) { \ + return sv##OP##_b##BITS(detail::Mask(d), m); \ + } + +HWY_SVE_FOREACH(HWY_SVE_COUNT_TRUE, CountTrue, cntp) +#undef HWY_SVE_COUNT_TRUE + +// For 16-bit Compress: full vector, not limited to SV_POW2. +namespace detail { + +#define HWY_SVE_COUNT_TRUE_FULL(BASE, CHAR, BITS, NAME, OP) \ + template <size_t N> \ + HWY_API size_t NAME(HWY_SVE_D(BASE, BITS, N) d, svbool_t m) { \ + return sv##OP##_b##BITS(svptrue_b##BITS(), m); \ + } + +HWY_SVE_FOREACH(HWY_SVE_COUNT_TRUE_FULL, CountTrueFull, cntp) +#undef HWY_SVE_COUNT_TRUE_FULL + +} // namespace detail + +// ------------------------------ AllFalse +template <typename T, size_t N> +HWY_API bool AllFalse(Simd<T, N> d, svbool_t m) { + return !svptest_any(detail::Mask(d), m); +} + +// ------------------------------ AllTrue +template <typename T, size_t N> +HWY_API bool AllTrue(Simd<T, N> d, svbool_t m) { + return CountTrue(d, m) == Lanes(d); +} + +// ------------------------------ FindFirstTrue +template <typename T, size_t N> +HWY_API intptr_t FindFirstTrue(Simd<T, N> d, svbool_t m) { + return AllFalse(d, m) ? -1 : CountTrue(d, svbrkb_b_z(detail::Mask(d), m)); +} + +// ------------------------------ IfThenElse +#define HWY_SVE_IF_THEN_ELSE(BASE, CHAR, BITS, NAME, OP) \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME(svbool_t m, HWY_SVE_V(BASE, BITS) yes, HWY_SVE_V(BASE, BITS) no) { \ + return sv##OP##_##CHAR##BITS(m, yes, no); \ + } + +HWY_SVE_FOREACH(HWY_SVE_IF_THEN_ELSE, IfThenElse, sel) +#undef HWY_SVE_IF_THEN_ELSE + +// ------------------------------ IfThenElseZero +template <class M, class V> +HWY_API V IfThenElseZero(const M mask, const V yes) { + return IfThenElse(mask, yes, Zero(DFromV<V>())); +} + +// ------------------------------ IfThenZeroElse +template <class M, class V> +HWY_API V IfThenZeroElse(const M mask, const V no) { + return IfThenElse(mask, Zero(DFromV<V>()), no); +} + +// ================================================== COMPARE + +// mask = f(vector, vector) +#define HWY_SVE_COMPARE(BASE, CHAR, BITS, NAME, OP) \ + HWY_API svbool_t NAME(HWY_SVE_V(BASE, BITS) a, HWY_SVE_V(BASE, BITS) b) { \ + return sv##OP##_##CHAR##BITS(HWY_SVE_PTRUE(BITS), a, b); \ + } +#define HWY_SVE_COMPARE_N(BASE, CHAR, BITS, NAME, OP) \ + HWY_API svbool_t NAME(HWY_SVE_V(BASE, BITS) a, HWY_SVE_T(BASE, BITS) b) { \ + return sv##OP##_##CHAR##BITS(HWY_SVE_PTRUE(BITS), a, b); \ + } + +// ------------------------------ Eq +HWY_SVE_FOREACH(HWY_SVE_COMPARE, Eq, cmpeq) + +// ------------------------------ Ne +HWY_SVE_FOREACH(HWY_SVE_COMPARE, Ne, cmpne) + +// ------------------------------ Lt +HWY_SVE_FOREACH(HWY_SVE_COMPARE, Lt, cmplt) +namespace detail { +HWY_SVE_FOREACH_IF(HWY_SVE_COMPARE_N, LtN, cmplt_n) +} // namespace detail + +// ------------------------------ Le +HWY_SVE_FOREACH_F(HWY_SVE_COMPARE, Le, cmple) + +#undef HWY_SVE_COMPARE +#undef HWY_SVE_COMPARE_N + +// ------------------------------ Gt/Ge (swapped order) + +template <class V> +HWY_API svbool_t Gt(const V a, const V b) { + return Lt(b, a); +} +template <class V> +HWY_API svbool_t Ge(const V a, const V b) { + return Le(b, a); +} + +// ------------------------------ TestBit +template <class V> +HWY_API svbool_t TestBit(const V a, const V bit) { + return Ne(And(a, bit), Zero(DFromV<V>())); +} + +// ------------------------------ MaskFromVec (Ne) +template <class V> +HWY_API svbool_t MaskFromVec(const V v) { + return Ne(v, Zero(DFromV<V>())); +} + +// ------------------------------ VecFromMask + +template <class D, HWY_IF_NOT_FLOAT_D(D)> +HWY_API VFromD<D> VecFromMask(const D d, svbool_t mask) { + const auto v0 = Zero(RebindToSigned<decltype(d)>()); + return BitCast(d, detail::SubN(mask, v0, 1)); +} + +template <class D, HWY_IF_FLOAT_D(D)> +HWY_API VFromD<D> VecFromMask(const D d, svbool_t mask) { + return BitCast(d, VecFromMask(RebindToUnsigned<D>(), mask)); +} + +// ================================================== MEMORY + +// ------------------------------ Load/MaskedLoad/LoadDup128/Store/Stream + +#define HWY_SVE_LOAD(BASE, CHAR, BITS, NAME, OP) \ + template <size_t N> \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME(HWY_SVE_D(BASE, BITS, N) d, \ + const HWY_SVE_T(BASE, BITS) * HWY_RESTRICT p) { \ + return sv##OP##_##CHAR##BITS(detail::Mask(d), p); \ + } + +#define HWY_SVE_MASKED_LOAD(BASE, CHAR, BITS, NAME, OP) \ + template <size_t N> \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME(svbool_t m, HWY_SVE_D(BASE, BITS, N) d, \ + const HWY_SVE_T(BASE, BITS) * HWY_RESTRICT p) { \ + return sv##OP##_##CHAR##BITS(m, p); \ + } + +#define HWY_SVE_LOAD_DUP128(BASE, CHAR, BITS, NAME, OP) \ + template <size_t N> \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME(HWY_SVE_D(BASE, BITS, N) d, \ + const HWY_SVE_T(BASE, BITS) * HWY_RESTRICT p) { \ + /* All-true predicate to load all 128 bits. */ \ + return sv##OP##_##CHAR##BITS(HWY_SVE_PTRUE(8), p); \ + } + +#define HWY_SVE_STORE(BASE, CHAR, BITS, NAME, OP) \ + template <size_t N> \ + HWY_API void NAME(HWY_SVE_V(BASE, BITS) v, HWY_SVE_D(BASE, BITS, N) d, \ + HWY_SVE_T(BASE, BITS) * HWY_RESTRICT p) { \ + sv##OP##_##CHAR##BITS(detail::Mask(d), p, v); \ + } + +#define HWY_SVE_MASKED_STORE(BASE, CHAR, BITS, NAME, OP) \ + template <size_t N> \ + HWY_API void NAME(svbool_t m, HWY_SVE_V(BASE, BITS) v, \ + HWY_SVE_D(BASE, BITS, N) d, \ + HWY_SVE_T(BASE, BITS) * HWY_RESTRICT p) { \ + sv##OP##_##CHAR##BITS(m, p, v); \ + } + +HWY_SVE_FOREACH(HWY_SVE_LOAD, Load, ld1) +HWY_SVE_FOREACH(HWY_SVE_MASKED_LOAD, MaskedLoad, ld1) +HWY_SVE_FOREACH(HWY_SVE_LOAD_DUP128, LoadDup128, ld1rq) +HWY_SVE_FOREACH(HWY_SVE_STORE, Store, st1) +HWY_SVE_FOREACH(HWY_SVE_STORE, Stream, stnt1) +HWY_SVE_FOREACH(HWY_SVE_MASKED_STORE, MaskedStore, st1) + +#undef HWY_SVE_LOAD +#undef HWY_SVE_MASKED_LOAD +#undef HWY_SVE_LOAD_DUP128 +#undef HWY_SVE_STORE +#undef HWY_SVE_MASKED_STORE + +// BF16 is the same as svuint16_t because BF16 is optional before v8.6. +template <size_t N> +HWY_API svuint16_t Load(Simd<bfloat16_t, N> d, + const bfloat16_t* HWY_RESTRICT p) { + return Load(RebindToUnsigned<decltype(d)>(), + reinterpret_cast<const uint16_t * HWY_RESTRICT>(p)); +} + +template <size_t N> +HWY_API void Store(svuint16_t v, Simd<bfloat16_t, N> d, + bfloat16_t* HWY_RESTRICT p) { + Store(v, RebindToUnsigned<decltype(d)>(), + reinterpret_cast<uint16_t * HWY_RESTRICT>(p)); +} + +// ------------------------------ Load/StoreU + +// SVE only requires lane alignment, not natural alignment of the entire +// vector. +template <class D> +HWY_API VFromD<D> LoadU(D d, const TFromD<D>* HWY_RESTRICT p) { + return Load(d, p); +} + +template <class V, class D> +HWY_API void StoreU(const V v, D d, TFromD<D>* HWY_RESTRICT p) { + Store(v, d, p); +} + +// ------------------------------ ScatterOffset/Index + +#define HWY_SVE_SCATTER_OFFSET(BASE, CHAR, BITS, NAME, OP) \ + template <size_t N> \ + HWY_API void NAME(HWY_SVE_V(BASE, BITS) v, HWY_SVE_D(BASE, BITS, N) d, \ + HWY_SVE_T(BASE, BITS) * HWY_RESTRICT base, \ + HWY_SVE_V(int, BITS) offset) { \ + sv##OP##_s##BITS##offset_##CHAR##BITS(detail::Mask(d), base, offset, v); \ + } + +#define HWY_SVE_SCATTER_INDEX(BASE, CHAR, BITS, NAME, OP) \ + template <size_t N> \ + HWY_API void NAME(HWY_SVE_V(BASE, BITS) v, HWY_SVE_D(BASE, BITS, N) d, \ + HWY_SVE_T(BASE, BITS) * HWY_RESTRICT base, \ + HWY_SVE_V(int, BITS) index) { \ + sv##OP##_s##BITS##index_##CHAR##BITS(detail::Mask(d), base, index, v); \ + } + +HWY_SVE_FOREACH_UIF3264(HWY_SVE_SCATTER_OFFSET, ScatterOffset, st1_scatter) +HWY_SVE_FOREACH_UIF3264(HWY_SVE_SCATTER_INDEX, ScatterIndex, st1_scatter) +#undef HWY_SVE_SCATTER_OFFSET +#undef HWY_SVE_SCATTER_INDEX + +// ------------------------------ GatherOffset/Index + +#define HWY_SVE_GATHER_OFFSET(BASE, CHAR, BITS, NAME, OP) \ + template <size_t N> \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME(HWY_SVE_D(BASE, BITS, N) d, \ + const HWY_SVE_T(BASE, BITS) * HWY_RESTRICT base, \ + HWY_SVE_V(int, BITS) offset) { \ + return sv##OP##_s##BITS##offset_##CHAR##BITS(detail::Mask(d), base, \ + offset); \ + } +#define HWY_SVE_GATHER_INDEX(BASE, CHAR, BITS, NAME, OP) \ + template <size_t N> \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME(HWY_SVE_D(BASE, BITS, N) d, \ + const HWY_SVE_T(BASE, BITS) * HWY_RESTRICT base, \ + HWY_SVE_V(int, BITS) index) { \ + return sv##OP##_s##BITS##index_##CHAR##BITS(detail::Mask(d), base, index); \ + } + +HWY_SVE_FOREACH_UIF3264(HWY_SVE_GATHER_OFFSET, GatherOffset, ld1_gather) +HWY_SVE_FOREACH_UIF3264(HWY_SVE_GATHER_INDEX, GatherIndex, ld1_gather) +#undef HWY_SVE_GATHER_OFFSET +#undef HWY_SVE_GATHER_INDEX + +// ------------------------------ StoreInterleaved3 + +#define HWY_SVE_STORE3(BASE, CHAR, BITS, NAME, OP) \ + template <size_t N> \ + HWY_API void NAME(HWY_SVE_V(BASE, BITS) v0, HWY_SVE_V(BASE, BITS) v1, \ + HWY_SVE_V(BASE, BITS) v2, HWY_SVE_D(BASE, BITS, N) d, \ + HWY_SVE_T(BASE, BITS) * HWY_RESTRICT unaligned) { \ + const sv##BASE##BITS##x3_t triple = svcreate3##_##CHAR##BITS(v0, v1, v2); \ + sv##OP##_##CHAR##BITS(detail::Mask(d), unaligned, triple); \ + } +HWY_SVE_FOREACH_U08(HWY_SVE_STORE3, StoreInterleaved3, st3) + +#undef HWY_SVE_STORE3 + +// ------------------------------ StoreInterleaved4 + +#define HWY_SVE_STORE4(BASE, CHAR, BITS, NAME, OP) \ + template <size_t N> \ + HWY_API void NAME(HWY_SVE_V(BASE, BITS) v0, HWY_SVE_V(BASE, BITS) v1, \ + HWY_SVE_V(BASE, BITS) v2, HWY_SVE_V(BASE, BITS) v3, \ + HWY_SVE_D(BASE, BITS, N) d, \ + HWY_SVE_T(BASE, BITS) * HWY_RESTRICT unaligned) { \ + const sv##BASE##BITS##x4_t quad = \ + svcreate4##_##CHAR##BITS(v0, v1, v2, v3); \ + sv##OP##_##CHAR##BITS(detail::Mask(d), unaligned, quad); \ + } +HWY_SVE_FOREACH_U08(HWY_SVE_STORE4, StoreInterleaved4, st4) + +#undef HWY_SVE_STORE4 + +// ================================================== CONVERT + +// ------------------------------ PromoteTo + +// Same sign +#define HWY_SVE_PROMOTE_TO(BASE, CHAR, BITS, NAME, OP) \ + template <size_t N> \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME(HWY_SVE_D(BASE, BITS, N) /* tag */, \ + VFromD<Simd<MakeNarrow<HWY_SVE_T(BASE, BITS)>, \ + HWY_LANES(HWY_SVE_T(BASE, BITS)) * 2>> \ + v) { \ + return sv##OP##_##CHAR##BITS(v); \ + } + +HWY_SVE_FOREACH_UI16(HWY_SVE_PROMOTE_TO, PromoteTo, unpklo) +HWY_SVE_FOREACH_UI32(HWY_SVE_PROMOTE_TO, PromoteTo, unpklo) +HWY_SVE_FOREACH_UI64(HWY_SVE_PROMOTE_TO, PromoteTo, unpklo) + +// 2x +template <size_t N> +HWY_API svuint32_t PromoteTo(Simd<uint32_t, N> dto, svuint8_t vfrom) { + const RepartitionToWide<DFromV<decltype(vfrom)>> d2; + return PromoteTo(dto, PromoteTo(d2, vfrom)); +} +template <size_t N> +HWY_API svint32_t PromoteTo(Simd<int32_t, N> dto, svint8_t vfrom) { + const RepartitionToWide<DFromV<decltype(vfrom)>> d2; + return PromoteTo(dto, PromoteTo(d2, vfrom)); +} +template <size_t N> +HWY_API svuint32_t U32FromU8(svuint8_t v) { + return PromoteTo(Simd<uint32_t, N>(), v); +} + +// Sign change +template <size_t N> +HWY_API svint16_t PromoteTo(Simd<int16_t, N> dto, svuint8_t vfrom) { + const RebindToUnsigned<decltype(dto)> du; + return BitCast(dto, PromoteTo(du, vfrom)); +} +template <size_t N> +HWY_API svint32_t PromoteTo(Simd<int32_t, N> dto, svuint16_t vfrom) { + const RebindToUnsigned<decltype(dto)> du; + return BitCast(dto, PromoteTo(du, vfrom)); +} +template <size_t N> +HWY_API svint32_t PromoteTo(Simd<int32_t, N> dto, svuint8_t vfrom) { + const Repartition<uint16_t, DFromV<decltype(vfrom)>> du16; + const Repartition<int16_t, decltype(du16)> di16; + return PromoteTo(dto, BitCast(di16, PromoteTo(du16, vfrom))); +} + +// ------------------------------ PromoteTo F + +template <size_t N> +HWY_API svfloat32_t PromoteTo(Simd<float32_t, N> /* d */, const svfloat16_t v) { + return svcvt_f32_f16_x(detail::PTrue(Simd<float16_t, N>()), v); +} + +template <size_t N> +HWY_API svfloat64_t PromoteTo(Simd<float64_t, N> /* d */, const svfloat32_t v) { + return svcvt_f64_f32_x(detail::PTrue(Simd<float32_t, N>()), v); +} + +template <size_t N> +HWY_API svfloat64_t PromoteTo(Simd<float64_t, N> /* d */, const svint32_t v) { + return svcvt_f64_s32_x(detail::PTrue(Simd<int32_t, N>()), v); +} + +// For 16-bit Compress +namespace detail { +HWY_SVE_FOREACH_UI32(HWY_SVE_PROMOTE_TO, PromoteUpperTo, unpkhi) +#undef HWY_SVE_PROMOTE_TO + +template <size_t N> +HWY_API svfloat32_t PromoteUpperTo(Simd<float, N> df, const svfloat16_t v) { + const RebindToUnsigned<decltype(df)> du; + const RepartitionToNarrow<decltype(du)> dn; + return BitCast(df, PromoteUpperTo(du, BitCast(dn, v))); +} + +} // namespace detail + +// ------------------------------ DemoteTo U + +namespace detail { + +// Saturates unsigned vectors to half/quarter-width TN. +template <typename TN, class VU> +VU SaturateU(VU v) { + return detail::MinN(v, static_cast<TFromV<VU>>(LimitsMax<TN>())); +} + +// Saturates unsigned vectors to half/quarter-width TN. +template <typename TN, class VI> +VI SaturateI(VI v) { + const DFromV<VI> di; + return detail::MinN(detail::MaxN(v, LimitsMin<TN>()), LimitsMax<TN>()); +} + +} // namespace detail + +template <size_t N> +HWY_API svuint8_t DemoteTo(Simd<uint8_t, N> dn, const svint16_t v) { + const DFromV<decltype(v)> di; + const RebindToUnsigned<decltype(di)> du; + using TN = TFromD<decltype(dn)>; + // First clamp negative numbers to zero and cast to unsigned. + const svuint16_t clamped = BitCast(du, Max(Zero(di), v)); + // Saturate to unsigned-max and halve the width. + const svuint8_t vn = BitCast(dn, detail::SaturateU<TN>(clamped)); + return svuzp1_u8(vn, vn); +} + +template <size_t N> +HWY_API svuint16_t DemoteTo(Simd<uint16_t, N> dn, const svint32_t v) { + const DFromV<decltype(v)> di; + const RebindToUnsigned<decltype(di)> du; + using TN = TFromD<decltype(dn)>; + // First clamp negative numbers to zero and cast to unsigned. + const svuint32_t clamped = BitCast(du, Max(Zero(di), v)); + // Saturate to unsigned-max and halve the width. + const svuint16_t vn = BitCast(dn, detail::SaturateU<TN>(clamped)); + return svuzp1_u16(vn, vn); +} + +template <size_t N> +HWY_API svuint8_t DemoteTo(Simd<uint8_t, N> dn, const svint32_t v) { + const DFromV<decltype(v)> di; + const RebindToUnsigned<decltype(di)> du; + const RepartitionToNarrow<decltype(du)> d2; + using TN = TFromD<decltype(dn)>; + // First clamp negative numbers to zero and cast to unsigned. + const svuint32_t clamped = BitCast(du, Max(Zero(di), v)); + // Saturate to unsigned-max and quarter the width. + const svuint16_t cast16 = BitCast(d2, detail::SaturateU<TN>(clamped)); + const svuint8_t x2 = BitCast(dn, svuzp1_u16(cast16, cast16)); + return svuzp1_u8(x2, x2); +} + +HWY_API svuint8_t U8FromU32(const svuint32_t v) { + const DFromV<svuint32_t> du32; + const RepartitionToNarrow<decltype(du32)> du16; + const RepartitionToNarrow<decltype(du16)> du8; + + const svuint16_t cast16 = BitCast(du16, v); + const svuint16_t x2 = svuzp1_u16(cast16, cast16); + const svuint8_t cast8 = BitCast(du8, x2); + return svuzp1_u8(cast8, cast8); +} + +// ------------------------------ DemoteTo I + +template <size_t N> +HWY_API svint8_t DemoteTo(Simd<int8_t, N> dn, const svint16_t v) { + const DFromV<decltype(v)> di; + using TN = TFromD<decltype(dn)>; +#if HWY_TARGET == HWY_SVE2 + const svint8_t vn = BitCast(dn, svqxtnb_s16(v)); +#else + const svint8_t vn = BitCast(dn, detail::SaturateI<TN>(v)); +#endif + return svuzp1_s8(vn, vn); +} + +template <size_t N> +HWY_API svint16_t DemoteTo(Simd<int16_t, N> dn, const svint32_t v) { + const DFromV<decltype(v)> di; + using TN = TFromD<decltype(dn)>; +#if HWY_TARGET == HWY_SVE2 + const svint16_t vn = BitCast(dn, svqxtnb_s32(v)); +#else + const svint16_t vn = BitCast(dn, detail::SaturateI<TN>(v)); +#endif + return svuzp1_s16(vn, vn); +} + +template <size_t N> +HWY_API svint8_t DemoteTo(Simd<int8_t, N> dn, const svint32_t v) { + const DFromV<decltype(v)> di; + using TN = TFromD<decltype(dn)>; + const RepartitionToWide<decltype(dn)> d2; +#if HWY_TARGET == HWY_SVE2 + const svint16_t cast16 = BitCast(d2, svqxtnb_s16(svqxtnb_s32(v))); +#else + const svint16_t cast16 = BitCast(d2, detail::SaturateI<TN>(v)); +#endif + const svint8_t v2 = BitCast(dn, svuzp1_s16(cast16, cast16)); + return BitCast(dn, svuzp1_s8(v2, v2)); +} + +// ------------------------------ ConcatEven/ConcatOdd + +// WARNING: the upper half of these needs fixing up (uzp1/uzp2 use the +// full vector length, not rounded down to a power of two as we require). +namespace detail { + +#define HWY_SVE_CONCAT_EVERY_SECOND(BASE, CHAR, BITS, NAME, OP) \ + HWY_INLINE HWY_SVE_V(BASE, BITS) \ + NAME(HWY_SVE_V(BASE, BITS) hi, HWY_SVE_V(BASE, BITS) lo) { \ + return sv##OP##_##CHAR##BITS(lo, hi); \ + } +HWY_SVE_FOREACH(HWY_SVE_CONCAT_EVERY_SECOND, ConcatEven, uzp1) +HWY_SVE_FOREACH(HWY_SVE_CONCAT_EVERY_SECOND, ConcatOdd, uzp2) +#undef HWY_SVE_CONCAT_EVERY_SECOND + +// Used to slide up / shift whole register left; mask indicates which range +// to take from lo, and the rest is filled from hi starting at its lowest. +#define HWY_SVE_SPLICE(BASE, CHAR, BITS, NAME, OP) \ + HWY_API HWY_SVE_V(BASE, BITS) NAME( \ + HWY_SVE_V(BASE, BITS) hi, HWY_SVE_V(BASE, BITS) lo, svbool_t mask) { \ + return sv##OP##_##CHAR##BITS(mask, lo, hi); \ + } +HWY_SVE_FOREACH(HWY_SVE_SPLICE, Splice, splice) +#undef HWY_SVE_SPLICE + +} // namespace detail + +template <class D> +HWY_API VFromD<D> ConcatOdd(D d, VFromD<D> hi, VFromD<D> lo) { +#if 0 // if we could assume VL is a power of two + return detail::ConcatOdd(hi, lo); +#else + const VFromD<D> hi_odd = detail::ConcatOdd(hi, hi); + const VFromD<D> lo_odd = detail::ConcatOdd(lo, lo); + return detail::Splice(hi_odd, lo_odd, FirstN(d, Lanes(d) / 2)); +#endif +} + +template <class D> +HWY_API VFromD<D> ConcatEven(D d, VFromD<D> hi, VFromD<D> lo) { +#if 0 // if we could assume VL is a power of two + return detail::ConcatEven(hi, lo); +#else + const VFromD<D> hi_odd = detail::ConcatEven(hi, hi); + const VFromD<D> lo_odd = detail::ConcatEven(lo, lo); + return detail::Splice(hi_odd, lo_odd, FirstN(d, Lanes(d) / 2)); +#endif +} + +// ------------------------------ DemoteTo F + +template <size_t N> +HWY_API svfloat16_t DemoteTo(Simd<float16_t, N> d, const svfloat32_t v) { + return svcvt_f16_f32_x(detail::PTrue(d), v); +} + +template <size_t N> +HWY_API svuint16_t DemoteTo(Simd<bfloat16_t, N> d, const svfloat32_t v) { + const svuint16_t halves = BitCast(Full<uint16_t>(), v); + return detail::ConcatOdd(halves, halves); // can ignore upper half of vec +} + +template <size_t N> +HWY_API svfloat32_t DemoteTo(Simd<float32_t, N> d, const svfloat64_t v) { + return svcvt_f32_f64_x(detail::PTrue(d), v); +} + +template <size_t N> +HWY_API svint32_t DemoteTo(Simd<int32_t, N> d, const svfloat64_t v) { + return svcvt_s32_f64_x(detail::PTrue(d), v); +} + +// ------------------------------ ConvertTo F + +#define HWY_SVE_CONVERT(BASE, CHAR, BITS, NAME, OP) \ + template <size_t N> \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME(HWY_SVE_D(BASE, BITS, N) /* d */, HWY_SVE_V(int, BITS) v) { \ + return sv##OP##_##CHAR##BITS##_s##BITS##_x(HWY_SVE_PTRUE(BITS), v); \ + } \ + /* Truncates (rounds toward zero). */ \ + template <size_t N> \ + HWY_API HWY_SVE_V(int, BITS) \ + NAME(HWY_SVE_D(int, BITS, N) /* d */, HWY_SVE_V(BASE, BITS) v) { \ + return sv##OP##_s##BITS##_##CHAR##BITS##_x(HWY_SVE_PTRUE(BITS), v); \ + } + +// API only requires f32 but we provide f64 for use by Iota. +HWY_SVE_FOREACH_F(HWY_SVE_CONVERT, ConvertTo, cvt) +#undef HWY_SVE_CONVERT + +// ------------------------------ NearestInt (Round, ConvertTo) + +template <class VF, class DI = RebindToSigned<DFromV<VF>>> +HWY_API VFromD<DI> NearestInt(VF v) { + // No single instruction, round then truncate. + return ConvertTo(DI(), Round(v)); +} + +// ------------------------------ Iota (Add, ConvertTo) + +#define HWY_SVE_IOTA(BASE, CHAR, BITS, NAME, OP) \ + template <size_t N> \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME(HWY_SVE_D(BASE, BITS, N) d, HWY_SVE_T(BASE, BITS) first) { \ + return sv##OP##_##CHAR##BITS(first, 1); \ + } + +HWY_SVE_FOREACH_UI(HWY_SVE_IOTA, Iota, index) +#undef HWY_SVE_IOTA + +template <class D, HWY_IF_FLOAT_D(D)> +HWY_API VFromD<D> Iota(const D d, TFromD<D> first) { + const RebindToSigned<D> di; + return detail::AddN(ConvertTo(d, Iota(di, 0)), first); +} + +// ================================================== COMBINE + +namespace detail { + +template <typename T, size_t N> +svbool_t MaskLowerHalf(Simd<T, N> d) { + return FirstN(d, Lanes(d) / 2); +} +template <typename T, size_t N> +svbool_t MaskUpperHalf(Simd<T, N> d) { + // For Splice to work as intended, make sure bits above Lanes(d) are zero. + return AndNot(MaskLowerHalf(d), detail::Mask(d)); +} + +// Right-shift vector pair by constexpr; can be used to slide down (=N) or up +// (=Lanes()-N). +#define HWY_SVE_EXT(BASE, CHAR, BITS, NAME, OP) \ + template <size_t kIndex> \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME(HWY_SVE_V(BASE, BITS) hi, HWY_SVE_V(BASE, BITS) lo) { \ + return sv##OP##_##CHAR##BITS(lo, hi, kIndex); \ + } +HWY_SVE_FOREACH(HWY_SVE_EXT, Ext, ext) +#undef HWY_SVE_EXT + +} // namespace detail + +// ------------------------------ ConcatUpperLower +template <class D, class V> +HWY_API V ConcatUpperLower(const D d, const V hi, const V lo) { + return IfThenElse(detail::MaskLowerHalf(d), lo, hi); +} + +// ------------------------------ ConcatLowerLower +template <class D, class V> +HWY_API V ConcatLowerLower(const D d, const V hi, const V lo) { + return detail::Splice(hi, lo, detail::MaskLowerHalf(d)); +} + +// ------------------------------ ConcatLowerUpper +template <class D, class V> +HWY_API V ConcatLowerUpper(const D d, const V hi, const V lo) { + return detail::Splice(hi, lo, detail::MaskUpperHalf(d)); +} + +// ------------------------------ ConcatUpperUpper +template <class D, class V> +HWY_API V ConcatUpperUpper(const D d, const V hi, const V lo) { + const svbool_t mask_upper = detail::MaskUpperHalf(d); + const V lo_upper = detail::Splice(lo, lo, mask_upper); + return IfThenElse(mask_upper, hi, lo_upper); +} + +// ------------------------------ Combine +template <class D, class V2> +HWY_API VFromD<D> Combine(const D d, const V2 hi, const V2 lo) { + return ConcatLowerLower(d, hi, lo); +} + +// ------------------------------ ZeroExtendVector + +template <class D, class V> +HWY_API V ZeroExtendVector(const D d, const V lo) { + return Combine(d, Zero(Half<D>()), lo); +} + +// ------------------------------ Lower/UpperHalf + +template <class D2, class V> +HWY_API V LowerHalf(D2 /* tag */, const V v) { + return v; +} + +template <class V> +HWY_API V LowerHalf(const V v) { + return v; +} + +template <class D2, class V> +HWY_API V UpperHalf(const D2 d2, const V v) { + return detail::Splice(v, v, detail::MaskUpperHalf(Twice<D2>())); +} + +// ================================================== SWIZZLE + +// ------------------------------ GetLane + +#define HWY_SVE_GET_LANE(BASE, CHAR, BITS, NAME, OP) \ + HWY_API HWY_SVE_T(BASE, BITS) NAME(HWY_SVE_V(BASE, BITS) v) { \ + return sv##OP##_##CHAR##BITS(detail::PFalse(), v); \ + } + +HWY_SVE_FOREACH(HWY_SVE_GET_LANE, GetLane, lasta) +#undef HWY_SVE_GET_LANE + +// ------------------------------ OddEven + +namespace detail { +HWY_SVE_FOREACH(HWY_SVE_RETV_ARGVN, Insert, insr_n) +HWY_SVE_FOREACH(HWY_SVE_RETV_ARGVV, InterleaveEven, trn1) +HWY_SVE_FOREACH(HWY_SVE_RETV_ARGVV, InterleaveOdd, trn2) +} // namespace detail + +template <class V> +HWY_API V OddEven(const V odd, const V even) { + const auto even_in_odd = detail::Insert(even, 0); + return detail::InterleaveOdd(even_in_odd, odd); +} + +// ------------------------------ OddEvenBlocks +template <class V> +HWY_API V OddEvenBlocks(const V odd, const V even) { + const RebindToUnsigned<DFromV<V>> du; + constexpr size_t kShift = CeilLog2(16 / sizeof(TFromV<V>)); + const auto idx_block = ShiftRight<kShift>(Iota(du, 0)); + const svbool_t is_even = Eq(detail::AndN(idx_block, 1), Zero(du)); + return IfThenElse(is_even, even, odd); +} + +// ------------------------------ SwapAdjacentBlocks + +namespace detail { + +template <typename T, size_t N> +constexpr size_t LanesPerBlock(Simd<T, N> /* tag */) { + // We might have a capped vector smaller than a block, so honor that. + return HWY_MIN(16 / sizeof(T), N); +} + +} // namespace detail + +template <class V> +HWY_API V SwapAdjacentBlocks(const V v) { + const DFromV<V> d; + constexpr size_t kLanesPerBlock = detail::LanesPerBlock(d); + const V down = detail::Ext<kLanesPerBlock>(v, v); + const V up = detail::Splice(v, v, FirstN(d, kLanesPerBlock)); + return OddEvenBlocks(up, down); +} + +// ------------------------------ TableLookupLanes + +template <class D, class VI> +HWY_API VFromD<RebindToUnsigned<D>> IndicesFromVec(D d, VI vec) { + static_assert(sizeof(TFromD<D>) == sizeof(TFromV<VI>), "Index != lane"); + const RebindToUnsigned<D> du; + const auto indices = BitCast(du, vec); +#if HWY_IS_DEBUG_BUILD + HWY_DASSERT(AllTrue(du, Lt(indices, Set(du, Lanes(d))))); +#endif + return indices; +} + +template <class D, typename TI> +HWY_API VFromD<RebindToUnsigned<D>> SetTableIndices(D d, const TI* idx) { + static_assert(sizeof(TFromD<D>) == sizeof(TI), "Index size must match lane"); + return IndicesFromVec(d, LoadU(Rebind<TI, D>(), idx)); +} + +// <32bit are not part of Highway API, but used in Broadcast. +#define HWY_SVE_TABLE(BASE, CHAR, BITS, NAME, OP) \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME(HWY_SVE_V(BASE, BITS) v, HWY_SVE_V(uint, BITS) idx) { \ + return sv##OP##_##CHAR##BITS(v, idx); \ + } + +HWY_SVE_FOREACH(HWY_SVE_TABLE, TableLookupLanes, tbl) +#undef HWY_SVE_TABLE + +// ------------------------------ Reverse + +#if 0 // if we could assume VL is a power of two +#error "Update macro" +#endif +#define HWY_SVE_REVERSE(BASE, CHAR, BITS, NAME, OP) \ + template <size_t N> \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME(Simd<HWY_SVE_T(BASE, BITS), N> d, HWY_SVE_V(BASE, BITS) v) { \ + const auto reversed = sv##OP##_##CHAR##BITS(v); \ + /* Shift right to remove extra (non-pow2 and remainder) lanes. */ \ + const size_t all_lanes = \ + detail::AllHardwareLanes(hwy::SizeTag<BITS / 8>()); \ + /* TODO(janwas): on SVE2, use whilege. */ \ + const svbool_t mask = Not(FirstN(d, all_lanes - Lanes(d))); \ + return detail::Splice(reversed, reversed, mask); \ + } + +HWY_SVE_FOREACH(HWY_SVE_REVERSE, Reverse, rev) +#undef HWY_SVE_REVERSE + +// ------------------------------ Compress (PromoteTo) + +#define HWY_SVE_COMPRESS(BASE, CHAR, BITS, NAME, OP) \ + HWY_API HWY_SVE_V(BASE, BITS) NAME(HWY_SVE_V(BASE, BITS) v, svbool_t mask) { \ + return sv##OP##_##CHAR##BITS(mask, v); \ + } + +HWY_SVE_FOREACH_UIF3264(HWY_SVE_COMPRESS, Compress, compact) +#undef HWY_SVE_COMPRESS + +template <class V, HWY_IF_LANE_SIZE_V(V, 2)> +HWY_API V Compress(V v, svbool_t mask16) { + static_assert(!IsSame<V, svfloat16_t>(), "Must use overload"); + const DFromV<V> d16; + + // Promote vector and mask to 32-bit + const RepartitionToWide<decltype(d16)> dw; + const auto v32L = PromoteTo(dw, v); + const auto v32H = detail::PromoteUpperTo(dw, v); + const svbool_t mask32L = svunpklo_b(mask16); + const svbool_t mask32H = svunpkhi_b(mask16); + + const auto compressedL = Compress(v32L, mask32L); + const auto compressedH = Compress(v32H, mask32H); + + // Demote to 16-bit (already in range) - separately so we can splice + const V evenL = BitCast(d16, compressedL); + const V evenH = BitCast(d16, compressedH); + const V v16L = detail::ConcatEven(evenL, evenL); // only lower half needed + const V v16H = detail::ConcatEven(evenH, evenH); + + // We need to combine two vectors of non-constexpr length, so the only option + // is Splice, which requires us to synthesize a mask. NOTE: this function uses + // full vectors (SV_ALL instead of SV_POW2), hence we need unmasked svcnt. + const size_t countL = detail::CountTrueFull(dw, mask32L); + const auto compressed_maskL = FirstN(d16, countL); + return detail::Splice(v16H, v16L, compressed_maskL); +} + +// Must treat float16_t as integers so we can ConcatEven. +HWY_API svfloat16_t Compress(svfloat16_t v, svbool_t mask16) { + const DFromV<decltype(v)> df; + const RebindToSigned<decltype(df)> di; + return BitCast(df, Compress(BitCast(di, v), mask16)); +} + +// ------------------------------ CompressStore + +template <class V, class M, class D> +HWY_API size_t CompressStore(const V v, const M mask, const D d, + TFromD<D>* HWY_RESTRICT unaligned) { + StoreU(Compress(v, mask), d, unaligned); + return CountTrue(d, mask); +} + +// ------------------------------ CompressBlendedStore + +template <class V, class M, class D> +HWY_API size_t CompressBlendedStore(const V v, const M mask, const D d, + TFromD<D>* HWY_RESTRICT unaligned) { + const size_t count = CountTrue(d, mask); + const svbool_t store_mask = FirstN(d, count); + MaskedStore(store_mask, Compress(v, mask), d, unaligned); + return count; +} + +// ================================================== BLOCKWISE + +// ------------------------------ CombineShiftRightBytes + +namespace detail { + +// For x86-compatible behaviour mandated by Highway API: TableLookupBytes +// offsets are implicitly relative to the start of their 128-bit block. +template <class D, class V> +HWY_INLINE V OffsetsOf128BitBlocks(const D d, const V iota0) { + using T = MakeUnsigned<TFromD<D>>; + return detail::AndNotN(static_cast<T>(LanesPerBlock(d) - 1), iota0); +} + +template <size_t kLanes, class D> +svbool_t FirstNPerBlock(D d) { + const RebindToSigned<D> di; + constexpr size_t kLanesPerBlock = detail::LanesPerBlock(di); + const auto idx_mod = detail::AndN(Iota(di, 0), kLanesPerBlock - 1); + return detail::LtN(BitCast(di, idx_mod), kLanes); +} + +} // namespace detail + +template <size_t kBytes, class D, class V = VFromD<D>> +HWY_API V CombineShiftRightBytes(const D d, const V hi, const V lo) { + const Repartition<uint8_t, decltype(d)> d8; + const auto hi8 = BitCast(d8, hi); + const auto lo8 = BitCast(d8, lo); + const auto hi_up = detail::Splice(hi8, hi8, FirstN(d8, 16 - kBytes)); + const auto lo_down = detail::Ext<kBytes>(lo8, lo8); + const svbool_t is_lo = detail::FirstNPerBlock<16 - kBytes>(d8); + return BitCast(d, IfThenElse(is_lo, lo_down, hi_up)); +} + +// ------------------------------ Shuffle2301 + +#define HWY_SVE_SHUFFLE_2301(BASE, CHAR, BITS, NAME, OP) \ + HWY_API HWY_SVE_V(BASE, BITS) NAME(HWY_SVE_V(BASE, BITS) v) { \ + const DFromV<decltype(v)> d; \ + const svuint64_t vu64 = BitCast(Repartition<uint64_t, decltype(d)>(), v); \ + return BitCast(d, sv##OP##_u64_x(HWY_SVE_PTRUE(64), vu64)); \ + } + +HWY_SVE_FOREACH_UI32(HWY_SVE_SHUFFLE_2301, Shuffle2301, revw) +#undef HWY_SVE_SHUFFLE_2301 + +template <class V, HWY_IF_FLOAT_V(V)> +HWY_API V Shuffle2301(const V v) { + const DFromV<V> df; + const RebindToUnsigned<decltype(df)> du; + return BitCast(df, Shuffle2301(BitCast(du, v))); +} + +// ------------------------------ Shuffle2103 +template <class V> +HWY_API V Shuffle2103(const V v) { + const DFromV<V> d; + const Repartition<uint8_t, decltype(d)> d8; + static_assert(sizeof(TFromD<decltype(d)>) == 4, "Defined for 32-bit types"); + const svuint8_t v8 = BitCast(d8, v); + return BitCast(d, CombineShiftRightBytes<12>(d8, v8, v8)); +} + +// ------------------------------ Shuffle0321 +template <class V> +HWY_API V Shuffle0321(const V v) { + const DFromV<V> d; + const Repartition<uint8_t, decltype(d)> d8; + static_assert(sizeof(TFromD<decltype(d)>) == 4, "Defined for 32-bit types"); + const svuint8_t v8 = BitCast(d8, v); + return BitCast(d, CombineShiftRightBytes<4>(d8, v8, v8)); +} + +// ------------------------------ Shuffle1032 +template <class V> +HWY_API V Shuffle1032(const V v) { + const DFromV<V> d; + const Repartition<uint8_t, decltype(d)> d8; + static_assert(sizeof(TFromD<decltype(d)>) == 4, "Defined for 32-bit types"); + const svuint8_t v8 = BitCast(d8, v); + return BitCast(d, CombineShiftRightBytes<8>(d8, v8, v8)); +} + +// ------------------------------ Shuffle01 +template <class V> +HWY_API V Shuffle01(const V v) { + const DFromV<V> d; + const Repartition<uint8_t, decltype(d)> d8; + static_assert(sizeof(TFromD<decltype(d)>) == 8, "Defined for 64-bit types"); + const svuint8_t v8 = BitCast(d8, v); + return BitCast(d, CombineShiftRightBytes<8>(d8, v8, v8)); +} + +// ------------------------------ Shuffle0123 +template <class V> +HWY_API V Shuffle0123(const V v) { + return Shuffle2301(Shuffle1032(v)); +} + +// ------------------------------ TableLookupBytes + +template <class V, class VI> +HWY_API VI TableLookupBytes(const V v, const VI idx) { + const DFromV<VI> d; + const Repartition<uint8_t, decltype(d)> du8; + const auto offsets128 = detail::OffsetsOf128BitBlocks(du8, Iota(du8, 0)); + const auto idx8 = Add(BitCast(du8, idx), offsets128); + return BitCast(d, TableLookupLanes(BitCast(du8, v), idx8)); +} + +template <class V, class VI> +HWY_API VI TableLookupBytesOr0(const V v, const VI idx) { + const DFromV<VI> d; + // Mask size must match vector type, so cast everything to this type. + const Repartition<int8_t, decltype(d)> di8; + + auto idx8 = BitCast(di8, idx); + const auto msb = Lt(idx8, Zero(di8)); +// Prevent overflow in table lookups (unnecessary if native) +#if defined(HWY_EMULATE_SVE) + idx8 = IfThenZeroElse(msb, idx8); +#endif + + const auto lookup = TableLookupBytes(BitCast(di8, v), idx8); + return BitCast(d, IfThenZeroElse(msb, lookup)); +} + +// ------------------------------ Broadcast + +template <int kLane, class V> +HWY_API V Broadcast(const V v) { + const DFromV<V> d; + const RebindToUnsigned<decltype(d)> du; + constexpr size_t kLanesPerBlock = detail::LanesPerBlock(du); + static_assert(0 <= kLane && kLane < kLanesPerBlock, "Invalid lane"); + auto idx = detail::OffsetsOf128BitBlocks(du, Iota(du, 0)); + if (kLane != 0) { + idx = detail::AddN(idx, kLane); + } + return TableLookupLanes(v, idx); +} + +// ------------------------------ ShiftLeftLanes + +template <size_t kLanes, class D, class V = VFromD<D>> +HWY_API V ShiftLeftLanes(D d, const V v) { + const RebindToSigned<decltype(d)> di; + const auto zero = Zero(d); + const auto shifted = detail::Splice(v, zero, FirstN(d, kLanes)); + // Match x86 semantics by zeroing lower lanes in 128-bit blocks + return IfThenElse(detail::FirstNPerBlock<kLanes>(d), zero, shifted); +} + +template <size_t kLanes, class V> +HWY_API V ShiftLeftLanes(const V v) { + return ShiftLeftLanes<kLanes>(DFromV<V>(), v); +} + +// ------------------------------ ShiftRightLanes +template <size_t kLanes, typename T, size_t N, class V = VFromD<Simd<T, N>>> +HWY_API V ShiftRightLanes(Simd<T, N> d, V v) { + const RebindToSigned<decltype(d)> di; + // For partial vectors, clear upper lanes so we shift in zeros. + if (N != HWY_LANES(T)) { + v = IfThenElseZero(detail::Mask(d), v); + } + + const auto shifted = detail::Ext<kLanes>(v, v); + // Match x86 semantics by zeroing upper lanes in 128-bit blocks + constexpr size_t kLanesPerBlock = detail::LanesPerBlock(d); + const svbool_t mask = detail::FirstNPerBlock<kLanesPerBlock - kLanes>(d); + return IfThenElseZero(mask, shifted); +} + +// ------------------------------ ShiftLeftBytes + +template <int kBytes, class D, class V = VFromD<D>> +HWY_API V ShiftLeftBytes(const D d, const V v) { + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, ShiftLeftLanes<kBytes>(BitCast(d8, v))); +} + +template <int kBytes, class V> +HWY_API V ShiftLeftBytes(const V v) { + return ShiftLeftBytes<kBytes>(DFromV<V>(), v); +} + +// ------------------------------ ShiftRightBytes +template <int kBytes, class D, class V = VFromD<D>> +HWY_API V ShiftRightBytes(const D d, const V v) { + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, ShiftRightLanes<kBytes>(d8, BitCast(d8, v))); +} + +// ------------------------------ InterleaveLower + +namespace detail { +HWY_SVE_FOREACH(HWY_SVE_RETV_ARGVV, ZipLower, zip1) +// Do not use zip2 to implement PromoteUpperTo or similar because vectors may be +// non-powers of two, so getting the actual "upper half" requires MaskUpperHalf. +} // namespace detail + +template <class D, class V> +HWY_API V InterleaveLower(D d, const V a, const V b) { + static_assert(IsSame<TFromD<D>, TFromV<V>>(), "D/V mismatch"); + // Move lower halves of blocks to lower half of vector. + const Repartition<uint64_t, decltype(d)> d64; + const auto a64 = BitCast(d64, a); + const auto b64 = BitCast(d64, b); + const auto a_blocks = detail::ConcatEven(a64, a64); // only lower half needed + const auto b_blocks = detail::ConcatEven(b64, b64); + + return detail::ZipLower(BitCast(d, a_blocks), BitCast(d, b_blocks)); +} + +template <class V> +HWY_API V InterleaveLower(const V a, const V b) { + return InterleaveLower(DFromV<V>(), a, b); +} + +// ------------------------------ InterleaveUpper + +// Full vector: guaranteed to have at least one block +template <typename T, class V = VFromD<Full<T>>> +HWY_API V InterleaveUpper(Simd<T, HWY_LANES(T)> d, const V a, const V b) { + // Move upper halves of blocks to lower half of vector. + const Repartition<uint64_t, decltype(d)> d64; + const auto a64 = BitCast(d64, a); + const auto b64 = BitCast(d64, b); + const auto a_blocks = detail::ConcatOdd(a64, a64); // only lower half needed + const auto b_blocks = detail::ConcatOdd(b64, b64); + return detail::ZipLower(BitCast(d, a_blocks), BitCast(d, b_blocks)); +} + +// Capped: less than one block +template <typename T, size_t N, HWY_IF_LE64(T, N), class V = VFromD<Simd<T, N>>> +HWY_API V InterleaveUpper(Simd<T, N> d, const V a, const V b) { + static_assert(IsSame<T, TFromV<V>>(), "D/V mismatch"); + const Half<decltype(d)> d2; + return InterleaveLower(d, UpperHalf(d2, a), UpperHalf(d2, b)); +} + +// Partial: need runtime check +template <typename T, size_t N, + hwy::EnableIf<(N < HWY_LANES(T) && N * sizeof(T) >= 16)>* = nullptr, + class V = VFromD<Simd<T, N>>> +HWY_API V InterleaveUpper(Simd<T, N> d, const V a, const V b) { + static_assert(IsSame<T, TFromV<V>>(), "D/V mismatch"); + // Less than one block: treat as capped + if (Lanes(d) * sizeof(T) < 16) { + const Half<decltype(d)> d2; + return InterleaveLower(d, UpperHalf(d2, a), UpperHalf(d2, b)); + } + return InterleaveUpper(Full<T>(), a, b); +} + +// ------------------------------ ZipLower + +template <class V, class DW = RepartitionToWide<DFromV<V>>> +HWY_API VFromD<DW> ZipLower(DW dw, V a, V b) { + const RepartitionToNarrow<DW> dn; + static_assert(IsSame<TFromD<decltype(dn)>, TFromV<V>>(), "D/V mismatch"); + return BitCast(dw, InterleaveLower(dn, a, b)); +} +template <class V, class D = DFromV<V>, class DW = RepartitionToWide<D>> +HWY_API VFromD<DW> ZipLower(const V a, const V b) { + return BitCast(DW(), InterleaveLower(D(), a, b)); +} + +// ------------------------------ ZipUpper +template <class V, class DW = RepartitionToWide<DFromV<V>>> +HWY_API VFromD<DW> ZipUpper(DW dw, V a, V b) { + const RepartitionToNarrow<DW> dn; + static_assert(IsSame<TFromD<decltype(dn)>, TFromV<V>>(), "D/V mismatch"); + return BitCast(dw, InterleaveUpper(dn, a, b)); +} + +// ================================================== REDUCE + +#define HWY_SVE_REDUCE(BASE, CHAR, BITS, NAME, OP) \ + template <size_t N> \ + HWY_API HWY_SVE_V(BASE, BITS) \ + NAME(HWY_SVE_D(BASE, BITS, N) d, HWY_SVE_V(BASE, BITS) v) { \ + return Set(d, sv##OP##_##CHAR##BITS(detail::Mask(d), v)); \ + } + +HWY_SVE_FOREACH(HWY_SVE_REDUCE, SumOfLanes, addv) +HWY_SVE_FOREACH_UI(HWY_SVE_REDUCE, MinOfLanes, minv) +HWY_SVE_FOREACH_UI(HWY_SVE_REDUCE, MaxOfLanes, maxv) +// NaN if all are +HWY_SVE_FOREACH_F(HWY_SVE_REDUCE, MinOfLanes, minnmv) +HWY_SVE_FOREACH_F(HWY_SVE_REDUCE, MaxOfLanes, maxnmv) + +#undef HWY_SVE_REDUCE + +// ================================================== Ops with dependencies + +// ------------------------------ PromoteTo bfloat16 (ZipLower) + +template <size_t N> +HWY_API svfloat32_t PromoteTo(Simd<float32_t, N> df32, const svuint16_t v) { + return BitCast(df32, detail::ZipLower(svdup_n_u16(0), v)); +} + +// ------------------------------ ReorderDemote2To (OddEven) + +template <size_t N> +HWY_API svuint16_t ReorderDemote2To(Simd<bfloat16_t, N> dbf16, svfloat32_t a, + svfloat32_t b) { + const RebindToUnsigned<decltype(dbf16)> du16; + const Repartition<uint32_t, decltype(dbf16)> du32; + const svuint32_t b_in_even = ShiftRight<16>(BitCast(du32, b)); + return BitCast(dbf16, OddEven(BitCast(du16, a), BitCast(du16, b_in_even))); +} + +// ------------------------------ ZeroIfNegative (Lt, IfThenElse) +template <class V> +HWY_API V ZeroIfNegative(const V v) { + const auto v0 = Zero(DFromV<V>()); + // We already have a zero constant, so avoid IfThenZeroElse. + return IfThenElse(Lt(v, v0), v0, v); +} + +// ------------------------------ BroadcastSignBit (ShiftRight) +template <class V> +HWY_API V BroadcastSignBit(const V v) { + return ShiftRight<sizeof(TFromV<V>) * 8 - 1>(v); +} + +// ------------------------------ AverageRound (ShiftRight) + +#if HWY_TARGET == HWY_SVE2 +HWY_SVE_FOREACH_U08(HWY_SVE_RETV_ARGPVV, AverageRound, rhadd) +HWY_SVE_FOREACH_U16(HWY_SVE_RETV_ARGPVV, AverageRound, rhadd) +#else +template <class V> +V AverageRound(const V a, const V b) { + return ShiftRight<1>(Add(Add(a, b), Set(DFromV<V>(), 1))); +} +#endif // HWY_TARGET == HWY_SVE2 + +// ------------------------------ LoadMaskBits (TestBit) + +// `p` points to at least 8 readable bytes, not all of which need be valid. +template <class D, HWY_IF_LANE_SIZE_D(D, 1)> +HWY_INLINE svbool_t LoadMaskBits(D d, const uint8_t* HWY_RESTRICT bits) { + const RebindToUnsigned<D> du; + const svuint8_t iota = Iota(du, 0); + + // Load correct number of bytes (bits/8) with 7 zeros after each. + const svuint8_t bytes = BitCast(du, svld1ub_u64(detail::PTrue(d), bits)); + // Replicate bytes 8x such that each byte contains the bit that governs it. + const svuint8_t rep8 = svtbl_u8(bytes, detail::AndNotN(7, iota)); + + // 1, 2, 4, 8, 16, 32, 64, 128, 1, 2 .. + const svuint8_t bit = Shl(Set(du, 1), detail::AndN(iota, 7)); + + return TestBit(rep8, bit); +} + +template <class D, HWY_IF_LANE_SIZE_D(D, 2)> +HWY_INLINE svbool_t LoadMaskBits(D /* tag */, + const uint8_t* HWY_RESTRICT bits) { + const RebindToUnsigned<D> du; + const Repartition<uint8_t, D> du8; + + // There may be up to 128 bits; avoid reading past the end. + const svuint8_t bytes = svld1(FirstN(du8, (Lanes(du) + 7) / 8), bits); + + // Replicate bytes 16x such that each lane contains the bit that governs it. + const svuint8_t rep16 = svtbl_u8(bytes, ShiftRight<4>(Iota(du8, 0))); + + // 1, 2, 4, 8, 16, 32, 64, 128, 1, 2 .. + const svuint16_t bit = Shl(Set(du, 1), detail::AndN(Iota(du, 0), 7)); + + return TestBit(BitCast(du, rep16), bit); +} + +template <class D, HWY_IF_LANE_SIZE_D(D, 4)> +HWY_INLINE svbool_t LoadMaskBits(D /* tag */, + const uint8_t* HWY_RESTRICT bits) { + const RebindToUnsigned<D> du; + const Repartition<uint8_t, D> du8; + + // Upper bound = 2048 bits / 32 bit = 64 bits; at least 8 bytes are readable, + // so we can skip computing the actual length (Lanes(du)+7)/8. + const svuint8_t bytes = svld1(FirstN(du8, 8), bits); + + // Replicate bytes 32x such that each lane contains the bit that governs it. + const svuint8_t rep32 = svtbl_u8(bytes, ShiftRight<5>(Iota(du8, 0))); + + // 1, 2, 4, 8, 16, 32, 64, 128, 1, 2 .. + const svuint32_t bit = Shl(Set(du, 1), detail::AndN(Iota(du, 0), 7)); + + return TestBit(BitCast(du, rep32), bit); +} + +template <class D, HWY_IF_LANE_SIZE_D(D, 8)> +HWY_INLINE svbool_t LoadMaskBits(D /* tag */, + const uint8_t* HWY_RESTRICT bits) { + const RebindToUnsigned<D> du; + + // Max 2048 bits = 32 lanes = 32 input bits; replicate those into each lane. + // The "at least 8 byte" guarantee in quick_reference ensures this is safe. + uint32_t mask_bits; + CopyBytes<4>(bits, &mask_bits); + const auto vbits = Set(du, mask_bits); + + // 2 ^ {0,1, .., 31}, will not have more lanes than that. + const svuint64_t bit = Shl(Set(du, 1), Iota(du, 0)); + + return TestBit(vbits, bit); +} + +// ------------------------------ StoreMaskBits + +namespace detail { + +// Returns mask ? 1 : 0 in BYTE lanes. +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 1)> +HWY_API svuint8_t BoolFromMask(Simd<T, N> d, svbool_t m) { + return svdup_n_u8_z(m, 1); +} +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 2)> +HWY_API svuint8_t BoolFromMask(Simd<T, N> d, svbool_t m) { + const Repartition<uint8_t, decltype(d)> d8; + const svuint8_t b16 = BitCast(d8, svdup_n_u16_z(m, 1)); + return detail::ConcatEven(b16, b16); // only lower half needed +} +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 4)> +HWY_API svuint8_t BoolFromMask(Simd<T, N> d, svbool_t m) { + return U8FromU32(svdup_n_u32_z(m, 1)); +} +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 8)> +HWY_API svuint8_t BoolFromMask(Simd<T, N> d, svbool_t m) { + const Repartition<uint32_t, decltype(d)> d32; + const svuint32_t b64 = BitCast(d32, svdup_n_u64_z(m, 1)); + return U8FromU32(detail::ConcatEven(b64, b64)); // only lower half needed +} + +} // namespace detail + +// `p` points to at least 8 writable bytes. +template <typename T, size_t N> +HWY_API size_t StoreMaskBits(Simd<T, N> d, svbool_t m, uint8_t* bits) { + const Repartition<uint8_t, decltype(d)> d8; + const Repartition<uint16_t, decltype(d)> d16; + const Repartition<uint32_t, decltype(d)> d32; + const Repartition<uint64_t, decltype(d)> d64; + auto x = detail::BoolFromMask(d, m); + // Compact bytes to bits. Could use SVE2 BDEP, but it's optional. + x = Or(x, BitCast(d8, ShiftRight<7>(BitCast(d16, x)))); + x = Or(x, BitCast(d8, ShiftRight<14>(BitCast(d32, x)))); + x = Or(x, BitCast(d8, ShiftRight<28>(BitCast(d64, x)))); + + const size_t num_bits = Lanes(d); + const size_t num_bytes = (num_bits + 8 - 1) / 8; // Round up, see below + + // Truncate to 8 bits and store. + svst1b_u64(FirstN(d64, num_bytes), bits, BitCast(d64, x)); + + // Non-full byte, need to clear the undefined upper bits. Can happen for + // capped/partial vectors or large T and small hardware vectors. + if (num_bits < 8) { + const int mask = (1 << num_bits) - 1; + bits[0] = static_cast<uint8_t>(bits[0] & mask); + } + // Else: we wrote full bytes because num_bits is a power of two >= 8. + + return num_bytes; +} + +// ------------------------------ CompressBits, CompressBitsStore (LoadMaskBits) + +template <class V> +HWY_INLINE V CompressBits(V v, const uint8_t* HWY_RESTRICT bits) { + return Compress(v, LoadMaskBits(DFromV<V>(), bits)); +} + +template <class D> +HWY_API size_t CompressBitsStore(VFromD<D> v, const uint8_t* HWY_RESTRICT bits, + D d, TFromD<D>* HWY_RESTRICT unaligned) { + return CompressStore(v, LoadMaskBits(d, bits), d, unaligned); +} + +// ------------------------------ MulEven (InterleaveEven) + +#if HWY_TARGET == HWY_SVE2 +namespace detail { +HWY_SVE_FOREACH_UI32(HWY_SVE_RETV_ARGPVV, MulEven, mullb) +} // namespace detail +#endif + +template <class V, class DW = RepartitionToWide<DFromV<V>>> +HWY_API VFromD<DW> MulEven(const V a, const V b) { +#if HWY_TARGET == HWY_SVE2 + return BitCast(DW(), detail::MulEven(a, b)); +#else + const auto lo = Mul(a, b); + const auto hi = detail::MulHigh(a, b); + return BitCast(DW(), detail::InterleaveEven(lo, hi)); +#endif +} + +HWY_API svuint64_t MulEven(const svuint64_t a, const svuint64_t b) { + const auto lo = Mul(a, b); + const auto hi = detail::MulHigh(a, b); + return detail::InterleaveEven(lo, hi); +} + +HWY_API svuint64_t MulOdd(const svuint64_t a, const svuint64_t b) { + const auto lo = Mul(a, b); + const auto hi = detail::MulHigh(a, b); + return detail::InterleaveOdd(lo, hi); +} + +// ------------------------------ ReorderWidenMulAccumulate (MulAdd, ZipLower) + +template <size_t N> +HWY_API svfloat32_t ReorderWidenMulAccumulate(Simd<float, N> df32, svuint16_t a, + svuint16_t b, + const svfloat32_t sum0, + svfloat32_t& sum1) { + // TODO(janwas): svbfmlalb_f32 if __ARM_FEATURE_SVE_BF16. + const Repartition<uint16_t, decltype(df32)> du16; + const RebindToUnsigned<decltype(df32)> du32; + const svuint16_t zero = Zero(du16); + const svuint32_t a0 = ZipLower(du32, zero, BitCast(du16, a)); + const svuint32_t a1 = ZipUpper(du32, zero, BitCast(du16, a)); + const svuint32_t b0 = ZipLower(du32, zero, BitCast(du16, b)); + const svuint32_t b1 = ZipUpper(du32, zero, BitCast(du16, b)); + sum1 = MulAdd(BitCast(df32, a1), BitCast(df32, b1), sum1); + return MulAdd(BitCast(df32, a0), BitCast(df32, b0), sum0); +} + +// ------------------------------ AESRound / CLMul + +#if defined(__ARM_FEATURE_SVE2_AES) + +// Per-target flag to prevent generic_ops-inl.h from defining AESRound. +#ifdef HWY_NATIVE_AES +#undef HWY_NATIVE_AES +#else +#define HWY_NATIVE_AES +#endif + +HWY_API svuint8_t AESRound(svuint8_t state, svuint8_t round_key) { + // NOTE: it is important that AESE and AESMC be consecutive instructions so + // they can be fused. AESE includes AddRoundKey, which is a different ordering + // than the AES-NI semantics we adopted, so XOR by 0 and later with the actual + // round key (the compiler will hopefully optimize this for multiple rounds). + const svuint8_t zero = svdup_n_u8(0); + return Xor(vaesmcq_u8(vaeseq_u8(state, zero), round_key)); +} + +HWY_API svuint64_t CLMulLower(const svuint64_t a, const svuint64_t b) { + return svpmullb_pair(a, b); +} + +HWY_API svuint64_t CLMulUpper(const svuint64_t a, const svuint64_t b) { + return svpmullt_pair(a, b); +} + +#endif // __ARM_FEATURE_SVE2_AES + +// ================================================== END MACROS +namespace detail { // for code folding +#undef HWY_IF_FLOAT_V +#undef HWY_IF_LANE_SIZE_V +#undef HWY_IF_SIGNED_V +#undef HWY_IF_UNSIGNED_V +#undef HWY_SVE_D +#undef HWY_SVE_FOREACH +#undef HWY_SVE_FOREACH_F +#undef HWY_SVE_FOREACH_F16 +#undef HWY_SVE_FOREACH_F32 +#undef HWY_SVE_FOREACH_F64 +#undef HWY_SVE_FOREACH_I +#undef HWY_SVE_FOREACH_I08 +#undef HWY_SVE_FOREACH_I16 +#undef HWY_SVE_FOREACH_I32 +#undef HWY_SVE_FOREACH_I64 +#undef HWY_SVE_FOREACH_IF +#undef HWY_SVE_FOREACH_U +#undef HWY_SVE_FOREACH_U08 +#undef HWY_SVE_FOREACH_U16 +#undef HWY_SVE_FOREACH_U32 +#undef HWY_SVE_FOREACH_U64 +#undef HWY_SVE_FOREACH_UI +#undef HWY_SVE_FOREACH_UI08 +#undef HWY_SVE_FOREACH_UI16 +#undef HWY_SVE_FOREACH_UI32 +#undef HWY_SVE_FOREACH_UI64 +#undef HWY_SVE_FOREACH_UIF3264 +#undef HWY_SVE_PTRUE +#undef HWY_SVE_RETV_ARGD +#undef HWY_SVE_RETV_ARGPV +#undef HWY_SVE_RETV_ARGPVN +#undef HWY_SVE_RETV_ARGPVV +#undef HWY_SVE_RETV_ARGV +#undef HWY_SVE_RETV_ARGVN +#undef HWY_SVE_RETV_ARGVV +#undef HWY_SVE_T +#undef HWY_SVE_V + +} // namespace detail +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); diff --git a/third_party/highway/hwy/ops/generic_ops-inl.h b/third_party/highway/hwy/ops/generic_ops-inl.h new file mode 100644 index 0000000..35cec12 --- /dev/null +++ b/third_party/highway/hwy/ops/generic_ops-inl.h @@ -0,0 +1,324 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Target-independent types/functions defined after target-specific ops. + +// Relies on the external include guard in highway.h. +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +// The lane type of a vector type, e.g. float for Vec<Simd<float, 4>>. +template <class V> +using LaneType = decltype(GetLane(V())); + +// Vector type, e.g. Vec128<float> for Simd<float, 4>. Useful as the return type +// of functions that do not take a vector argument, or as an argument type if +// the function only has a template argument for D, or for explicit type names +// instead of auto. This may be a built-in type. +template <class D> +using Vec = decltype(Zero(D())); + +// Mask type. Useful as the return type of functions that do not take a mask +// argument, or as an argument type if the function only has a template argument +// for D, or for explicit type names instead of auto. +template <class D> +using Mask = decltype(MaskFromVec(Zero(D()))); + +// Returns the closest value to v within [lo, hi]. +template <class V> +HWY_API V Clamp(const V v, const V lo, const V hi) { + return Min(Max(lo, v), hi); +} + +// CombineShiftRightBytes (and -Lanes) are not available for the scalar target, +// and RVV has its own implementation of -Lanes. +#if HWY_TARGET != HWY_SCALAR && HWY_TARGET != HWY_RVV + +template <size_t kLanes, class D, class V = VFromD<D>> +HWY_API V CombineShiftRightLanes(D d, const V hi, const V lo) { + constexpr size_t kBytes = kLanes * sizeof(LaneType<V>); + static_assert(kBytes < 16, "Shift count is per-block"); + return CombineShiftRightBytes<kBytes>(d, hi, lo); +} + +// DEPRECATED +template <size_t kLanes, class V> +HWY_API V CombineShiftRightLanes(const V hi, const V lo) { + return CombineShiftRightLanes<kLanes>(DFromV<V>(), hi, lo); +} + +#endif + +// Returns lanes with the most significant bit set and all other bits zero. +template <class D> +HWY_API Vec<D> SignBit(D d) { + using Unsigned = MakeUnsigned<TFromD<D>>; + const Unsigned bit = Unsigned(1) << (sizeof(Unsigned) * 8 - 1); + return BitCast(d, Set(Rebind<Unsigned, D>(), bit)); +} + +// Returns quiet NaN. +template <class D> +HWY_API Vec<D> NaN(D d) { + const RebindToSigned<D> di; + // LimitsMax sets all exponent and mantissa bits to 1. The exponent plus + // mantissa MSB (to indicate quiet) would be sufficient. + return BitCast(d, Set(di, LimitsMax<TFromD<decltype(di)>>())); +} + +// ------------------------------ AESRound + +// Cannot implement on scalar: need at least 16 bytes for TableLookupBytes. +#if HWY_TARGET != HWY_SCALAR + +// Define for white-box testing, even if native instructions are available. +namespace detail { + +// Constant-time: computes inverse in GF(2^4) based on "Accelerating AES with +// Vector Permute Instructions" and the accompanying assembly language +// implementation: https://crypto.stanford.edu/vpaes/vpaes.tgz. See also Botan: +// https://botan.randombit.net/doxygen/aes__vperm_8cpp_source.html . +// +// A brute-force 256 byte table lookup can also be made constant-time, and +// possibly competitive on NEON, but this is more performance-portable +// especially for x86 and large vectors. +template <class V> // u8 +HWY_INLINE V SubBytes(V state) { + const DFromV<V> du; + const auto mask = Set(du, 0xF); + + // Change polynomial basis to GF(2^4) + { + alignas(16) static constexpr uint8_t basisL[16] = { + 0x00, 0x70, 0x2A, 0x5A, 0x98, 0xE8, 0xB2, 0xC2, + 0x08, 0x78, 0x22, 0x52, 0x90, 0xE0, 0xBA, 0xCA}; + alignas(16) static constexpr uint8_t basisU[16] = { + 0x00, 0x4D, 0x7C, 0x31, 0x7D, 0x30, 0x01, 0x4C, + 0x81, 0xCC, 0xFD, 0xB0, 0xFC, 0xB1, 0x80, 0xCD}; + const auto sL = And(state, mask); + const auto sU = ShiftRight<4>(state); // byte shift => upper bits are zero + const auto gf4L = TableLookupBytes(LoadDup128(du, basisL), sL); + const auto gf4U = TableLookupBytes(LoadDup128(du, basisU), sU); + state = Xor(gf4L, gf4U); + } + + // Inversion in GF(2^4). Elements 0 represent "infinity" (division by 0) and + // cause TableLookupBytesOr0 to return 0. + alignas(16) static constexpr uint8_t kZetaInv[16] = { + 0x80, 7, 11, 15, 6, 10, 4, 1, 9, 8, 5, 2, 12, 14, 13, 3}; + alignas(16) static constexpr uint8_t kInv[16] = { + 0x80, 1, 8, 13, 15, 6, 5, 14, 2, 12, 11, 10, 9, 3, 7, 4}; + const auto tbl = LoadDup128(du, kInv); + const auto sL = And(state, mask); // L=low nibble, U=upper + const auto sU = ShiftRight<4>(state); // byte shift => upper bits are zero + const auto sX = Xor(sU, sL); + const auto invL = TableLookupBytes(LoadDup128(du, kZetaInv), sL); + const auto invU = TableLookupBytes(tbl, sU); + const auto invX = TableLookupBytes(tbl, sX); + const auto outL = Xor(sX, TableLookupBytesOr0(tbl, Xor(invL, invU))); + const auto outU = Xor(sU, TableLookupBytesOr0(tbl, Xor(invL, invX))); + + // Linear skew (cannot bake 0x63 bias into the table because out* indices + // may have the infinity flag set). + alignas(16) static constexpr uint8_t kAffineL[16] = { + 0x00, 0xC7, 0xBD, 0x6F, 0x17, 0x6D, 0xD2, 0xD0, + 0x78, 0xA8, 0x02, 0xC5, 0x7A, 0xBF, 0xAA, 0x15}; + alignas(16) static constexpr uint8_t kAffineU[16] = { + 0x00, 0x6A, 0xBB, 0x5F, 0xA5, 0x74, 0xE4, 0xCF, + 0xFA, 0x35, 0x2B, 0x41, 0xD1, 0x90, 0x1E, 0x8E}; + const auto affL = TableLookupBytesOr0(LoadDup128(du, kAffineL), outL); + const auto affU = TableLookupBytesOr0(LoadDup128(du, kAffineU), outU); + return Xor(Xor(affL, affU), Set(du, 0x63)); +} + +} // namespace detail + +#endif // HWY_TARGET != HWY_SCALAR + +// "Include guard": skip if native AES instructions are available. +#if (defined(HWY_NATIVE_AES) == defined(HWY_TARGET_TOGGLE)) +#ifdef HWY_NATIVE_AES +#undef HWY_NATIVE_AES +#else +#define HWY_NATIVE_AES +#endif + +// (Must come after HWY_TARGET_TOGGLE, else we don't reset it for scalar) +#if HWY_TARGET != HWY_SCALAR + +namespace detail { + +template <class V> // u8 +HWY_API V ShiftRows(const V state) { + const DFromV<V> du; + alignas(16) static constexpr uint8_t kShiftRow[16] = { + 0, 5, 10, 15, // transposed: state is column major + 4, 9, 14, 3, // + 8, 13, 2, 7, // + 12, 1, 6, 11}; + const auto shift_row = LoadDup128(du, kShiftRow); + return TableLookupBytes(state, shift_row); +} + +template <class V> // u8 +HWY_API V MixColumns(const V state) { + const DFromV<V> du; + // For each column, the rows are the sum of GF(2^8) matrix multiplication by: + // 2 3 1 1 // Let s := state*1, d := state*2, t := state*3. + // 1 2 3 1 // d are on diagonal, no permutation needed. + // 1 1 2 3 // t1230 indicates column indices of threes for the 4 rows. + // 3 1 1 2 // We also need to compute s2301 and s3012 (=1230 o 2301). + alignas(16) static constexpr uint8_t k2301[16] = { + 2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13}; + alignas(16) static constexpr uint8_t k1230[16] = { + 1, 2, 3, 0, 5, 6, 7, 4, 9, 10, 11, 8, 13, 14, 15, 12}; + const RebindToSigned<decltype(du)> di; // can only do signed comparisons + const auto msb = Lt(BitCast(di, state), Zero(di)); + const auto overflow = BitCast(du, IfThenElseZero(msb, Set(di, 0x1B))); + const auto d = Xor(Add(state, state), overflow); // = state*2 in GF(2^8). + const auto s2301 = TableLookupBytes(state, LoadDup128(du, k2301)); + const auto d_s2301 = Xor(d, s2301); + const auto t_s2301 = Xor(state, d_s2301); // t(s*3) = XOR-sum {s, d(s*2)} + const auto t1230_s3012 = TableLookupBytes(t_s2301, LoadDup128(du, k1230)); + return Xor(d_s2301, t1230_s3012); // XOR-sum of 4 terms +} + +} // namespace detail + +template <class V> // u8 +HWY_API V AESRound(V state, const V round_key) { + // Intel docs swap the first two steps, but it does not matter because + // ShiftRows is a permutation and SubBytes is independent of lane index. + state = detail::SubBytes(state); + state = detail::ShiftRows(state); + state = detail::MixColumns(state); + state = Xor(state, round_key); // AddRoundKey + return state; +} + +// Constant-time implementation inspired by +// https://www.bearssl.org/constanttime.html, but about half the cost because we +// use 64x64 multiplies and 128-bit XORs. +template <class V> +HWY_API V CLMulLower(V a, V b) { + const DFromV<V> d; + static_assert(IsSame<TFromD<decltype(d)>, uint64_t>(), "V must be u64"); + const auto k1 = Set(d, 0x1111111111111111ULL); + const auto k2 = Set(d, 0x2222222222222222ULL); + const auto k4 = Set(d, 0x4444444444444444ULL); + const auto k8 = Set(d, 0x8888888888888888ULL); + const auto a0 = And(a, k1); + const auto a1 = And(a, k2); + const auto a2 = And(a, k4); + const auto a3 = And(a, k8); + const auto b0 = And(b, k1); + const auto b1 = And(b, k2); + const auto b2 = And(b, k4); + const auto b3 = And(b, k8); + + auto m0 = Xor(MulEven(a0, b0), MulEven(a1, b3)); + auto m1 = Xor(MulEven(a0, b1), MulEven(a1, b0)); + auto m2 = Xor(MulEven(a0, b2), MulEven(a1, b1)); + auto m3 = Xor(MulEven(a0, b3), MulEven(a1, b2)); + m0 = Xor(m0, Xor(MulEven(a2, b2), MulEven(a3, b1))); + m1 = Xor(m1, Xor(MulEven(a2, b3), MulEven(a3, b2))); + m2 = Xor(m2, Xor(MulEven(a2, b0), MulEven(a3, b3))); + m3 = Xor(m3, Xor(MulEven(a2, b1), MulEven(a3, b0))); + return Or(Or(And(m0, k1), And(m1, k2)), Or(And(m2, k4), And(m3, k8))); +} + +template <class V> +HWY_API V CLMulUpper(V a, V b) { + const DFromV<V> d; + static_assert(IsSame<TFromD<decltype(d)>, uint64_t>(), "V must be u64"); + const auto k1 = Set(d, 0x1111111111111111ULL); + const auto k2 = Set(d, 0x2222222222222222ULL); + const auto k4 = Set(d, 0x4444444444444444ULL); + const auto k8 = Set(d, 0x8888888888888888ULL); + const auto a0 = And(a, k1); + const auto a1 = And(a, k2); + const auto a2 = And(a, k4); + const auto a3 = And(a, k8); + const auto b0 = And(b, k1); + const auto b1 = And(b, k2); + const auto b2 = And(b, k4); + const auto b3 = And(b, k8); + + auto m0 = Xor(MulOdd(a0, b0), MulOdd(a1, b3)); + auto m1 = Xor(MulOdd(a0, b1), MulOdd(a1, b0)); + auto m2 = Xor(MulOdd(a0, b2), MulOdd(a1, b1)); + auto m3 = Xor(MulOdd(a0, b3), MulOdd(a1, b2)); + m0 = Xor(m0, Xor(MulOdd(a2, b2), MulOdd(a3, b1))); + m1 = Xor(m1, Xor(MulOdd(a2, b3), MulOdd(a3, b2))); + m2 = Xor(m2, Xor(MulOdd(a2, b0), MulOdd(a3, b3))); + m3 = Xor(m3, Xor(MulOdd(a2, b1), MulOdd(a3, b0))); + return Or(Or(And(m0, k1), And(m1, k2)), Or(And(m2, k4), And(m3, k8))); +} + +#endif // HWY_NATIVE_AES +#endif // HWY_TARGET != HWY_SCALAR + +// "Include guard": skip if native POPCNT-related instructions are available. +#if (defined(HWY_NATIVE_POPCNT) == defined(HWY_TARGET_TOGGLE)) +#ifdef HWY_NATIVE_POPCNT +#undef HWY_NATIVE_POPCNT +#else +#define HWY_NATIVE_POPCNT +#endif + +template <typename V, HWY_IF_LANES_ARE(uint8_t, V)> +HWY_API V PopulationCount(V v) { + constexpr DFromV<V> d; + HWY_ALIGN constexpr uint8_t kLookup[16] = { + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, + }; + auto lo = And(v, Set(d, 0xF)); + auto hi = ShiftRight<4>(v); + auto lookup = LoadDup128(Simd<uint8_t, HWY_MAX(16, MaxLanes(d))>(), kLookup); + return Add(TableLookupBytes(lookup, hi), TableLookupBytes(lookup, lo)); +} + +template <typename V, HWY_IF_LANES_ARE(uint16_t, V)> +HWY_API V PopulationCount(V v) { + const DFromV<V> d; + Repartition<uint8_t, decltype(d)> d8; + auto vals = BitCast(d, PopulationCount(BitCast(d8, v))); + return Add(ShiftRight<8>(vals), And(vals, Set(d, 0xFF))); +} + +template <typename V, HWY_IF_LANES_ARE(uint32_t, V)> +HWY_API V PopulationCount(V v) { + const DFromV<V> d; + Repartition<uint16_t, decltype(d)> d16; + auto vals = BitCast(d, PopulationCount(BitCast(d16, v))); + return Add(ShiftRight<16>(vals), And(vals, Set(d, 0xFF))); +} + +#if HWY_CAP_INTEGER64 +template <typename V, HWY_IF_LANES_ARE(uint64_t, V)> +HWY_API V PopulationCount(V v) { + const DFromV<V> d; + Repartition<uint32_t, decltype(d)> d32; + auto vals = BitCast(d, PopulationCount(BitCast(d32, v))); + return Add(ShiftRight<32>(vals), And(vals, Set(d, 0xFF))); +} +#endif + +#endif // HWY_NATIVE_POPCNT + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); diff --git a/third_party/highway/hwy/ops/rvv-inl.h b/third_party/highway/hwy/ops/rvv-inl.h new file mode 100644 index 0000000..14a0306 --- /dev/null +++ b/third_party/highway/hwy/ops/rvv-inl.h @@ -0,0 +1,2183 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// RISC-V V vectors (length not known at compile time). +// External include guard in highway.h - see comment there. + +#include <riscv_vector.h> +#include <stddef.h> +#include <stdint.h> + +#include "hwy/base.h" +#include "hwy/ops/shared-inl.h" + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +template <class V> +struct DFromV_t {}; // specialized in macros +template <class V> +using DFromV = typename DFromV_t<RemoveConst<V>>::type; + +template <class V> +using TFromV = TFromD<DFromV<V>>; + +template <typename T, size_t N> +HWY_INLINE constexpr size_t MLenFromD(Simd<T, N> /* tag */) { + // Returns divisor = type bits / LMUL + return sizeof(T) * 8 / (N / HWY_LANES(T)); +} + +// kShift = log2 of multiplier: 0 for m1, 1 for m2, -2 for mf4 +template <typename T, int kShift = 0> +using Full = Simd<T, (kShift < 0) ? (HWY_LANES(T) >> (-kShift)) + : (HWY_LANES(T) << kShift)>; + +// ================================================== MACROS + +// Generate specializations and function definitions using X macros. Although +// harder to read and debug, writing everything manually is too bulky. + +namespace detail { // for code folding + +// For all mask sizes MLEN: (1/Nth of a register, one bit per lane) +// The first two arguments are SEW and SHIFT such that SEW >> SHIFT = MLEN. +#define HWY_RVV_FOREACH_B(X_MACRO, NAME, OP) \ + X_MACRO(64, 0, 64, NAME, OP) \ + X_MACRO(32, 0, 32, NAME, OP) \ + X_MACRO(16, 0, 16, NAME, OP) \ + X_MACRO(8, 0, 8, NAME, OP) \ + X_MACRO(8, 1, 4, NAME, OP) \ + X_MACRO(8, 2, 2, NAME, OP) \ + X_MACRO(8, 3, 1, NAME, OP) + +// For given SEW, iterate over all LMUL. Precompute SEW/LMUL => MLEN because we +// need to token-paste the result. For the same reason, we also pass the +// twice-as-long and half-as-long LMUL suffixes as arguments. +// TODO(janwas): add fractional LMUL +#define HWY_RVV_FOREACH_08(X_MACRO, BASE, CHAR, NAME, OP) \ + X_MACRO(BASE, CHAR, 8, m1, m2, mf2, /*kShift=*/0, /*MLEN=*/8, NAME, OP) \ + X_MACRO(BASE, CHAR, 8, m2, m4, m1, /*kShift=*/1, /*MLEN=*/4, NAME, OP) \ + X_MACRO(BASE, CHAR, 8, m4, m8, m2, /*kShift=*/2, /*MLEN=*/2, NAME, OP) \ + X_MACRO(BASE, CHAR, 8, m8, __, m4, /*kShift=*/3, /*MLEN=*/1, NAME, OP) + +#define HWY_RVV_FOREACH_16(X_MACRO, BASE, CHAR, NAME, OP) \ + X_MACRO(BASE, CHAR, 16, m1, m2, mf2, /*kShift=*/0, /*MLEN=*/16, NAME, OP) \ + X_MACRO(BASE, CHAR, 16, m2, m4, m1, /*kShift=*/1, /*MLEN=*/8, NAME, OP) \ + X_MACRO(BASE, CHAR, 16, m4, m8, m2, /*kShift=*/2, /*MLEN=*/4, NAME, OP) \ + X_MACRO(BASE, CHAR, 16, m8, __, m4, /*kShift=*/3, /*MLEN=*/2, NAME, OP) + +#define HWY_RVV_FOREACH_32(X_MACRO, BASE, CHAR, NAME, OP) \ + X_MACRO(BASE, CHAR, 32, m1, m2, mf2, /*kShift=*/0, /*MLEN=*/32, NAME, OP) \ + X_MACRO(BASE, CHAR, 32, m2, m4, m1, /*kShift=*/1, /*MLEN=*/16, NAME, OP) \ + X_MACRO(BASE, CHAR, 32, m4, m8, m2, /*kShift=*/2, /*MLEN=*/8, NAME, OP) \ + X_MACRO(BASE, CHAR, 32, m8, __, m4, /*kShift=*/3, /*MLEN=*/4, NAME, OP) + +#define HWY_RVV_FOREACH_64(X_MACRO, BASE, CHAR, NAME, OP) \ + X_MACRO(BASE, CHAR, 64, m1, m2, mf2, /*kShift=*/0, /*MLEN=*/64, NAME, OP) \ + X_MACRO(BASE, CHAR, 64, m2, m4, m1, /*kShift=*/1, /*MLEN=*/32, NAME, OP) \ + X_MACRO(BASE, CHAR, 64, m4, m8, m2, /*kShift=*/2, /*MLEN=*/16, NAME, OP) \ + X_MACRO(BASE, CHAR, 64, m8, __, m4, /*kShift=*/3, /*MLEN=*/8, NAME, OP) + +// SEW for unsigned: +#define HWY_RVV_FOREACH_U08(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_08(X_MACRO, uint, u, NAME, OP) +#define HWY_RVV_FOREACH_U16(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_16(X_MACRO, uint, u, NAME, OP) +#define HWY_RVV_FOREACH_U32(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_32(X_MACRO, uint, u, NAME, OP) +#define HWY_RVV_FOREACH_U64(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_64(X_MACRO, uint, u, NAME, OP) + +// SEW for signed: +#define HWY_RVV_FOREACH_I08(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_08(X_MACRO, int, i, NAME, OP) +#define HWY_RVV_FOREACH_I16(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_16(X_MACRO, int, i, NAME, OP) +#define HWY_RVV_FOREACH_I32(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_32(X_MACRO, int, i, NAME, OP) +#define HWY_RVV_FOREACH_I64(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_64(X_MACRO, int, i, NAME, OP) + +// SEW for float: +#define HWY_RVV_FOREACH_F16(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_16(X_MACRO, float, f, NAME, OP) +#define HWY_RVV_FOREACH_F32(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_32(X_MACRO, float, f, NAME, OP) +#define HWY_RVV_FOREACH_F64(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_64(X_MACRO, float, f, NAME, OP) + +// For all combinations of SEW: +#define HWY_RVV_FOREACH_U(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_U08(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_U16(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_U32(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_U64(X_MACRO, NAME, OP) + +#define HWY_RVV_FOREACH_I(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_I08(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_I16(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_I32(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_I64(X_MACRO, NAME, OP) + +#if HWY_CAP_FLOAT16 +#define HWY_RVV_FOREACH_F(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_F16(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_F32(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_F64(X_MACRO, NAME, OP) +#else +#define HWY_RVV_FOREACH_F(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_F32(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_F64(X_MACRO, NAME, OP) +#endif + +// Commonly used type categories for a given SEW: +#define HWY_RVV_FOREACH_UI16(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_U16(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_I16(X_MACRO, NAME, OP) + +#define HWY_RVV_FOREACH_UI32(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_U32(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_I32(X_MACRO, NAME, OP) + +#define HWY_RVV_FOREACH_UI64(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_U64(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_I64(X_MACRO, NAME, OP) + +// Commonly used type categories: +#define HWY_RVV_FOREACH_UI(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_U(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_I(X_MACRO, NAME, OP) + +#define HWY_RVV_FOREACH(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_U(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_I(X_MACRO, NAME, OP) \ + HWY_RVV_FOREACH_F(X_MACRO, NAME, OP) + +// Assemble types for use in x-macros +#define HWY_RVV_T(BASE, SEW) BASE##SEW##_t +#define HWY_RVV_D(CHAR, SEW, LMUL) D##CHAR##SEW##LMUL +#define HWY_RVV_V(BASE, SEW, LMUL) v##BASE##SEW##LMUL##_t +#define HWY_RVV_M(MLEN) vbool##MLEN##_t + +} // namespace detail + +// TODO(janwas): remove typedefs and only use HWY_RVV_V etc. directly + +// Until we have full intrinsic support for fractional LMUL, mixed-precision +// code can use LMUL 1..8 (adequate unless they need many registers). +#define HWY_SPECIALIZE(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, OP) \ + using HWY_RVV_D(CHAR, SEW, LMUL) = Full<HWY_RVV_T(BASE, SEW), SHIFT>; \ + using V##CHAR##SEW##LMUL = HWY_RVV_V(BASE, SEW, LMUL); \ + template <> \ + struct DFromV_t<HWY_RVV_V(BASE, SEW, LMUL)> { \ + using Lane = HWY_RVV_T(BASE, SEW); \ + using type = Full<Lane, SHIFT>; \ + }; +#if HWY_CAP_FLOAT16 +using Vf16m1 = vfloat16m1_t; +using Vf16m2 = vfloat16m2_t; +using Vf16m4 = vfloat16m4_t; +using Vf16m8 = vfloat16m8_t; +using Df16m1 = Full<float16_t, 0>; +using Df16m2 = Full<float16_t, 1>; +using Df16m4 = Full<float16_t, 2>; +using Df16m8 = Full<float16_t, 3>; +#endif + +HWY_RVV_FOREACH(HWY_SPECIALIZE, _, _) +#undef HWY_SPECIALIZE + +// ------------------------------ Lanes + +// WARNING: we want to query VLMAX/sizeof(T), but this actually changes VL! +// vlenb is not exposed through intrinsics and vreadvl is not VLMAX. +#define HWY_RVV_LANES(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, OP) \ + HWY_API size_t NAME(HWY_RVV_D(CHAR, SEW, LMUL) /* d */) { \ + return v##OP##SEW##LMUL(); \ + } + +HWY_RVV_FOREACH(HWY_RVV_LANES, Lanes, setvlmax_e) +#undef HWY_RVV_LANES + +// Capped +template <typename T, size_t N, + hwy::EnableIf<(N < HWY_LANES(T) / 8)>* = nullptr> +HWY_API size_t Lanes(Simd<T, N> /* tag*/) { + return HWY_MIN(N, Lanes(Full<T>())); +} + +template <size_t N> +HWY_API size_t Lanes(Simd<bfloat16_t, N> /* tag*/) { + return Lanes(Simd<uint16_t, N>()); +} + +// ------------------------------ Common x-macros + +// Last argument to most intrinsics. Use when the op has no d arg of its own. +#define HWY_RVV_AVL(SEW, SHIFT) Lanes(Full<HWY_RVV_T(uint, SEW), SHIFT>()) + +// vector = f(vector), e.g. Not +#define HWY_RVV_RETV_ARGV(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, \ + OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) NAME(HWY_RVV_V(BASE, SEW, LMUL) v) { \ + return v##OP##_v_##CHAR##SEW##LMUL(v, HWY_RVV_AVL(SEW, SHIFT)); \ + } + +// vector = f(vector, scalar), e.g. detail::AddS +#define HWY_RVV_RETV_ARGVS(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, \ + OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME(HWY_RVV_V(BASE, SEW, LMUL) a, HWY_RVV_T(BASE, SEW) b) { \ + return v##OP##_##CHAR##SEW##LMUL(a, b, HWY_RVV_AVL(SEW, SHIFT)); \ + } + +// vector = f(vector, vector), e.g. Add +#define HWY_RVV_RETV_ARGVV(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, \ + OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME(HWY_RVV_V(BASE, SEW, LMUL) a, HWY_RVV_V(BASE, SEW, LMUL) b) { \ + return v##OP##_vv_##CHAR##SEW##LMUL(a, b, HWY_RVV_AVL(SEW, SHIFT)); \ + } + +// ================================================== INIT + +// ------------------------------ Set + +#define HWY_RVV_SET(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME(HWY_RVV_D(CHAR, SEW, LMUL) d, HWY_RVV_T(BASE, SEW) arg) { \ + return v##OP##_##CHAR##SEW##LMUL(arg, Lanes(d)); \ + } + +HWY_RVV_FOREACH_UI(HWY_RVV_SET, Set, mv_v_x) +HWY_RVV_FOREACH_F(HWY_RVV_SET, Set, fmv_v_f) +#undef HWY_RVV_SET + +// Treat bfloat16_t as uint16_t (using the previously defined Set overloads); +// required for Zero and VFromD. +template <size_t N> +decltype(Set(Simd<uint16_t, N>(), 0)) Set(Simd<bfloat16_t, N> d, + bfloat16_t arg) { + return Set(RebindToUnsigned<decltype(d)>(), arg.bits); +} + +// Capped vectors +template <typename T, size_t N, + hwy::EnableIf<(N < HWY_LANES(T) / 8)>* = nullptr> +HWY_API decltype(Set(Full<T>(), T{0})) Set(Simd<T, N> /*tag*/, T arg) { + return Set(Full<T>(), arg); +} + +template <class D> +using VFromD = decltype(Set(D(), TFromD<D>())); + +// ------------------------------ Zero + +template <typename T, size_t N> +HWY_API VFromD<Simd<T, N>> Zero(Simd<T, N> d) { + return Set(d, T(0)); +} + +// ------------------------------ Undefined + +// RVV vundefined is 'poisoned' such that even XORing a _variable_ initialized +// by it gives unpredictable results. It should only be used for maskoff, so +// keep it internal. For the Highway op, just use Zero (single instruction). +namespace detail { +#define HWY_RVV_UNDEFINED(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, \ + OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME(HWY_RVV_D(CHAR, SEW, LMUL) /* tag */) { \ + return v##OP##_##CHAR##SEW##LMUL(); /* no AVL */ \ + } + +HWY_RVV_FOREACH(HWY_RVV_UNDEFINED, Undefined, undefined) +#undef HWY_RVV_UNDEFINED +} // namespace detail + +template <class D> +HWY_API VFromD<D> Undefined(D d) { + return Zero(d); +} + +// ------------------------------ BitCast + +namespace detail { + +// There is no reinterpret from u8 <-> u8, so just return. +#define HWY_RVV_CAST_U8(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, \ + OP) \ + HWY_API vuint8##LMUL##_t BitCastToByte(vuint8##LMUL##_t v) { return v; } \ + HWY_API vuint8##LMUL##_t BitCastFromByte(HWY_RVV_D(CHAR, SEW, LMUL) /* d */, \ + vuint8##LMUL##_t v) { \ + return v; \ + } + +// For i8, need a single reinterpret (HWY_RVV_CAST_IF does two). +#define HWY_RVV_CAST_I8(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, \ + OP) \ + HWY_API vuint8##LMUL##_t BitCastToByte(vint8##LMUL##_t v) { \ + return vreinterpret_v_i8##LMUL##_u8##LMUL(v); \ + } \ + HWY_API vint8##LMUL##_t BitCastFromByte(HWY_RVV_D(CHAR, SEW, LMUL) /* d */, \ + vuint8##LMUL##_t v) { \ + return vreinterpret_v_u8##LMUL##_i8##LMUL(v); \ + } + +// Separate u/i because clang only provides signed <-> unsigned reinterpret for +// the same SEW. +#define HWY_RVV_CAST_U(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, OP) \ + HWY_API vuint8##LMUL##_t BitCastToByte(HWY_RVV_V(BASE, SEW, LMUL) v) { \ + return v##OP##_v_##CHAR##SEW##LMUL##_u8##LMUL(v); \ + } \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) BitCastFromByte( \ + HWY_RVV_D(CHAR, SEW, LMUL) /* d */, vuint8##LMUL##_t v) { \ + return v##OP##_v_u8##LMUL##_##CHAR##SEW##LMUL(v); \ + } + +// Signed/Float: first cast to/from unsigned +#define HWY_RVV_CAST_IF(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, \ + OP) \ + HWY_API vuint8##LMUL##_t BitCastToByte(HWY_RVV_V(BASE, SEW, LMUL) v) { \ + return v##OP##_v_u##SEW##LMUL##_u8##LMUL( \ + v##OP##_v_##CHAR##SEW##LMUL##_u##SEW##LMUL(v)); \ + } \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) BitCastFromByte( \ + HWY_RVV_D(CHAR, SEW, LMUL) /* d */, vuint8##LMUL##_t v) { \ + return v##OP##_v_u##SEW##LMUL##_##CHAR##SEW##LMUL( \ + v##OP##_v_u8##LMUL##_u##SEW##LMUL(v)); \ + } + +HWY_RVV_FOREACH_U08(HWY_RVV_CAST_U8, _, reinterpret) +HWY_RVV_FOREACH_I08(HWY_RVV_CAST_I8, _, reinterpret) +HWY_RVV_FOREACH_U16(HWY_RVV_CAST_U, _, reinterpret) +HWY_RVV_FOREACH_U32(HWY_RVV_CAST_U, _, reinterpret) +HWY_RVV_FOREACH_U64(HWY_RVV_CAST_U, _, reinterpret) +HWY_RVV_FOREACH_I16(HWY_RVV_CAST_IF, _, reinterpret) +HWY_RVV_FOREACH_I32(HWY_RVV_CAST_IF, _, reinterpret) +HWY_RVV_FOREACH_I64(HWY_RVV_CAST_IF, _, reinterpret) +HWY_RVV_FOREACH_F(HWY_RVV_CAST_IF, _, reinterpret) + +#undef HWY_RVV_CAST_U8 +#undef HWY_RVV_CAST_I8 +#undef HWY_RVV_CAST_U +#undef HWY_RVV_CAST_IF + +template <size_t N> +HWY_INLINE VFromD<Simd<uint16_t, N>> BitCastFromByte( + Simd<bfloat16_t, N> /* d */, VFromD<Simd<uint8_t, N * 2>> v) { + return BitCastFromByte(Simd<uint16_t, N>(), v); +} + +} // namespace detail + +template <class D, class FromV> +HWY_API VFromD<D> BitCast(D d, FromV v) { + return detail::BitCastFromByte(d, detail::BitCastToByte(v)); +} + +// Capped +template <typename T, size_t N, class FromV, + hwy::EnableIf<(N < HWY_LANES(T) / 8)>* = nullptr> +HWY_API VFromD<Simd<T, N>> BitCast(Simd<T, N> /*tag*/, FromV v) { + return BitCast(Full<T>(), v); +} + +namespace detail { + +template <class V, class DU = RebindToUnsigned<DFromV<V>>> +HWY_INLINE VFromD<DU> BitCastToUnsigned(V v) { + return BitCast(DU(), v); +} + +} // namespace detail + +// ------------------------------ Iota + +namespace detail { + +#define HWY_RVV_IOTA(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) NAME(HWY_RVV_D(CHAR, SEW, LMUL) d) { \ + return v##OP##_##CHAR##SEW##LMUL(Lanes(d)); \ + } + +HWY_RVV_FOREACH_U(HWY_RVV_IOTA, Iota0, id_v) +#undef HWY_RVV_IOTA + +template <class D, class DU = RebindToUnsigned<D>> +HWY_INLINE VFromD<DU> Iota0(const D /*d*/) { + return BitCastToUnsigned(Iota0(DU())); +} + +// Capped +template <typename T, size_t N, typename TU = MakeUnsigned<T>, + hwy::EnableIf<(N < HWY_LANES(T) / 8)>* = nullptr> +HWY_INLINE VFromD<Full<TU>> Iota0(Simd<T, N> /*tag*/) { + return Iota0(Full<TU>()); +} + +} // namespace detail + +// ================================================== LOGICAL + +// ------------------------------ Not + +HWY_RVV_FOREACH_UI(HWY_RVV_RETV_ARGV, Not, not ) + +template <class V, HWY_IF_FLOAT_V(V)> +HWY_API V Not(const V v) { + using DF = DFromV<V>; + using DU = RebindToUnsigned<DF>; + return BitCast(DF(), Not(BitCast(DU(), v))); +} + +// ------------------------------ And + +// Non-vector version (ideally immediate) for use with Iota0 +namespace detail { +HWY_RVV_FOREACH_UI(HWY_RVV_RETV_ARGVS, AndS, and_vx) +} // namespace detail + +HWY_RVV_FOREACH_UI(HWY_RVV_RETV_ARGVV, And, and) + +template <class V, HWY_IF_FLOAT_V(V)> +HWY_API V And(const V a, const V b) { + using DF = DFromV<V>; + using DU = RebindToUnsigned<DF>; + return BitCast(DF(), And(BitCast(DU(), a), BitCast(DU(), b))); +} + +// ------------------------------ Or + +#undef HWY_RVV_OR_MASK + +HWY_RVV_FOREACH_UI(HWY_RVV_RETV_ARGVV, Or, or) + +template <class V, HWY_IF_FLOAT_V(V)> +HWY_API V Or(const V a, const V b) { + using DF = DFromV<V>; + using DU = RebindToUnsigned<DF>; + return BitCast(DF(), Or(BitCast(DU(), a), BitCast(DU(), b))); +} + +// ------------------------------ Xor + +// Non-vector version (ideally immediate) for use with Iota0 +namespace detail { +HWY_RVV_FOREACH_UI(HWY_RVV_RETV_ARGVS, XorS, xor_vx) +} // namespace detail + +HWY_RVV_FOREACH_UI(HWY_RVV_RETV_ARGVV, Xor, xor) + +template <class V, HWY_IF_FLOAT_V(V)> +HWY_API V Xor(const V a, const V b) { + using DF = DFromV<V>; + using DU = RebindToUnsigned<DF>; + return BitCast(DF(), Xor(BitCast(DU(), a), BitCast(DU(), b))); +} + +// ------------------------------ AndNot + +template <class V> +HWY_API V AndNot(const V not_a, const V b) { + return And(Not(not_a), b); +} + +// ------------------------------ CopySign + +HWY_RVV_FOREACH_F(HWY_RVV_RETV_ARGVV, CopySign, fsgnj) + +template <class V> +HWY_API V CopySignToAbs(const V abs, const V sign) { + // RVV can also handle abs < 0, so no extra action needed. + return CopySign(abs, sign); +} + +// ================================================== ARITHMETIC + +// ------------------------------ Add + +namespace detail { +HWY_RVV_FOREACH_UI(HWY_RVV_RETV_ARGVS, AddS, add_vx) +HWY_RVV_FOREACH_F(HWY_RVV_RETV_ARGVS, AddS, fadd_vf) +} // namespace detail + +HWY_RVV_FOREACH_UI(HWY_RVV_RETV_ARGVV, Add, add) +HWY_RVV_FOREACH_F(HWY_RVV_RETV_ARGVV, Add, fadd) + +// ------------------------------ Sub +HWY_RVV_FOREACH_UI(HWY_RVV_RETV_ARGVV, Sub, sub) +HWY_RVV_FOREACH_F(HWY_RVV_RETV_ARGVV, Sub, fsub) + +// ------------------------------ SaturatedAdd + +HWY_RVV_FOREACH_U08(HWY_RVV_RETV_ARGVV, SaturatedAdd, saddu) +HWY_RVV_FOREACH_U16(HWY_RVV_RETV_ARGVV, SaturatedAdd, saddu) + +HWY_RVV_FOREACH_I08(HWY_RVV_RETV_ARGVV, SaturatedAdd, sadd) +HWY_RVV_FOREACH_I16(HWY_RVV_RETV_ARGVV, SaturatedAdd, sadd) + +// ------------------------------ SaturatedSub + +HWY_RVV_FOREACH_U08(HWY_RVV_RETV_ARGVV, SaturatedSub, ssubu) +HWY_RVV_FOREACH_U16(HWY_RVV_RETV_ARGVV, SaturatedSub, ssubu) + +HWY_RVV_FOREACH_I08(HWY_RVV_RETV_ARGVV, SaturatedSub, ssub) +HWY_RVV_FOREACH_I16(HWY_RVV_RETV_ARGVV, SaturatedSub, ssub) + +// ------------------------------ AverageRound + +// TODO(janwas): check vxrm rounding mode +HWY_RVV_FOREACH_U08(HWY_RVV_RETV_ARGVV, AverageRound, aaddu) +HWY_RVV_FOREACH_U16(HWY_RVV_RETV_ARGVV, AverageRound, aaddu) + +// ------------------------------ ShiftLeft[Same] + +// Intrinsics do not define .vi forms, so use .vx instead. +#define HWY_RVV_SHIFT(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, OP) \ + template <int kBits> \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) NAME(HWY_RVV_V(BASE, SEW, LMUL) v) { \ + return v##OP##_vx_##CHAR##SEW##LMUL(v, kBits, HWY_RVV_AVL(SEW, SHIFT)); \ + } \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME##Same(HWY_RVV_V(BASE, SEW, LMUL) v, int bits) { \ + return v##OP##_vx_##CHAR##SEW##LMUL(v, static_cast<uint8_t>(bits), \ + HWY_RVV_AVL(SEW, SHIFT)); \ + } + +HWY_RVV_FOREACH_UI(HWY_RVV_SHIFT, ShiftLeft, sll) + +// ------------------------------ ShiftRight[Same] + +HWY_RVV_FOREACH_U(HWY_RVV_SHIFT, ShiftRight, srl) +HWY_RVV_FOREACH_I(HWY_RVV_SHIFT, ShiftRight, sra) + +#undef HWY_RVV_SHIFT + +// ------------------------------ RotateRight +template <int kBits, class V> +HWY_API V RotateRight(const V v) { + constexpr size_t kSizeInBits = sizeof(TFromV<V>) * 8; + static_assert(0 <= kBits && kBits < kSizeInBits, "Invalid shift count"); + if (kBits == 0) return v; + return Or(ShiftRight<kBits>(v), ShiftLeft<kSizeInBits - kBits>(v)); +} + +// ------------------------------ Shl +#define HWY_RVV_SHIFT_VV(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, \ + OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME(HWY_RVV_V(BASE, SEW, LMUL) v, HWY_RVV_V(BASE, SEW, LMUL) bits) { \ + return v##OP##_vv_##CHAR##SEW##LMUL(v, bits, HWY_RVV_AVL(SEW, SHIFT)); \ + } + +HWY_RVV_FOREACH_U(HWY_RVV_SHIFT_VV, Shl, sll) + +#define HWY_RVV_SHIFT_II(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, \ + OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME(HWY_RVV_V(BASE, SEW, LMUL) v, HWY_RVV_V(BASE, SEW, LMUL) bits) { \ + return v##OP##_vv_##CHAR##SEW##LMUL(v, detail::BitCastToUnsigned(bits), \ + HWY_RVV_AVL(SEW, SHIFT)); \ + } + +HWY_RVV_FOREACH_I(HWY_RVV_SHIFT_II, Shl, sll) + +// ------------------------------ Shr + +HWY_RVV_FOREACH_U(HWY_RVV_SHIFT_VV, Shr, srl) +HWY_RVV_FOREACH_I(HWY_RVV_SHIFT_II, Shr, sra) + +#undef HWY_RVV_SHIFT_II +#undef HWY_RVV_SHIFT_VV + +// ------------------------------ Min + +HWY_RVV_FOREACH_U(HWY_RVV_RETV_ARGVV, Min, minu) +HWY_RVV_FOREACH_I(HWY_RVV_RETV_ARGVV, Min, min) +HWY_RVV_FOREACH_F(HWY_RVV_RETV_ARGVV, Min, fmin) + +// ------------------------------ Max + +namespace detail { + +HWY_RVV_FOREACH_U(HWY_RVV_RETV_ARGVS, MaxS, maxu_vx) +HWY_RVV_FOREACH_I(HWY_RVV_RETV_ARGVS, MaxS, max_vx) +HWY_RVV_FOREACH_F(HWY_RVV_RETV_ARGVS, MaxS, fmax_vf) + +} // namespace detail + +HWY_RVV_FOREACH_U(HWY_RVV_RETV_ARGVV, Max, maxu) +HWY_RVV_FOREACH_I(HWY_RVV_RETV_ARGVV, Max, max) +HWY_RVV_FOREACH_F(HWY_RVV_RETV_ARGVV, Max, fmax) + +// ------------------------------ Mul + +// Only for internal use (Highway only promises Mul for 16/32-bit inputs). +// Used by MulLower. +namespace detail { +HWY_RVV_FOREACH_U64(HWY_RVV_RETV_ARGVV, Mul, mul) +} // namespace detail + +HWY_RVV_FOREACH_UI16(HWY_RVV_RETV_ARGVV, Mul, mul) +HWY_RVV_FOREACH_UI32(HWY_RVV_RETV_ARGVV, Mul, mul) +HWY_RVV_FOREACH_F(HWY_RVV_RETV_ARGVV, Mul, fmul) + +// ------------------------------ MulHigh + +// Only for internal use (Highway only promises MulHigh for 16-bit inputs). +// Used by MulEven; vwmul does not work for m8. +namespace detail { +HWY_RVV_FOREACH_I32(HWY_RVV_RETV_ARGVV, MulHigh, mulh) +HWY_RVV_FOREACH_U32(HWY_RVV_RETV_ARGVV, MulHigh, mulhu) +HWY_RVV_FOREACH_U64(HWY_RVV_RETV_ARGVV, MulHigh, mulhu) +} // namespace detail + +HWY_RVV_FOREACH_U16(HWY_RVV_RETV_ARGVV, MulHigh, mulhu) +HWY_RVV_FOREACH_I16(HWY_RVV_RETV_ARGVV, MulHigh, mulh) + +// ------------------------------ Div +HWY_RVV_FOREACH_F(HWY_RVV_RETV_ARGVV, Div, fdiv) + +// ------------------------------ ApproximateReciprocal +HWY_RVV_FOREACH_F32(HWY_RVV_RETV_ARGV, ApproximateReciprocal, frec7) + +// ------------------------------ Sqrt +HWY_RVV_FOREACH_F(HWY_RVV_RETV_ARGV, Sqrt, fsqrt) + +// ------------------------------ ApproximateReciprocalSqrt +HWY_RVV_FOREACH_F32(HWY_RVV_RETV_ARGV, ApproximateReciprocalSqrt, frsqrt7) + +// ------------------------------ MulAdd +// Note: op is still named vv, not vvv. +#define HWY_RVV_FMA(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME(HWY_RVV_V(BASE, SEW, LMUL) mul, HWY_RVV_V(BASE, SEW, LMUL) x, \ + HWY_RVV_V(BASE, SEW, LMUL) add) { \ + return v##OP##_vv_##CHAR##SEW##LMUL(add, mul, x, HWY_RVV_AVL(SEW, SHIFT)); \ + } + +HWY_RVV_FOREACH_F(HWY_RVV_FMA, MulAdd, fmacc) + +// ------------------------------ NegMulAdd +HWY_RVV_FOREACH_F(HWY_RVV_FMA, NegMulAdd, fnmsac) + +// ------------------------------ MulSub +HWY_RVV_FOREACH_F(HWY_RVV_FMA, MulSub, fmsac) + +// ------------------------------ NegMulSub +HWY_RVV_FOREACH_F(HWY_RVV_FMA, NegMulSub, fnmacc) + +#undef HWY_RVV_FMA + +// ================================================== COMPARE + +// Comparisons set a mask bit to 1 if the condition is true, else 0. The XX in +// vboolXX_t is a power of two divisor for vector bits. SLEN 8 / LMUL 1 = 1/8th +// of all bits; SLEN 8 / LMUL 4 = half of all bits. + +// mask = f(vector, vector) +#define HWY_RVV_RETM_ARGVV(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, \ + OP) \ + HWY_API HWY_RVV_M(MLEN) \ + NAME(HWY_RVV_V(BASE, SEW, LMUL) a, HWY_RVV_V(BASE, SEW, LMUL) b) { \ + return v##OP##_vv_##CHAR##SEW##LMUL##_b##MLEN(a, b, \ + HWY_RVV_AVL(SEW, SHIFT)); \ + } + +// mask = f(vector, scalar) +#define HWY_RVV_RETM_ARGVS(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, \ + OP) \ + HWY_API HWY_RVV_M(MLEN) \ + NAME(HWY_RVV_V(BASE, SEW, LMUL) a, HWY_RVV_T(BASE, SEW) b) { \ + return v##OP##_vx_##CHAR##SEW##LMUL##_b##MLEN(a, b, \ + HWY_RVV_AVL(SEW, SHIFT)); \ + } + +// ------------------------------ Eq +HWY_RVV_FOREACH_UI(HWY_RVV_RETM_ARGVV, Eq, mseq) +HWY_RVV_FOREACH_F(HWY_RVV_RETM_ARGVV, Eq, mfeq) + +// ------------------------------ Ne +HWY_RVV_FOREACH_UI(HWY_RVV_RETM_ARGVV, Ne, msne) +HWY_RVV_FOREACH_F(HWY_RVV_RETM_ARGVV, Ne, mfne) + +// ------------------------------ Lt +HWY_RVV_FOREACH_U(HWY_RVV_RETM_ARGVV, Lt, msltu) +HWY_RVV_FOREACH_I(HWY_RVV_RETM_ARGVV, Lt, mslt) +HWY_RVV_FOREACH_F(HWY_RVV_RETM_ARGVV, Lt, mflt) + +namespace detail { +HWY_RVV_FOREACH_I(HWY_RVV_RETM_ARGVS, LtS, mslt) +} // namespace detail + +// ------------------------------ Le +HWY_RVV_FOREACH_F(HWY_RVV_RETM_ARGVV, Le, mfle) + +#undef HWY_RVV_RETM_ARGVV +#undef HWY_RVV_RETM_ARGVS + +// ------------------------------ Gt/Ge + +template <class V> +HWY_API auto Ge(const V a, const V b) -> decltype(Le(a, b)) { + return Le(b, a); +} + +template <class V> +HWY_API auto Gt(const V a, const V b) -> decltype(Lt(a, b)) { + return Lt(b, a); +} + +// ------------------------------ TestBit +template <class V> +HWY_API auto TestBit(const V a, const V bit) -> decltype(Eq(a, bit)) { + return Ne(And(a, bit), Zero(DFromV<V>())); +} + +// ------------------------------ Not + +// mask = f(mask) +#define HWY_RVV_RETM_ARGM(SEW, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_M(MLEN) NAME(HWY_RVV_M(MLEN) m) { \ + return vm##OP##_m_b##MLEN(m, ~0ull); \ + } + +HWY_RVV_FOREACH_B(HWY_RVV_RETM_ARGM, Not, not ) + +#undef HWY_RVV_RETM_ARGM + +// ------------------------------ And + +// mask = f(mask_a, mask_b) (note arg2,arg1 order!) +#define HWY_RVV_RETM_ARGMM(SEW, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_M(MLEN) NAME(HWY_RVV_M(MLEN) a, HWY_RVV_M(MLEN) b) { \ + return vm##OP##_mm_b##MLEN(b, a, HWY_RVV_AVL(SEW, SHIFT)); \ + } + +HWY_RVV_FOREACH_B(HWY_RVV_RETM_ARGMM, And, and) + +// ------------------------------ AndNot +HWY_RVV_FOREACH_B(HWY_RVV_RETM_ARGMM, AndNot, andn) + +// ------------------------------ Or +HWY_RVV_FOREACH_B(HWY_RVV_RETM_ARGMM, Or, or) + +// ------------------------------ Xor +HWY_RVV_FOREACH_B(HWY_RVV_RETM_ARGMM, Xor, xor) + +#undef HWY_RVV_RETM_ARGMM + +// ------------------------------ IfThenElse +#define HWY_RVV_IF_THEN_ELSE(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, \ + NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME(HWY_RVV_M(MLEN) m, HWY_RVV_V(BASE, SEW, LMUL) yes, \ + HWY_RVV_V(BASE, SEW, LMUL) no) { \ + return v##OP##_vvm_##CHAR##SEW##LMUL(m, no, yes, HWY_RVV_AVL(SEW, SHIFT)); \ + } + +HWY_RVV_FOREACH(HWY_RVV_IF_THEN_ELSE, IfThenElse, merge) + +#undef HWY_RVV_IF_THEN_ELSE + +// ------------------------------ IfThenElseZero +template <class M, class V> +HWY_API V IfThenElseZero(const M mask, const V yes) { + return IfThenElse(mask, yes, Zero(DFromV<V>())); +} + +// ------------------------------ IfThenZeroElse +template <class M, class V> +HWY_API V IfThenZeroElse(const M mask, const V no) { + return IfThenElse(mask, Zero(DFromV<V>()), no); +} + +// ------------------------------ MaskFromVec + +template <class V> +HWY_API auto MaskFromVec(const V v) -> decltype(Eq(v, v)) { + return Ne(v, Zero(DFromV<V>())); +} + +template <class D> +using MFromD = decltype(MaskFromVec(Zero(D()))); + +template <class D, typename MFrom> +HWY_API MFromD<D> RebindMask(const D /*d*/, const MFrom mask) { + // No need to check lane size/LMUL are the same: if not, casting MFrom to + // MFromD<D> would fail. + return mask; +} + +// ------------------------------ VecFromMask + +namespace detail { +#define HWY_RVV_VEC_FROM_MASK(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, \ + NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME(HWY_RVV_V(BASE, SEW, LMUL) v0, HWY_RVV_M(MLEN) m) { \ + return v##OP##_##CHAR##SEW##LMUL##_m(m, v0, v0, 1, \ + HWY_RVV_AVL(SEW, SHIFT)); \ + } + +HWY_RVV_FOREACH_UI(HWY_RVV_VEC_FROM_MASK, SubS, sub_vx) +#undef HWY_RVV_VEC_FROM_MASK +} // namespace detail + +template <class D, HWY_IF_NOT_FLOAT_D(D)> +HWY_API VFromD<D> VecFromMask(const D d, MFromD<D> mask) { + return detail::SubS(Zero(d), mask); +} + +template <class D, HWY_IF_FLOAT_D(D)> +HWY_API VFromD<D> VecFromMask(const D d, MFromD<D> mask) { + return BitCast(d, VecFromMask(RebindToUnsigned<D>(), mask)); +} + +// ------------------------------ ZeroIfNegative +template <class V> +HWY_API V ZeroIfNegative(const V v) { + const auto v0 = Zero(DFromV<V>()); + // We already have a zero constant, so avoid IfThenZeroElse. + return IfThenElse(Lt(v, v0), v0, v); +} + +// ------------------------------ BroadcastSignBit +template <class V> +HWY_API V BroadcastSignBit(const V v) { + return ShiftRight<sizeof(TFromV<V>) * 8 - 1>(v); +} + +// ------------------------------ FindFirstTrue + +#define HWY_RVV_FIND_FIRST_TRUE(SEW, SHIFT, MLEN, NAME, OP) \ + template <class D> \ + HWY_API intptr_t FindFirstTrue(D d, HWY_RVV_M(MLEN) m) { \ + static_assert(MLenFromD(d) == MLEN, "Type mismatch"); \ + return vfirst_m_b##MLEN(m, Lanes(d)); \ + } + +HWY_RVV_FOREACH_B(HWY_RVV_FIND_FIRST_TRUE, _, _) +#undef HWY_RVV_FIND_FIRST_TRUE + +// ------------------------------ AllFalse +template <class D> +HWY_API bool AllFalse(D d, MFromD<D> m) { + return FindFirstTrue(d, m) < 0; +} + +// ------------------------------ AllTrue + +#define HWY_RVV_ALL_TRUE(SEW, SHIFT, MLEN, NAME, OP) \ + template <class D> \ + HWY_API bool AllTrue(D d, HWY_RVV_M(MLEN) m) { \ + static_assert(MLenFromD(d) == MLEN, "Type mismatch"); \ + return AllFalse(d, vmnot_m_b##MLEN(m, Lanes(d))); \ + } + +HWY_RVV_FOREACH_B(HWY_RVV_ALL_TRUE, _, _) +#undef HWY_RVV_ALL_TRUE + +// ------------------------------ CountTrue + +#define HWY_RVV_COUNT_TRUE(SEW, SHIFT, MLEN, NAME, OP) \ + template <class D> \ + HWY_API size_t CountTrue(D d, HWY_RVV_M(MLEN) m) { \ + static_assert(MLenFromD(d) == MLEN, "Type mismatch"); \ + return vcpop_m_b##MLEN(m, Lanes(d)); \ + } + +HWY_RVV_FOREACH_B(HWY_RVV_COUNT_TRUE, _, _) +#undef HWY_RVV_COUNT_TRUE + +// ================================================== MEMORY + +// ------------------------------ Load + +#define HWY_RVV_LOAD(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME(HWY_RVV_D(CHAR, SEW, LMUL) d, \ + const HWY_RVV_T(BASE, SEW) * HWY_RESTRICT p) { \ + return v##OP##SEW##_v_##CHAR##SEW##LMUL(p, Lanes(d)); \ + } +HWY_RVV_FOREACH(HWY_RVV_LOAD, Load, le) +#undef HWY_RVV_LOAD + +// Capped +template <typename T, size_t N, + hwy::EnableIf<(N < HWY_LANES(T) / 8)>* = nullptr> +HWY_API VFromD<Simd<T, N>> Load(Simd<T, N> d, const T* HWY_RESTRICT p) { + return Load(d, p); +} + +// There is no native BF16, treat as uint16_t. +template <size_t N> +HWY_API VFromD<Simd<uint16_t, N>> Load(Simd<bfloat16_t, N> d, + const bfloat16_t* HWY_RESTRICT p) { + return Load(RebindToUnsigned<decltype(d)>(), + reinterpret_cast<const uint16_t * HWY_RESTRICT>(p)); +} + +template <size_t N> +HWY_API void Store(VFromD<Simd<uint16_t, N>> v, Simd<bfloat16_t, N> d, + bfloat16_t* HWY_RESTRICT p) { + Store(v, RebindToUnsigned<decltype(d)>(), + reinterpret_cast<uint16_t * HWY_RESTRICT>(p)); +} + +// ------------------------------ LoadU + +// RVV only requires lane alignment, not natural alignment of the entire vector. +template <class D> +HWY_API VFromD<D> LoadU(D d, const TFromD<D>* HWY_RESTRICT p) { + return Load(d, p); +} + +// ------------------------------ MaskedLoad + +#define HWY_RVV_MASKED_LOAD(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, \ + NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME(HWY_RVV_M(MLEN) m, HWY_RVV_D(CHAR, SEW, LMUL) d, \ + const HWY_RVV_T(BASE, SEW) * HWY_RESTRICT p) { \ + return v##OP##SEW##_v_##CHAR##SEW##LMUL##_m(m, Zero(d), p, Lanes(d)); \ + } +HWY_RVV_FOREACH(HWY_RVV_MASKED_LOAD, MaskedLoad, le) +#undef HWY_RVV_MASKED_LOAD + +// ------------------------------ Store + +#define HWY_RVV_RET_ARGVDP(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, \ + OP) \ + HWY_API void NAME(HWY_RVV_V(BASE, SEW, LMUL) v, \ + HWY_RVV_D(CHAR, SEW, LMUL) d, \ + HWY_RVV_T(BASE, SEW) * HWY_RESTRICT p) { \ + return v##OP##SEW##_v_##CHAR##SEW##LMUL(p, v, Lanes(d)); \ + } +HWY_RVV_FOREACH(HWY_RVV_RET_ARGVDP, Store, se) +#undef HWY_RVV_RET_ARGVDP + +// Capped +template <typename T, size_t N, + hwy::EnableIf<(N < HWY_LANES(T) / 8)>* = nullptr> +HWY_API void Store(VFromD<Simd<T, N>> v, Simd<T, N> /* d */, + T* HWY_RESTRICT p) { + return Store(v, Full<T>(), p); +} + +// ------------------------------ MaskedStore + +#define HWY_RVV_RET_ARGMVDP(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, \ + NAME, OP) \ + HWY_API void NAME(HWY_RVV_M(MLEN) m, HWY_RVV_V(BASE, SEW, LMUL) v, \ + HWY_RVV_D(CHAR, SEW, LMUL) d, \ + HWY_RVV_T(BASE, SEW) * HWY_RESTRICT p) { \ + return v##OP##SEW##_v_##CHAR##SEW##LMUL##_m(m, p, v, Lanes(d)); \ + } +HWY_RVV_FOREACH(HWY_RVV_RET_ARGMVDP, MaskedStore, se) +#undef HWY_RVV_RET_ARGMVDP + +namespace detail { + +#define HWY_RVV_RET_ARGNVDP(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, \ + NAME, OP) \ + HWY_API void NAME(size_t count, HWY_RVV_V(BASE, SEW, LMUL) v, \ + HWY_RVV_D(CHAR, SEW, LMUL) /* d */, \ + HWY_RVV_T(BASE, SEW) * HWY_RESTRICT p) { \ + return v##OP##SEW##_v_##CHAR##SEW##LMUL(p, v, count); \ + } +HWY_RVV_FOREACH(HWY_RVV_RET_ARGNVDP, StoreN, se) +#undef HWY_RVV_RET_ARGNVDP + +} // namespace detail + +// ------------------------------ StoreU + +// RVV only requires lane alignment, not natural alignment of the entire vector. +template <class V, class D> +HWY_API void StoreU(const V v, D d, TFromD<D>* HWY_RESTRICT p) { + Store(v, d, p); +} + +// ------------------------------ Stream +template <class V, class D, typename T> +HWY_API void Stream(const V v, D d, T* HWY_RESTRICT aligned) { + Store(v, d, aligned); +} + +// ------------------------------ ScatterOffset + +#define HWY_RVV_SCATTER(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, \ + OP) \ + HWY_API void NAME(HWY_RVV_V(BASE, SEW, LMUL) v, \ + HWY_RVV_D(CHAR, SEW, LMUL) d, \ + HWY_RVV_T(BASE, SEW) * HWY_RESTRICT base, \ + HWY_RVV_V(int, SEW, LMUL) offset) { \ + return v##OP##ei##SEW##_v_##CHAR##SEW##LMUL( \ + base, detail::BitCastToUnsigned(offset), v, Lanes(d)); \ + } +HWY_RVV_FOREACH(HWY_RVV_SCATTER, ScatterOffset, sux) +#undef HWY_RVV_SCATTER + +// Capped +template <typename T, size_t N, + hwy::EnableIf<(N < HWY_LANES(T) / 8)>* = nullptr> +HWY_API void ScatterOffset(VFromD<Simd<T, N>> v, Simd<T, N> /* d */, + T* HWY_RESTRICT base, + VFromD<Simd<MakeSigned<T>, N>> offset) { + return ScatterOffset(v, Full<T>(), base, offset); +} + +// ------------------------------ ScatterIndex + +template <class D, HWY_IF_LANE_SIZE_D(D, 4)> +HWY_API void ScatterIndex(VFromD<D> v, D d, TFromD<D>* HWY_RESTRICT base, + const VFromD<RebindToSigned<D>> index) { + return ScatterOffset(v, d, base, ShiftLeft<2>(index)); +} + +template <class D, HWY_IF_LANE_SIZE_D(D, 8)> +HWY_API void ScatterIndex(VFromD<D> v, D d, TFromD<D>* HWY_RESTRICT base, + const VFromD<RebindToSigned<D>> index) { + return ScatterOffset(v, d, base, ShiftLeft<3>(index)); +} + +// ------------------------------ GatherOffset + +#define HWY_RVV_GATHER(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME(HWY_RVV_D(CHAR, SEW, LMUL) d, \ + const HWY_RVV_T(BASE, SEW) * HWY_RESTRICT base, \ + HWY_RVV_V(int, SEW, LMUL) offset) { \ + return v##OP##ei##SEW##_v_##CHAR##SEW##LMUL( \ + base, detail::BitCastToUnsigned(offset), Lanes(d)); \ + } +HWY_RVV_FOREACH(HWY_RVV_GATHER, GatherOffset, lux) +#undef HWY_RVV_GATHER + +// Capped +template <typename T, size_t N, + hwy::EnableIf<(N < HWY_LANES(T) / 8)>* = nullptr> +HWY_API VFromD<Simd<T, N>> GatherOffset(Simd<T, N> /* d */, + const T* HWY_RESTRICT base, + VFromD<Simd<MakeSigned<T>, N>> offset) { + return GatherOffset(Full<T>(), base, offset); +} + +// ------------------------------ GatherIndex + +template <class D, HWY_IF_LANE_SIZE_D(D, 4)> +HWY_API VFromD<D> GatherIndex(D d, const TFromD<D>* HWY_RESTRICT base, + const VFromD<RebindToSigned<D>> index) { + return GatherOffset(d, base, ShiftLeft<2>(index)); +} + +template <class D, HWY_IF_LANE_SIZE_D(D, 8)> +HWY_API VFromD<D> GatherIndex(D d, const TFromD<D>* HWY_RESTRICT base, + const VFromD<RebindToSigned<D>> index) { + return GatherOffset(d, base, ShiftLeft<3>(index)); +} + +// TODO(janwas): wait for https://github.com/riscv/rvv-intrinsic-doc/issues/95 +#if HWY_COMPILER_GCC && !HWY_COMPILER_CLANG + +// ------------------------------ StoreInterleaved3 + +#define HWY_RVV_STORE3(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, OP) \ + HWY_API void NAME( \ + HWY_RVV_V(BASE, SEW, LMUL) v0, HWY_RVV_V(BASE, SEW, LMUL) v1, \ + HWY_RVV_V(BASE, SEW, LMUL) v2, HWY_RVV_D(CHAR, SEW, LMUL) d, \ + HWY_RVV_T(BASE, SEW) * HWY_RESTRICT unaligned) { \ + const v##BASE##SEW##LMUL##x3_t triple = \ + vcreate_##CHAR##SEW##LMUL##x3(v0, v1, v2); \ + return v##OP##e8_v_##CHAR##SEW##LMUL##x3(unaligned, triple, Lanes(d)); \ + } +// Segments are limited to 8 registers, so we can only go up to LMUL=2. +HWY_RVV_STORE3(uint, u, 8, m1, /*kShift=*/0, 8, StoreInterleaved3, sseg3) +HWY_RVV_STORE3(uint, u, 8, m2, /*kShift=*/1, 4, StoreInterleaved3, sseg3) + +#undef HWY_RVV_STORE3 + +// Capped +template <typename T, size_t N, + hwy::EnableIf<(N < HWY_LANES(T) / 8)>* = nullptr> +HWY_API void StoreInterleaved3(VFromD<Simd<T, N>> v0, VFromD<Simd<T, N>> v1, + VFromD<Simd<T, N>> v2, Simd<T, N> /*tag*/, + T* unaligned) { + return StoreInterleaved3(v0, v1, v2, Full<T>(), unaligned); +} + +// ------------------------------ StoreInterleaved4 + +#define HWY_RVV_STORE4(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, OP) \ + HWY_API void NAME( \ + HWY_RVV_V(BASE, SEW, LMUL) v0, HWY_RVV_V(BASE, SEW, LMUL) v1, \ + HWY_RVV_V(BASE, SEW, LMUL) v2, HWY_RVV_V(BASE, SEW, LMUL) v3, \ + HWY_RVV_D(CHAR, SEW, LMUL) d, \ + HWY_RVV_T(BASE, SEW) * HWY_RESTRICT aligned) { \ + const v##BASE##SEW##LMUL##x4_t quad = \ + vcreate_##CHAR##SEW##LMUL##x4(v0, v1, v2, v3); \ + return v##OP##e8_v_##CHAR##SEW##LMUL##x4(aligned, quad, Lanes(d)); \ + } +// Segments are limited to 8 registers, so we can only go up to LMUL=2. +HWY_RVV_STORE4(uint, u, 8, m1, /*kShift=*/0, 8, StoreInterleaved4, sseg4) +HWY_RVV_STORE4(uint, u, 8, m2, /*kShift=*/1, 4, StoreInterleaved4, sseg4) + +#undef HWY_RVV_STORE4 + +// Capped +template <typename T, size_t N, + hwy::EnableIf<(N < HWY_LANES(T) / 8)>* = nullptr> +HWY_API void StoreInterleaved4(VFromD<Simd<T, N>> v0, VFromD<Simd<T, N>> v1, + VFromD<Simd<T, N>> v2, VFromD<Simd<T, N>> v3, + Simd<T, N> /*tag*/, T* unaligned) { + return StoreInterleaved4(v0, v1, v2, v3, Full<T>(), unaligned); +} + +#endif // GCC + +// ================================================== CONVERT + +#define HWY_RVV_PROMOTE(OP, BASE, CHAR, BITS, BASE_IN, BITS_IN, LMUL, LMUL_IN) \ + HWY_API HWY_RVV_V(BASE, BITS, LMUL) PromoteTo( \ + HWY_RVV_D(CHAR, BITS, LMUL) d, HWY_RVV_V(BASE_IN, BITS_IN, LMUL_IN) v) { \ + return OP##CHAR##BITS##LMUL(v, Lanes(d)); \ + } + +#define HWY_RVV_PROMOTE_X2(OP, BASE, CHAR, BITS, BASE_IN, BITS_IN) \ + HWY_RVV_PROMOTE(OP, BASE, CHAR, BITS, BASE_IN, BITS_IN, m1, mf2) \ + HWY_RVV_PROMOTE(OP, BASE, CHAR, BITS, BASE_IN, BITS_IN, m2, m1) \ + HWY_RVV_PROMOTE(OP, BASE, CHAR, BITS, BASE_IN, BITS_IN, m4, m2) \ + HWY_RVV_PROMOTE(OP, BASE, CHAR, BITS, BASE_IN, BITS_IN, m8, m4) + +#define HWY_RVV_PROMOTE_X4(OP, BASE, CHAR, BITS, BASE_IN, BITS_IN) \ + HWY_RVV_PROMOTE(OP, BASE, CHAR, BITS, BASE_IN, BITS_IN, m1, mf4) \ + HWY_RVV_PROMOTE(OP, BASE, CHAR, BITS, BASE_IN, BITS_IN, m2, mf2) \ + HWY_RVV_PROMOTE(OP, BASE, CHAR, BITS, BASE_IN, BITS_IN, m4, m1) \ + HWY_RVV_PROMOTE(OP, BASE, CHAR, BITS, BASE_IN, BITS_IN, m8, m2) + +// ------------------------------ PromoteTo + +HWY_RVV_PROMOTE_X2(vzext_vf2_, uint, u, 16, uint, 8) +HWY_RVV_PROMOTE_X2(vzext_vf2_, uint, u, 32, uint, 16) +HWY_RVV_PROMOTE_X2(vzext_vf2_, uint, u, 64, uint, 32) +HWY_RVV_PROMOTE_X4(vzext_vf4_, uint, u, 32, uint, 8) + +HWY_RVV_PROMOTE_X2(vsext_vf2_, int, i, 16, int, 8) +HWY_RVV_PROMOTE_X2(vsext_vf2_, int, i, 32, int, 16) +HWY_RVV_PROMOTE_X2(vsext_vf2_, int, i, 64, int, 32) +HWY_RVV_PROMOTE_X4(vsext_vf4_, int, i, 32, int, 8) + +#if HWY_CAP_FLOAT16 +HWY_RVV_PROMOTE_X2(vfwcvt_f_f_v_, float, f, 32, float, 16) +#endif +HWY_RVV_PROMOTE_X2(vfwcvt_f_f_v_, float, f, 64, float, 32) + +// i32 to f64 +HWY_RVV_PROMOTE_X2(vfwcvt_f_x_v_, float, f, 64, int, 32) + +#undef HWY_RVV_PROMOTE_X4 +#undef HWY_RVV_PROMOTE_X2 +#undef HWY_RVV_PROMOTE + +template <size_t N> +HWY_API auto PromoteTo(Simd<int16_t, N> d, VFromD<Simd<uint8_t, N>> v) + -> VFromD<decltype(d)> { + return BitCast(d, PromoteTo(RebindToUnsigned<decltype(d)>(), v)); +} + +template <size_t N> +HWY_API auto PromoteTo(Simd<int32_t, N> d, VFromD<Simd<uint8_t, N>> v) + -> VFromD<decltype(d)> { + return BitCast(d, PromoteTo(RebindToUnsigned<decltype(d)>(), v)); +} + +template <size_t N> +HWY_API auto PromoteTo(Simd<int32_t, N> d, VFromD<Simd<uint16_t, N>> v) + -> VFromD<decltype(d)> { + return BitCast(d, PromoteTo(RebindToUnsigned<decltype(d)>(), v)); +} + +template <size_t N> +HWY_API auto PromoteTo(Simd<float32_t, N> d, VFromD<Simd<bfloat16_t, N>> v) + -> VFromD<decltype(d)> { + const RebindToSigned<decltype(d)> di32; + const Rebind<uint16_t, decltype(d)> du16; + return BitCast(d, ShiftLeft<16>(PromoteTo(di32, BitCast(du16, v)))); +} + +// ------------------------------ DemoteTo U + +// Unsigned -> unsigned (also used for bf16) +namespace detail { + +HWY_INLINE Vu16m1 DemoteTo(Du16m1 d, const Vu32m2 v) { + return vnclipu_wx_u16m1(v, 0, Lanes(d)); +} +HWY_INLINE Vu16m2 DemoteTo(Du16m2 d, const Vu32m4 v) { + return vnclipu_wx_u16m2(v, 0, Lanes(d)); +} +HWY_INLINE Vu16m4 DemoteTo(Du16m4 d, const Vu32m8 v) { + return vnclipu_wx_u16m4(v, 0, Lanes(d)); +} + +HWY_INLINE Vu8m1 DemoteTo(Du8m1 d, const Vu16m2 v) { + return vnclipu_wx_u8m1(v, 0, Lanes(d)); +} +HWY_INLINE Vu8m2 DemoteTo(Du8m2 d, const Vu16m4 v) { + return vnclipu_wx_u8m2(v, 0, Lanes(d)); +} +HWY_INLINE Vu8m4 DemoteTo(Du8m4 d, const Vu16m8 v) { + return vnclipu_wx_u8m4(v, 0, Lanes(d)); +} + +} // namespace detail + +// First clamp negative numbers to zero to match x86 packus. +HWY_API Vu16m1 DemoteTo(Du16m1 d, const Vi32m2 v) { + return detail::DemoteTo(d, detail::BitCastToUnsigned(detail::MaxS(v, 0))); +} +HWY_API Vu16m2 DemoteTo(Du16m2 d, const Vi32m4 v) { + return detail::DemoteTo(d, detail::BitCastToUnsigned(detail::MaxS(v, 0))); +} +HWY_API Vu16m4 DemoteTo(Du16m4 d, const Vi32m8 v) { + return detail::DemoteTo(d, detail::BitCastToUnsigned(detail::MaxS(v, 0))); +} + +HWY_API Vu8m1 DemoteTo(Du8m1 d, const Vi32m4 v) { + return vnclipu_wx_u8m1(DemoteTo(Du16m2(), v), 0, Lanes(d)); +} +HWY_API Vu8m2 DemoteTo(Du8m2 d, const Vi32m8 v) { + return vnclipu_wx_u8m2(DemoteTo(Du16m4(), v), 0, Lanes(d)); +} + +HWY_API Vu8m1 DemoteTo(Du8m1 d, const Vi16m2 v) { + return detail::DemoteTo(d, detail::BitCastToUnsigned(detail::MaxS(v, 0))); +} +HWY_API Vu8m2 DemoteTo(Du8m2 d, const Vi16m4 v) { + return detail::DemoteTo(d, detail::BitCastToUnsigned(detail::MaxS(v, 0))); +} +HWY_API Vu8m4 DemoteTo(Du8m4 d, const Vi16m8 v) { + return detail::DemoteTo(d, detail::BitCastToUnsigned(detail::MaxS(v, 0))); +} + +HWY_API Vu8m1 U8FromU32(const Vu32m4 v) { + const size_t avl = Lanes(Full<uint8_t>()); + return vnclipu_wx_u8m1(vnclipu_wx_u16m2(v, 0, avl), 0, avl); +} +HWY_API Vu8m2 U8FromU32(const Vu32m8 v) { + const size_t avl = Lanes(Full<uint8_t, 2>()); + return vnclipu_wx_u8m2(vnclipu_wx_u16m4(v, 0, avl), 0, avl); +} + +// ------------------------------ DemoteTo I + +HWY_API Vi8m1 DemoteTo(Di8m1 d, const Vi16m2 v) { + return vnclip_wx_i8m1(v, 0, Lanes(d)); +} +HWY_API Vi8m2 DemoteTo(Di8m2 d, const Vi16m4 v) { + return vnclip_wx_i8m2(v, 0, Lanes(d)); +} +HWY_API Vi8m4 DemoteTo(Di8m4 d, const Vi16m8 v) { + return vnclip_wx_i8m4(v, 0, Lanes(d)); +} + +HWY_API Vi16m1 DemoteTo(Di16m1 d, const Vi32m2 v) { + return vnclip_wx_i16m1(v, 0, Lanes(d)); +} +HWY_API Vi16m2 DemoteTo(Di16m2 d, const Vi32m4 v) { + return vnclip_wx_i16m2(v, 0, Lanes(d)); +} +HWY_API Vi16m4 DemoteTo(Di16m4 d, const Vi32m8 v) { + return vnclip_wx_i16m4(v, 0, Lanes(d)); +} + +HWY_API Vi8m1 DemoteTo(Di8m1 d, const Vi32m4 v) { + return DemoteTo(d, DemoteTo(Di16m2(), v)); +} +HWY_API Vi8m2 DemoteTo(Di8m2 d, const Vi32m8 v) { + return DemoteTo(d, DemoteTo(Di16m4(), v)); +} + +// ------------------------------ DemoteTo F + +#if HWY_CAP_FLOAT16 +HWY_API Vf16m1 DemoteTo(Df16m1 d, const Vf32m2 v) { + return vfncvt_rod_f_f_w_f16m1(v, Lanes(d)); +} +HWY_API Vf16m2 DemoteTo(Df16m2 d, const Vf32m4 v) { + return vfncvt_rod_f_f_w_f16m2(v, Lanes(d)); +} +HWY_API Vf16m4 DemoteTo(Df16m4 d, const Vf32m8 v) { + return vfncvt_rod_f_f_w_f16m4(v, Lanes(d)); +} +#endif + +HWY_API Vf32m1 DemoteTo(Df32m1 d, const Vf64m2 v) { + return vfncvt_rod_f_f_w_f32m1(v, Lanes(d)); +} +HWY_API Vf32m2 DemoteTo(Df32m2 d, const Vf64m4 v) { + return vfncvt_rod_f_f_w_f32m2(v, Lanes(d)); +} +HWY_API Vf32m4 DemoteTo(Df32m4 d, const Vf64m8 v) { + return vfncvt_rod_f_f_w_f32m4(v, Lanes(d)); +} + +HWY_API Vi32m1 DemoteTo(Di32m1 d, const Vf64m2 v) { + return vfncvt_rtz_x_f_w_i32m1(v, Lanes(d)); +} +HWY_API Vi32m2 DemoteTo(Di32m2 d, const Vf64m4 v) { + return vfncvt_rtz_x_f_w_i32m2(v, Lanes(d)); +} +HWY_API Vi32m4 DemoteTo(Di32m4 d, const Vf64m8 v) { + return vfncvt_rtz_x_f_w_i32m4(v, Lanes(d)); +} + +template <size_t N> +HWY_API VFromD<Simd<uint16_t, N>> DemoteTo(Simd<bfloat16_t, N> d, + VFromD<Simd<float, N>> v) { + const RebindToUnsigned<decltype(d)> du16; + const Rebind<uint32_t, decltype(d)> du32; + return DemoteTo(du16, BitCast(du32, v)); +} + +// ------------------------------ ConvertTo F + +#define HWY_RVV_CONVERT(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, \ + OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + ConvertTo(HWY_RVV_D(CHAR, SEW, LMUL) d, HWY_RVV_V(int, SEW, LMUL) v) { \ + return vfcvt_f_x_v_f##SEW##LMUL(v, Lanes(d)); \ + } \ + /* Truncates (rounds toward zero). */ \ + HWY_API HWY_RVV_V(int, SEW, LMUL) \ + ConvertTo(HWY_RVV_D(i, SEW, LMUL) d, HWY_RVV_V(BASE, SEW, LMUL) v) { \ + return vfcvt_rtz_x_f_v_i##SEW##LMUL(v, Lanes(d)); \ + } \ + /* Uses default rounding mode. */ \ + HWY_API HWY_RVV_V(int, SEW, LMUL) NearestInt(HWY_RVV_V(BASE, SEW, LMUL) v) { \ + return vfcvt_x_f_v_i##SEW##LMUL(v, HWY_RVV_AVL(SEW, SHIFT)); \ + } + +// API only requires f32 but we provide f64 for internal use (otherwise, it +// seems difficult to implement Iota without a _mf2 vector half). +HWY_RVV_FOREACH_F(HWY_RVV_CONVERT, _, _) +#undef HWY_RVV_CONVERT + +// Capped +template <typename T, size_t N, class FromV, + hwy::EnableIf<(N < HWY_LANES(T) / 8)>* = nullptr> +HWY_API VFromD<Simd<T, N>> ConvertTo(Simd<T, N> /*tag*/, FromV v) { + return ConvertTo(Full<T>(), v); +} + +// ================================================== COMBINE + +namespace detail { + +// For x86-compatible behaviour mandated by Highway API: TableLookupBytes +// offsets are implicitly relative to the start of their 128-bit block. +template <typename T, size_t N> +constexpr size_t LanesPerBlock(Simd<T, N> /* tag */) { + // Also cap to the limit imposed by D (for fixed-size <= 128-bit vectors). + return HWY_MIN(16 / sizeof(T), N); +} + +template <class D, class V> +HWY_INLINE V OffsetsOf128BitBlocks(const D d, const V iota0) { + using T = MakeUnsigned<TFromD<D>>; + return AndS(iota0, static_cast<T>(~(LanesPerBlock(d) - 1))); +} + +template <size_t kLanes, class D> +HWY_INLINE MFromD<D> FirstNPerBlock(D /* tag */) { + const RebindToUnsigned<D> du; + const RebindToSigned<D> di; + constexpr size_t kLanesPerBlock = LanesPerBlock(du); + const auto idx_mod = AndS(Iota0(du), kLanesPerBlock - 1); + return LtS(BitCast(di, idx_mod), static_cast<TFromD<decltype(di)>>(kLanes)); +} + +// vector = f(vector, vector, size_t) +#define HWY_RVV_SLIDE(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME(HWY_RVV_V(BASE, SEW, LMUL) dst, HWY_RVV_V(BASE, SEW, LMUL) src, \ + size_t lanes) { \ + return v##OP##_vx_##CHAR##SEW##LMUL(dst, src, lanes, \ + HWY_RVV_AVL(SEW, SHIFT)); \ + } + +HWY_RVV_FOREACH(HWY_RVV_SLIDE, SlideUp, slideup) +HWY_RVV_FOREACH(HWY_RVV_SLIDE, SlideDown, slidedown) + +#undef HWY_RVV_SLIDE + +} // namespace detail + +// ------------------------------ ConcatUpperLower +template <class D, class V> +HWY_API V ConcatUpperLower(D d, const V hi, const V lo) { + return IfThenElse(FirstN(d, Lanes(d) / 2), lo, hi); +} + +// ------------------------------ ConcatLowerLower +template <class D, class V> +HWY_API V ConcatLowerLower(D d, const V hi, const V lo) { + return detail::SlideUp(lo, hi, Lanes(d) / 2); +} + +// ------------------------------ ConcatUpperUpper +template <class D, class V> +HWY_API V ConcatUpperUpper(D d, const V hi, const V lo) { + // Move upper half into lower + const auto lo_down = detail::SlideDown(lo, lo, Lanes(d) / 2); + return ConcatUpperLower(d, hi, lo_down); +} + +// ------------------------------ ConcatLowerUpper +template <class D, class V> +HWY_API V ConcatLowerUpper(D d, const V hi, const V lo) { + // Move half of both inputs to the other half + const auto hi_up = detail::SlideUp(hi, hi, Lanes(d) / 2); + const auto lo_down = detail::SlideDown(lo, lo, Lanes(d) / 2); + return ConcatUpperLower(d, hi_up, lo_down); +} + +// ------------------------------ Combine + +// TODO(janwas): implement after LMUL ext/trunc +#if 0 + +template <class V> +HWY_API V Combine(const V a, const V b) { + using D = DFromV<V>; + // double LMUL of inputs, then SlideUp with Lanes(). +} + +#endif + +// ------------------------------ ZeroExtendVector + +template <class V> +HWY_API V ZeroExtendVector(const V lo) { + return Combine(Xor(lo, lo), lo); +} + +// ------------------------------ Lower/UpperHalf + +namespace detail { +#define HWY_RVV_TRUNC(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, HALF) NAME(HWY_RVV_V(BASE, SEW, LMUL) v) { \ + return v##OP##_v_##CHAR##SEW##LMUL##_##CHAR##SEW##HALF(v); /* no AVL */ \ + } +HWY_RVV_FOREACH_U08(HWY_RVV_TRUNC, Trunc, lmul_trunc) +HWY_RVV_FOREACH_I08(HWY_RVV_TRUNC, Trunc, lmul_trunc) +HWY_RVV_FOREACH_UI16(HWY_RVV_TRUNC, Trunc, lmul_trunc) +HWY_RVV_FOREACH_UI32(HWY_RVV_TRUNC, Trunc, lmul_trunc) +#if HWY_CAP_FLOAT16 +HWY_RVV_FOREACH_F16(HWY_RVV_TRUNC, Trunc, lmul_trunc) +#endif +HWY_RVV_FOREACH_F32(HWY_RVV_TRUNC, Trunc, lmul_trunc) +#undef HWY_RVV_TRUNC +} // namespace detail + +template <class D, HWY_IF_NOT_LANE_SIZE_D(D, 8)> +HWY_API VFromD<D> LowerHalf(const D /* tag */, const VFromD<D> v) { + return detail::Trunc(v); +} + +// Intrinsics do not provide mf2 for 64-bit T because VLEN might only be 64, +// so "half-vectors" might not exist. However, the application processor profile +// requires VLEN >= 128. Bypass this by casting to 32-bit. +template <class D, HWY_IF_LANE_SIZE_D(D, 8)> +HWY_API VFromD<D> LowerHalf(const D d, const VFromD<D> v) { + const Repartition<uint32_t, D> d32; + return BitCast(d, detail::Trunc(BitCast(Twice<decltype(d32)>(), v))); +} + +template <class D, HWY_IF_NOT_LANE_SIZE_D(D, 8)> +HWY_API VFromD<D> UpperHalf(const D d, const VFromD<D> v) { + return LowerHalf(d, detail::SlideDown(v, v, Lanes(d))); +} + +// ================================================== SWIZZLE + +// ------------------------------ GetLane + +#define HWY_RVV_GET_LANE(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, \ + OP) \ + HWY_API HWY_RVV_T(BASE, SEW) NAME(HWY_RVV_V(BASE, SEW, LMUL) v) { \ + return v##OP##_s_##CHAR##SEW##LMUL##_##CHAR##SEW(v); /* no AVL */ \ + } + +HWY_RVV_FOREACH_UI(HWY_RVV_GET_LANE, GetLane, mv_x) +HWY_RVV_FOREACH_F(HWY_RVV_GET_LANE, GetLane, fmv_f) +#undef HWY_RVV_GET_LANE + +// ------------------------------ OddEven +template <class V> +HWY_API V OddEven(const V a, const V b) { + const RebindToUnsigned<DFromV<V>> du; // Iota0 is unsigned only + const auto is_even = Eq(detail::AndS(detail::Iota0(du), 1), Zero(du)); + return IfThenElse(is_even, b, a); +} + +// ------------------------------ OddEvenBlocks +template <class V> +HWY_API V OddEvenBlocks(const V a, const V b) { + const RebindToUnsigned<DFromV<V>> du; // Iota0 is unsigned only + constexpr size_t kShift = CeilLog2(16 / sizeof(TFromV<V>)); + const auto idx_block = ShiftRight<kShift>(detail::Iota0(du)); + const auto is_even = Eq(detail::AndS(idx_block, 1), Zero(du)); + return IfThenElse(is_even, b, a); +} + +// ------------------------------ SwapAdjacentBlocks + +template <class V> +HWY_API V SwapAdjacentBlocks(const V v) { + const DFromV<V> d; + constexpr size_t kLanesPerBlock = detail::LanesPerBlock(d); + const V down = detail::SlideDown(v, v, kLanesPerBlock); + const V up = detail::SlideUp(v, v, kLanesPerBlock); + return OddEvenBlocks(up, down); +} + +// ------------------------------ TableLookupLanes + +template <class D, class VI> +HWY_API VFromD<RebindToUnsigned<D>> IndicesFromVec(D d, VI vec) { + static_assert(sizeof(TFromD<D>) == sizeof(TFromV<VI>), "Index != lane"); + const RebindToUnsigned<decltype(d)> du; // instead of <D>: avoids unused d. + const auto indices = BitCast(du, vec); +#if HWY_IS_DEBUG_BUILD + HWY_DASSERT(AllTrue(du, Lt(indices, Set(du, Lanes(d))))); +#endif + return indices; +} + +template <class D, typename TI> +HWY_API VFromD<RebindToUnsigned<D>> SetTableIndices(D d, const TI* idx) { + static_assert(sizeof(TFromD<D>) == sizeof(TI), "Index size must match lane"); + return IndicesFromVec(d, LoadU(Rebind<TI, D>(), idx)); +} + +// <32bit are not part of Highway API, but used in Broadcast. This limits VLMAX +// to 2048! We could instead use vrgatherei16. +#define HWY_RVV_TABLE(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME(HWY_RVV_V(BASE, SEW, LMUL) v, HWY_RVV_V(uint, SEW, LMUL) idx) { \ + return v##OP##_vv_##CHAR##SEW##LMUL(v, idx, HWY_RVV_AVL(SEW, SHIFT)); \ + } + +HWY_RVV_FOREACH(HWY_RVV_TABLE, TableLookupLanes, rgather) +#undef HWY_RVV_TABLE + +// ------------------------------ Reverse +template <class D> +HWY_API VFromD<D> Reverse(D /* tag */, VFromD<D> v) { + const RebindToUnsigned<D> du; + using TU = TFromD<decltype(du)>; + const size_t N = Lanes(du); + const auto idx = Sub(Set(du, static_cast<TU>(N - 1)), detail::Iota0(du)); + return TableLookupLanes(v, idx); +} + +// ------------------------------ Compress + +#define HWY_RVV_COMPRESS(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, \ + OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME(HWY_RVV_V(BASE, SEW, LMUL) v, HWY_RVV_M(MLEN) mask) { \ + return v##OP##_vm_##CHAR##SEW##LMUL(mask, v, v, HWY_RVV_AVL(SEW, SHIFT)); \ + } + +HWY_RVV_FOREACH_UI16(HWY_RVV_COMPRESS, Compress, compress) +HWY_RVV_FOREACH_UI32(HWY_RVV_COMPRESS, Compress, compress) +HWY_RVV_FOREACH_UI64(HWY_RVV_COMPRESS, Compress, compress) +HWY_RVV_FOREACH_F(HWY_RVV_COMPRESS, Compress, compress) +#undef HWY_RVV_COMPRESS + +// ------------------------------ CompressStore +template <class V, class M, class D> +HWY_API size_t CompressStore(const V v, const M mask, const D d, + TFromD<D>* HWY_RESTRICT unaligned) { + StoreU(Compress(v, mask), d, unaligned); + return CountTrue(d, mask); +} + +// ------------------------------ CompressBlendedStore +template <class V, class M, class D> +HWY_API size_t CompressBlendedStore(const V v, const M mask, const D d, + TFromD<D>* HWY_RESTRICT unaligned) { + const size_t count = CountTrue(d, mask); + detail::StoreN(count, Compress(v, mask), d, unaligned); + return count; +} + +// ================================================== BLOCKWISE + +// ------------------------------ CombineShiftRightBytes +template <size_t kBytes, class D, class V = VFromD<D>> +HWY_API V CombineShiftRightBytes(const D d, const V hi, V lo) { + const Repartition<uint8_t, decltype(d)> d8; + const auto hi8 = BitCast(d8, hi); + const auto lo8 = BitCast(d8, lo); + const auto hi_up = detail::SlideUp(hi8, hi8, 16 - kBytes); + const auto lo_down = detail::SlideDown(lo8, lo8, kBytes); + const auto is_lo = detail::FirstNPerBlock<16 - kBytes>(d8); + return BitCast(d, IfThenElse(is_lo, lo_down, hi_up)); +} + +// ------------------------------ CombineShiftRightLanes +template <size_t kLanes, class D, class V = VFromD<D>> +HWY_API V CombineShiftRightLanes(const D d, const V hi, V lo) { + constexpr size_t kLanesUp = 16 / sizeof(TFromV<V>) - kLanes; + const auto hi_up = detail::SlideUp(hi, hi, kLanesUp); + const auto lo_down = detail::SlideDown(lo, lo, kLanes); + const auto is_lo = detail::FirstNPerBlock<kLanesUp>(d); + return IfThenElse(is_lo, lo_down, hi_up); +} + +// ------------------------------ Shuffle2301 (ShiftLeft) +template <class V> +HWY_API V Shuffle2301(const V v) { + const DFromV<V> d; + static_assert(sizeof(TFromD<decltype(d)>) == 4, "Defined for 32-bit types"); + const Repartition<uint64_t, decltype(d)> du64; + const auto v64 = BitCast(du64, v); + return BitCast(d, Or(ShiftRight<32>(v64), ShiftLeft<32>(v64))); +} + +// ------------------------------ Shuffle2103 +template <class V> +HWY_API V Shuffle2103(const V v) { + const DFromV<V> d; + static_assert(sizeof(TFromD<decltype(d)>) == 4, "Defined for 32-bit types"); + return CombineShiftRightLanes<3>(d, v, v); +} + +// ------------------------------ Shuffle0321 +template <class V> +HWY_API V Shuffle0321(const V v) { + const DFromV<V> d; + static_assert(sizeof(TFromD<decltype(d)>) == 4, "Defined for 32-bit types"); + return CombineShiftRightLanes<1>(d, v, v); +} + +// ------------------------------ Shuffle1032 +template <class V> +HWY_API V Shuffle1032(const V v) { + const DFromV<V> d; + static_assert(sizeof(TFromD<decltype(d)>) == 4, "Defined for 32-bit types"); + return CombineShiftRightLanes<2>(d, v, v); +} + +// ------------------------------ Shuffle01 +template <class V> +HWY_API V Shuffle01(const V v) { + const DFromV<V> d; + static_assert(sizeof(TFromD<decltype(d)>) == 8, "Defined for 64-bit types"); + return CombineShiftRightLanes<1>(d, v, v); +} + +// ------------------------------ Shuffle0123 +template <class V> +HWY_API V Shuffle0123(const V v) { + return Shuffle2301(Shuffle1032(v)); +} + +// ------------------------------ TableLookupBytes + +template <class V, class VI> +HWY_API VI TableLookupBytes(const V v, const VI idx) { + const DFromV<VI> d; + const Repartition<uint8_t, decltype(d)> d8; + const auto offsets128 = detail::OffsetsOf128BitBlocks(d8, detail::Iota0(d8)); + const auto idx8 = Add(BitCast(d8, idx), offsets128); + return BitCast(d, TableLookupLanes(BitCast(d8, v), idx8)); +} + +template <class V, class VI> +HWY_API VI TableLookupBytesOr0(const V v, const VI idx) { + const DFromV<VI> d; + // Mask size must match vector type, so cast everything to this type. + const Repartition<int8_t, decltype(d)> di8; + const auto lookup = TableLookupBytes(BitCast(di8, v), BitCast(di8, idx)); + const auto msb = Lt(BitCast(di8, idx), Zero(di8)); + return BitCast(d, IfThenZeroElse(msb, lookup)); +} + +// ------------------------------ Broadcast +template <int kLane, class V> +HWY_API V Broadcast(const V v) { + const DFromV<V> d; + constexpr size_t kLanesPerBlock = detail::LanesPerBlock(d); + static_assert(0 <= kLane && kLane < kLanesPerBlock, "Invalid lane"); + auto idx = detail::OffsetsOf128BitBlocks(d, detail::Iota0(d)); + if (kLane != 0) { + idx = detail::AddS(idx, kLane); + } + return TableLookupLanes(v, idx); +} + +// ------------------------------ ShiftLeftLanes + +template <size_t kLanes, class D, class V = VFromD<D>> +HWY_API V ShiftLeftLanes(const D d, const V v) { + const RebindToSigned<decltype(d)> di; + const auto shifted = detail::SlideUp(v, v, kLanes); + // Match x86 semantics by zeroing lower lanes in 128-bit blocks + constexpr size_t kLanesPerBlock = detail::LanesPerBlock(di); + const auto idx_mod = detail::AndS(detail::Iota0(di), kLanesPerBlock - 1); + const auto clear = Lt(BitCast(di, idx_mod), Set(di, kLanes)); + return IfThenZeroElse(clear, shifted); +} + +template <size_t kLanes, class V> +HWY_API V ShiftLeftLanes(const V v) { + return ShiftLeftLanes<kLanes>(DFromV<V>(), v); +} + +// ------------------------------ ShiftLeftBytes + +template <int kBytes, class V> +HWY_API V ShiftLeftBytes(DFromV<V> d, const V v) { + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, ShiftLeftLanes<kBytes>(BitCast(d8, v))); +} + +template <int kBytes, class V> +HWY_API V ShiftLeftBytes(const V v) { + return ShiftLeftBytes<kBytes>(DFromV<V>(), v); +} + +// ------------------------------ ShiftRightLanes +template <size_t kLanes, typename T, size_t N, class V = VFromD<Simd<T, N>>> +HWY_API V ShiftRightLanes(const Simd<T, N> d, V v) { + const RebindToSigned<decltype(d)> di; + // For partial vectors, clear upper lanes so we shift in zeros. + if (N <= 16 / sizeof(T)) { + v = IfThenElseZero(FirstN(d, N), v); + } + + const auto shifted = detail::SlideDown(v, v, kLanes); + // Match x86 semantics by zeroing upper lanes in 128-bit blocks + constexpr size_t kLanesPerBlock = detail::LanesPerBlock(di); + const auto idx_mod = detail::AndS(detail::Iota0(di), kLanesPerBlock - 1); + const auto keep = Lt(BitCast(di, idx_mod), Set(di, kLanesPerBlock - kLanes)); + return IfThenElseZero(keep, shifted); +} + +// ------------------------------ ShiftRightBytes +template <int kBytes, class D, class V = VFromD<D>> +HWY_API V ShiftRightBytes(const D d, const V v) { + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, ShiftRightLanes<kBytes>(d8, BitCast(d8, v))); +} + +// ------------------------------ InterleaveLower + +// TODO(janwas): PromoteTo(LowerHalf), slide1up, add +template <class D, class V> +HWY_API V InterleaveLower(D d, const V a, const V b) { + static_assert(IsSame<TFromD<D>, TFromV<V>>(), "D/V mismatch"); + const RebindToUnsigned<decltype(d)> du; + constexpr size_t kLanesPerBlock = detail::LanesPerBlock(du); + const auto i = detail::Iota0(du); + const auto idx_mod = ShiftRight<1>(detail::AndS(i, kLanesPerBlock - 1)); + const auto idx = Add(idx_mod, detail::OffsetsOf128BitBlocks(d, i)); + const auto is_even = Eq(detail::AndS(i, 1), Zero(du)); + return IfThenElse(is_even, TableLookupLanes(a, idx), + TableLookupLanes(b, idx)); +} + +template <class V> +HWY_API V InterleaveLower(const V a, const V b) { + return InterleaveLower(DFromV<V>(), a, b); +} + +// ------------------------------ InterleaveUpper + +template <class D, class V> +HWY_API V InterleaveUpper(const D d, const V a, const V b) { + static_assert(IsSame<TFromD<D>, TFromV<V>>(), "D/V mismatch"); + const RebindToUnsigned<decltype(d)> du; + constexpr size_t kLanesPerBlock = detail::LanesPerBlock(du); + const auto i = detail::Iota0(du); + const auto idx_mod = ShiftRight<1>(detail::AndS(i, kLanesPerBlock - 1)); + const auto idx_lower = Add(idx_mod, detail::OffsetsOf128BitBlocks(d, i)); + const auto idx = detail::AddS(idx_lower, kLanesPerBlock / 2); + const auto is_even = Eq(detail::AndS(i, 1), Zero(du)); + return IfThenElse(is_even, TableLookupLanes(a, idx), + TableLookupLanes(b, idx)); +} + +// ------------------------------ ZipLower + +template <class V, class DW = RepartitionToWide<DFromV<V>>> +HWY_API VFromD<DW> ZipLower(DW dw, V a, V b) { + const RepartitionToNarrow<DW> dn; + static_assert(IsSame<TFromD<decltype(dn)>, TFromV<V>>(), "D/V mismatch"); + return BitCast(dw, InterleaveLower(dn, a, b)); +} + +template <class V, class DW = RepartitionToWide<DFromV<V>>> +HWY_API VFromD<DW> ZipLower(V a, V b) { + return BitCast(DW(), InterleaveLower(a, b)); +} + +// ------------------------------ ZipUpper +template <class DW, class V> +HWY_API VFromD<DW> ZipUpper(DW dw, V a, V b) { + const RepartitionToNarrow<DW> dn; + static_assert(IsSame<TFromD<decltype(dn)>, TFromV<V>>(), "D/V mismatch"); + return BitCast(dw, InterleaveUpper(dn, a, b)); +} + +// ================================================== REDUCE + +// vector = f(vector, zero_m1) +#define HWY_RVV_REDUCE(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) \ + NAME(HWY_RVV_V(BASE, SEW, LMUL) v, HWY_RVV_V(BASE, SEW, m1) v0) { \ + return Set(HWY_RVV_D(CHAR, SEW, LMUL)(), \ + GetLane(v##OP##_vs_##CHAR##SEW##LMUL##_##CHAR##SEW##m1( \ + v0, v, v0, HWY_RVV_AVL(SEW, SHIFT)))); \ + } + +// ------------------------------ SumOfLanes + +namespace detail { +HWY_RVV_FOREACH_UI(HWY_RVV_REDUCE, RedSum, redsum) +HWY_RVV_FOREACH_F(HWY_RVV_REDUCE, RedSum, fredusum) +} // namespace detail + +template <class D> +HWY_API VFromD<D> SumOfLanes(D /* d */, const VFromD<D> v) { + const auto v0 = Zero(Full<TFromD<D>>()); // always m1 + return detail::RedSum(v, v0); +} + +// ------------------------------ MinOfLanes +namespace detail { +HWY_RVV_FOREACH_U(HWY_RVV_REDUCE, RedMin, redminu) +HWY_RVV_FOREACH_I(HWY_RVV_REDUCE, RedMin, redmin) +HWY_RVV_FOREACH_F(HWY_RVV_REDUCE, RedMin, fredmin) +} // namespace detail + +template <class D> +HWY_API VFromD<D> MinOfLanes(D /* d */, const VFromD<D> v) { + using T = TFromD<D>; + const Full<T> d1; // always m1 + const auto neutral = Set(d1, HighestValue<T>()); + return detail::RedMin(v, neutral); +} + +// ------------------------------ MaxOfLanes +namespace detail { +HWY_RVV_FOREACH_U(HWY_RVV_REDUCE, RedMax, redmaxu) +HWY_RVV_FOREACH_I(HWY_RVV_REDUCE, RedMax, redmax) +HWY_RVV_FOREACH_F(HWY_RVV_REDUCE, RedMax, fredmax) +} // namespace detail + +template <class D> +HWY_API VFromD<D> MaxOfLanes(D /* d */, const VFromD<D> v) { + using T = TFromD<D>; + const Full<T> d1; // always m1 + const auto neutral = Set(d1, LowestValue<T>()); + return detail::RedMax(v, neutral); +} + +#undef HWY_RVV_REDUCE + +// ================================================== Ops with dependencies + +// ------------------------------ LoadDup128 + +template <class D> +HWY_API VFromD<D> LoadDup128(D d, const TFromD<D>* const HWY_RESTRICT p) { + const auto loaded = Load(d, p); + constexpr size_t kLanesPerBlock = detail::LanesPerBlock(d); + // Broadcast the first block + const auto idx = detail::AndS(detail::Iota0(d), kLanesPerBlock - 1); + return TableLookupLanes(loaded, idx); +} + +// ------------------------------ StoreMaskBits +#define HWY_RVV_STORE_MASK_BITS(SEW, SHIFT, MLEN, NAME, OP) \ + /* DEPRECATED */ \ + HWY_API size_t StoreMaskBits(HWY_RVV_M(MLEN) m, uint8_t* bits) { \ + /* LMUL=1 is always enough */ \ + Full<uint8_t> d8; \ + const size_t num_bytes = (Lanes(d8) + MLEN - 1) / MLEN; \ + /* TODO(janwas): how to convert vbool* to vuint?*/ \ + /*Store(m, d8, bits);*/ \ + (void)m; \ + (void)bits; \ + return num_bytes; \ + } \ + template <class D> \ + HWY_API size_t StoreMaskBits(D /* tag */, HWY_RVV_M(MLEN) m, \ + uint8_t* bits) { \ + return StoreMaskBits(m, bits); \ + } +HWY_RVV_FOREACH_B(HWY_RVV_STORE_MASK_BITS, _, _) +#undef HWY_RVV_STORE_MASK_BITS + +// ------------------------------ FirstN (Iota0, Lt, RebindMask, SlideUp) + +// Disallow for 8-bit because Iota is likely to overflow. +template <class D, HWY_IF_NOT_LANE_SIZE_D(D, 1)> +HWY_API MFromD<D> FirstN(const D d, const size_t n) { + const RebindToSigned<D> di; + using TI = TFromD<decltype(di)>; + return RebindMask( + d, Lt(BitCast(di, detail::Iota0(d)), Set(di, static_cast<TI>(n)))); +} + +template <class D, HWY_IF_LANE_SIZE_D(D, 1)> +HWY_API MFromD<D> FirstN(const D d, const size_t n) { + const auto zero = Zero(d); + const auto one = Set(d, 1); + return Eq(detail::SlideUp(one, zero, n), one); +} + +// ------------------------------ Neg (Sub) + +template <class V, HWY_IF_SIGNED_V(V)> +HWY_API V Neg(const V v) { + return Sub(Zero(DFromV<V>()), v); +} + +// vector = f(vector), but argument is repeated +#define HWY_RVV_RETV_ARGV2(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, \ + OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) NAME(HWY_RVV_V(BASE, SEW, LMUL) v) { \ + return v##OP##_vv_##CHAR##SEW##LMUL(v, v, HWY_RVV_AVL(SEW, SHIFT)); \ + } + +HWY_RVV_FOREACH_F(HWY_RVV_RETV_ARGV2, Neg, fsgnjn) + +// ------------------------------ Abs (Max, Neg) + +template <class V, HWY_IF_SIGNED_V(V)> +HWY_API V Abs(const V v) { + return Max(v, Neg(v)); +} + +HWY_RVV_FOREACH_F(HWY_RVV_RETV_ARGV2, Abs, fsgnjx) + +#undef HWY_RVV_RETV_ARGV2 + +// ------------------------------ AbsDiff (Abs, Sub) +template <class V> +HWY_API V AbsDiff(const V a, const V b) { + return Abs(Sub(a, b)); +} + +// ------------------------------ Round (NearestInt, ConvertTo, CopySign) + +// IEEE-754 roundToIntegralTiesToEven returns floating-point, but we do not have +// a dedicated instruction for that. Rounding to integer and converting back to +// float is correct except when the input magnitude is large, in which case the +// input was already an integer (because mantissa >> exponent is zero). + +namespace detail { +enum RoundingModes { kNear, kTrunc, kDown, kUp }; + +template <class V> +HWY_INLINE auto UseInt(const V v) -> decltype(MaskFromVec(v)) { + return Lt(Abs(v), Set(DFromV<V>(), MantissaEnd<TFromV<V>>())); +} + +} // namespace detail + +template <class V> +HWY_API V Round(const V v) { + const DFromV<V> df; + + const auto integer = NearestInt(v); // round using current mode + const auto int_f = ConvertTo(df, integer); + + return IfThenElse(detail::UseInt(v), CopySign(int_f, v), v); +} + +// ------------------------------ Trunc (ConvertTo) +template <class V> +HWY_API V Trunc(const V v) { + const DFromV<V> df; + const RebindToSigned<decltype(df)> di; + + const auto integer = ConvertTo(di, v); // round toward 0 + const auto int_f = ConvertTo(df, integer); + + return IfThenElse(detail::UseInt(v), CopySign(int_f, v), v); +} + +// ------------------------------ Ceil +template <class V> +HWY_API V Ceil(const V v) { + asm volatile("fsrm %0" ::"r"(detail::kUp)); + const auto ret = Round(v); + asm volatile("fsrm %0" ::"r"(detail::kNear)); + return ret; +} + +// ------------------------------ Floor +template <class V> +HWY_API V Floor(const V v) { + asm volatile("fsrm %0" ::"r"(detail::kDown)); + const auto ret = Round(v); + asm volatile("fsrm %0" ::"r"(detail::kNear)); + return ret; +} + +// ------------------------------ Iota (ConvertTo) + +template <class D, HWY_IF_UNSIGNED_D(D)> +HWY_API VFromD<D> Iota(const D d, TFromD<D> first) { + return Add(detail::Iota0(d), Set(d, first)); +} + +template <class D, HWY_IF_SIGNED_D(D)> +HWY_API VFromD<D> Iota(const D d, TFromD<D> first) { + const RebindToUnsigned<D> du; + return Add(BitCast(d, detail::Iota0(du)), Set(d, first)); +} + +template <class D, HWY_IF_FLOAT_D(D)> +HWY_API VFromD<D> Iota(const D d, TFromD<D> first) { + const RebindToUnsigned<D> du; + const RebindToSigned<D> di; + return detail::AddS(ConvertTo(d, BitCast(di, detail::Iota0(du))), first); +} + +// ------------------------------ MulEven/Odd (Mul, OddEven) + +namespace detail { +// Special instruction for 1 lane is presumably faster? +#define HWY_RVV_SLIDE1(BASE, CHAR, SEW, LMUL, X2, HALF, SHIFT, MLEN, NAME, OP) \ + HWY_API HWY_RVV_V(BASE, SEW, LMUL) NAME(HWY_RVV_V(BASE, SEW, LMUL) v) { \ + return v##OP##_vx_##CHAR##SEW##LMUL(v, 0, HWY_RVV_AVL(SEW, SHIFT)); \ + } + +HWY_RVV_FOREACH_UI32(HWY_RVV_SLIDE1, Slide1Up, slide1up) +HWY_RVV_FOREACH_U64(HWY_RVV_SLIDE1, Slide1Up, slide1up) +HWY_RVV_FOREACH_UI32(HWY_RVV_SLIDE1, Slide1Down, slide1down) +HWY_RVV_FOREACH_U64(HWY_RVV_SLIDE1, Slide1Down, slide1down) +#undef HWY_RVV_SLIDE1 +} // namespace detail + +template <class V, HWY_IF_LANE_SIZE_V(V, 4)> +HWY_API VFromD<RepartitionToWide<DFromV<V>>> MulEven(const V a, const V b) { + const auto lo = Mul(a, b); + const auto hi = detail::MulHigh(a, b); + const RepartitionToWide<DFromV<V>> dw; + return BitCast(dw, OddEven(detail::Slide1Up(hi), lo)); +} + +// There is no 64x64 vwmul. +template <class V, HWY_IF_LANE_SIZE_V(V, 8)> +HWY_INLINE V MulEven(const V a, const V b) { + const auto lo = detail::Mul(a, b); + const auto hi = detail::MulHigh(a, b); + return OddEven(detail::Slide1Up(hi), lo); +} + +template <class V, HWY_IF_LANE_SIZE_V(V, 8)> +HWY_INLINE V MulOdd(const V a, const V b) { + const auto lo = detail::Mul(a, b); + const auto hi = detail::MulHigh(a, b); + return OddEven(hi, detail::Slide1Down(lo)); +} + +// ------------------------------ ReorderDemote2To (OddEven) + +template <size_t N, class DF = Simd<float, N / 2>> +HWY_API VFromD<Simd<uint16_t, N>> ReorderDemote2To(Simd<bfloat16_t, N> dbf16, + VFromD<DF> a, VFromD<DF> b) { + const RebindToUnsigned<decltype(dbf16)> du16; + const RebindToUnsigned<DF> du32; + const VFromD<decltype(du32)> b_in_even = ShiftRight<16>(BitCast(du32, b)); + return BitCast(dbf16, OddEven(BitCast(du16, a), BitCast(du16, b_in_even))); +} + +// ------------------------------ ReorderWidenMulAccumulate (MulAdd, ZipLower) + +template <size_t N, class DU16 = Simd<uint16_t, N * 2>> +HWY_API auto ReorderWidenMulAccumulate(Simd<float, N> df32, VFromD<DU16> a, + VFromD<DU16> b, + const VFromD<decltype(df32)> sum0, + VFromD<decltype(df32)>& sum1) + -> VFromD<decltype(df32)> { + const DU16 du16; + const RebindToUnsigned<decltype(df32)> du32; + using VU32 = VFromD<decltype(du32)>; + const VFromD<DU16> zero = Zero(du16); + const VU32 a0 = ZipLower(du32, zero, BitCast(du16, a)); + const VU32 a1 = ZipUpper(du32, zero, BitCast(du16, a)); + const VU32 b0 = ZipLower(du32, zero, BitCast(du16, b)); + const VU32 b1 = ZipUpper(du32, zero, BitCast(du16, b)); + sum1 = MulAdd(BitCast(df32, a1), BitCast(df32, b1), sum1); + return MulAdd(BitCast(df32, a0), BitCast(df32, b0), sum0); +} + +// ================================================== END MACROS +namespace detail { // for code folding +#undef HWY_IF_FLOAT_V +#undef HWY_IF_SIGNED_V +#undef HWY_IF_UNSIGNED_V + +#undef HWY_RVV_FOREACH +#undef HWY_RVV_FOREACH_08 +#undef HWY_RVV_FOREACH_16 +#undef HWY_RVV_FOREACH_32 +#undef HWY_RVV_FOREACH_64 +#undef HWY_RVV_FOREACH_B +#undef HWY_RVV_FOREACH_F +#undef HWY_RVV_FOREACH_F32 +#undef HWY_RVV_FOREACH_F64 +#undef HWY_RVV_FOREACH_I +#undef HWY_RVV_FOREACH_I08 +#undef HWY_RVV_FOREACH_I16 +#undef HWY_RVV_FOREACH_I32 +#undef HWY_RVV_FOREACH_I64 +#undef HWY_RVV_FOREACH_U +#undef HWY_RVV_FOREACH_U08 +#undef HWY_RVV_FOREACH_U16 +#undef HWY_RVV_FOREACH_U32 +#undef HWY_RVV_FOREACH_U64 +#undef HWY_RVV_FOREACH_UI +#undef HWY_RVV_FOREACH_UI16 +#undef HWY_RVV_FOREACH_UI32 +#undef HWY_RVV_FOREACH_UI64 + +#undef HWY_RVV_RETV_ARGV +#undef HWY_RVV_RETV_ARGVS +#undef HWY_RVV_RETV_ARGVV + +#undef HWY_RVV_T +#undef HWY_RVV_D +#undef HWY_RVV_V +#undef HWY_RVV_M + +} // namespace detail +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); diff --git a/third_party/highway/hwy/ops/scalar-inl.h b/third_party/highway/hwy/ops/scalar-inl.h new file mode 100644 index 0000000..3e7758f --- /dev/null +++ b/third_party/highway/hwy/ops/scalar-inl.h @@ -0,0 +1,1405 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Single-element vectors and operations. +// External include guard in highway.h - see comment there. + +#include <stddef.h> +#include <stdint.h> + +#include "hwy/base.h" +#include "hwy/ops/shared-inl.h" + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +// Single instruction, single data. +template <typename T> +using Sisd = Simd<T, 1>; + +// (Wrapper class required for overloading comparison operators.) +template <typename T> +struct Vec1 { + HWY_INLINE Vec1() = default; + Vec1(const Vec1&) = default; + Vec1& operator=(const Vec1&) = default; + HWY_INLINE explicit Vec1(const T t) : raw(t) {} + + HWY_INLINE Vec1& operator*=(const Vec1 other) { + return *this = (*this * other); + } + HWY_INLINE Vec1& operator/=(const Vec1 other) { + return *this = (*this / other); + } + HWY_INLINE Vec1& operator+=(const Vec1 other) { + return *this = (*this + other); + } + HWY_INLINE Vec1& operator-=(const Vec1 other) { + return *this = (*this - other); + } + HWY_INLINE Vec1& operator&=(const Vec1 other) { + return *this = (*this & other); + } + HWY_INLINE Vec1& operator|=(const Vec1 other) { + return *this = (*this | other); + } + HWY_INLINE Vec1& operator^=(const Vec1 other) { + return *this = (*this ^ other); + } + + T raw; +}; + +// 0 or FF..FF, same size as Vec1. +template <typename T> +class Mask1 { + using Raw = hwy::MakeUnsigned<T>; + + public: + static HWY_INLINE Mask1<T> FromBool(bool b) { + Mask1<T> mask; + mask.bits = b ? ~Raw(0) : 0; + return mask; + } + + Raw bits; +}; + +namespace detail { + +// Deduce Sisd<T> from Vec1<T> +struct Deduce1 { + template <typename T> + Sisd<T> operator()(Vec1<T>) const { + return Sisd<T>(); + } +}; + +} // namespace detail + +template <class V> +using DFromV = decltype(detail::Deduce1()(V())); + +template <class V> +using TFromV = TFromD<DFromV<V>>; + +// ------------------------------ BitCast + +template <typename T, typename FromT> +HWY_API Vec1<T> BitCast(Sisd<T> /* tag */, Vec1<FromT> v) { + static_assert(sizeof(T) <= sizeof(FromT), "Promoting is undefined"); + T to; + CopyBytes<sizeof(FromT)>(&v.raw, &to); + return Vec1<T>(to); +} + +// ------------------------------ Set + +template <typename T> +HWY_API Vec1<T> Zero(Sisd<T> /* tag */) { + return Vec1<T>(T(0)); +} + +template <typename T, typename T2> +HWY_API Vec1<T> Set(Sisd<T> /* tag */, const T2 t) { + return Vec1<T>(static_cast<T>(t)); +} + +template <typename T> +HWY_API Vec1<T> Undefined(Sisd<T> d) { + return Zero(d); +} + +template <typename T, typename T2> +HWY_API Vec1<T> Iota(const Sisd<T> /* tag */, const T2 first) { + return Vec1<T>(static_cast<T>(first)); +} + +// ================================================== LOGICAL + +// ------------------------------ Not + +template <typename T> +HWY_API Vec1<T> Not(const Vec1<T> v) { + using TU = MakeUnsigned<T>; + const Sisd<TU> du; + return BitCast(Sisd<T>(), Vec1<TU>(static_cast<TU>(~BitCast(du, v).raw))); +} + +// ------------------------------ And + +template <typename T> +HWY_API Vec1<T> And(const Vec1<T> a, const Vec1<T> b) { + using TU = MakeUnsigned<T>; + const Sisd<TU> du; + return BitCast(Sisd<T>(), Vec1<TU>(BitCast(du, a).raw & BitCast(du, b).raw)); +} +template <typename T> +HWY_API Vec1<T> operator&(const Vec1<T> a, const Vec1<T> b) { + return And(a, b); +} + +// ------------------------------ AndNot + +template <typename T> +HWY_API Vec1<T> AndNot(const Vec1<T> a, const Vec1<T> b) { + using TU = MakeUnsigned<T>; + const Sisd<TU> du; + return BitCast(Sisd<T>(), Vec1<TU>(static_cast<TU>(~BitCast(du, a).raw & + BitCast(du, b).raw))); +} + +// ------------------------------ Or + +template <typename T> +HWY_API Vec1<T> Or(const Vec1<T> a, const Vec1<T> b) { + using TU = MakeUnsigned<T>; + const Sisd<TU> du; + return BitCast(Sisd<T>(), Vec1<TU>(BitCast(du, a).raw | BitCast(du, b).raw)); +} +template <typename T> +HWY_API Vec1<T> operator|(const Vec1<T> a, const Vec1<T> b) { + return Or(a, b); +} + +// ------------------------------ Xor + +template <typename T> +HWY_API Vec1<T> Xor(const Vec1<T> a, const Vec1<T> b) { + using TU = MakeUnsigned<T>; + const Sisd<TU> du; + return BitCast(Sisd<T>(), Vec1<TU>(BitCast(du, a).raw ^ BitCast(du, b).raw)); +} +template <typename T> +HWY_API Vec1<T> operator^(const Vec1<T> a, const Vec1<T> b) { + return Xor(a, b); +} + +// ------------------------------ CopySign + +template <typename T> +HWY_API Vec1<T> CopySign(const Vec1<T> magn, const Vec1<T> sign) { + static_assert(IsFloat<T>(), "Only makes sense for floating-point"); + const auto msb = SignBit(Sisd<T>()); + return Or(AndNot(msb, magn), And(msb, sign)); +} + +template <typename T> +HWY_API Vec1<T> CopySignToAbs(const Vec1<T> abs, const Vec1<T> sign) { + static_assert(IsFloat<T>(), "Only makes sense for floating-point"); + return Or(abs, And(SignBit(Sisd<T>()), sign)); +} + +// ------------------------------ BroadcastSignBit + +template <typename T> +HWY_API Vec1<T> BroadcastSignBit(const Vec1<T> v) { + // This is used inside ShiftRight, so we cannot implement in terms of it. + return v.raw < 0 ? Vec1<T>(T(-1)) : Vec1<T>(0); +} + +// ------------------------------ PopulationCount + +#ifdef HWY_NATIVE_POPCNT +#undef HWY_NATIVE_POPCNT +#else +#define HWY_NATIVE_POPCNT +#endif + +template <typename T> +HWY_API Vec1<T> PopulationCount(Vec1<T> v) { + return Vec1<T>(static_cast<T>(PopCount(v.raw))); +} + +// ------------------------------ Mask + +template <typename TFrom, typename TTo> +HWY_API Mask1<TTo> RebindMask(Sisd<TTo> /*tag*/, Mask1<TFrom> m) { + static_assert(sizeof(TFrom) == sizeof(TTo), "Must have same size"); + return Mask1<TTo>{m.bits}; +} + +// v must be 0 or FF..FF. +template <typename T> +HWY_API Mask1<T> MaskFromVec(const Vec1<T> v) { + Mask1<T> mask; + CopyBytes<sizeof(mask.bits)>(&v.raw, &mask.bits); + return mask; +} + +template <typename T> +Vec1<T> VecFromMask(const Mask1<T> mask) { + Vec1<T> v; + CopyBytes<sizeof(v.raw)>(&mask.bits, &v.raw); + return v; +} + +template <typename T> +Vec1<T> VecFromMask(Sisd<T> /* tag */, const Mask1<T> mask) { + Vec1<T> v; + CopyBytes<sizeof(v.raw)>(&mask.bits, &v.raw); + return v; +} + +template <typename T> +HWY_API Mask1<T> FirstN(Sisd<T> /*tag*/, size_t n) { + return Mask1<T>::FromBool(n != 0); +} + +// Returns mask ? yes : no. +template <typename T> +HWY_API Vec1<T> IfThenElse(const Mask1<T> mask, const Vec1<T> yes, + const Vec1<T> no) { + return mask.bits ? yes : no; +} + +template <typename T> +HWY_API Vec1<T> IfThenElseZero(const Mask1<T> mask, const Vec1<T> yes) { + return mask.bits ? yes : Vec1<T>(0); +} + +template <typename T> +HWY_API Vec1<T> IfThenZeroElse(const Mask1<T> mask, const Vec1<T> no) { + return mask.bits ? Vec1<T>(0) : no; +} + +template <typename T> +HWY_API Vec1<T> ZeroIfNegative(const Vec1<T> v) { + return v.raw < 0 ? Vec1<T>(0) : v; +} + +// ------------------------------ Mask logical + +template <typename T> +HWY_API Mask1<T> Not(const Mask1<T> m) { + return MaskFromVec(Not(VecFromMask(Sisd<T>(), m))); +} + +template <typename T> +HWY_API Mask1<T> And(const Mask1<T> a, Mask1<T> b) { + const Sisd<T> d; + return MaskFromVec(And(VecFromMask(d, a), VecFromMask(d, b))); +} + +template <typename T> +HWY_API Mask1<T> AndNot(const Mask1<T> a, Mask1<T> b) { + const Sisd<T> d; + return MaskFromVec(AndNot(VecFromMask(d, a), VecFromMask(d, b))); +} + +template <typename T> +HWY_API Mask1<T> Or(const Mask1<T> a, Mask1<T> b) { + const Sisd<T> d; + return MaskFromVec(Or(VecFromMask(d, a), VecFromMask(d, b))); +} + +template <typename T> +HWY_API Mask1<T> Xor(const Mask1<T> a, Mask1<T> b) { + const Sisd<T> d; + return MaskFromVec(Xor(VecFromMask(d, a), VecFromMask(d, b))); +} + +// ================================================== SHIFTS + +// ------------------------------ ShiftLeft/ShiftRight (BroadcastSignBit) + +template <int kBits, typename T> +HWY_API Vec1<T> ShiftLeft(const Vec1<T> v) { + static_assert(0 <= kBits && kBits < sizeof(T) * 8, "Invalid shift"); + return Vec1<T>(static_cast<hwy::MakeUnsigned<T>>(v.raw) << kBits); +} + +template <int kBits, typename T> +HWY_API Vec1<T> ShiftRight(const Vec1<T> v) { + static_assert(0 <= kBits && kBits < sizeof(T) * 8, "Invalid shift"); +#if __cplusplus >= 202002L + // Signed right shift is now guaranteed to be arithmetic (rounding toward + // negative infinity, i.e. shifting in the sign bit). + return Vec1<T>(v.raw >> kBits); +#else + if (IsSigned<T>()) { + // Emulate arithmetic shift using only logical (unsigned) shifts, because + // signed shifts are still implementation-defined. + using TU = hwy::MakeUnsigned<T>; + const Sisd<TU> du; + const TU shifted = BitCast(du, v).raw >> kBits; + const TU sign = BitCast(du, BroadcastSignBit(v)).raw; + const TU upper = sign << (sizeof(TU) * 8 - 1 - kBits); + return BitCast(Sisd<T>(), Vec1<TU>(shifted | upper)); + } else { + return Vec1<T>(v.raw >> kBits); // unsigned, logical shift + } +#endif +} + +// ------------------------------ RotateRight (ShiftRight) + +template <int kBits, typename T> +HWY_API Vec1<T> RotateRight(const Vec1<T> v) { + static_assert(0 <= kBits && kBits < sizeof(T) * 8, "Invalid shift"); + if (kBits == 0) return v; + return Or(ShiftRight<kBits>(v), ShiftLeft<sizeof(T) * 8 - kBits>(v)); +} + +// ------------------------------ ShiftLeftSame (BroadcastSignBit) + +template <typename T> +HWY_API Vec1<T> ShiftLeftSame(const Vec1<T> v, int bits) { + return Vec1<T>(static_cast<hwy::MakeUnsigned<T>>(v.raw) << bits); +} + +template <typename T> +HWY_API Vec1<T> ShiftRightSame(const Vec1<T> v, int bits) { +#if __cplusplus >= 202002L + // Signed right shift is now guaranteed to be arithmetic (rounding toward + // negative infinity, i.e. shifting in the sign bit). + return Vec1<T>(v.raw >> bits); +#else + if (IsSigned<T>()) { + // Emulate arithmetic shift using only logical (unsigned) shifts, because + // signed shifts are still implementation-defined. + using TU = hwy::MakeUnsigned<T>; + const Sisd<TU> du; + const TU shifted = BitCast(du, v).raw >> bits; + const TU sign = BitCast(du, BroadcastSignBit(v)).raw; + const TU upper = sign << (sizeof(TU) * 8 - 1 - bits); + return BitCast(Sisd<T>(), Vec1<TU>(shifted | upper)); + } else { + return Vec1<T>(v.raw >> bits); // unsigned, logical shift + } +#endif +} + +// ------------------------------ Shl + +// Single-lane => same as ShiftLeftSame except for the argument type. +template <typename T> +HWY_API Vec1<T> operator<<(const Vec1<T> v, const Vec1<T> bits) { + return ShiftLeftSame(v, static_cast<int>(bits.raw)); +} + +template <typename T> +HWY_API Vec1<T> operator>>(const Vec1<T> v, const Vec1<T> bits) { + return ShiftRightSame(v, static_cast<int>(bits.raw)); +} + +// ================================================== ARITHMETIC + +template <typename T> +HWY_API Vec1<T> operator+(Vec1<T> a, Vec1<T> b) { + const uint64_t a64 = static_cast<uint64_t>(a.raw); + const uint64_t b64 = static_cast<uint64_t>(b.raw); + return Vec1<T>(static_cast<T>((a64 + b64) & static_cast<uint64_t>(~T(0)))); +} +HWY_API Vec1<float> operator+(const Vec1<float> a, const Vec1<float> b) { + return Vec1<float>(a.raw + b.raw); +} +HWY_API Vec1<double> operator+(const Vec1<double> a, const Vec1<double> b) { + return Vec1<double>(a.raw + b.raw); +} + +template <typename T> +HWY_API Vec1<T> operator-(Vec1<T> a, Vec1<T> b) { + const uint64_t a64 = static_cast<uint64_t>(a.raw); + const uint64_t b64 = static_cast<uint64_t>(b.raw); + return Vec1<T>(static_cast<T>((a64 - b64) & static_cast<uint64_t>(~T(0)))); +} +HWY_API Vec1<float> operator-(const Vec1<float> a, const Vec1<float> b) { + return Vec1<float>(a.raw - b.raw); +} +HWY_API Vec1<double> operator-(const Vec1<double> a, const Vec1<double> b) { + return Vec1<double>(a.raw - b.raw); +} + +// ------------------------------ Saturating addition + +// Returns a + b clamped to the destination range. + +// Unsigned +HWY_API Vec1<uint8_t> SaturatedAdd(const Vec1<uint8_t> a, + const Vec1<uint8_t> b) { + return Vec1<uint8_t>( + static_cast<uint8_t>(HWY_MIN(HWY_MAX(0, a.raw + b.raw), 255))); +} +HWY_API Vec1<uint16_t> SaturatedAdd(const Vec1<uint16_t> a, + const Vec1<uint16_t> b) { + return Vec1<uint16_t>( + static_cast<uint16_t>(HWY_MIN(HWY_MAX(0, a.raw + b.raw), 65535))); +} + +// Signed +HWY_API Vec1<int8_t> SaturatedAdd(const Vec1<int8_t> a, const Vec1<int8_t> b) { + return Vec1<int8_t>( + static_cast<int8_t>(HWY_MIN(HWY_MAX(-128, a.raw + b.raw), 127))); +} +HWY_API Vec1<int16_t> SaturatedAdd(const Vec1<int16_t> a, + const Vec1<int16_t> b) { + return Vec1<int16_t>( + static_cast<int16_t>(HWY_MIN(HWY_MAX(-32768, a.raw + b.raw), 32767))); +} + +// ------------------------------ Saturating subtraction + +// Returns a - b clamped to the destination range. + +// Unsigned +HWY_API Vec1<uint8_t> SaturatedSub(const Vec1<uint8_t> a, + const Vec1<uint8_t> b) { + return Vec1<uint8_t>( + static_cast<uint8_t>(HWY_MIN(HWY_MAX(0, a.raw - b.raw), 255))); +} +HWY_API Vec1<uint16_t> SaturatedSub(const Vec1<uint16_t> a, + const Vec1<uint16_t> b) { + return Vec1<uint16_t>( + static_cast<uint16_t>(HWY_MIN(HWY_MAX(0, a.raw - b.raw), 65535))); +} + +// Signed +HWY_API Vec1<int8_t> SaturatedSub(const Vec1<int8_t> a, const Vec1<int8_t> b) { + return Vec1<int8_t>( + static_cast<int8_t>(HWY_MIN(HWY_MAX(-128, a.raw - b.raw), 127))); +} +HWY_API Vec1<int16_t> SaturatedSub(const Vec1<int16_t> a, + const Vec1<int16_t> b) { + return Vec1<int16_t>( + static_cast<int16_t>(HWY_MIN(HWY_MAX(-32768, a.raw - b.raw), 32767))); +} + +// ------------------------------ Average + +// Returns (a + b + 1) / 2 + +HWY_API Vec1<uint8_t> AverageRound(const Vec1<uint8_t> a, + const Vec1<uint8_t> b) { + return Vec1<uint8_t>(static_cast<uint8_t>((a.raw + b.raw + 1) / 2)); +} +HWY_API Vec1<uint16_t> AverageRound(const Vec1<uint16_t> a, + const Vec1<uint16_t> b) { + return Vec1<uint16_t>(static_cast<uint16_t>((a.raw + b.raw + 1) / 2)); +} + +// ------------------------------ Absolute value + +template <typename T> +HWY_API Vec1<T> Abs(const Vec1<T> a) { + const T i = a.raw; + return (i >= 0 || i == hwy::LimitsMin<T>()) ? a : Vec1<T>(-i); +} +HWY_API Vec1<float> Abs(const Vec1<float> a) { + return Vec1<float>(std::abs(a.raw)); +} +HWY_API Vec1<double> Abs(const Vec1<double> a) { + return Vec1<double>(std::abs(a.raw)); +} + +// ------------------------------ min/max + +template <typename T, HWY_IF_NOT_FLOAT(T)> +HWY_API Vec1<T> Min(const Vec1<T> a, const Vec1<T> b) { + return Vec1<T>(HWY_MIN(a.raw, b.raw)); +} + +template <typename T, HWY_IF_FLOAT(T)> +HWY_API Vec1<T> Min(const Vec1<T> a, const Vec1<T> b) { + if (std::isnan(a.raw)) return b; + if (std::isnan(b.raw)) return a; + return Vec1<T>(HWY_MIN(a.raw, b.raw)); +} + +template <typename T, HWY_IF_NOT_FLOAT(T)> +HWY_API Vec1<T> Max(const Vec1<T> a, const Vec1<T> b) { + return Vec1<T>(HWY_MAX(a.raw, b.raw)); +} + +template <typename T, HWY_IF_FLOAT(T)> +HWY_API Vec1<T> Max(const Vec1<T> a, const Vec1<T> b) { + if (std::isnan(a.raw)) return b; + if (std::isnan(b.raw)) return a; + return Vec1<T>(HWY_MAX(a.raw, b.raw)); +} + +// ------------------------------ Floating-point negate + +template <typename T, HWY_IF_FLOAT(T)> +HWY_API Vec1<T> Neg(const Vec1<T> v) { + return Xor(v, SignBit(Sisd<T>())); +} + +template <typename T, HWY_IF_NOT_FLOAT(T)> +HWY_API Vec1<T> Neg(const Vec1<T> v) { + return Zero(Sisd<T>()) - v; +} + +// ------------------------------ mul/div + +template <typename T, HWY_IF_FLOAT(T)> +HWY_API Vec1<T> operator*(const Vec1<T> a, const Vec1<T> b) { + return Vec1<T>(static_cast<T>(double(a.raw) * b.raw)); +} + +template <typename T, HWY_IF_SIGNED(T)> +HWY_API Vec1<T> operator*(const Vec1<T> a, const Vec1<T> b) { + return Vec1<T>(static_cast<T>(int64_t(a.raw) * b.raw)); +} + +template <typename T, HWY_IF_UNSIGNED(T)> +HWY_API Vec1<T> operator*(const Vec1<T> a, const Vec1<T> b) { + return Vec1<T>(static_cast<T>(uint64_t(a.raw) * b.raw)); +} + +template <typename T> +HWY_API Vec1<T> operator/(const Vec1<T> a, const Vec1<T> b) { + return Vec1<T>(a.raw / b.raw); +} + +// Returns the upper 16 bits of a * b in each lane. +HWY_API Vec1<int16_t> MulHigh(const Vec1<int16_t> a, const Vec1<int16_t> b) { + return Vec1<int16_t>(static_cast<int16_t>((a.raw * b.raw) >> 16)); +} +HWY_API Vec1<uint16_t> MulHigh(const Vec1<uint16_t> a, const Vec1<uint16_t> b) { + // Cast to uint32_t first to prevent overflow. Otherwise the result of + // uint16_t * uint16_t is in "int" which may overflow. In practice the result + // is the same but this way it is also defined. + return Vec1<uint16_t>(static_cast<uint16_t>( + (static_cast<uint32_t>(a.raw) * static_cast<uint32_t>(b.raw)) >> 16)); +} + +// Multiplies even lanes (0, 2 ..) and returns the double-wide result. +HWY_API Vec1<int64_t> MulEven(const Vec1<int32_t> a, const Vec1<int32_t> b) { + const int64_t a64 = a.raw; + return Vec1<int64_t>(a64 * b.raw); +} +HWY_API Vec1<uint64_t> MulEven(const Vec1<uint32_t> a, const Vec1<uint32_t> b) { + const uint64_t a64 = a.raw; + return Vec1<uint64_t>(a64 * b.raw); +} + +// Approximate reciprocal +HWY_API Vec1<float> ApproximateReciprocal(const Vec1<float> v) { + // Zero inputs are allowed, but callers are responsible for replacing the + // return value with something else (typically using IfThenElse). This check + // avoids a ubsan error. The return value is arbitrary. + if (v.raw == 0.0f) return Vec1<float>(0.0f); + return Vec1<float>(1.0f / v.raw); +} + +// Absolute value of difference. +HWY_API Vec1<float> AbsDiff(const Vec1<float> a, const Vec1<float> b) { + return Abs(a - b); +} + +// ------------------------------ Floating-point multiply-add variants + +template <typename T> +HWY_API Vec1<T> MulAdd(const Vec1<T> mul, const Vec1<T> x, const Vec1<T> add) { + return mul * x + add; +} + +template <typename T> +HWY_API Vec1<T> NegMulAdd(const Vec1<T> mul, const Vec1<T> x, + const Vec1<T> add) { + return add - mul * x; +} + +template <typename T> +HWY_API Vec1<T> MulSub(const Vec1<T> mul, const Vec1<T> x, const Vec1<T> sub) { + return mul * x - sub; +} + +template <typename T> +HWY_API Vec1<T> NegMulSub(const Vec1<T> mul, const Vec1<T> x, + const Vec1<T> sub) { + return Neg(mul) * x - sub; +} + +// ------------------------------ Floating-point square root + +// Approximate reciprocal square root +HWY_API Vec1<float> ApproximateReciprocalSqrt(const Vec1<float> v) { + float f = v.raw; + const float half = f * 0.5f; + uint32_t bits; + CopyBytes<4>(&f, &bits); + // Initial guess based on log2(f) + bits = 0x5F3759DF - (bits >> 1); + CopyBytes<4>(&bits, &f); + // One Newton-Raphson iteration + return Vec1<float>(f * (1.5f - (half * f * f))); +} + +// Square root +HWY_API Vec1<float> Sqrt(const Vec1<float> v) { + return Vec1<float>(std::sqrt(v.raw)); +} +HWY_API Vec1<double> Sqrt(const Vec1<double> v) { + return Vec1<double>(std::sqrt(v.raw)); +} + +// ------------------------------ Floating-point rounding + +template <typename T> +HWY_API Vec1<T> Round(const Vec1<T> v) { + using TI = MakeSigned<T>; + if (!(Abs(v).raw < MantissaEnd<T>())) { // Huge or NaN + return v; + } + const T bias = v.raw < T(0.0) ? T(-0.5) : T(0.5); + const TI rounded = static_cast<TI>(v.raw + bias); + if (rounded == 0) return CopySignToAbs(Vec1<T>(0), v); + // Round to even + if ((rounded & 1) && std::abs(rounded - v.raw) == T(0.5)) { + return Vec1<T>(static_cast<T>(rounded - (v.raw < T(0) ? -1 : 1))); + } + return Vec1<T>(static_cast<T>(rounded)); +} + +// Round-to-nearest even. +HWY_API Vec1<int32_t> NearestInt(const Vec1<float> v) { + using T = float; + using TI = int32_t; + + const T abs = Abs(v).raw; + const bool signbit = std::signbit(v.raw); + + if (!(abs < MantissaEnd<T>())) { // Huge or NaN + // Check if too large to cast or NaN + if (!(abs <= static_cast<T>(LimitsMax<TI>()))) { + return Vec1<TI>(signbit ? LimitsMin<TI>() : LimitsMax<TI>()); + } + return Vec1<int32_t>(static_cast<TI>(v.raw)); + } + const T bias = v.raw < T(0.0) ? T(-0.5) : T(0.5); + const TI rounded = static_cast<TI>(v.raw + bias); + if (rounded == 0) return Vec1<int32_t>(0); + // Round to even + if ((rounded & 1) && std::abs(static_cast<T>(rounded) - v.raw) == T(0.5)) { + return Vec1<TI>(rounded - (signbit ? -1 : 1)); + } + return Vec1<TI>(rounded); +} + +template <typename T> +HWY_API Vec1<T> Trunc(const Vec1<T> v) { + using TI = MakeSigned<T>; + if (!(Abs(v).raw <= MantissaEnd<T>())) { // Huge or NaN + return v; + } + const TI truncated = static_cast<TI>(v.raw); + if (truncated == 0) return CopySignToAbs(Vec1<T>(0), v); + return Vec1<T>(static_cast<T>(truncated)); +} + +template <typename Float, typename Bits, int kMantissaBits, int kExponentBits, + class V> +V Ceiling(const V v) { + const Bits kExponentMask = (1ull << kExponentBits) - 1; + const Bits kMantissaMask = (1ull << kMantissaBits) - 1; + const Bits kBias = kExponentMask / 2; + + Float f = v.raw; + const bool positive = f > Float(0.0); + + Bits bits; + CopyBytes<sizeof(Bits)>(&v, &bits); + + const int exponent = + static_cast<int>(((bits >> kMantissaBits) & kExponentMask) - kBias); + // Already an integer. + if (exponent >= kMantissaBits) return v; + // |v| <= 1 => 0 or 1. + if (exponent < 0) return positive ? V(1) : V(-0.0); + + const Bits mantissa_mask = kMantissaMask >> exponent; + // Already an integer + if ((bits & mantissa_mask) == 0) return v; + + // Clear fractional bits and round up + if (positive) bits += (kMantissaMask + 1) >> exponent; + bits &= ~mantissa_mask; + + CopyBytes<sizeof(Bits)>(&bits, &f); + return V(f); +} + +template <typename Float, typename Bits, int kMantissaBits, int kExponentBits, + class V> +V Floor(const V v) { + const Bits kExponentMask = (1ull << kExponentBits) - 1; + const Bits kMantissaMask = (1ull << kMantissaBits) - 1; + const Bits kBias = kExponentMask / 2; + + Float f = v.raw; + const bool negative = f < Float(0.0); + + Bits bits; + CopyBytes<sizeof(Bits)>(&v, &bits); + + const int exponent = + static_cast<int>(((bits >> kMantissaBits) & kExponentMask) - kBias); + // Already an integer. + if (exponent >= kMantissaBits) return v; + // |v| <= 1 => -1 or 0. + if (exponent < 0) return V(negative ? Float(-1.0) : Float(0.0)); + + const Bits mantissa_mask = kMantissaMask >> exponent; + // Already an integer + if ((bits & mantissa_mask) == 0) return v; + + // Clear fractional bits and round down + if (negative) bits += (kMantissaMask + 1) >> exponent; + bits &= ~mantissa_mask; + + CopyBytes<sizeof(Bits)>(&bits, &f); + return V(f); +} + +// Toward +infinity, aka ceiling +HWY_API Vec1<float> Ceil(const Vec1<float> v) { + return Ceiling<float, uint32_t, 23, 8>(v); +} +HWY_API Vec1<double> Ceil(const Vec1<double> v) { + return Ceiling<double, uint64_t, 52, 11>(v); +} + +// Toward -infinity, aka floor +HWY_API Vec1<float> Floor(const Vec1<float> v) { + return Floor<float, uint32_t, 23, 8>(v); +} +HWY_API Vec1<double> Floor(const Vec1<double> v) { + return Floor<double, uint64_t, 52, 11>(v); +} + +// ================================================== COMPARE + +template <typename T> +HWY_API Mask1<T> operator==(const Vec1<T> a, const Vec1<T> b) { + return Mask1<T>::FromBool(a.raw == b.raw); +} + +template <typename T> +HWY_API Mask1<T> operator!=(const Vec1<T> a, const Vec1<T> b) { + return Mask1<T>::FromBool(a.raw != b.raw); +} + +template <typename T> +HWY_API Mask1<T> TestBit(const Vec1<T> v, const Vec1<T> bit) { + static_assert(!hwy::IsFloat<T>(), "Only integer vectors supported"); + return (v & bit) == bit; +} + +template <typename T> +HWY_API Mask1<T> operator<(const Vec1<T> a, const Vec1<T> b) { + return Mask1<T>::FromBool(a.raw < b.raw); +} +template <typename T> +HWY_API Mask1<T> operator>(const Vec1<T> a, const Vec1<T> b) { + return Mask1<T>::FromBool(a.raw > b.raw); +} + +template <typename T> +HWY_API Mask1<T> operator<=(const Vec1<T> a, const Vec1<T> b) { + return Mask1<T>::FromBool(a.raw <= b.raw); +} +template <typename T> +HWY_API Mask1<T> operator>=(const Vec1<T> a, const Vec1<T> b) { + return Mask1<T>::FromBool(a.raw >= b.raw); +} + +// ================================================== MEMORY + +// ------------------------------ Load + +template <typename T> +HWY_API Vec1<T> Load(Sisd<T> /* tag */, const T* HWY_RESTRICT aligned) { + T t; + CopyBytes<sizeof(T)>(aligned, &t); + return Vec1<T>(t); +} + +template <typename T> +HWY_API Vec1<T> MaskedLoad(Mask1<T> m, Sisd<T> d, + const T* HWY_RESTRICT aligned) { + return IfThenElseZero(m, Load(d, aligned)); +} + +template <typename T> +HWY_API Vec1<T> LoadU(Sisd<T> d, const T* HWY_RESTRICT p) { + return Load(d, p); +} + +// In some use cases, "load single lane" is sufficient; otherwise avoid this. +template <typename T> +HWY_API Vec1<T> LoadDup128(Sisd<T> d, const T* HWY_RESTRICT aligned) { + return Load(d, aligned); +} + +// ------------------------------ Store + +template <typename T> +HWY_API void Store(const Vec1<T> v, Sisd<T> /* tag */, + T* HWY_RESTRICT aligned) { + CopyBytes<sizeof(T)>(&v.raw, aligned); +} + +template <typename T> +HWY_API void StoreU(const Vec1<T> v, Sisd<T> d, T* HWY_RESTRICT p) { + return Store(v, d, p); +} + +// ------------------------------ StoreInterleaved3 + +HWY_API void StoreInterleaved3(const Vec1<uint8_t> v0, const Vec1<uint8_t> v1, + const Vec1<uint8_t> v2, Sisd<uint8_t> d, + uint8_t* HWY_RESTRICT unaligned) { + StoreU(v0, d, unaligned + 0); + StoreU(v1, d, unaligned + 1); + StoreU(v2, d, unaligned + 2); +} + +HWY_API void StoreInterleaved4(const Vec1<uint8_t> v0, const Vec1<uint8_t> v1, + const Vec1<uint8_t> v2, const Vec1<uint8_t> v3, + Sisd<uint8_t> d, + uint8_t* HWY_RESTRICT unaligned) { + StoreU(v0, d, unaligned + 0); + StoreU(v1, d, unaligned + 1); + StoreU(v2, d, unaligned + 2); + StoreU(v3, d, unaligned + 3); +} + +// ------------------------------ Stream + +template <typename T> +HWY_API void Stream(const Vec1<T> v, Sisd<T> d, T* HWY_RESTRICT aligned) { + return Store(v, d, aligned); +} + +// ------------------------------ Scatter + +template <typename T, typename Offset> +HWY_API void ScatterOffset(Vec1<T> v, Sisd<T> d, T* base, + const Vec1<Offset> offset) { + static_assert(sizeof(T) == sizeof(Offset), "Must match for portability"); + uint8_t* const base8 = reinterpret_cast<uint8_t*>(base) + offset.raw; + return Store(v, d, reinterpret_cast<T*>(base8)); +} + +template <typename T, typename Index> +HWY_API void ScatterIndex(Vec1<T> v, Sisd<T> d, T* HWY_RESTRICT base, + const Vec1<Index> index) { + static_assert(sizeof(T) == sizeof(Index), "Must match for portability"); + return Store(v, d, base + index.raw); +} + +// ------------------------------ Gather + +template <typename T, typename Offset> +HWY_API Vec1<T> GatherOffset(Sisd<T> d, const T* base, + const Vec1<Offset> offset) { + static_assert(sizeof(T) == sizeof(Offset), "Must match for portability"); + const uintptr_t addr = reinterpret_cast<uintptr_t>(base) + offset.raw; + return Load(d, reinterpret_cast<const T*>(addr)); +} + +template <typename T, typename Index> +HWY_API Vec1<T> GatherIndex(Sisd<T> d, const T* HWY_RESTRICT base, + const Vec1<Index> index) { + static_assert(sizeof(T) == sizeof(Index), "Must match for portability"); + return Load(d, base + index.raw); +} + +// ================================================== CONVERT + +// ConvertTo and DemoteTo with floating-point input and integer output truncate +// (rounding toward zero). + +template <typename FromT, typename ToT> +HWY_API Vec1<ToT> PromoteTo(Sisd<ToT> /* tag */, Vec1<FromT> from) { + static_assert(sizeof(ToT) > sizeof(FromT), "Not promoting"); + // For bits Y > X, floatX->floatY and intX->intY are always representable. + return Vec1<ToT>(static_cast<ToT>(from.raw)); +} + +template <typename FromT, typename ToT, HWY_IF_FLOAT(FromT)> +HWY_API Vec1<ToT> DemoteTo(Sisd<ToT> /* tag */, Vec1<FromT> from) { + static_assert(sizeof(ToT) < sizeof(FromT), "Not demoting"); + + // Prevent ubsan errors when converting float to narrower integer/float + if (std::isinf(from.raw) || + std::fabs(from.raw) > static_cast<FromT>(HighestValue<ToT>())) { + return Vec1<ToT>(std::signbit(from.raw) ? LowestValue<ToT>() + : HighestValue<ToT>()); + } + return Vec1<ToT>(static_cast<ToT>(from.raw)); +} + +template <typename FromT, typename ToT, HWY_IF_NOT_FLOAT(FromT)> +HWY_API Vec1<ToT> DemoteTo(Sisd<ToT> /* tag */, Vec1<FromT> from) { + static_assert(sizeof(ToT) < sizeof(FromT), "Not demoting"); + + // Int to int: choose closest value in ToT to `from` (avoids UB) + from.raw = HWY_MIN(HWY_MAX(LimitsMin<ToT>(), from.raw), LimitsMax<ToT>()); + return Vec1<ToT>(static_cast<ToT>(from.raw)); +} + +HWY_API Vec1<float> PromoteTo(Sisd<float> /* tag */, const Vec1<float16_t> v) { +#if HWY_NATIVE_FLOAT16 + uint16_t bits16; + CopyBytes<2>(&v.raw, &bits16); +#else + const uint16_t bits16 = v.raw.bits; +#endif + const uint32_t sign = static_cast<uint32_t>(bits16 >> 15); + const uint32_t biased_exp = (bits16 >> 10) & 0x1F; + const uint32_t mantissa = bits16 & 0x3FF; + + // Subnormal or zero + if (biased_exp == 0) { + const float subnormal = + (1.0f / 16384) * (static_cast<float>(mantissa) * (1.0f / 1024)); + return Vec1<float>(sign ? -subnormal : subnormal); + } + + // Normalized: convert the representation directly (faster than ldexp/tables). + const uint32_t biased_exp32 = biased_exp + (127 - 15); + const uint32_t mantissa32 = mantissa << (23 - 10); + const uint32_t bits32 = (sign << 31) | (biased_exp32 << 23) | mantissa32; + float out; + CopyBytes<4>(&bits32, &out); + return Vec1<float>(out); +} + +HWY_API Vec1<float> PromoteTo(Sisd<float> d, const Vec1<bfloat16_t> v) { + return Set(d, F32FromBF16(v.raw)); +} + +HWY_API Vec1<float16_t> DemoteTo(Sisd<float16_t> /* tag */, + const Vec1<float> v) { + uint32_t bits32; + CopyBytes<4>(&v.raw, &bits32); + const uint32_t sign = bits32 >> 31; + const uint32_t biased_exp32 = (bits32 >> 23) & 0xFF; + const uint32_t mantissa32 = bits32 & 0x7FFFFF; + + const int32_t exp = HWY_MIN(static_cast<int32_t>(biased_exp32) - 127, 15); + + // Tiny or zero => zero. + Vec1<float16_t> out; + if (exp < -24) { +#if HWY_NATIVE_FLOAT16 + const uint16_t zero = 0; + CopyBytes<2>(&zero, &out.raw); +#else + out.raw.bits = 0; +#endif + return out; + } + + uint32_t biased_exp16, mantissa16; + + // exp = [-24, -15] => subnormal + if (exp < -14) { + biased_exp16 = 0; + const uint32_t sub_exp = static_cast<uint32_t>(-14 - exp); + HWY_DASSERT(1 <= sub_exp && sub_exp < 11); + mantissa16 = static_cast<uint32_t>((1u << (10 - sub_exp)) + + (mantissa32 >> (13 + sub_exp))); + } else { + // exp = [-14, 15] + biased_exp16 = static_cast<uint32_t>(exp + 15); + HWY_DASSERT(1 <= biased_exp16 && biased_exp16 < 31); + mantissa16 = mantissa32 >> 13; + } + + HWY_DASSERT(mantissa16 < 1024); + const uint32_t bits16 = (sign << 15) | (biased_exp16 << 10) | mantissa16; + HWY_DASSERT(bits16 < 0x10000); +#if HWY_NATIVE_FLOAT16 + const uint16_t narrowed = static_cast<uint16_t>(bits16); // big-endian safe + CopyBytes<2>(&narrowed, &out.raw); +#else + out.raw.bits = static_cast<uint16_t>(bits16); +#endif + return out; +} + +HWY_API Vec1<bfloat16_t> DemoteTo(Sisd<bfloat16_t> d, const Vec1<float> v) { + return Set(d, BF16FromF32(v.raw)); +} + +template <typename FromT, typename ToT, HWY_IF_FLOAT(FromT)> +HWY_API Vec1<ToT> ConvertTo(Sisd<ToT> /* tag */, Vec1<FromT> from) { + static_assert(sizeof(ToT) == sizeof(FromT), "Should have same size"); + // float## -> int##: return closest representable value. We cannot exactly + // represent LimitsMax<ToT> in FromT, so use double. + const double f = static_cast<double>(from.raw); + if (std::isinf(from.raw) || + std::fabs(f) > static_cast<double>(LimitsMax<ToT>())) { + return Vec1<ToT>(std::signbit(from.raw) ? LimitsMin<ToT>() + : LimitsMax<ToT>()); + } + return Vec1<ToT>(static_cast<ToT>(from.raw)); +} + +template <typename FromT, typename ToT, HWY_IF_NOT_FLOAT(FromT)> +HWY_API Vec1<ToT> ConvertTo(Sisd<ToT> /* tag */, Vec1<FromT> from) { + static_assert(sizeof(ToT) == sizeof(FromT), "Should have same size"); + // int## -> float##: no check needed + return Vec1<ToT>(static_cast<ToT>(from.raw)); +} + +HWY_API Vec1<uint8_t> U8FromU32(const Vec1<uint32_t> v) { + return DemoteTo(Sisd<uint8_t>(), v); +} + +// ================================================== COMBINE +// UpperHalf, ZeroExtendVector, Combine, Concat* are unsupported. + +template <typename T> +HWY_API Vec1<T> LowerHalf(Vec1<T> v) { + return v; +} + +template <typename T> +HWY_API Vec1<T> LowerHalf(Sisd<T> /* tag */, Vec1<T> v) { + return v; +} + +// ================================================== SWIZZLE + +template <typename T> +HWY_API T GetLane(const Vec1<T> v) { + return v.raw; +} + +template <typename T> +HWY_API Vec1<T> OddEven(Vec1<T> /* odd */, Vec1<T> even) { + return even; +} + +template <typename T> +HWY_API Vec1<T> OddEvenBlocks(Vec1<T> /* odd */, Vec1<T> even) { + return even; +} + +// ------------------------------ SwapAdjacentBlocks + +template <typename T> +HWY_API Vec1<T> SwapAdjacentBlocks(Vec1<T> v) { + return v; +} + +// ------------------------------ TableLookupLanes + +// Returned by SetTableIndices for use by TableLookupLanes. +template <typename T> +struct Indices1 { + MakeSigned<T> raw; +}; + +template <typename T, typename TI> +HWY_API Indices1<T> IndicesFromVec(Sisd<T>, Vec1<TI> vec) { + static_assert(sizeof(T) == sizeof(TI), "Index size must match lane size"); + HWY_DASSERT(vec.raw == 0); + return Indices1<T>{vec.raw}; +} + +template <typename T, typename TI> +HWY_API Indices1<T> SetTableIndices(Sisd<T> d, const TI* idx) { + return IndicesFromVec(d, LoadU(idx)); +} + +template <typename T> +HWY_API Vec1<T> TableLookupLanes(const Vec1<T> v, const Indices1<T> /* idx */) { + return v; +} + +// ------------------------------ Reverse + +template <typename T> +HWY_API Vec1<T> Reverse(Sisd<T> /* tag */, const Vec1<T> v) { + return v; +} + +// ================================================== BLOCKWISE +// Shift*Bytes, CombineShiftRightBytes, Interleave*, Shuffle* are unsupported. + +// ------------------------------ Broadcast/splat any lane + +template <int kLane, typename T> +HWY_API Vec1<T> Broadcast(const Vec1<T> v) { + static_assert(kLane == 0, "Scalar only has one lane"); + return v; +} + +// ------------------------------ TableLookupBytes, TableLookupBytesOr0 + +template <typename T, typename TI> +HWY_API Vec1<TI> TableLookupBytes(const Vec1<T> in, const Vec1<TI> indices) { + uint8_t in_bytes[sizeof(T)]; + uint8_t idx_bytes[sizeof(T)]; + uint8_t out_bytes[sizeof(T)]; + CopyBytes<sizeof(T)>(&in, &in_bytes); + CopyBytes<sizeof(T)>(&indices, &idx_bytes); + for (size_t i = 0; i < sizeof(T); ++i) { + out_bytes[i] = in_bytes[idx_bytes[i]]; + } + TI out; + CopyBytes<sizeof(TI)>(&out_bytes, &out); + return Vec1<TI>{out}; +} + +template <typename T, typename TI> +HWY_API Vec1<TI> TableLookupBytesOr0(const Vec1<T> in, const Vec1<TI> indices) { + uint8_t in_bytes[sizeof(T)]; + uint8_t idx_bytes[sizeof(T)]; + uint8_t out_bytes[sizeof(T)]; + CopyBytes<sizeof(T)>(&in, &in_bytes); + CopyBytes<sizeof(T)>(&indices, &idx_bytes); + for (size_t i = 0; i < sizeof(T); ++i) { + out_bytes[i] = idx_bytes[i] & 0x80 ? 0 : in_bytes[idx_bytes[i]]; + } + TI out; + CopyBytes<sizeof(TI)>(&out_bytes, &out); + return Vec1<TI>{out}; +} + +// ------------------------------ ZipLower + +HWY_API Vec1<uint16_t> ZipLower(const Vec1<uint8_t> a, const Vec1<uint8_t> b) { + return Vec1<uint16_t>(static_cast<uint16_t>((uint32_t(b.raw) << 8) + a.raw)); +} +HWY_API Vec1<uint32_t> ZipLower(const Vec1<uint16_t> a, + const Vec1<uint16_t> b) { + return Vec1<uint32_t>((uint32_t(b.raw) << 16) + a.raw); +} +HWY_API Vec1<uint64_t> ZipLower(const Vec1<uint32_t> a, + const Vec1<uint32_t> b) { + return Vec1<uint64_t>((uint64_t(b.raw) << 32) + a.raw); +} +HWY_API Vec1<int16_t> ZipLower(const Vec1<int8_t> a, const Vec1<int8_t> b) { + return Vec1<int16_t>(static_cast<int16_t>((int32_t(b.raw) << 8) + a.raw)); +} +HWY_API Vec1<int32_t> ZipLower(const Vec1<int16_t> a, const Vec1<int16_t> b) { + return Vec1<int32_t>((int32_t(b.raw) << 16) + a.raw); +} +HWY_API Vec1<int64_t> ZipLower(const Vec1<int32_t> a, const Vec1<int32_t> b) { + return Vec1<int64_t>((int64_t(b.raw) << 32) + a.raw); +} + +template <typename T, typename TW = MakeWide<T>, class VW = Vec1<TW>> +HWY_API VW ZipLower(Sisd<TW> /* tag */, Vec1<T> a, Vec1<T> b) { + return VW(static_cast<TW>((TW{b.raw} << (sizeof(T) * 8)) + a.raw)); +} + +// ================================================== MASK + +template <typename T> +HWY_API bool AllFalse(Sisd<T> /* tag */, const Mask1<T> mask) { + return mask.bits == 0; +} + +template <typename T> +HWY_API bool AllTrue(Sisd<T> /* tag */, const Mask1<T> mask) { + return mask.bits != 0; +} + +// `p` points to at least 8 readable bytes, not all of which need be valid. +template <typename T> +HWY_API Mask1<T> LoadMaskBits(Sisd<T> /* tag */, + const uint8_t* HWY_RESTRICT bits) { + return Mask1<T>::FromBool((bits[0] & 1) != 0); +} + +// `p` points to at least 8 writable bytes. +template <typename T> +HWY_API size_t StoreMaskBits(Sisd<T> d, const Mask1<T> mask, uint8_t* bits) { + *bits = AllTrue(d, mask); + return 1; +} + +template <typename T> +HWY_API size_t CountTrue(Sisd<T> /* tag */, const Mask1<T> mask) { + return mask.bits == 0 ? 0 : 1; +} + +template <typename T> +HWY_API intptr_t FindFirstTrue(Sisd<T> /* tag */, const Mask1<T> mask) { + return mask.bits == 0 ? -1 : 0; +} + +// ------------------------------ Compress, CompressBits + +template <typename T> +HWY_API Vec1<T> Compress(Vec1<T> v, const Mask1<T> /* mask */) { + // Upper lanes are undefined, so result is the same independent of mask. + return v; +} + +template <typename T> +HWY_API Vec1<T> Compress(Vec1<T> v, const uint8_t* HWY_RESTRICT /* bits */) { + return v; +} + +// ------------------------------ CompressStore + +template <typename T> +HWY_API size_t CompressStore(Vec1<T> v, const Mask1<T> mask, Sisd<T> d, + T* HWY_RESTRICT unaligned) { + StoreU(Compress(v, mask), d, unaligned); + return CountTrue(d, mask); +} + +// ------------------------------ CompressBlendedStore + +template <typename T> +HWY_API size_t CompressBlendedStore(Vec1<T> v, const Mask1<T> mask, Sisd<T> d, + T* HWY_RESTRICT unaligned) { + if (!mask.bits) return 0; + StoreU(v, d, unaligned); + return 1; +} + +// ------------------------------ CompressBitsStore + +template <typename T> +HWY_API size_t CompressBitsStore(Vec1<T> v, const uint8_t* HWY_RESTRICT bits, + Sisd<T> d, T* HWY_RESTRICT unaligned) { + const Mask1<T> mask = LoadMaskBits(d, bits); + StoreU(Compress(v, mask), d, unaligned); + return CountTrue(d, mask); +} + +// ------------------------------ ReorderWidenMulAccumulate (MulAdd, ZipLower) + +HWY_API Vec1<float> ReorderWidenMulAccumulate(Sisd<float> /* tag */, + Vec1<bfloat16_t> a, + Vec1<bfloat16_t> b, + const Vec1<float> sum0, + Vec1<float>& /* sum1 */) { + return MulAdd(Vec1<float>(F32FromBF16(a.raw)), + Vec1<float>(F32FromBF16(b.raw)), sum0); +} + +// ================================================== REDUCTIONS + +// Sum of all lanes, i.e. the only one. +template <typename T> +HWY_API Vec1<T> SumOfLanes(Sisd<T> /* tag */, const Vec1<T> v) { + return v; +} +template <typename T> +HWY_API Vec1<T> MinOfLanes(Sisd<T> /* tag */, const Vec1<T> v) { + return v; +} +template <typename T> +HWY_API Vec1<T> MaxOfLanes(Sisd<T> /* tag */, const Vec1<T> v) { + return v; +} + +// ================================================== DEPRECATED + +template <typename T> +HWY_API size_t StoreMaskBits(const Mask1<T> mask, uint8_t* bits) { + return StoreMaskBits(Sisd<T>(), mask, bits); +} + +template <typename T> +HWY_API bool AllTrue(const Mask1<T> mask) { + return AllTrue(Sisd<T>(), mask); +} + +template <typename T> +HWY_API bool AllFalse(const Mask1<T> mask) { + return AllFalse(Sisd<T>(), mask); +} + +template <typename T> +HWY_API size_t CountTrue(const Mask1<T> mask) { + return CountTrue(Sisd<T>(), mask); +} + +template <typename T> +HWY_API Vec1<T> SumOfLanes(const Vec1<T> v) { + return SumOfLanes(Sisd<T>(), v); +} +template <typename T> +HWY_API Vec1<T> MinOfLanes(const Vec1<T> v) { + return MinOfLanes(Sisd<T>(), v); +} +template <typename T> +HWY_API Vec1<T> MaxOfLanes(const Vec1<T> v) { + return MaxOfLanes(Sisd<T>(), v); +} + +// ================================================== Operator wrapper + +template <class V> +HWY_API V Add(V a, V b) { + return a + b; +} +template <class V> +HWY_API V Sub(V a, V b) { + return a - b; +} + +template <class V> +HWY_API V Mul(V a, V b) { + return a * b; +} +template <class V> +HWY_API V Div(V a, V b) { + return a / b; +} + +template <class V> +V Shl(V a, V b) { + return a << b; +} +template <class V> +V Shr(V a, V b) { + return a >> b; +} + +template <class V> +HWY_API auto Eq(V a, V b) -> decltype(a == b) { + return a == b; +} +template <class V> +HWY_API auto Ne(V a, V b) -> decltype(a == b) { + return a != b; +} +template <class V> +HWY_API auto Lt(V a, V b) -> decltype(a == b) { + return a < b; +} + +template <class V> +HWY_API auto Gt(V a, V b) -> decltype(a == b) { + return a > b; +} +template <class V> +HWY_API auto Ge(V a, V b) -> decltype(a == b) { + return a >= b; +} + +template <class V> +HWY_API auto Le(V a, V b) -> decltype(a == b) { + return a <= b; +} + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); diff --git a/third_party/highway/hwy/ops/set_macros-inl.h b/third_party/highway/hwy/ops/set_macros-inl.h new file mode 100644 index 0000000..1da80cd --- /dev/null +++ b/third_party/highway/hwy/ops/set_macros-inl.h @@ -0,0 +1,350 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Sets macros based on HWY_TARGET. + +// This include guard is toggled by foreach_target, so avoid the usual _H_ +// suffix to prevent copybara from renaming it. +#if defined(HWY_SET_MACROS_PER_TARGET) == defined(HWY_TARGET_TOGGLE) +#ifdef HWY_SET_MACROS_PER_TARGET +#undef HWY_SET_MACROS_PER_TARGET +#else +#define HWY_SET_MACROS_PER_TARGET +#endif + +#endif // HWY_SET_MACROS_PER_TARGET + +#include "hwy/detect_targets.h" + +#undef HWY_NAMESPACE +#undef HWY_ALIGN +#undef HWY_MAX_BYTES +#undef HWY_LANES + +#undef HWY_CAP_INTEGER64 +#undef HWY_CAP_FLOAT16 +#undef HWY_CAP_FLOAT64 +#undef HWY_CAP_GE256 +#undef HWY_CAP_GE512 + +#undef HWY_TARGET_STR + +#if defined(HWY_DISABLE_PCLMUL_AES) +#define HWY_TARGET_STR_PCLMUL_AES "" +#else +#define HWY_TARGET_STR_PCLMUL_AES ",pclmul,aes" +#endif + +#if defined(HWY_DISABLE_BMI2_FMA) +#define HWY_TARGET_STR_BMI2_FMA "" +#else +#define HWY_TARGET_STR_BMI2_FMA ",bmi,bmi2,fma" +#endif + +#if defined(HWY_DISABLE_F16C) +#define HWY_TARGET_STR_F16C "" +#else +#define HWY_TARGET_STR_F16C ",f16c" +#endif + +#define HWY_TARGET_STR_SSSE3 "sse2,ssse3" + +#define HWY_TARGET_STR_SSE4 \ + HWY_TARGET_STR_SSSE3 ",sse4.1,sse4.2" HWY_TARGET_STR_PCLMUL_AES +// Include previous targets, which are the half-vectors of the next target. +#define HWY_TARGET_STR_AVX2 \ + HWY_TARGET_STR_SSE4 ",avx,avx2" HWY_TARGET_STR_BMI2_FMA HWY_TARGET_STR_F16C +#define HWY_TARGET_STR_AVX3 \ + HWY_TARGET_STR_AVX2 ",avx512f,avx512vl,avx512dq,avx512bw" + +// Before include guard so we redefine HWY_TARGET_STR on each include, +// governed by the current HWY_TARGET. +//----------------------------------------------------------------------------- +// SSSE3 +#if HWY_TARGET == HWY_SSSE3 + +#define HWY_NAMESPACE N_SSSE3 +#define HWY_ALIGN alignas(16) +#define HWY_MAX_BYTES 16 +#define HWY_LANES(T) (16 / sizeof(T)) + +#define HWY_CAP_INTEGER64 1 +#define HWY_CAP_FLOAT16 1 +#define HWY_CAP_FLOAT64 1 +#define HWY_CAP_AES 0 +#define HWY_CAP_GE256 0 +#define HWY_CAP_GE512 0 + +#define HWY_TARGET_STR HWY_TARGET_STR_SSSE3 +//----------------------------------------------------------------------------- +// SSE4 +#elif HWY_TARGET == HWY_SSE4 + +#define HWY_NAMESPACE N_SSE4 +#define HWY_ALIGN alignas(16) +#define HWY_MAX_BYTES 16 +#define HWY_LANES(T) (16 / sizeof(T)) + +#define HWY_CAP_INTEGER64 1 +#define HWY_CAP_FLOAT16 1 +#define HWY_CAP_FLOAT64 1 +#define HWY_CAP_GE256 0 +#define HWY_CAP_GE512 0 + +#define HWY_TARGET_STR HWY_TARGET_STR_SSE4 + +//----------------------------------------------------------------------------- +// AVX2 +#elif HWY_TARGET == HWY_AVX2 + +#define HWY_NAMESPACE N_AVX2 +#define HWY_ALIGN alignas(32) +#define HWY_MAX_BYTES 32 +#define HWY_LANES(T) (32 / sizeof(T)) + +#define HWY_CAP_INTEGER64 1 +#define HWY_CAP_FLOAT16 1 +#define HWY_CAP_FLOAT64 1 +#define HWY_CAP_GE256 1 +#define HWY_CAP_GE512 0 + +#define HWY_TARGET_STR HWY_TARGET_STR_AVX2 + +//----------------------------------------------------------------------------- +// AVX3[_DL] +#elif HWY_TARGET == HWY_AVX3 || HWY_TARGET == HWY_AVX3_DL + +#define HWY_ALIGN alignas(64) +#define HWY_MAX_BYTES 64 +#define HWY_LANES(T) (64 / sizeof(T)) + +#define HWY_CAP_INTEGER64 1 +#define HWY_CAP_FLOAT16 1 +#define HWY_CAP_FLOAT64 1 +#define HWY_CAP_GE256 1 +#define HWY_CAP_GE512 1 + +#if HWY_TARGET == HWY_AVX3 + +#define HWY_NAMESPACE N_AVX3 +#define HWY_TARGET_STR HWY_TARGET_STR_AVX3 + +#elif HWY_TARGET == HWY_AVX3_DL + +#define HWY_NAMESPACE N_AVX3_DL +#define HWY_TARGET_STR \ + HWY_TARGET_STR_AVX3 \ + ",vpclmulqdq,avx512vbmi2,vaes,avxvnni,avx512bitalg,avx512vpopcntdq" + +#else +#error "Logic error" +#endif // HWY_TARGET == HWY_AVX3_DL + +//----------------------------------------------------------------------------- +// PPC8 +#elif HWY_TARGET == HWY_PPC8 + +#define HWY_ALIGN alignas(16) +#define HWY_MAX_BYTES 16 +#define HWY_LANES(T) (16 / sizeof(T)) + +#define HWY_CAP_INTEGER64 1 +#define HWY_CAP_FLOAT16 0 +#define HWY_CAP_FLOAT64 1 +#define HWY_CAP_GE256 0 +#define HWY_CAP_GE512 0 + +#define HWY_NAMESPACE N_PPC8 + +#define HWY_TARGET_STR "altivec,vsx" + +//----------------------------------------------------------------------------- +// NEON +#elif HWY_TARGET == HWY_NEON + +#define HWY_ALIGN alignas(16) +#define HWY_MAX_BYTES 16 +#define HWY_LANES(T) (16 / sizeof(T)) + +#define HWY_CAP_INTEGER64 1 +#define HWY_CAP_FLOAT16 1 +#define HWY_CAP_GE256 0 +#define HWY_CAP_GE512 0 + +#if HWY_ARCH_ARM_A64 +#define HWY_CAP_FLOAT64 1 +#else +#define HWY_CAP_FLOAT64 0 +#endif + +#define HWY_NAMESPACE N_NEON + +// HWY_TARGET_STR remains undefined so HWY_ATTR is a no-op. + +//----------------------------------------------------------------------------- +// SVE[2] +#elif HWY_TARGET == HWY_SVE2 || HWY_TARGET == HWY_SVE + +#if defined(HWY_EMULATE_SVE) && !defined(__F16C__) +#error "Disable HWY_CAP_FLOAT16 or ensure farm_sve actually converts to f16" +#endif + +// SVE only requires lane alignment, not natural alignment of the entire vector. +#define HWY_ALIGN alignas(8) + +#define HWY_MAX_BYTES 256 + +// <= HWY_MAX_BYTES / sizeof(T): exact size. Otherwise a fraction 1/div (div = +// 1,2,4,8) is encoded as HWY_LANES(T) / div. This value leaves enough room for +// div=8 and demoting to 1/8 the lane width while still exceeding HWY_MAX_BYTES. +#define HWY_LANES(T) (32768 / sizeof(T)) + +#define HWY_CAP_INTEGER64 1 +#define HWY_CAP_FLOAT16 1 +#define HWY_CAP_FLOAT64 1 +#define HWY_CAP_GE256 0 +#define HWY_CAP_GE512 0 + +#if HWY_TARGET == HWY_SVE2 +#define HWY_NAMESPACE N_SVE2 +#else +#define HWY_NAMESPACE N_SVE +#endif + +// HWY_TARGET_STR remains undefined + +//----------------------------------------------------------------------------- +// WASM +#elif HWY_TARGET == HWY_WASM + +#define HWY_ALIGN alignas(16) +#define HWY_MAX_BYTES 16 +#define HWY_LANES(T) (16 / sizeof(T)) + +#define HWY_CAP_INTEGER64 0 +#define HWY_CAP_FLOAT16 1 +#define HWY_CAP_FLOAT64 0 +#define HWY_CAP_GE256 0 +#define HWY_CAP_GE512 0 + +#define HWY_NAMESPACE N_WASM + +#define HWY_TARGET_STR "simd128" + +//----------------------------------------------------------------------------- +// WASM2 +#elif HWY_TARGET == HWY_WASM2 + +#define HWY_ALIGN alignas(32) +#define HWY_MAX_BYTES 32 +#define HWY_LANES(T) (32 / sizeof(T)) + +#define HWY_CAP_INTEGER64 0 +#define HWY_CAP_FLOAT16 1 +#define HWY_CAP_FLOAT64 0 +#define HWY_CAP_GE256 0 +#define HWY_CAP_GE512 0 + +#define HWY_NAMESPACE N_WASM2 + +#define HWY_TARGET_STR "simd128" + +//----------------------------------------------------------------------------- +// RVV +#elif HWY_TARGET == HWY_RVV + +// RVV only requires lane alignment, not natural alignment of the entire vector, +// and the compiler already aligns builtin types, so nothing to do here. +#define HWY_ALIGN + +// The spec requires VLEN <= 2^16 bits, so the limit is 2^16 bytes (LMUL=8). +#define HWY_MAX_BYTES 65536 + +// <= HWY_MAX_BYTES / sizeof(T): exact size. Otherwise a fraction 1/div (div = +// 1,2,4,8) is encoded as HWY_LANES(T) / div. This value leaves enough room for +// div=8 and demoting to 1/8 the lane width while still exceeding HWY_MAX_BYTES. +#define HWY_LANES(T) (8388608 / sizeof(T)) + +#define HWY_CAP_INTEGER64 1 +#define HWY_CAP_FLOAT64 1 +#define HWY_CAP_GE256 0 +#define HWY_CAP_GE512 0 + +#if defined(__riscv_zfh) +#define HWY_CAP_FLOAT16 1 +#else +#define HWY_CAP_FLOAT16 0 +#endif + +#define HWY_NAMESPACE N_RVV + +// HWY_TARGET_STR remains undefined so HWY_ATTR is a no-op. +// (rv64gcv is not a valid target) + +//----------------------------------------------------------------------------- +// SCALAR +#elif HWY_TARGET == HWY_SCALAR + +#define HWY_ALIGN +#define HWY_MAX_BYTES 8 +#define HWY_LANES(T) 1 + +#define HWY_CAP_INTEGER64 1 +#define HWY_CAP_FLOAT16 1 +#define HWY_CAP_FLOAT64 1 +#define HWY_CAP_GE256 0 +#define HWY_CAP_GE512 0 + +#define HWY_NAMESPACE N_SCALAR + +// HWY_TARGET_STR remains undefined so HWY_ATTR is a no-op. + +#else +#pragma message("HWY_TARGET does not match any known target") +#endif // HWY_TARGET + +// Clang <9 requires this be invoked at file scope, before any namespace. +#undef HWY_BEFORE_NAMESPACE +#if defined(HWY_TARGET_STR) +#define HWY_BEFORE_NAMESPACE() \ + HWY_PUSH_ATTRIBUTES(HWY_TARGET_STR) \ + static_assert(true, "For requiring trailing semicolon") +#else +// avoids compiler warning if no HWY_TARGET_STR +#define HWY_BEFORE_NAMESPACE() \ + static_assert(true, "For requiring trailing semicolon") +#endif + +// Clang <9 requires any namespaces be closed before this macro. +#undef HWY_AFTER_NAMESPACE +#if defined(HWY_TARGET_STR) +#define HWY_AFTER_NAMESPACE() \ + HWY_POP_ATTRIBUTES \ + static_assert(true, "For requiring trailing semicolon") +#else +// avoids compiler warning if no HWY_TARGET_STR +#define HWY_AFTER_NAMESPACE() \ + static_assert(true, "For requiring trailing semicolon") +#endif + +#undef HWY_ATTR +#if defined(HWY_TARGET_STR) && HWY_HAS_ATTRIBUTE(target) +#define HWY_ATTR __attribute__((target(HWY_TARGET_STR))) +#else +#define HWY_ATTR +#endif + +// DEPRECATED +#undef HWY_GATHER_LANES +#define HWY_GATHER_LANES(T) HWY_LANES(T) diff --git a/third_party/highway/hwy/ops/shared-inl.h b/third_party/highway/hwy/ops/shared-inl.h new file mode 100644 index 0000000..4a4ed1e --- /dev/null +++ b/third_party/highway/hwy/ops/shared-inl.h @@ -0,0 +1,232 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Per-target definitions shared by ops/*.h and user code. + +#include <cmath> + +#include "hwy/base.h" + +// Separate header because foreach_target.h re-enables its include guard. +#include "hwy/ops/set_macros-inl.h" + +// Relies on the external include guard in highway.h. +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +// SIMD operations are implemented as overloaded functions selected using a tag +// type D := Simd<T, N>. T is the lane type, N an opaque integer for internal +// use only. Users create D via aliases ScalableTag<T>() (a full vector), +// CappedTag<T, kLimit> or FixedTag<T, kNumLanes>. The actual number of lanes +// (always a power of two) is Lanes(D()). +template <typename Lane, size_t N> +struct Simd { + constexpr Simd() = default; + using T = Lane; + static_assert((N & (N - 1)) == 0 && N != 0, "N must be a power of two"); + + // Widening/narrowing ops change the number of lanes and/or their type. + // To initialize such vectors, we need the corresponding tag types: + + // PromoteTo/DemoteTo() with another lane type, but same number of lanes. + template <typename NewLane> + using Rebind = Simd<NewLane, N>; + + // MulEven() with another lane type, but same total size. + // Round up to correctly handle scalars with N=1. + template <typename NewLane> + using Repartition = + Simd<NewLane, (N * sizeof(Lane) + sizeof(NewLane) - 1) / sizeof(NewLane)>; + + // LowerHalf() with the same lane type, but half the lanes. + // Round up to correctly handle scalars with N=1. + using Half = Simd<T, (N + 1) / 2>; + + // Combine() with the same lane type, but twice the lanes. + using Twice = Simd<T, 2 * N>; +}; + +namespace detail { + +// Given N from HWY_LANES(T), returns N for use in Simd<T, N> to describe: +// - a full vector (pow2 = 0); +// - 2,4,8 regs on RVV, otherwise a full vector (pow2 [1,3]); +// - a fraction of a register from 1/8 to 1/2 (pow2 [-3,-1]). +constexpr size_t ScaleByPower(size_t N, int pow2) { +#if HWY_TARGET == HWY_RVV + // For fractions, if N == 1 ensure we still return at least one lane. + return pow2 >= 0 ? (N << pow2) : HWY_MAX(1, (N >> (-pow2))); +#else + // If pow2 > 0, replace it with 0 (there is nothing wider than a full vector). + return HWY_MAX(1, N >> HWY_MAX(-pow2, 0)); +#endif +} + +// Struct wrappers enable validation of arguments via static_assert. +template <typename T, int kPow2> +struct ScalableTagChecker { + static_assert(-3 <= kPow2 && kPow2 <= 3, "Fraction must be 1/8 to 8"); + using type = Simd<T, ScaleByPower(HWY_LANES(T), kPow2)>; +}; + +template <typename T, size_t kLimit> +struct CappedTagChecker { + static_assert(kLimit != 0, "Does not make sense to have zero lanes"); + using type = Simd<T, HWY_MIN(kLimit, HWY_LANES(T))>; +}; + +template <typename T, size_t kNumLanes> +struct FixedTagChecker { + static_assert(kNumLanes != 0, "Does not make sense to have zero lanes"); + static_assert(kNumLanes * sizeof(T) <= HWY_MAX_BYTES, "Too many lanes"); +#if HWY_TARGET == HWY_SCALAR + // HWY_MAX_BYTES would still allow uint8x8, which is not supported. + static_assert(kNumLanes == 1, "Scalar only supports one lane"); +#endif + using type = Simd<T, kNumLanes>; +}; + +} // namespace detail + +// Alias for a tag describing a full vector (kPow2 == 0: the most common usage, +// e.g. 1D loops where the application does not care about the vector size) or a +// fraction/multiple of one. Multiples are the same as full vectors for all +// targets except RVV. Fractions (kPow2 < 0) are useful as the argument/return +// value of type promotion and demotion. +template <typename T, int kPow2 = 0> +using ScalableTag = typename detail::ScalableTagChecker<T, kPow2>::type; + +// Alias for a tag describing a vector with *up to* kLimit active lanes, even on +// targets with scalable vectors and HWY_SCALAR. The runtime lane count +// `Lanes(tag)` may be less than kLimit, and is 1 on HWY_SCALAR. This alias is +// typically used for 1D loops with a relatively low application-defined upper +// bound, e.g. for 8x8 DCTs. However, it is better if data structures are +// designed to be vector-length-agnostic (e.g. a hybrid SoA where there are +// chunks of say 256 DC components followed by 256 AC1 and finally 256 AC63; +// this would enable vector-length-agnostic loops using ScalableTag). +template <typename T, size_t kLimit> +using CappedTag = typename detail::CappedTagChecker<T, kLimit>::type; + +// Alias for a tag describing a vector with *exactly* kNumLanes active lanes, +// even on targets with scalable vectors. All targets except HWY_SCALAR support +// up to 16 / sizeof(T). Other targets may allow larger kNumLanes, but relying +// on that is non-portable and discouraged. +// +// NOTE: if the application does not need to support HWY_SCALAR (+), use this +// instead of CappedTag to emphasize that there will be exactly kNumLanes lanes. +// This is useful for data structures that rely on exactly 128-bit SIMD, but +// these are discouraged because they cannot benefit from wider vectors. +// Instead, applications would ideally define a larger problem size and loop +// over it with the (unknown size) vectors from ScalableTag. +// +// + e.g. if the baseline is known to support SIMD, or the application requires +// ops such as TableLookupBytes not supported by HWY_SCALAR. +template <typename T, size_t kNumLanes> +using FixedTag = typename detail::FixedTagChecker<T, kNumLanes>::type; + +template <class D> +using TFromD = typename D::T; + +// Tag for the same number of lanes as D, but with the LaneType T. +template <class T, class D> +using Rebind = typename D::template Rebind<T>; + +template <class D> +using RebindToSigned = Rebind<MakeSigned<TFromD<D>>, D>; +template <class D> +using RebindToUnsigned = Rebind<MakeUnsigned<TFromD<D>>, D>; +template <class D> +using RebindToFloat = Rebind<MakeFloat<TFromD<D>>, D>; + +// Tag for the same total size as D, but with the LaneType T. +template <class T, class D> +using Repartition = typename D::template Repartition<T>; + +template <class D> +using RepartitionToWide = Repartition<MakeWide<TFromD<D>>, D>; +template <class D> +using RepartitionToNarrow = Repartition<MakeNarrow<TFromD<D>>, D>; + +// Tag for the same lane type as D, but half the lanes. +template <class D> +using Half = typename D::Half; + +// Descriptor for the same lane type as D, but twice the lanes. +template <class D> +using Twice = typename D::Twice; + +// Same as base.h macros but with a Simd<T, N> argument instead of T. +#define HWY_IF_UNSIGNED_D(D) HWY_IF_UNSIGNED(TFromD<D>) +#define HWY_IF_SIGNED_D(D) HWY_IF_SIGNED(TFromD<D>) +#define HWY_IF_FLOAT_D(D) HWY_IF_FLOAT(TFromD<D>) +#define HWY_IF_NOT_FLOAT_D(D) HWY_IF_NOT_FLOAT(TFromD<D>) +#define HWY_IF_LANE_SIZE_D(D, bytes) HWY_IF_LANE_SIZE(TFromD<D>, bytes) +#define HWY_IF_NOT_LANE_SIZE_D(D, bytes) HWY_IF_NOT_LANE_SIZE(TFromD<D>, bytes) + +// Same, but with a vector argument. +#define HWY_IF_UNSIGNED_V(V) HWY_IF_UNSIGNED(TFromV<V>) +#define HWY_IF_SIGNED_V(V) HWY_IF_SIGNED(TFromV<V>) +#define HWY_IF_FLOAT_V(V) HWY_IF_FLOAT(TFromV<V>) +#define HWY_IF_LANE_SIZE_V(V, bytes) HWY_IF_LANE_SIZE(TFromV<V>, bytes) + +// For implementing functions for a specific type. +// IsSame<...>() in template arguments is broken on MSVC2015. +#define HWY_IF_LANES_ARE(T, V) \ + EnableIf<IsSameT<T, TFromD<DFromV<V>>>::value>* = nullptr + +// Compile-time-constant, (typically but not guaranteed) an upper bound on the +// number of lanes. +// Prefer instead using Lanes() and dynamic allocation, or Rebind, or +// `#if HWY_CAP_GE*`. +template <typename T, size_t N> +HWY_INLINE HWY_MAYBE_UNUSED constexpr size_t MaxLanes(Simd<T, N>) { + return N; +} + +// Targets with non-constexpr Lanes define this themselves. +#if HWY_TARGET != HWY_RVV && HWY_TARGET != HWY_SVE2 && HWY_TARGET != HWY_SVE + +// (Potentially) non-constant actual size of the vector at runtime, subject to +// the limit imposed by the Simd. Useful for advancing loop counters. +template <typename T, size_t N> +HWY_INLINE HWY_MAYBE_UNUSED size_t Lanes(Simd<T, N>) { + return N; +} + +#endif + +// NOTE: GCC generates incorrect code for vector arguments to non-inlined +// functions in two situations: +// - on Windows and GCC 10.3, passing by value crashes due to unaligned loads: +// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54412. +// - on ARM64 and GCC 9.3.0 or 11.2.1, passing by const& causes many (but not +// all) tests to fail. +// +// We therefore pass by const& only on GCC and (Windows or ARM64). This alias +// must be used for all vector/mask parameters of functions marked HWY_NOINLINE, +// and possibly also other functions that are not inlined. +#if HWY_COMPILER_GCC && !HWY_COMPILER_CLANG && \ + ((defined(_WIN32) || defined(_WIN64)) || HWY_ARCH_ARM64) +template <class V> +using VecArg = const V&; +#else +template <class V> +using VecArg = V; +#endif + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); diff --git a/third_party/highway/hwy/ops/wasm_128-inl.h b/third_party/highway/hwy/ops/wasm_128-inl.h new file mode 100644 index 0000000..fbb9acf --- /dev/null +++ b/third_party/highway/hwy/ops/wasm_128-inl.h @@ -0,0 +1,3702 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// 128-bit WASM vectors and operations. +// External include guard in highway.h - see comment there. + +#include <stddef.h> +#include <stdint.h> +#include <wasm_simd128.h> + +#include "hwy/base.h" +#include "hwy/ops/shared-inl.h" + +#ifdef HWY_WASM_OLD_NAMES +#define wasm_i8x16_shuffle wasm_v8x16_shuffle +#define wasm_i16x8_shuffle wasm_v16x8_shuffle +#define wasm_i32x4_shuffle wasm_v32x4_shuffle +#define wasm_i64x2_shuffle wasm_v64x2_shuffle +#define wasm_u16x8_extend_low_u8x16 wasm_i16x8_widen_low_u8x16 +#define wasm_u32x4_extend_low_u16x8 wasm_i32x4_widen_low_u16x8 +#define wasm_i32x4_extend_low_i16x8 wasm_i32x4_widen_low_i16x8 +#define wasm_i16x8_extend_low_i8x16 wasm_i16x8_widen_low_i8x16 +#define wasm_u32x4_extend_high_u16x8 wasm_i32x4_widen_high_u16x8 +#define wasm_i32x4_extend_high_i16x8 wasm_i32x4_widen_high_i16x8 +#define wasm_i32x4_trunc_sat_f32x4 wasm_i32x4_trunc_saturate_f32x4 +#define wasm_u8x16_add_sat wasm_u8x16_add_saturate +#define wasm_u8x16_sub_sat wasm_u8x16_sub_saturate +#define wasm_u16x8_add_sat wasm_u16x8_add_saturate +#define wasm_u16x8_sub_sat wasm_u16x8_sub_saturate +#define wasm_i8x16_add_sat wasm_i8x16_add_saturate +#define wasm_i8x16_sub_sat wasm_i8x16_sub_saturate +#define wasm_i16x8_add_sat wasm_i16x8_add_saturate +#define wasm_i16x8_sub_sat wasm_i16x8_sub_saturate +#endif + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +template <typename T> +using Full128 = Simd<T, 16 / sizeof(T)>; + +namespace detail { + +template <typename T> +struct Raw128 { + using type = __v128_u; +}; +template <> +struct Raw128<float> { + using type = __f32x4; +}; + +} // namespace detail + +template <typename T, size_t N = 16 / sizeof(T)> +class Vec128 { + using Raw = typename detail::Raw128<T>::type; + + public: + // Compound assignment. Only usable if there is a corresponding non-member + // binary operator overload. For example, only f32 and f64 support division. + HWY_INLINE Vec128& operator*=(const Vec128 other) { + return *this = (*this * other); + } + HWY_INLINE Vec128& operator/=(const Vec128 other) { + return *this = (*this / other); + } + HWY_INLINE Vec128& operator+=(const Vec128 other) { + return *this = (*this + other); + } + HWY_INLINE Vec128& operator-=(const Vec128 other) { + return *this = (*this - other); + } + HWY_INLINE Vec128& operator&=(const Vec128 other) { + return *this = (*this & other); + } + HWY_INLINE Vec128& operator|=(const Vec128 other) { + return *this = (*this | other); + } + HWY_INLINE Vec128& operator^=(const Vec128 other) { + return *this = (*this ^ other); + } + + Raw raw; +}; + +// FF..FF or 0. +template <typename T, size_t N = 16 / sizeof(T)> +struct Mask128 { + typename detail::Raw128<T>::type raw; +}; + +namespace detail { + +// Deduce Simd<T, N> from Vec128<T, N> +struct DeduceD { + template <typename T, size_t N> + Simd<T, N> operator()(Vec128<T, N>) const { + return Simd<T, N>(); + } +}; + +} // namespace detail + +template <class V> +using DFromV = decltype(detail::DeduceD()(V())); + +template <class V> +using TFromV = TFromD<DFromV<V>>; + +// ------------------------------ BitCast + +namespace detail { + +HWY_INLINE __v128_u BitCastToInteger(__v128_u v) { return v; } +HWY_INLINE __v128_u BitCastToInteger(__f32x4 v) { + return static_cast<__v128_u>(v); +} +HWY_INLINE __v128_u BitCastToInteger(__f64x2 v) { + return static_cast<__v128_u>(v); +} + +template <typename T, size_t N> +HWY_INLINE Vec128<uint8_t, N * sizeof(T)> BitCastToByte(Vec128<T, N> v) { + return Vec128<uint8_t, N * sizeof(T)>{BitCastToInteger(v.raw)}; +} + +// Cannot rely on function overloading because return types differ. +template <typename T> +struct BitCastFromInteger128 { + HWY_INLINE __v128_u operator()(__v128_u v) { return v; } +}; +template <> +struct BitCastFromInteger128<float> { + HWY_INLINE __f32x4 operator()(__v128_u v) { return static_cast<__f32x4>(v); } +}; + +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> BitCastFromByte(Simd<T, N> /* tag */, + Vec128<uint8_t, N * sizeof(T)> v) { + return Vec128<T, N>{BitCastFromInteger128<T>()(v.raw)}; +} + +} // namespace detail + +template <typename T, size_t N, typename FromT> +HWY_API Vec128<T, N> BitCast(Simd<T, N> d, + Vec128<FromT, N * sizeof(T) / sizeof(FromT)> v) { + return detail::BitCastFromByte(d, detail::BitCastToByte(v)); +} + +// ------------------------------ Zero + +// Returns an all-zero vector/part. +template <typename T, size_t N, HWY_IF_LE128(T, N)> +HWY_API Vec128<T, N> Zero(Simd<T, N> /* tag */) { + return Vec128<T, N>{wasm_i32x4_splat(0)}; +} +template <size_t N, HWY_IF_LE128(float, N)> +HWY_API Vec128<float, N> Zero(Simd<float, N> /* tag */) { + return Vec128<float, N>{wasm_f32x4_splat(0.0f)}; +} + +template <class D> +using VFromD = decltype(Zero(D())); + +// ------------------------------ Set + +// Returns a vector/part with all lanes set to "t". +template <size_t N, HWY_IF_LE128(uint8_t, N)> +HWY_API Vec128<uint8_t, N> Set(Simd<uint8_t, N> /* tag */, const uint8_t t) { + return Vec128<uint8_t, N>{wasm_i8x16_splat(static_cast<int8_t>(t))}; +} +template <size_t N, HWY_IF_LE128(uint16_t, N)> +HWY_API Vec128<uint16_t, N> Set(Simd<uint16_t, N> /* tag */, const uint16_t t) { + return Vec128<uint16_t, N>{wasm_i16x8_splat(static_cast<int16_t>(t))}; +} +template <size_t N, HWY_IF_LE128(uint32_t, N)> +HWY_API Vec128<uint32_t, N> Set(Simd<uint32_t, N> /* tag */, const uint32_t t) { + return Vec128<uint32_t, N>{wasm_i32x4_splat(static_cast<int32_t>(t))}; +} +template <size_t N, HWY_IF_LE128(uint64_t, N)> +HWY_API Vec128<uint64_t, N> Set(Simd<uint64_t, N> /* tag */, const uint64_t t) { + return Vec128<uint64_t, N>{wasm_i64x2_splat(static_cast<int64_t>(t))}; +} + +template <size_t N, HWY_IF_LE128(int8_t, N)> +HWY_API Vec128<int8_t, N> Set(Simd<int8_t, N> /* tag */, const int8_t t) { + return Vec128<int8_t, N>{wasm_i8x16_splat(t)}; +} +template <size_t N, HWY_IF_LE128(int16_t, N)> +HWY_API Vec128<int16_t, N> Set(Simd<int16_t, N> /* tag */, const int16_t t) { + return Vec128<int16_t, N>{wasm_i16x8_splat(t)}; +} +template <size_t N, HWY_IF_LE128(int32_t, N)> +HWY_API Vec128<int32_t, N> Set(Simd<int32_t, N> /* tag */, const int32_t t) { + return Vec128<int32_t, N>{wasm_i32x4_splat(t)}; +} +template <size_t N, HWY_IF_LE128(int64_t, N)> +HWY_API Vec128<int64_t, N> Set(Simd<int64_t, N> /* tag */, const int64_t t) { + return Vec128<int64_t, N>{wasm_i64x2_splat(t)}; +} + +template <size_t N, HWY_IF_LE128(float, N)> +HWY_API Vec128<float, N> Set(Simd<float, N> /* tag */, const float t) { + return Vec128<float, N>{wasm_f32x4_splat(t)}; +} + +HWY_DIAGNOSTICS(push) +HWY_DIAGNOSTICS_OFF(disable : 4700, ignored "-Wuninitialized") + +// Returns a vector with uninitialized elements. +template <typename T, size_t N, HWY_IF_LE128(T, N)> +HWY_API Vec128<T, N> Undefined(Simd<T, N> d) { + return Zero(d); +} + +HWY_DIAGNOSTICS(pop) + +// Returns a vector with lane i=[0, N) set to "first" + i. +template <typename T, size_t N, typename T2> +Vec128<T, N> Iota(const Simd<T, N> d, const T2 first) { + HWY_ALIGN T lanes[16 / sizeof(T)]; + for (size_t i = 0; i < 16 / sizeof(T); ++i) { + lanes[i] = static_cast<T>(first + static_cast<T2>(i)); + } + return Load(d, lanes); +} + +// ================================================== ARITHMETIC + +// ------------------------------ Addition + +// Unsigned +template <size_t N> +HWY_API Vec128<uint8_t, N> operator+(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{wasm_i8x16_add(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> operator+(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{wasm_i16x8_add(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> operator+(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { + return Vec128<uint32_t, N>{wasm_i32x4_add(a.raw, b.raw)}; +} + +// Signed +template <size_t N> +HWY_API Vec128<int8_t, N> operator+(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Vec128<int8_t, N>{wasm_i8x16_add(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> operator+(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{wasm_i16x8_add(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> operator+(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + return Vec128<int32_t, N>{wasm_i32x4_add(a.raw, b.raw)}; +} + +// Float +template <size_t N> +HWY_API Vec128<float, N> operator+(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Vec128<float, N>{wasm_f32x4_add(a.raw, b.raw)}; +} + +// ------------------------------ Subtraction + +// Unsigned +template <size_t N> +HWY_API Vec128<uint8_t, N> operator-(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{wasm_i8x16_sub(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> operator-(Vec128<uint16_t, N> a, + Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{wasm_i16x8_sub(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> operator-(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { + return Vec128<uint32_t, N>{wasm_i32x4_sub(a.raw, b.raw)}; +} + +// Signed +template <size_t N> +HWY_API Vec128<int8_t, N> operator-(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Vec128<int8_t, N>{wasm_i8x16_sub(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> operator-(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{wasm_i16x8_sub(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> operator-(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + return Vec128<int32_t, N>{wasm_i32x4_sub(a.raw, b.raw)}; +} + +// Float +template <size_t N> +HWY_API Vec128<float, N> operator-(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Vec128<float, N>{wasm_f32x4_sub(a.raw, b.raw)}; +} + +// ------------------------------ Saturating addition + +// Returns a + b clamped to the destination range. + +// Unsigned +template <size_t N> +HWY_API Vec128<uint8_t, N> SaturatedAdd(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{wasm_u8x16_add_sat(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> SaturatedAdd(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{wasm_u16x8_add_sat(a.raw, b.raw)}; +} + +// Signed +template <size_t N> +HWY_API Vec128<int8_t, N> SaturatedAdd(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Vec128<int8_t, N>{wasm_i8x16_add_sat(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> SaturatedAdd(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{wasm_i16x8_add_sat(a.raw, b.raw)}; +} + +// ------------------------------ Saturating subtraction + +// Returns a - b clamped to the destination range. + +// Unsigned +template <size_t N> +HWY_API Vec128<uint8_t, N> SaturatedSub(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{wasm_u8x16_sub_sat(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> SaturatedSub(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{wasm_u16x8_sub_sat(a.raw, b.raw)}; +} + +// Signed +template <size_t N> +HWY_API Vec128<int8_t, N> SaturatedSub(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Vec128<int8_t, N>{wasm_i8x16_sub_sat(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> SaturatedSub(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{wasm_i16x8_sub_sat(a.raw, b.raw)}; +} + +// ------------------------------ Average + +// Returns (a + b + 1) / 2 + +// Unsigned +template <size_t N> +HWY_API Vec128<uint8_t, N> AverageRound(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{wasm_u8x16_avgr(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> AverageRound(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{wasm_u16x8_avgr(a.raw, b.raw)}; +} + +// ------------------------------ Absolute value + +// Returns absolute value, except that LimitsMin() maps to LimitsMax() + 1. +template <size_t N> +HWY_API Vec128<int8_t, N> Abs(const Vec128<int8_t, N> v) { + return Vec128<int8_t, N>{wasm_i8x16_abs(v.raw)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> Abs(const Vec128<int16_t, N> v) { + return Vec128<int16_t, N>{wasm_i16x8_abs(v.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> Abs(const Vec128<int32_t, N> v) { + return Vec128<int32_t, N>{wasm_i32x4_abs(v.raw)}; +} +template <size_t N> +HWY_API Vec128<int64_t, N> Abs(const Vec128<int64_t, N> v) { + return Vec128<int32_t, N>{wasm_i62x2_abs(v.raw)}; +} + +template <size_t N> +HWY_API Vec128<float, N> Abs(const Vec128<float, N> v) { + return Vec128<float, N>{wasm_f32x4_abs(v.raw)}; +} + +// ------------------------------ Shift lanes by constant #bits + +// Unsigned +template <int kBits, size_t N> +HWY_API Vec128<uint16_t, N> ShiftLeft(const Vec128<uint16_t, N> v) { + return Vec128<uint16_t, N>{wasm_i16x8_shl(v.raw, kBits)}; +} +template <int kBits, size_t N> +HWY_API Vec128<uint16_t, N> ShiftRight(const Vec128<uint16_t, N> v) { + return Vec128<uint16_t, N>{wasm_u16x8_shr(v.raw, kBits)}; +} +template <int kBits, size_t N> +HWY_API Vec128<uint32_t, N> ShiftLeft(const Vec128<uint32_t, N> v) { + return Vec128<uint32_t, N>{wasm_i32x4_shl(v.raw, kBits)}; +} +template <int kBits, size_t N> +HWY_API Vec128<uint32_t, N> ShiftRight(const Vec128<uint32_t, N> v) { + return Vec128<uint32_t, N>{wasm_u32x4_shr(v.raw, kBits)}; +} + +// Signed +template <int kBits, size_t N> +HWY_API Vec128<int16_t, N> ShiftLeft(const Vec128<int16_t, N> v) { + return Vec128<int16_t, N>{wasm_i16x8_shl(v.raw, kBits)}; +} +template <int kBits, size_t N> +HWY_API Vec128<int16_t, N> ShiftRight(const Vec128<int16_t, N> v) { + return Vec128<int16_t, N>{wasm_i16x8_shr(v.raw, kBits)}; +} +template <int kBits, size_t N> +HWY_API Vec128<int32_t, N> ShiftLeft(const Vec128<int32_t, N> v) { + return Vec128<int32_t, N>{wasm_i32x4_shl(v.raw, kBits)}; +} +template <int kBits, size_t N> +HWY_API Vec128<int32_t, N> ShiftRight(const Vec128<int32_t, N> v) { + return Vec128<int32_t, N>{wasm_i32x4_shr(v.raw, kBits)}; +} + +// 8-bit +template <int kBits, typename T, size_t N, HWY_IF_LANE_SIZE(T, 1)> +HWY_API Vec128<T, N> ShiftLeft(const Vec128<T, N> v) { + const Simd<T, N> d8; + // Use raw instead of BitCast to support N=1. + const Vec128<T, N> shifted{ShiftLeft<kBits>(Vec128<MakeWide<T>>{v.raw}).raw}; + return kBits == 1 + ? (v + v) + : (shifted & Set(d8, static_cast<T>((0xFF << kBits) & 0xFF))); +} + +template <int kBits, size_t N> +HWY_API Vec128<uint8_t, N> ShiftRight(const Vec128<uint8_t, N> v) { + const Simd<uint8_t, N> d8; + // Use raw instead of BitCast to support N=1. + const Vec128<uint8_t, N> shifted{ + ShiftRight<kBits>(Vec128<uint16_t>{v.raw}).raw}; + return shifted & Set(d8, 0xFF >> kBits); +} + +template <int kBits, size_t N> +HWY_API Vec128<int8_t, N> ShiftRight(const Vec128<int8_t, N> v) { + const Simd<int8_t, N> di; + const Simd<uint8_t, N> du; + const auto shifted = BitCast(di, ShiftRight<kBits>(BitCast(du, v))); + const auto shifted_sign = BitCast(di, Set(du, 0x80 >> kBits)); + return (shifted ^ shifted_sign) - shifted_sign; +} + +// ------------------------------ RotateRight (ShiftRight, Or) +template <int kBits, typename T, size_t N> +HWY_API Vec128<T, N> RotateRight(const Vec128<T, N> v) { + constexpr size_t kSizeInBits = sizeof(T) * 8; + static_assert(0 <= kBits && kBits < kSizeInBits, "Invalid shift count"); + if (kBits == 0) return v; + return Or(ShiftRight<kBits>(v), ShiftLeft<kSizeInBits - kBits>(v)); +} + +// ------------------------------ Shift lanes by same variable #bits + +// Unsigned +template <size_t N> +HWY_API Vec128<uint16_t, N> ShiftLeftSame(const Vec128<uint16_t, N> v, + const int bits) { + return Vec128<uint16_t, N>{wasm_i16x8_shl(v.raw, bits)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> ShiftRightSame(const Vec128<uint16_t, N> v, + const int bits) { + return Vec128<uint16_t, N>{wasm_u16x8_shr(v.raw, bits)}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> ShiftLeftSame(const Vec128<uint32_t, N> v, + const int bits) { + return Vec128<uint32_t, N>{wasm_i32x4_shl(v.raw, bits)}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> ShiftRightSame(const Vec128<uint32_t, N> v, + const int bits) { + return Vec128<uint32_t, N>{wasm_u32x4_shr(v.raw, bits)}; +} + +// Signed +template <size_t N> +HWY_API Vec128<int16_t, N> ShiftLeftSame(const Vec128<int16_t, N> v, + const int bits) { + return Vec128<int16_t, N>{wasm_i16x8_shl(v.raw, bits)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> ShiftRightSame(const Vec128<int16_t, N> v, + const int bits) { + return Vec128<int16_t, N>{wasm_i16x8_shr(v.raw, bits)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> ShiftLeftSame(const Vec128<int32_t, N> v, + const int bits) { + return Vec128<int32_t, N>{wasm_i32x4_shl(v.raw, bits)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> ShiftRightSame(const Vec128<int32_t, N> v, + const int bits) { + return Vec128<int32_t, N>{wasm_i32x4_shr(v.raw, bits)}; +} + +// 8-bit +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 1)> +HWY_API Vec128<T, N> ShiftLeftSame(const Vec128<T, N> v, const int bits) { + const Simd<T, N> d8; + // Use raw instead of BitCast to support N=1. + const Vec128<T, N> shifted{ + ShiftLeftSame(Vec128<MakeWide<T>>{v.raw}, bits).raw}; + return shifted & Set(d8, static_cast<T>((0xFF << bits) & 0xFF)); +} + +template <size_t N> +HWY_API Vec128<uint8_t, N> ShiftRightSame(Vec128<uint8_t, N> v, + const int bits) { + const Simd<uint8_t, N> d8; + // Use raw instead of BitCast to support N=1. + const Vec128<uint8_t, N> shifted{ + ShiftRightSame(Vec128<uint16_t>{v.raw}, bits).raw}; + return shifted & Set(d8, 0xFF >> bits); +} + +template <size_t N> +HWY_API Vec128<int8_t, N> ShiftRightSame(Vec128<int8_t, N> v, const int bits) { + const Simd<int8_t, N> di; + const Simd<uint8_t, N> du; + const auto shifted = BitCast(di, ShiftRightSame(BitCast(du, v), bits)); + const auto shifted_sign = BitCast(di, Set(du, 0x80 >> bits)); + return (shifted ^ shifted_sign) - shifted_sign; +} + +// ------------------------------ Minimum + +// Unsigned +template <size_t N> +HWY_API Vec128<uint8_t, N> Min(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{wasm_u8x16_min(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> Min(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{wasm_u16x8_min(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> Min(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { + return Vec128<uint32_t, N>{wasm_u32x4_min(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint64_t, N> Min(const Vec128<uint64_t, N> a, + const Vec128<uint64_t, N> b) { + alignas(16) float min[4]; + min[0] = + HWY_MIN(wasm_u64x2_extract_lane(a, 0), wasm_u64x2_extract_lane(b, 0)); + min[1] = + HWY_MIN(wasm_u64x2_extract_lane(a, 1), wasm_u64x2_extract_lane(b, 1)); + return Vec128<uint64_t, N>{wasm_v128_load(min)}; +} + +// Signed +template <size_t N> +HWY_API Vec128<int8_t, N> Min(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Vec128<int8_t, N>{wasm_i8x16_min(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> Min(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{wasm_i16x8_min(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> Min(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + return Vec128<int32_t, N>{wasm_i32x4_min(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int64_t, N> Min(const Vec128<int64_t, N> a, + const Vec128<int64_t, N> b) { + alignas(16) float min[4]; + min[0] = + HWY_MIN(wasm_i64x2_extract_lane(a, 0), wasm_i64x2_extract_lane(b, 0)); + min[1] = + HWY_MIN(wasm_i64x2_extract_lane(a, 1), wasm_i64x2_extract_lane(b, 1)); + return Vec128<int64_t, N>{wasm_v128_load(min)}; +} + +// Float +template <size_t N> +HWY_API Vec128<float, N> Min(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Vec128<float, N>{wasm_f32x4_min(a.raw, b.raw)}; +} + +// ------------------------------ Maximum + +// Unsigned +template <size_t N> +HWY_API Vec128<uint8_t, N> Max(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{wasm_u8x16_max(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> Max(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{wasm_u16x8_max(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> Max(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { + return Vec128<uint32_t, N>{wasm_u32x4_max(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint64_t, N> Max(const Vec128<uint64_t, N> a, + const Vec128<uint64_t, N> b) { + alignas(16) float max[4]; + max[0] = + HWY_MAX(wasm_u64x2_extract_lane(a, 0), wasm_u64x2_extract_lane(b, 0)); + max[1] = + HWY_MAX(wasm_u64x2_extract_lane(a, 1), wasm_u64x2_extract_lane(b, 1)); + return Vec128<int64_t, N>{wasm_v128_load(max)}; +} + +// Signed +template <size_t N> +HWY_API Vec128<int8_t, N> Max(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Vec128<int8_t, N>{wasm_i8x16_max(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> Max(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{wasm_i16x8_max(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> Max(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + return Vec128<int32_t, N>{wasm_i32x4_max(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int64_t, N> Max(const Vec128<int64_t, N> a, + const Vec128<int64_t, N> b) { + alignas(16) float max[4]; + max[0] = + HWY_MAX(wasm_i64x2_extract_lane(a, 0), wasm_i64x2_extract_lane(b, 0)); + max[1] = + HWY_MAX(wasm_i64x2_extract_lane(a, 1), wasm_i64x2_extract_lane(b, 1)); + return Vec128<int64_t, N>{wasm_v128_load(max)}; +} + +// Float +template <size_t N> +HWY_API Vec128<float, N> Max(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Vec128<float, N>{wasm_f32x4_max(a.raw, b.raw)}; +} + +// ------------------------------ Integer multiplication + +// Unsigned +template <size_t N> +HWY_API Vec128<uint16_t, N> operator*(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{wasm_i16x8_mul(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> operator*(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { + return Vec128<uint32_t, N>{wasm_i32x4_mul(a.raw, b.raw)}; +} + +// Signed +template <size_t N> +HWY_API Vec128<int16_t, N> operator*(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{wasm_i16x8_mul(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> operator*(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + return Vec128<int32_t, N>{wasm_i32x4_mul(a.raw, b.raw)}; +} + +// Returns the upper 16 bits of a * b in each lane. +template <size_t N> +HWY_API Vec128<uint16_t, N> MulHigh(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + // TODO(eustas): replace, when implemented in WASM. + const auto al = wasm_u32x4_extend_low_u16x8(a.raw); + const auto ah = wasm_u32x4_extend_high_u16x8(a.raw); + const auto bl = wasm_u32x4_extend_low_u16x8(b.raw); + const auto bh = wasm_u32x4_extend_high_u16x8(b.raw); + const auto l = wasm_i32x4_mul(al, bl); + const auto h = wasm_i32x4_mul(ah, bh); + // TODO(eustas): shift-right + narrow? + return Vec128<uint16_t, N>{ + wasm_i16x8_shuffle(l, h, 1, 3, 5, 7, 9, 11, 13, 15)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> MulHigh(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + // TODO(eustas): replace, when implemented in WASM. + const auto al = wasm_i32x4_extend_low_i16x8(a.raw); + const auto ah = wasm_i32x4_extend_high_i16x8(a.raw); + const auto bl = wasm_i32x4_extend_low_i16x8(b.raw); + const auto bh = wasm_i32x4_extend_high_i16x8(b.raw); + const auto l = wasm_i32x4_mul(al, bl); + const auto h = wasm_i32x4_mul(ah, bh); + // TODO(eustas): shift-right + narrow? + return Vec128<int16_t, N>{ + wasm_i16x8_shuffle(l, h, 1, 3, 5, 7, 9, 11, 13, 15)}; +} + +// Multiplies even lanes (0, 2 ..) and returns the double-width result. +template <size_t N> +HWY_API Vec128<int64_t, (N + 1) / 2> MulEven(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + // TODO(eustas): replace, when implemented in WASM. + const auto kEvenMask = wasm_i32x4_make(-1, 0, -1, 0); + const auto ae = wasm_v128_and(a.raw, kEvenMask); + const auto be = wasm_v128_and(b.raw, kEvenMask); + return Vec128<int64_t, (N + 1) / 2>{wasm_i64x2_mul(ae, be)}; +} +template <size_t N> +HWY_API Vec128<uint64_t, (N + 1) / 2> MulEven(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { + // TODO(eustas): replace, when implemented in WASM. + const auto kEvenMask = wasm_i32x4_make(-1, 0, -1, 0); + const auto ae = wasm_v128_and(a.raw, kEvenMask); + const auto be = wasm_v128_and(b.raw, kEvenMask); + return Vec128<uint64_t, (N + 1) / 2>{wasm_i64x2_mul(ae, be)}; +} + +// ------------------------------ Negate + +template <typename T, size_t N, HWY_IF_FLOAT(T)> +HWY_API Vec128<T, N> Neg(const Vec128<T, N> v) { + return Xor(v, SignBit(Simd<T, N>())); +} + +template <size_t N> +HWY_API Vec128<int8_t, N> Neg(const Vec128<int8_t, N> v) { + return Vec128<int8_t, N>{wasm_i8x16_neg(v.raw)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> Neg(const Vec128<int16_t, N> v) { + return Vec128<int16_t, N>{wasm_i16x8_neg(v.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> Neg(const Vec128<int32_t, N> v) { + return Vec128<int32_t, N>{wasm_i32x4_neg(v.raw)}; +} +template <size_t N> +HWY_API Vec128<int64_t, N> Neg(const Vec128<int64_t, N> v) { + return Vec128<int64_t, N>{wasm_i64x2_neg(v.raw)}; +} + +// ------------------------------ Floating-point mul / div + +template <size_t N> +HWY_API Vec128<float, N> operator*(Vec128<float, N> a, Vec128<float, N> b) { + return Vec128<float, N>{wasm_f32x4_mul(a.raw, b.raw)}; +} + +template <size_t N> +HWY_API Vec128<float, N> operator/(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Vec128<float, N>{wasm_f32x4_div(a.raw, b.raw)}; +} + +// Approximate reciprocal +template <size_t N> +HWY_API Vec128<float, N> ApproximateReciprocal(const Vec128<float, N> v) { + const Vec128<float, N> one = Vec128<float, N>{wasm_f32x4_splat(1.0f)}; + return one / v; +} + +// Absolute value of difference. +template <size_t N> +HWY_API Vec128<float, N> AbsDiff(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Abs(a - b); +} + +// ------------------------------ Floating-point multiply-add variants + +// Returns mul * x + add +template <size_t N> +HWY_API Vec128<float, N> MulAdd(const Vec128<float, N> mul, + const Vec128<float, N> x, + const Vec128<float, N> add) { + // TODO(eustas): replace, when implemented in WASM. + // TODO(eustas): is it wasm_f32x4_qfma? + return mul * x + add; +} + +// Returns add - mul * x +template <size_t N> +HWY_API Vec128<float, N> NegMulAdd(const Vec128<float, N> mul, + const Vec128<float, N> x, + const Vec128<float, N> add) { + // TODO(eustas): replace, when implemented in WASM. + return add - mul * x; +} + +// Returns mul * x - sub +template <size_t N> +HWY_API Vec128<float, N> MulSub(const Vec128<float, N> mul, + const Vec128<float, N> x, + const Vec128<float, N> sub) { + // TODO(eustas): replace, when implemented in WASM. + // TODO(eustas): is it wasm_f32x4_qfms? + return mul * x - sub; +} + +// Returns -mul * x - sub +template <size_t N> +HWY_API Vec128<float, N> NegMulSub(const Vec128<float, N> mul, + const Vec128<float, N> x, + const Vec128<float, N> sub) { + // TODO(eustas): replace, when implemented in WASM. + return Neg(mul) * x - sub; +} + +// ------------------------------ Floating-point square root + +// Full precision square root +template <size_t N> +HWY_API Vec128<float, N> Sqrt(const Vec128<float, N> v) { + return Vec128<float, N>{wasm_f32x4_sqrt(v.raw)}; +} + +// Approximate reciprocal square root +template <size_t N> +HWY_API Vec128<float, N> ApproximateReciprocalSqrt(const Vec128<float, N> v) { + // TODO(eustas): find cheaper a way to calculate this. + const Vec128<float, N> one = Vec128<float, N>{wasm_f32x4_splat(1.0f)}; + return one / Sqrt(v); +} + +// ------------------------------ Floating-point rounding + +// Toward nearest integer, ties to even +template <size_t N> +HWY_API Vec128<float, N> Round(const Vec128<float, N> v) { + return Vec128<float, N>{wasm_f32x4_nearest(v.raw)}; +} + +// Toward zero, aka truncate +template <size_t N> +HWY_API Vec128<float, N> Trunc(const Vec128<float, N> v) { + return Vec128<float, N>{wasm_f32x4_trunc(v.raw)}; +} + +// Toward +infinity, aka ceiling +template <size_t N> +HWY_API Vec128<float, N> Ceil(const Vec128<float, N> v) { + return Vec128<float, N>{wasm_f32x4_ceil(v.raw)}; +} + +// Toward -infinity, aka floor +template <size_t N> +HWY_API Vec128<float, N> Floor(const Vec128<float, N> v) { + return Vec128<float, N>{wasm_f32x4_floor(v.raw)}; +} + +// ================================================== COMPARE + +// Comparisons fill a lane with 1-bits if the condition is true, else 0. + +template <typename TFrom, typename TTo, size_t N> +HWY_API Mask128<TTo, N> RebindMask(Simd<TTo, N> /*tag*/, Mask128<TFrom, N> m) { + static_assert(sizeof(TFrom) == sizeof(TTo), "Must have same size"); + return Mask128<TTo, N>{m.raw}; +} + +template <typename T, size_t N> +HWY_API Mask128<T, N> TestBit(Vec128<T, N> v, Vec128<T, N> bit) { + static_assert(!hwy::IsFloat<T>(), "Only integer vectors supported"); + return (v & bit) == bit; +} + +// ------------------------------ Equality + +// Unsigned +template <size_t N> +HWY_API Mask128<uint8_t, N> operator==(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Mask128<uint8_t, N>{wasm_i8x16_eq(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<uint16_t, N> operator==(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Mask128<uint16_t, N>{wasm_i16x8_eq(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<uint32_t, N> operator==(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { + return Mask128<uint32_t, N>{wasm_i32x4_eq(a.raw, b.raw)}; +} + +// Signed +template <size_t N> +HWY_API Mask128<int8_t, N> operator==(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Mask128<int8_t, N>{wasm_i8x16_eq(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<int16_t, N> operator==(Vec128<int16_t, N> a, + Vec128<int16_t, N> b) { + return Mask128<int16_t, N>{wasm_i16x8_eq(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<int32_t, N> operator==(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + return Mask128<int32_t, N>{wasm_i32x4_eq(a.raw, b.raw)}; +} + +// Float +template <size_t N> +HWY_API Mask128<float, N> operator==(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Mask128<float, N>{wasm_f32x4_eq(a.raw, b.raw)}; +} + +// ------------------------------ Inequality + +// Unsigned +template <size_t N> +HWY_API Mask128<uint8_t, N> operator!=(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Mask128<uint8_t, N>{wasm_i8x16_ne(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<uint16_t, N> operator!=(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Mask128<uint16_t, N>{wasm_i16x8_ne(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<uint32_t, N> operator!=(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { + return Mask128<uint32_t, N>{wasm_i32x4_ne(a.raw, b.raw)}; +} + +// Signed +template <size_t N> +HWY_API Mask128<int8_t, N> operator!=(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Mask128<int8_t, N>{wasm_i8x16_ne(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<int16_t, N> operator!=(Vec128<int16_t, N> a, + Vec128<int16_t, N> b) { + return Mask128<int16_t, N>{wasm_i16x8_ne(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<int32_t, N> operator!=(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + return Mask128<int32_t, N>{wasm_i32x4_ne(a.raw, b.raw)}; +} + +// Float +template <size_t N> +HWY_API Mask128<float, N> operator!=(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Mask128<float, N>{wasm_f32x4_ne(a.raw, b.raw)}; +} + +// ------------------------------ Strict inequality + +template <size_t N> +HWY_API Mask128<int8_t, N> operator>(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Mask128<int8_t, N>{wasm_i8x16_gt(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<int16_t, N> operator>(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Mask128<int16_t, N>{wasm_i16x8_gt(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<int32_t, N> operator>(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + return Mask128<int32_t, N>{wasm_i32x4_gt(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<int64_t, N> operator>(const Vec128<int64_t, N> a, + const Vec128<int64_t, N> b) { + const Simd<int32_t, N * 2> d32; + const auto a32 = BitCast(d32, a); + const auto b32 = BitCast(d32, b); + // If the upper half is less than or greater, this is the answer. + const auto m_gt = a32 < b32; + + // Otherwise, the lower half decides. + const auto m_eq = a32 == b32; + const auto lo_in_hi = wasm_i32x4_shuffle(m_gt, m_gt, 2, 2, 0, 0); + const auto lo_gt = And(m_eq, lo_in_hi); + + const auto gt = Or(lo_gt, m_gt); + // Copy result in upper 32 bits to lower 32 bits. + return Mask128<int64_t, N>{wasm_i32x4_shuffle(gt, gt, 3, 3, 1, 1)}; +} + +template <typename T, size_t N, HWY_IF_UNSIGNED(T)> +HWY_API Mask128<T, N> operator>(Vec128<T, N> a, Vec128<T, N> b) { + const Simd<T, N> du; + const RebindToSigned<decltype(du)> di; + const Vec128<T, N> msb = Set(du, (LimitsMax<T>() >> 1) + 1); + return RebindMask(du, BitCast(di, Xor(a, msb)) > BitCast(di, Xor(b, msb))); +} + +template <size_t N> +HWY_API Mask128<float, N> operator>(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Mask128<float, N>{wasm_f32x4_gt(a.raw, b.raw)}; +} + +template <typename T, size_t N> +HWY_API Mask128<T, N> operator<(const Vec128<T, N> a, const Vec128<T, N> b) { + return operator>(b, a); +} + +// ------------------------------ Weak inequality + +// Float <= >= +template <size_t N> +HWY_API Mask128<float, N> operator<=(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Mask128<float, N>{wasm_f32x4_le(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<float, N> operator>=(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Mask128<float, N>{wasm_f32x4_ge(a.raw, b.raw)}; +} + +// ------------------------------ FirstN (Iota, Lt) + +template <typename T, size_t N> +HWY_API Mask128<T, N> FirstN(const Simd<T, N> d, size_t num) { + const RebindToSigned<decltype(d)> di; // Signed comparisons may be cheaper. + return RebindMask(d, Iota(di, 0) < Set(di, static_cast<MakeSigned<T>>(num))); +} + +// ================================================== LOGICAL + +// ------------------------------ Not + +template <typename T, size_t N> +HWY_API Vec128<T, N> Not(Vec128<T, N> v) { + return Vec128<T, N>{wasm_v128_not(v.raw)}; +} + +// ------------------------------ And + +template <typename T, size_t N> +HWY_API Vec128<T, N> And(Vec128<T, N> a, Vec128<T, N> b) { + return Vec128<T, N>{wasm_v128_and(a.raw, b.raw)}; +} + +// ------------------------------ AndNot + +// Returns ~not_mask & mask. +template <typename T, size_t N> +HWY_API Vec128<T, N> AndNot(Vec128<T, N> not_mask, Vec128<T, N> mask) { + return Vec128<T, N>{wasm_v128_andnot(mask.raw, not_mask.raw)}; +} + +// ------------------------------ Or + +template <typename T, size_t N> +HWY_API Vec128<T, N> Or(Vec128<T, N> a, Vec128<T, N> b) { + return Vec128<T, N>{wasm_v128_or(a.raw, b.raw)}; +} + +// ------------------------------ Xor + +template <typename T, size_t N> +HWY_API Vec128<T, N> Xor(Vec128<T, N> a, Vec128<T, N> b) { + return Vec128<T, N>{wasm_v128_xor(a.raw, b.raw)}; +} + +// ------------------------------ Operator overloads (internal-only if float) + +template <typename T, size_t N> +HWY_API Vec128<T, N> operator&(const Vec128<T, N> a, const Vec128<T, N> b) { + return And(a, b); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> operator|(const Vec128<T, N> a, const Vec128<T, N> b) { + return Or(a, b); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> operator^(const Vec128<T, N> a, const Vec128<T, N> b) { + return Xor(a, b); +} + +// ------------------------------ CopySign + +template <typename T, size_t N> +HWY_API Vec128<T, N> CopySign(const Vec128<T, N> magn, + const Vec128<T, N> sign) { + static_assert(IsFloat<T>(), "Only makes sense for floating-point"); + const auto msb = SignBit(Simd<T, N>()); + return Or(AndNot(msb, magn), And(msb, sign)); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> CopySignToAbs(const Vec128<T, N> abs, + const Vec128<T, N> sign) { + static_assert(IsFloat<T>(), "Only makes sense for floating-point"); + return Or(abs, And(SignBit(Simd<T, N>()), sign)); +} + +// ------------------------------ BroadcastSignBit (compare) + +template <typename T, size_t N, HWY_IF_NOT_LANE_SIZE(T, 1)> +HWY_API Vec128<T, N> BroadcastSignBit(const Vec128<T, N> v) { + return ShiftRight<sizeof(T) * 8 - 1>(v); +} +template <size_t N> +HWY_API Vec128<int8_t, N> BroadcastSignBit(const Vec128<int8_t, N> v) { + return VecFromMask(Simd<int8_t, N>(), v < Zero(Simd<int8_t, N>())); +} + +// ------------------------------ Mask + +// Mask and Vec are the same (true = FF..FF). +template <typename T, size_t N> +HWY_API Mask128<T, N> MaskFromVec(const Vec128<T, N> v) { + return Mask128<T, N>{v.raw}; +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> VecFromMask(Simd<T, N> /* tag */, Mask128<T, N> v) { + return Vec128<T, N>{v.raw}; +} + +// DEPRECATED +template <typename T, size_t N> +HWY_API Vec128<T, N> VecFromMask(const Mask128<T, N> v) { + return Vec128<T, N>{v.raw}; +} + +// mask ? yes : no +template <typename T, size_t N> +HWY_API Vec128<T, N> IfThenElse(Mask128<T, N> mask, Vec128<T, N> yes, + Vec128<T, N> no) { + return Vec128<T, N>{wasm_v128_bitselect(yes.raw, no.raw, mask.raw)}; +} + +// mask ? yes : 0 +template <typename T, size_t N> +HWY_API Vec128<T, N> IfThenElseZero(Mask128<T, N> mask, Vec128<T, N> yes) { + return yes & VecFromMask(Simd<T, N>(), mask); +} + +// mask ? 0 : no +template <typename T, size_t N> +HWY_API Vec128<T, N> IfThenZeroElse(Mask128<T, N> mask, Vec128<T, N> no) { + return AndNot(VecFromMask(Simd<T, N>(), mask), no); +} + +template <typename T, size_t N, HWY_IF_FLOAT(T)> +HWY_API Vec128<T, N> ZeroIfNegative(Vec128<T, N> v) { + const Simd<T, N> d; + const auto zero = Zero(d); + return IfThenElse(Mask128<T, N>{(v > zero).raw}, v, zero); +} + +// ------------------------------ Mask logical + +template <typename T, size_t N> +HWY_API Mask128<T, N> Not(const Mask128<T, N> m) { + return MaskFromVec(Not(VecFromMask(Simd<T, N>(), m))); +} + +template <typename T, size_t N> +HWY_API Mask128<T, N> And(const Mask128<T, N> a, Mask128<T, N> b) { + const Simd<T, N> d; + return MaskFromVec(And(VecFromMask(d, a), VecFromMask(d, b))); +} + +template <typename T, size_t N> +HWY_API Mask128<T, N> AndNot(const Mask128<T, N> a, Mask128<T, N> b) { + const Simd<T, N> d; + return MaskFromVec(AndNot(VecFromMask(d, a), VecFromMask(d, b))); +} + +template <typename T, size_t N> +HWY_API Mask128<T, N> Or(const Mask128<T, N> a, Mask128<T, N> b) { + const Simd<T, N> d; + return MaskFromVec(Or(VecFromMask(d, a), VecFromMask(d, b))); +} + +template <typename T, size_t N> +HWY_API Mask128<T, N> Xor(const Mask128<T, N> a, Mask128<T, N> b) { + const Simd<T, N> d; + return MaskFromVec(Xor(VecFromMask(d, a), VecFromMask(d, b))); +} + +// ------------------------------ Shl (BroadcastSignBit, IfThenElse) + +// The x86 multiply-by-Pow2() trick will not work because WASM saturates +// float->int correctly to 2^31-1 (not 2^31). Because WASM's shifts take a +// scalar count operand, per-lane shift instructions would require extract_lane +// for each lane, and hoping that shuffle is correctly mapped to a native +// instruction. Using non-vector shifts would incur a store-load forwarding +// stall when loading the result vector. We instead test bits of the shift +// count to "predicate" a shift of the entire vector by a constant. + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Vec128<T, N> operator<<(Vec128<T, N> v, const Vec128<T, N> bits) { + const Simd<T, N> d; + Mask128<T, N> mask; + // Need a signed type for BroadcastSignBit. + auto test = BitCast(RebindToSigned<decltype(d)>(), bits); + // Move the highest valid bit of the shift count into the sign bit. + test = ShiftLeft<12>(test); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftLeft<8>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftLeft<4>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftLeft<2>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + return IfThenElse(mask, ShiftLeft<1>(v), v); +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T, N> operator<<(Vec128<T, N> v, const Vec128<T, N> bits) { + const Simd<T, N> d; + Mask128<T, N> mask; + // Need a signed type for BroadcastSignBit. + auto test = BitCast(RebindToSigned<decltype(d)>(), bits); + // Move the highest valid bit of the shift count into the sign bit. + test = ShiftLeft<27>(test); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftLeft<16>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftLeft<8>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftLeft<4>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftLeft<2>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + return IfThenElse(mask, ShiftLeft<1>(v), v); +} + +// ------------------------------ Shr (BroadcastSignBit, IfThenElse) + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Vec128<T, N> operator>>(Vec128<T, N> v, const Vec128<T, N> bits) { + const Simd<T, N> d; + Mask128<T, N> mask; + // Need a signed type for BroadcastSignBit. + auto test = BitCast(RebindToSigned<decltype(d)>(), bits); + // Move the highest valid bit of the shift count into the sign bit. + test = ShiftLeft<12>(test); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftRight<8>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftRight<4>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftRight<2>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + return IfThenElse(mask, ShiftRight<1>(v), v); +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T, N> operator>>(Vec128<T, N> v, const Vec128<T, N> bits) { + const Simd<T, N> d; + Mask128<T, N> mask; + // Need a signed type for BroadcastSignBit. + auto test = BitCast(RebindToSigned<decltype(d)>(), bits); + // Move the highest valid bit of the shift count into the sign bit. + test = ShiftLeft<27>(test); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftRight<16>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftRight<8>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftRight<4>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftRight<2>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + return IfThenElse(mask, ShiftRight<1>(v), v); +} + +// ================================================== MEMORY + +// ------------------------------ Load + +template <typename T> +HWY_API Vec128<T> Load(Full128<T> /* tag */, const T* HWY_RESTRICT aligned) { + return Vec128<T>{wasm_v128_load(aligned)}; +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> MaskedLoad(Mask128<T, N> m, Simd<T, N> d, + const T* HWY_RESTRICT aligned) { + return IfThenElseZero(m, Load(d, aligned)); +} + +// Partial load. +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_API Vec128<T, N> Load(Simd<T, N> /* tag */, const T* HWY_RESTRICT p) { + Vec128<T, N> v; + CopyBytes<sizeof(T) * N>(p, &v); + return v; +} + +// LoadU == Load. +template <typename T, size_t N> +HWY_API Vec128<T, N> LoadU(Simd<T, N> d, const T* HWY_RESTRICT p) { + return Load(d, p); +} + +// 128-bit SIMD => nothing to duplicate, same as an unaligned load. +template <typename T, size_t N, HWY_IF_LE128(T, N)> +HWY_API Vec128<T, N> LoadDup128(Simd<T, N> d, const T* HWY_RESTRICT p) { + return Load(d, p); +} + +// ------------------------------ Store + +template <typename T> +HWY_API void Store(Vec128<T> v, Full128<T> /* tag */, T* HWY_RESTRICT aligned) { + wasm_v128_store(aligned, v.raw); +} + +// Partial store. +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_API void Store(Vec128<T, N> v, Simd<T, N> /* tag */, T* HWY_RESTRICT p) { + CopyBytes<sizeof(T) * N>(&v, p); +} + +HWY_API void Store(const Vec128<float, 1> v, Simd<float, 1> /* tag */, + float* HWY_RESTRICT p) { + *p = wasm_f32x4_extract_lane(v.raw, 0); +} + +// StoreU == Store. +template <typename T, size_t N> +HWY_API void StoreU(Vec128<T, N> v, Simd<T, N> d, T* HWY_RESTRICT p) { + Store(v, d, p); +} + +// ------------------------------ Non-temporal stores + +// Same as aligned stores on non-x86. + +template <typename T, size_t N> +HWY_API void Stream(Vec128<T, N> v, Simd<T, N> /* tag */, + T* HWY_RESTRICT aligned) { + wasm_v128_store(aligned, v.raw); +} + +// ------------------------------ Scatter (Store) + +template <typename T, size_t N, typename Offset, HWY_IF_LE128(T, N)> +HWY_API void ScatterOffset(Vec128<T, N> v, Simd<T, N> d, T* HWY_RESTRICT base, + const Vec128<Offset, N> offset) { + static_assert(sizeof(T) == sizeof(Offset), "Must match for portability"); + + alignas(16) T lanes[N]; + Store(v, d, lanes); + + alignas(16) Offset offset_lanes[N]; + Store(offset, Simd<Offset, N>(), offset_lanes); + + uint8_t* base_bytes = reinterpret_cast<uint8_t*>(base); + for (size_t i = 0; i < N; ++i) { + CopyBytes<sizeof(T)>(&lanes[i], base_bytes + offset_lanes[i]); + } +} + +template <typename T, size_t N, typename Index, HWY_IF_LE128(T, N)> +HWY_API void ScatterIndex(Vec128<T, N> v, Simd<T, N> d, T* HWY_RESTRICT base, + const Vec128<Index, N> index) { + static_assert(sizeof(T) == sizeof(Index), "Must match for portability"); + + alignas(16) T lanes[N]; + Store(v, d, lanes); + + alignas(16) Index index_lanes[N]; + Store(index, Simd<Index, N>(), index_lanes); + + for (size_t i = 0; i < N; ++i) { + base[index_lanes[i]] = lanes[i]; + } +} + +// ------------------------------ Gather (Load/Store) + +template <typename T, size_t N, typename Offset> +HWY_API Vec128<T, N> GatherOffset(const Simd<T, N> d, + const T* HWY_RESTRICT base, + const Vec128<Offset, N> offset) { + static_assert(sizeof(T) == sizeof(Offset), "Must match for portability"); + + alignas(16) Offset offset_lanes[N]; + Store(offset, Simd<Offset, N>(), offset_lanes); + + alignas(16) T lanes[N]; + const uint8_t* base_bytes = reinterpret_cast<const uint8_t*>(base); + for (size_t i = 0; i < N; ++i) { + CopyBytes<sizeof(T)>(base_bytes + offset_lanes[i], &lanes[i]); + } + return Load(d, lanes); +} + +template <typename T, size_t N, typename Index> +HWY_API Vec128<T, N> GatherIndex(const Simd<T, N> d, const T* HWY_RESTRICT base, + const Vec128<Index, N> index) { + static_assert(sizeof(T) == sizeof(Index), "Must match for portability"); + + alignas(16) Index index_lanes[N]; + Store(index, Simd<Index, N>(), index_lanes); + + alignas(16) T lanes[N]; + for (size_t i = 0; i < N; ++i) { + lanes[i] = base[index_lanes[i]]; + } + return Load(d, lanes); +} + +// ================================================== SWIZZLE + +// ------------------------------ Extract lane + +// Gets the single value stored in a vector/part. +template <size_t N> +HWY_API uint8_t GetLane(const Vec128<uint8_t, N> v) { + return static_cast<uint8_t>(wasm_i8x16_extract_lane(v.raw, 0)); +} +template <size_t N> +HWY_API int8_t GetLane(const Vec128<int8_t, N> v) { + return static_cast<int8_t>(wasm_i8x16_extract_lane(v.raw, 0)); +} +template <size_t N> +HWY_API uint16_t GetLane(const Vec128<uint16_t, N> v) { + return static_cast<uint16_t>(wasm_i16x8_extract_lane(v.raw, 0)); +} +template <size_t N> +HWY_API int16_t GetLane(const Vec128<int16_t, N> v) { + return static_cast<int16_t>(wasm_i16x8_extract_lane(v.raw, 0)); +} +template <size_t N> +HWY_API uint32_t GetLane(const Vec128<uint32_t, N> v) { + return static_cast<uint32_t>(wasm_i32x4_extract_lane(v.raw, 0)); +} +template <size_t N> +HWY_API int32_t GetLane(const Vec128<int32_t, N> v) { + return static_cast<int32_t>(wasm_i32x4_extract_lane(v.raw, 0)); +} +template <size_t N> +HWY_API uint64_t GetLane(const Vec128<uint64_t, N> v) { + return static_cast<uint64_t>(wasm_i64x2_extract_lane(v.raw, 0)); +} +template <size_t N> +HWY_API int64_t GetLane(const Vec128<int64_t, N> v) { + return static_cast<int64_t>(wasm_i64x2_extract_lane(v.raw, 0)); +} + +template <size_t N> +HWY_API float GetLane(const Vec128<float, N> v) { + return wasm_f32x4_extract_lane(v.raw, 0); +} + +// ------------------------------ LowerHalf + +template <typename T, size_t N> +HWY_API Vec128<T, N / 2> LowerHalf(Simd<T, N / 2> /* tag */, Vec128<T, N> v) { + return Vec128<T, N / 2>{v.raw}; +} + +template <typename T, size_t N> +HWY_API Vec128<T, N / 2> LowerHalf(Vec128<T, N> v) { + return LowerHalf(Simd<T, N / 2>(), v); +} + +// ------------------------------ ShiftLeftBytes + +// 0x01..0F, kBytes = 1 => 0x02..0F00 +template <int kBytes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftLeftBytes(Simd<T, N> /* tag */, Vec128<T, N> v) { + static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes"); + const __i8x16 zero = wasm_i8x16_splat(0); + switch (kBytes) { + case 0: + return v; + + case 1: + return Vec128<T, N>{wasm_i8x16_shuffle(v.raw, zero, 16, 0, 1, 2, 3, 4, 5, + 6, 7, 8, 9, 10, 11, 12, 13, 14)}; + + case 2: + return Vec128<T, N>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 0, 1, 2, 3, 4, + 5, 6, 7, 8, 9, 10, 11, 12, 13)}; + + case 3: + return Vec128<T, N>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 0, 1, 2, + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12)}; + + case 4: + return Vec128<T, N>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 0, 1, + 2, 3, 4, 5, 6, 7, 8, 9, 10, 11)}; + + case 5: + return Vec128<T, N>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, 0, + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10)}; + + case 6: + return Vec128<T, N>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, + 16, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9)}; + + case 7: + return Vec128<T, N>{wasm_i8x16_shuffle( + v.raw, zero, 16, 16, 16, 16, 16, 16, 16, 0, 1, 2, 3, 4, 5, 6, 7, 8)}; + + case 8: + return Vec128<T, N>{wasm_i8x16_shuffle( + v.raw, zero, 16, 16, 16, 16, 16, 16, 16, 16, 0, 1, 2, 3, 4, 5, 6, 7)}; + + case 9: + return Vec128<T, N>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 0, 1, 2, 3, 4, 5, + 6)}; + + case 10: + return Vec128<T, N>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 0, 1, 2, 3, 4, + 5)}; + + case 11: + return Vec128<T, N>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 0, 1, 2, 3, + 4)}; + + case 12: + return Vec128<T, N>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 0, 1, + 2, 3)}; + + case 13: + return Vec128<T, N>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 0, + 1, 2)}; + + case 14: + return Vec128<T, N>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, + 0, 1)}; + + case 15: + return Vec128<T, N>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 0)}; + } + return Vec128<T, N>{zero}; +} + +template <int kBytes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftLeftBytes(Vec128<T, N> v) { + return ShiftLeftBytes<kBytes>(Simd<T, N>(), v); +} + +// ------------------------------ ShiftLeftLanes + +template <int kLanes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftLeftLanes(Simd<T, N> d, const Vec128<T, N> v) { + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, ShiftLeftBytes<kLanes * sizeof(T)>(BitCast(d8, v))); +} + +template <int kLanes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftLeftLanes(const Vec128<T, N> v) { + return ShiftLeftLanes<kLanes>(Simd<T, N>(), v); +} + +// ------------------------------ ShiftRightBytes +namespace detail { + +// Helper function allows zeroing invalid lanes in caller. +template <int kBytes, typename T, size_t N> +HWY_API __i8x16 ShrBytes(const Vec128<T, N> v) { + static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes"); + const __i8x16 zero = wasm_i8x16_splat(0); + + switch (kBytes) { + case 0: + return v.raw; + + case 1: + return wasm_i8x16_shuffle(v.raw, zero, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16); + + case 2: + return wasm_i8x16_shuffle(v.raw, zero, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16, 16); + + case 3: + return wasm_i8x16_shuffle(v.raw, zero, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16, 16, 16); + + case 4: + return wasm_i8x16_shuffle(v.raw, zero, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 16, 16, 16, 16); + + case 5: + return wasm_i8x16_shuffle(v.raw, zero, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 16, 16, 16, 16); + + case 6: + return wasm_i8x16_shuffle(v.raw, zero, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 16, 16, 16, 16); + + case 7: + return wasm_i8x16_shuffle(v.raw, zero, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 16, 16, 16, 16, 16); + + case 8: + return wasm_i8x16_shuffle(v.raw, zero, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 16, 16, 16, 16, 16, 16, 16); + + case 9: + return wasm_i8x16_shuffle(v.raw, zero, 9, 10, 11, 12, 13, 14, 15, 16, 16, + 16, 16, 16, 16, 16, 16, 16); + + case 10: + return wasm_i8x16_shuffle(v.raw, zero, 10, 11, 12, 13, 14, 15, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16); + + case 11: + return wasm_i8x16_shuffle(v.raw, zero, 11, 12, 13, 14, 15, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16); + + case 12: + return wasm_i8x16_shuffle(v.raw, zero, 12, 13, 14, 15, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16); + + case 13: + return wasm_i8x16_shuffle(v.raw, zero, 13, 14, 15, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16); + + case 14: + return wasm_i8x16_shuffle(v.raw, zero, 14, 15, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16); + + case 15: + return wasm_i8x16_shuffle(v.raw, zero, 15, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16); + case 16: + return zero; + } +} + +} // namespace detail + +// 0x01..0F, kBytes = 1 => 0x0001..0E +template <int kBytes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftRightBytes(Simd<T, N> /* tag */, Vec128<T, N> v) { + // For partial vectors, clear upper lanes so we shift in zeros. + if (N != 16 / sizeof(T)) { + const Vec128<T> vfull{v.raw}; + v = Vec128<T, N>{IfThenElseZero(FirstN(Full128<T>(), N), vfull).raw}; + } + return Vec128<T, N>{detail::ShrBytes<kBytes>(v)}; +} + +// ------------------------------ ShiftRightLanes +template <int kLanes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftRightLanes(Simd<T, N> d, const Vec128<T, N> v) { + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, ShiftRightBytes<kLanes * sizeof(T)>(BitCast(d8, v))); +} + +// ------------------------------ UpperHalf (ShiftRightBytes) + +// Full input: copy hi into lo (smaller instruction encoding than shifts). +template <typename T> +HWY_API Vec128<T, 8 / sizeof(T)> UpperHalf(Half<Full128<T>> /* tag */, + const Vec128<T> v) { + return Vec128<T, 8 / sizeof(T)>{wasm_i32x4_shuffle(v.raw, v.raw, 2, 3, 2, 3)}; +} +HWY_API Vec128<float, 2> UpperHalf(Half<Full128<float>> /* tag */, + const Vec128<float> v) { + return Vec128<float, 2>{wasm_i32x4_shuffle(v.raw, v.raw, 2, 3, 2, 3)}; +} + +// Partial +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_API Vec128<T, (N + 1) / 2> UpperHalf(Half<Simd<T, N>> /* tag */, + Vec128<T, N> v) { + const Simd<T, N> d; + const auto vu = BitCast(RebindToUnsigned<decltype(d)>(), v); + const auto upper = BitCast(d, ShiftRightBytes<N * sizeof(T) / 2>(vu)); + return Vec128<T, (N + 1) / 2>{upper.raw}; +} + +// ------------------------------ CombineShiftRightBytes + +template <int kBytes, typename T, class V = Vec128<T>> +HWY_API V CombineShiftRightBytes(Full128<T> /* tag */, V hi, V lo) { + static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes"); + switch (kBytes) { + case 0: + return lo; + + case 1: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16)}; + + case 2: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17)}; + + case 3: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16, 17, 18)}; + + case 4: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 4, 5, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16, 17, 18, 19)}; + + case 5: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 5, 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 16, 17, 18, 19, 20)}; + + case 6: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 16, 17, 18, 19, 20, 21)}; + + case 7: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22)}; + + case 8: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23)}; + + case 9: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24)}; + + case 10: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25)}; + + case 11: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 11, 12, 13, 14, 15, 16, 17, + 18, 19, 20, 21, 22, 23, 24, 25, 26)}; + + case 12: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27)}; + + case 13: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28)}; + + case 14: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29)}; + + case 15: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30)}; + } + return hi; +} + +template <int kBytes, typename T, size_t N, HWY_IF_LE64(T, N), + class V = Vec128<T, N>> +HWY_API V CombineShiftRightBytes(Simd<T, N> d, V hi, V lo) { + constexpr size_t kSize = N * sizeof(T); + static_assert(0 < kBytes && kBytes < kSize, "kBytes invalid"); + const Repartition<uint8_t, decltype(d)> d8; + const Full128<uint8_t> d_full8; + using V8 = VFromD<decltype(d_full8)>; + const V8 hi8{BitCast(d8, hi).raw}; + // Move into most-significant bytes + const V8 lo8 = ShiftLeftBytes<16 - kSize>(V8{BitCast(d8, lo).raw}); + const V8 r = CombineShiftRightBytes<16 - kSize + kBytes>(d_full8, hi8, lo8); + return V{BitCast(Full128<T>(), r).raw}; +} + +// ------------------------------ Broadcast/splat any lane + +// Unsigned +template <int kLane, size_t N> +HWY_API Vec128<uint16_t, N> Broadcast(const Vec128<uint16_t, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<uint16_t, N>{wasm_i16x8_shuffle( + v.raw, v.raw, kLane, kLane, kLane, kLane, kLane, kLane, kLane, kLane)}; +} +template <int kLane, size_t N> +HWY_API Vec128<uint32_t, N> Broadcast(const Vec128<uint32_t, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<uint32_t, N>{ + wasm_i32x4_shuffle(v.raw, v.raw, kLane, kLane, kLane, kLane)}; +} + +// Signed +template <int kLane, size_t N> +HWY_API Vec128<int16_t, N> Broadcast(const Vec128<int16_t, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<int16_t, N>{wasm_i16x8_shuffle( + v.raw, v.raw, kLane, kLane, kLane, kLane, kLane, kLane, kLane, kLane)}; +} +template <int kLane, size_t N> +HWY_API Vec128<int32_t, N> Broadcast(const Vec128<int32_t, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<int32_t, N>{ + wasm_i32x4_shuffle(v.raw, v.raw, kLane, kLane, kLane, kLane)}; +} + +// Float +template <int kLane, size_t N> +HWY_API Vec128<float, N> Broadcast(const Vec128<float, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<float, N>{ + wasm_i32x4_shuffle(v.raw, v.raw, kLane, kLane, kLane, kLane)}; +} + +// ------------------------------ TableLookupBytes + +// Returns vector of bytes[from[i]]. "from" is also interpreted as bytes, i.e. +// lane indices in [0, 16). +template <typename T, size_t N, typename TI, size_t NI> +HWY_API Vec128<TI, NI> TableLookupBytes(const Vec128<T, N> bytes, + const Vec128<TI, NI> from) { +// Not yet available in all engines, see +// https://github.com/WebAssembly/simd/blob/bdcc304b2d379f4601c2c44ea9b44ed9484fde7e/proposals/simd/ImplementationStatus.md +// V8 implementation of this had a bug, fixed on 2021-04-03: +// https://chromium-review.googlesource.com/c/v8/v8/+/2822951 +#if 0 + return Vec128<TI, NI>{wasm_i8x16_swizzle(bytes.raw, from.raw)}; +#else + alignas(16) uint8_t control[16]; + alignas(16) uint8_t input[16]; + alignas(16) uint8_t output[16]; + wasm_v128_store(control, from.raw); + wasm_v128_store(input, bytes.raw); + for (size_t i = 0; i < 16; ++i) { + output[i] = control[i] < 16 ? input[control[i]] : 0; + } + return Vec128<TI, NI>{wasm_v128_load(output)}; +#endif +} + +template <typename T, size_t N, typename TI, size_t NI> +HWY_API Vec128<TI, NI> TableLookupBytesOr0(const Vec128<T, N> bytes, + const Vec128<TI, NI> from) { + const Simd<TI, NI> d; + // Mask size must match vector type, so cast everything to this type. + Repartition<int8_t, decltype(d)> di8; + Repartition<int8_t, Simd<T, N>> d_bytes8; + const auto msb = BitCast(di8, from) < Zero(di8); + const auto lookup = + TableLookupBytes(BitCast(d_bytes8, bytes), BitCast(di8, from)); + return BitCast(d, IfThenZeroElse(msb, lookup)); +} + +// ------------------------------ Hard-coded shuffles + +// Notation: let Vec128<int32_t> have lanes 3,2,1,0 (0 is least-significant). +// Shuffle0321 rotates one lane to the right (the previous least-significant +// lane is now most-significant). These could also be implemented via +// CombineShiftRightBytes but the shuffle_abcd notation is more convenient. + +// Swap 32-bit halves in 64-bit halves. +HWY_API Vec128<uint32_t> Shuffle2301(const Vec128<uint32_t> v) { + return Vec128<uint32_t>{wasm_i32x4_shuffle(v.raw, v.raw, 1, 0, 3, 2)}; +} +HWY_API Vec128<int32_t> Shuffle2301(const Vec128<int32_t> v) { + return Vec128<int32_t>{wasm_i32x4_shuffle(v.raw, v.raw, 1, 0, 3, 2)}; +} +HWY_API Vec128<float> Shuffle2301(const Vec128<float> v) { + return Vec128<float>{wasm_i32x4_shuffle(v.raw, v.raw, 1, 0, 3, 2)}; +} + +// Swap 64-bit halves +HWY_API Vec128<uint32_t> Shuffle1032(const Vec128<uint32_t> v) { + return Vec128<uint32_t>{wasm_i64x2_shuffle(v.raw, v.raw, 1, 0)}; +} +HWY_API Vec128<int32_t> Shuffle1032(const Vec128<int32_t> v) { + return Vec128<int32_t>{wasm_i64x2_shuffle(v.raw, v.raw, 1, 0)}; +} +HWY_API Vec128<float> Shuffle1032(const Vec128<float> v) { + return Vec128<float>{wasm_i64x2_shuffle(v.raw, v.raw, 1, 0)}; +} + +// Rotate right 32 bits +HWY_API Vec128<uint32_t> Shuffle0321(const Vec128<uint32_t> v) { + return Vec128<uint32_t>{wasm_i32x4_shuffle(v.raw, v.raw, 1, 2, 3, 0)}; +} +HWY_API Vec128<int32_t> Shuffle0321(const Vec128<int32_t> v) { + return Vec128<int32_t>{wasm_i32x4_shuffle(v.raw, v.raw, 1, 2, 3, 0)}; +} +HWY_API Vec128<float> Shuffle0321(const Vec128<float> v) { + return Vec128<float>{wasm_i32x4_shuffle(v.raw, v.raw, 1, 2, 3, 0)}; +} +// Rotate left 32 bits +HWY_API Vec128<uint32_t> Shuffle2103(const Vec128<uint32_t> v) { + return Vec128<uint32_t>{wasm_i32x4_shuffle(v.raw, v.raw, 3, 0, 1, 2)}; +} +HWY_API Vec128<int32_t> Shuffle2103(const Vec128<int32_t> v) { + return Vec128<int32_t>{wasm_i32x4_shuffle(v.raw, v.raw, 3, 0, 1, 2)}; +} +HWY_API Vec128<float> Shuffle2103(const Vec128<float> v) { + return Vec128<float>{wasm_i32x4_shuffle(v.raw, v.raw, 3, 0, 1, 2)}; +} + +// Reverse +HWY_API Vec128<uint32_t> Shuffle0123(const Vec128<uint32_t> v) { + return Vec128<uint32_t>{wasm_i32x4_shuffle(v.raw, v.raw, 3, 2, 1, 0)}; +} +HWY_API Vec128<int32_t> Shuffle0123(const Vec128<int32_t> v) { + return Vec128<int32_t>{wasm_i32x4_shuffle(v.raw, v.raw, 3, 2, 1, 0)}; +} +HWY_API Vec128<float> Shuffle0123(const Vec128<float> v) { + return Vec128<float>{wasm_i32x4_shuffle(v.raw, v.raw, 3, 2, 1, 0)}; +} + +// ------------------------------ TableLookupLanes + +// Returned by SetTableIndices for use by TableLookupLanes. +template <typename T, size_t N> +struct Indices128 { + __v128_u raw; +}; + +template <typename T, size_t N, typename TI, HWY_IF_LE128(T, N)> +HWY_API Indices128<T, N> IndicesFromVec(Simd<T, N> d, Vec128<TI, N> vec) { + static_assert(sizeof(T) == sizeof(TI), "Index size must match lane"); +#if HWY_IS_DEBUG_BUILD + const Simd<TI, N> di; + HWY_DASSERT(AllFalse(di, Lt(vec, Zero(di))) && + AllTrue(di, Lt(vec, Set(di, static_cast<TI>(N))))); +#endif + + const Repartition<uint8_t, decltype(d)> d8; + using V8 = VFromD<decltype(d8)>; + const Repartition<uint16_t, decltype(d)> d16; + + // Broadcast each lane index to all bytes of T and shift to bytes + static_assert(sizeof(T) == 4 || sizeof(T) == 8, ""); + if (sizeof(T) == 4) { + alignas(16) constexpr uint8_t kBroadcastLaneBytes[16] = { + 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12}; + const V8 lane_indices = + TableLookupBytes(BitCast(d8, vec), Load(d8, kBroadcastLaneBytes)); + const V8 byte_indices = + BitCast(d8, ShiftLeft<2>(BitCast(d16, lane_indices))); + alignas(16) constexpr uint8_t kByteOffsets[16] = {0, 1, 2, 3, 0, 1, 2, 3, + 0, 1, 2, 3, 0, 1, 2, 3}; + return Indices128<T, N>{Add(byte_indices, Load(d8, kByteOffsets)).raw}; + } else { + alignas(16) constexpr uint8_t kBroadcastLaneBytes[16] = { + 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 8, 8, 8, 8}; + const V8 lane_indices = + TableLookupBytes(BitCast(d8, vec), Load(d8, kBroadcastLaneBytes)); + const V8 byte_indices = + BitCast(d8, ShiftLeft<3>(BitCast(d16, lane_indices))); + alignas(16) constexpr uint8_t kByteOffsets[16] = {0, 1, 2, 3, 4, 5, 6, 7, + 0, 1, 2, 3, 4, 5, 6, 7}; + return Indices128<T, N>{Add(byte_indices, Load(d8, kByteOffsets)).raw}; + } +} + +template <typename T, size_t N, typename TI, HWY_IF_LE128(T, N)> +HWY_API Indices128<T, N> SetTableIndices(Simd<T, N> d, const TI* idx) { + const Rebind<TI, decltype(d)> di; + return IndicesFromVec(d, LoadU(di, idx)); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> TableLookupLanes(Vec128<T, N> v, Indices128<T, N> idx) { + using TI = MakeSigned<T>; + const Simd<T, N> d; + const Simd<TI, N> di; + return BitCast(d, TableLookupBytes(BitCast(di, v), Vec128<TI, N>{idx.raw})); +} + +// ------------------------------ Reverse (Shuffle0123, Shuffle2301, Shuffle01) + +// Single lane: no change +template <typename T> +HWY_API Vec128<T, 1> Reverse(Simd<T, 1> /* tag */, const Vec128<T, 1> v) { + return v; +} + +// Two lanes: shuffle +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T, 2> Reverse(Simd<T, 2> /* tag */, const Vec128<T, 2> v) { + return Vec128<T, 2>{Shuffle2301(Vec128<T>{v.raw}).raw}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec128<T> Reverse(Full128<T> /* tag */, const Vec128<T> v) { + return Shuffle01(v); +} + +// Four lanes: shuffle +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T> Reverse(Full128<T> /* tag */, const Vec128<T> v) { + return Shuffle0123(v); +} + +// 16-bit +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Vec128<T, N> Reverse(Simd<T, N> d, const Vec128<T, N> v) { + const RepartitionToWide<RebindToUnsigned<decltype(d)>> du32; + return BitCast(d, RotateRight<16>(Reverse(du32, BitCast(du32, v)))); +} + +// ------------------------------ InterleaveLower + +template <size_t N> +HWY_API Vec128<uint8_t, N> InterleaveLower(Vec128<uint8_t, N> a, + Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{wasm_i8x16_shuffle( + a.raw, b.raw, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> InterleaveLower(Vec128<uint16_t, N> a, + Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{ + wasm_i16x8_shuffle(a.raw, b.raw, 0, 8, 1, 9, 2, 10, 3, 11)}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> InterleaveLower(Vec128<uint32_t, N> a, + Vec128<uint32_t, N> b) { + return Vec128<uint32_t, N>{wasm_i32x4_shuffle(a.raw, b.raw, 0, 4, 1, 5)}; +} +template <size_t N> +HWY_API Vec128<uint64_t, N> InterleaveLower(Vec128<uint64_t, N> a, + Vec128<uint64_t, N> b) { + return Vec128<uint64_t, N>{wasm_i64x2_shuffle(a.raw, b.raw, 0, 2)}; +} + +template <size_t N> +HWY_API Vec128<int8_t, N> InterleaveLower(Vec128<int8_t, N> a, + Vec128<int8_t, N> b) { + return Vec128<int8_t, N>{wasm_i8x16_shuffle( + a.raw, b.raw, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> InterleaveLower(Vec128<int16_t, N> a, + Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{ + wasm_i16x8_shuffle(a.raw, b.raw, 0, 8, 1, 9, 2, 10, 3, 11)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> InterleaveLower(Vec128<int32_t, N> a, + Vec128<int32_t, N> b) { + return Vec128<int32_t, N>{wasm_i32x4_shuffle(a.raw, b.raw, 0, 4, 1, 5)}; +} +template <size_t N> +HWY_API Vec128<int64_t, N> InterleaveLower(Vec128<int64_t, N> a, + Vec128<int64_t, N> b) { + return Vec128<int64_t, N>{wasm_i64x2_shuffle(a.raw, b.raw, 0, 2)}; +} + +template <size_t N> +HWY_API Vec128<float, N> InterleaveLower(Vec128<float, N> a, + Vec128<float, N> b) { + return Vec128<float, N>{wasm_i32x4_shuffle(a.raw, b.raw, 0, 4, 1, 5)}; +} + +// Additional overload for the optional Simd<> tag. +template <typename T, size_t N, class V = Vec128<T, N>> +HWY_API V InterleaveLower(Simd<T, N> /* tag */, V a, V b) { + return InterleaveLower(a, b); +} + +// ------------------------------ InterleaveUpper (UpperHalf) + +// All functions inside detail lack the required D parameter. +namespace detail { + +template <size_t N> +HWY_API Vec128<uint8_t, N> InterleaveUpper(Vec128<uint8_t, N> a, + Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{wasm_i8x16_shuffle(a.raw, b.raw, 8, 24, 9, 25, 10, + 26, 11, 27, 12, 28, 13, 29, 14, + 30, 15, 31)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> InterleaveUpper(Vec128<uint16_t, N> a, + Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{ + wasm_i16x8_shuffle(a.raw, b.raw, 4, 12, 5, 13, 6, 14, 7, 15)}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> InterleaveUpper(Vec128<uint32_t, N> a, + Vec128<uint32_t, N> b) { + return Vec128<uint32_t, N>{wasm_i32x4_shuffle(a.raw, b.raw, 2, 6, 3, 7)}; +} +template <size_t N> +HWY_API Vec128<uint64_t, N> InterleaveUpper(Vec128<uint64_t, N> a, + Vec128<uint64_t, N> b) { + return Vec128<uint64_t, N>{wasm_i64x2_shuffle(a.raw, b.raw, 1, 3)}; +} + +template <size_t N> +HWY_API Vec128<int8_t, N> InterleaveUpper(Vec128<int8_t, N> a, + Vec128<int8_t, N> b) { + return Vec128<int8_t, N>{wasm_i8x16_shuffle(a.raw, b.raw, 8, 24, 9, 25, 10, + 26, 11, 27, 12, 28, 13, 29, 14, + 30, 15, 31)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> InterleaveUpper(Vec128<int16_t, N> a, + Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{ + wasm_i16x8_shuffle(a.raw, b.raw, 4, 12, 5, 13, 6, 14, 7, 15)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> InterleaveUpper(Vec128<int32_t, N> a, + Vec128<int32_t, N> b) { + return Vec128<int32_t, N>{wasm_i32x4_shuffle(a.raw, b.raw, 2, 6, 3, 7)}; +} +template <size_t N> +HWY_API Vec128<int64_t, N> InterleaveUpper(Vec128<int64_t, N> a, + Vec128<int64_t, N> b) { + return Vec128<int64_t, N>{wasm_i64x2_shuffle(a.raw, b.raw, 1, 3)}; +} + +template <size_t N> +HWY_API Vec128<float, N> InterleaveUpper(Vec128<float, N> a, + Vec128<float, N> b) { + return Vec128<float, N>{wasm_i32x4_shuffle(a.raw, b.raw, 2, 6, 3, 7)}; +} + +} // namespace detail + +// Full +template <typename T, class V = Vec128<T>> +HWY_API V InterleaveUpper(Full128<T> /* tag */, V a, V b) { + return detail::InterleaveUpper(a, b); +} + +// Partial +template <typename T, size_t N, HWY_IF_LE64(T, N), class V = Vec128<T, N>> +HWY_API V InterleaveUpper(Simd<T, N> d, V a, V b) { + const Half<decltype(d)> d2; + return InterleaveLower(d, V{UpperHalf(d2, a).raw}, V{UpperHalf(d2, b).raw}); +} + +// ------------------------------ ZipLower/ZipUpper (InterleaveLower) + +// Same as Interleave*, except that the return lanes are double-width integers; +// this is necessary because the single-lane scalar cannot return two values. +template <typename T, size_t N, class DW = RepartitionToWide<Simd<T, N>>> +HWY_API VFromD<DW> ZipLower(Vec128<T, N> a, Vec128<T, N> b) { + return BitCast(DW(), InterleaveLower(a, b)); +} +template <typename T, size_t N, class D = Simd<T, N>, + class DW = RepartitionToWide<D>> +HWY_API VFromD<DW> ZipLower(DW dw, Vec128<T, N> a, Vec128<T, N> b) { + return BitCast(dw, InterleaveLower(D(), a, b)); +} + +template <typename T, size_t N, class D = Simd<T, N>, + class DW = RepartitionToWide<D>> +HWY_API VFromD<DW> ZipUpper(DW dw, Vec128<T, N> a, Vec128<T, N> b) { + return BitCast(dw, InterleaveUpper(D(), a, b)); +} + +// ================================================== COMBINE + +// ------------------------------ Combine (InterleaveLower) + +// N = N/2 + N/2 (upper half undefined) +template <typename T, size_t N> +HWY_API Vec128<T, N> Combine(Simd<T, N> d, Vec128<T, N / 2> hi_half, + Vec128<T, N / 2> lo_half) { + const Half<decltype(d)> d2; + const RebindToUnsigned<decltype(d2)> du2; + // Treat half-width input as one lane, and expand to two lanes. + using VU = Vec128<UnsignedFromSize<N * sizeof(T) / 2>, 2>; + const VU lo{BitCast(du2, lo_half).raw}; + const VU hi{BitCast(du2, hi_half).raw}; + return BitCast(d, InterleaveLower(lo, hi)); +} + +// ------------------------------ ZeroExtendVector (Combine, IfThenElseZero) + +template <typename T, size_t N> +HWY_API Vec128<T, N> ZeroExtendVector(Simd<T, N> d, Vec128<T, N / 2> lo) { + return IfThenElseZero(FirstN(d, N / 2), Vec128<T, N>{lo.raw}); +} + +// ------------------------------ ConcatLowerLower + +// hiH,hiL loH,loL |-> hiL,loL (= lower halves) +template <typename T> +HWY_API Vec128<T> ConcatLowerLower(Full128<T> /* tag */, const Vec128<T> hi, + const Vec128<T> lo) { + return Vec128<T>{wasm_i64x2_shuffle(lo.raw, hi.raw, 0, 2)}; +} +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_API Vec128<T, N> ConcatLowerLower(Simd<T, N> d, const Vec128<T, N> hi, + const Vec128<T, N> lo) { + const Half<decltype(d)> d2; + return Combine(LowerHalf(d2, hi), LowerHalf(d2, lo)); +} + +// ------------------------------ ConcatUpperUpper + +template <typename T> +HWY_API Vec128<T> ConcatUpperUpper(Full128<T> /* tag */, const Vec128<T> hi, + const Vec128<T> lo) { + return Vec128<T>{wasm_i64x2_shuffle(lo.raw, hi.raw, 1, 3)}; +} +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_API Vec128<T, N> ConcatUpperUpper(Simd<T, N> d, const Vec128<T, N> hi, + const Vec128<T, N> lo) { + const Half<decltype(d)> d2; + return Combine(UpperHalf(d2, hi), UpperHalf(d2, lo)); +} + +// ------------------------------ ConcatLowerUpper + +template <typename T> +HWY_API Vec128<T> ConcatLowerUpper(Full128<T> d, const Vec128<T> hi, + const Vec128<T> lo) { + return CombineShiftRightBytes<8>(d, hi, lo); +} +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_API Vec128<T, N> ConcatLowerUpper(Simd<T, N> d, const Vec128<T, N> hi, + const Vec128<T, N> lo) { + const Half<decltype(d)> d2; + return Combine(LowerHalf(d2, hi), UpperHalf(d2, lo)); +} + +// ------------------------------ ConcatUpperLower +template <typename T, size_t N> +HWY_API Vec128<T, N> ConcatUpperLower(Simd<T, N> d, const Vec128<T, N> hi, + const Vec128<T, N> lo) { + return IfThenElse(FirstN(d, Lanes(d) / 2), lo, hi); +} + +// ------------------------------ ConcatOdd + +// 32-bit full +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T> ConcatOdd(Full128<T> /* tag */, Vec128<T> hi, Vec128<T> lo) { + return Vec128<T>{wasm_i32x4_shuffle(lo.raw, hi.raw, 1, 3, 5, 7)}; +} + +// 32-bit partial +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T, 2> ConcatOdd(Simd<T, 2> /* tag */, Vec128<T, 2> hi, + Vec128<T, 2> lo) { + return InterleaveUpper(Simd<T, 2>(), lo, hi); +} + +// 64-bit full - no partial because we need at least two inputs to have +// even/odd. +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec128<T> ConcatOdd(Full128<T> /* tag */, Vec128<T> hi, Vec128<T> lo) { + return InterleaveUpper(Full128<T>(), lo, hi); +} + +// ------------------------------ ConcatEven (InterleaveLower) + +// 32-bit full +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T> ConcatEven(Full128<T> /* tag */, Vec128<T> hi, Vec128<T> lo) { + return Vec128<T>{wasm_i32x4_shuffle(lo.raw, hi.raw, 0, 2, 4, 6)}; +} + +// 32-bit partial +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T, 2> ConcatEven(Simd<T, 2> /* tag */, Vec128<T, 2> hi, + Vec128<T, 2> lo) { + return InterleaveLower(Simd<T, 2>(), lo, hi); +} + +// 64-bit full - no partial because we need at least two inputs to have +// even/odd. +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec128<T> ConcatEven(Full128<T> /* tag */, Vec128<T> hi, Vec128<T> lo) { + return InterleaveLower(Full128<T>(), lo, hi); +} + +// ------------------------------ OddEven + +namespace detail { + +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> OddEven(hwy::SizeTag<1> /* tag */, const Vec128<T, N> a, + const Vec128<T, N> b) { + const Simd<T, N> d; + const Repartition<uint8_t, decltype(d)> d8; + alignas(16) constexpr uint8_t mask[16] = {0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, + 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0}; + return IfThenElse(MaskFromVec(BitCast(d, Load(d8, mask))), b, a); +} +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> OddEven(hwy::SizeTag<2> /* tag */, const Vec128<T, N> a, + const Vec128<T, N> b) { + return Vec128<T, N>{ + wasm_i16x8_shuffle(a.raw, b.raw, 8, 1, 10, 3, 12, 5, 14, 7)}; +} +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> OddEven(hwy::SizeTag<4> /* tag */, const Vec128<T, N> a, + const Vec128<T, N> b) { + return Vec128<T, N>{wasm_i32x4_shuffle(a.raw, b.raw, 4, 1, 6, 3)}; +} +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> OddEven(hwy::SizeTag<8> /* tag */, const Vec128<T, N> a, + const Vec128<T, N> b) { + return Vec128<T, N>{wasm_i64x2_shuffle(a.raw, b.raw, 2, 1)}; +} + +} // namespace detail + +template <typename T, size_t N> +HWY_API Vec128<T, N> OddEven(const Vec128<T, N> a, const Vec128<T, N> b) { + return detail::OddEven(hwy::SizeTag<sizeof(T)>(), a, b); +} +template <size_t N> +HWY_API Vec128<float, N> OddEven(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Vec128<float, N>{wasm_i32x4_shuffle(a.raw, b.raw, 4, 1, 6, 3)}; +} + +// ------------------------------ OddEvenBlocks +template <typename T, size_t N> +HWY_API Vec128<T, N> OddEvenBlocks(Vec128<T, N> /* odd */, Vec128<T, N> even) { + return even; +} + +// ------------------------------ SwapAdjacentBlocks + +template <typename T, size_t N> +HWY_API Vec128<T, N> SwapAdjacentBlocks(Vec128<T, N> v) { + return v; +} + +// ================================================== CONVERT + +// ------------------------------ Promotions (part w/ narrow lanes -> full) + +// Unsigned: zero-extend. +template <size_t N> +HWY_API Vec128<uint16_t, N> PromoteTo(Simd<uint16_t, N> /* tag */, + const Vec128<uint8_t, N> v) { + return Vec128<uint16_t, N>{wasm_u16x8_extend_low_u8x16(v.raw)}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> PromoteTo(Simd<uint32_t, N> /* tag */, + const Vec128<uint8_t, N> v) { + return Vec128<uint32_t, N>{ + wasm_u32x4_extend_low_u16x8(wasm_u16x8_extend_low_u8x16(v.raw))}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> PromoteTo(Simd<int16_t, N> /* tag */, + const Vec128<uint8_t, N> v) { + return Vec128<int16_t, N>{wasm_u16x8_extend_low_u8x16(v.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> PromoteTo(Simd<int32_t, N> /* tag */, + const Vec128<uint8_t, N> v) { + return Vec128<int32_t, N>{ + wasm_u32x4_extend_low_u16x8(wasm_u16x8_extend_low_u8x16(v.raw))}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> PromoteTo(Simd<uint32_t, N> /* tag */, + const Vec128<uint16_t, N> v) { + return Vec128<uint32_t, N>{wasm_u32x4_extend_low_u16x8(v.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> PromoteTo(Simd<int32_t, N> /* tag */, + const Vec128<uint16_t, N> v) { + return Vec128<int32_t, N>{wasm_u32x4_extend_low_u16x8(v.raw)}; +} + +// Signed: replicate sign bit. +template <size_t N> +HWY_API Vec128<int16_t, N> PromoteTo(Simd<int16_t, N> /* tag */, + const Vec128<int8_t, N> v) { + return Vec128<int16_t, N>{wasm_i16x8_extend_low_i8x16(v.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> PromoteTo(Simd<int32_t, N> /* tag */, + const Vec128<int8_t, N> v) { + return Vec128<int32_t, N>{ + wasm_i32x4_extend_low_i16x8(wasm_i16x8_extend_low_i8x16(v.raw))}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> PromoteTo(Simd<int32_t, N> /* tag */, + const Vec128<int16_t, N> v) { + return Vec128<int32_t, N>{wasm_i32x4_extend_low_i16x8(v.raw)}; +} + +template <size_t N> +HWY_API Vec128<double, N> PromoteTo(Simd<double, N> /* tag */, + const Vec128<int32_t, N> v) { + return Vec128<double, N>{wasm_f64x2_convert_low_i32x4(v.raw)}; +} + +template <size_t N> +HWY_API Vec128<float, N> PromoteTo(Simd<float, N> /* tag */, + const Vec128<float16_t, N> v) { + const Simd<int32_t, N> di32; + const Simd<uint32_t, N> du32; + const Simd<float, N> df32; + // Expand to u32 so we can shift. + const auto bits16 = PromoteTo(du32, Vec128<uint16_t, N>{v.raw}); + const auto sign = ShiftRight<15>(bits16); + const auto biased_exp = ShiftRight<10>(bits16) & Set(du32, 0x1F); + const auto mantissa = bits16 & Set(du32, 0x3FF); + const auto subnormal = + BitCast(du32, ConvertTo(df32, BitCast(di32, mantissa)) * + Set(df32, 1.0f / 16384 / 1024)); + + const auto biased_exp32 = biased_exp + Set(du32, 127 - 15); + const auto mantissa32 = ShiftLeft<23 - 10>(mantissa); + const auto normal = ShiftLeft<23>(biased_exp32) | mantissa32; + const auto bits32 = IfThenElse(biased_exp == Zero(du32), subnormal, normal); + return BitCast(df32, ShiftLeft<31>(sign) | bits32); +} + +template <size_t N> +HWY_API Vec128<float, N> PromoteTo(Simd<float, N> df32, + const Vec128<bfloat16_t, N> v) { + const Rebind<uint16_t, decltype(df32)> du16; + const RebindToSigned<decltype(df32)> di32; + return BitCast(df32, ShiftLeft<16>(PromoteTo(di32, BitCast(du16, v)))); +} + +// ------------------------------ Demotions (full -> part w/ narrow lanes) + +template <size_t N> +HWY_API Vec128<uint16_t, N> DemoteTo(Simd<uint16_t, N> /* tag */, + const Vec128<int32_t, N> v) { + return Vec128<uint16_t, N>{wasm_u16x8_narrow_i32x4(v.raw, v.raw)}; +} + +template <size_t N> +HWY_API Vec128<int16_t, N> DemoteTo(Simd<int16_t, N> /* tag */, + const Vec128<int32_t, N> v) { + return Vec128<int16_t, N>{wasm_i16x8_narrow_i32x4(v.raw, v.raw)}; +} + +template <size_t N> +HWY_API Vec128<uint8_t, N> DemoteTo(Simd<uint8_t, N> /* tag */, + const Vec128<int32_t, N> v) { + const auto intermediate = wasm_i16x8_narrow_i32x4(v.raw, v.raw); + return Vec128<uint8_t, N>{ + wasm_u8x16_narrow_i16x8(intermediate, intermediate)}; +} + +template <size_t N> +HWY_API Vec128<uint8_t, N> DemoteTo(Simd<uint8_t, N> /* tag */, + const Vec128<int16_t, N> v) { + return Vec128<uint8_t, N>{wasm_u8x16_narrow_i16x8(v.raw, v.raw)}; +} + +template <size_t N> +HWY_API Vec128<int8_t, N> DemoteTo(Simd<int8_t, N> /* tag */, + const Vec128<int32_t, N> v) { + const auto intermediate = wasm_i16x8_narrow_i32x4(v.raw, v.raw); + return Vec128<int8_t, N>{wasm_i8x16_narrow_i16x8(intermediate, intermediate)}; +} + +template <size_t N> +HWY_API Vec128<int8_t, N> DemoteTo(Simd<int8_t, N> /* tag */, + const Vec128<int16_t, N> v) { + return Vec128<int8_t, N>{wasm_i8x16_narrow_i16x8(v.raw, v.raw)}; +} + +template <size_t N> +HWY_API Vec128<int32_t, N> DemoteTo(Simd<int32_t, N> /* di */, + const Vec128<double, N> v) { + return Vec128<int32_t, N>{wasm_i32x4_trunc_sat_f64x2_zero(v.raw)}; +} + +template <size_t N> +HWY_API Vec128<float16_t, N> DemoteTo(Simd<float16_t, N> /* tag */, + const Vec128<float, N> v) { + const Simd<int32_t, N> di; + const Simd<uint32_t, N> du; + const Simd<uint16_t, N> du16; + const auto bits32 = BitCast(du, v); + const auto sign = ShiftRight<31>(bits32); + const auto biased_exp32 = ShiftRight<23>(bits32) & Set(du, 0xFF); + const auto mantissa32 = bits32 & Set(du, 0x7FFFFF); + + const auto k15 = Set(di, 15); + const auto exp = Min(BitCast(di, biased_exp32) - Set(di, 127), k15); + const auto is_tiny = exp < Set(di, -24); + + const auto is_subnormal = exp < Set(di, -14); + const auto biased_exp16 = + BitCast(du, IfThenZeroElse(is_subnormal, exp + k15)); + const auto sub_exp = BitCast(du, Set(di, -14) - exp); // [1, 11) + const auto sub_m = (Set(du, 1) << (Set(du, 10) - sub_exp)) + + (mantissa32 >> (Set(du, 13) + sub_exp)); + const auto mantissa16 = IfThenElse(RebindMask(du, is_subnormal), sub_m, + ShiftRight<13>(mantissa32)); // <1024 + + const auto sign16 = ShiftLeft<15>(sign); + const auto normal16 = sign16 | ShiftLeft<10>(biased_exp16) | mantissa16; + const auto bits16 = IfThenZeroElse(is_tiny, BitCast(di, normal16)); + return Vec128<float16_t, N>{DemoteTo(du16, bits16).raw}; +} + +template <size_t N> +HWY_API Vec128<bfloat16_t, N> DemoteTo(Simd<bfloat16_t, N> dbf16, + const Vec128<float, N> v) { + const Rebind<int32_t, decltype(dbf16)> di32; + const Rebind<uint32_t, decltype(dbf16)> du32; // for logical shift right + const Rebind<uint16_t, decltype(dbf16)> du16; + const auto bits_in_32 = BitCast(di32, ShiftRight<16>(BitCast(du32, v))); + return BitCast(dbf16, DemoteTo(du16, bits_in_32)); +} + +template <size_t N> +HWY_API Vec128<bfloat16_t, 2 * N> ReorderDemote2To( + Simd<bfloat16_t, 2 * N> dbf16, Vec128<float, N> a, Vec128<float, N> b) { + const RebindToUnsigned<decltype(dbf16)> du16; + const Repartition<uint32_t, decltype(dbf16)> du32; + const Vec128<uint32_t, N> b_in_even = ShiftRight<16>(BitCast(du32, b)); + return BitCast(dbf16, OddEven(BitCast(du16, a), BitCast(du16, b_in_even))); +} + +// For already range-limited input [0, 255]. +template <size_t N> +HWY_API Vec128<uint8_t, N> U8FromU32(const Vec128<uint32_t, N> v) { + const auto intermediate = wasm_i16x8_narrow_i32x4(v.raw, v.raw); + return Vec128<uint8_t, N>{ + wasm_u8x16_narrow_i16x8(intermediate, intermediate)}; +} + +// ------------------------------ Convert i32 <=> f32 (Round) + +template <size_t N> +HWY_API Vec128<float, N> ConvertTo(Simd<float, N> /* tag */, + const Vec128<int32_t, N> v) { + return Vec128<float, N>{wasm_f32x4_convert_i32x4(v.raw)}; +} +// Truncates (rounds toward zero). +template <size_t N> +HWY_API Vec128<int32_t, N> ConvertTo(Simd<int32_t, N> /* tag */, + const Vec128<float, N> v) { + return Vec128<int32_t, N>{wasm_i32x4_trunc_sat_f32x4(v.raw)}; +} + +template <size_t N> +HWY_API Vec128<int32_t, N> NearestInt(const Vec128<float, N> v) { + return ConvertTo(Simd<int32_t, N>(), Round(v)); +} + +// ================================================== MISC + +// ------------------------------ LoadMaskBits (TestBit) + +namespace detail { + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 1)> +HWY_INLINE Mask128<T, N> LoadMaskBits(Simd<T, N> d, uint64_t bits) { + const RebindToUnsigned<decltype(d)> du; + // Easier than Set(), which would require an >8-bit type, which would not + // compile for T=uint8_t, N=1. + const Vec128<T, N> vbits{wasm_i32x4_splat(static_cast<int32_t>(bits))}; + + // Replicate bytes 8x such that each byte contains the bit that governs it. + alignas(16) constexpr uint8_t kRep8[16] = {0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1}; + const auto rep8 = TableLookupBytes(vbits, Load(du, kRep8)); + + alignas(16) constexpr uint8_t kBit[16] = {1, 2, 4, 8, 16, 32, 64, 128, + 1, 2, 4, 8, 16, 32, 64, 128}; + return RebindMask(d, TestBit(rep8, LoadDup128(du, kBit))); +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 2)> +HWY_INLINE Mask128<T, N> LoadMaskBits(Simd<T, N> d, uint64_t bits) { + const RebindToUnsigned<decltype(d)> du; + alignas(16) constexpr uint16_t kBit[8] = {1, 2, 4, 8, 16, 32, 64, 128}; + return RebindMask( + d, TestBit(Set(du, static_cast<uint16_t>(bits)), Load(du, kBit))); +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 4)> +HWY_INLINE Mask128<T, N> LoadMaskBits(Simd<T, N> d, uint64_t bits) { + const RebindToUnsigned<decltype(d)> du; + alignas(16) constexpr uint32_t kBit[8] = {1, 2, 4, 8}; + return RebindMask( + d, TestBit(Set(du, static_cast<uint32_t>(bits)), Load(du, kBit))); +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 8)> +HWY_INLINE Mask128<T, N> LoadMaskBits(Simd<T, N> d, uint64_t bits) { + const RebindToUnsigned<decltype(d)> du; + alignas(16) constexpr uint64_t kBit[8] = {1, 2}; + return RebindMask(d, TestBit(Set(du, bits), Load(du, kBit))); +} + +} // namespace detail + +// `p` points to at least 8 readable bytes, not all of which need be valid. +template <typename T, size_t N, HWY_IF_LE128(T, N)> +HWY_API Mask128<T, N> LoadMaskBits(Simd<T, N> d, + const uint8_t* HWY_RESTRICT bits) { + uint64_t mask_bits = 0; + CopyBytes<(N + 7) / 8>(bits, &mask_bits); + return detail::LoadMaskBits(d, mask_bits); +} + +// ------------------------------ Mask + +namespace detail { + +// Full +template <typename T> +HWY_INLINE uint64_t BitsFromMask(hwy::SizeTag<1> /*tag*/, + const Mask128<T> mask) { + alignas(16) uint64_t lanes[2]; + wasm_v128_store(lanes, mask.raw); + + constexpr uint64_t kMagic = 0x103070F1F3F80ULL; + const uint64_t lo = ((lanes[0] * kMagic) >> 56); + const uint64_t hi = ((lanes[1] * kMagic) >> 48) & 0xFF00; + return (hi + lo); +} + +// 64-bit +template <typename T> +HWY_INLINE uint64_t BitsFromMask(hwy::SizeTag<1> /*tag*/, + const Mask128<T, 8> mask) { + constexpr uint64_t kMagic = 0x103070F1F3F80ULL; + return (static_cast<uint64_t>(wasm_i64x2_extract_lane(mask.raw, 0)) * + kMagic) >> + 56; +} + +// 32-bit or less: need masking +template <typename T, size_t N, HWY_IF_LE32(T, N)> +HWY_INLINE uint64_t BitsFromMask(hwy::SizeTag<1> /*tag*/, + const Mask128<T, N> mask) { + uint64_t bytes = static_cast<uint64_t>(wasm_i64x2_extract_lane(mask.raw, 0)); + // Clear potentially undefined bytes. + bytes &= (1ULL << (N * 8)) - 1; + constexpr uint64_t kMagic = 0x103070F1F3F80ULL; + return (bytes * kMagic) >> 56; +} + +template <typename T, size_t N> +HWY_INLINE uint64_t BitsFromMask(hwy::SizeTag<2> /*tag*/, + const Mask128<T, N> mask) { + // Remove useless lower half of each u16 while preserving the sign bit. + const __i16x8 zero = wasm_i16x8_splat(0); + const Mask128<uint8_t, N> mask8{wasm_i8x16_narrow_i16x8(mask.raw, zero)}; + return BitsFromMask(hwy::SizeTag<1>(), mask8); +} + +template <typename T, size_t N> +HWY_INLINE uint64_t BitsFromMask(hwy::SizeTag<4> /*tag*/, + const Mask128<T, N> mask) { + const __i32x4 mask_i = static_cast<__i32x4>(mask.raw); + const __i32x4 slice = wasm_i32x4_make(1, 2, 4, 8); + const __i32x4 sliced_mask = wasm_v128_and(mask_i, slice); + alignas(16) uint32_t lanes[4]; + wasm_v128_store(lanes, sliced_mask); + return lanes[0] | lanes[1] | lanes[2] | lanes[3]; +} + +// Returns the lowest N bits for the BitsFromMask result. +template <typename T, size_t N> +constexpr uint64_t OnlyActive(uint64_t bits) { + return ((N * sizeof(T)) == 16) ? bits : bits & ((1ull << N) - 1); +} + +// Returns 0xFF for bytes with index >= N, otherwise 0. +template <size_t N> +constexpr __i8x16 BytesAbove() { + return /**/ + (N == 0) ? wasm_i32x4_make(-1, -1, -1, -1) + : (N == 4) ? wasm_i32x4_make(0, -1, -1, -1) + : (N == 8) ? wasm_i32x4_make(0, 0, -1, -1) + : (N == 12) ? wasm_i32x4_make(0, 0, 0, -1) + : (N == 16) ? wasm_i32x4_make(0, 0, 0, 0) + : (N == 2) ? wasm_i16x8_make(0, -1, -1, -1, -1, -1, -1, -1) + : (N == 6) ? wasm_i16x8_make(0, 0, 0, -1, -1, -1, -1, -1) + : (N == 10) ? wasm_i16x8_make(0, 0, 0, 0, 0, -1, -1, -1) + : (N == 14) ? wasm_i16x8_make(0, 0, 0, 0, 0, 0, 0, -1) + : (N == 1) ? wasm_i8x16_make(0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1) + : (N == 3) ? wasm_i8x16_make(0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1) + : (N == 5) ? wasm_i8x16_make(0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1) + : (N == 7) ? wasm_i8x16_make(0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, + -1, -1, -1) + : (N == 9) ? wasm_i8x16_make(0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, + -1, -1, -1) + : (N == 11) + ? wasm_i8x16_make(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1) + : (N == 13) + ? wasm_i8x16_make(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1) + : wasm_i8x16_make(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1); +} + +template <typename T, size_t N> +HWY_INLINE uint64_t BitsFromMask(const Mask128<T, N> mask) { + return OnlyActive<T, N>(BitsFromMask(hwy::SizeTag<sizeof(T)>(), mask)); +} + +template <typename T> +HWY_INLINE size_t CountTrue(hwy::SizeTag<1> tag, const Mask128<T> m) { + return PopCount(BitsFromMask(tag, m)); +} + +template <typename T> +HWY_INLINE size_t CountTrue(hwy::SizeTag<2> tag, const Mask128<T> m) { + return PopCount(BitsFromMask(tag, m)); +} + +template <typename T> +HWY_INLINE size_t CountTrue(hwy::SizeTag<4> /*tag*/, const Mask128<T> m) { + const __i32x4 var_shift = wasm_i32x4_make(1, 2, 4, 8); + const __i32x4 shifted_bits = wasm_v128_and(m.raw, var_shift); + alignas(16) uint64_t lanes[2]; + wasm_v128_store(lanes, shifted_bits); + return PopCount(lanes[0] | lanes[1]); +} + +} // namespace detail + +// `p` points to at least 8 writable bytes. +template <typename T, size_t N> +HWY_API size_t StoreMaskBits(const Simd<T, N> /* tag */, + const Mask128<T, N> mask, uint8_t* bits) { + const uint64_t mask_bits = detail::BitsFromMask(mask); + const size_t kNumBytes = (N + 7) / 8; + CopyBytes<kNumBytes>(&mask_bits, bits); + return kNumBytes; +} + +template <typename T, size_t N> +HWY_API size_t CountTrue(const Simd<T, N> /* tag */, const Mask128<T> m) { + return detail::CountTrue(hwy::SizeTag<sizeof(T)>(), m); +} + +// Partial vector +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_API size_t CountTrue(const Simd<T, N> d, const Mask128<T, N> m) { + // Ensure all undefined bytes are 0. + const Mask128<T, N> mask{detail::BytesAbove<N * sizeof(T)>()}; + return CountTrue(d, Mask128<T>{AndNot(mask, m).raw}); +} + +// Full vector +template <typename T> +HWY_API bool AllFalse(const Full128<T> d, const Mask128<T> m) { +#if 0 + // Casting followed by wasm_i8x16_any_true results in wasm error: + // i32.eqz[0] expected type i32, found i8x16.popcnt of type s128 + const auto v8 = BitCast(Full128<int8_t>(), VecFromMask(d, m)); + return !wasm_i8x16_any_true(v8.raw); +#else + (void)d; + return (wasm_i64x2_extract_lane(m.raw, 0) | + wasm_i64x2_extract_lane(m.raw, 1)) == 0; +#endif +} + +// Full vector +namespace detail { +template <typename T> +HWY_INLINE bool AllTrue(hwy::SizeTag<1> /*tag*/, const Mask128<T> m) { + return wasm_i8x16_all_true(m.raw); +} +template <typename T> +HWY_INLINE bool AllTrue(hwy::SizeTag<2> /*tag*/, const Mask128<T> m) { + return wasm_i16x8_all_true(m.raw); +} +template <typename T> +HWY_INLINE bool AllTrue(hwy::SizeTag<4> /*tag*/, const Mask128<T> m) { + return wasm_i32x4_all_true(m.raw); +} + +} // namespace detail + +template <typename T, size_t N> +HWY_API bool AllTrue(const Simd<T, N> /* tag */, const Mask128<T> m) { + return detail::AllTrue(hwy::SizeTag<sizeof(T)>(), m); +} + +// Partial vectors + +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_API bool AllFalse(Simd<T, N> /* tag */, const Mask128<T, N> m) { + // Ensure all undefined bytes are 0. + const Mask128<T, N> mask{detail::BytesAbove<N * sizeof(T)>()}; + return AllFalse(Mask128<T>{AndNot(mask, m).raw}); +} + +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_API bool AllTrue(const Simd<T, N> d, const Mask128<T, N> m) { + // Ensure all undefined bytes are FF. + const Mask128<T, N> mask{detail::BytesAbove<N * sizeof(T)>()}; + return AllTrue(d, Mask128<T>{Or(mask, m).raw}); +} + +template <typename T, size_t N> +HWY_API intptr_t FindFirstTrue(const Simd<T, N> /* tag */, + const Mask128<T, N> mask) { + const uint64_t bits = detail::BitsFromMask(mask); + return bits ? static_cast<intptr_t>(Num0BitsBelowLS1Bit_Nonzero64(bits)) : -1; +} + +// ------------------------------ Compress + +namespace detail { + +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> Idx16x8FromBits(const uint64_t mask_bits) { + HWY_DASSERT(mask_bits < 256); + const Simd<T, N> d; + const Rebind<uint8_t, decltype(d)> d8; + const Simd<uint16_t, N> du; + + // We need byte indices for TableLookupBytes (one vector's worth for each of + // 256 combinations of 8 mask bits). Loading them directly requires 4 KiB. We + // can instead store lane indices and convert to byte indices (2*lane + 0..1), + // with the doubling baked into the table. Unpacking nibbles is likely more + // costly than the higher cache footprint from storing bytes. + alignas(16) constexpr uint8_t table[256 * 8] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, + 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, + 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, + 0, 6, 0, 0, 0, 0, 0, 0, 2, 6, 0, 0, 0, 0, 0, 0, 0, 2, + 6, 0, 0, 0, 0, 0, 4, 6, 0, 0, 0, 0, 0, 0, 0, 4, 6, 0, + 0, 0, 0, 0, 2, 4, 6, 0, 0, 0, 0, 0, 0, 2, 4, 6, 0, 0, + 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, + 2, 8, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 0, 0, 4, 8, + 0, 0, 0, 0, 0, 0, 0, 4, 8, 0, 0, 0, 0, 0, 2, 4, 8, 0, + 0, 0, 0, 0, 0, 2, 4, 8, 0, 0, 0, 0, 6, 8, 0, 0, 0, 0, + 0, 0, 0, 6, 8, 0, 0, 0, 0, 0, 2, 6, 8, 0, 0, 0, 0, 0, + 0, 2, 6, 8, 0, 0, 0, 0, 4, 6, 8, 0, 0, 0, 0, 0, 0, 4, + 6, 8, 0, 0, 0, 0, 2, 4, 6, 8, 0, 0, 0, 0, 0, 2, 4, 6, + 8, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, + 0, 0, 2, 10, 0, 0, 0, 0, 0, 0, 0, 2, 10, 0, 0, 0, 0, 0, + 4, 10, 0, 0, 0, 0, 0, 0, 0, 4, 10, 0, 0, 0, 0, 0, 2, 4, + 10, 0, 0, 0, 0, 0, 0, 2, 4, 10, 0, 0, 0, 0, 6, 10, 0, 0, + 0, 0, 0, 0, 0, 6, 10, 0, 0, 0, 0, 0, 2, 6, 10, 0, 0, 0, + 0, 0, 0, 2, 6, 10, 0, 0, 0, 0, 4, 6, 10, 0, 0, 0, 0, 0, + 0, 4, 6, 10, 0, 0, 0, 0, 2, 4, 6, 10, 0, 0, 0, 0, 0, 2, + 4, 6, 10, 0, 0, 0, 8, 10, 0, 0, 0, 0, 0, 0, 0, 8, 10, 0, + 0, 0, 0, 0, 2, 8, 10, 0, 0, 0, 0, 0, 0, 2, 8, 10, 0, 0, + 0, 0, 4, 8, 10, 0, 0, 0, 0, 0, 0, 4, 8, 10, 0, 0, 0, 0, + 2, 4, 8, 10, 0, 0, 0, 0, 0, 2, 4, 8, 10, 0, 0, 0, 6, 8, + 10, 0, 0, 0, 0, 0, 0, 6, 8, 10, 0, 0, 0, 0, 2, 6, 8, 10, + 0, 0, 0, 0, 0, 2, 6, 8, 10, 0, 0, 0, 4, 6, 8, 10, 0, 0, + 0, 0, 0, 4, 6, 8, 10, 0, 0, 0, 2, 4, 6, 8, 10, 0, 0, 0, + 0, 2, 4, 6, 8, 10, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 12, + 0, 0, 0, 0, 0, 0, 2, 12, 0, 0, 0, 0, 0, 0, 0, 2, 12, 0, + 0, 0, 0, 0, 4, 12, 0, 0, 0, 0, 0, 0, 0, 4, 12, 0, 0, 0, + 0, 0, 2, 4, 12, 0, 0, 0, 0, 0, 0, 2, 4, 12, 0, 0, 0, 0, + 6, 12, 0, 0, 0, 0, 0, 0, 0, 6, 12, 0, 0, 0, 0, 0, 2, 6, + 12, 0, 0, 0, 0, 0, 0, 2, 6, 12, 0, 0, 0, 0, 4, 6, 12, 0, + 0, 0, 0, 0, 0, 4, 6, 12, 0, 0, 0, 0, 2, 4, 6, 12, 0, 0, + 0, 0, 0, 2, 4, 6, 12, 0, 0, 0, 8, 12, 0, 0, 0, 0, 0, 0, + 0, 8, 12, 0, 0, 0, 0, 0, 2, 8, 12, 0, 0, 0, 0, 0, 0, 2, + 8, 12, 0, 0, 0, 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 4, 8, 12, + 0, 0, 0, 0, 2, 4, 8, 12, 0, 0, 0, 0, 0, 2, 4, 8, 12, 0, + 0, 0, 6, 8, 12, 0, 0, 0, 0, 0, 0, 6, 8, 12, 0, 0, 0, 0, + 2, 6, 8, 12, 0, 0, 0, 0, 0, 2, 6, 8, 12, 0, 0, 0, 4, 6, + 8, 12, 0, 0, 0, 0, 0, 4, 6, 8, 12, 0, 0, 0, 2, 4, 6, 8, + 12, 0, 0, 0, 0, 2, 4, 6, 8, 12, 0, 0, 10, 12, 0, 0, 0, 0, + 0, 0, 0, 10, 12, 0, 0, 0, 0, 0, 2, 10, 12, 0, 0, 0, 0, 0, + 0, 2, 10, 12, 0, 0, 0, 0, 4, 10, 12, 0, 0, 0, 0, 0, 0, 4, + 10, 12, 0, 0, 0, 0, 2, 4, 10, 12, 0, 0, 0, 0, 0, 2, 4, 10, + 12, 0, 0, 0, 6, 10, 12, 0, 0, 0, 0, 0, 0, 6, 10, 12, 0, 0, + 0, 0, 2, 6, 10, 12, 0, 0, 0, 0, 0, 2, 6, 10, 12, 0, 0, 0, + 4, 6, 10, 12, 0, 0, 0, 0, 0, 4, 6, 10, 12, 0, 0, 0, 2, 4, + 6, 10, 12, 0, 0, 0, 0, 2, 4, 6, 10, 12, 0, 0, 8, 10, 12, 0, + 0, 0, 0, 0, 0, 8, 10, 12, 0, 0, 0, 0, 2, 8, 10, 12, 0, 0, + 0, 0, 0, 2, 8, 10, 12, 0, 0, 0, 4, 8, 10, 12, 0, 0, 0, 0, + 0, 4, 8, 10, 12, 0, 0, 0, 2, 4, 8, 10, 12, 0, 0, 0, 0, 2, + 4, 8, 10, 12, 0, 0, 6, 8, 10, 12, 0, 0, 0, 0, 0, 6, 8, 10, + 12, 0, 0, 0, 2, 6, 8, 10, 12, 0, 0, 0, 0, 2, 6, 8, 10, 12, + 0, 0, 4, 6, 8, 10, 12, 0, 0, 0, 0, 4, 6, 8, 10, 12, 0, 0, + 2, 4, 6, 8, 10, 12, 0, 0, 0, 2, 4, 6, 8, 10, 12, 0, 14, 0, + 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 2, 14, 0, 0, + 0, 0, 0, 0, 0, 2, 14, 0, 0, 0, 0, 0, 4, 14, 0, 0, 0, 0, + 0, 0, 0, 4, 14, 0, 0, 0, 0, 0, 2, 4, 14, 0, 0, 0, 0, 0, + 0, 2, 4, 14, 0, 0, 0, 0, 6, 14, 0, 0, 0, 0, 0, 0, 0, 6, + 14, 0, 0, 0, 0, 0, 2, 6, 14, 0, 0, 0, 0, 0, 0, 2, 6, 14, + 0, 0, 0, 0, 4, 6, 14, 0, 0, 0, 0, 0, 0, 4, 6, 14, 0, 0, + 0, 0, 2, 4, 6, 14, 0, 0, 0, 0, 0, 2, 4, 6, 14, 0, 0, 0, + 8, 14, 0, 0, 0, 0, 0, 0, 0, 8, 14, 0, 0, 0, 0, 0, 2, 8, + 14, 0, 0, 0, 0, 0, 0, 2, 8, 14, 0, 0, 0, 0, 4, 8, 14, 0, + 0, 0, 0, 0, 0, 4, 8, 14, 0, 0, 0, 0, 2, 4, 8, 14, 0, 0, + 0, 0, 0, 2, 4, 8, 14, 0, 0, 0, 6, 8, 14, 0, 0, 0, 0, 0, + 0, 6, 8, 14, 0, 0, 0, 0, 2, 6, 8, 14, 0, 0, 0, 0, 0, 2, + 6, 8, 14, 0, 0, 0, 4, 6, 8, 14, 0, 0, 0, 0, 0, 4, 6, 8, + 14, 0, 0, 0, 2, 4, 6, 8, 14, 0, 0, 0, 0, 2, 4, 6, 8, 14, + 0, 0, 10, 14, 0, 0, 0, 0, 0, 0, 0, 10, 14, 0, 0, 0, 0, 0, + 2, 10, 14, 0, 0, 0, 0, 0, 0, 2, 10, 14, 0, 0, 0, 0, 4, 10, + 14, 0, 0, 0, 0, 0, 0, 4, 10, 14, 0, 0, 0, 0, 2, 4, 10, 14, + 0, 0, 0, 0, 0, 2, 4, 10, 14, 0, 0, 0, 6, 10, 14, 0, 0, 0, + 0, 0, 0, 6, 10, 14, 0, 0, 0, 0, 2, 6, 10, 14, 0, 0, 0, 0, + 0, 2, 6, 10, 14, 0, 0, 0, 4, 6, 10, 14, 0, 0, 0, 0, 0, 4, + 6, 10, 14, 0, 0, 0, 2, 4, 6, 10, 14, 0, 0, 0, 0, 2, 4, 6, + 10, 14, 0, 0, 8, 10, 14, 0, 0, 0, 0, 0, 0, 8, 10, 14, 0, 0, + 0, 0, 2, 8, 10, 14, 0, 0, 0, 0, 0, 2, 8, 10, 14, 0, 0, 0, + 4, 8, 10, 14, 0, 0, 0, 0, 0, 4, 8, 10, 14, 0, 0, 0, 2, 4, + 8, 10, 14, 0, 0, 0, 0, 2, 4, 8, 10, 14, 0, 0, 6, 8, 10, 14, + 0, 0, 0, 0, 0, 6, 8, 10, 14, 0, 0, 0, 2, 6, 8, 10, 14, 0, + 0, 0, 0, 2, 6, 8, 10, 14, 0, 0, 4, 6, 8, 10, 14, 0, 0, 0, + 0, 4, 6, 8, 10, 14, 0, 0, 2, 4, 6, 8, 10, 14, 0, 0, 0, 2, + 4, 6, 8, 10, 14, 0, 12, 14, 0, 0, 0, 0, 0, 0, 0, 12, 14, 0, + 0, 0, 0, 0, 2, 12, 14, 0, 0, 0, 0, 0, 0, 2, 12, 14, 0, 0, + 0, 0, 4, 12, 14, 0, 0, 0, 0, 0, 0, 4, 12, 14, 0, 0, 0, 0, + 2, 4, 12, 14, 0, 0, 0, 0, 0, 2, 4, 12, 14, 0, 0, 0, 6, 12, + 14, 0, 0, 0, 0, 0, 0, 6, 12, 14, 0, 0, 0, 0, 2, 6, 12, 14, + 0, 0, 0, 0, 0, 2, 6, 12, 14, 0, 0, 0, 4, 6, 12, 14, 0, 0, + 0, 0, 0, 4, 6, 12, 14, 0, 0, 0, 2, 4, 6, 12, 14, 0, 0, 0, + 0, 2, 4, 6, 12, 14, 0, 0, 8, 12, 14, 0, 0, 0, 0, 0, 0, 8, + 12, 14, 0, 0, 0, 0, 2, 8, 12, 14, 0, 0, 0, 0, 0, 2, 8, 12, + 14, 0, 0, 0, 4, 8, 12, 14, 0, 0, 0, 0, 0, 4, 8, 12, 14, 0, + 0, 0, 2, 4, 8, 12, 14, 0, 0, 0, 0, 2, 4, 8, 12, 14, 0, 0, + 6, 8, 12, 14, 0, 0, 0, 0, 0, 6, 8, 12, 14, 0, 0, 0, 2, 6, + 8, 12, 14, 0, 0, 0, 0, 2, 6, 8, 12, 14, 0, 0, 4, 6, 8, 12, + 14, 0, 0, 0, 0, 4, 6, 8, 12, 14, 0, 0, 2, 4, 6, 8, 12, 14, + 0, 0, 0, 2, 4, 6, 8, 12, 14, 0, 10, 12, 14, 0, 0, 0, 0, 0, + 0, 10, 12, 14, 0, 0, 0, 0, 2, 10, 12, 14, 0, 0, 0, 0, 0, 2, + 10, 12, 14, 0, 0, 0, 4, 10, 12, 14, 0, 0, 0, 0, 0, 4, 10, 12, + 14, 0, 0, 0, 2, 4, 10, 12, 14, 0, 0, 0, 0, 2, 4, 10, 12, 14, + 0, 0, 6, 10, 12, 14, 0, 0, 0, 0, 0, 6, 10, 12, 14, 0, 0, 0, + 2, 6, 10, 12, 14, 0, 0, 0, 0, 2, 6, 10, 12, 14, 0, 0, 4, 6, + 10, 12, 14, 0, 0, 0, 0, 4, 6, 10, 12, 14, 0, 0, 2, 4, 6, 10, + 12, 14, 0, 0, 0, 2, 4, 6, 10, 12, 14, 0, 8, 10, 12, 14, 0, 0, + 0, 0, 0, 8, 10, 12, 14, 0, 0, 0, 2, 8, 10, 12, 14, 0, 0, 0, + 0, 2, 8, 10, 12, 14, 0, 0, 4, 8, 10, 12, 14, 0, 0, 0, 0, 4, + 8, 10, 12, 14, 0, 0, 2, 4, 8, 10, 12, 14, 0, 0, 0, 2, 4, 8, + 10, 12, 14, 0, 6, 8, 10, 12, 14, 0, 0, 0, 0, 6, 8, 10, 12, 14, + 0, 0, 2, 6, 8, 10, 12, 14, 0, 0, 0, 2, 6, 8, 10, 12, 14, 0, + 4, 6, 8, 10, 12, 14, 0, 0, 0, 4, 6, 8, 10, 12, 14, 0, 2, 4, + 6, 8, 10, 12, 14, 0, 0, 2, 4, 6, 8, 10, 12, 14}; + + const Vec128<uint8_t, 2 * N> byte_idx{Load(d8, table + mask_bits * 8).raw}; + const Vec128<uint16_t, N> pairs = ZipLower(byte_idx, byte_idx); + return BitCast(d, pairs + Set(du, 0x0100)); +} + +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> Idx32x4FromBits(const uint64_t mask_bits) { + HWY_DASSERT(mask_bits < 16); + + // There are only 4 lanes, so we can afford to load the index vector directly. + alignas(16) constexpr uint8_t packed_array[16 * 16] = { + 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, // + 4, 5, 6, 7, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 0, 1, 2, 3, // + 8, 9, 10, 11, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 8, 9, 10, 11, 0, 1, 2, 3, 0, 1, 2, 3, // + 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, // + 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3, // + 4, 5, 6, 7, 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 15, 0, 1, 2, 3, // + 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, // + 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, // + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + + const Simd<T, N> d; + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, Load(d8, packed_array + 16 * mask_bits)); +} + +#if HWY_CAP_INTEGER64 || HWY_CAP_FLOAT64 + +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> Idx64x2FromBits(const uint64_t mask_bits) { + HWY_DASSERT(mask_bits < 4); + + // There are only 2 lanes, so we can afford to load the index vector directly. + alignas(16) constexpr uint8_t packed_array[4 * 16] = { + 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, // + 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, // + 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, // + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + + const Simd<T, N> d; + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, Load(d8, packed_array + 16 * mask_bits)); +} + +#endif + +// Helper functions called by both Compress and CompressStore - avoids a +// redundant BitsFromMask in the latter. + +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> Compress(hwy::SizeTag<2> /*tag*/, Vec128<T, N> v, + const uint64_t mask_bits) { + const auto idx = detail::Idx16x8FromBits<T, N>(mask_bits); + using D = Simd<T, N>; + const RebindToSigned<D> di; + return BitCast(D(), TableLookupBytes(BitCast(di, v), BitCast(di, idx))); +} + +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> Compress(hwy::SizeTag<4> /*tag*/, Vec128<T, N> v, + const uint64_t mask_bits) { + const auto idx = detail::Idx32x4FromBits<T, N>(mask_bits); + using D = Simd<T, N>; + const RebindToSigned<D> di; + return BitCast(D(), TableLookupBytes(BitCast(di, v), BitCast(di, idx))); +} + +#if HWY_CAP_INTEGER64 || HWY_CAP_FLOAT64 + +template <typename T, size_t N> +HWY_INLINE Vec128<uint64_t, N> Compress(hwy::SizeTag<8> /*tag*/, + Vec128<uint64_t, N> v, + const uint64_t mask_bits) { + const auto idx = detail::Idx64x2FromBits<uint64_t, N>(mask_bits); + using D = Simd<T, N>; + const RebindToSigned<D> di; + return BitCast(D(), TableLookupBytes(BitCast(di, v), BitCast(di, idx))); +} + +#endif + +} // namespace detail + +template <typename T, size_t N> +HWY_API Vec128<T, N> Compress(Vec128<T, N> v, const Mask128<T, N> mask) { + const uint64_t mask_bits = detail::BitsFromMask(mask); + return detail::Compress(hwy::SizeTag<sizeof(T)>(), v, mask_bits); +} + +// ------------------------------ CompressBits + +template <typename T, size_t N> +HWY_API Vec128<T, N> CompressBits(Vec128<T, N> v, + const uint8_t* HWY_RESTRICT bits) { + uint64_t mask_bits = 0; + constexpr size_t kNumBytes = (N + 7) / 8; + CopyBytes<kNumBytes>(bits, &mask_bits); + if (N < 8) { + mask_bits &= (1ull << N) - 1; + } + + return detail::Compress(hwy::SizeTag<sizeof(T)>(), v, mask_bits); +} + +// ------------------------------ CompressStore +template <typename T, size_t N> +HWY_API size_t CompressStore(Vec128<T, N> v, const Mask128<T, N> mask, + Simd<T, N> d, T* HWY_RESTRICT unaligned) { + const uint64_t mask_bits = detail::BitsFromMask(mask); + const auto c = detail::Compress(hwy::SizeTag<sizeof(T)>(), v, mask_bits); + StoreU(c, d, unaligned); + return PopCount(mask_bits); +} + +// ------------------------------ CompressBlendedStore +template <typename T, size_t N> +HWY_API size_t CompressBlendedStore(Vec128<T, N> v, Mask128<T, N> m, + Simd<T, N> d, T* HWY_RESTRICT unaligned) { + const RebindToUnsigned<decltype(d)> du; // so we can support fp16/bf16 + using TU = TFromD<decltype(du)>; + const uint64_t mask_bits = detail::BitsFromMask(m); + const size_t count = PopCount(mask_bits); + const Mask128<TU, N> store_mask = FirstN(du, count); + const Vec128<TU, N> compressed = + detail::Compress(hwy::SizeTag<sizeof(T)>(), BitCast(du, v), mask_bits); + const Vec128<TU, N> prev = BitCast(du, LoadU(d, unaligned)); + StoreU(BitCast(d, IfThenElse(store_mask, compressed, prev)), d, unaligned); + return count; +} + +// ------------------------------ CompressBitsStore + +template <typename T, size_t N> +HWY_API size_t CompressBitsStore(Vec128<T, N> v, + const uint8_t* HWY_RESTRICT bits, Simd<T, N> d, + T* HWY_RESTRICT unaligned) { + uint64_t mask_bits = 0; + constexpr size_t kNumBytes = (N + 7) / 8; + CopyBytes<kNumBytes>(bits, &mask_bits); + if (N < 8) { + mask_bits &= (1ull << N) - 1; + } + + const auto c = detail::Compress(hwy::SizeTag<sizeof(T)>(), v, mask_bits); + StoreU(c, d, unaligned); + return PopCount(mask_bits); +} + +// ------------------------------ StoreInterleaved3 (CombineShiftRightBytes, +// TableLookupBytes) + +// 128 bits +HWY_API void StoreInterleaved3(const Vec128<uint8_t> a, const Vec128<uint8_t> b, + const Vec128<uint8_t> c, Full128<uint8_t> d, + uint8_t* HWY_RESTRICT unaligned) { + const auto k5 = Set(d, 5); + const auto k6 = Set(d, 6); + + // Shuffle (a,b,c) vector bytes to (MSB on left): r5, bgr[4:0]. + // 0x80 so lanes to be filled from other vectors are 0 for blending. + alignas(16) static constexpr uint8_t tbl_r0[16] = { + 0, 0x80, 0x80, 1, 0x80, 0x80, 2, 0x80, 0x80, // + 3, 0x80, 0x80, 4, 0x80, 0x80, 5}; + alignas(16) static constexpr uint8_t tbl_g0[16] = { + 0x80, 0, 0x80, 0x80, 1, 0x80, // + 0x80, 2, 0x80, 0x80, 3, 0x80, 0x80, 4, 0x80, 0x80}; + const auto shuf_r0 = Load(d, tbl_r0); + const auto shuf_g0 = Load(d, tbl_g0); // cannot reuse r0 due to 5 in MSB + const auto shuf_b0 = CombineShiftRightBytes<15>(d, shuf_g0, shuf_g0); + const auto r0 = TableLookupBytes(a, shuf_r0); // 5..4..3..2..1..0 + const auto g0 = TableLookupBytes(b, shuf_g0); // ..4..3..2..1..0. + const auto b0 = TableLookupBytes(c, shuf_b0); // .4..3..2..1..0.. + const auto int0 = r0 | g0 | b0; + StoreU(int0, d, unaligned + 0 * 16); + + // Second vector: g10,r10, bgr[9:6], b5,g5 + const auto shuf_r1 = shuf_b0 + k6; // .A..9..8..7..6.. + const auto shuf_g1 = shuf_r0 + k5; // A..9..8..7..6..5 + const auto shuf_b1 = shuf_g0 + k5; // ..9..8..7..6..5. + const auto r1 = TableLookupBytes(a, shuf_r1); + const auto g1 = TableLookupBytes(b, shuf_g1); + const auto b1 = TableLookupBytes(c, shuf_b1); + const auto int1 = r1 | g1 | b1; + StoreU(int1, d, unaligned + 1 * 16); + + // Third vector: bgr[15:11], b10 + const auto shuf_r2 = shuf_b1 + k6; // ..F..E..D..C..B. + const auto shuf_g2 = shuf_r1 + k5; // .F..E..D..C..B.. + const auto shuf_b2 = shuf_g1 + k5; // F..E..D..C..B..A + const auto r2 = TableLookupBytes(a, shuf_r2); + const auto g2 = TableLookupBytes(b, shuf_g2); + const auto b2 = TableLookupBytes(c, shuf_b2); + const auto int2 = r2 | g2 | b2; + StoreU(int2, d, unaligned + 2 * 16); +} + +// 64 bits +HWY_API void StoreInterleaved3(const Vec128<uint8_t, 8> a, + const Vec128<uint8_t, 8> b, + const Vec128<uint8_t, 8> c, Simd<uint8_t, 8> d, + uint8_t* HWY_RESTRICT unaligned) { + // Use full vectors for the shuffles and first result. + const Full128<uint8_t> d_full; + const auto k5 = Set(d_full, 5); + const auto k6 = Set(d_full, 6); + + const Vec128<uint8_t> full_a{a.raw}; + const Vec128<uint8_t> full_b{b.raw}; + const Vec128<uint8_t> full_c{c.raw}; + + // Shuffle (a,b,c) vector bytes to (MSB on left): r5, bgr[4:0]. + // 0x80 so lanes to be filled from other vectors are 0 for blending. + alignas(16) static constexpr uint8_t tbl_r0[16] = { + 0, 0x80, 0x80, 1, 0x80, 0x80, 2, 0x80, 0x80, // + 3, 0x80, 0x80, 4, 0x80, 0x80, 5}; + alignas(16) static constexpr uint8_t tbl_g0[16] = { + 0x80, 0, 0x80, 0x80, 1, 0x80, // + 0x80, 2, 0x80, 0x80, 3, 0x80, 0x80, 4, 0x80, 0x80}; + const auto shuf_r0 = Load(d_full, tbl_r0); + const auto shuf_g0 = Load(d_full, tbl_g0); // cannot reuse r0 due to 5 in MSB + const auto shuf_b0 = CombineShiftRightBytes<15>(d_full, shuf_g0, shuf_g0); + const auto r0 = TableLookupBytes(full_a, shuf_r0); // 5..4..3..2..1..0 + const auto g0 = TableLookupBytes(full_b, shuf_g0); // ..4..3..2..1..0. + const auto b0 = TableLookupBytes(full_c, shuf_b0); // .4..3..2..1..0.. + const auto int0 = r0 | g0 | b0; + StoreU(int0, d_full, unaligned + 0 * 16); + + // Second (HALF) vector: bgr[7:6], b5,g5 + const auto shuf_r1 = shuf_b0 + k6; // ..7..6.. + const auto shuf_g1 = shuf_r0 + k5; // .7..6..5 + const auto shuf_b1 = shuf_g0 + k5; // 7..6..5. + const auto r1 = TableLookupBytes(full_a, shuf_r1); + const auto g1 = TableLookupBytes(full_b, shuf_g1); + const auto b1 = TableLookupBytes(full_c, shuf_b1); + const decltype(Zero(d)) int1{(r1 | g1 | b1).raw}; + StoreU(int1, d, unaligned + 1 * 16); +} + +// <= 32 bits +template <size_t N, HWY_IF_LE32(uint8_t, N)> +HWY_API void StoreInterleaved3(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b, + const Vec128<uint8_t, N> c, + Simd<uint8_t, N> /*tag*/, + uint8_t* HWY_RESTRICT unaligned) { + // Use full vectors for the shuffles and result. + const Full128<uint8_t> d_full; + + const Vec128<uint8_t> full_a{a.raw}; + const Vec128<uint8_t> full_b{b.raw}; + const Vec128<uint8_t> full_c{c.raw}; + + // Shuffle (a,b,c) vector bytes to bgr[3:0]. + // 0x80 so lanes to be filled from other vectors are 0 for blending. + alignas(16) static constexpr uint8_t tbl_r0[16] = { + 0, 0x80, 0x80, 1, 0x80, 0x80, 2, 0x80, 0x80, 3, 0x80, 0x80, // + 0x80, 0x80, 0x80, 0x80}; + const auto shuf_r0 = Load(d_full, tbl_r0); + const auto shuf_g0 = CombineShiftRightBytes<15>(d_full, shuf_r0, shuf_r0); + const auto shuf_b0 = CombineShiftRightBytes<14>(d_full, shuf_r0, shuf_r0); + const auto r0 = TableLookupBytes(full_a, shuf_r0); // ......3..2..1..0 + const auto g0 = TableLookupBytes(full_b, shuf_g0); // .....3..2..1..0. + const auto b0 = TableLookupBytes(full_c, shuf_b0); // ....3..2..1..0.. + const auto int0 = r0 | g0 | b0; + alignas(16) uint8_t buf[16]; + StoreU(int0, d_full, buf); + CopyBytes<N * 3>(buf, unaligned); +} + +// ------------------------------ StoreInterleaved4 + +// 128 bits +HWY_API void StoreInterleaved4(const Vec128<uint8_t> v0, + const Vec128<uint8_t> v1, + const Vec128<uint8_t> v2, + const Vec128<uint8_t> v3, Full128<uint8_t> d8, + uint8_t* HWY_RESTRICT unaligned) { + const RepartitionToWide<decltype(d8)> d16; + const RepartitionToWide<decltype(d16)> d32; + // let a,b,c,d denote v0..3. + const auto ba0 = ZipLower(d16, v0, v1); // b7 a7 .. b0 a0 + const auto dc0 = ZipLower(d16, v2, v3); // d7 c7 .. d0 c0 + const auto ba8 = ZipUpper(d16, v0, v1); + const auto dc8 = ZipUpper(d16, v2, v3); + const auto dcba_0 = ZipLower(d32, ba0, dc0); // d..a3 d..a0 + const auto dcba_4 = ZipUpper(d32, ba0, dc0); // d..a7 d..a4 + const auto dcba_8 = ZipLower(d32, ba8, dc8); // d..aB d..a8 + const auto dcba_C = ZipUpper(d32, ba8, dc8); // d..aF d..aC + StoreU(BitCast(d8, dcba_0), d8, unaligned + 0 * 16); + StoreU(BitCast(d8, dcba_4), d8, unaligned + 1 * 16); + StoreU(BitCast(d8, dcba_8), d8, unaligned + 2 * 16); + StoreU(BitCast(d8, dcba_C), d8, unaligned + 3 * 16); +} + +// 64 bits +HWY_API void StoreInterleaved4(const Vec128<uint8_t, 8> in0, + const Vec128<uint8_t, 8> in1, + const Vec128<uint8_t, 8> in2, + const Vec128<uint8_t, 8> in3, + Simd<uint8_t, 8> /* tag */, + uint8_t* HWY_RESTRICT unaligned) { + // Use full vectors to reduce the number of stores. + const Full128<uint8_t> d_full8; + const RepartitionToWide<decltype(d_full8)> d16; + const RepartitionToWide<decltype(d16)> d32; + const Vec128<uint8_t> v0{in0.raw}; + const Vec128<uint8_t> v1{in1.raw}; + const Vec128<uint8_t> v2{in2.raw}; + const Vec128<uint8_t> v3{in3.raw}; + // let a,b,c,d denote v0..3. + const auto ba0 = ZipLower(d16, v0, v1); // b7 a7 .. b0 a0 + const auto dc0 = ZipLower(d16, v2, v3); // d7 c7 .. d0 c0 + const auto dcba_0 = ZipLower(d32, ba0, dc0); // d..a3 d..a0 + const auto dcba_4 = ZipUpper(d32, ba0, dc0); // d..a7 d..a4 + StoreU(BitCast(d_full8, dcba_0), d_full8, unaligned + 0 * 16); + StoreU(BitCast(d_full8, dcba_4), d_full8, unaligned + 1 * 16); +} + +// <= 32 bits +template <size_t N, HWY_IF_LE32(uint8_t, N)> +HWY_API void StoreInterleaved4(const Vec128<uint8_t, N> in0, + const Vec128<uint8_t, N> in1, + const Vec128<uint8_t, N> in2, + const Vec128<uint8_t, N> in3, + Simd<uint8_t, N> /*tag*/, + uint8_t* HWY_RESTRICT unaligned) { + // Use full vectors to reduce the number of stores. + const Full128<uint8_t> d_full8; + const RepartitionToWide<decltype(d_full8)> d16; + const RepartitionToWide<decltype(d16)> d32; + const Vec128<uint8_t> v0{in0.raw}; + const Vec128<uint8_t> v1{in1.raw}; + const Vec128<uint8_t> v2{in2.raw}; + const Vec128<uint8_t> v3{in3.raw}; + // let a,b,c,d denote v0..3. + const auto ba0 = ZipLower(d16, v0, v1); // b3 a3 .. b0 a0 + const auto dc0 = ZipLower(d16, v2, v3); // d3 c3 .. d0 c0 + const auto dcba_0 = ZipLower(d32, ba0, dc0); // d..a3 d..a0 + alignas(16) uint8_t buf[16]; + StoreU(BitCast(d_full8, dcba_0), d_full8, buf); + CopyBytes<4 * N>(buf, unaligned); +} + +// ------------------------------ MulEven/Odd (Load) + +HWY_INLINE Vec128<uint64_t> MulEven(const Vec128<uint64_t> a, + const Vec128<uint64_t> b) { + alignas(16) uint64_t mul[2]; + mul[0] = + Mul128(static_cast<uint64_t>(wasm_i64x2_extract_lane(a.raw, 0)), + static_cast<uint64_t>(wasm_i64x2_extract_lane(b.raw, 0)), &mul[1]); + return Load(Full128<uint64_t>(), mul); +} + +HWY_INLINE Vec128<uint64_t> MulOdd(const Vec128<uint64_t> a, + const Vec128<uint64_t> b) { + alignas(16) uint64_t mul[2]; + mul[0] = + Mul128(static_cast<uint64_t>(wasm_i64x2_extract_lane(a.raw, 1)), + static_cast<uint64_t>(wasm_i64x2_extract_lane(b.raw, 1)), &mul[1]); + return Load(Full128<uint64_t>(), mul); +} + +// ------------------------------ ReorderWidenMulAccumulate (MulAdd, ZipLower) + +template <size_t N> +HWY_API Vec128<float, N> ReorderWidenMulAccumulate(Simd<float, N> df32, + Vec128<bfloat16_t, 2 * N> a, + Vec128<bfloat16_t, 2 * N> b, + const Vec128<float, N> sum0, + Vec128<float, N>& sum1) { + const Repartition<uint16_t, decltype(df32)> du16; + const RebindToUnsigned<decltype(df32)> du32; + const Vec128<uint16_t, 2 * N> zero = Zero(du16); + const Vec128<uint32_t, N> a0 = ZipLower(du32, zero, BitCast(du16, a)); + const Vec128<uint32_t, N> a1 = ZipUpper(du32, zero, BitCast(du16, a)); + const Vec128<uint32_t, N> b0 = ZipLower(du32, zero, BitCast(du16, b)); + const Vec128<uint32_t, N> b1 = ZipUpper(du32, zero, BitCast(du16, b)); + sum1 = MulAdd(BitCast(df32, a1), BitCast(df32, b1), sum1); + return MulAdd(BitCast(df32, a0), BitCast(df32, b0), sum0); +} + +// ------------------------------ Reductions + +namespace detail { + +// N=1 for any T: no-op +template <typename T> +HWY_INLINE Vec128<T, 1> SumOfLanes(hwy::SizeTag<sizeof(T)> /* tag */, + const Vec128<T, 1> v) { + return v; +} +template <typename T> +HWY_INLINE Vec128<T, 1> MinOfLanes(hwy::SizeTag<sizeof(T)> /* tag */, + const Vec128<T, 1> v) { + return v; +} +template <typename T> +HWY_INLINE Vec128<T, 1> MaxOfLanes(hwy::SizeTag<sizeof(T)> /* tag */, + const Vec128<T, 1> v) { + return v; +} + +// u32/i32/f32: + +// N=2 +template <typename T> +HWY_INLINE Vec128<T, 2> SumOfLanes(hwy::SizeTag<4> /* tag */, + const Vec128<T, 2> v10) { + return v10 + Vec128<T, 2>{Shuffle2301(Vec128<T>{v10.raw}).raw}; +} +template <typename T> +HWY_INLINE Vec128<T, 2> MinOfLanes(hwy::SizeTag<4> /* tag */, + const Vec128<T, 2> v10) { + return Min(v10, Vec128<T, 2>{Shuffle2301(Vec128<T>{v10.raw}).raw}); +} +template <typename T> +HWY_INLINE Vec128<T, 2> MaxOfLanes(hwy::SizeTag<4> /* tag */, + const Vec128<T, 2> v10) { + return Max(v10, Vec128<T, 2>{Shuffle2301(Vec128<T>{v10.raw}).raw}); +} + +// N=4 (full) +template <typename T> +HWY_INLINE Vec128<T> SumOfLanes(hwy::SizeTag<4> /* tag */, + const Vec128<T> v3210) { + const Vec128<T> v1032 = Shuffle1032(v3210); + const Vec128<T> v31_20_31_20 = v3210 + v1032; + const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20); + return v20_31_20_31 + v31_20_31_20; +} +template <typename T> +HWY_INLINE Vec128<T> MinOfLanes(hwy::SizeTag<4> /* tag */, + const Vec128<T> v3210) { + const Vec128<T> v1032 = Shuffle1032(v3210); + const Vec128<T> v31_20_31_20 = Min(v3210, v1032); + const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20); + return Min(v20_31_20_31, v31_20_31_20); +} +template <typename T> +HWY_INLINE Vec128<T> MaxOfLanes(hwy::SizeTag<4> /* tag */, + const Vec128<T> v3210) { + const Vec128<T> v1032 = Shuffle1032(v3210); + const Vec128<T> v31_20_31_20 = Max(v3210, v1032); + const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20); + return Max(v20_31_20_31, v31_20_31_20); +} + +// u64/i64/f64: + +// N=2 (full) +template <typename T> +HWY_INLINE Vec128<T> SumOfLanes(hwy::SizeTag<8> /* tag */, + const Vec128<T> v10) { + const Vec128<T> v01 = Shuffle01(v10); + return v10 + v01; +} +template <typename T> +HWY_INLINE Vec128<T> MinOfLanes(hwy::SizeTag<8> /* tag */, + const Vec128<T> v10) { + const Vec128<T> v01 = Shuffle01(v10); + return Min(v10, v01); +} +template <typename T> +HWY_INLINE Vec128<T> MaxOfLanes(hwy::SizeTag<8> /* tag */, + const Vec128<T> v10) { + const Vec128<T> v01 = Shuffle01(v10); + return Max(v10, v01); +} + +// u16/i16 +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 2), HWY_IF_GE32(T, N)> +HWY_API Vec128<T, N> MinOfLanes(hwy::SizeTag<2> /* tag */, Vec128<T, N> v) { + const Repartition<int32_t, Simd<T, N>> d32; + const auto even = And(BitCast(d32, v), Set(d32, 0xFFFF)); + const auto odd = ShiftRight<16>(BitCast(d32, v)); + const auto min = MinOfLanes(d32, Min(even, odd)); + // Also broadcast into odd lanes. + return BitCast(Simd<T, N>(), Or(min, ShiftLeft<16>(min))); +} +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 2), HWY_IF_GE32(T, N)> +HWY_API Vec128<T, N> MaxOfLanes(hwy::SizeTag<2> /* tag */, Vec128<T, N> v) { + const Repartition<int32_t, Simd<T, N>> d32; + const auto even = And(BitCast(d32, v), Set(d32, 0xFFFF)); + const auto odd = ShiftRight<16>(BitCast(d32, v)); + const auto min = MaxOfLanes(d32, Max(even, odd)); + // Also broadcast into odd lanes. + return BitCast(Simd<T, N>(), Or(min, ShiftLeft<16>(min))); +} + +} // namespace detail + +// Supported for u/i/f 32/64. Returns the same value in each lane. +template <typename T, size_t N> +HWY_API Vec128<T, N> SumOfLanes(Simd<T, N> /* tag */, const Vec128<T, N> v) { + return detail::SumOfLanes(hwy::SizeTag<sizeof(T)>(), v); +} +template <typename T, size_t N> +HWY_API Vec128<T, N> MinOfLanes(Simd<T, N> /* tag */, const Vec128<T, N> v) { + return detail::MinOfLanes(hwy::SizeTag<sizeof(T)>(), v); +} +template <typename T, size_t N> +HWY_API Vec128<T, N> MaxOfLanes(Simd<T, N> /* tag */, const Vec128<T, N> v) { + return detail::MaxOfLanes(hwy::SizeTag<sizeof(T)>(), v); +} + +// ================================================== DEPRECATED + +template <typename T, size_t N> +HWY_API size_t StoreMaskBits(const Mask128<T, N> mask, uint8_t* bits) { + return StoreMaskBits(Simd<T, N>(), mask, bits); +} + +template <typename T, size_t N> +HWY_API bool AllTrue(const Mask128<T, N> mask) { + return AllTrue(Simd<T, N>(), mask); +} + +template <typename T, size_t N> +HWY_API bool AllFalse(const Mask128<T, N> mask) { + return AllFalse(Simd<T, N>(), mask); +} + +template <typename T, size_t N> +HWY_API size_t CountTrue(const Mask128<T, N> mask) { + return CountTrue(Simd<T, N>(), mask); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> SumOfLanes(const Vec128<T, N> v) { + return SumOfLanes(Simd<T, N>(), v); +} +template <typename T, size_t N> +HWY_API Vec128<T, N> MinOfLanes(const Vec128<T, N> v) { + return MinOfLanes(Simd<T, N>(), v); +} +template <typename T, size_t N> +HWY_API Vec128<T, N> MaxOfLanes(const Vec128<T, N> v) { + return MaxOfLanes(Simd<T, N>(), v); +} + +template <typename T, size_t N> +HWY_API Vec128<T, (N + 1) / 2> UpperHalf(Vec128<T, N> v) { + return UpperHalf(Half<Simd<T, N>>(), v); +} + +template <int kBytes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftRightBytes(const Vec128<T, N> v) { + return ShiftRightBytes<kBytes>(Simd<T, N>(), v); +} + +template <int kLanes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftRightLanes(const Vec128<T, N> v) { + return ShiftRightLanes<kLanes>(Simd<T, N>(), v); +} + +template <size_t kBytes, typename T, size_t N> +HWY_API Vec128<T, N> CombineShiftRightBytes(Vec128<T, N> hi, Vec128<T, N> lo) { + return CombineShiftRightBytes<kBytes>(Simd<T, N>(), hi, lo); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> InterleaveUpper(Vec128<T, N> a, Vec128<T, N> b) { + return InterleaveUpper(Simd<T, N>(), a, b); +} + +template <typename T, size_t N, class D = Simd<T, N>> +HWY_API VFromD<RepartitionToWide<D>> ZipUpper(Vec128<T, N> a, Vec128<T, N> b) { + return InterleaveUpper(RepartitionToWide<D>(), a, b); +} + +template <typename T, size_t N2> +HWY_API Vec128<T, N2 * 2> Combine(Vec128<T, N2> hi2, Vec128<T, N2> lo2) { + return Combine(Simd<T, N2 * 2>(), hi2, lo2); +} + +template <typename T, size_t N2, HWY_IF_LE64(T, N2)> +HWY_API Vec128<T, N2 * 2> ZeroExtendVector(Vec128<T, N2> lo) { + return ZeroExtendVector(Simd<T, N2 * 2>(), lo); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> ConcatLowerLower(Vec128<T, N> hi, Vec128<T, N> lo) { + return ConcatLowerLower(Simd<T, N>(), hi, lo); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> ConcatUpperUpper(Vec128<T, N> hi, Vec128<T, N> lo) { + return ConcatUpperUpper(Simd<T, N>(), hi, lo); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> ConcatLowerUpper(const Vec128<T, N> hi, + const Vec128<T, N> lo) { + return ConcatLowerUpper(Simd<T, N>(), hi, lo); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> ConcatUpperLower(Vec128<T, N> hi, Vec128<T, N> lo) { + return ConcatUpperLower(Simd<T, N>(), hi, lo); +} + +// ================================================== Operator wrapper + +template <class V> +HWY_API V Add(V a, V b) { + return a + b; +} +template <class V> +HWY_API V Sub(V a, V b) { + return a - b; +} + +template <class V> +HWY_API V Mul(V a, V b) { + return a * b; +} +template <class V> +HWY_API V Div(V a, V b) { + return a / b; +} + +template <class V> +V Shl(V a, V b) { + return a << b; +} +template <class V> +V Shr(V a, V b) { + return a >> b; +} + +template <class V> +HWY_API auto Eq(V a, V b) -> decltype(a == b) { + return a == b; +} +template <class V> +HWY_API auto Ne(V a, V b) -> decltype(a == b) { + return a != b; +} +template <class V> +HWY_API auto Lt(V a, V b) -> decltype(a == b) { + return a < b; +} + +template <class V> +HWY_API auto Gt(V a, V b) -> decltype(a == b) { + return a > b; +} +template <class V> +HWY_API auto Ge(V a, V b) -> decltype(a == b) { + return a >= b; +} + +template <class V> +HWY_API auto Le(V a, V b) -> decltype(a == b) { + return a <= b; +} + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); diff --git a/third_party/highway/hwy/ops/wasm_256-inl.h b/third_party/highway/hwy/ops/wasm_256-inl.h new file mode 100644 index 0000000..f66e257 --- /dev/null +++ b/third_party/highway/hwy/ops/wasm_256-inl.h @@ -0,0 +1,3513 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// 256-bit WASM vectors and operations. Experimental. +// External include guard in highway.h - see comment there. + +#include <stddef.h> +#include <stdint.h> +#include <wasm_simd128.h> + +#include "hwy/base.h" +#include "hwy/ops/shared-inl.h" +#include "hwy/ops/wasm_128-inl.h" + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +template <typename T> +using Full256 = Simd<T, 32 / sizeof(T)>; + +// TODO(richardwinterton): add this to DeduceD in wasm_128 similar to x86_128. +template <typename T> +class Vec256 { + public: + // Compound assignment. Only usable if there is a corresponding non-member + // binary operator overload. For example, only f32 and f64 support division. + HWY_INLINE Vec256& operator*=(const Vec256 other) { + return *this = (*this * other); + } + HWY_INLINE Vec256& operator/=(const Vec256 other) { + return *this = (*this / other); + } + HWY_INLINE Vec256& operator+=(const Vec256 other) { + return *this = (*this + other); + } + HWY_INLINE Vec256& operator-=(const Vec256 other) { + return *this = (*this - other); + } + HWY_INLINE Vec256& operator&=(const Vec256 other) { + return *this = (*this & other); + } + HWY_INLINE Vec256& operator|=(const Vec256 other) { + return *this = (*this | other); + } + HWY_INLINE Vec256& operator^=(const Vec256 other) { + return *this = (*this ^ other); + } + + Vec128<T> v0; + Vec128<T> v1; +}; + +template <typename T> +struct Mask256 { + Mask128<T> m0; + Mask128<T> m1; +}; + +// ------------------------------ BitCast + +template <typename T, size_t N, typename FromT> +HWY_API Vec256<T> BitCast(Simd<T, N> d, Vec256<FromT> v) { + const Half<decltype(d)> dh; + Vec256<T> ret; + ret.v0 = BitCast(dh, v.v0); + ret.v1 = BitCast(dh, v.v1); + return ret; + + // TODO(richardwinterton): implement other ops like this +} + +// ------------------------------ Zero + +// Returns an all-zero vector/part. +template <typename T, HWY_IF_GT128(T, N)> +HWY_API Vec256<T> Zero(Simd<T, N> /* tag */) { + return Vec256<T>{wasm_i32x4_splat(0)}; +} +template <size_t N, HWY_IF_GT128(float, N)> +HWY_API Vec128<float, N> Zero(Simd<float, N> /* tag */) { + return Vec128<float, N>{wasm_f32x4_splat(0.0f)}; +} + +template <class D> +using VFromD = decltype(Zero(D())); + +// ------------------------------ Set + +// Returns a vector/part with all lanes set to "t". +template <size_t N, HWY_IF_GT128(uint8_t, N)> +HWY_API Vec128<uint8_t, N> Set(Simd<uint8_t, N> /* tag */, const uint8_t t) { + return Vec128<uint8_t, N>{wasm_i8x16_splat(static_cast<int8_t>(t))}; +} +template <size_t N, HWY_IF_GT128(uint16_t, N)> +HWY_API Vec128<uint16_t, N> Set(Simd<uint16_t, N> /* tag */, const uint16_t t) { + return Vec128<uint16_t, N>{wasm_i16x8_splat(static_cast<int16_t>(t))}; +} +template <size_t N, HWY_IF_GT128(uint32_t, N)> +HWY_API Vec128<uint32_t, N> Set(Simd<uint32_t, N> /* tag */, const uint32_t t) { + return Vec128<uint32_t, N>{wasm_i32x4_splat(static_cast<int32_t>(t))}; +} +template <size_t N, HWY_IF_GT128(uint64_t, N)> +HWY_API Vec128<uint64_t, N> Set(Simd<uint64_t, N> /* tag */, const uint64_t t) { + return Vec128<uint64_t, N>{wasm_i64x2_splat(static_cast<int64_t>(t))}; +} + +template <size_t N, HWY_IF_GT128(int8_t, N)> +HWY_API Vec128<int8_t, N> Set(Simd<int8_t, N> /* tag */, const int8_t t) { + return Vec128<int8_t, N>{wasm_i8x16_splat(t)}; +} +template <size_t N, HWY_IF_GT128(int16_t, N)> +HWY_API Vec128<int16_t, N> Set(Simd<int16_t, N> /* tag */, const int16_t t) { + return Vec128<int16_t, N>{wasm_i16x8_splat(t)}; +} +template <size_t N, HWY_IF_GT128(int32_t, N)> +HWY_API Vec128<int32_t, N> Set(Simd<int32_t, N> /* tag */, const int32_t t) { + return Vec128<int32_t, N>{wasm_i32x4_splat(t)}; +} +template <size_t N, HWY_IF_GT128(int64_t, N)> +HWY_API Vec128<int64_t, N> Set(Simd<int64_t, N> /* tag */, const int64_t t) { + return Vec128<int64_t, N>{wasm_i64x2_splat(t)}; +} + +template <size_t N, HWY_IF_GT128(float, N)> +HWY_API Vec128<float, N> Set(Simd<float, N> /* tag */, const float t) { + return Vec128<float, N>{wasm_f32x4_splat(t)}; +} + +HWY_DIAGNOSTICS(push) +HWY_DIAGNOSTICS_OFF(disable : 4700, ignored "-Wuninitialized") + +// Returns a vector with uninitialized elements. +template <typename T, HWY_IF_GT128(T, N)> +HWY_API Vec256<T> Undefined(Simd<T, N> d) { + return Zero(d); +} + +HWY_DIAGNOSTICS(pop) + +// Returns a vector with lane i=[0, N) set to "first" + i. +template <typename T, typename T2> +Vec256<T> Iota(const Simd<T, N> d, const T2 first) { + HWY_ALIGN T lanes[16 / sizeof(T)]; + for (size_t i = 0; i < 16 / sizeof(T); ++i) { + lanes[i] = static_cast<T>(first + static_cast<T2>(i)); + } + return Load(d, lanes); +} + +// ================================================== ARITHMETIC + +// ------------------------------ Addition + +// Unsigned +template <size_t N> +HWY_API Vec128<uint8_t, N> operator+(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{wasm_i8x16_add(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> operator+(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{wasm_i16x8_add(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> operator+(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { + return Vec128<uint32_t, N>{wasm_i32x4_add(a.raw, b.raw)}; +} + +// Signed +template <size_t N> +HWY_API Vec128<int8_t, N> operator+(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Vec128<int8_t, N>{wasm_i8x16_add(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> operator+(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{wasm_i16x8_add(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> operator+(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + return Vec128<int32_t, N>{wasm_i32x4_add(a.raw, b.raw)}; +} + +// Float +template <size_t N> +HWY_API Vec128<float, N> operator+(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Vec128<float, N>{wasm_f32x4_add(a.raw, b.raw)}; +} + +// ------------------------------ Subtraction + +// Unsigned +template <size_t N> +HWY_API Vec128<uint8_t, N> operator-(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{wasm_i8x16_sub(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> operator-(Vec128<uint16_t, N> a, + Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{wasm_i16x8_sub(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> operator-(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { + return Vec128<uint32_t, N>{wasm_i32x4_sub(a.raw, b.raw)}; +} + +// Signed +template <size_t N> +HWY_API Vec128<int8_t, N> operator-(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Vec128<int8_t, N>{wasm_i8x16_sub(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> operator-(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{wasm_i16x8_sub(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> operator-(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + return Vec128<int32_t, N>{wasm_i32x4_sub(a.raw, b.raw)}; +} + +// Float +template <size_t N> +HWY_API Vec128<float, N> operator-(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Vec128<float, N>{wasm_f32x4_sub(a.raw, b.raw)}; +} + +// ------------------------------ Saturating addition + +// Returns a + b clamped to the destination range. + +// Unsigned +template <size_t N> +HWY_API Vec128<uint8_t, N> SaturatedAdd(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{wasm_u8x16_add_sat(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> SaturatedAdd(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{wasm_u16x8_add_sat(a.raw, b.raw)}; +} + +// Signed +template <size_t N> +HWY_API Vec128<int8_t, N> SaturatedAdd(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Vec128<int8_t, N>{wasm_i8x16_add_sat(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> SaturatedAdd(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{wasm_i16x8_add_sat(a.raw, b.raw)}; +} + +// ------------------------------ Saturating subtraction + +// Returns a - b clamped to the destination range. + +// Unsigned +template <size_t N> +HWY_API Vec128<uint8_t, N> SaturatedSub(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{wasm_u8x16_sub_sat(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> SaturatedSub(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{wasm_u16x8_sub_sat(a.raw, b.raw)}; +} + +// Signed +template <size_t N> +HWY_API Vec128<int8_t, N> SaturatedSub(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Vec128<int8_t, N>{wasm_i8x16_sub_sat(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> SaturatedSub(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{wasm_i16x8_sub_sat(a.raw, b.raw)}; +} + +// ------------------------------ Average + +// Returns (a + b + 1) / 2 + +// Unsigned +template <size_t N> +HWY_API Vec128<uint8_t, N> AverageRound(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{wasm_u8x16_avgr(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> AverageRound(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{wasm_u16x8_avgr(a.raw, b.raw)}; +} + +// ------------------------------ Absolute value + +// Returns absolute value, except that LimitsMin() maps to LimitsMax() + 1. +template <size_t N> +HWY_API Vec128<int8_t, N> Abs(const Vec128<int8_t, N> v) { + return Vec128<int8_t, N>{wasm_i8x16_abs(v.raw)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> Abs(const Vec128<int16_t, N> v) { + return Vec128<int16_t, N>{wasm_i16x8_abs(v.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> Abs(const Vec128<int32_t, N> v) { + return Vec128<int32_t, N>{wasm_i32x4_abs(v.raw)}; +} +template <size_t N> +HWY_API Vec128<int64_t, N> Abs(const Vec128<int64_t, N> v) { + return Vec128<int32_t, N>{wasm_i62x2_abs(v.raw)}; +} + +template <size_t N> +HWY_API Vec128<float, N> Abs(const Vec128<float, N> v) { + return Vec128<float, N>{wasm_f32x4_abs(v.raw)}; +} + +// ------------------------------ Shift lanes by constant #bits + +// Unsigned +template <int kBits, size_t N> +HWY_API Vec128<uint16_t, N> ShiftLeft(const Vec128<uint16_t, N> v) { + return Vec128<uint16_t, N>{wasm_i16x8_shl(v.raw, kBits)}; +} +template <int kBits, size_t N> +HWY_API Vec128<uint16_t, N> ShiftRight(const Vec128<uint16_t, N> v) { + return Vec128<uint16_t, N>{wasm_u16x8_shr(v.raw, kBits)}; +} +template <int kBits, size_t N> +HWY_API Vec128<uint32_t, N> ShiftLeft(const Vec128<uint32_t, N> v) { + return Vec128<uint32_t, N>{wasm_i32x4_shl(v.raw, kBits)}; +} +template <int kBits, size_t N> +HWY_API Vec128<uint32_t, N> ShiftRight(const Vec128<uint32_t, N> v) { + return Vec128<uint32_t, N>{wasm_u32x4_shr(v.raw, kBits)}; +} + +// Signed +template <int kBits, size_t N> +HWY_API Vec128<int16_t, N> ShiftLeft(const Vec128<int16_t, N> v) { + return Vec128<int16_t, N>{wasm_i16x8_shl(v.raw, kBits)}; +} +template <int kBits, size_t N> +HWY_API Vec128<int16_t, N> ShiftRight(const Vec128<int16_t, N> v) { + return Vec128<int16_t, N>{wasm_i16x8_shr(v.raw, kBits)}; +} +template <int kBits, size_t N> +HWY_API Vec128<int32_t, N> ShiftLeft(const Vec128<int32_t, N> v) { + return Vec128<int32_t, N>{wasm_i32x4_shl(v.raw, kBits)}; +} +template <int kBits, size_t N> +HWY_API Vec128<int32_t, N> ShiftRight(const Vec128<int32_t, N> v) { + return Vec128<int32_t, N>{wasm_i32x4_shr(v.raw, kBits)}; +} + +// 8-bit +template <int kBits, typename T, HWY_IF_LANE_SIZE(T, 1)> +HWY_API Vec256<T> ShiftLeft(const Vec256<T> v) { + const Simd<T, N> d8; + // Use raw instead of BitCast to support N=1. + const Vec256<T> shifted{ShiftLeft<kBits>(Vec128<MakeWide<T>>{v.raw}).raw}; + return kBits == 1 + ? (v + v) + : (shifted & Set(d8, static_cast<T>((0xFF << kBits) & 0xFF))); +} + +template <int kBits, size_t N> +HWY_API Vec128<uint8_t, N> ShiftRight(const Vec128<uint8_t, N> v) { + const Simd<uint8_t, N> d8; + // Use raw instead of BitCast to support N=1. + const Vec128<uint8_t, N> shifted{ + ShiftRight<kBits>(Vec128<uint16_t>{v.raw}).raw}; + return shifted & Set(d8, 0xFF >> kBits); +} + +template <int kBits, size_t N> +HWY_API Vec128<int8_t, N> ShiftRight(const Vec128<int8_t, N> v) { + const Simd<int8_t, N> di; + const Simd<uint8_t, N> du; + const auto shifted = BitCast(di, ShiftRight<kBits>(BitCast(du, v))); + const auto shifted_sign = BitCast(di, Set(du, 0x80 >> kBits)); + return (shifted ^ shifted_sign) - shifted_sign; +} + +// ------------------------------ RotateRight (ShiftRight, Or) +template <int kBits, typename T> +HWY_API Vec256<T> RotateRight(const Vec256<T> v) { + constexpr size_t kSizeInBits = sizeof(T) * 8; + static_assert(0 <= kBits && kBits < kSizeInBits, "Invalid shift count"); + if (kBits == 0) return v; + return Or(ShiftRight<kBits>(v), ShiftLeft<kSizeInBits - kBits>(v)); +} + +// ------------------------------ Shift lanes by same variable #bits + +// Unsigned +template <size_t N> +HWY_API Vec128<uint16_t, N> ShiftLeftSame(const Vec128<uint16_t, N> v, + const int bits) { + return Vec128<uint16_t, N>{wasm_i16x8_shl(v.raw, bits)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> ShiftRightSame(const Vec128<uint16_t, N> v, + const int bits) { + return Vec128<uint16_t, N>{wasm_u16x8_shr(v.raw, bits)}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> ShiftLeftSame(const Vec128<uint32_t, N> v, + const int bits) { + return Vec128<uint32_t, N>{wasm_i32x4_shl(v.raw, bits)}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> ShiftRightSame(const Vec128<uint32_t, N> v, + const int bits) { + return Vec128<uint32_t, N>{wasm_u32x4_shr(v.raw, bits)}; +} + +// Signed +template <size_t N> +HWY_API Vec128<int16_t, N> ShiftLeftSame(const Vec128<int16_t, N> v, + const int bits) { + return Vec128<int16_t, N>{wasm_i16x8_shl(v.raw, bits)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> ShiftRightSame(const Vec128<int16_t, N> v, + const int bits) { + return Vec128<int16_t, N>{wasm_i16x8_shr(v.raw, bits)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> ShiftLeftSame(const Vec128<int32_t, N> v, + const int bits) { + return Vec128<int32_t, N>{wasm_i32x4_shl(v.raw, bits)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> ShiftRightSame(const Vec128<int32_t, N> v, + const int bits) { + return Vec128<int32_t, N>{wasm_i32x4_shr(v.raw, bits)}; +} + +// 8-bit +template <typename T, HWY_IF_LANE_SIZE(T, 1)> +HWY_API Vec256<T> ShiftLeftSame(const Vec256<T> v, const int bits) { + const Simd<T, N> d8; + // Use raw instead of BitCast to support N=1. + const Vec256<T> shifted{ShiftLeftSame(Vec128<MakeWide<T>>{v.raw}, bits).raw}; + return shifted & Set(d8, (0xFF << bits) & 0xFF); +} + +template <size_t N> +HWY_API Vec128<uint8_t, N> ShiftRightSame(Vec128<uint8_t, N> v, + const int bits) { + const Simd<uint8_t, N> d8; + // Use raw instead of BitCast to support N=1. + const Vec128<uint8_t, N> shifted{ + ShiftRightSame(Vec128<uint16_t>{v.raw}, bits).raw}; + return shifted & Set(d8, 0xFF >> bits); +} + +template <size_t N> +HWY_API Vec128<int8_t, N> ShiftRightSame(Vec128<int8_t, N> v, const int bits) { + const Simd<int8_t, N> di; + const Simd<uint8_t, N> du; + const auto shifted = BitCast(di, ShiftRightSame(BitCast(du, v), bits)); + const auto shifted_sign = BitCast(di, Set(du, 0x80 >> bits)); + return (shifted ^ shifted_sign) - shifted_sign; +} + +// ------------------------------ Minimum + +// Unsigned +template <size_t N> +HWY_API Vec128<uint8_t, N> Min(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{wasm_u8x16_min(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> Min(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{wasm_u16x8_min(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> Min(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { + return Vec128<uint32_t, N>{wasm_u32x4_min(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint64_t, N> Min(const Vec128<uint64_t, N> a, + const Vec128<uint64_t, N> b) { + alignas(16) float min[4]; + min[0] = + HWY_MIN(wasm_u64x2_extract_lane(a, 0), wasm_u64x2_extract_lane(b, 0)); + min[1] = + HWY_MIN(wasm_u64x2_extract_lane(a, 1), wasm_u64x2_extract_lane(b, 1)); + return Vec128<uint64_t, N>{wasm_v128_load(min)}; +} + +// Signed +template <size_t N> +HWY_API Vec128<int8_t, N> Min(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Vec128<int8_t, N>{wasm_i8x16_min(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> Min(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{wasm_i16x8_min(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> Min(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + return Vec128<int32_t, N>{wasm_i32x4_min(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int64_t, N> Min(const Vec128<int64_t, N> a, + const Vec128<int64_t, N> b) { + alignas(16) float min[4]; + min[0] = + HWY_MIN(wasm_i64x2_extract_lane(a, 0), wasm_i64x2_extract_lane(b, 0)); + min[1] = + HWY_MIN(wasm_i64x2_extract_lane(a, 1), wasm_i64x2_extract_lane(b, 1)); + return Vec128<int64_t, N>{wasm_v128_load(min)}; +} + +// Float +template <size_t N> +HWY_API Vec128<float, N> Min(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Vec128<float, N>{wasm_f32x4_min(a.raw, b.raw)}; +} + +// ------------------------------ Maximum + +// Unsigned +template <size_t N> +HWY_API Vec128<uint8_t, N> Max(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{wasm_u8x16_max(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> Max(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{wasm_u16x8_max(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> Max(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { + return Vec128<uint32_t, N>{wasm_u32x4_max(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint64_t, N> Max(const Vec128<uint64_t, N> a, + const Vec128<uint64_t, N> b) { + alignas(16) float max[4]; + max[0] = + HWY_MAX(wasm_u64x2_extract_lane(a, 0), wasm_u64x2_extract_lane(b, 0)); + max[1] = + HWY_MAX(wasm_u64x2_extract_lane(a, 1), wasm_u64x2_extract_lane(b, 1)); + return Vec128<int64_t, N>{wasm_v128_load(max)}; +} + +// Signed +template <size_t N> +HWY_API Vec128<int8_t, N> Max(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Vec128<int8_t, N>{wasm_i8x16_max(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> Max(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{wasm_i16x8_max(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> Max(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + return Vec128<int32_t, N>{wasm_i32x4_max(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int64_t, N> Max(const Vec128<int64_t, N> a, + const Vec128<int64_t, N> b) { + alignas(16) float max[4]; + max[0] = + HWY_MAX(wasm_i64x2_extract_lane(a, 0), wasm_i64x2_extract_lane(b, 0)); + max[1] = + HWY_MAX(wasm_i64x2_extract_lane(a, 1), wasm_i64x2_extract_lane(b, 1)); + return Vec128<int64_t, N>{wasm_v128_load(max)}; +} + +// Float +template <size_t N> +HWY_API Vec128<float, N> Max(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Vec128<float, N>{wasm_f32x4_max(a.raw, b.raw)}; +} + +// ------------------------------ Integer multiplication + +// Unsigned +template <size_t N> +HWY_API Vec128<uint16_t, N> operator*(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{wasm_i16x8_mul(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> operator*(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { + return Vec128<uint32_t, N>{wasm_i32x4_mul(a.raw, b.raw)}; +} + +// Signed +template <size_t N> +HWY_API Vec128<int16_t, N> operator*(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{wasm_i16x8_mul(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> operator*(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + return Vec128<int32_t, N>{wasm_i32x4_mul(a.raw, b.raw)}; +} + +// Returns the upper 16 bits of a * b in each lane. +template <size_t N> +HWY_API Vec128<uint16_t, N> MulHigh(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + // TODO(eustas): replace, when implemented in WASM. + const auto al = wasm_u32x4_extend_low_u16x8(a.raw); + const auto ah = wasm_u32x4_extend_high_u16x8(a.raw); + const auto bl = wasm_u32x4_extend_low_u16x8(b.raw); + const auto bh = wasm_u32x4_extend_high_u16x8(b.raw); + const auto l = wasm_i32x4_mul(al, bl); + const auto h = wasm_i32x4_mul(ah, bh); + // TODO(eustas): shift-right + narrow? + return Vec128<uint16_t, N>{ + wasm_i16x8_shuffle(l, h, 1, 3, 5, 7, 9, 11, 13, 15)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> MulHigh(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + // TODO(eustas): replace, when implemented in WASM. + const auto al = wasm_i32x4_extend_low_i16x8(a.raw); + const auto ah = wasm_i32x4_extend_high_i16x8(a.raw); + const auto bl = wasm_i32x4_extend_low_i16x8(b.raw); + const auto bh = wasm_i32x4_extend_high_i16x8(b.raw); + const auto l = wasm_i32x4_mul(al, bl); + const auto h = wasm_i32x4_mul(ah, bh); + // TODO(eustas): shift-right + narrow? + return Vec128<int16_t, N>{ + wasm_i16x8_shuffle(l, h, 1, 3, 5, 7, 9, 11, 13, 15)}; +} + +// Multiplies even lanes (0, 2 ..) and returns the double-width result. +template <size_t N> +HWY_API Vec128<int64_t, (N + 1) / 2> MulEven(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + // TODO(eustas): replace, when implemented in WASM. + const auto kEvenMask = wasm_i32x4_make(-1, 0, -1, 0); + const auto ae = wasm_v128_and(a.raw, kEvenMask); + const auto be = wasm_v128_and(b.raw, kEvenMask); + return Vec128<int64_t, (N + 1) / 2>{wasm_i64x2_mul(ae, be)}; +} +template <size_t N> +HWY_API Vec128<uint64_t, (N + 1) / 2> MulEven(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { + // TODO(eustas): replace, when implemented in WASM. + const auto kEvenMask = wasm_i32x4_make(-1, 0, -1, 0); + const auto ae = wasm_v128_and(a.raw, kEvenMask); + const auto be = wasm_v128_and(b.raw, kEvenMask); + return Vec128<uint64_t, (N + 1) / 2>{wasm_i64x2_mul(ae, be)}; +} + +// ------------------------------ Negate + +template <typename T, HWY_IF_FLOAT(T)> +HWY_API Vec256<T> Neg(const Vec256<T> v) { + return Xor(v, SignBit(Simd<T, N>())); +} + +template <size_t N> +HWY_API Vec128<int8_t, N> Neg(const Vec128<int8_t, N> v) { + return Vec128<int8_t, N>{wasm_i8x16_neg(v.raw)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> Neg(const Vec128<int16_t, N> v) { + return Vec128<int16_t, N>{wasm_i16x8_neg(v.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> Neg(const Vec128<int32_t, N> v) { + return Vec128<int32_t, N>{wasm_i32x4_neg(v.raw)}; +} +template <size_t N> +HWY_API Vec128<int64_t, N> Neg(const Vec128<int64_t, N> v) { + return Vec128<int64_t, N>{wasm_i64x2_neg(v.raw)}; +} + +// ------------------------------ Floating-point mul / div + +template <size_t N> +HWY_API Vec128<float, N> operator*(Vec128<float, N> a, Vec128<float, N> b) { + return Vec128<float, N>{wasm_f32x4_mul(a.raw, b.raw)}; +} + +template <size_t N> +HWY_API Vec128<float, N> operator/(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Vec128<float, N>{wasm_f32x4_div(a.raw, b.raw)}; +} + +// Approximate reciprocal +template <size_t N> +HWY_API Vec128<float, N> ApproximateReciprocal(const Vec128<float, N> v) { + const Vec128<float, N> one = Vec128<float, N>{wasm_f32x4_splat(1.0f)}; + return one / v; +} + +// Absolute value of difference. +template <size_t N> +HWY_API Vec128<float, N> AbsDiff(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Abs(a - b); +} + +// ------------------------------ Floating-point multiply-add variants + +// Returns mul * x + add +template <size_t N> +HWY_API Vec128<float, N> MulAdd(const Vec128<float, N> mul, + const Vec128<float, N> x, + const Vec128<float, N> add) { + // TODO(eustas): replace, when implemented in WASM. + // TODO(eustas): is it wasm_f32x4_qfma? + return mul * x + add; +} + +// Returns add - mul * x +template <size_t N> +HWY_API Vec128<float, N> NegMulAdd(const Vec128<float, N> mul, + const Vec128<float, N> x, + const Vec128<float, N> add) { + // TODO(eustas): replace, when implemented in WASM. + return add - mul * x; +} + +// Returns mul * x - sub +template <size_t N> +HWY_API Vec128<float, N> MulSub(const Vec128<float, N> mul, + const Vec128<float, N> x, + const Vec128<float, N> sub) { + // TODO(eustas): replace, when implemented in WASM. + // TODO(eustas): is it wasm_f32x4_qfms? + return mul * x - sub; +} + +// Returns -mul * x - sub +template <size_t N> +HWY_API Vec128<float, N> NegMulSub(const Vec128<float, N> mul, + const Vec128<float, N> x, + const Vec128<float, N> sub) { + // TODO(eustas): replace, when implemented in WASM. + return Neg(mul) * x - sub; +} + +// ------------------------------ Floating-point square root + +// Full precision square root +template <size_t N> +HWY_API Vec128<float, N> Sqrt(const Vec128<float, N> v) { + return Vec128<float, N>{wasm_f32x4_sqrt(v.raw)}; +} + +// Approximate reciprocal square root +template <size_t N> +HWY_API Vec128<float, N> ApproximateReciprocalSqrt(const Vec128<float, N> v) { + // TODO(eustas): find cheaper a way to calculate this. + const Vec128<float, N> one = Vec128<float, N>{wasm_f32x4_splat(1.0f)}; + return one / Sqrt(v); +} + +// ------------------------------ Floating-point rounding + +// Toward nearest integer, ties to even +template <size_t N> +HWY_API Vec128<float, N> Round(const Vec128<float, N> v) { + return Vec128<float, N>{wasm_f32x4_nearest(v.raw)}; +} + +// Toward zero, aka truncate +template <size_t N> +HWY_API Vec128<float, N> Trunc(const Vec128<float, N> v) { + return Vec128<float, N>{wasm_f32x4_trunc(v.raw)}; +} + +// Toward +infinity, aka ceiling +template <size_t N> +HWY_API Vec128<float, N> Ceil(const Vec128<float, N> v) { + return Vec128<float, N>{wasm_f32x4_ceil(v.raw)}; +} + +// Toward -infinity, aka floor +template <size_t N> +HWY_API Vec128<float, N> Floor(const Vec128<float, N> v) { + return Vec128<float, N>{wasm_f32x4_floor(v.raw)}; +} + +// ================================================== COMPARE + +// Comparisons fill a lane with 1-bits if the condition is true, else 0. + +template <typename TFrom, typename TTo, size_t N> +HWY_API Mask128<TTo, N> RebindMask(Simd<TTo, N> /*tag*/, Mask128<TFrom, N> m) { + static_assert(sizeof(TFrom) == sizeof(TTo), "Must have same size"); + return Mask128<TTo, N>{m.raw}; +} + +template <typename T> +HWY_API Mask128<T, N> TestBit(Vec256<T> v, Vec256<T> bit) { + static_assert(!hwy::IsFloat<T>(), "Only integer vectors supported"); + return (v & bit) == bit; +} + +// ------------------------------ Equality + +// Unsigned +template <size_t N> +HWY_API Mask128<uint8_t, N> operator==(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Mask128<uint8_t, N>{wasm_i8x16_eq(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<uint16_t, N> operator==(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Mask128<uint16_t, N>{wasm_i16x8_eq(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<uint32_t, N> operator==(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { + return Mask128<uint32_t, N>{wasm_i32x4_eq(a.raw, b.raw)}; +} + +// Signed +template <size_t N> +HWY_API Mask128<int8_t, N> operator==(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Mask128<int8_t, N>{wasm_i8x16_eq(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<int16_t, N> operator==(Vec128<int16_t, N> a, + Vec128<int16_t, N> b) { + return Mask128<int16_t, N>{wasm_i16x8_eq(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<int32_t, N> operator==(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + return Mask128<int32_t, N>{wasm_i32x4_eq(a.raw, b.raw)}; +} + +// Float +template <size_t N> +HWY_API Mask128<float, N> operator==(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Mask128<float, N>{wasm_f32x4_eq(a.raw, b.raw)}; +} + +// ------------------------------ Inequality + +// Unsigned +template <size_t N> +HWY_API Mask128<uint8_t, N> operator!=(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Mask128<uint8_t, N>{wasm_i8x16_ne(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<uint16_t, N> operator!=(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Mask128<uint16_t, N>{wasm_i16x8_ne(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<uint32_t, N> operator!=(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { + return Mask128<uint32_t, N>{wasm_i32x4_ne(a.raw, b.raw)}; +} + +// Signed +template <size_t N> +HWY_API Mask128<int8_t, N> operator!=(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Mask128<int8_t, N>{wasm_i8x16_ne(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<int16_t, N> operator!=(Vec128<int16_t, N> a, + Vec128<int16_t, N> b) { + return Mask128<int16_t, N>{wasm_i16x8_ne(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<int32_t, N> operator!=(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + return Mask128<int32_t, N>{wasm_i32x4_ne(a.raw, b.raw)}; +} + +// Float +template <size_t N> +HWY_API Mask128<float, N> operator!=(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Mask128<float, N>{wasm_f32x4_ne(a.raw, b.raw)}; +} + +// ------------------------------ Strict inequality + +template <size_t N> +HWY_API Mask128<int8_t, N> operator>(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Mask128<int8_t, N>{wasm_i8x16_gt(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<int16_t, N> operator>(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Mask128<int16_t, N>{wasm_i16x8_gt(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<int32_t, N> operator>(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + return Mask128<int32_t, N>{wasm_i32x4_gt(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<int64_t, N> operator>(const Vec128<int64_t, N> a, + const Vec128<int64_t, N> b) { + const Simd<int32_t, N * 2> d32; + const auto a32 = BitCast(d32, a); + const auto b32 = BitCast(d32, b); + // If the upper half is less than or greater, this is the answer. + const auto m_gt = a32 < b32; + + // Otherwise, the lower half decides. + const auto m_eq = a32 == b32; + const auto lo_in_hi = wasm_i32x4_shuffle(m_gt, m_gt, 2, 2, 0, 0); + const auto lo_gt = And(m_eq, lo_in_hi); + + const auto gt = Or(lo_gt, m_gt); + // Copy result in upper 32 bits to lower 32 bits. + return Mask128<int64_t, N>{wasm_i32x4_shuffle(gt, gt, 3, 3, 1, 1)}; +} + +template <typename T, HWY_IF_UNSIGNED(T)> +HWY_API Mask128<T, N> operator>(Vec256<T> a, Vec256<T> b) { + const Simd<T, N> du; + const RebindToSigned<decltype(du)> di; + const Vec256<T> msb = Set(du, (LimitsMax<T>() >> 1) + 1); + return RebindMask(du, BitCast(di, Xor(a, msb)) > BitCast(di, Xor(b, msb))); +} + +template <size_t N> +HWY_API Mask128<float, N> operator>(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Mask128<float, N>{wasm_f32x4_gt(a.raw, b.raw)}; +} + +template <typename T> +HWY_API Mask128<T, N> operator<(const Vec256<T> a, const Vec256<T> b) { + return operator>(b, a); +} + +// ------------------------------ Weak inequality + +// Float <= >= +template <size_t N> +HWY_API Mask128<float, N> operator<=(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Mask128<float, N>{wasm_f32x4_le(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<float, N> operator>=(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Mask128<float, N>{wasm_f32x4_ge(a.raw, b.raw)}; +} + +// ------------------------------ FirstN (Iota, Lt) + +template <typename T> +HWY_API Mask128<T, N> FirstN(const Simd<T, N> d, size_t num) { + const RebindToSigned<decltype(d)> di; // Signed comparisons may be cheaper. + return RebindMask(d, Iota(di, 0) < Set(di, static_cast<MakeSigned<T>>(num))); +} + +// ================================================== LOGICAL + +// ------------------------------ Not + +template <typename T> +HWY_API Vec256<T> Not(Vec256<T> v) { + return Vec256<T>{wasm_v128_not(v.raw)}; +} + +// ------------------------------ And + +template <typename T> +HWY_API Vec256<T> And(Vec256<T> a, Vec256<T> b) { + return Vec256<T>{wasm_v128_and(a.raw, b.raw)}; +} + +// ------------------------------ AndNot + +// Returns ~not_mask & mask. +template <typename T> +HWY_API Vec256<T> AndNot(Vec256<T> not_mask, Vec256<T> mask) { + return Vec256<T>{wasm_v128_andnot(mask.raw, not_mask.raw)}; +} + +// ------------------------------ Or + +template <typename T> +HWY_API Vec256<T> Or(Vec256<T> a, Vec256<T> b) { + return Vec256<T>{wasm_v128_or(a.raw, b.raw)}; +} + +// ------------------------------ Xor + +template <typename T> +HWY_API Vec256<T> Xor(Vec256<T> a, Vec256<T> b) { + return Vec256<T>{wasm_v128_xor(a.raw, b.raw)}; +} + +// ------------------------------ Operator overloads (internal-only if float) + +template <typename T> +HWY_API Vec256<T> operator&(const Vec256<T> a, const Vec256<T> b) { + return And(a, b); +} + +template <typename T> +HWY_API Vec256<T> operator|(const Vec256<T> a, const Vec256<T> b) { + return Or(a, b); +} + +template <typename T> +HWY_API Vec256<T> operator^(const Vec256<T> a, const Vec256<T> b) { + return Xor(a, b); +} + +// ------------------------------ CopySign + +template <typename T> +HWY_API Vec256<T> CopySign(const Vec256<T> magn, const Vec256<T> sign) { + static_assert(IsFloat<T>(), "Only makes sense for floating-point"); + const auto msb = SignBit(Simd<T, N>()); + return Or(AndNot(msb, magn), And(msb, sign)); +} + +template <typename T> +HWY_API Vec256<T> CopySignToAbs(const Vec256<T> abs, const Vec256<T> sign) { + static_assert(IsFloat<T>(), "Only makes sense for floating-point"); + return Or(abs, And(SignBit(Simd<T, N>()), sign)); +} + +// ------------------------------ BroadcastSignBit (compare) + +template <typename T, HWY_IF_NOT_LANE_SIZE(T, 1)> +HWY_API Vec256<T> BroadcastSignBit(const Vec256<T> v) { + return ShiftRight<sizeof(T) * 8 - 1>(v); +} +template <size_t N> +HWY_API Vec128<int8_t, N> BroadcastSignBit(const Vec128<int8_t, N> v) { + return VecFromMask(Simd<int8_t, N>(), v < Zero(Simd<int8_t, N>())); +} + +// ------------------------------ Mask + +// Mask and Vec are the same (true = FF..FF). +template <typename T> +HWY_API Mask128<T, N> MaskFromVec(const Vec256<T> v) { + return Mask128<T, N>{v.raw}; +} + +template <typename T> +HWY_API Vec256<T> VecFromMask(Simd<T, N> /* tag */, Mask128<T, N> v) { + return Vec256<T>{v.raw}; +} + +// DEPRECATED +template <typename T> +HWY_API Vec256<T> VecFromMask(const Mask128<T, N> v) { + return Vec256<T>{v.raw}; +} + +// mask ? yes : no +template <typename T> +HWY_API Vec256<T> IfThenElse(Mask128<T, N> mask, Vec256<T> yes, Vec256<T> no) { + return Vec256<T>{wasm_v128_bitselect(yes.raw, no.raw, mask.raw)}; +} + +// mask ? yes : 0 +template <typename T> +HWY_API Vec256<T> IfThenElseZero(Mask128<T, N> mask, Vec256<T> yes) { + return yes & VecFromMask(Simd<T, N>(), mask); +} + +// mask ? 0 : no +template <typename T> +HWY_API Vec256<T> IfThenZeroElse(Mask128<T, N> mask, Vec256<T> no) { + return AndNot(VecFromMask(Simd<T, N>(), mask), no); +} + +template <typename T, HWY_IF_FLOAT(T)> +HWY_API Vec256<T> ZeroIfNegative(Vec256<T> v) { + const Simd<T, N> d; + const auto zero = Zero(d); + return IfThenElse(Mask128<T, N>{(v > zero).raw}, v, zero); +} + +// ------------------------------ Mask logical + +template <typename T> +HWY_API Mask128<T, N> Not(const Mask128<T, N> m) { + return MaskFromVec(Not(VecFromMask(Simd<T, N>(), m))); +} + +template <typename T> +HWY_API Mask128<T, N> And(const Mask128<T, N> a, Mask128<T, N> b) { + const Simd<T, N> d; + return MaskFromVec(And(VecFromMask(d, a), VecFromMask(d, b))); +} + +template <typename T> +HWY_API Mask128<T, N> AndNot(const Mask128<T, N> a, Mask128<T, N> b) { + const Simd<T, N> d; + return MaskFromVec(AndNot(VecFromMask(d, a), VecFromMask(d, b))); +} + +template <typename T> +HWY_API Mask128<T, N> Or(const Mask128<T, N> a, Mask128<T, N> b) { + const Simd<T, N> d; + return MaskFromVec(Or(VecFromMask(d, a), VecFromMask(d, b))); +} + +template <typename T> +HWY_API Mask128<T, N> Xor(const Mask128<T, N> a, Mask128<T, N> b) { + const Simd<T, N> d; + return MaskFromVec(Xor(VecFromMask(d, a), VecFromMask(d, b))); +} + +// ------------------------------ Shl (BroadcastSignBit, IfThenElse) + +// The x86 multiply-by-Pow2() trick will not work because WASM saturates +// float->int correctly to 2^31-1 (not 2^31). Because WASM's shifts take a +// scalar count operand, per-lane shift instructions would require extract_lane +// for each lane, and hoping that shuffle is correctly mapped to a native +// instruction. Using non-vector shifts would incur a store-load forwarding +// stall when loading the result vector. We instead test bits of the shift +// count to "predicate" a shift of the entire vector by a constant. + +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Vec256<T> operator<<(Vec256<T> v, const Vec256<T> bits) { + const Simd<T, N> d; + Mask128<T, N> mask; + // Need a signed type for BroadcastSignBit. + auto test = BitCast(RebindToSigned<decltype(d)>(), bits); + // Move the highest valid bit of the shift count into the sign bit. + test = ShiftLeft<12>(test); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftLeft<8>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftLeft<4>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftLeft<2>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + return IfThenElse(mask, ShiftLeft<1>(v), v); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec256<T> operator<<(Vec256<T> v, const Vec256<T> bits) { + const Simd<T, N> d; + Mask128<T, N> mask; + // Need a signed type for BroadcastSignBit. + auto test = BitCast(RebindToSigned<decltype(d)>(), bits); + // Move the highest valid bit of the shift count into the sign bit. + test = ShiftLeft<27>(test); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftLeft<16>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftLeft<8>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftLeft<4>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftLeft<2>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + return IfThenElse(mask, ShiftLeft<1>(v), v); +} + +// ------------------------------ Shr (BroadcastSignBit, IfThenElse) + +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Vec256<T> operator>>(Vec256<T> v, const Vec256<T> bits) { + const Simd<T, N> d; + Mask128<T, N> mask; + // Need a signed type for BroadcastSignBit. + auto test = BitCast(RebindToSigned<decltype(d)>(), bits); + // Move the highest valid bit of the shift count into the sign bit. + test = ShiftLeft<12>(test); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftRight<8>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftRight<4>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftRight<2>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + return IfThenElse(mask, ShiftRight<1>(v), v); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec256<T> operator>>(Vec256<T> v, const Vec256<T> bits) { + const Simd<T, N> d; + Mask128<T, N> mask; + // Need a signed type for BroadcastSignBit. + auto test = BitCast(RebindToSigned<decltype(d)>(), bits); + // Move the highest valid bit of the shift count into the sign bit. + test = ShiftLeft<27>(test); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftRight<16>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftRight<8>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftRight<4>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + test = ShiftLeft<1>(test); // next bit (descending order) + v = IfThenElse(mask, ShiftRight<2>(v), v); + + mask = RebindMask(d, MaskFromVec(BroadcastSignBit(test))); + return IfThenElse(mask, ShiftRight<1>(v), v); +} + +// ================================================== MEMORY + +// ------------------------------ Load + +template <typename T> +HWY_API Vec128<T> Load(Full256<T> /* tag */, const T* HWY_RESTRICT aligned) { + return Vec128<T>{wasm_v128_load(aligned)}; +} + +template <typename T> +HWY_API Vec256<T> MaskedLoad(Mask128<T, N> m, Simd<T, N> d, + const T* HWY_RESTRICT aligned) { + return IfThenElseZero(m, Load(d, aligned)); +} + +// Partial load. +template <typename T, HWY_IF_LE64(T, N)> +HWY_API Vec256<T> Load(Simd<T, N> /* tag */, const T* HWY_RESTRICT p) { + Vec256<T> v; + CopyBytes<sizeof(T) * N>(p, &v); + return v; +} + +// LoadU == Load. +template <typename T> +HWY_API Vec256<T> LoadU(Simd<T, N> d, const T* HWY_RESTRICT p) { + return Load(d, p); +} + +// 128-bit SIMD => nothing to duplicate, same as an unaligned load. +template <typename T, HWY_IF_GT128(T, N)> +HWY_API Vec256<T> LoadDup128(Simd<T, N> d, const T* HWY_RESTRICT p) { + return Load(d, p); +} + +// ------------------------------ Store + +template <typename T> +HWY_API void Store(Vec128<T> v, Full256<T> /* tag */, T* HWY_RESTRICT aligned) { + wasm_v128_store(aligned, v.raw); +} + +// Partial store. +template <typename T, HWY_IF_LE64(T, N)> +HWY_API void Store(Vec256<T> v, Simd<T, N> /* tag */, T* HWY_RESTRICT p) { + CopyBytes<sizeof(T) * N>(&v, p); +} + +HWY_API void Store(const Vec128<float, 1> v, Simd<float, 1> /* tag */, + float* HWY_RESTRICT p) { + *p = wasm_f32x4_extract_lane(v.raw, 0); +} + +// StoreU == Store. +template <typename T> +HWY_API void StoreU(Vec256<T> v, Simd<T, N> d, T* HWY_RESTRICT p) { + Store(v, d, p); +} + +// ------------------------------ Non-temporal stores + +// Same as aligned stores on non-x86. + +template <typename T> +HWY_API void Stream(Vec256<T> v, Simd<T, N> /* tag */, + T* HWY_RESTRICT aligned) { + wasm_v128_store(aligned, v.raw); +} + +// ------------------------------ Scatter (Store) + +template <typename T, typename Offset, HWY_IF_GT128(T, N)> +HWY_API void ScatterOffset(Vec256<T> v, Simd<T, N> d, T* HWY_RESTRICT base, + const Vec128<Offset, N> offset) { + static_assert(sizeof(T) == sizeof(Offset), "Must match for portability"); + + alignas(16) T lanes[N]; + Store(v, d, lanes); + + alignas(16) Offset offset_lanes[N]; + Store(offset, Simd<Offset, N>(), offset_lanes); + + uint8_t* base_bytes = reinterpret_cast<uint8_t*>(base); + for (size_t i = 0; i < N; ++i) { + CopyBytes<sizeof(T)>(&lanes[i], base_bytes + offset_lanes[i]); + } +} + +template <typename T, typename Index, HWY_IF_GT128(T, N)> +HWY_API void ScatterIndex(Vec256<T> v, Simd<T, N> d, T* HWY_RESTRICT base, + const Vec128<Index, N> index) { + static_assert(sizeof(T) == sizeof(Index), "Must match for portability"); + + alignas(16) T lanes[N]; + Store(v, d, lanes); + + alignas(16) Index index_lanes[N]; + Store(index, Simd<Index, N>(), index_lanes); + + for (size_t i = 0; i < N; ++i) { + base[index_lanes[i]] = lanes[i]; + } +} + +// ------------------------------ Gather (Load/Store) + +template <typename T, typename Offset> +HWY_API Vec256<T> GatherOffset(const Simd<T, N> d, const T* HWY_RESTRICT base, + const Vec128<Offset, N> offset) { + static_assert(sizeof(T) == sizeof(Offset), "Must match for portability"); + + alignas(16) Offset offset_lanes[N]; + Store(offset, Simd<Offset, N>(), offset_lanes); + + alignas(16) T lanes[N]; + const uint8_t* base_bytes = reinterpret_cast<const uint8_t*>(base); + for (size_t i = 0; i < N; ++i) { + CopyBytes<sizeof(T)>(base_bytes + offset_lanes[i], &lanes[i]); + } + return Load(d, lanes); +} + +template <typename T, typename Index> +HWY_API Vec256<T> GatherIndex(const Simd<T, N> d, const T* HWY_RESTRICT base, + const Vec128<Index, N> index) { + static_assert(sizeof(T) == sizeof(Index), "Must match for portability"); + + alignas(16) Index index_lanes[N]; + Store(index, Simd<Index, N>(), index_lanes); + + alignas(16) T lanes[N]; + for (size_t i = 0; i < N; ++i) { + lanes[i] = base[index_lanes[i]]; + } + return Load(d, lanes); +} + +// ================================================== SWIZZLE + +// ------------------------------ Extract lane + +// Gets the single value stored in a vector/part. +template <size_t N> +HWY_API uint8_t GetLane(const Vec128<uint8_t, N> v) { + return wasm_i8x16_extract_lane(v.raw, 0); +} +template <size_t N> +HWY_API int8_t GetLane(const Vec128<int8_t, N> v) { + return wasm_i8x16_extract_lane(v.raw, 0); +} +template <size_t N> +HWY_API uint16_t GetLane(const Vec128<uint16_t, N> v) { + return wasm_i16x8_extract_lane(v.raw, 0); +} +template <size_t N> +HWY_API int16_t GetLane(const Vec128<int16_t, N> v) { + return wasm_i16x8_extract_lane(v.raw, 0); +} +template <size_t N> +HWY_API uint32_t GetLane(const Vec128<uint32_t, N> v) { + return wasm_i32x4_extract_lane(v.raw, 0); +} +template <size_t N> +HWY_API int32_t GetLane(const Vec128<int32_t, N> v) { + return wasm_i32x4_extract_lane(v.raw, 0); +} +template <size_t N> +HWY_API uint64_t GetLane(const Vec128<uint64_t, N> v) { + return wasm_i64x2_extract_lane(v.raw, 0); +} +template <size_t N> +HWY_API int64_t GetLane(const Vec128<int64_t, N> v) { + return wasm_i64x2_extract_lane(v.raw, 0); +} + +template <size_t N> +HWY_API float GetLane(const Vec128<float, N> v) { + return wasm_f32x4_extract_lane(v.raw, 0); +} + +// ------------------------------ LowerHalf + +template <typename T> +HWY_API Vec128<T, N / 2> LowerHalf(Simd<T, N / 2> /* tag */, Vec256<T> v) { + return Vec128<T, N / 2>{v.raw}; +} + +template <typename T> +HWY_API Vec128<T, N / 2> LowerHalf(Vec256<T> v) { + return LowerHalf(Simd<T, N / 2>(), v); +} + +// ------------------------------ ShiftLeftBytes + +// 0x01..0F, kBytes = 1 => 0x02..0F00 +template <int kBytes, typename T> +HWY_API Vec256<T> ShiftLeftBytes(Simd<T, N> /* tag */, Vec256<T> v) { + static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes"); + const __i8x16 zero = wasm_i8x16_splat(0); + switch (kBytes) { + case 0: + return v; + + case 1: + return Vec256<T>{wasm_i8x16_shuffle(v.raw, zero, 16, 0, 1, 2, 3, 4, 5, 6, + 7, 8, 9, 10, 11, 12, 13, 14)}; + + case 2: + return Vec256<T>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 0, 1, 2, 3, 4, 5, + 6, 7, 8, 9, 10, 11, 12, 13)}; + + case 3: + return Vec256<T>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 0, 1, 2, 3, + 4, 5, 6, 7, 8, 9, 10, 11, 12)}; + + case 4: + return Vec256<T>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 0, 1, 2, + 3, 4, 5, 6, 7, 8, 9, 10, 11)}; + + case 5: + return Vec256<T>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, 0, 1, + 2, 3, 4, 5, 6, 7, 8, 9, 10)}; + + case 6: + return Vec256<T>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, 16, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9)}; + + case 7: + return Vec256<T>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, 16, + 16, 0, 1, 2, 3, 4, 5, 6, 7, 8)}; + + case 8: + return Vec256<T>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, 16, + 16, 16, 0, 1, 2, 3, 4, 5, 6, 7)}; + + case 9: + return Vec256<T>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 0, 1, 2, 3, 4, 5, 6)}; + + case 10: + return Vec256<T>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 0, 1, 2, 3, 4, 5)}; + + case 11: + return Vec256<T>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 0, 1, 2, 3, 4)}; + + case 12: + return Vec256<T>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 0, 1, 2, 3)}; + + case 13: + return Vec256<T>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 0, 1, 2)}; + + case 14: + return Vec256<T>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 0, + 1)}; + + case 15: + return Vec256<T>{wasm_i8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 16, + 0)}; + } + return Vec256<T>{zero}; +} + +template <int kBytes, typename T> +HWY_API Vec256<T> ShiftLeftBytes(Vec256<T> v) { + return ShiftLeftBytes<kBytes>(Simd<T, N>(), v); +} + +// ------------------------------ ShiftLeftLanes + +template <int kLanes, typename T> +HWY_API Vec256<T> ShiftLeftLanes(Simd<T, N> d, const Vec256<T> v) { + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, ShiftLeftBytes<kLanes * sizeof(T)>(BitCast(d8, v))); +} + +template <int kLanes, typename T> +HWY_API Vec256<T> ShiftLeftLanes(const Vec256<T> v) { + return ShiftLeftLanes<kLanes>(Simd<T, N>(), v); +} + +// ------------------------------ ShiftRightBytes +namespace detail { + +// Helper function allows zeroing invalid lanes in caller. +template <int kBytes, typename T> +HWY_API __i8x16 ShrBytes(const Vec256<T> v) { + static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes"); + const __i8x16 zero = wasm_i8x16_splat(0); + + switch (kBytes) { + case 0: + return v.raw; + + case 1: + return wasm_i8x16_shuffle(v.raw, zero, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16); + + case 2: + return wasm_i8x16_shuffle(v.raw, zero, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16, 16); + + case 3: + return wasm_i8x16_shuffle(v.raw, zero, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16, 16, 16); + + case 4: + return wasm_i8x16_shuffle(v.raw, zero, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 16, 16, 16, 16); + + case 5: + return wasm_i8x16_shuffle(v.raw, zero, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 16, 16, 16, 16); + + case 6: + return wasm_i8x16_shuffle(v.raw, zero, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 16, 16, 16, 16); + + case 7: + return wasm_i8x16_shuffle(v.raw, zero, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 16, 16, 16, 16, 16); + + case 8: + return wasm_i8x16_shuffle(v.raw, zero, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 16, 16, 16, 16, 16, 16, 16); + + case 9: + return wasm_i8x16_shuffle(v.raw, zero, 9, 10, 11, 12, 13, 14, 15, 16, 16, + 16, 16, 16, 16, 16, 16, 16); + + case 10: + return wasm_i8x16_shuffle(v.raw, zero, 10, 11, 12, 13, 14, 15, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16); + + case 11: + return wasm_i8x16_shuffle(v.raw, zero, 11, 12, 13, 14, 15, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16); + + case 12: + return wasm_i8x16_shuffle(v.raw, zero, 12, 13, 14, 15, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16); + + case 13: + return wasm_i8x16_shuffle(v.raw, zero, 13, 14, 15, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16); + + case 14: + return wasm_i8x16_shuffle(v.raw, zero, 14, 15, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16); + + case 15: + return wasm_i8x16_shuffle(v.raw, zero, 15, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16); + case 16: + return zero; + } +} + +} // namespace detail + +// 0x01..0F, kBytes = 1 => 0x0001..0E +template <int kBytes, typename T> +HWY_API Vec256<T> ShiftRightBytes(Simd<T, N> /* tag */, Vec256<T> v) { + // For partial vectors, clear upper lanes so we shift in zeros. + if (N != 16 / sizeof(T)) { + const Vec128<T> vfull{v.raw}; + v = Vec256<T>{IfThenElseZero(FirstN(Full256<T>(), N), vfull).raw}; + } + return Vec256<T>{detail::ShrBytes<kBytes>(v)}; +} + +// ------------------------------ ShiftRightLanes +template <int kLanes, typename T> +HWY_API Vec256<T> ShiftRightLanes(Simd<T, N> d, const Vec256<T> v) { + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, ShiftRightBytes<kLanes * sizeof(T)>(BitCast(d8, v))); +} + +// ------------------------------ UpperHalf (ShiftRightBytes) + +// Full input: copy hi into lo (smaller instruction encoding than shifts). +template <typename T> +HWY_API Vec128<T, 8 / sizeof(T)> UpperHalf(Half<Full256<T>> /* tag */, + const Vec128<T> v) { + return Vec128<T, 8 / sizeof(T)>{wasm_i32x4_shuffle(v.raw, v.raw, 2, 3, 2, 3)}; +} +HWY_API Vec128<float, 2> UpperHalf(Half<Full256<float>> /* tag */, + const Vec128<float> v) { + return Vec128<float, 2>{wasm_i32x4_shuffle(v.raw, v.raw, 2, 3, 2, 3)}; +} + +// Partial +template <typename T, HWY_IF_LE64(T, N)> +HWY_API Vec128<T, (N + 1) / 2> UpperHalf(Half<Simd<T, N>> /* tag */, + Vec256<T> v) { + const Simd<T, N> d; + const auto vu = BitCast(RebindToUnsigned<decltype(d)>(), v); + const auto upper = BitCast(d, ShiftRightBytes<N * sizeof(T) / 2>(vu)); + return Vec128<T, (N + 1) / 2>{upper.raw}; +} + +// ------------------------------ CombineShiftRightBytes + +template <int kBytes, typename T, class V = Vec128<T>> +HWY_API V CombineShiftRightBytes(Full256<T> /* tag */, V hi, V lo) { + static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes"); + switch (kBytes) { + case 0: + return lo; + + case 1: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16)}; + + case 2: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 2, 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17)}; + + case 3: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 3, 4, 5, 6, 7, 8, 9, 10, 11, + 12, 13, 14, 15, 16, 17, 18)}; + + case 4: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 4, 5, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16, 17, 18, 19)}; + + case 5: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 5, 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 16, 17, 18, 19, 20)}; + + case 6: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 6, 7, 8, 9, 10, 11, 12, 13, + 14, 15, 16, 17, 18, 19, 20, 21)}; + + case 7: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22)}; + + case 8: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23)}; + + case 9: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24)}; + + case 10: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22, 23, 24, 25)}; + + case 11: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 11, 12, 13, 14, 15, 16, 17, + 18, 19, 20, 21, 22, 23, 24, 25, 26)}; + + case 12: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27)}; + + case 13: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28)}; + + case 14: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29)}; + + case 15: + return V{wasm_i8x16_shuffle(lo.raw, hi.raw, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 25, 26, 27, 28, 29, 30)}; + } + return hi; +} + +template <int kBytes, typename T, HWY_IF_LE64(T, N), class V = Vec256<T>> +HWY_API V CombineShiftRightBytes(Simd<T, N> d, V hi, V lo) { + constexpr size_t kSize = N * sizeof(T); + static_assert(0 < kBytes && kBytes < kSize, "kBytes invalid"); + const Repartition<uint8_t, decltype(d)> d8; + const Full256<uint8_t> d_full8; + using V8 = VFromD<decltype(d_full8)>; + const V8 hi8{BitCast(d8, hi).raw}; + // Move into most-significant bytes + const V8 lo8 = ShiftLeftBytes<16 - kSize>(V8{BitCast(d8, lo).raw}); + const V8 r = CombineShiftRightBytes<16 - kSize + kBytes>(d_full8, hi8, lo8); + return V{BitCast(Full256<T>(), r).raw}; +} + +// ------------------------------ Broadcast/splat any lane + +// Unsigned +template <int kLane, size_t N> +HWY_API Vec128<uint16_t, N> Broadcast(const Vec128<uint16_t, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<uint16_t, N>{wasm_i16x8_shuffle( + v.raw, v.raw, kLane, kLane, kLane, kLane, kLane, kLane, kLane, kLane)}; +} +template <int kLane, size_t N> +HWY_API Vec128<uint32_t, N> Broadcast(const Vec128<uint32_t, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<uint32_t, N>{ + wasm_i32x4_shuffle(v.raw, v.raw, kLane, kLane, kLane, kLane)}; +} + +// Signed +template <int kLane, size_t N> +HWY_API Vec128<int16_t, N> Broadcast(const Vec128<int16_t, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<int16_t, N>{wasm_i16x8_shuffle( + v.raw, v.raw, kLane, kLane, kLane, kLane, kLane, kLane, kLane, kLane)}; +} +template <int kLane, size_t N> +HWY_API Vec128<int32_t, N> Broadcast(const Vec128<int32_t, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<int32_t, N>{ + wasm_i32x4_shuffle(v.raw, v.raw, kLane, kLane, kLane, kLane)}; +} + +// Float +template <int kLane, size_t N> +HWY_API Vec128<float, N> Broadcast(const Vec128<float, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<float, N>{ + wasm_i32x4_shuffle(v.raw, v.raw, kLane, kLane, kLane, kLane)}; +} + +// ------------------------------ TableLookupBytes + +// Returns vector of bytes[from[i]]. "from" is also interpreted as bytes, i.e. +// lane indices in [0, 16). +template <typename T, typename TI, size_t NI> +HWY_API Vec128<TI, NI> TableLookupBytes(const Vec256<T> bytes, + const Vec128<TI, NI> from) { +// Not yet available in all engines, see +// https://github.com/WebAssembly/simd/blob/bdcc304b2d379f4601c2c44ea9b44ed9484fde7e/proposals/simd/ImplementationStatus.md +// V8 implementation of this had a bug, fixed on 2021-04-03: +// https://chromium-review.googlesource.com/c/v8/v8/+/2822951 +#if 0 + return Vec128<TI, NI>{wasm_i8x16_swizzle(bytes.raw, from.raw)}; +#else + alignas(16) uint8_t control[16]; + alignas(16) uint8_t input[16]; + alignas(16) uint8_t output[16]; + wasm_v128_store(control, from.raw); + wasm_v128_store(input, bytes.raw); + for (size_t i = 0; i < 16; ++i) { + output[i] = control[i] < 16 ? input[control[i]] : 0; + } + return Vec128<TI, NI>{wasm_v128_load(output)}; +#endif +} + +template <typename T, typename TI, size_t NI> +HWY_API Vec128<TI, NI> TableLookupBytesOr0(const Vec256<T> bytes, + const Vec128<TI, NI> from) { + const Simd<TI, NI> d; + // Mask size must match vector type, so cast everything to this type. + Repartition<int8_t, decltype(d)> di8; + Repartition<int8_t, Simd<T, N>> d_bytes8; + const auto msb = BitCast(di8, from) < Zero(di8); + const auto lookup = + TableLookupBytes(BitCast(d_bytes8, bytes), BitCast(di8, from)); + return BitCast(d, IfThenZeroElse(msb, lookup)); +} + +// ------------------------------ Hard-coded shuffles + +// Notation: let Vec128<int32_t> have lanes 3,2,1,0 (0 is least-significant). +// Shuffle0321 rotates one lane to the right (the previous least-significant +// lane is now most-significant). These could also be implemented via +// CombineShiftRightBytes but the shuffle_abcd notation is more convenient. + +// Swap 32-bit halves in 64-bit halves. +HWY_API Vec128<uint32_t> Shuffle2301(const Vec128<uint32_t> v) { + return Vec128<uint32_t>{wasm_i32x4_shuffle(v.raw, v.raw, 1, 0, 3, 2)}; +} +HWY_API Vec128<int32_t> Shuffle2301(const Vec128<int32_t> v) { + return Vec128<int32_t>{wasm_i32x4_shuffle(v.raw, v.raw, 1, 0, 3, 2)}; +} +HWY_API Vec128<float> Shuffle2301(const Vec128<float> v) { + return Vec128<float>{wasm_i32x4_shuffle(v.raw, v.raw, 1, 0, 3, 2)}; +} + +// Swap 64-bit halves +HWY_API Vec128<uint32_t> Shuffle1032(const Vec128<uint32_t> v) { + return Vec128<uint32_t>{wasm_i64x2_shuffle(v.raw, v.raw, 1, 0)}; +} +HWY_API Vec128<int32_t> Shuffle1032(const Vec128<int32_t> v) { + return Vec128<int32_t>{wasm_i64x2_shuffle(v.raw, v.raw, 1, 0)}; +} +HWY_API Vec128<float> Shuffle1032(const Vec128<float> v) { + return Vec128<float>{wasm_i64x2_shuffle(v.raw, v.raw, 1, 0)}; +} + +// Rotate right 32 bits +HWY_API Vec128<uint32_t> Shuffle0321(const Vec128<uint32_t> v) { + return Vec128<uint32_t>{wasm_i32x4_shuffle(v.raw, v.raw, 1, 2, 3, 0)}; +} +HWY_API Vec128<int32_t> Shuffle0321(const Vec128<int32_t> v) { + return Vec128<int32_t>{wasm_i32x4_shuffle(v.raw, v.raw, 1, 2, 3, 0)}; +} +HWY_API Vec128<float> Shuffle0321(const Vec128<float> v) { + return Vec128<float>{wasm_i32x4_shuffle(v.raw, v.raw, 1, 2, 3, 0)}; +} +// Rotate left 32 bits +HWY_API Vec128<uint32_t> Shuffle2103(const Vec128<uint32_t> v) { + return Vec128<uint32_t>{wasm_i32x4_shuffle(v.raw, v.raw, 3, 0, 1, 2)}; +} +HWY_API Vec128<int32_t> Shuffle2103(const Vec128<int32_t> v) { + return Vec128<int32_t>{wasm_i32x4_shuffle(v.raw, v.raw, 3, 0, 1, 2)}; +} +HWY_API Vec128<float> Shuffle2103(const Vec128<float> v) { + return Vec128<float>{wasm_i32x4_shuffle(v.raw, v.raw, 3, 0, 1, 2)}; +} + +// Reverse +HWY_API Vec128<uint32_t> Shuffle0123(const Vec128<uint32_t> v) { + return Vec128<uint32_t>{wasm_i32x4_shuffle(v.raw, v.raw, 3, 2, 1, 0)}; +} +HWY_API Vec128<int32_t> Shuffle0123(const Vec128<int32_t> v) { + return Vec128<int32_t>{wasm_i32x4_shuffle(v.raw, v.raw, 3, 2, 1, 0)}; +} +HWY_API Vec128<float> Shuffle0123(const Vec128<float> v) { + return Vec128<float>{wasm_i32x4_shuffle(v.raw, v.raw, 3, 2, 1, 0)}; +} + +// ------------------------------ TableLookupLanes + +// Returned by SetTableIndices for use by TableLookupLanes. +template <typename T> +struct Indices256 { + __v128_u raw; +}; + +template <typename T, size_t N, typename TI> +HWY_API Indices256<T, N> IndicesFromVec(Simd<T, N> d, Vec256<TI, N> vec) { + static_assert(sizeof(T) == sizeof(TI), "Index size must match lane"); + return Indices256<T, N>{}; +} + +template <typename T, size_t N, typename TI, HWY_IF_LE256(T, N)> +HWY_API Indices256<T, N> SetTableIndices(Simd<T, N> d, const TI* idx) { + const Rebind<TI, decltype(d)> di; + return IndicesFromVec(d, LoadU(di, idx)); +} + +template <typename T> +HWY_API Vec256<T> TableLookupLanes(Vec256<T> v, Indices256<T, N> idx) { + using TI = MakeSigned<T>; + const Simd<T, N> d; + const Simd<TI, N> di; + return BitCast(d, TableLookupBytes(BitCast(di, v), Vec256<TI, N>{idx.raw})); +} + +// ------------------------------ Reverse (Shuffle0123, Shuffle2301, Shuffle01) + +// Single lane: no change +template <typename T> +HWY_API Vec128<T, 1> Reverse(Simd<T, 1> /* tag */, const Vec128<T, 1> v) { + return v; +} + +// Two lanes: shuffle +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T, 2> Reverse(Simd<T, 2> /* tag */, const Vec128<T, 2> v) { + return Vec128<T, 2>{Shuffle2301(Vec128<T>{v.raw}).raw}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec128<T> Reverse(Full256<T> /* tag */, const Vec128<T> v) { + return Shuffle01(v); +} + +// Four lanes: shuffle +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T> Reverse(Full256<T> /* tag */, const Vec128<T> v) { + return Shuffle0123(v); +} + +// 16-bit +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Vec256<T> Reverse(Simd<T, N> d, const Vec256<T> v) { + const RepartitionToWide<RebindToUnsigned<decltype(d)>> du32; + return BitCast(d, RotateRight<16>(Reverse(du32, BitCast(du32, v)))); +} + +// ------------------------------ InterleaveLower + +template <size_t N> +HWY_API Vec128<uint8_t, N> InterleaveLower(Vec128<uint8_t, N> a, + Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{wasm_i8x16_shuffle( + a.raw, b.raw, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> InterleaveLower(Vec128<uint16_t, N> a, + Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{ + wasm_i16x8_shuffle(a.raw, b.raw, 0, 8, 1, 9, 2, 10, 3, 11)}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> InterleaveLower(Vec128<uint32_t, N> a, + Vec128<uint32_t, N> b) { + return Vec128<uint32_t, N>{wasm_i32x4_shuffle(a.raw, b.raw, 0, 4, 1, 5)}; +} +template <size_t N> +HWY_API Vec128<uint64_t, N> InterleaveLower(Vec128<uint64_t, N> a, + Vec128<uint64_t, N> b) { + return Vec128<uint64_t, N>{wasm_i64x2_shuffle(a.raw, b.raw, 0, 2)}; +} + +template <size_t N> +HWY_API Vec128<int8_t, N> InterleaveLower(Vec128<int8_t, N> a, + Vec128<int8_t, N> b) { + return Vec128<int8_t, N>{wasm_i8x16_shuffle( + a.raw, b.raw, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> InterleaveLower(Vec128<int16_t, N> a, + Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{ + wasm_i16x8_shuffle(a.raw, b.raw, 0, 8, 1, 9, 2, 10, 3, 11)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> InterleaveLower(Vec128<int32_t, N> a, + Vec128<int32_t, N> b) { + return Vec128<int32_t, N>{wasm_i32x4_shuffle(a.raw, b.raw, 0, 4, 1, 5)}; +} +template <size_t N> +HWY_API Vec128<int64_t, N> InterleaveLower(Vec128<int64_t, N> a, + Vec128<int64_t, N> b) { + return Vec128<int64_t, N>{wasm_i64x2_shuffle(a.raw, b.raw, 0, 2)}; +} + +template <size_t N> +HWY_API Vec128<float, N> InterleaveLower(Vec128<float, N> a, + Vec128<float, N> b) { + return Vec128<float, N>{wasm_i32x4_shuffle(a.raw, b.raw, 0, 4, 1, 5)}; +} + +// Additional overload for the optional Simd<> tag. +template <typename T, class V = Vec256<T>> +HWY_API V InterleaveLower(Simd<T, N> /* tag */, V a, V b) { + return InterleaveLower(a, b); +} + +// ------------------------------ InterleaveUpper (UpperHalf) + +// All functions inside detail lack the required D parameter. +namespace detail { + +template <size_t N> +HWY_API Vec128<uint8_t, N> InterleaveUpper(Vec128<uint8_t, N> a, + Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{wasm_i8x16_shuffle(a.raw, b.raw, 8, 24, 9, 25, 10, + 26, 11, 27, 12, 28, 13, 29, 14, + 30, 15, 31)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> InterleaveUpper(Vec128<uint16_t, N> a, + Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{ + wasm_i16x8_shuffle(a.raw, b.raw, 4, 12, 5, 13, 6, 14, 7, 15)}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> InterleaveUpper(Vec128<uint32_t, N> a, + Vec128<uint32_t, N> b) { + return Vec128<uint32_t, N>{wasm_i32x4_shuffle(a.raw, b.raw, 2, 6, 3, 7)}; +} +template <size_t N> +HWY_API Vec128<uint64_t, N> InterleaveUpper(Vec128<uint64_t, N> a, + Vec128<uint64_t, N> b) { + return Vec128<uint64_t, N>{wasm_i64x2_shuffle(a.raw, b.raw, 1, 3)}; +} + +template <size_t N> +HWY_API Vec128<int8_t, N> InterleaveUpper(Vec128<int8_t, N> a, + Vec128<int8_t, N> b) { + return Vec128<int8_t, N>{wasm_i8x16_shuffle(a.raw, b.raw, 8, 24, 9, 25, 10, + 26, 11, 27, 12, 28, 13, 29, 14, + 30, 15, 31)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> InterleaveUpper(Vec128<int16_t, N> a, + Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{ + wasm_i16x8_shuffle(a.raw, b.raw, 4, 12, 5, 13, 6, 14, 7, 15)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> InterleaveUpper(Vec128<int32_t, N> a, + Vec128<int32_t, N> b) { + return Vec128<int32_t, N>{wasm_i32x4_shuffle(a.raw, b.raw, 2, 6, 3, 7)}; +} +template <size_t N> +HWY_API Vec128<int64_t, N> InterleaveUpper(Vec128<int64_t, N> a, + Vec128<int64_t, N> b) { + return Vec128<int64_t, N>{wasm_i64x2_shuffle(a.raw, b.raw, 1, 3)}; +} + +template <size_t N> +HWY_API Vec128<float, N> InterleaveUpper(Vec128<float, N> a, + Vec128<float, N> b) { + return Vec128<float, N>{wasm_i32x4_shuffle(a.raw, b.raw, 2, 6, 3, 7)}; +} + +} // namespace detail + +// Full +template <typename T, class V = Vec128<T>> +HWY_API V InterleaveUpper(Full256<T> /* tag */, V a, V b) { + return detail::InterleaveUpper(a, b); +} + +// Partial +template <typename T, HWY_IF_LE64(T, N), class V = Vec256<T>> +HWY_API V InterleaveUpper(Simd<T, N> d, V a, V b) { + const Half<decltype(d)> d2; + return InterleaveLower(d, V{UpperHalf(d2, a).raw}, V{UpperHalf(d2, b).raw}); +} + +// ------------------------------ ZipLower/ZipUpper (InterleaveLower) + +// Same as Interleave*, except that the return lanes are double-width integers; +// this is necessary because the single-lane scalar cannot return two values. +template <typename T, class DW = RepartitionToWide<Simd<T, N>>> +HWY_API VFromD<DW> ZipLower(Vec256<T> a, Vec256<T> b) { + return BitCast(DW(), InterleaveLower(a, b)); +} +template <typename T, class D = Simd<T, N>, class DW = RepartitionToWide<D>> +HWY_API VFromD<DW> ZipLower(DW dw, Vec256<T> a, Vec256<T> b) { + return BitCast(dw, InterleaveLower(D(), a, b)); +} + +template <typename T, class D = Simd<T, N>, class DW = RepartitionToWide<D>> +HWY_API VFromD<DW> ZipUpper(DW dw, Vec256<T> a, Vec256<T> b) { + return BitCast(dw, InterleaveUpper(D(), a, b)); +} + +// ================================================== COMBINE + +// ------------------------------ Combine (InterleaveLower) + +// N = N/2 + N/2 (upper half undefined) +template <typename T> +HWY_API Vec256<T> Combine(Simd<T, N> d, Vec128<T, N / 2> hi_half, + Vec128<T, N / 2> lo_half) { + const Half<decltype(d)> d2; + const RebindToUnsigned<decltype(d2)> du2; + // Treat half-width input as one lane, and expand to two lanes. + using VU = Vec128<UnsignedFromSize<N * sizeof(T) / 2>, 2>; + const VU lo{BitCast(du2, lo_half).raw}; + const VU hi{BitCast(du2, hi_half).raw}; + return BitCast(d, InterleaveLower(lo, hi)); +} + +// ------------------------------ ZeroExtendVector (Combine, IfThenElseZero) + +template <typename T> +HWY_API Vec256<T> ZeroExtendVector(Simd<T, N> d, Vec128<T, N / 2> lo) { + return IfThenElseZero(FirstN(d, N / 2), Vec256<T>{lo.raw}); +} + +// ------------------------------ ConcatLowerLower + +// hiH,hiL loH,loL |-> hiL,loL (= lower halves) +template <typename T> +HWY_API Vec128<T> ConcatLowerLower(Full256<T> /* tag */, const Vec128<T> hi, + const Vec128<T> lo) { + return Vec128<T>{wasm_i64x2_shuffle(lo.raw, hi.raw, 0, 2)}; +} +template <typename T, HWY_IF_LE64(T, N)> +HWY_API Vec256<T> ConcatLowerLower(Simd<T, N> d, const Vec256<T> hi, + const Vec256<T> lo) { + const Half<decltype(d)> d2; + return Combine(LowerHalf(d2, hi), LowerHalf(d2, lo)); +} + +// ------------------------------ ConcatUpperUpper + +template <typename T> +HWY_API Vec128<T> ConcatUpperUpper(Full256<T> /* tag */, const Vec128<T> hi, + const Vec128<T> lo) { + return Vec128<T>{wasm_i64x2_shuffle(lo.raw, hi.raw, 1, 3)}; +} +template <typename T, HWY_IF_LE64(T, N)> +HWY_API Vec256<T> ConcatUpperUpper(Simd<T, N> d, const Vec256<T> hi, + const Vec256<T> lo) { + const Half<decltype(d)> d2; + return Combine(UpperHalf(d2, hi), UpperHalf(d2, lo)); +} + +// ------------------------------ ConcatLowerUpper + +template <typename T> +HWY_API Vec128<T> ConcatLowerUpper(Full256<T> d, const Vec128<T> hi, + const Vec128<T> lo) { + return CombineShiftRightBytes<8>(d, hi, lo); +} +template <typename T, HWY_IF_LE64(T, N)> +HWY_API Vec256<T> ConcatLowerUpper(Simd<T, N> d, const Vec256<T> hi, + const Vec256<T> lo) { + const Half<decltype(d)> d2; + return Combine(LowerHalf(d2, hi), UpperHalf(d2, lo)); +} + +// ------------------------------ ConcatUpperLower +template <typename T> +HWY_API Vec256<T> ConcatUpperLower(Simd<T, N> d, const Vec256<T> hi, + const Vec256<T> lo) { + return IfThenElse(FirstN(d, Lanes(d) / 2), lo, hi); +} + +// ------------------------------ ConcatOdd + +// 32-bit full +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T> ConcatOdd(Full256<T> /* tag */, Vec128<T> hi, Vec128<T> lo) { + return Vec128<T>{wasm_i32x4_shuffle(lo.raw, hi.raw, 1, 3, 5, 7)}; +} + +// 32-bit partial +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T, 2> ConcatOdd(Simd<T, 2> /* tag */, Vec128<T, 2> hi, + Vec128<T, 2> lo) { + return InterleaveUpper(Simd<T, 2>(), lo, hi); +} + +// 64-bit full - no partial because we need at least two inputs to have +// even/odd. +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec128<T> ConcatOdd(Full256<T> /* tag */, Vec128<T> hi, Vec128<T> lo) { + return InterleaveUpper(Full256<T>(), lo, hi); +} + +// ------------------------------ ConcatEven (InterleaveLower) + +// 32-bit full +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T> ConcatEven(Full256<T> /* tag */, Vec128<T> hi, Vec128<T> lo) { + return Vec128<T>{wasm_i32x4_shuffle(lo.raw, hi.raw, 0, 2, 4, 6)}; +} + +// 32-bit partial +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T, 2> ConcatEven(Simd<T, 2> /* tag */, Vec128<T, 2> hi, + Vec128<T, 2> lo) { + return InterleaveLower(Simd<T, 2>(), lo, hi); +} + +// 64-bit full - no partial because we need at least two inputs to have +// even/odd. +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec128<T> ConcatEven(Full256<T> /* tag */, Vec128<T> hi, Vec128<T> lo) { + return InterleaveLower(Full256<T>(), lo, hi); +} + +// ------------------------------ OddEven + +namespace detail { + +template <typename T> +HWY_INLINE Vec256<T> OddEven(hwy::SizeTag<1> /* tag */, const Vec256<T> a, + const Vec256<T> b) { + const Simd<T, N> d; + const Repartition<uint8_t, decltype(d)> d8; + alignas(16) constexpr uint8_t mask[16] = {0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, + 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0}; + return IfThenElse(MaskFromVec(BitCast(d, Load(d8, mask))), b, a); +} +template <typename T> +HWY_INLINE Vec256<T> OddEven(hwy::SizeTag<2> /* tag */, const Vec256<T> a, + const Vec256<T> b) { + return Vec256<T>{wasm_i16x8_shuffle(a.raw, b.raw, 8, 1, 10, 3, 12, 5, 14, 7)}; +} +template <typename T> +HWY_INLINE Vec256<T> OddEven(hwy::SizeTag<4> /* tag */, const Vec256<T> a, + const Vec256<T> b) { + return Vec256<T>{wasm_i32x4_shuffle(a.raw, b.raw, 4, 1, 6, 3)}; +} +template <typename T> +HWY_INLINE Vec256<T> OddEven(hwy::SizeTag<8> /* tag */, const Vec256<T> a, + const Vec256<T> b) { + return Vec256<T>{wasm_i64x2_shuffle(a.raw, b.raw, 2, 1)}; +} + +} // namespace detail + +template <typename T> +HWY_API Vec256<T> OddEven(const Vec256<T> a, const Vec256<T> b) { + return detail::OddEven(hwy::SizeTag<sizeof(T)>(), a, b); +} +template <size_t N> +HWY_API Vec128<float, N> OddEven(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Vec128<float, N>{wasm_i32x4_shuffle(a.raw, b.raw, 4, 1, 6, 3)}; +} + +// ------------------------------ OddEvenBlocks +template <typename T> +HWY_API Vec256<T> OddEvenBlocks(Vec256<T> /* odd */, Vec256<T> even) { + return even; +} + +// ------------------------------ SwapAdjacentBlocks + +template <typename T> +HWY_API Vec256<T> SwapAdjacentBlocks(Vec256<T> v) { + return v; +} + +// ================================================== CONVERT + +// ------------------------------ Promotions (part w/ narrow lanes -> full) + +// Unsigned: zero-extend. +template <size_t N> +HWY_API Vec128<uint16_t, N> PromoteTo(Simd<uint16_t, N> /* tag */, + const Vec128<uint8_t, N> v) { + return Vec128<uint16_t, N>{wasm_u16x8_extend_low_u8x16(v.raw)}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> PromoteTo(Simd<uint32_t, N> /* tag */, + const Vec128<uint8_t, N> v) { + return Vec128<uint32_t, N>{ + wasm_u32x4_extend_low_u16x8(wasm_u16x8_extend_low_u8x16(v.raw))}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> PromoteTo(Simd<int16_t, N> /* tag */, + const Vec128<uint8_t, N> v) { + return Vec128<int16_t, N>{wasm_u16x8_extend_low_u8x16(v.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> PromoteTo(Simd<int32_t, N> /* tag */, + const Vec128<uint8_t, N> v) { + return Vec128<int32_t, N>{ + wasm_u32x4_extend_low_u16x8(wasm_u16x8_extend_low_u8x16(v.raw))}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> PromoteTo(Simd<uint32_t, N> /* tag */, + const Vec128<uint16_t, N> v) { + return Vec128<uint32_t, N>{wasm_u32x4_extend_low_u16x8(v.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> PromoteTo(Simd<int32_t, N> /* tag */, + const Vec128<uint16_t, N> v) { + return Vec128<int32_t, N>{wasm_u32x4_extend_low_u16x8(v.raw)}; +} + +// Signed: replicate sign bit. +template <size_t N> +HWY_API Vec128<int16_t, N> PromoteTo(Simd<int16_t, N> /* tag */, + const Vec128<int8_t, N> v) { + return Vec128<int16_t, N>{wasm_i16x8_extend_low_i8x16(v.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> PromoteTo(Simd<int32_t, N> /* tag */, + const Vec128<int8_t, N> v) { + return Vec128<int32_t, N>{ + wasm_i32x4_extend_low_i16x8(wasm_i16x8_extend_low_i8x16(v.raw))}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> PromoteTo(Simd<int32_t, N> /* tag */, + const Vec128<int16_t, N> v) { + return Vec128<int32_t, N>{wasm_i32x4_extend_low_i16x8(v.raw)}; +} + +template <size_t N> +HWY_API Vec128<double, N> PromoteTo(Simd<double, N> /* tag */, + const Vec128<int32_t, N> v) { + return Vec128<double, N>{wasm_f64x2_convert_low_i32x4(v.raw)}; +} + +template <size_t N> +HWY_API Vec128<float, N> PromoteTo(Simd<float, N> /* tag */, + const Vec128<float16_t, N> v) { + const Simd<int32_t, N> di32; + const Simd<uint32_t, N> du32; + const Simd<float, N> df32; + // Expand to u32 so we can shift. + const auto bits16 = PromoteTo(du32, Vec128<uint16_t, N>{v.raw}); + const auto sign = ShiftRight<15>(bits16); + const auto biased_exp = ShiftRight<10>(bits16) & Set(du32, 0x1F); + const auto mantissa = bits16 & Set(du32, 0x3FF); + const auto subnormal = + BitCast(du32, ConvertTo(df32, BitCast(di32, mantissa)) * + Set(df32, 1.0f / 16384 / 1024)); + + const auto biased_exp32 = biased_exp + Set(du32, 127 - 15); + const auto mantissa32 = ShiftLeft<23 - 10>(mantissa); + const auto normal = ShiftLeft<23>(biased_exp32) | mantissa32; + const auto bits32 = IfThenElse(biased_exp == Zero(du32), subnormal, normal); + return BitCast(df32, ShiftLeft<31>(sign) | bits32); +} + +template <size_t N> +HWY_API Vec128<float, N> PromoteTo(Simd<float, N> df32, + const Vec128<bfloat16_t, N> v) { + const Rebind<uint16_t, decltype(df32)> du16; + const RebindToSigned<decltype(df32)> di32; + return BitCast(df32, ShiftLeft<16>(PromoteTo(di32, BitCast(du16, v)))); +} + +// ------------------------------ Demotions (full -> part w/ narrow lanes) + +template <size_t N> +HWY_API Vec128<uint16_t, N> DemoteTo(Simd<uint16_t, N> /* tag */, + const Vec128<int32_t, N> v) { + return Vec128<uint16_t, N>{wasm_u16x8_narrow_i32x4(v.raw, v.raw)}; +} + +template <size_t N> +HWY_API Vec128<int16_t, N> DemoteTo(Simd<int16_t, N> /* tag */, + const Vec128<int32_t, N> v) { + return Vec128<int16_t, N>{wasm_i16x8_narrow_i32x4(v.raw, v.raw)}; +} + +template <size_t N> +HWY_API Vec128<uint8_t, N> DemoteTo(Simd<uint8_t, N> /* tag */, + const Vec128<int32_t, N> v) { + const auto intermediate = wasm_i16x8_narrow_i32x4(v.raw, v.raw); + return Vec128<uint8_t, N>{ + wasm_u8x16_narrow_i16x8(intermediate, intermediate)}; +} + +template <size_t N> +HWY_API Vec128<uint8_t, N> DemoteTo(Simd<uint8_t, N> /* tag */, + const Vec128<int16_t, N> v) { + return Vec128<uint8_t, N>{wasm_u8x16_narrow_i16x8(v.raw, v.raw)}; +} + +template <size_t N> +HWY_API Vec128<int8_t, N> DemoteTo(Simd<int8_t, N> /* tag */, + const Vec128<int32_t, N> v) { + const auto intermediate = wasm_i16x8_narrow_i32x4(v.raw, v.raw); + return Vec128<int8_t, N>{wasm_i8x16_narrow_i16x8(intermediate, intermediate)}; +} + +template <size_t N> +HWY_API Vec128<int8_t, N> DemoteTo(Simd<int8_t, N> /* tag */, + const Vec128<int16_t, N> v) { + return Vec128<int8_t, N>{wasm_i8x16_narrow_i16x8(v.raw, v.raw)}; +} + +template <size_t N> +HWY_API Vec128<int32_t, N> DemoteTo(Simd<int32_t, N> /* di */, + const Vec128<double, N> v) { + return Vec128<int32_t, N>{wasm_i32x4_trunc_sat_f64x2_zero(v.raw)}; +} + +template <size_t N> +HWY_API Vec128<float16_t, N> DemoteTo(Simd<float16_t, N> /* tag */, + const Vec128<float, N> v) { + const Simd<int32_t, N> di; + const Simd<uint32_t, N> du; + const Simd<uint16_t, N> du16; + const auto bits32 = BitCast(du, v); + const auto sign = ShiftRight<31>(bits32); + const auto biased_exp32 = ShiftRight<23>(bits32) & Set(du, 0xFF); + const auto mantissa32 = bits32 & Set(du, 0x7FFFFF); + + const auto k15 = Set(di, 15); + const auto exp = Min(BitCast(di, biased_exp32) - Set(di, 127), k15); + const auto is_tiny = exp < Set(di, -24); + + const auto is_subnormal = exp < Set(di, -14); + const auto biased_exp16 = + BitCast(du, IfThenZeroElse(is_subnormal, exp + k15)); + const auto sub_exp = BitCast(du, Set(di, -14) - exp); // [1, 11) + const auto sub_m = (Set(du, 1) << (Set(du, 10) - sub_exp)) + + (mantissa32 >> (Set(du, 13) + sub_exp)); + const auto mantissa16 = IfThenElse(RebindMask(du, is_subnormal), sub_m, + ShiftRight<13>(mantissa32)); // <1024 + + const auto sign16 = ShiftLeft<15>(sign); + const auto normal16 = sign16 | ShiftLeft<10>(biased_exp16) | mantissa16; + const auto bits16 = IfThenZeroElse(is_tiny, BitCast(di, normal16)); + return Vec128<float16_t, N>{DemoteTo(du16, bits16).raw}; +} + +template <size_t N> +HWY_API Vec128<bfloat16_t, N> DemoteTo(Simd<bfloat16_t, N> dbf16, + const Vec128<float, N> v) { + const Rebind<int32_t, decltype(dbf16)> di32; + const Rebind<uint32_t, decltype(dbf16)> du32; // for logical shift right + const Rebind<uint16_t, decltype(dbf16)> du16; + const auto bits_in_32 = BitCast(di32, ShiftRight<16>(BitCast(du32, v))); + return BitCast(dbf16, DemoteTo(du16, bits_in_32)); +} + +template <size_t N> +HWY_API Vec128<bfloat16_t, 2 * N> ReorderDemote2To( + Simd<bfloat16_t, 2 * N> dbf16, Vec128<float, N> a, Vec128<float, N> b) { + const RebindToUnsigned<decltype(dbf16)> du16; + const Repartition<uint32_t, decltype(dbf16)> du32; + const Vec128<uint32_t, N> b_in_even = ShiftRight<16>(BitCast(du32, b)); + return BitCast(dbf16, OddEven(BitCast(du16, a), BitCast(du16, b_in_even))); +} + +// For already range-limited input [0, 255]. +template <size_t N> +HWY_API Vec128<uint8_t, N> U8FromU32(const Vec128<uint32_t, N> v) { + const auto intermediate = wasm_i16x8_narrow_i32x4(v.raw, v.raw); + return Vec128<uint8_t, N>{ + wasm_u8x16_narrow_i16x8(intermediate, intermediate)}; +} + +// ------------------------------ Convert i32 <=> f32 (Round) + +template <size_t N> +HWY_API Vec128<float, N> ConvertTo(Simd<float, N> /* tag */, + const Vec128<int32_t, N> v) { + return Vec128<float, N>{wasm_f32x4_convert_i32x4(v.raw)}; +} +// Truncates (rounds toward zero). +template <size_t N> +HWY_API Vec128<int32_t, N> ConvertTo(Simd<int32_t, N> /* tag */, + const Vec128<float, N> v) { + return Vec128<int32_t, N>{wasm_i32x4_trunc_sat_f32x4(v.raw)}; +} + +template <size_t N> +HWY_API Vec128<int32_t, N> NearestInt(const Vec128<float, N> v) { + return ConvertTo(Simd<int32_t, N>(), Round(v)); +} + +// ================================================== MISC + +// ------------------------------ LoadMaskBits (TestBit) + +namespace detail { + +template <typename T, HWY_IF_LANE_SIZE(T, 1)> +HWY_INLINE Mask128<T, N> LoadMaskBits(Simd<T, N> d, uint64_t bits) { + const RebindToUnsigned<decltype(d)> du; + // Easier than Set(), which would require an >8-bit type, which would not + // compile for T=uint8_t, N=1. + const Vec256<T> vbits{wasm_i32x4_splat(static_cast<int32_t>(bits))}; + + // Replicate bytes 8x such that each byte contains the bit that governs it. + alignas(16) constexpr uint8_t kRep8[16] = {0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1}; + const auto rep8 = TableLookupBytes(vbits, Load(du, kRep8)); + + alignas(16) constexpr uint8_t kBit[16] = {1, 2, 4, 8, 16, 32, 64, 128, + 1, 2, 4, 8, 16, 32, 64, 128}; + return RebindMask(d, TestBit(rep8, LoadDup128(du, kBit))); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_INLINE Mask128<T, N> LoadMaskBits(Simd<T, N> d, uint64_t bits) { + const RebindToUnsigned<decltype(d)> du; + alignas(16) constexpr uint16_t kBit[8] = {1, 2, 4, 8, 16, 32, 64, 128}; + return RebindMask(d, TestBit(Set(du, bits), Load(du, kBit))); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_INLINE Mask128<T, N> LoadMaskBits(Simd<T, N> d, uint64_t bits) { + const RebindToUnsigned<decltype(d)> du; + alignas(16) constexpr uint32_t kBit[8] = {1, 2, 4, 8}; + return RebindMask(d, TestBit(Set(du, bits), Load(du, kBit))); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_INLINE Mask128<T, N> LoadMaskBits(Simd<T, N> d, uint64_t bits) { + const RebindToUnsigned<decltype(d)> du; + alignas(16) constexpr uint64_t kBit[8] = {1, 2}; + return RebindMask(d, TestBit(Set(du, bits), Load(du, kBit))); +} + +} // namespace detail + +// `p` points to at least 8 readable bytes, not all of which need be valid. +template <typename T, HWY_IF_GT128(T, N)> +HWY_API Mask128<T, N> LoadMaskBits(Simd<T, N> d, + const uint8_t* HWY_RESTRICT bits) { + uint64_t mask_bits = 0; + CopyBytes<(N + 7) / 8>(bits, &mask_bits); + return detail::LoadMaskBits(d, mask_bits); +} + +// ------------------------------ Mask + +namespace detail { + +// Full +template <typename T> +HWY_INLINE uint64_t BitsFromMask(hwy::SizeTag<1> /*tag*/, + const Mask128<T> mask) { + alignas(16) uint64_t lanes[2]; + wasm_v128_store(lanes, mask.raw); + + constexpr uint64_t kMagic = 0x103070F1F3F80ULL; + const uint64_t lo = ((lanes[0] * kMagic) >> 56); + const uint64_t hi = ((lanes[1] * kMagic) >> 48) & 0xFF00; + return (hi + lo); +} + +// 64-bit +template <typename T> +HWY_INLINE uint64_t BitsFromMask(hwy::SizeTag<1> /*tag*/, + const Mask128<T, 8> mask) { + constexpr uint64_t kMagic = 0x103070F1F3F80ULL; + return (wasm_i64x2_extract_lane(mask.raw, 0) * kMagic) >> 56; +} + +// 32-bit or less: need masking +template <typename T, HWY_IF_LE32(T, N)> +HWY_INLINE uint64_t BitsFromMask(hwy::SizeTag<1> /*tag*/, + const Mask128<T, N> mask) { + uint64_t bytes = wasm_i64x2_extract_lane(mask.raw, 0); + // Clear potentially undefined bytes. + bytes &= (1ULL << (N * 8)) - 1; + constexpr uint64_t kMagic = 0x103070F1F3F80ULL; + return (bytes * kMagic) >> 56; +} + +template <typename T> +HWY_INLINE uint64_t BitsFromMask(hwy::SizeTag<2> /*tag*/, + const Mask128<T, N> mask) { + // Remove useless lower half of each u16 while preserving the sign bit. + const __i16x8 zero = wasm_i16x8_splat(0); + const Mask128<uint8_t, N> mask8{wasm_i8x16_narrow_i16x8(mask.raw, zero)}; + return BitsFromMask(hwy::SizeTag<1>(), mask8); +} + +template <typename T> +HWY_INLINE uint64_t BitsFromMask(hwy::SizeTag<4> /*tag*/, + const Mask128<T, N> mask) { + const __i32x4 mask_i = static_cast<__i32x4>(mask.raw); + const __i32x4 slice = wasm_i32x4_make(1, 2, 4, 8); + const __i32x4 sliced_mask = wasm_v128_and(mask_i, slice); + alignas(16) uint32_t lanes[4]; + wasm_v128_store(lanes, sliced_mask); + return lanes[0] | lanes[1] | lanes[2] | lanes[3]; +} + +// Returns the lowest N bits for the BitsFromMask result. +template <typename T> +constexpr uint64_t OnlyActive(uint64_t bits) { + return ((N * sizeof(T)) == 16) ? bits : bits & ((1ull << N) - 1); +} + +// Returns 0xFF for bytes with index >= N, otherwise 0. +template <size_t N> +constexpr __i8x16 BytesAbove() { + return /**/ + (N == 0) ? wasm_i32x4_make(-1, -1, -1, -1) + : (N == 4) ? wasm_i32x4_make(0, -1, -1, -1) + : (N == 8) ? wasm_i32x4_make(0, 0, -1, -1) + : (N == 12) ? wasm_i32x4_make(0, 0, 0, -1) + : (N == 16) ? wasm_i32x4_make(0, 0, 0, 0) + : (N == 2) ? wasm_i16x8_make(0, -1, -1, -1, -1, -1, -1, -1) + : (N == 6) ? wasm_i16x8_make(0, 0, 0, -1, -1, -1, -1, -1) + : (N == 10) ? wasm_i16x8_make(0, 0, 0, 0, 0, -1, -1, -1) + : (N == 14) ? wasm_i16x8_make(0, 0, 0, 0, 0, 0, 0, -1) + : (N == 1) ? wasm_i8x16_make(0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1) + : (N == 3) ? wasm_i8x16_make(0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1) + : (N == 5) ? wasm_i8x16_make(0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1) + : (N == 7) ? wasm_i8x16_make(0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, + -1, -1, -1) + : (N == 9) ? wasm_i8x16_make(0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, + -1, -1, -1) + : (N == 11) + ? wasm_i8x16_make(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1, -1, -1) + : (N == 13) + ? wasm_i8x16_make(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1, -1, -1) + : wasm_i8x16_make(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -1); +} + +template <typename T> +HWY_INLINE uint64_t BitsFromMask(const Mask128<T, N> mask) { + return OnlyActive<T, N>(BitsFromMask(hwy::SizeTag<sizeof(T)>(), mask)); +} + +template <typename T> +HWY_INLINE size_t CountTrue(hwy::SizeTag<1> tag, const Mask128<T> m) { + return PopCount(BitsFromMask(tag, m)); +} + +template <typename T> +HWY_INLINE size_t CountTrue(hwy::SizeTag<2> tag, const Mask128<T> m) { + return PopCount(BitsFromMask(tag, m)); +} + +template <typename T> +HWY_INLINE size_t CountTrue(hwy::SizeTag<4> /*tag*/, const Mask128<T> m) { + const __i32x4 var_shift = wasm_i32x4_make(1, 2, 4, 8); + const __i32x4 shifted_bits = wasm_v128_and(m.raw, var_shift); + alignas(16) uint64_t lanes[2]; + wasm_v128_store(lanes, shifted_bits); + return PopCount(lanes[0] | lanes[1]); +} + +} // namespace detail + +// `p` points to at least 8 writable bytes. +template <typename T> +HWY_API size_t StoreMaskBits(const Simd<T, N> /* tag */, + const Mask128<T, N> mask, uint8_t* bits) { + const uint64_t mask_bits = detail::BitsFromMask(mask); + const size_t kNumBytes = (N + 7) / 8; + CopyBytes<kNumBytes>(&mask_bits, bits); + return kNumBytes; +} + +template <typename T> +HWY_API size_t CountTrue(const Simd<T, N> /* tag */, const Mask128<T> m) { + return detail::CountTrue(hwy::SizeTag<sizeof(T)>(), m); +} + +// Partial vector +template <typename T, HWY_IF_LE64(T, N)> +HWY_API size_t CountTrue(const Simd<T, N> d, const Mask128<T, N> m) { + // Ensure all undefined bytes are 0. + const Mask128<T, N> mask{detail::BytesAbove<N * sizeof(T)>()}; + return CountTrue(d, Mask128<T>{AndNot(mask, m).raw}); +} + +// Full vector +template <typename T> +HWY_API bool AllFalse(const Full256<T> d, const Mask128<T> m) { +#if 0 + // Casting followed by wasm_i8x16_any_true results in wasm error: + // i32.eqz[0] expected type i32, found i8x16.popcnt of type s128 + const auto v8 = BitCast(Full256<int8_t>(), VecFromMask(d, m)); + return !wasm_i8x16_any_true(v8.raw); +#else + (void)d; + return (wasm_i64x2_extract_lane(m.raw, 0) | + wasm_i64x2_extract_lane(m.raw, 1)) == 0; +#endif +} + +// Full vector +namespace detail { +template <typename T> +HWY_INLINE bool AllTrue(hwy::SizeTag<1> /*tag*/, const Mask128<T> m) { + return wasm_i8x16_all_true(m.raw); +} +template <typename T> +HWY_INLINE bool AllTrue(hwy::SizeTag<2> /*tag*/, const Mask128<T> m) { + return wasm_i16x8_all_true(m.raw); +} +template <typename T> +HWY_INLINE bool AllTrue(hwy::SizeTag<4> /*tag*/, const Mask128<T> m) { + return wasm_i32x4_all_true(m.raw); +} + +} // namespace detail + +template <typename T> +HWY_API bool AllTrue(const Simd<T, N> /* tag */, const Mask128<T> m) { + return detail::AllTrue(hwy::SizeTag<sizeof(T)>(), m); +} + +// Partial vectors + +template <typename T, HWY_IF_LE64(T, N)> +HWY_API bool AllFalse(Simd<T, N> /* tag */, const Mask128<T, N> m) { + // Ensure all undefined bytes are 0. + const Mask128<T, N> mask{detail::BytesAbove<N * sizeof(T)>()}; + return AllFalse(Mask128<T>{AndNot(mask, m).raw}); +} + +template <typename T, HWY_IF_LE64(T, N)> +HWY_API bool AllTrue(const Simd<T, N> d, const Mask128<T, N> m) { + // Ensure all undefined bytes are FF. + const Mask128<T, N> mask{detail::BytesAbove<N * sizeof(T)>()}; + return AllTrue(d, Mask128<T>{Or(mask, m).raw}); +} + +template <typename T> +HWY_API intptr_t FindFirstTrue(const Simd<T, N> /* tag */, + const Mask128<T, N> mask) { + const uint64_t bits = detail::BitsFromMask(mask); + return bits ? Num0BitsBelowLS1Bit_Nonzero64(bits) : -1; +} + +// ------------------------------ Compress + +namespace detail { + +template <typename T> +HWY_INLINE Vec256<T> Idx16x8FromBits(const uint64_t mask_bits) { + HWY_DASSERT(mask_bits < 256); + const Simd<T, N> d; + const Rebind<uint8_t, decltype(d)> d8; + const Simd<uint16_t, N> du; + + // We need byte indices for TableLookupBytes (one vector's worth for each of + // 256 combinations of 8 mask bits). Loading them directly requires 4 KiB. We + // can instead store lane indices and convert to byte indices (2*lane + 0..1), + // with the doubling baked into the table. Unpacking nibbles is likely more + // costly than the higher cache footprint from storing bytes. + alignas(16) constexpr uint8_t table[256 * 8] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, + 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, + 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, + 0, 6, 0, 0, 0, 0, 0, 0, 2, 6, 0, 0, 0, 0, 0, 0, 0, 2, + 6, 0, 0, 0, 0, 0, 4, 6, 0, 0, 0, 0, 0, 0, 0, 4, 6, 0, + 0, 0, 0, 0, 2, 4, 6, 0, 0, 0, 0, 0, 0, 2, 4, 6, 0, 0, + 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, + 2, 8, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 0, 0, 4, 8, + 0, 0, 0, 0, 0, 0, 0, 4, 8, 0, 0, 0, 0, 0, 2, 4, 8, 0, + 0, 0, 0, 0, 0, 2, 4, 8, 0, 0, 0, 0, 6, 8, 0, 0, 0, 0, + 0, 0, 0, 6, 8, 0, 0, 0, 0, 0, 2, 6, 8, 0, 0, 0, 0, 0, + 0, 2, 6, 8, 0, 0, 0, 0, 4, 6, 8, 0, 0, 0, 0, 0, 0, 4, + 6, 8, 0, 0, 0, 0, 2, 4, 6, 8, 0, 0, 0, 0, 0, 2, 4, 6, + 8, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, + 0, 0, 2, 10, 0, 0, 0, 0, 0, 0, 0, 2, 10, 0, 0, 0, 0, 0, + 4, 10, 0, 0, 0, 0, 0, 0, 0, 4, 10, 0, 0, 0, 0, 0, 2, 4, + 10, 0, 0, 0, 0, 0, 0, 2, 4, 10, 0, 0, 0, 0, 6, 10, 0, 0, + 0, 0, 0, 0, 0, 6, 10, 0, 0, 0, 0, 0, 2, 6, 10, 0, 0, 0, + 0, 0, 0, 2, 6, 10, 0, 0, 0, 0, 4, 6, 10, 0, 0, 0, 0, 0, + 0, 4, 6, 10, 0, 0, 0, 0, 2, 4, 6, 10, 0, 0, 0, 0, 0, 2, + 4, 6, 10, 0, 0, 0, 8, 10, 0, 0, 0, 0, 0, 0, 0, 8, 10, 0, + 0, 0, 0, 0, 2, 8, 10, 0, 0, 0, 0, 0, 0, 2, 8, 10, 0, 0, + 0, 0, 4, 8, 10, 0, 0, 0, 0, 0, 0, 4, 8, 10, 0, 0, 0, 0, + 2, 4, 8, 10, 0, 0, 0, 0, 0, 2, 4, 8, 10, 0, 0, 0, 6, 8, + 10, 0, 0, 0, 0, 0, 0, 6, 8, 10, 0, 0, 0, 0, 2, 6, 8, 10, + 0, 0, 0, 0, 0, 2, 6, 8, 10, 0, 0, 0, 4, 6, 8, 10, 0, 0, + 0, 0, 0, 4, 6, 8, 10, 0, 0, 0, 2, 4, 6, 8, 10, 0, 0, 0, + 0, 2, 4, 6, 8, 10, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 12, + 0, 0, 0, 0, 0, 0, 2, 12, 0, 0, 0, 0, 0, 0, 0, 2, 12, 0, + 0, 0, 0, 0, 4, 12, 0, 0, 0, 0, 0, 0, 0, 4, 12, 0, 0, 0, + 0, 0, 2, 4, 12, 0, 0, 0, 0, 0, 0, 2, 4, 12, 0, 0, 0, 0, + 6, 12, 0, 0, 0, 0, 0, 0, 0, 6, 12, 0, 0, 0, 0, 0, 2, 6, + 12, 0, 0, 0, 0, 0, 0, 2, 6, 12, 0, 0, 0, 0, 4, 6, 12, 0, + 0, 0, 0, 0, 0, 4, 6, 12, 0, 0, 0, 0, 2, 4, 6, 12, 0, 0, + 0, 0, 0, 2, 4, 6, 12, 0, 0, 0, 8, 12, 0, 0, 0, 0, 0, 0, + 0, 8, 12, 0, 0, 0, 0, 0, 2, 8, 12, 0, 0, 0, 0, 0, 0, 2, + 8, 12, 0, 0, 0, 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 4, 8, 12, + 0, 0, 0, 0, 2, 4, 8, 12, 0, 0, 0, 0, 0, 2, 4, 8, 12, 0, + 0, 0, 6, 8, 12, 0, 0, 0, 0, 0, 0, 6, 8, 12, 0, 0, 0, 0, + 2, 6, 8, 12, 0, 0, 0, 0, 0, 2, 6, 8, 12, 0, 0, 0, 4, 6, + 8, 12, 0, 0, 0, 0, 0, 4, 6, 8, 12, 0, 0, 0, 2, 4, 6, 8, + 12, 0, 0, 0, 0, 2, 4, 6, 8, 12, 0, 0, 10, 12, 0, 0, 0, 0, + 0, 0, 0, 10, 12, 0, 0, 0, 0, 0, 2, 10, 12, 0, 0, 0, 0, 0, + 0, 2, 10, 12, 0, 0, 0, 0, 4, 10, 12, 0, 0, 0, 0, 0, 0, 4, + 10, 12, 0, 0, 0, 0, 2, 4, 10, 12, 0, 0, 0, 0, 0, 2, 4, 10, + 12, 0, 0, 0, 6, 10, 12, 0, 0, 0, 0, 0, 0, 6, 10, 12, 0, 0, + 0, 0, 2, 6, 10, 12, 0, 0, 0, 0, 0, 2, 6, 10, 12, 0, 0, 0, + 4, 6, 10, 12, 0, 0, 0, 0, 0, 4, 6, 10, 12, 0, 0, 0, 2, 4, + 6, 10, 12, 0, 0, 0, 0, 2, 4, 6, 10, 12, 0, 0, 8, 10, 12, 0, + 0, 0, 0, 0, 0, 8, 10, 12, 0, 0, 0, 0, 2, 8, 10, 12, 0, 0, + 0, 0, 0, 2, 8, 10, 12, 0, 0, 0, 4, 8, 10, 12, 0, 0, 0, 0, + 0, 4, 8, 10, 12, 0, 0, 0, 2, 4, 8, 10, 12, 0, 0, 0, 0, 2, + 4, 8, 10, 12, 0, 0, 6, 8, 10, 12, 0, 0, 0, 0, 0, 6, 8, 10, + 12, 0, 0, 0, 2, 6, 8, 10, 12, 0, 0, 0, 0, 2, 6, 8, 10, 12, + 0, 0, 4, 6, 8, 10, 12, 0, 0, 0, 0, 4, 6, 8, 10, 12, 0, 0, + 2, 4, 6, 8, 10, 12, 0, 0, 0, 2, 4, 6, 8, 10, 12, 0, 14, 0, + 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 2, 14, 0, 0, + 0, 0, 0, 0, 0, 2, 14, 0, 0, 0, 0, 0, 4, 14, 0, 0, 0, 0, + 0, 0, 0, 4, 14, 0, 0, 0, 0, 0, 2, 4, 14, 0, 0, 0, 0, 0, + 0, 2, 4, 14, 0, 0, 0, 0, 6, 14, 0, 0, 0, 0, 0, 0, 0, 6, + 14, 0, 0, 0, 0, 0, 2, 6, 14, 0, 0, 0, 0, 0, 0, 2, 6, 14, + 0, 0, 0, 0, 4, 6, 14, 0, 0, 0, 0, 0, 0, 4, 6, 14, 0, 0, + 0, 0, 2, 4, 6, 14, 0, 0, 0, 0, 0, 2, 4, 6, 14, 0, 0, 0, + 8, 14, 0, 0, 0, 0, 0, 0, 0, 8, 14, 0, 0, 0, 0, 0, 2, 8, + 14, 0, 0, 0, 0, 0, 0, 2, 8, 14, 0, 0, 0, 0, 4, 8, 14, 0, + 0, 0, 0, 0, 0, 4, 8, 14, 0, 0, 0, 0, 2, 4, 8, 14, 0, 0, + 0, 0, 0, 2, 4, 8, 14, 0, 0, 0, 6, 8, 14, 0, 0, 0, 0, 0, + 0, 6, 8, 14, 0, 0, 0, 0, 2, 6, 8, 14, 0, 0, 0, 0, 0, 2, + 6, 8, 14, 0, 0, 0, 4, 6, 8, 14, 0, 0, 0, 0, 0, 4, 6, 8, + 14, 0, 0, 0, 2, 4, 6, 8, 14, 0, 0, 0, 0, 2, 4, 6, 8, 14, + 0, 0, 10, 14, 0, 0, 0, 0, 0, 0, 0, 10, 14, 0, 0, 0, 0, 0, + 2, 10, 14, 0, 0, 0, 0, 0, 0, 2, 10, 14, 0, 0, 0, 0, 4, 10, + 14, 0, 0, 0, 0, 0, 0, 4, 10, 14, 0, 0, 0, 0, 2, 4, 10, 14, + 0, 0, 0, 0, 0, 2, 4, 10, 14, 0, 0, 0, 6, 10, 14, 0, 0, 0, + 0, 0, 0, 6, 10, 14, 0, 0, 0, 0, 2, 6, 10, 14, 0, 0, 0, 0, + 0, 2, 6, 10, 14, 0, 0, 0, 4, 6, 10, 14, 0, 0, 0, 0, 0, 4, + 6, 10, 14, 0, 0, 0, 2, 4, 6, 10, 14, 0, 0, 0, 0, 2, 4, 6, + 10, 14, 0, 0, 8, 10, 14, 0, 0, 0, 0, 0, 0, 8, 10, 14, 0, 0, + 0, 0, 2, 8, 10, 14, 0, 0, 0, 0, 0, 2, 8, 10, 14, 0, 0, 0, + 4, 8, 10, 14, 0, 0, 0, 0, 0, 4, 8, 10, 14, 0, 0, 0, 2, 4, + 8, 10, 14, 0, 0, 0, 0, 2, 4, 8, 10, 14, 0, 0, 6, 8, 10, 14, + 0, 0, 0, 0, 0, 6, 8, 10, 14, 0, 0, 0, 2, 6, 8, 10, 14, 0, + 0, 0, 0, 2, 6, 8, 10, 14, 0, 0, 4, 6, 8, 10, 14, 0, 0, 0, + 0, 4, 6, 8, 10, 14, 0, 0, 2, 4, 6, 8, 10, 14, 0, 0, 0, 2, + 4, 6, 8, 10, 14, 0, 12, 14, 0, 0, 0, 0, 0, 0, 0, 12, 14, 0, + 0, 0, 0, 0, 2, 12, 14, 0, 0, 0, 0, 0, 0, 2, 12, 14, 0, 0, + 0, 0, 4, 12, 14, 0, 0, 0, 0, 0, 0, 4, 12, 14, 0, 0, 0, 0, + 2, 4, 12, 14, 0, 0, 0, 0, 0, 2, 4, 12, 14, 0, 0, 0, 6, 12, + 14, 0, 0, 0, 0, 0, 0, 6, 12, 14, 0, 0, 0, 0, 2, 6, 12, 14, + 0, 0, 0, 0, 0, 2, 6, 12, 14, 0, 0, 0, 4, 6, 12, 14, 0, 0, + 0, 0, 0, 4, 6, 12, 14, 0, 0, 0, 2, 4, 6, 12, 14, 0, 0, 0, + 0, 2, 4, 6, 12, 14, 0, 0, 8, 12, 14, 0, 0, 0, 0, 0, 0, 8, + 12, 14, 0, 0, 0, 0, 2, 8, 12, 14, 0, 0, 0, 0, 0, 2, 8, 12, + 14, 0, 0, 0, 4, 8, 12, 14, 0, 0, 0, 0, 0, 4, 8, 12, 14, 0, + 0, 0, 2, 4, 8, 12, 14, 0, 0, 0, 0, 2, 4, 8, 12, 14, 0, 0, + 6, 8, 12, 14, 0, 0, 0, 0, 0, 6, 8, 12, 14, 0, 0, 0, 2, 6, + 8, 12, 14, 0, 0, 0, 0, 2, 6, 8, 12, 14, 0, 0, 4, 6, 8, 12, + 14, 0, 0, 0, 0, 4, 6, 8, 12, 14, 0, 0, 2, 4, 6, 8, 12, 14, + 0, 0, 0, 2, 4, 6, 8, 12, 14, 0, 10, 12, 14, 0, 0, 0, 0, 0, + 0, 10, 12, 14, 0, 0, 0, 0, 2, 10, 12, 14, 0, 0, 0, 0, 0, 2, + 10, 12, 14, 0, 0, 0, 4, 10, 12, 14, 0, 0, 0, 0, 0, 4, 10, 12, + 14, 0, 0, 0, 2, 4, 10, 12, 14, 0, 0, 0, 0, 2, 4, 10, 12, 14, + 0, 0, 6, 10, 12, 14, 0, 0, 0, 0, 0, 6, 10, 12, 14, 0, 0, 0, + 2, 6, 10, 12, 14, 0, 0, 0, 0, 2, 6, 10, 12, 14, 0, 0, 4, 6, + 10, 12, 14, 0, 0, 0, 0, 4, 6, 10, 12, 14, 0, 0, 2, 4, 6, 10, + 12, 14, 0, 0, 0, 2, 4, 6, 10, 12, 14, 0, 8, 10, 12, 14, 0, 0, + 0, 0, 0, 8, 10, 12, 14, 0, 0, 0, 2, 8, 10, 12, 14, 0, 0, 0, + 0, 2, 8, 10, 12, 14, 0, 0, 4, 8, 10, 12, 14, 0, 0, 0, 0, 4, + 8, 10, 12, 14, 0, 0, 2, 4, 8, 10, 12, 14, 0, 0, 0, 2, 4, 8, + 10, 12, 14, 0, 6, 8, 10, 12, 14, 0, 0, 0, 0, 6, 8, 10, 12, 14, + 0, 0, 2, 6, 8, 10, 12, 14, 0, 0, 0, 2, 6, 8, 10, 12, 14, 0, + 4, 6, 8, 10, 12, 14, 0, 0, 0, 4, 6, 8, 10, 12, 14, 0, 2, 4, + 6, 8, 10, 12, 14, 0, 0, 2, 4, 6, 8, 10, 12, 14}; + + const Vec128<uint8_t, 2 * N> byte_idx{Load(d8, table + mask_bits * 8).raw}; + const Vec128<uint16_t, N> pairs = ZipLower(byte_idx, byte_idx); + return BitCast(d, pairs + Set(du, 0x0100)); +} + +template <typename T> +HWY_INLINE Vec256<T> Idx32x4FromBits(const uint64_t mask_bits) { + HWY_DASSERT(mask_bits < 16); + + // There are only 4 lanes, so we can afford to load the index vector directly. + alignas(16) constexpr uint8_t packed_array[16 * 16] = { + 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, // + 4, 5, 6, 7, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 0, 1, 2, 3, // + 8, 9, 10, 11, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 8, 9, 10, 11, 0, 1, 2, 3, 0, 1, 2, 3, // + 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, // + 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3, // + 4, 5, 6, 7, 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 15, 0, 1, 2, 3, // + 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, // + 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, // + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + + const Simd<T, N> d; + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, Load(d8, packed_array + 16 * mask_bits)); +} + +#if HWY_CAP_INTEGER64 || HWY_CAP_FLOAT64 + +template <typename T> +HWY_INLINE Vec256<T> Idx64x2FromBits(const uint64_t mask_bits) { + HWY_DASSERT(mask_bits < 4); + + // There are only 2 lanes, so we can afford to load the index vector directly. + alignas(16) constexpr uint8_t packed_array[4 * 16] = { + 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, // + 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, // + 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, // + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + + const Simd<T, N> d; + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, Load(d8, packed_array + 16 * mask_bits)); +} + +#endif + +// Helper functions called by both Compress and CompressStore - avoids a +// redundant BitsFromMask in the latter. + +template <typename T> +HWY_INLINE Vec256<T> Compress(hwy::SizeTag<2> /*tag*/, Vec256<T> v, + const uint64_t mask_bits) { + const auto idx = detail::Idx16x8FromBits<T, N>(mask_bits); + using D = Simd<T, N>; + const RebindToSigned<D> di; + return BitCast(D(), TableLookupBytes(BitCast(di, v), BitCast(di, idx))); +} + +template <typename T> +HWY_INLINE Vec256<T> Compress(hwy::SizeTag<4> /*tag*/, Vec256<T> v, + const uint64_t mask_bits) { + const auto idx = detail::Idx32x4FromBits<T, N>(mask_bits); + using D = Simd<T, N>; + const RebindToSigned<D> di; + return BitCast(D(), TableLookupBytes(BitCast(di, v), BitCast(di, idx))); +} + +#if HWY_CAP_INTEGER64 || HWY_CAP_FLOAT64 + +template <typename T> +HWY_INLINE Vec128<uint64_t, N> Compress(hwy::SizeTag<8> /*tag*/, + Vec128<uint64_t, N> v, + const uint64_t mask_bits) { + const auto idx = detail::Idx64x2FromBits<uint64_t, N>(mask_bits); + using D = Simd<T, N>; + const RebindToSigned<D> di; + return BitCast(D(), TableLookupBytes(BitCast(di, v), BitCast(di, idx))); +} + +#endif + +} // namespace detail + +template <typename T> +HWY_API Vec256<T> Compress(Vec256<T> v, const Mask128<T, N> mask) { + const uint64_t mask_bits = detail::BitsFromMask(mask); + return detail::Compress(hwy::SizeTag<sizeof(T)>(), v, mask_bits); +} + +// ------------------------------ CompressBits + +template <typename T> +HWY_API Vec256<T> CompressBits(Vec256<T> v, const uint8_t* HWY_RESTRICT bits) { + uint64_t mask_bits = 0; + constexpr size_t kNumBytes = (N + 7) / 8; + CopyBytes<kNumBytes>(bits, &mask_bits); + if (N < 8) { + mask_bits &= (1ull << N) - 1; + } + + return detail::Compress(hwy::SizeTag<sizeof(T)>(), v, mask_bits); +} + +// ------------------------------ CompressStore +template <typename T> +HWY_API size_t CompressStore(Vec256<T> v, const Mask128<T, N> mask, + Simd<T, N> d, T* HWY_RESTRICT unaligned) { + const uint64_t mask_bits = detail::BitsFromMask(mask); + const auto c = detail::Compress(hwy::SizeTag<sizeof(T)>(), v, mask_bits); + StoreU(c, d, unaligned); + return PopCount(mask_bits); +} + +// ------------------------------ CompressBlendedStore +template <typename T> +HWY_API size_t CompressBlendedStore(Vec256<T> v, Mask128<T, N> m, Simd<T, N> d, + T* HWY_RESTRICT unaligned) { + const RebindToUnsigned<decltype(d)> du; // so we can support fp16/bf16 + using TU = TFromD<decltype(du)>; + const uint64_t mask_bits = detail::BitsFromMask(m); + const size_t count = PopCount(mask_bits); + const Mask128<TU, N> store_mask = FirstN(du, count); + const Vec128<TU, N> compressed = + detail::Compress(hwy::SizeTag<sizeof(T)>(), BitCast(du, v), mask_bits); + const Vec128<TU, N> prev = BitCast(du, LoadU(d, unaligned)); + StoreU(BitCast(d, IfThenElse(store_mask, compressed, prev)), d, unaligned); + return count; +} + +// ------------------------------ CompressBitsStore + +template <typename T> +HWY_API size_t CompressBitsStore(Vec256<T> v, const uint8_t* HWY_RESTRICT bits, + Simd<T, N> d, T* HWY_RESTRICT unaligned) { + uint64_t mask_bits = 0; + constexpr size_t kNumBytes = (N + 7) / 8; + CopyBytes<kNumBytes>(bits, &mask_bits); + if (N < 8) { + mask_bits &= (1ull << N) - 1; + } + + const auto c = detail::Compress(hwy::SizeTag<sizeof(T)>(), v, mask_bits); + StoreU(c, d, unaligned); + return PopCount(mask_bits); +} + +// ------------------------------ StoreInterleaved3 (CombineShiftRightBytes, +// TableLookupBytes) + +// 128 bits +HWY_API void StoreInterleaved3(const Vec128<uint8_t> a, const Vec128<uint8_t> b, + const Vec128<uint8_t> c, Full256<uint8_t> d, + uint8_t* HWY_RESTRICT unaligned) { + const auto k5 = Set(d, 5); + const auto k6 = Set(d, 6); + + // Shuffle (a,b,c) vector bytes to (MSB on left): r5, bgr[4:0]. + // 0x80 so lanes to be filled from other vectors are 0 for blending. + alignas(16) static constexpr uint8_t tbl_r0[16] = { + 0, 0x80, 0x80, 1, 0x80, 0x80, 2, 0x80, 0x80, // + 3, 0x80, 0x80, 4, 0x80, 0x80, 5}; + alignas(16) static constexpr uint8_t tbl_g0[16] = { + 0x80, 0, 0x80, 0x80, 1, 0x80, // + 0x80, 2, 0x80, 0x80, 3, 0x80, 0x80, 4, 0x80, 0x80}; + const auto shuf_r0 = Load(d, tbl_r0); + const auto shuf_g0 = Load(d, tbl_g0); // cannot reuse r0 due to 5 in MSB + const auto shuf_b0 = CombineShiftRightBytes<15>(d, shuf_g0, shuf_g0); + const auto r0 = TableLookupBytes(a, shuf_r0); // 5..4..3..2..1..0 + const auto g0 = TableLookupBytes(b, shuf_g0); // ..4..3..2..1..0. + const auto b0 = TableLookupBytes(c, shuf_b0); // .4..3..2..1..0.. + const auto int0 = r0 | g0 | b0; + StoreU(int0, d, unaligned + 0 * 16); + + // Second vector: g10,r10, bgr[9:6], b5,g5 + const auto shuf_r1 = shuf_b0 + k6; // .A..9..8..7..6.. + const auto shuf_g1 = shuf_r0 + k5; // A..9..8..7..6..5 + const auto shuf_b1 = shuf_g0 + k5; // ..9..8..7..6..5. + const auto r1 = TableLookupBytes(a, shuf_r1); + const auto g1 = TableLookupBytes(b, shuf_g1); + const auto b1 = TableLookupBytes(c, shuf_b1); + const auto int1 = r1 | g1 | b1; + StoreU(int1, d, unaligned + 1 * 16); + + // Third vector: bgr[15:11], b10 + const auto shuf_r2 = shuf_b1 + k6; // ..F..E..D..C..B. + const auto shuf_g2 = shuf_r1 + k5; // .F..E..D..C..B.. + const auto shuf_b2 = shuf_g1 + k5; // F..E..D..C..B..A + const auto r2 = TableLookupBytes(a, shuf_r2); + const auto g2 = TableLookupBytes(b, shuf_g2); + const auto b2 = TableLookupBytes(c, shuf_b2); + const auto int2 = r2 | g2 | b2; + StoreU(int2, d, unaligned + 2 * 16); +} + +// 64 bits +HWY_API void StoreInterleaved3(const Vec128<uint8_t, 8> a, + const Vec128<uint8_t, 8> b, + const Vec128<uint8_t, 8> c, Simd<uint8_t, 8> d, + uint8_t* HWY_RESTRICT unaligned) { + // Use full vectors for the shuffles and first result. + const Full256<uint8_t> d_full; + const auto k5 = Set(d_full, 5); + const auto k6 = Set(d_full, 6); + + const Vec128<uint8_t> full_a{a.raw}; + const Vec128<uint8_t> full_b{b.raw}; + const Vec128<uint8_t> full_c{c.raw}; + + // Shuffle (a,b,c) vector bytes to (MSB on left): r5, bgr[4:0]. + // 0x80 so lanes to be filled from other vectors are 0 for blending. + alignas(16) static constexpr uint8_t tbl_r0[16] = { + 0, 0x80, 0x80, 1, 0x80, 0x80, 2, 0x80, 0x80, // + 3, 0x80, 0x80, 4, 0x80, 0x80, 5}; + alignas(16) static constexpr uint8_t tbl_g0[16] = { + 0x80, 0, 0x80, 0x80, 1, 0x80, // + 0x80, 2, 0x80, 0x80, 3, 0x80, 0x80, 4, 0x80, 0x80}; + const auto shuf_r0 = Load(d_full, tbl_r0); + const auto shuf_g0 = Load(d_full, tbl_g0); // cannot reuse r0 due to 5 in MSB + const auto shuf_b0 = CombineShiftRightBytes<15>(d_full, shuf_g0, shuf_g0); + const auto r0 = TableLookupBytes(full_a, shuf_r0); // 5..4..3..2..1..0 + const auto g0 = TableLookupBytes(full_b, shuf_g0); // ..4..3..2..1..0. + const auto b0 = TableLookupBytes(full_c, shuf_b0); // .4..3..2..1..0.. + const auto int0 = r0 | g0 | b0; + StoreU(int0, d_full, unaligned + 0 * 16); + + // Second (HALF) vector: bgr[7:6], b5,g5 + const auto shuf_r1 = shuf_b0 + k6; // ..7..6.. + const auto shuf_g1 = shuf_r0 + k5; // .7..6..5 + const auto shuf_b1 = shuf_g0 + k5; // 7..6..5. + const auto r1 = TableLookupBytes(full_a, shuf_r1); + const auto g1 = TableLookupBytes(full_b, shuf_g1); + const auto b1 = TableLookupBytes(full_c, shuf_b1); + const decltype(Zero(d)) int1{(r1 | g1 | b1).raw}; + StoreU(int1, d, unaligned + 1 * 16); +} + +// <= 32 bits +template <size_t N, HWY_IF_LE32(uint8_t, N)> +HWY_API void StoreInterleaved3(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b, + const Vec128<uint8_t, N> c, + Simd<uint8_t, N> /*tag*/, + uint8_t* HWY_RESTRICT unaligned) { + // Use full vectors for the shuffles and result. + const Full256<uint8_t> d_full; + + const Vec128<uint8_t> full_a{a.raw}; + const Vec128<uint8_t> full_b{b.raw}; + const Vec128<uint8_t> full_c{c.raw}; + + // Shuffle (a,b,c) vector bytes to bgr[3:0]. + // 0x80 so lanes to be filled from other vectors are 0 for blending. + alignas(16) static constexpr uint8_t tbl_r0[16] = { + 0, 0x80, 0x80, 1, 0x80, 0x80, 2, 0x80, 0x80, 3, 0x80, 0x80, // + 0x80, 0x80, 0x80, 0x80}; + const auto shuf_r0 = Load(d_full, tbl_r0); + const auto shuf_g0 = CombineShiftRightBytes<15>(d_full, shuf_r0, shuf_r0); + const auto shuf_b0 = CombineShiftRightBytes<14>(d_full, shuf_r0, shuf_r0); + const auto r0 = TableLookupBytes(full_a, shuf_r0); // ......3..2..1..0 + const auto g0 = TableLookupBytes(full_b, shuf_g0); // .....3..2..1..0. + const auto b0 = TableLookupBytes(full_c, shuf_b0); // ....3..2..1..0.. + const auto int0 = r0 | g0 | b0; + alignas(16) uint8_t buf[16]; + StoreU(int0, d_full, buf); + CopyBytes<N * 3>(buf, unaligned); +} + +// ------------------------------ StoreInterleaved4 + +// 128 bits +HWY_API void StoreInterleaved4(const Vec128<uint8_t> v0, + const Vec128<uint8_t> v1, + const Vec128<uint8_t> v2, + const Vec128<uint8_t> v3, Full256<uint8_t> d8, + uint8_t* HWY_RESTRICT unaligned) { + const RepartitionToWide<decltype(d8)> d16; + const RepartitionToWide<decltype(d16)> d32; + // let a,b,c,d denote v0..3. + const auto ba0 = ZipLower(d16, v0, v1); // b7 a7 .. b0 a0 + const auto dc0 = ZipLower(d16, v2, v3); // d7 c7 .. d0 c0 + const auto ba8 = ZipUpper(d16, v0, v1); + const auto dc8 = ZipUpper(d16, v2, v3); + const auto dcba_0 = ZipLower(d32, ba0, dc0); // d..a3 d..a0 + const auto dcba_4 = ZipUpper(d32, ba0, dc0); // d..a7 d..a4 + const auto dcba_8 = ZipLower(d32, ba8, dc8); // d..aB d..a8 + const auto dcba_C = ZipUpper(d32, ba8, dc8); // d..aF d..aC + StoreU(BitCast(d8, dcba_0), d8, unaligned + 0 * 16); + StoreU(BitCast(d8, dcba_4), d8, unaligned + 1 * 16); + StoreU(BitCast(d8, dcba_8), d8, unaligned + 2 * 16); + StoreU(BitCast(d8, dcba_C), d8, unaligned + 3 * 16); +} + +// 64 bits +HWY_API void StoreInterleaved4(const Vec128<uint8_t, 8> in0, + const Vec128<uint8_t, 8> in1, + const Vec128<uint8_t, 8> in2, + const Vec128<uint8_t, 8> in3, + Simd<uint8_t, 8> /* tag */, + uint8_t* HWY_RESTRICT unaligned) { + // Use full vectors to reduce the number of stores. + const Full256<uint8_t> d_full8; + const RepartitionToWide<decltype(d_full8)> d16; + const RepartitionToWide<decltype(d16)> d32; + const Vec128<uint8_t> v0{in0.raw}; + const Vec128<uint8_t> v1{in1.raw}; + const Vec128<uint8_t> v2{in2.raw}; + const Vec128<uint8_t> v3{in3.raw}; + // let a,b,c,d denote v0..3. + const auto ba0 = ZipLower(d16, v0, v1); // b7 a7 .. b0 a0 + const auto dc0 = ZipLower(d16, v2, v3); // d7 c7 .. d0 c0 + const auto dcba_0 = ZipLower(d32, ba0, dc0); // d..a3 d..a0 + const auto dcba_4 = ZipUpper(d32, ba0, dc0); // d..a7 d..a4 + StoreU(BitCast(d_full8, dcba_0), d_full8, unaligned + 0 * 16); + StoreU(BitCast(d_full8, dcba_4), d_full8, unaligned + 1 * 16); +} + +// <= 32 bits +template <size_t N, HWY_IF_LE32(uint8_t, N)> +HWY_API void StoreInterleaved4(const Vec128<uint8_t, N> in0, + const Vec128<uint8_t, N> in1, + const Vec128<uint8_t, N> in2, + const Vec128<uint8_t, N> in3, + Simd<uint8_t, N> /*tag*/, + uint8_t* HWY_RESTRICT unaligned) { + // Use full vectors to reduce the number of stores. + const Full256<uint8_t> d_full8; + const RepartitionToWide<decltype(d_full8)> d16; + const RepartitionToWide<decltype(d16)> d32; + const Vec128<uint8_t> v0{in0.raw}; + const Vec128<uint8_t> v1{in1.raw}; + const Vec128<uint8_t> v2{in2.raw}; + const Vec128<uint8_t> v3{in3.raw}; + // let a,b,c,d denote v0..3. + const auto ba0 = ZipLower(d16, v0, v1); // b3 a3 .. b0 a0 + const auto dc0 = ZipLower(d16, v2, v3); // d3 c3 .. d0 c0 + const auto dcba_0 = ZipLower(d32, ba0, dc0); // d..a3 d..a0 + alignas(16) uint8_t buf[16]; + StoreU(BitCast(d_full8, dcba_0), d_full8, buf); + CopyBytes<4 * N>(buf, unaligned); +} + +// ------------------------------ MulEven/Odd (Load) + +HWY_INLINE Vec128<uint64_t> MulEven(const Vec128<uint64_t> a, + const Vec128<uint64_t> b) { + alignas(16) uint64_t mul[2]; + mul[0] = + Mul128(static_cast<uint64_t>(wasm_i64x2_extract_lane(a.raw, 0)), + static_cast<uint64_t>(wasm_i64x2_extract_lane(b.raw, 0)), &mul[1]); + return Load(Full256<uint64_t>(), mul); +} + +HWY_INLINE Vec128<uint64_t> MulOdd(const Vec128<uint64_t> a, + const Vec128<uint64_t> b) { + alignas(16) uint64_t mul[2]; + mul[0] = + Mul128(static_cast<uint64_t>(wasm_i64x2_extract_lane(a.raw, 1)), + static_cast<uint64_t>(wasm_i64x2_extract_lane(b.raw, 1)), &mul[1]); + return Load(Full256<uint64_t>(), mul); +} + +// ------------------------------ ReorderWidenMulAccumulate (MulAdd, ZipLower) + +template <size_t N> +HWY_API Vec128<float, N> ReorderWidenMulAccumulate(Simd<float, N> df32, + Vec128<bfloat16_t, 2 * N> a, + Vec128<bfloat16_t, 2 * N> b, + const Vec128<float, N> sum0, + Vec128<float, N>& sum1) { + const Repartition<uint16_t, decltype(df32)> du16; + const RebindToUnsigned<decltype(df32)> du32; + const Vec128<uint16_t, 2 * N> zero = Zero(du16); + const Vec128<uint32_t, N> a0 = ZipLower(du32, zero, BitCast(du16, a)); + const Vec128<uint32_t, N> a1 = ZipUpper(du32, zero, BitCast(du16, a)); + const Vec128<uint32_t, N> b0 = ZipLower(du32, zero, BitCast(du16, b)); + const Vec128<uint32_t, N> b1 = ZipUpper(du32, zero, BitCast(du16, b)); + sum1 = MulAdd(BitCast(df32, a1), BitCast(df32, b1), sum1); + return MulAdd(BitCast(df32, a0), BitCast(df32, b0), sum0); +} + +// ------------------------------ Reductions + +namespace detail { + +// N=1 for any T: no-op +template <typename T> +HWY_INLINE Vec128<T, 1> SumOfLanes(hwy::SizeTag<sizeof(T)> /* tag */, + const Vec128<T, 1> v) { + return v; +} +template <typename T> +HWY_INLINE Vec128<T, 1> MinOfLanes(hwy::SizeTag<sizeof(T)> /* tag */, + const Vec128<T, 1> v) { + return v; +} +template <typename T> +HWY_INLINE Vec128<T, 1> MaxOfLanes(hwy::SizeTag<sizeof(T)> /* tag */, + const Vec128<T, 1> v) { + return v; +} + +// u32/i32/f32: + +// N=2 +template <typename T> +HWY_INLINE Vec128<T, 2> SumOfLanes(hwy::SizeTag<4> /* tag */, + const Vec128<T, 2> v10) { + return v10 + Vec128<T, 2>{Shuffle2301(Vec128<T>{v10.raw}).raw}; +} +template <typename T> +HWY_INLINE Vec128<T, 2> MinOfLanes(hwy::SizeTag<4> /* tag */, + const Vec128<T, 2> v10) { + return Min(v10, Vec128<T, 2>{Shuffle2301(Vec128<T>{v10.raw}).raw}); +} +template <typename T> +HWY_INLINE Vec128<T, 2> MaxOfLanes(hwy::SizeTag<4> /* tag */, + const Vec128<T, 2> v10) { + return Max(v10, Vec128<T, 2>{Shuffle2301(Vec128<T>{v10.raw}).raw}); +} + +// N=4 (full) +template <typename T> +HWY_INLINE Vec128<T> SumOfLanes(hwy::SizeTag<4> /* tag */, + const Vec128<T> v3210) { + const Vec128<T> v1032 = Shuffle1032(v3210); + const Vec128<T> v31_20_31_20 = v3210 + v1032; + const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20); + return v20_31_20_31 + v31_20_31_20; +} +template <typename T> +HWY_INLINE Vec128<T> MinOfLanes(hwy::SizeTag<4> /* tag */, + const Vec128<T> v3210) { + const Vec128<T> v1032 = Shuffle1032(v3210); + const Vec128<T> v31_20_31_20 = Min(v3210, v1032); + const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20); + return Min(v20_31_20_31, v31_20_31_20); +} +template <typename T> +HWY_INLINE Vec128<T> MaxOfLanes(hwy::SizeTag<4> /* tag */, + const Vec128<T> v3210) { + const Vec128<T> v1032 = Shuffle1032(v3210); + const Vec128<T> v31_20_31_20 = Max(v3210, v1032); + const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20); + return Max(v20_31_20_31, v31_20_31_20); +} + +// u64/i64/f64: + +// N=2 (full) +template <typename T> +HWY_INLINE Vec128<T> SumOfLanes(hwy::SizeTag<8> /* tag */, + const Vec128<T> v10) { + const Vec128<T> v01 = Shuffle01(v10); + return v10 + v01; +} +template <typename T> +HWY_INLINE Vec128<T> MinOfLanes(hwy::SizeTag<8> /* tag */, + const Vec128<T> v10) { + const Vec128<T> v01 = Shuffle01(v10); + return Min(v10, v01); +} +template <typename T> +HWY_INLINE Vec128<T> MaxOfLanes(hwy::SizeTag<8> /* tag */, + const Vec128<T> v10) { + const Vec128<T> v01 = Shuffle01(v10); + return Max(v10, v01); +} + +// u16/i16 +template <typename T, HWY_IF_LANE_SIZE(T, 2), HWY_IF_GE32(T, N)> +HWY_API Vec256<T> MinOfLanes(hwy::SizeTag<2> /* tag */, Vec256<T> v) { + const Repartition<int32_t, Simd<T, N>> d32; + const auto even = And(BitCast(d32, v), Set(d32, 0xFFFF)); + const auto odd = ShiftRight<16>(BitCast(d32, v)); + const auto min = MinOfLanes(d32, Min(even, odd)); + // Also broadcast into odd lanes. + return BitCast(Simd<T, N>(), Or(min, ShiftLeft<16>(min))); +} +template <typename T, HWY_IF_LANE_SIZE(T, 2), HWY_IF_GE32(T, N)> +HWY_API Vec256<T> MaxOfLanes(hwy::SizeTag<2> /* tag */, Vec256<T> v) { + const Repartition<int32_t, Simd<T, N>> d32; + const auto even = And(BitCast(d32, v), Set(d32, 0xFFFF)); + const auto odd = ShiftRight<16>(BitCast(d32, v)); + const auto min = MaxOfLanes(d32, Max(even, odd)); + // Also broadcast into odd lanes. + return BitCast(Simd<T, N>(), Or(min, ShiftLeft<16>(min))); +} + +} // namespace detail + +// Supported for u/i/f 32/64. Returns the same value in each lane. +template <typename T> +HWY_API Vec256<T> SumOfLanes(Simd<T, N> /* tag */, const Vec256<T> v) { + return detail::SumOfLanes(hwy::SizeTag<sizeof(T)>(), v); +} +template <typename T> +HWY_API Vec256<T> MinOfLanes(Simd<T, N> /* tag */, const Vec256<T> v) { + return detail::MinOfLanes(hwy::SizeTag<sizeof(T)>(), v); +} +template <typename T> +HWY_API Vec256<T> MaxOfLanes(Simd<T, N> /* tag */, const Vec256<T> v) { + return detail::MaxOfLanes(hwy::SizeTag<sizeof(T)>(), v); +} + +// ================================================== DEPRECATED + +template <typename T> +HWY_API size_t StoreMaskBits(const Mask128<T, N> mask, uint8_t* bits) { + return StoreMaskBits(Simd<T, N>(), mask, bits); +} + +template <typename T> +HWY_API bool AllTrue(const Mask128<T, N> mask) { + return AllTrue(Simd<T, N>(), mask); +} + +template <typename T> +HWY_API bool AllFalse(const Mask128<T, N> mask) { + return AllFalse(Simd<T, N>(), mask); +} + +template <typename T> +HWY_API size_t CountTrue(const Mask128<T, N> mask) { + return CountTrue(Simd<T, N>(), mask); +} + +template <typename T> +HWY_API Vec256<T> SumOfLanes(const Vec256<T> v) { + return SumOfLanes(Simd<T, N>(), v); +} +template <typename T> +HWY_API Vec256<T> MinOfLanes(const Vec256<T> v) { + return MinOfLanes(Simd<T, N>(), v); +} +template <typename T> +HWY_API Vec256<T> MaxOfLanes(const Vec256<T> v) { + return MaxOfLanes(Simd<T, N>(), v); +} + +template <typename T> +HWY_API Vec128<T, (N + 1) / 2> UpperHalf(Vec256<T> v) { + return UpperHalf(Half<Simd<T, N>>(), v); +} + +template <int kBytes, typename T> +HWY_API Vec256<T> ShiftRightBytes(const Vec256<T> v) { + return ShiftRightBytes<kBytes>(Simd<T, N>(), v); +} + +template <int kLanes, typename T> +HWY_API Vec256<T> ShiftRightLanes(const Vec256<T> v) { + return ShiftRightLanes<kLanes>(Simd<T, N>(), v); +} + +template <size_t kBytes, typename T> +HWY_API Vec256<T> CombineShiftRightBytes(Vec256<T> hi, Vec256<T> lo) { + return CombineShiftRightBytes<kBytes>(Simd<T, N>(), hi, lo); +} + +template <typename T> +HWY_API Vec256<T> InterleaveUpper(Vec256<T> a, Vec256<T> b) { + return InterleaveUpper(Simd<T, N>(), a, b); +} + +template <typename T, class D = Simd<T, N>> +HWY_API VFromD<RepartitionToWide<D>> ZipUpper(Vec256<T> a, Vec256<T> b) { + return InterleaveUpper(RepartitionToWide<D>(), a, b); +} + +template <typename T2> +HWY_API Vec128<T, N2 * 2> Combine(Vec128<T, N2> hi2, Vec128<T, N2> lo2) { + return Combine(Simd<T, N2 * 2>(), hi2, lo2); +} + +template <typename T2, HWY_IF_LE64(T, N2)> +HWY_API Vec128<T, N2 * 2> ZeroExtendVector(Vec128<T, N2> lo) { + return ZeroExtendVector(Simd<T, N2 * 2>(), lo); +} + +template <typename T> +HWY_API Vec256<T> ConcatLowerLower(Vec256<T> hi, Vec256<T> lo) { + return ConcatLowerLower(Simd<T, N>(), hi, lo); +} + +template <typename T> +HWY_API Vec256<T> ConcatUpperUpper(Vec256<T> hi, Vec256<T> lo) { + return ConcatUpperUpper(Simd<T, N>(), hi, lo); +} + +template <typename T> +HWY_API Vec256<T> ConcatLowerUpper(const Vec256<T> hi, const Vec256<T> lo) { + return ConcatLowerUpper(Simd<T, N>(), hi, lo); +} + +template <typename T> +HWY_API Vec256<T> ConcatUpperLower(Vec256<T> hi, Vec256<T> lo) { + return ConcatUpperLower(Simd<T, N>(), hi, lo); +} + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); diff --git a/third_party/highway/hwy/ops/x86_128-inl.h b/third_party/highway/hwy/ops/x86_128-inl.h new file mode 100644 index 0000000..0bb7e26 --- /dev/null +++ b/third_party/highway/hwy/ops/x86_128-inl.h @@ -0,0 +1,5891 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// 128-bit vectors and SSE4 instructions, plus some AVX2 and AVX512-VL +// operations when compiling for those targets. +// External include guard in highway.h - see comment there. + +#include <emmintrin.h> +#if HWY_TARGET == HWY_SSSE3 +#include <tmmintrin.h> // SSSE3 +#else +#include <smmintrin.h> // SSE4 +#include <wmmintrin.h> // CLMUL +#endif +#include <stddef.h> +#include <stdint.h> + +#include "hwy/base.h" +#include "hwy/ops/shared-inl.h" + +// Clang 3.9 generates VINSERTF128 instead of the desired VBROADCASTF128, +// which would free up port5. However, inline assembly isn't supported on +// MSVC, results in incorrect output on GCC 8.3, and raises "invalid output size +// for constraint" errors on Clang (https://gcc.godbolt.org/z/-Jt_-F), hence we +// disable it. +#ifndef HWY_LOADDUP_ASM +#define HWY_LOADDUP_ASM 0 +#endif + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +template <typename T> +using Full128 = Simd<T, 16 / sizeof(T)>; + +namespace detail { + +template <typename T> +struct Raw128 { + using type = __m128i; +}; +template <> +struct Raw128<float> { + using type = __m128; +}; +template <> +struct Raw128<double> { + using type = __m128d; +}; + +} // namespace detail + +template <typename T, size_t N = 16 / sizeof(T)> +class Vec128 { + using Raw = typename detail::Raw128<T>::type; + + public: + // Compound assignment. Only usable if there is a corresponding non-member + // binary operator overload. For example, only f32 and f64 support division. + HWY_INLINE Vec128& operator*=(const Vec128 other) { + return *this = (*this * other); + } + HWY_INLINE Vec128& operator/=(const Vec128 other) { + return *this = (*this / other); + } + HWY_INLINE Vec128& operator+=(const Vec128 other) { + return *this = (*this + other); + } + HWY_INLINE Vec128& operator-=(const Vec128 other) { + return *this = (*this - other); + } + HWY_INLINE Vec128& operator&=(const Vec128 other) { + return *this = (*this & other); + } + HWY_INLINE Vec128& operator|=(const Vec128 other) { + return *this = (*this | other); + } + HWY_INLINE Vec128& operator^=(const Vec128 other) { + return *this = (*this ^ other); + } + + Raw raw; +}; + +// Forward-declare for use by DeduceD, see below. +template <typename T> +class Vec256; +template <typename T> +class Vec512; + +#if HWY_TARGET <= HWY_AVX3 + +namespace detail { + +// Template arg: sizeof(lane type) +template <size_t size> +struct RawMask128 {}; +template <> +struct RawMask128<1> { + using type = __mmask16; +}; +template <> +struct RawMask128<2> { + using type = __mmask8; +}; +template <> +struct RawMask128<4> { + using type = __mmask8; +}; +template <> +struct RawMask128<8> { + using type = __mmask8; +}; + +} // namespace detail + +template <typename T, size_t N> +struct Mask128 { + using Raw = typename detail::RawMask128<sizeof(T)>::type; + + static Mask128<T, N> FromBits(uint64_t mask_bits) { + return Mask128<T, N>{static_cast<Raw>(mask_bits)}; + } + + Raw raw; +}; + +#else // AVX2 or below + +// FF..FF or 0. +template <typename T, size_t N = 16 / sizeof(T)> +struct Mask128 { + typename detail::Raw128<T>::type raw; +}; + +#endif // HWY_TARGET <= HWY_AVX3 + +namespace detail { + +// Deduce Simd<T, N> from Vec*<T, N> (pointers because Vec256/512 may be +// incomplete types at this point; this is simpler than avoiding multiple +// definitions of DFromV via #if) +struct DeduceD { + template <typename T, size_t N> + Simd<T, N> operator()(const Vec128<T, N>*) const { + return Simd<T, N>(); + } + template <typename T> + Simd<T, 32 / sizeof(T)> operator()(const Vec256<T>*) const { + return Simd<T, 32 / sizeof(T)>(); + } + template <typename T> + Simd<T, 64 / sizeof(T)> operator()(const Vec512<T>*) const { + return Simd<T, 64 / sizeof(T)>(); + } +}; + +// Workaround for MSVC v19.14: alias with a dependent type fails to specialize. +template <class V> +struct ExpandDFromV { + using type = decltype(DeduceD()(static_cast<V*>(nullptr))); +}; + +} // namespace detail + +template <class V> +using DFromV = typename detail::ExpandDFromV<V>::type; + +template <class V> +using TFromV = TFromD<DFromV<V>>; + +// ------------------------------ BitCast + +namespace detail { + +HWY_INLINE __m128i BitCastToInteger(__m128i v) { return v; } +HWY_INLINE __m128i BitCastToInteger(__m128 v) { return _mm_castps_si128(v); } +HWY_INLINE __m128i BitCastToInteger(__m128d v) { return _mm_castpd_si128(v); } + +template <typename T, size_t N> +HWY_INLINE Vec128<uint8_t, N * sizeof(T)> BitCastToByte(Vec128<T, N> v) { + return Vec128<uint8_t, N * sizeof(T)>{BitCastToInteger(v.raw)}; +} + +// Cannot rely on function overloading because return types differ. +template <typename T> +struct BitCastFromInteger128 { + HWY_INLINE __m128i operator()(__m128i v) { return v; } +}; +template <> +struct BitCastFromInteger128<float> { + HWY_INLINE __m128 operator()(__m128i v) { return _mm_castsi128_ps(v); } +}; +template <> +struct BitCastFromInteger128<double> { + HWY_INLINE __m128d operator()(__m128i v) { return _mm_castsi128_pd(v); } +}; + +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> BitCastFromByte(Simd<T, N> /* tag */, + Vec128<uint8_t, N * sizeof(T)> v) { + return Vec128<T, N>{BitCastFromInteger128<T>()(v.raw)}; +} + +} // namespace detail + +template <typename T, size_t N, typename FromT> +HWY_API Vec128<T, N> BitCast(Simd<T, N> d, + Vec128<FromT, N * sizeof(T) / sizeof(FromT)> v) { + return detail::BitCastFromByte(d, detail::BitCastToByte(v)); +} + +// ------------------------------ Zero + +// Returns an all-zero vector/part. +template <typename T, size_t N, HWY_IF_LE128(T, N)> +HWY_API Vec128<T, N> Zero(Simd<T, N> /* tag */) { + return Vec128<T, N>{_mm_setzero_si128()}; +} +template <size_t N, HWY_IF_LE128(float, N)> +HWY_API Vec128<float, N> Zero(Simd<float, N> /* tag */) { + return Vec128<float, N>{_mm_setzero_ps()}; +} +template <size_t N, HWY_IF_LE128(double, N)> +HWY_API Vec128<double, N> Zero(Simd<double, N> /* tag */) { + return Vec128<double, N>{_mm_setzero_pd()}; +} + +template <class D> +using VFromD = decltype(Zero(D())); + +// ------------------------------ Set + +// Returns a vector/part with all lanes set to "t". +template <size_t N, HWY_IF_LE128(uint8_t, N)> +HWY_API Vec128<uint8_t, N> Set(Simd<uint8_t, N> /* tag */, const uint8_t t) { + return Vec128<uint8_t, N>{_mm_set1_epi8(static_cast<char>(t))}; // NOLINT +} +template <size_t N, HWY_IF_LE128(uint16_t, N)> +HWY_API Vec128<uint16_t, N> Set(Simd<uint16_t, N> /* tag */, const uint16_t t) { + return Vec128<uint16_t, N>{_mm_set1_epi16(static_cast<short>(t))}; // NOLINT +} +template <size_t N, HWY_IF_LE128(uint32_t, N)> +HWY_API Vec128<uint32_t, N> Set(Simd<uint32_t, N> /* tag */, const uint32_t t) { + return Vec128<uint32_t, N>{_mm_set1_epi32(static_cast<int>(t))}; +} +template <size_t N, HWY_IF_LE128(uint64_t, N)> +HWY_API Vec128<uint64_t, N> Set(Simd<uint64_t, N> /* tag */, const uint64_t t) { + return Vec128<uint64_t, N>{ + _mm_set1_epi64x(static_cast<long long>(t))}; // NOLINT +} +template <size_t N, HWY_IF_LE128(int8_t, N)> +HWY_API Vec128<int8_t, N> Set(Simd<int8_t, N> /* tag */, const int8_t t) { + return Vec128<int8_t, N>{_mm_set1_epi8(static_cast<char>(t))}; // NOLINT +} +template <size_t N, HWY_IF_LE128(int16_t, N)> +HWY_API Vec128<int16_t, N> Set(Simd<int16_t, N> /* tag */, const int16_t t) { + return Vec128<int16_t, N>{_mm_set1_epi16(static_cast<short>(t))}; // NOLINT +} +template <size_t N, HWY_IF_LE128(int32_t, N)> +HWY_API Vec128<int32_t, N> Set(Simd<int32_t, N> /* tag */, const int32_t t) { + return Vec128<int32_t, N>{_mm_set1_epi32(t)}; +} +template <size_t N, HWY_IF_LE128(int64_t, N)> +HWY_API Vec128<int64_t, N> Set(Simd<int64_t, N> /* tag */, const int64_t t) { + return Vec128<int64_t, N>{ + _mm_set1_epi64x(static_cast<long long>(t))}; // NOLINT +} +template <size_t N, HWY_IF_LE128(float, N)> +HWY_API Vec128<float, N> Set(Simd<float, N> /* tag */, const float t) { + return Vec128<float, N>{_mm_set1_ps(t)}; +} +template <size_t N, HWY_IF_LE128(double, N)> +HWY_API Vec128<double, N> Set(Simd<double, N> /* tag */, const double t) { + return Vec128<double, N>{_mm_set1_pd(t)}; +} + +HWY_DIAGNOSTICS(push) +HWY_DIAGNOSTICS_OFF(disable : 4700, ignored "-Wuninitialized") + +// Returns a vector with uninitialized elements. +template <typename T, size_t N, HWY_IF_LE128(T, N)> +HWY_API Vec128<T, N> Undefined(Simd<T, N> /* tag */) { + // Available on Clang 6.0, GCC 6.2, ICC 16.03, MSVC 19.14. All but ICC + // generate an XOR instruction. + return Vec128<T, N>{_mm_undefined_si128()}; +} +template <size_t N, HWY_IF_LE128(float, N)> +HWY_API Vec128<float, N> Undefined(Simd<float, N> /* tag */) { + return Vec128<float, N>{_mm_undefined_ps()}; +} +template <size_t N, HWY_IF_LE128(double, N)> +HWY_API Vec128<double, N> Undefined(Simd<double, N> /* tag */) { + return Vec128<double, N>{_mm_undefined_pd()}; +} + +HWY_DIAGNOSTICS(pop) + +// ------------------------------ GetLane + +// Gets the single value stored in a vector/part. +template <size_t N> +HWY_API uint8_t GetLane(const Vec128<uint8_t, N> v) { + return static_cast<uint8_t>(_mm_cvtsi128_si32(v.raw) & 0xFF); +} +template <size_t N> +HWY_API int8_t GetLane(const Vec128<int8_t, N> v) { + return static_cast<int8_t>(_mm_cvtsi128_si32(v.raw) & 0xFF); +} +template <size_t N> +HWY_API uint16_t GetLane(const Vec128<uint16_t, N> v) { + return static_cast<uint16_t>(_mm_cvtsi128_si32(v.raw) & 0xFFFF); +} +template <size_t N> +HWY_API int16_t GetLane(const Vec128<int16_t, N> v) { + return static_cast<int16_t>(_mm_cvtsi128_si32(v.raw) & 0xFFFF); +} +template <size_t N> +HWY_API uint32_t GetLane(const Vec128<uint32_t, N> v) { + return static_cast<uint32_t>(_mm_cvtsi128_si32(v.raw)); +} +template <size_t N> +HWY_API int32_t GetLane(const Vec128<int32_t, N> v) { + return _mm_cvtsi128_si32(v.raw); +} +template <size_t N> +HWY_API float GetLane(const Vec128<float, N> v) { + return _mm_cvtss_f32(v.raw); +} +template <size_t N> +HWY_API uint64_t GetLane(const Vec128<uint64_t, N> v) { +#if HWY_ARCH_X86_32 + alignas(16) uint64_t lanes[2]; + Store(v, Simd<uint64_t, N>(), lanes); + return lanes[0]; +#else + return static_cast<uint64_t>(_mm_cvtsi128_si64(v.raw)); +#endif +} +template <size_t N> +HWY_API int64_t GetLane(const Vec128<int64_t, N> v) { +#if HWY_ARCH_X86_32 + alignas(16) int64_t lanes[2]; + Store(v, Simd<int64_t, N>(), lanes); + return lanes[0]; +#else + return _mm_cvtsi128_si64(v.raw); +#endif +} +template <size_t N> +HWY_API double GetLane(const Vec128<double, N> v) { + return _mm_cvtsd_f64(v.raw); +} + +// ================================================== LOGICAL + +// ------------------------------ And + +template <typename T, size_t N> +HWY_API Vec128<T, N> And(Vec128<T, N> a, Vec128<T, N> b) { + return Vec128<T, N>{_mm_and_si128(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<float, N> And(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Vec128<float, N>{_mm_and_ps(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<double, N> And(const Vec128<double, N> a, + const Vec128<double, N> b) { + return Vec128<double, N>{_mm_and_pd(a.raw, b.raw)}; +} + +// ------------------------------ AndNot + +// Returns ~not_mask & mask. +template <typename T, size_t N> +HWY_API Vec128<T, N> AndNot(Vec128<T, N> not_mask, Vec128<T, N> mask) { + return Vec128<T, N>{_mm_andnot_si128(not_mask.raw, mask.raw)}; +} +template <size_t N> +HWY_API Vec128<float, N> AndNot(const Vec128<float, N> not_mask, + const Vec128<float, N> mask) { + return Vec128<float, N>{_mm_andnot_ps(not_mask.raw, mask.raw)}; +} +template <size_t N> +HWY_API Vec128<double, N> AndNot(const Vec128<double, N> not_mask, + const Vec128<double, N> mask) { + return Vec128<double, N>{_mm_andnot_pd(not_mask.raw, mask.raw)}; +} + +// ------------------------------ Or + +template <typename T, size_t N> +HWY_API Vec128<T, N> Or(Vec128<T, N> a, Vec128<T, N> b) { + return Vec128<T, N>{_mm_or_si128(a.raw, b.raw)}; +} + +template <size_t N> +HWY_API Vec128<float, N> Or(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Vec128<float, N>{_mm_or_ps(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<double, N> Or(const Vec128<double, N> a, + const Vec128<double, N> b) { + return Vec128<double, N>{_mm_or_pd(a.raw, b.raw)}; +} + +// ------------------------------ Xor + +template <typename T, size_t N> +HWY_API Vec128<T, N> Xor(Vec128<T, N> a, Vec128<T, N> b) { + return Vec128<T, N>{_mm_xor_si128(a.raw, b.raw)}; +} + +template <size_t N> +HWY_API Vec128<float, N> Xor(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Vec128<float, N>{_mm_xor_ps(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<double, N> Xor(const Vec128<double, N> a, + const Vec128<double, N> b) { + return Vec128<double, N>{_mm_xor_pd(a.raw, b.raw)}; +} + +// ------------------------------ Not + +template <typename T, size_t N> +HWY_API Vec128<T, N> Not(const Vec128<T, N> v) { + using TU = MakeUnsigned<T>; +#if HWY_TARGET <= HWY_AVX3 + const __m128i vu = BitCast(Simd<TU, N>(), v).raw; + return BitCast(Simd<T, N>(), + Vec128<TU, N>{_mm_ternarylogic_epi32(vu, vu, vu, 0x55)}); +#else + return Xor(v, BitCast(Simd<T, N>(), Vec128<TU, N>{_mm_set1_epi32(-1)})); +#endif +} + +// ------------------------------ Operator overloads (internal-only if float) + +template <typename T, size_t N> +HWY_API Vec128<T, N> operator&(const Vec128<T, N> a, const Vec128<T, N> b) { + return And(a, b); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> operator|(const Vec128<T, N> a, const Vec128<T, N> b) { + return Or(a, b); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> operator^(const Vec128<T, N> a, const Vec128<T, N> b) { + return Xor(a, b); +} + +// ------------------------------ PopulationCount + +// 8/16 require BITALG, 32/64 require VPOPCNTDQ. +#if HWY_TARGET == HWY_AVX3_DL + +#ifdef HWY_NATIVE_POPCNT +#undef HWY_NATIVE_POPCNT +#else +#define HWY_NATIVE_POPCNT +#endif + +namespace detail { + +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> PopulationCount(hwy::SizeTag<1> /* tag */, + Vec128<T, N> v) { + return Vec128<T, N>{_mm_popcnt_epi8(v.raw)}; +} +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> PopulationCount(hwy::SizeTag<2> /* tag */, + Vec128<T, N> v) { + return Vec128<T, N>{_mm_popcnt_epi16(v.raw)}; +} +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> PopulationCount(hwy::SizeTag<4> /* tag */, + Vec128<T, N> v) { + return Vec128<T, N>{_mm_popcnt_epi32(v.raw)}; +} +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> PopulationCount(hwy::SizeTag<8> /* tag */, + Vec128<T, N> v) { + return Vec128<T, N>{_mm_popcnt_epi64(v.raw)}; +} + +} // namespace detail + +template <typename T, size_t N> +HWY_API Vec128<T, N> PopulationCount(Vec128<T, N> v) { + return detail::PopulationCount(hwy::SizeTag<sizeof(T)>(), v); +} + +#endif // HWY_TARGET == HWY_AVX3_DL + +// ================================================== SIGN + +// ------------------------------ Neg + +template <typename T, size_t N, HWY_IF_FLOAT(T)> +HWY_API Vec128<T, N> Neg(const Vec128<T, N> v) { + return Xor(v, SignBit(Simd<T, N>())); +} + +template <typename T, size_t N, HWY_IF_NOT_FLOAT(T)> +HWY_API Vec128<T, N> Neg(const Vec128<T, N> v) { + return Zero(Simd<T, N>()) - v; +} + +// ------------------------------ Abs + +// Returns absolute value, except that LimitsMin() maps to LimitsMax() + 1. +template <size_t N> +HWY_API Vec128<int8_t, N> Abs(const Vec128<int8_t, N> v) { +#if HWY_COMPILER_MSVC + // Workaround for incorrect codegen? (reaches breakpoint) + const auto zero = Zero(Simd<int8_t, N>()); + return Vec128<int8_t, N>{_mm_max_epi8(v.raw, (zero - v).raw)}; +#else + return Vec128<int8_t, N>{_mm_abs_epi8(v.raw)}; +#endif +} +template <size_t N> +HWY_API Vec128<int16_t, N> Abs(const Vec128<int16_t, N> v) { + return Vec128<int16_t, N>{_mm_abs_epi16(v.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> Abs(const Vec128<int32_t, N> v) { + return Vec128<int32_t, N>{_mm_abs_epi32(v.raw)}; +} +// i64 is implemented after BroadcastSignBit. +template <size_t N> +HWY_API Vec128<float, N> Abs(const Vec128<float, N> v) { + const Vec128<int32_t, N> mask{_mm_set1_epi32(0x7FFFFFFF)}; + return v & BitCast(Simd<float, N>(), mask); +} +template <size_t N> +HWY_API Vec128<double, N> Abs(const Vec128<double, N> v) { + const Vec128<int64_t, N> mask{_mm_set1_epi64x(0x7FFFFFFFFFFFFFFFLL)}; + return v & BitCast(Simd<double, N>(), mask); +} + +// ------------------------------ CopySign + +template <typename T, size_t N> +HWY_API Vec128<T, N> CopySign(const Vec128<T, N> magn, + const Vec128<T, N> sign) { + static_assert(IsFloat<T>(), "Only makes sense for floating-point"); + + const Simd<T, N> d; + const auto msb = SignBit(d); + +#if HWY_TARGET <= HWY_AVX3 + const Rebind<MakeUnsigned<T>, decltype(d)> du; + // Truth table for msb, magn, sign | bitwise msb ? sign : mag + // 0 0 0 | 0 + // 0 0 1 | 0 + // 0 1 0 | 1 + // 0 1 1 | 1 + // 1 0 0 | 0 + // 1 0 1 | 1 + // 1 1 0 | 0 + // 1 1 1 | 1 + // The lane size does not matter because we are not using predication. + const __m128i out = _mm_ternarylogic_epi32( + BitCast(du, msb).raw, BitCast(du, magn).raw, BitCast(du, sign).raw, 0xAC); + return BitCast(d, decltype(Zero(du)){out}); +#else + return Or(AndNot(msb, magn), And(msb, sign)); +#endif +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> CopySignToAbs(const Vec128<T, N> abs, + const Vec128<T, N> sign) { +#if HWY_TARGET <= HWY_AVX3 + // AVX3 can also handle abs < 0, so no extra action needed. + return CopySign(abs, sign); +#else + return Or(abs, And(SignBit(Simd<T, N>()), sign)); +#endif +} + +// ================================================== MASK + +#if HWY_TARGET <= HWY_AVX3 + +// ------------------------------ IfThenElse + +// Returns mask ? b : a. + +namespace detail { + +// Templates for signed/unsigned integer of a particular size. +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> IfThenElse(hwy::SizeTag<1> /* tag */, + Mask128<T, N> mask, Vec128<T, N> yes, + Vec128<T, N> no) { + return Vec128<T, N>{_mm_mask_mov_epi8(no.raw, mask.raw, yes.raw)}; +} +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> IfThenElse(hwy::SizeTag<2> /* tag */, + Mask128<T, N> mask, Vec128<T, N> yes, + Vec128<T, N> no) { + return Vec128<T, N>{_mm_mask_mov_epi16(no.raw, mask.raw, yes.raw)}; +} +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> IfThenElse(hwy::SizeTag<4> /* tag */, + Mask128<T, N> mask, Vec128<T, N> yes, + Vec128<T, N> no) { + return Vec128<T, N>{_mm_mask_mov_epi32(no.raw, mask.raw, yes.raw)}; +} +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> IfThenElse(hwy::SizeTag<8> /* tag */, + Mask128<T, N> mask, Vec128<T, N> yes, + Vec128<T, N> no) { + return Vec128<T, N>{_mm_mask_mov_epi64(no.raw, mask.raw, yes.raw)}; +} + +} // namespace detail + +template <typename T, size_t N> +HWY_API Vec128<T, N> IfThenElse(Mask128<T, N> mask, Vec128<T, N> yes, + Vec128<T, N> no) { + return detail::IfThenElse(hwy::SizeTag<sizeof(T)>(), mask, yes, no); +} + +template <size_t N> +HWY_API Vec128<float, N> IfThenElse(Mask128<float, N> mask, + Vec128<float, N> yes, Vec128<float, N> no) { + return Vec128<float, N>{_mm_mask_mov_ps(no.raw, mask.raw, yes.raw)}; +} + +template <size_t N> +HWY_API Vec128<double, N> IfThenElse(Mask128<double, N> mask, + Vec128<double, N> yes, + Vec128<double, N> no) { + return Vec128<double, N>{_mm_mask_mov_pd(no.raw, mask.raw, yes.raw)}; +} + +namespace detail { + +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> IfThenElseZero(hwy::SizeTag<1> /* tag */, + Mask128<T, N> mask, Vec128<T, N> yes) { + return Vec128<T, N>{_mm_maskz_mov_epi8(mask.raw, yes.raw)}; +} +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> IfThenElseZero(hwy::SizeTag<2> /* tag */, + Mask128<T, N> mask, Vec128<T, N> yes) { + return Vec128<T, N>{_mm_maskz_mov_epi16(mask.raw, yes.raw)}; +} +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> IfThenElseZero(hwy::SizeTag<4> /* tag */, + Mask128<T, N> mask, Vec128<T, N> yes) { + return Vec128<T, N>{_mm_maskz_mov_epi32(mask.raw, yes.raw)}; +} +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> IfThenElseZero(hwy::SizeTag<8> /* tag */, + Mask128<T, N> mask, Vec128<T, N> yes) { + return Vec128<T, N>{_mm_maskz_mov_epi64(mask.raw, yes.raw)}; +} + +} // namespace detail + +template <typename T, size_t N> +HWY_API Vec128<T, N> IfThenElseZero(Mask128<T, N> mask, Vec128<T, N> yes) { + return detail::IfThenElseZero(hwy::SizeTag<sizeof(T)>(), mask, yes); +} + +template <size_t N> +HWY_API Vec128<float, N> IfThenElseZero(Mask128<float, N> mask, + Vec128<float, N> yes) { + return Vec128<float, N>{_mm_maskz_mov_ps(mask.raw, yes.raw)}; +} + +template <size_t N> +HWY_API Vec128<double, N> IfThenElseZero(Mask128<double, N> mask, + Vec128<double, N> yes) { + return Vec128<double, N>{_mm_maskz_mov_pd(mask.raw, yes.raw)}; +} + +namespace detail { + +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> IfThenZeroElse(hwy::SizeTag<1> /* tag */, + Mask128<T, N> mask, Vec128<T, N> no) { + // xor_epi8/16 are missing, but we have sub, which is just as fast for u8/16. + return Vec128<T, N>{_mm_mask_sub_epi8(no.raw, mask.raw, no.raw, no.raw)}; +} +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> IfThenZeroElse(hwy::SizeTag<2> /* tag */, + Mask128<T, N> mask, Vec128<T, N> no) { + return Vec128<T, N>{_mm_mask_sub_epi16(no.raw, mask.raw, no.raw, no.raw)}; +} +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> IfThenZeroElse(hwy::SizeTag<4> /* tag */, + Mask128<T, N> mask, Vec128<T, N> no) { + return Vec128<T, N>{_mm_mask_xor_epi32(no.raw, mask.raw, no.raw, no.raw)}; +} +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> IfThenZeroElse(hwy::SizeTag<8> /* tag */, + Mask128<T, N> mask, Vec128<T, N> no) { + return Vec128<T, N>{_mm_mask_xor_epi64(no.raw, mask.raw, no.raw, no.raw)}; +} + +} // namespace detail + +template <typename T, size_t N> +HWY_API Vec128<T, N> IfThenZeroElse(Mask128<T, N> mask, Vec128<T, N> no) { + return detail::IfThenZeroElse(hwy::SizeTag<sizeof(T)>(), mask, no); +} + +template <size_t N> +HWY_API Vec128<float, N> IfThenZeroElse(Mask128<float, N> mask, + Vec128<float, N> no) { + return Vec128<float, N>{_mm_mask_xor_ps(no.raw, mask.raw, no.raw, no.raw)}; +} + +template <size_t N> +HWY_API Vec128<double, N> IfThenZeroElse(Mask128<double, N> mask, + Vec128<double, N> no) { + return Vec128<double, N>{_mm_mask_xor_pd(no.raw, mask.raw, no.raw, no.raw)}; +} + +// ------------------------------ Mask logical + +// For Clang and GCC, mask intrinsics (KORTEST) weren't added until recently. +#if !defined(HWY_COMPILER_HAS_MASK_INTRINSICS) && \ + (HWY_COMPILER_MSVC != 0 || HWY_COMPILER_GCC >= 700 || \ + HWY_COMPILER_CLANG >= 800) +#define HWY_COMPILER_HAS_MASK_INTRINSICS 1 +#else +#define HWY_COMPILER_HAS_MASK_INTRINSICS 0 +#endif + +namespace detail { + +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> And(hwy::SizeTag<1> /*tag*/, const Mask128<T, N> a, + const Mask128<T, N> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask128<T, N>{_kand_mask16(a.raw, b.raw)}; +#else + return Mask128<T, N>{static_cast<__mmask16>(a.raw & b.raw)}; +#endif +} +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> And(hwy::SizeTag<2> /*tag*/, const Mask128<T, N> a, + const Mask128<T, N> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask128<T, N>{_kand_mask8(a.raw, b.raw)}; +#else + return Mask128<T, N>{static_cast<__mmask8>(a.raw & b.raw)}; +#endif +} +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> And(hwy::SizeTag<4> /*tag*/, const Mask128<T, N> a, + const Mask128<T, N> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask128<T, N>{_kand_mask8(a.raw, b.raw)}; +#else + return Mask128<T, N>{static_cast<__mmask8>(a.raw & b.raw)}; +#endif +} +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> And(hwy::SizeTag<8> /*tag*/, const Mask128<T, N> a, + const Mask128<T, N> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask128<T, N>{_kand_mask8(a.raw, b.raw)}; +#else + return Mask128<T, N>{static_cast<__mmask8>(a.raw & b.raw)}; +#endif +} + +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> AndNot(hwy::SizeTag<1> /*tag*/, const Mask128<T, N> a, + const Mask128<T, N> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask128<T, N>{_kandn_mask16(a.raw, b.raw)}; +#else + return Mask128<T, N>{static_cast<__mmask16>(~a.raw & b.raw)}; +#endif +} +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> AndNot(hwy::SizeTag<2> /*tag*/, const Mask128<T, N> a, + const Mask128<T, N> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask128<T, N>{_kandn_mask8(a.raw, b.raw)}; +#else + return Mask128<T, N>{static_cast<__mmask8>(~a.raw & b.raw)}; +#endif +} +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> AndNot(hwy::SizeTag<4> /*tag*/, const Mask128<T, N> a, + const Mask128<T, N> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask128<T, N>{_kandn_mask8(a.raw, b.raw)}; +#else + return Mask128<T, N>{static_cast<__mmask8>(~a.raw & b.raw)}; +#endif +} +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> AndNot(hwy::SizeTag<8> /*tag*/, const Mask128<T, N> a, + const Mask128<T, N> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask128<T, N>{_kandn_mask8(a.raw, b.raw)}; +#else + return Mask128<T, N>{static_cast<__mmask8>(~a.raw & b.raw)}; +#endif +} + +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> Or(hwy::SizeTag<1> /*tag*/, const Mask128<T, N> a, + const Mask128<T, N> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask128<T, N>{_kor_mask16(a.raw, b.raw)}; +#else + return Mask128<T, N>{static_cast<__mmask16>(a.raw | b.raw)}; +#endif +} +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> Or(hwy::SizeTag<2> /*tag*/, const Mask128<T, N> a, + const Mask128<T, N> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask128<T, N>{_kor_mask8(a.raw, b.raw)}; +#else + return Mask128<T, N>{static_cast<__mmask8>(a.raw | b.raw)}; +#endif +} +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> Or(hwy::SizeTag<4> /*tag*/, const Mask128<T, N> a, + const Mask128<T, N> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask128<T, N>{_kor_mask8(a.raw, b.raw)}; +#else + return Mask128<T, N>{static_cast<__mmask8>(a.raw | b.raw)}; +#endif +} +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> Or(hwy::SizeTag<8> /*tag*/, const Mask128<T, N> a, + const Mask128<T, N> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask128<T, N>{_kor_mask8(a.raw, b.raw)}; +#else + return Mask128<T, N>{static_cast<__mmask8>(a.raw | b.raw)}; +#endif +} + +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> Xor(hwy::SizeTag<1> /*tag*/, const Mask128<T, N> a, + const Mask128<T, N> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask128<T, N>{_kxor_mask16(a.raw, b.raw)}; +#else + return Mask128<T, N>{static_cast<__mmask16>(a.raw ^ b.raw)}; +#endif +} +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> Xor(hwy::SizeTag<2> /*tag*/, const Mask128<T, N> a, + const Mask128<T, N> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask128<T, N>{_kxor_mask8(a.raw, b.raw)}; +#else + return Mask128<T, N>{static_cast<__mmask8>(a.raw ^ b.raw)}; +#endif +} +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> Xor(hwy::SizeTag<4> /*tag*/, const Mask128<T, N> a, + const Mask128<T, N> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask128<T, N>{_kxor_mask8(a.raw, b.raw)}; +#else + return Mask128<T, N>{static_cast<__mmask8>(a.raw ^ b.raw)}; +#endif +} +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> Xor(hwy::SizeTag<8> /*tag*/, const Mask128<T, N> a, + const Mask128<T, N> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask128<T, N>{_kxor_mask8(a.raw, b.raw)}; +#else + return Mask128<T, N>{static_cast<__mmask8>(a.raw ^ b.raw)}; +#endif +} + +} // namespace detail + +template <typename T, size_t N> +HWY_API Mask128<T, N> And(const Mask128<T, N> a, Mask128<T, N> b) { + return detail::And(hwy::SizeTag<sizeof(T)>(), a, b); +} + +template <typename T, size_t N> +HWY_API Mask128<T, N> AndNot(const Mask128<T, N> a, Mask128<T, N> b) { + return detail::AndNot(hwy::SizeTag<sizeof(T)>(), a, b); +} + +template <typename T, size_t N> +HWY_API Mask128<T, N> Or(const Mask128<T, N> a, Mask128<T, N> b) { + return detail::Or(hwy::SizeTag<sizeof(T)>(), a, b); +} + +template <typename T, size_t N> +HWY_API Mask128<T, N> Xor(const Mask128<T, N> a, Mask128<T, N> b) { + return detail::Xor(hwy::SizeTag<sizeof(T)>(), a, b); +} + +template <typename T, size_t N> +HWY_API Mask128<T, N> Not(const Mask128<T, N> m) { + // Flip only the valid bits. + return Xor(m, Mask128<T, N>::FromBits((1ull << N) - 1)); +} + +#else // AVX2 or below + +// ------------------------------ Mask + +// Mask and Vec are the same (true = FF..FF). +template <typename T, size_t N> +HWY_API Mask128<T, N> MaskFromVec(const Vec128<T, N> v) { + return Mask128<T, N>{v.raw}; +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> VecFromMask(const Mask128<T, N> v) { + return Vec128<T, N>{v.raw}; +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> VecFromMask(const Simd<T, N> /* tag */, + const Mask128<T, N> v) { + return Vec128<T, N>{v.raw}; +} + +#if HWY_TARGET == HWY_SSSE3 + +// mask ? yes : no +template <typename T, size_t N> +HWY_API Vec128<T, N> IfThenElse(Mask128<T, N> mask, Vec128<T, N> yes, + Vec128<T, N> no) { + const auto vmask = VecFromMask(Simd<T, N>(), mask); + return Or(And(vmask, yes), AndNot(vmask, no)); +} + +#else // HWY_TARGET == HWY_SSSE3 + +// mask ? yes : no +template <typename T, size_t N> +HWY_API Vec128<T, N> IfThenElse(Mask128<T, N> mask, Vec128<T, N> yes, + Vec128<T, N> no) { + return Vec128<T, N>{_mm_blendv_epi8(no.raw, yes.raw, mask.raw)}; +} +template <size_t N> +HWY_API Vec128<float, N> IfThenElse(const Mask128<float, N> mask, + const Vec128<float, N> yes, + const Vec128<float, N> no) { + return Vec128<float, N>{_mm_blendv_ps(no.raw, yes.raw, mask.raw)}; +} +template <size_t N> +HWY_API Vec128<double, N> IfThenElse(const Mask128<double, N> mask, + const Vec128<double, N> yes, + const Vec128<double, N> no) { + return Vec128<double, N>{_mm_blendv_pd(no.raw, yes.raw, mask.raw)}; +} + +#endif // HWY_TARGET == HWY_SSSE3 + +// mask ? yes : 0 +template <typename T, size_t N> +HWY_API Vec128<T, N> IfThenElseZero(Mask128<T, N> mask, Vec128<T, N> yes) { + return yes & VecFromMask(Simd<T, N>(), mask); +} + +// mask ? 0 : no +template <typename T, size_t N> +HWY_API Vec128<T, N> IfThenZeroElse(Mask128<T, N> mask, Vec128<T, N> no) { + return AndNot(VecFromMask(Simd<T, N>(), mask), no); +} + +// ------------------------------ Mask logical + +template <typename T, size_t N> +HWY_API Mask128<T, N> Not(const Mask128<T, N> m) { + return MaskFromVec(Not(VecFromMask(Simd<T, N>(), m))); +} + +template <typename T, size_t N> +HWY_API Mask128<T, N> And(const Mask128<T, N> a, Mask128<T, N> b) { + const Simd<T, N> d; + return MaskFromVec(And(VecFromMask(d, a), VecFromMask(d, b))); +} + +template <typename T, size_t N> +HWY_API Mask128<T, N> AndNot(const Mask128<T, N> a, Mask128<T, N> b) { + const Simd<T, N> d; + return MaskFromVec(AndNot(VecFromMask(d, a), VecFromMask(d, b))); +} + +template <typename T, size_t N> +HWY_API Mask128<T, N> Or(const Mask128<T, N> a, Mask128<T, N> b) { + const Simd<T, N> d; + return MaskFromVec(Or(VecFromMask(d, a), VecFromMask(d, b))); +} + +template <typename T, size_t N> +HWY_API Mask128<T, N> Xor(const Mask128<T, N> a, Mask128<T, N> b) { + const Simd<T, N> d; + return MaskFromVec(Xor(VecFromMask(d, a), VecFromMask(d, b))); +} + +#endif // HWY_TARGET <= HWY_AVX3 + +// ================================================== SWIZZLE (1) + +// ------------------------------ Hard-coded shuffles + +// Notation: let Vec128<int32_t> have lanes 3,2,1,0 (0 is least-significant). +// Shuffle0321 rotates one lane to the right (the previous least-significant +// lane is now most-significant). These could also be implemented via +// CombineShiftRightBytes but the shuffle_abcd notation is more convenient. + +// Swap 32-bit halves in 64-bit halves. +template <size_t N> +HWY_API Vec128<uint32_t, N> Shuffle2301(const Vec128<uint32_t, N> v) { + static_assert(N == 2 || N == 4, "Does not make sense for N=1"); + return Vec128<uint32_t, N>{_mm_shuffle_epi32(v.raw, 0xB1)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> Shuffle2301(const Vec128<int32_t, N> v) { + static_assert(N == 2 || N == 4, "Does not make sense for N=1"); + return Vec128<int32_t, N>{_mm_shuffle_epi32(v.raw, 0xB1)}; +} +template <size_t N> +HWY_API Vec128<float, N> Shuffle2301(const Vec128<float, N> v) { + static_assert(N == 2 || N == 4, "Does not make sense for N=1"); + return Vec128<float, N>{_mm_shuffle_ps(v.raw, v.raw, 0xB1)}; +} + +// Swap 64-bit halves +HWY_API Vec128<uint32_t> Shuffle1032(const Vec128<uint32_t> v) { + return Vec128<uint32_t>{_mm_shuffle_epi32(v.raw, 0x4E)}; +} +HWY_API Vec128<int32_t> Shuffle1032(const Vec128<int32_t> v) { + return Vec128<int32_t>{_mm_shuffle_epi32(v.raw, 0x4E)}; +} +HWY_API Vec128<float> Shuffle1032(const Vec128<float> v) { + return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw, 0x4E)}; +} +HWY_API Vec128<uint64_t> Shuffle01(const Vec128<uint64_t> v) { + return Vec128<uint64_t>{_mm_shuffle_epi32(v.raw, 0x4E)}; +} +HWY_API Vec128<int64_t> Shuffle01(const Vec128<int64_t> v) { + return Vec128<int64_t>{_mm_shuffle_epi32(v.raw, 0x4E)}; +} +HWY_API Vec128<double> Shuffle01(const Vec128<double> v) { + return Vec128<double>{_mm_shuffle_pd(v.raw, v.raw, 1)}; +} + +// Rotate right 32 bits +HWY_API Vec128<uint32_t> Shuffle0321(const Vec128<uint32_t> v) { + return Vec128<uint32_t>{_mm_shuffle_epi32(v.raw, 0x39)}; +} +HWY_API Vec128<int32_t> Shuffle0321(const Vec128<int32_t> v) { + return Vec128<int32_t>{_mm_shuffle_epi32(v.raw, 0x39)}; +} +HWY_API Vec128<float> Shuffle0321(const Vec128<float> v) { + return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw, 0x39)}; +} +// Rotate left 32 bits +HWY_API Vec128<uint32_t> Shuffle2103(const Vec128<uint32_t> v) { + return Vec128<uint32_t>{_mm_shuffle_epi32(v.raw, 0x93)}; +} +HWY_API Vec128<int32_t> Shuffle2103(const Vec128<int32_t> v) { + return Vec128<int32_t>{_mm_shuffle_epi32(v.raw, 0x93)}; +} +HWY_API Vec128<float> Shuffle2103(const Vec128<float> v) { + return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw, 0x93)}; +} + +// Reverse +HWY_API Vec128<uint32_t> Shuffle0123(const Vec128<uint32_t> v) { + return Vec128<uint32_t>{_mm_shuffle_epi32(v.raw, 0x1B)}; +} +HWY_API Vec128<int32_t> Shuffle0123(const Vec128<int32_t> v) { + return Vec128<int32_t>{_mm_shuffle_epi32(v.raw, 0x1B)}; +} +HWY_API Vec128<float> Shuffle0123(const Vec128<float> v) { + return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw, 0x1B)}; +} + +// ================================================== COMPARE + +#if HWY_TARGET <= HWY_AVX3 + +// Comparisons set a mask bit to 1 if the condition is true, else 0. + +template <typename TFrom, size_t NFrom, typename TTo, size_t NTo> +HWY_API Mask128<TTo, NTo> RebindMask(Simd<TTo, NTo> /*tag*/, + Mask128<TFrom, NFrom> m) { + static_assert(sizeof(TFrom) == sizeof(TTo), "Must have same size"); + return Mask128<TTo, NTo>{m.raw}; +} + +namespace detail { + +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> TestBit(hwy::SizeTag<1> /*tag*/, const Vec128<T, N> v, + const Vec128<T, N> bit) { + return Mask128<T, N>{_mm_test_epi8_mask(v.raw, bit.raw)}; +} +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> TestBit(hwy::SizeTag<2> /*tag*/, const Vec128<T, N> v, + const Vec128<T, N> bit) { + return Mask128<T, N>{_mm_test_epi16_mask(v.raw, bit.raw)}; +} +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> TestBit(hwy::SizeTag<4> /*tag*/, const Vec128<T, N> v, + const Vec128<T, N> bit) { + return Mask128<T, N>{_mm_test_epi32_mask(v.raw, bit.raw)}; +} +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> TestBit(hwy::SizeTag<8> /*tag*/, const Vec128<T, N> v, + const Vec128<T, N> bit) { + return Mask128<T, N>{_mm_test_epi64_mask(v.raw, bit.raw)}; +} + +} // namespace detail + +template <typename T, size_t N> +HWY_API Mask128<T, N> TestBit(const Vec128<T, N> v, const Vec128<T, N> bit) { + static_assert(!hwy::IsFloat<T>(), "Only integer vectors supported"); + return detail::TestBit(hwy::SizeTag<sizeof(T)>(), v, bit); +} + +// ------------------------------ Equality + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 1)> +HWY_API Mask128<T, N> operator==(const Vec128<T, N> a, const Vec128<T, N> b) { + return Mask128<T, N>{_mm_cmpeq_epi8_mask(a.raw, b.raw)}; +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Mask128<T, N> operator==(const Vec128<T, N> a, const Vec128<T, N> b) { + return Mask128<T, N>{_mm_cmpeq_epi16_mask(a.raw, b.raw)}; +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Mask128<T, N> operator==(const Vec128<T, N> a, const Vec128<T, N> b) { + return Mask128<T, N>{_mm_cmpeq_epi32_mask(a.raw, b.raw)}; +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Mask128<T, N> operator==(const Vec128<T, N> a, const Vec128<T, N> b) { + return Mask128<T, N>{_mm_cmpeq_epi64_mask(a.raw, b.raw)}; +} + +template <size_t N> +HWY_API Mask128<float, N> operator==(Vec128<float, N> a, Vec128<float, N> b) { + return Mask128<float, N>{_mm_cmp_ps_mask(a.raw, b.raw, _CMP_EQ_OQ)}; +} + +template <size_t N> +HWY_API Mask128<double, N> operator==(Vec128<double, N> a, + Vec128<double, N> b) { + return Mask128<double, N>{_mm_cmp_pd_mask(a.raw, b.raw, _CMP_EQ_OQ)}; +} + +// ------------------------------ Inequality + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 1)> +HWY_API Mask128<T, N> operator!=(const Vec128<T, N> a, const Vec128<T, N> b) { + return Mask128<T, N>{_mm_cmpneq_epi8_mask(a.raw, b.raw)}; +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Mask128<T, N> operator!=(const Vec128<T, N> a, const Vec128<T, N> b) { + return Mask128<T, N>{_mm_cmpneq_epi16_mask(a.raw, b.raw)}; +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Mask128<T, N> operator!=(const Vec128<T, N> a, const Vec128<T, N> b) { + return Mask128<T, N>{_mm_cmpneq_epi32_mask(a.raw, b.raw)}; +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Mask128<T, N> operator!=(const Vec128<T, N> a, const Vec128<T, N> b) { + return Mask128<T, N>{_mm_cmpneq_epi64_mask(a.raw, b.raw)}; +} + +template <size_t N> +HWY_API Mask128<float, N> operator!=(Vec128<float, N> a, Vec128<float, N> b) { + return Mask128<float, N>{_mm_cmp_ps_mask(a.raw, b.raw, _CMP_NEQ_OQ)}; +} + +template <size_t N> +HWY_API Mask128<double, N> operator!=(Vec128<double, N> a, + Vec128<double, N> b) { + return Mask128<double, N>{_mm_cmp_pd_mask(a.raw, b.raw, _CMP_NEQ_OQ)}; +} + +// ------------------------------ Strict inequality + +// Signed/float < +template <size_t N> +HWY_API Mask128<int8_t, N> operator>(Vec128<int8_t, N> a, Vec128<int8_t, N> b) { + return Mask128<int8_t, N>{_mm_cmpgt_epi8_mask(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<int16_t, N> operator>(Vec128<int16_t, N> a, + Vec128<int16_t, N> b) { + return Mask128<int16_t, N>{_mm_cmpgt_epi16_mask(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<int32_t, N> operator>(Vec128<int32_t, N> a, + Vec128<int32_t, N> b) { + return Mask128<int32_t, N>{_mm_cmpgt_epi32_mask(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<int64_t, N> operator>(Vec128<int64_t, N> a, + Vec128<int64_t, N> b) { + return Mask128<int64_t, N>{_mm_cmpgt_epi64_mask(a.raw, b.raw)}; +} + +template <size_t N> +HWY_API Mask128<uint8_t, N> operator>(Vec128<uint8_t, N> a, + Vec128<uint8_t, N> b) { + return Mask128<uint8_t, N>{_mm_cmpgt_epu8_mask(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<uint16_t, N> operator>(Vec128<uint16_t, N> a, + Vec128<uint16_t, N> b) { + return Mask128<uint16_t, N>{_mm_cmpgt_epu16_mask(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<uint32_t, N> operator>(Vec128<uint32_t, N> a, + Vec128<uint32_t, N> b) { + return Mask128<uint32_t, N>{_mm_cmpgt_epu32_mask(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<uint64_t, N> operator>(Vec128<uint64_t, N> a, + Vec128<uint64_t, N> b) { + return Mask128<uint64_t, N>{_mm_cmpgt_epu64_mask(a.raw, b.raw)}; +} + +template <size_t N> +HWY_API Mask128<float, N> operator>(Vec128<float, N> a, Vec128<float, N> b) { + return Mask128<float, N>{_mm_cmp_ps_mask(a.raw, b.raw, _CMP_GT_OQ)}; +} +template <size_t N> +HWY_API Mask128<double, N> operator>(Vec128<double, N> a, Vec128<double, N> b) { + return Mask128<double, N>{_mm_cmp_pd_mask(a.raw, b.raw, _CMP_GT_OQ)}; +} + +// ------------------------------ Weak inequality + +template <size_t N> +HWY_API Mask128<float, N> operator>=(Vec128<float, N> a, Vec128<float, N> b) { + return Mask128<float, N>{_mm_cmp_ps_mask(a.raw, b.raw, _CMP_GE_OQ)}; +} +template <size_t N> +HWY_API Mask128<double, N> operator>=(Vec128<double, N> a, + Vec128<double, N> b) { + return Mask128<double, N>{_mm_cmp_pd_mask(a.raw, b.raw, _CMP_GE_OQ)}; +} + +// ------------------------------ Mask + +namespace detail { + +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> MaskFromVec(hwy::SizeTag<1> /*tag*/, + const Vec128<T, N> v) { + return Mask128<T, N>{_mm_movepi8_mask(v.raw)}; +} +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> MaskFromVec(hwy::SizeTag<2> /*tag*/, + const Vec128<T, N> v) { + return Mask128<T, N>{_mm_movepi16_mask(v.raw)}; +} +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> MaskFromVec(hwy::SizeTag<4> /*tag*/, + const Vec128<T, N> v) { + return Mask128<T, N>{_mm_movepi32_mask(v.raw)}; +} +template <typename T, size_t N> +HWY_INLINE Mask128<T, N> MaskFromVec(hwy::SizeTag<8> /*tag*/, + const Vec128<T, N> v) { + return Mask128<T, N>{_mm_movepi64_mask(v.raw)}; +} + +} // namespace detail + +template <typename T, size_t N> +HWY_API Mask128<T, N> MaskFromVec(const Vec128<T, N> v) { + return detail::MaskFromVec(hwy::SizeTag<sizeof(T)>(), v); +} +// There do not seem to be native floating-point versions of these instructions. +template <size_t N> +HWY_API Mask128<float, N> MaskFromVec(const Vec128<float, N> v) { + return Mask128<float, N>{MaskFromVec(BitCast(Simd<int32_t, N>(), v)).raw}; +} +template <size_t N> +HWY_API Mask128<double, N> MaskFromVec(const Vec128<double, N> v) { + return Mask128<double, N>{MaskFromVec(BitCast(Simd<int64_t, N>(), v)).raw}; +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 1)> +HWY_API Vec128<T, N> VecFromMask(const Mask128<T, N> v) { + return Vec128<T, N>{_mm_movm_epi8(v.raw)}; +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Vec128<T, N> VecFromMask(const Mask128<T, N> v) { + return Vec128<T, N>{_mm_movm_epi16(v.raw)}; +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T, N> VecFromMask(const Mask128<T, N> v) { + return Vec128<T, N>{_mm_movm_epi32(v.raw)}; +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec128<T, N> VecFromMask(const Mask128<T, N> v) { + return Vec128<T, N>{_mm_movm_epi64(v.raw)}; +} + +template <size_t N> +HWY_API Vec128<float, N> VecFromMask(const Mask128<float, N> v) { + return Vec128<float, N>{_mm_castsi128_ps(_mm_movm_epi32(v.raw))}; +} + +template <size_t N> +HWY_API Vec128<double, N> VecFromMask(const Mask128<double, N> v) { + return Vec128<double, N>{_mm_castsi128_pd(_mm_movm_epi64(v.raw))}; +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> VecFromMask(Simd<T, N> /* tag */, const Mask128<T, N> v) { + return VecFromMask(v); +} + +#else // AVX2 or below + +// Comparisons fill a lane with 1-bits if the condition is true, else 0. + +template <typename TFrom, typename TTo, size_t N> +HWY_API Mask128<TTo, N> RebindMask(Simd<TTo, N> /*tag*/, Mask128<TFrom, N> m) { + static_assert(sizeof(TFrom) == sizeof(TTo), "Must have same size"); + const Simd<TFrom, N> d; + return MaskFromVec(BitCast(Simd<TTo, N>(), VecFromMask(d, m))); +} + +template <typename T, size_t N> +HWY_API Mask128<T, N> TestBit(Vec128<T, N> v, Vec128<T, N> bit) { + static_assert(!hwy::IsFloat<T>(), "Only integer vectors supported"); + return (v & bit) == bit; +} + +// ------------------------------ Equality + +// Unsigned +template <size_t N> +HWY_API Mask128<uint8_t, N> operator==(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Mask128<uint8_t, N>{_mm_cmpeq_epi8(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<uint16_t, N> operator==(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Mask128<uint16_t, N>{_mm_cmpeq_epi16(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<uint32_t, N> operator==(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { + return Mask128<uint32_t, N>{_mm_cmpeq_epi32(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<uint64_t, N> operator==(const Vec128<uint64_t, N> a, + const Vec128<uint64_t, N> b) { +#if HWY_TARGET == HWY_SSSE3 + const Simd<uint32_t, N * 2> d32; + const Simd<uint64_t, N> d64; + const auto cmp32 = VecFromMask(d32, Eq(BitCast(d32, a), BitCast(d32, b))); + const auto cmp64 = cmp32 & Shuffle2301(cmp32); + return MaskFromVec(BitCast(d64, cmp64)); +#else + return Mask128<uint64_t, N>{_mm_cmpeq_epi64(a.raw, b.raw)}; +#endif +} + +// Signed +template <size_t N> +HWY_API Mask128<int8_t, N> operator==(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Mask128<int8_t, N>{_mm_cmpeq_epi8(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<int16_t, N> operator==(Vec128<int16_t, N> a, + Vec128<int16_t, N> b) { + return Mask128<int16_t, N>{_mm_cmpeq_epi16(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<int32_t, N> operator==(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + return Mask128<int32_t, N>{_mm_cmpeq_epi32(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<int64_t, N> operator==(const Vec128<int64_t, N> a, + const Vec128<int64_t, N> b) { + // Same as signed ==; avoid duplicating the SSSE3 version. + const Simd<uint64_t, N> du; + return RebindMask(Simd<int64_t, N>(), BitCast(du, a) == BitCast(du, b)); +} + +// Float +template <size_t N> +HWY_API Mask128<float, N> operator==(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Mask128<float, N>{_mm_cmpeq_ps(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<double, N> operator==(const Vec128<double, N> a, + const Vec128<double, N> b) { + return Mask128<double, N>{_mm_cmpeq_pd(a.raw, b.raw)}; +} + +// ------------------------------ Inequality + +template <typename T, size_t N, HWY_IF_NOT_FLOAT(T)> +HWY_API Mask128<T, N> operator!=(const Vec128<T, N> a, const Vec128<T, N> b) { + return Not(a == b); +} + +template <size_t N> +HWY_API Mask128<float, N> operator!=(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Mask128<float, N>{_mm_cmpneq_ps(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<double, N> operator!=(const Vec128<double, N> a, + const Vec128<double, N> b) { + return Mask128<double, N>{_mm_cmpneq_pd(a.raw, b.raw)}; +} + +// ------------------------------ Strict inequality + +// Signed/float < +template <size_t N> +HWY_API Mask128<int8_t, N> operator>(Vec128<int8_t, N> a, Vec128<int8_t, N> b) { + return Mask128<int8_t, N>{_mm_cmpgt_epi8(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<int16_t, N> operator>(Vec128<int16_t, N> a, + Vec128<int16_t, N> b) { + return Mask128<int16_t, N>{_mm_cmpgt_epi16(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<int32_t, N> operator>(Vec128<int32_t, N> a, + Vec128<int32_t, N> b) { + return Mask128<int32_t, N>{_mm_cmpgt_epi32(a.raw, b.raw)}; +} + +template <typename T, size_t N, HWY_IF_UNSIGNED(T)> +HWY_API Mask128<T, N> operator>(Vec128<T, N> a, Vec128<T, N> b) { + const Simd<T, N> du; + const RebindToSigned<decltype(du)> di; + const Vec128<T, N> msb = Set(du, (LimitsMax<T>() >> 1) + 1); + return RebindMask(du, BitCast(di, Xor(a, msb)) > BitCast(di, Xor(b, msb))); +} + +template <size_t N> +HWY_API Mask128<float, N> operator>(Vec128<float, N> a, Vec128<float, N> b) { + return Mask128<float, N>{_mm_cmpgt_ps(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<double, N> operator>(Vec128<double, N> a, Vec128<double, N> b) { + return Mask128<double, N>{_mm_cmpgt_pd(a.raw, b.raw)}; +} + +template <size_t N> +HWY_API Mask128<int64_t, N> operator>(const Vec128<int64_t, N> a, + const Vec128<int64_t, N> b) { +#if HWY_TARGET == HWY_SSSE3 + // If the upper half is less than or greater, this is the answer. + const __m128i m_gt = _mm_cmpgt_epi32(a.raw, b.raw); + + // Otherwise, the lower half decides. + const __m128i m_eq = _mm_cmpeq_epi32(a.raw, b.raw); + const __m128i lo_in_hi = _mm_shuffle_epi32(m_gt, _MM_SHUFFLE(2, 2, 0, 0)); + const __m128i lo_gt = _mm_and_si128(m_eq, lo_in_hi); + + const __m128i gt = _mm_or_si128(lo_gt, m_gt); + // Copy result in upper 32 bits to lower 32 bits. + return Mask128<int64_t, N>{_mm_shuffle_epi32(gt, _MM_SHUFFLE(3, 3, 1, 1))}; +#else + return Mask128<int64_t, N>{_mm_cmpgt_epi64(a.raw, b.raw)}; // SSE4.2 +#endif +} + +// ------------------------------ Weak inequality + +template <size_t N> +HWY_API Mask128<float, N> operator>=(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Mask128<float, N>{_mm_cmpge_ps(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Mask128<double, N> operator>=(const Vec128<double, N> a, + const Vec128<double, N> b) { + return Mask128<double, N>{_mm_cmpge_pd(a.raw, b.raw)}; +} + +#endif // HWY_TARGET <= HWY_AVX3 + +// ------------------------------ Reversed comparisons + +template <typename T, size_t N> +HWY_API Mask128<T, N> operator<(Vec128<T, N> a, Vec128<T, N> b) { + return b > a; +} + +template <typename T, size_t N> +HWY_API Mask128<T, N> operator<=(Vec128<T, N> a, Vec128<T, N> b) { + return b >= a; +} + +// ------------------------------ FirstN (Iota, Lt) + +template <typename T, size_t N, HWY_IF_LE128(T, N)> +HWY_API Mask128<T, N> FirstN(const Simd<T, N> d, size_t num) { +#if HWY_TARGET <= HWY_AVX3 + (void)d; + const uint64_t all = (1ull << N) - 1; + // BZHI only looks at the lower 8 bits of num! + const uint64_t bits = (num > 255) ? all : _bzhi_u64(all, num); + return Mask128<T, N>::FromBits(bits); +#else + const RebindToSigned<decltype(d)> di; // Signed comparisons are cheaper. + return RebindMask(d, Iota(di, 0) < Set(di, static_cast<MakeSigned<T>>(num))); +#endif +} + +template <class D> +using MFromD = decltype(FirstN(D(), 0)); + +// ================================================== MEMORY (1) + +// Clang static analysis claims the memory immediately after a partial vector +// store is uninitialized, and also flags the input to partial loads (at least +// for loadl_pd) as "garbage". This is a false alarm because msan does not +// raise errors. We work around this by using CopyBytes instead of intrinsics, +// but only for the analyzer to avoid potentially bad code generation. +// Unfortunately __clang_analyzer__ was not defined for clang-tidy prior to v7. +#ifndef HWY_SAFE_PARTIAL_LOAD_STORE +#if defined(__clang_analyzer__) || \ + (HWY_COMPILER_CLANG != 0 && HWY_COMPILER_CLANG < 700) +#define HWY_SAFE_PARTIAL_LOAD_STORE 1 +#else +#define HWY_SAFE_PARTIAL_LOAD_STORE 0 +#endif +#endif // HWY_SAFE_PARTIAL_LOAD_STORE + +// ------------------------------ Load + +template <typename T> +HWY_API Vec128<T> Load(Full128<T> /* tag */, const T* HWY_RESTRICT aligned) { + return Vec128<T>{_mm_load_si128(reinterpret_cast<const __m128i*>(aligned))}; +} +HWY_API Vec128<float> Load(Full128<float> /* tag */, + const float* HWY_RESTRICT aligned) { + return Vec128<float>{_mm_load_ps(aligned)}; +} +HWY_API Vec128<double> Load(Full128<double> /* tag */, + const double* HWY_RESTRICT aligned) { + return Vec128<double>{_mm_load_pd(aligned)}; +} + +template <typename T> +HWY_API Vec128<T> LoadU(Full128<T> /* tag */, const T* HWY_RESTRICT p) { + return Vec128<T>{_mm_loadu_si128(reinterpret_cast<const __m128i*>(p))}; +} +HWY_API Vec128<float> LoadU(Full128<float> /* tag */, + const float* HWY_RESTRICT p) { + return Vec128<float>{_mm_loadu_ps(p)}; +} +HWY_API Vec128<double> LoadU(Full128<double> /* tag */, + const double* HWY_RESTRICT p) { + return Vec128<double>{_mm_loadu_pd(p)}; +} + +template <typename T> +HWY_API Vec128<T, 8 / sizeof(T)> Load(Simd<T, 8 / sizeof(T)> /* tag */, + const T* HWY_RESTRICT p) { +#if HWY_SAFE_PARTIAL_LOAD_STORE + __m128i v = _mm_setzero_si128(); + CopyBytes<8>(p, &v); + return Vec128<T, 8 / sizeof(T)>{v}; +#else + return Vec128<T, 8 / sizeof(T)>{ + _mm_loadl_epi64(reinterpret_cast<const __m128i*>(p))}; +#endif +} + +HWY_API Vec128<float, 2> Load(Simd<float, 2> /* tag */, + const float* HWY_RESTRICT p) { +#if HWY_SAFE_PARTIAL_LOAD_STORE + __m128 v = _mm_setzero_ps(); + CopyBytes<8>(p, &v); + return Vec128<float, 2>{v}; +#else + const __m128 hi = _mm_setzero_ps(); + return Vec128<float, 2>{_mm_loadl_pi(hi, reinterpret_cast<const __m64*>(p))}; +#endif +} + +HWY_API Vec128<double, 1> Load(Simd<double, 1> /* tag */, + const double* HWY_RESTRICT p) { +#if HWY_SAFE_PARTIAL_LOAD_STORE + __m128d v = _mm_setzero_pd(); + CopyBytes<8>(p, &v); + return Vec128<double, 1>{v}; +#else + return Vec128<double, 1>{_mm_load_sd(p)}; +#endif +} + +HWY_API Vec128<float, 1> Load(Simd<float, 1> /* tag */, + const float* HWY_RESTRICT p) { +#if HWY_SAFE_PARTIAL_LOAD_STORE + __m128 v = _mm_setzero_ps(); + CopyBytes<4>(p, &v); + return Vec128<float, 1>{v}; +#else + return Vec128<float, 1>{_mm_load_ss(p)}; +#endif +} + +// Any <= 32 bit except <float, 1> +template <typename T, size_t N, HWY_IF_LE32(T, N)> +HWY_API Vec128<T, N> Load(Simd<T, N> /* tag */, const T* HWY_RESTRICT p) { + constexpr size_t kSize = sizeof(T) * N; +#if HWY_SAFE_PARTIAL_LOAD_STORE + __m128 v = _mm_setzero_ps(); + CopyBytes<kSize>(p, &v); + return Vec128<T, N>{v}; +#else + int32_t bits; + CopyBytes<kSize>(p, &bits); + return Vec128<T, N>{_mm_cvtsi32_si128(bits)}; +#endif +} + +// For < 128 bit, LoadU == Load. +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_API Vec128<T, N> LoadU(Simd<T, N> d, const T* HWY_RESTRICT p) { + return Load(d, p); +} + +// 128-bit SIMD => nothing to duplicate, same as an unaligned load. +template <typename T, size_t N, HWY_IF_LE128(T, N)> +HWY_API Vec128<T, N> LoadDup128(Simd<T, N> d, const T* HWY_RESTRICT p) { + return LoadU(d, p); +} + +// Returns a vector with lane i=[0, N) set to "first" + i. +template <typename T, size_t N, typename T2, HWY_IF_LE128(T, N)> +HWY_API Vec128<T, N> Iota(const Simd<T, N> d, const T2 first) { + HWY_ALIGN T lanes[16 / sizeof(T)]; + for (size_t i = 0; i < 16 / sizeof(T); ++i) { + lanes[i] = static_cast<T>(first + static_cast<T2>(i)); + } + return Load(d, lanes); +} + +// ------------------------------ MaskedLoad + +#if HWY_TARGET <= HWY_AVX3 + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T, N> MaskedLoad(Mask128<T, N> m, Simd<T, N> /* tag */, + const T* HWY_RESTRICT aligned) { + return Vec128<T, N>{_mm_maskz_load_epi32(m.raw, aligned)}; +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec128<T, N> MaskedLoad(Mask128<T, N> m, Simd<T, N> /* tag */, + const T* HWY_RESTRICT aligned) { + return Vec128<T, N>{_mm_maskz_load_epi64(m.raw, aligned)}; +} + +template <size_t N> +HWY_API Vec128<float, N> MaskedLoad(Mask128<float, N> m, + Simd<float, N> /* tag */, + const float* HWY_RESTRICT aligned) { + return Vec128<float, N>{_mm_maskz_load_ps(m.raw, aligned)}; +} + +template <size_t N> +HWY_API Vec128<double, N> MaskedLoad(Mask128<double, N> m, + Simd<double, N> /* tag */, + const double* HWY_RESTRICT aligned) { + return Vec128<double, N>{_mm_maskz_load_pd(m.raw, aligned)}; +} + +// There is no load_epi8/16, so use loadu instead. +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 1)> +HWY_API Vec128<T, N> MaskedLoad(Mask128<T, N> m, Simd<T, N> /* tag */, + const T* HWY_RESTRICT aligned) { + return Vec128<T, N>{_mm_maskz_loadu_epi8(m.raw, aligned)}; +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Vec128<T, N> MaskedLoad(Mask128<T, N> m, Simd<T, N> /* tag */, + const T* HWY_RESTRICT aligned) { + return Vec128<T, N>{_mm_maskz_loadu_epi16(m.raw, aligned)}; +} + +#elif HWY_TARGET == HWY_AVX2 + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T, N> MaskedLoad(Mask128<T, N> m, Simd<T, N> /* tag */, + const T* HWY_RESTRICT aligned) { + auto aligned_p = reinterpret_cast<const int*>(aligned); // NOLINT + return Vec128<T, N>{_mm_maskload_epi32(aligned_p, m.raw)}; +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec128<T, N> MaskedLoad(Mask128<T, N> m, Simd<T, N> /* tag */, + const T* HWY_RESTRICT aligned) { + auto aligned_p = reinterpret_cast<const long long*>(aligned); // NOLINT + return Vec128<T, N>{_mm_maskload_epi64(aligned_p, m.raw)}; +} + +template <size_t N> +HWY_API Vec128<float, N> MaskedLoad(Mask128<float, N> m, Simd<float, N> d, + const float* HWY_RESTRICT aligned) { + const Vec128<int32_t, N> mi = + BitCast(RebindToSigned<decltype(d)>(), VecFromMask(d, m)); + return Vec128<float, N>{_mm_maskload_ps(aligned, mi.raw)}; +} + +template <size_t N> +HWY_API Vec128<double, N> MaskedLoad(Mask128<double, N> m, Simd<double, N> d, + const double* HWY_RESTRICT aligned) { + const Vec128<int64_t, N> mi = + BitCast(RebindToSigned<decltype(d)>(), VecFromMask(d, m)); + return Vec128<double, N>{_mm_maskload_pd(aligned, mi.raw)}; +} + +// There is no maskload_epi8/16, so blend instead. +template <typename T, size_t N, hwy::EnableIf<sizeof(T) <= 2>* = nullptr> +HWY_API Vec128<T, N> MaskedLoad(Mask128<T, N> m, Simd<T, N> d, + const T* HWY_RESTRICT aligned) { + return IfThenElseZero(m, Load(d, aligned)); +} + +#else // <= SSE4 + +// Avoid maskmov* - its nontemporal 'hint' causes it to bypass caches (slow). +template <typename T, size_t N> +HWY_API Vec128<T, N> MaskedLoad(Mask128<T, N> m, Simd<T, N> d, + const T* HWY_RESTRICT aligned) { + return IfThenElseZero(m, Load(d, aligned)); +} + +#endif + +// ------------------------------ Store + +template <typename T> +HWY_API void Store(Vec128<T> v, Full128<T> /* tag */, T* HWY_RESTRICT aligned) { + _mm_store_si128(reinterpret_cast<__m128i*>(aligned), v.raw); +} +HWY_API void Store(const Vec128<float> v, Full128<float> /* tag */, + float* HWY_RESTRICT aligned) { + _mm_store_ps(aligned, v.raw); +} +HWY_API void Store(const Vec128<double> v, Full128<double> /* tag */, + double* HWY_RESTRICT aligned) { + _mm_store_pd(aligned, v.raw); +} + +template <typename T> +HWY_API void StoreU(Vec128<T> v, Full128<T> /* tag */, T* HWY_RESTRICT p) { + _mm_storeu_si128(reinterpret_cast<__m128i*>(p), v.raw); +} +HWY_API void StoreU(const Vec128<float> v, Full128<float> /* tag */, + float* HWY_RESTRICT p) { + _mm_storeu_ps(p, v.raw); +} +HWY_API void StoreU(const Vec128<double> v, Full128<double> /* tag */, + double* HWY_RESTRICT p) { + _mm_storeu_pd(p, v.raw); +} + +template <typename T> +HWY_API void Store(Vec128<T, 8 / sizeof(T)> v, Simd<T, 8 / sizeof(T)> /* tag */, + T* HWY_RESTRICT p) { +#if HWY_SAFE_PARTIAL_LOAD_STORE + CopyBytes<8>(&v, p); +#else + _mm_storel_epi64(reinterpret_cast<__m128i*>(p), v.raw); +#endif +} +HWY_API void Store(const Vec128<float, 2> v, Simd<float, 2> /* tag */, + float* HWY_RESTRICT p) { +#if HWY_SAFE_PARTIAL_LOAD_STORE + CopyBytes<8>(&v, p); +#else + _mm_storel_pi(reinterpret_cast<__m64*>(p), v.raw); +#endif +} +HWY_API void Store(const Vec128<double, 1> v, Simd<double, 1> /* tag */, + double* HWY_RESTRICT p) { +#if HWY_SAFE_PARTIAL_LOAD_STORE + CopyBytes<8>(&v, p); +#else + _mm_storel_pd(p, v.raw); +#endif +} + +// Any <= 32 bit except <float, 1> +template <typename T, size_t N, HWY_IF_LE32(T, N)> +HWY_API void Store(Vec128<T, N> v, Simd<T, N> /* tag */, T* HWY_RESTRICT p) { + CopyBytes<sizeof(T) * N>(&v, p); +} +HWY_API void Store(const Vec128<float, 1> v, Simd<float, 1> /* tag */, + float* HWY_RESTRICT p) { +#if HWY_SAFE_PARTIAL_LOAD_STORE + CopyBytes<4>(&v, p); +#else + _mm_store_ss(p, v.raw); +#endif +} + +// For < 128 bit, StoreU == Store. +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_API void StoreU(const Vec128<T, N> v, Simd<T, N> d, T* HWY_RESTRICT p) { + Store(v, d, p); +} + +// ================================================== ARITHMETIC + +// ------------------------------ Addition + +// Unsigned +template <size_t N> +HWY_API Vec128<uint8_t, N> operator+(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{_mm_add_epi8(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> operator+(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{_mm_add_epi16(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> operator+(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { + return Vec128<uint32_t, N>{_mm_add_epi32(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint64_t, N> operator+(const Vec128<uint64_t, N> a, + const Vec128<uint64_t, N> b) { + return Vec128<uint64_t, N>{_mm_add_epi64(a.raw, b.raw)}; +} + +// Signed +template <size_t N> +HWY_API Vec128<int8_t, N> operator+(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Vec128<int8_t, N>{_mm_add_epi8(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> operator+(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{_mm_add_epi16(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> operator+(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + return Vec128<int32_t, N>{_mm_add_epi32(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int64_t, N> operator+(const Vec128<int64_t, N> a, + const Vec128<int64_t, N> b) { + return Vec128<int64_t, N>{_mm_add_epi64(a.raw, b.raw)}; +} + +// Float +template <size_t N> +HWY_API Vec128<float, N> operator+(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Vec128<float, N>{_mm_add_ps(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<double, N> operator+(const Vec128<double, N> a, + const Vec128<double, N> b) { + return Vec128<double, N>{_mm_add_pd(a.raw, b.raw)}; +} + +// ------------------------------ Subtraction + +// Unsigned +template <size_t N> +HWY_API Vec128<uint8_t, N> operator-(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{_mm_sub_epi8(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> operator-(Vec128<uint16_t, N> a, + Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{_mm_sub_epi16(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> operator-(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { + return Vec128<uint32_t, N>{_mm_sub_epi32(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint64_t, N> operator-(const Vec128<uint64_t, N> a, + const Vec128<uint64_t, N> b) { + return Vec128<uint64_t, N>{_mm_sub_epi64(a.raw, b.raw)}; +} + +// Signed +template <size_t N> +HWY_API Vec128<int8_t, N> operator-(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Vec128<int8_t, N>{_mm_sub_epi8(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> operator-(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{_mm_sub_epi16(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> operator-(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + return Vec128<int32_t, N>{_mm_sub_epi32(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int64_t, N> operator-(const Vec128<int64_t, N> a, + const Vec128<int64_t, N> b) { + return Vec128<int64_t, N>{_mm_sub_epi64(a.raw, b.raw)}; +} + +// Float +template <size_t N> +HWY_API Vec128<float, N> operator-(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Vec128<float, N>{_mm_sub_ps(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<double, N> operator-(const Vec128<double, N> a, + const Vec128<double, N> b) { + return Vec128<double, N>{_mm_sub_pd(a.raw, b.raw)}; +} + +// ------------------------------ Saturating addition + +// Returns a + b clamped to the destination range. + +// Unsigned +template <size_t N> +HWY_API Vec128<uint8_t, N> SaturatedAdd(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{_mm_adds_epu8(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> SaturatedAdd(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{_mm_adds_epu16(a.raw, b.raw)}; +} + +// Signed +template <size_t N> +HWY_API Vec128<int8_t, N> SaturatedAdd(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Vec128<int8_t, N>{_mm_adds_epi8(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> SaturatedAdd(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{_mm_adds_epi16(a.raw, b.raw)}; +} + +// ------------------------------ Saturating subtraction + +// Returns a - b clamped to the destination range. + +// Unsigned +template <size_t N> +HWY_API Vec128<uint8_t, N> SaturatedSub(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{_mm_subs_epu8(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> SaturatedSub(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{_mm_subs_epu16(a.raw, b.raw)}; +} + +// Signed +template <size_t N> +HWY_API Vec128<int8_t, N> SaturatedSub(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Vec128<int8_t, N>{_mm_subs_epi8(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> SaturatedSub(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{_mm_subs_epi16(a.raw, b.raw)}; +} + +// ------------------------------ AverageRound + +// Returns (a + b + 1) / 2 + +// Unsigned +template <size_t N> +HWY_API Vec128<uint8_t, N> AverageRound(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{_mm_avg_epu8(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> AverageRound(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{_mm_avg_epu16(a.raw, b.raw)}; +} + +// ------------------------------ Integer multiplication + +template <size_t N> +HWY_API Vec128<uint16_t, N> operator*(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{_mm_mullo_epi16(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> operator*(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{_mm_mullo_epi16(a.raw, b.raw)}; +} + +// Returns the upper 16 bits of a * b in each lane. +template <size_t N> +HWY_API Vec128<uint16_t, N> MulHigh(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{_mm_mulhi_epu16(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int16_t, N> MulHigh(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{_mm_mulhi_epi16(a.raw, b.raw)}; +} + +// Multiplies even lanes (0, 2 ..) and places the double-wide result into +// even and the upper half into its odd neighbor lane. +template <size_t N> +HWY_API Vec128<uint64_t, (N + 1) / 2> MulEven(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { + return Vec128<uint64_t, (N + 1) / 2>{_mm_mul_epu32(a.raw, b.raw)}; +} + +#if HWY_TARGET == HWY_SSSE3 + +template <size_t N, HWY_IF_LE64(int32_t, N)> // N=1 or 2 +HWY_API Vec128<int64_t, (N + 1) / 2> MulEven(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + return Set(Simd<int64_t, (N + 1) / 2>(), int64_t(GetLane(a)) * GetLane(b)); +} +HWY_API Vec128<int64_t> MulEven(const Vec128<int32_t> a, + const Vec128<int32_t> b) { + alignas(16) int32_t a_lanes[4]; + alignas(16) int32_t b_lanes[4]; + const Full128<int32_t> di32; + Store(a, di32, a_lanes); + Store(b, di32, b_lanes); + alignas(16) int64_t mul[2]; + mul[0] = int64_t(a_lanes[0]) * b_lanes[0]; + mul[1] = int64_t(a_lanes[2]) * b_lanes[2]; + return Load(Full128<int64_t>(), mul); +} + +#else // HWY_TARGET == HWY_SSSE3 + +template <size_t N> +HWY_API Vec128<int64_t, (N + 1) / 2> MulEven(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + return Vec128<int64_t, (N + 1) / 2>{_mm_mul_epi32(a.raw, b.raw)}; +} + +#endif // HWY_TARGET == HWY_SSSE3 + +template <size_t N> +HWY_API Vec128<uint32_t, N> operator*(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { +#if HWY_TARGET == HWY_SSSE3 + // Not as inefficient as it looks: _mm_mullo_epi32 has 10 cycle latency. + // 64-bit right shift would also work but also needs port 5, so no benefit. + // Notation: x=don't care, z=0. + const __m128i a_x3x1 = _mm_shuffle_epi32(a.raw, _MM_SHUFFLE(3, 3, 1, 1)); + const auto mullo_x2x0 = MulEven(a, b); + const __m128i b_x3x1 = _mm_shuffle_epi32(b.raw, _MM_SHUFFLE(3, 3, 1, 1)); + const auto mullo_x3x1 = + MulEven(Vec128<uint32_t, N>{a_x3x1}, Vec128<uint32_t, N>{b_x3x1}); + // We could _mm_slli_epi64 by 32 to get 3z1z and OR with z2z0, but generating + // the latter requires one more instruction or a constant. + const __m128i mul_20 = + _mm_shuffle_epi32(mullo_x2x0.raw, _MM_SHUFFLE(2, 0, 2, 0)); + const __m128i mul_31 = + _mm_shuffle_epi32(mullo_x3x1.raw, _MM_SHUFFLE(2, 0, 2, 0)); + return Vec128<uint32_t, N>{_mm_unpacklo_epi32(mul_20, mul_31)}; +#else + return Vec128<uint32_t, N>{_mm_mullo_epi32(a.raw, b.raw)}; +#endif +} + +template <size_t N> +HWY_API Vec128<int32_t, N> operator*(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + // Same as unsigned; avoid duplicating the SSSE3 code. + const Simd<uint32_t, N> du; + return BitCast(Simd<int32_t, N>(), BitCast(du, a) * BitCast(du, b)); +} + +// ------------------------------ ShiftLeft + +template <int kBits, size_t N> +HWY_API Vec128<uint16_t, N> ShiftLeft(const Vec128<uint16_t, N> v) { + return Vec128<uint16_t, N>{_mm_slli_epi16(v.raw, kBits)}; +} + +template <int kBits, size_t N> +HWY_API Vec128<uint32_t, N> ShiftLeft(const Vec128<uint32_t, N> v) { + return Vec128<uint32_t, N>{_mm_slli_epi32(v.raw, kBits)}; +} + +template <int kBits, size_t N> +HWY_API Vec128<uint64_t, N> ShiftLeft(const Vec128<uint64_t, N> v) { + return Vec128<uint64_t, N>{_mm_slli_epi64(v.raw, kBits)}; +} + +template <int kBits, size_t N> +HWY_API Vec128<int16_t, N> ShiftLeft(const Vec128<int16_t, N> v) { + return Vec128<int16_t, N>{_mm_slli_epi16(v.raw, kBits)}; +} +template <int kBits, size_t N> +HWY_API Vec128<int32_t, N> ShiftLeft(const Vec128<int32_t, N> v) { + return Vec128<int32_t, N>{_mm_slli_epi32(v.raw, kBits)}; +} +template <int kBits, size_t N> +HWY_API Vec128<int64_t, N> ShiftLeft(const Vec128<int64_t, N> v) { + return Vec128<int64_t, N>{_mm_slli_epi64(v.raw, kBits)}; +} + +template <int kBits, typename T, size_t N, HWY_IF_LANE_SIZE(T, 1)> +HWY_API Vec128<T, N> ShiftLeft(const Vec128<T, N> v) { + const Simd<T, N> d8; + // Use raw instead of BitCast to support N=1. + const Vec128<T, N> shifted{ShiftLeft<kBits>(Vec128<MakeWide<T>>{v.raw}).raw}; + return kBits == 1 + ? (v + v) + : (shifted & Set(d8, static_cast<T>((0xFF << kBits) & 0xFF))); +} + +// ------------------------------ ShiftRight + +template <int kBits, size_t N> +HWY_API Vec128<uint16_t, N> ShiftRight(const Vec128<uint16_t, N> v) { + return Vec128<uint16_t, N>{_mm_srli_epi16(v.raw, kBits)}; +} +template <int kBits, size_t N> +HWY_API Vec128<uint32_t, N> ShiftRight(const Vec128<uint32_t, N> v) { + return Vec128<uint32_t, N>{_mm_srli_epi32(v.raw, kBits)}; +} +template <int kBits, size_t N> +HWY_API Vec128<uint64_t, N> ShiftRight(const Vec128<uint64_t, N> v) { + return Vec128<uint64_t, N>{_mm_srli_epi64(v.raw, kBits)}; +} + +template <int kBits, size_t N> +HWY_API Vec128<uint8_t, N> ShiftRight(const Vec128<uint8_t, N> v) { + const Simd<uint8_t, N> d8; + // Use raw instead of BitCast to support N=1. + const Vec128<uint8_t, N> shifted{ + ShiftRight<kBits>(Vec128<uint16_t>{v.raw}).raw}; + return shifted & Set(d8, 0xFF >> kBits); +} + +template <int kBits, size_t N> +HWY_API Vec128<int16_t, N> ShiftRight(const Vec128<int16_t, N> v) { + return Vec128<int16_t, N>{_mm_srai_epi16(v.raw, kBits)}; +} +template <int kBits, size_t N> +HWY_API Vec128<int32_t, N> ShiftRight(const Vec128<int32_t, N> v) { + return Vec128<int32_t, N>{_mm_srai_epi32(v.raw, kBits)}; +} + +template <int kBits, size_t N> +HWY_API Vec128<int8_t, N> ShiftRight(const Vec128<int8_t, N> v) { + const Simd<int8_t, N> di; + const Simd<uint8_t, N> du; + const auto shifted = BitCast(di, ShiftRight<kBits>(BitCast(du, v))); + const auto shifted_sign = BitCast(di, Set(du, 0x80 >> kBits)); + return (shifted ^ shifted_sign) - shifted_sign; +} + +// i64 is implemented after BroadcastSignBit. + +// ------------------------------ RotateRight (ShiftRight, Or) + +template <int kBits, size_t N> +HWY_API Vec128<uint32_t, N> RotateRight(const Vec128<uint32_t, N> v) { + static_assert(0 <= kBits && kBits < 32, "Invalid shift count"); +#if HWY_TARGET <= HWY_AVX3 + return Vec128<uint32_t, N>{_mm_ror_epi32(v.raw, kBits)}; +#else + if (kBits == 0) return v; + return Or(ShiftRight<kBits>(v), ShiftLeft<HWY_MIN(31, 32 - kBits)>(v)); +#endif +} + +template <int kBits, size_t N> +HWY_API Vec128<uint64_t, N> RotateRight(const Vec128<uint64_t, N> v) { + static_assert(0 <= kBits && kBits < 64, "Invalid shift count"); +#if HWY_TARGET <= HWY_AVX3 + return Vec128<uint64_t, N>{_mm_ror_epi64(v.raw, kBits)}; +#else + if (kBits == 0) return v; + return Or(ShiftRight<kBits>(v), ShiftLeft<HWY_MIN(63, 64 - kBits)>(v)); +#endif +} + +// ------------------------------ BroadcastSignBit (ShiftRight, compare, mask) + +template <size_t N> +HWY_API Vec128<int8_t, N> BroadcastSignBit(const Vec128<int8_t, N> v) { + return VecFromMask(v < Zero(Simd<int8_t, N>())); +} + +template <size_t N> +HWY_API Vec128<int16_t, N> BroadcastSignBit(const Vec128<int16_t, N> v) { + return ShiftRight<15>(v); +} + +template <size_t N> +HWY_API Vec128<int32_t, N> BroadcastSignBit(const Vec128<int32_t, N> v) { + return ShiftRight<31>(v); +} + +template <size_t N> +HWY_API Vec128<int64_t, N> BroadcastSignBit(const Vec128<int64_t, N> v) { +#if HWY_TARGET <= HWY_AVX3 + return Vec128<int64_t, N>{_mm_srai_epi64(v.raw, 63)}; +#elif HWY_TARGET == HWY_AVX2 || HWY_TARGET == HWY_SSE4 + return VecFromMask(v < Zero(Simd<int64_t, N>())); +#else + // Efficient Lt() requires SSE4.2 and BLENDVPD requires SSE4.1. 32-bit shift + // avoids generating a zero. + const Simd<int32_t, N * 2> d32; + const auto sign = ShiftRight<31>(BitCast(d32, v)); + return Vec128<int64_t, N>{ + _mm_shuffle_epi32(sign.raw, _MM_SHUFFLE(3, 3, 1, 1))}; +#endif +} + +template <size_t N> +HWY_API Vec128<int64_t, N> Abs(const Vec128<int64_t, N> v) { +#if HWY_TARGET <= HWY_AVX3 + return Vec128<int64_t, N>{_mm_abs_epi64(v.raw)}; +#else + const auto zero = Zero(Simd<int64_t, N>()); + return IfThenElse(MaskFromVec(BroadcastSignBit(v)), zero - v, v); +#endif +} + +template <int kBits, size_t N> +HWY_API Vec128<int64_t, N> ShiftRight(const Vec128<int64_t, N> v) { +#if HWY_TARGET <= HWY_AVX3 + return Vec128<int64_t, N>{_mm_srai_epi64(v.raw, kBits)}; +#else + const Simd<int64_t, N> di; + const Simd<uint64_t, N> du; + const auto right = BitCast(di, ShiftRight<kBits>(BitCast(du, v))); + const auto sign = ShiftLeft<64 - kBits>(BroadcastSignBit(v)); + return right | sign; +#endif +} + +// ------------------------------ ZeroIfNegative (BroadcastSignBit) +template <typename T, size_t N, HWY_IF_FLOAT(T)> +HWY_API Vec128<T, N> ZeroIfNegative(Vec128<T, N> v) { + const Simd<T, N> d; +#if HWY_TARGET == HWY_SSSE3 + const RebindToSigned<decltype(d)> di; + const auto mask = MaskFromVec(BitCast(d, BroadcastSignBit(BitCast(di, v)))); +#else + const auto mask = MaskFromVec(v); // MSB is sufficient for BLENDVPS +#endif + return IfThenElse(mask, Zero(d), v); +} + +// ------------------------------ ShiftLeftSame + +template <size_t N> +HWY_API Vec128<uint16_t, N> ShiftLeftSame(const Vec128<uint16_t, N> v, + const int bits) { + return Vec128<uint16_t, N>{_mm_sll_epi16(v.raw, _mm_cvtsi32_si128(bits))}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> ShiftLeftSame(const Vec128<uint32_t, N> v, + const int bits) { + return Vec128<uint32_t, N>{_mm_sll_epi32(v.raw, _mm_cvtsi32_si128(bits))}; +} +template <size_t N> +HWY_API Vec128<uint64_t, N> ShiftLeftSame(const Vec128<uint64_t, N> v, + const int bits) { + return Vec128<uint64_t, N>{_mm_sll_epi64(v.raw, _mm_cvtsi32_si128(bits))}; +} + +template <size_t N> +HWY_API Vec128<int16_t, N> ShiftLeftSame(const Vec128<int16_t, N> v, + const int bits) { + return Vec128<int16_t, N>{_mm_sll_epi16(v.raw, _mm_cvtsi32_si128(bits))}; +} + +template <size_t N> +HWY_API Vec128<int32_t, N> ShiftLeftSame(const Vec128<int32_t, N> v, + const int bits) { + return Vec128<int32_t, N>{_mm_sll_epi32(v.raw, _mm_cvtsi32_si128(bits))}; +} + +template <size_t N> +HWY_API Vec128<int64_t, N> ShiftLeftSame(const Vec128<int64_t, N> v, + const int bits) { + return Vec128<int64_t, N>{_mm_sll_epi64(v.raw, _mm_cvtsi32_si128(bits))}; +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 1)> +HWY_API Vec128<T, N> ShiftLeftSame(const Vec128<T, N> v, const int bits) { + const Simd<T, N> d8; + // Use raw instead of BitCast to support N=1. + const Vec128<T, N> shifted{ + ShiftLeftSame(Vec128<MakeWide<T>>{v.raw}, bits).raw}; + return shifted & Set(d8, static_cast<T>((0xFF << bits) & 0xFF)); +} + +// ------------------------------ ShiftRightSame (BroadcastSignBit) + +template <size_t N> +HWY_API Vec128<uint16_t, N> ShiftRightSame(const Vec128<uint16_t, N> v, + const int bits) { + return Vec128<uint16_t, N>{_mm_srl_epi16(v.raw, _mm_cvtsi32_si128(bits))}; +} +template <size_t N> +HWY_API Vec128<uint32_t, N> ShiftRightSame(const Vec128<uint32_t, N> v, + const int bits) { + return Vec128<uint32_t, N>{_mm_srl_epi32(v.raw, _mm_cvtsi32_si128(bits))}; +} +template <size_t N> +HWY_API Vec128<uint64_t, N> ShiftRightSame(const Vec128<uint64_t, N> v, + const int bits) { + return Vec128<uint64_t, N>{_mm_srl_epi64(v.raw, _mm_cvtsi32_si128(bits))}; +} + +template <size_t N> +HWY_API Vec128<uint8_t, N> ShiftRightSame(Vec128<uint8_t, N> v, + const int bits) { + const Simd<uint8_t, N> d8; + // Use raw instead of BitCast to support N=1. + const Vec128<uint8_t, N> shifted{ + ShiftRightSame(Vec128<uint16_t>{v.raw}, bits).raw}; + return shifted & Set(d8, static_cast<uint8_t>(0xFF >> bits)); +} + +template <size_t N> +HWY_API Vec128<int16_t, N> ShiftRightSame(const Vec128<int16_t, N> v, + const int bits) { + return Vec128<int16_t, N>{_mm_sra_epi16(v.raw, _mm_cvtsi32_si128(bits))}; +} + +template <size_t N> +HWY_API Vec128<int32_t, N> ShiftRightSame(const Vec128<int32_t, N> v, + const int bits) { + return Vec128<int32_t, N>{_mm_sra_epi32(v.raw, _mm_cvtsi32_si128(bits))}; +} +template <size_t N> +HWY_API Vec128<int64_t, N> ShiftRightSame(const Vec128<int64_t, N> v, + const int bits) { +#if HWY_TARGET <= HWY_AVX3 + return Vec128<int64_t, N>{_mm_sra_epi64(v.raw, _mm_cvtsi32_si128(bits))}; +#else + const Simd<int64_t, N> di; + const Simd<uint64_t, N> du; + const auto right = BitCast(di, ShiftRightSame(BitCast(du, v), bits)); + const auto sign = ShiftLeftSame(BroadcastSignBit(v), 64 - bits); + return right | sign; +#endif +} + +template <size_t N> +HWY_API Vec128<int8_t, N> ShiftRightSame(Vec128<int8_t, N> v, const int bits) { + const Simd<int8_t, N> di; + const Simd<uint8_t, N> du; + const auto shifted = BitCast(di, ShiftRightSame(BitCast(du, v), bits)); + const auto shifted_sign = + BitCast(di, Set(du, static_cast<uint8_t>(0x80 >> bits))); + return (shifted ^ shifted_sign) - shifted_sign; +} + +// ------------------------------ Floating-point mul / div + +template <size_t N> +HWY_API Vec128<float, N> operator*(Vec128<float, N> a, Vec128<float, N> b) { + return Vec128<float, N>{_mm_mul_ps(a.raw, b.raw)}; +} +HWY_API Vec128<float, 1> operator*(const Vec128<float, 1> a, + const Vec128<float, 1> b) { + return Vec128<float, 1>{_mm_mul_ss(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<double, N> operator*(const Vec128<double, N> a, + const Vec128<double, N> b) { + return Vec128<double, N>{_mm_mul_pd(a.raw, b.raw)}; +} +HWY_API Vec128<double, 1> operator*(const Vec128<double, 1> a, + const Vec128<double, 1> b) { + return Vec128<double, 1>{_mm_mul_sd(a.raw, b.raw)}; +} + +template <size_t N> +HWY_API Vec128<float, N> operator/(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Vec128<float, N>{_mm_div_ps(a.raw, b.raw)}; +} +HWY_API Vec128<float, 1> operator/(const Vec128<float, 1> a, + const Vec128<float, 1> b) { + return Vec128<float, 1>{_mm_div_ss(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<double, N> operator/(const Vec128<double, N> a, + const Vec128<double, N> b) { + return Vec128<double, N>{_mm_div_pd(a.raw, b.raw)}; +} +HWY_API Vec128<double, 1> operator/(const Vec128<double, 1> a, + const Vec128<double, 1> b) { + return Vec128<double, 1>{_mm_div_sd(a.raw, b.raw)}; +} + +// Approximate reciprocal +template <size_t N> +HWY_API Vec128<float, N> ApproximateReciprocal(const Vec128<float, N> v) { + return Vec128<float, N>{_mm_rcp_ps(v.raw)}; +} +HWY_API Vec128<float, 1> ApproximateReciprocal(const Vec128<float, 1> v) { + return Vec128<float, 1>{_mm_rcp_ss(v.raw)}; +} + +// Absolute value of difference. +template <size_t N> +HWY_API Vec128<float, N> AbsDiff(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Abs(a - b); +} + +// ------------------------------ Floating-point multiply-add variants + +// Returns mul * x + add +template <size_t N> +HWY_API Vec128<float, N> MulAdd(const Vec128<float, N> mul, + const Vec128<float, N> x, + const Vec128<float, N> add) { +#if HWY_TARGET == HWY_SSSE3 || HWY_TARGET == HWY_SSE4 + return mul * x + add; +#else + return Vec128<float, N>{_mm_fmadd_ps(mul.raw, x.raw, add.raw)}; +#endif +} +template <size_t N> +HWY_API Vec128<double, N> MulAdd(const Vec128<double, N> mul, + const Vec128<double, N> x, + const Vec128<double, N> add) { +#if HWY_TARGET == HWY_SSSE3 || HWY_TARGET == HWY_SSE4 + return mul * x + add; +#else + return Vec128<double, N>{_mm_fmadd_pd(mul.raw, x.raw, add.raw)}; +#endif +} + +// Returns add - mul * x +template <size_t N> +HWY_API Vec128<float, N> NegMulAdd(const Vec128<float, N> mul, + const Vec128<float, N> x, + const Vec128<float, N> add) { +#if HWY_TARGET == HWY_SSSE3 || HWY_TARGET == HWY_SSE4 + return add - mul * x; +#else + return Vec128<float, N>{_mm_fnmadd_ps(mul.raw, x.raw, add.raw)}; +#endif +} +template <size_t N> +HWY_API Vec128<double, N> NegMulAdd(const Vec128<double, N> mul, + const Vec128<double, N> x, + const Vec128<double, N> add) { +#if HWY_TARGET == HWY_SSSE3 || HWY_TARGET == HWY_SSE4 + return add - mul * x; +#else + return Vec128<double, N>{_mm_fnmadd_pd(mul.raw, x.raw, add.raw)}; +#endif +} + +// Returns mul * x - sub +template <size_t N> +HWY_API Vec128<float, N> MulSub(const Vec128<float, N> mul, + const Vec128<float, N> x, + const Vec128<float, N> sub) { +#if HWY_TARGET == HWY_SSSE3 || HWY_TARGET == HWY_SSE4 + return mul * x - sub; +#else + return Vec128<float, N>{_mm_fmsub_ps(mul.raw, x.raw, sub.raw)}; +#endif +} +template <size_t N> +HWY_API Vec128<double, N> MulSub(const Vec128<double, N> mul, + const Vec128<double, N> x, + const Vec128<double, N> sub) { +#if HWY_TARGET == HWY_SSSE3 || HWY_TARGET == HWY_SSE4 + return mul * x - sub; +#else + return Vec128<double, N>{_mm_fmsub_pd(mul.raw, x.raw, sub.raw)}; +#endif +} + +// Returns -mul * x - sub +template <size_t N> +HWY_API Vec128<float, N> NegMulSub(const Vec128<float, N> mul, + const Vec128<float, N> x, + const Vec128<float, N> sub) { +#if HWY_TARGET == HWY_SSSE3 || HWY_TARGET == HWY_SSE4 + return Neg(mul) * x - sub; +#else + return Vec128<float, N>{_mm_fnmsub_ps(mul.raw, x.raw, sub.raw)}; +#endif +} +template <size_t N> +HWY_API Vec128<double, N> NegMulSub(const Vec128<double, N> mul, + const Vec128<double, N> x, + const Vec128<double, N> sub) { +#if HWY_TARGET == HWY_SSSE3 || HWY_TARGET == HWY_SSE4 + return Neg(mul) * x - sub; +#else + return Vec128<double, N>{_mm_fnmsub_pd(mul.raw, x.raw, sub.raw)}; +#endif +} + +// ------------------------------ Floating-point square root + +// Full precision square root +template <size_t N> +HWY_API Vec128<float, N> Sqrt(const Vec128<float, N> v) { + return Vec128<float, N>{_mm_sqrt_ps(v.raw)}; +} +HWY_API Vec128<float, 1> Sqrt(const Vec128<float, 1> v) { + return Vec128<float, 1>{_mm_sqrt_ss(v.raw)}; +} +template <size_t N> +HWY_API Vec128<double, N> Sqrt(const Vec128<double, N> v) { + return Vec128<double, N>{_mm_sqrt_pd(v.raw)}; +} +HWY_API Vec128<double, 1> Sqrt(const Vec128<double, 1> v) { + return Vec128<double, 1>{_mm_sqrt_sd(_mm_setzero_pd(), v.raw)}; +} + +// Approximate reciprocal square root +template <size_t N> +HWY_API Vec128<float, N> ApproximateReciprocalSqrt(const Vec128<float, N> v) { + return Vec128<float, N>{_mm_rsqrt_ps(v.raw)}; +} +HWY_API Vec128<float, 1> ApproximateReciprocalSqrt(const Vec128<float, 1> v) { + return Vec128<float, 1>{_mm_rsqrt_ss(v.raw)}; +} + +// ------------------------------ Min (Gt, IfThenElse) + +namespace detail { + +template <typename T, size_t N> +HWY_INLINE HWY_MAYBE_UNUSED Vec128<T, N> MinU(const Vec128<T, N> a, + const Vec128<T, N> b) { + const Simd<T, N> du; + const RebindToSigned<decltype(du)> di; + const auto msb = Set(du, static_cast<T>(T(1) << (sizeof(T) * 8 - 1))); + const auto gt = RebindMask(du, BitCast(di, a ^ msb) > BitCast(di, b ^ msb)); + return IfThenElse(gt, b, a); +} + +} // namespace detail + +// Unsigned +template <size_t N> +HWY_API Vec128<uint8_t, N> Min(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{_mm_min_epu8(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> Min(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { +#if HWY_TARGET == HWY_SSSE3 + return detail::MinU(a, b); +#else + return Vec128<uint16_t, N>{_mm_min_epu16(a.raw, b.raw)}; +#endif +} +template <size_t N> +HWY_API Vec128<uint32_t, N> Min(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { +#if HWY_TARGET == HWY_SSSE3 + return detail::MinU(a, b); +#else + return Vec128<uint32_t, N>{_mm_min_epu32(a.raw, b.raw)}; +#endif +} +template <size_t N> +HWY_API Vec128<uint64_t, N> Min(const Vec128<uint64_t, N> a, + const Vec128<uint64_t, N> b) { +#if HWY_TARGET <= HWY_AVX3 + return Vec128<uint64_t, N>{_mm_min_epu64(a.raw, b.raw)}; +#else + return detail::MinU(a, b); +#endif +} + +// Signed +template <size_t N> +HWY_API Vec128<int8_t, N> Min(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { +#if HWY_TARGET == HWY_SSSE3 + return IfThenElse(a < b, a, b); +#else + return Vec128<int8_t, N>{_mm_min_epi8(a.raw, b.raw)}; +#endif +} +template <size_t N> +HWY_API Vec128<int16_t, N> Min(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{_mm_min_epi16(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> Min(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { +#if HWY_TARGET == HWY_SSSE3 + return IfThenElse(a < b, a, b); +#else + return Vec128<int32_t, N>{_mm_min_epi32(a.raw, b.raw)}; +#endif +} +template <size_t N> +HWY_API Vec128<int64_t, N> Min(const Vec128<int64_t, N> a, + const Vec128<int64_t, N> b) { +#if HWY_TARGET <= HWY_AVX3 + return Vec128<int64_t, N>{_mm_min_epi64(a.raw, b.raw)}; +#else + return IfThenElse(a < b, a, b); +#endif +} + +// Float +template <size_t N> +HWY_API Vec128<float, N> Min(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Vec128<float, N>{_mm_min_ps(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<double, N> Min(const Vec128<double, N> a, + const Vec128<double, N> b) { + return Vec128<double, N>{_mm_min_pd(a.raw, b.raw)}; +} + +// ------------------------------ Max (Gt, IfThenElse) + +namespace detail { +template <typename T, size_t N> +HWY_INLINE HWY_MAYBE_UNUSED Vec128<T, N> MaxU(const Vec128<T, N> a, + const Vec128<T, N> b) { + const Simd<T, N> du; + const RebindToSigned<decltype(du)> di; + const auto msb = Set(du, static_cast<T>(T(1) << (sizeof(T) * 8 - 1))); + const auto gt = RebindMask(du, BitCast(di, a ^ msb) > BitCast(di, b ^ msb)); + return IfThenElse(gt, a, b); +} + +} // namespace detail + +// Unsigned +template <size_t N> +HWY_API Vec128<uint8_t, N> Max(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{_mm_max_epu8(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<uint16_t, N> Max(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { +#if HWY_TARGET == HWY_SSSE3 + return detail::MaxU(a, b); +#else + return Vec128<uint16_t, N>{_mm_max_epu16(a.raw, b.raw)}; +#endif +} +template <size_t N> +HWY_API Vec128<uint32_t, N> Max(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { +#if HWY_TARGET == HWY_SSSE3 + return detail::MaxU(a, b); +#else + return Vec128<uint32_t, N>{_mm_max_epu32(a.raw, b.raw)}; +#endif +} +template <size_t N> +HWY_API Vec128<uint64_t, N> Max(const Vec128<uint64_t, N> a, + const Vec128<uint64_t, N> b) { +#if HWY_TARGET <= HWY_AVX3 + return Vec128<uint64_t, N>{_mm_max_epu64(a.raw, b.raw)}; +#else + return detail::MaxU(a, b); +#endif +} + +// Signed +template <size_t N> +HWY_API Vec128<int8_t, N> Max(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { +#if HWY_TARGET == HWY_SSSE3 + return IfThenElse(a < b, b, a); +#else + return Vec128<int8_t, N>{_mm_max_epi8(a.raw, b.raw)}; +#endif +} +template <size_t N> +HWY_API Vec128<int16_t, N> Max(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{_mm_max_epi16(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<int32_t, N> Max(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { +#if HWY_TARGET == HWY_SSSE3 + return IfThenElse(a < b, b, a); +#else + return Vec128<int32_t, N>{_mm_max_epi32(a.raw, b.raw)}; +#endif +} +template <size_t N> +HWY_API Vec128<int64_t, N> Max(const Vec128<int64_t, N> a, + const Vec128<int64_t, N> b) { +#if HWY_TARGET <= HWY_AVX3 + return Vec128<int64_t, N>{_mm_max_epi64(a.raw, b.raw)}; +#else + return IfThenElse(a < b, b, a); +#endif +} + +// Float +template <size_t N> +HWY_API Vec128<float, N> Max(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Vec128<float, N>{_mm_max_ps(a.raw, b.raw)}; +} +template <size_t N> +HWY_API Vec128<double, N> Max(const Vec128<double, N> a, + const Vec128<double, N> b) { + return Vec128<double, N>{_mm_max_pd(a.raw, b.raw)}; +} + +// ================================================== MEMORY (2) + +// ------------------------------ Non-temporal stores + +// On clang6, we see incorrect code generated for _mm_stream_pi, so +// round even partial vectors up to 16 bytes. +template <typename T, size_t N> +HWY_API void Stream(Vec128<T, N> v, Simd<T, N> /* tag */, + T* HWY_RESTRICT aligned) { + _mm_stream_si128(reinterpret_cast<__m128i*>(aligned), v.raw); +} +template <size_t N> +HWY_API void Stream(const Vec128<float, N> v, Simd<float, N> /* tag */, + float* HWY_RESTRICT aligned) { + _mm_stream_ps(aligned, v.raw); +} +template <size_t N> +HWY_API void Stream(const Vec128<double, N> v, Simd<double, N> /* tag */, + double* HWY_RESTRICT aligned) { + _mm_stream_pd(aligned, v.raw); +} + +// ------------------------------ Scatter + +// Work around warnings in the intrinsic definitions (passing -1 as a mask). +HWY_DIAGNOSTICS(push) +HWY_DIAGNOSTICS_OFF(disable : 4245 4365, ignored "-Wsign-conversion") + +// Unfortunately the GCC/Clang intrinsics do not accept int64_t*. +using GatherIndex64 = long long int; // NOLINT(google-runtime-int) +static_assert(sizeof(GatherIndex64) == 8, "Must be 64-bit type"); + +#if HWY_TARGET <= HWY_AVX3 +namespace detail { + +template <typename T, size_t N> +HWY_INLINE void ScatterOffset(hwy::SizeTag<4> /* tag */, Vec128<T, N> v, + Simd<T, N> /* tag */, T* HWY_RESTRICT base, + const Vec128<int32_t, N> offset) { + if (N == 4) { + _mm_i32scatter_epi32(base, offset.raw, v.raw, 1); + } else { + const __mmask8 mask = (1u << N) - 1; + _mm_mask_i32scatter_epi32(base, mask, offset.raw, v.raw, 1); + } +} +template <typename T, size_t N> +HWY_INLINE void ScatterIndex(hwy::SizeTag<4> /* tag */, Vec128<T, N> v, + Simd<T, N> /* tag */, T* HWY_RESTRICT base, + const Vec128<int32_t, N> index) { + if (N == 4) { + _mm_i32scatter_epi32(base, index.raw, v.raw, 4); + } else { + const __mmask8 mask = (1u << N) - 1; + _mm_mask_i32scatter_epi32(base, mask, index.raw, v.raw, 4); + } +} + +template <typename T, size_t N> +HWY_INLINE void ScatterOffset(hwy::SizeTag<8> /* tag */, Vec128<T, N> v, + Simd<T, N> /* tag */, T* HWY_RESTRICT base, + const Vec128<int64_t, N> offset) { + if (N == 2) { + _mm_i64scatter_epi64(base, offset.raw, v.raw, 1); + } else { + const __mmask8 mask = (1u << N) - 1; + _mm_mask_i64scatter_epi64(base, mask, offset.raw, v.raw, 1); + } +} +template <typename T, size_t N> +HWY_INLINE void ScatterIndex(hwy::SizeTag<8> /* tag */, Vec128<T, N> v, + Simd<T, N> /* tag */, T* HWY_RESTRICT base, + const Vec128<int64_t, N> index) { + if (N == 2) { + _mm_i64scatter_epi64(base, index.raw, v.raw, 8); + } else { + const __mmask8 mask = (1u << N) - 1; + _mm_mask_i64scatter_epi64(base, mask, index.raw, v.raw, 8); + } +} + +} // namespace detail + +template <typename T, size_t N, typename Offset> +HWY_API void ScatterOffset(Vec128<T, N> v, Simd<T, N> d, T* HWY_RESTRICT base, + const Vec128<Offset, N> offset) { + static_assert(sizeof(T) == sizeof(Offset), "Must match for portability"); + return detail::ScatterOffset(hwy::SizeTag<sizeof(T)>(), v, d, base, offset); +} +template <typename T, size_t N, typename Index> +HWY_API void ScatterIndex(Vec128<T, N> v, Simd<T, N> d, T* HWY_RESTRICT base, + const Vec128<Index, N> index) { + static_assert(sizeof(T) == sizeof(Index), "Must match for portability"); + return detail::ScatterIndex(hwy::SizeTag<sizeof(T)>(), v, d, base, index); +} + +template <size_t N> +HWY_API void ScatterOffset(Vec128<float, N> v, Simd<float, N> /* tag */, + float* HWY_RESTRICT base, + const Vec128<int32_t, N> offset) { + if (N == 4) { + _mm_i32scatter_ps(base, offset.raw, v.raw, 1); + } else { + const __mmask8 mask = (1u << N) - 1; + _mm_mask_i32scatter_ps(base, mask, offset.raw, v.raw, 1); + } +} +template <size_t N> +HWY_API void ScatterIndex(Vec128<float, N> v, Simd<float, N> /* tag */, + float* HWY_RESTRICT base, + const Vec128<int32_t, N> index) { + if (N == 4) { + _mm_i32scatter_ps(base, index.raw, v.raw, 4); + } else { + const __mmask8 mask = (1u << N) - 1; + _mm_mask_i32scatter_ps(base, mask, index.raw, v.raw, 4); + } +} + +template <size_t N> +HWY_API void ScatterOffset(Vec128<double, N> v, Simd<double, N> /* tag */, + double* HWY_RESTRICT base, + const Vec128<int64_t, N> offset) { + if (N == 2) { + _mm_i64scatter_pd(base, offset.raw, v.raw, 1); + } else { + const __mmask8 mask = (1u << N) - 1; + _mm_mask_i64scatter_pd(base, mask, offset.raw, v.raw, 1); + } +} +template <size_t N> +HWY_API void ScatterIndex(Vec128<double, N> v, Simd<double, N> /* tag */, + double* HWY_RESTRICT base, + const Vec128<int64_t, N> index) { + if (N == 2) { + _mm_i64scatter_pd(base, index.raw, v.raw, 8); + } else { + const __mmask8 mask = (1u << N) - 1; + _mm_mask_i64scatter_pd(base, mask, index.raw, v.raw, 8); + } +} +#else // HWY_TARGET <= HWY_AVX3 + +template <typename T, size_t N, typename Offset, HWY_IF_LE128(T, N)> +HWY_API void ScatterOffset(Vec128<T, N> v, Simd<T, N> d, T* HWY_RESTRICT base, + const Vec128<Offset, N> offset) { + static_assert(sizeof(T) == sizeof(Offset), "Must match for portability"); + + alignas(16) T lanes[N]; + Store(v, d, lanes); + + alignas(16) Offset offset_lanes[N]; + Store(offset, Simd<Offset, N>(), offset_lanes); + + uint8_t* base_bytes = reinterpret_cast<uint8_t*>(base); + for (size_t i = 0; i < N; ++i) { + CopyBytes<sizeof(T)>(&lanes[i], base_bytes + offset_lanes[i]); + } +} + +template <typename T, size_t N, typename Index, HWY_IF_LE128(T, N)> +HWY_API void ScatterIndex(Vec128<T, N> v, Simd<T, N> d, T* HWY_RESTRICT base, + const Vec128<Index, N> index) { + static_assert(sizeof(T) == sizeof(Index), "Must match for portability"); + + alignas(16) T lanes[N]; + Store(v, d, lanes); + + alignas(16) Index index_lanes[N]; + Store(index, Simd<Index, N>(), index_lanes); + + for (size_t i = 0; i < N; ++i) { + base[index_lanes[i]] = lanes[i]; + } +} + +#endif + +// ------------------------------ Gather (Load/Store) + +#if HWY_TARGET == HWY_SSSE3 || HWY_TARGET == HWY_SSE4 + +template <typename T, size_t N, typename Offset> +HWY_API Vec128<T, N> GatherOffset(const Simd<T, N> d, + const T* HWY_RESTRICT base, + const Vec128<Offset, N> offset) { + static_assert(sizeof(T) == sizeof(Offset), "Must match for portability"); + + alignas(16) Offset offset_lanes[N]; + Store(offset, Simd<Offset, N>(), offset_lanes); + + alignas(16) T lanes[N]; + const uint8_t* base_bytes = reinterpret_cast<const uint8_t*>(base); + for (size_t i = 0; i < N; ++i) { + CopyBytes<sizeof(T)>(base_bytes + offset_lanes[i], &lanes[i]); + } + return Load(d, lanes); +} + +template <typename T, size_t N, typename Index> +HWY_API Vec128<T, N> GatherIndex(const Simd<T, N> d, const T* HWY_RESTRICT base, + const Vec128<Index, N> index) { + static_assert(sizeof(T) == sizeof(Index), "Must match for portability"); + + alignas(16) Index index_lanes[N]; + Store(index, Simd<Index, N>(), index_lanes); + + alignas(16) T lanes[N]; + for (size_t i = 0; i < N; ++i) { + lanes[i] = base[index_lanes[i]]; + } + return Load(d, lanes); +} + +#else + +namespace detail { + +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> GatherOffset(hwy::SizeTag<4> /* tag */, + Simd<T, N> /* d */, + const T* HWY_RESTRICT base, + const Vec128<int32_t, N> offset) { + return Vec128<T, N>{_mm_i32gather_epi32( + reinterpret_cast<const int32_t*>(base), offset.raw, 1)}; +} +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> GatherIndex(hwy::SizeTag<4> /* tag */, + Simd<T, N> /* d */, + const T* HWY_RESTRICT base, + const Vec128<int32_t, N> index) { + return Vec128<T, N>{_mm_i32gather_epi32( + reinterpret_cast<const int32_t*>(base), index.raw, 4)}; +} + +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> GatherOffset(hwy::SizeTag<8> /* tag */, + Simd<T, N> /* d */, + const T* HWY_RESTRICT base, + const Vec128<int64_t, N> offset) { + return Vec128<T, N>{_mm_i64gather_epi64( + reinterpret_cast<const GatherIndex64*>(base), offset.raw, 1)}; +} +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> GatherIndex(hwy::SizeTag<8> /* tag */, + Simd<T, N> /* d */, + const T* HWY_RESTRICT base, + const Vec128<int64_t, N> index) { + return Vec128<T, N>{_mm_i64gather_epi64( + reinterpret_cast<const GatherIndex64*>(base), index.raw, 8)}; +} + +} // namespace detail + +template <typename T, size_t N, typename Offset> +HWY_API Vec128<T, N> GatherOffset(Simd<T, N> d, const T* HWY_RESTRICT base, + const Vec128<Offset, N> offset) { + return detail::GatherOffset(hwy::SizeTag<sizeof(T)>(), d, base, offset); +} +template <typename T, size_t N, typename Index> +HWY_API Vec128<T, N> GatherIndex(Simd<T, N> d, const T* HWY_RESTRICT base, + const Vec128<Index, N> index) { + return detail::GatherIndex(hwy::SizeTag<sizeof(T)>(), d, base, index); +} + +template <size_t N> +HWY_API Vec128<float, N> GatherOffset(Simd<float, N> /* tag */, + const float* HWY_RESTRICT base, + const Vec128<int32_t, N> offset) { + return Vec128<float, N>{_mm_i32gather_ps(base, offset.raw, 1)}; +} +template <size_t N> +HWY_API Vec128<float, N> GatherIndex(Simd<float, N> /* tag */, + const float* HWY_RESTRICT base, + const Vec128<int32_t, N> index) { + return Vec128<float, N>{_mm_i32gather_ps(base, index.raw, 4)}; +} + +template <size_t N> +HWY_API Vec128<double, N> GatherOffset(Simd<double, N> /* tag */, + const double* HWY_RESTRICT base, + const Vec128<int64_t, N> offset) { + return Vec128<double, N>{_mm_i64gather_pd(base, offset.raw, 1)}; +} +template <size_t N> +HWY_API Vec128<double, N> GatherIndex(Simd<double, N> /* tag */, + const double* HWY_RESTRICT base, + const Vec128<int64_t, N> index) { + return Vec128<double, N>{_mm_i64gather_pd(base, index.raw, 8)}; +} + +#endif // HWY_TARGET == HWY_SSSE3 || HWY_TARGET == HWY_SSE4 + +HWY_DIAGNOSTICS(pop) + +// ================================================== SWIZZLE (2) + +// ------------------------------ LowerHalf + +// Returns upper/lower half of a vector. +template <typename T, size_t N> +HWY_API Vec128<T, N / 2> LowerHalf(Simd<T, N / 2> /* tag */, Vec128<T, N> v) { + return Vec128<T, N / 2>{v.raw}; +} + +template <typename T, size_t N> +HWY_API Vec128<T, N / 2> LowerHalf(Vec128<T, N> v) { + return LowerHalf(Simd<T, N / 2>(), v); +} + +// ------------------------------ ShiftLeftBytes + +template <int kBytes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftLeftBytes(Simd<T, N> /* tag */, Vec128<T, N> v) { + static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes"); + return Vec128<T, N>{_mm_slli_si128(v.raw, kBytes)}; +} + +template <int kBytes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftLeftBytes(const Vec128<T, N> v) { + return ShiftLeftBytes<kBytes>(Simd<T, N>(), v); +} + +// ------------------------------ ShiftLeftLanes + +template <int kLanes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftLeftLanes(Simd<T, N> d, const Vec128<T, N> v) { + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, ShiftLeftBytes<kLanes * sizeof(T)>(BitCast(d8, v))); +} + +template <int kLanes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftLeftLanes(const Vec128<T, N> v) { + return ShiftLeftLanes<kLanes>(Simd<T, N>(), v); +} + +// ------------------------------ ShiftRightBytes +template <int kBytes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftRightBytes(Simd<T, N> /* tag */, Vec128<T, N> v) { + static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes"); + // For partial vectors, clear upper lanes so we shift in zeros. + if (N != 16 / sizeof(T)) { + const Vec128<T> vfull{v.raw}; + v = Vec128<T, N>{IfThenElseZero(FirstN(Full128<T>(), N), vfull).raw}; + } + return Vec128<T, N>{_mm_srli_si128(v.raw, kBytes)}; +} + +// ------------------------------ ShiftRightLanes +template <int kLanes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftRightLanes(Simd<T, N> d, const Vec128<T, N> v) { + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, ShiftRightBytes<kLanes * sizeof(T)>(BitCast(d8, v))); +} + +// ------------------------------ UpperHalf (ShiftRightBytes) + +// Full input: copy hi into lo (smaller instruction encoding than shifts). +template <typename T> +HWY_API Vec128<T, 8 / sizeof(T)> UpperHalf(Half<Full128<T>> /* tag */, + Vec128<T> v) { + return Vec128<T, 8 / sizeof(T)>{_mm_unpackhi_epi64(v.raw, v.raw)}; +} +HWY_API Vec128<float, 2> UpperHalf(Simd<float, 2> /* tag */, Vec128<float> v) { + return Vec128<float, 2>{_mm_movehl_ps(v.raw, v.raw)}; +} +HWY_API Vec128<double, 1> UpperHalf(Simd<double, 1> /* tag */, + Vec128<double> v) { + return Vec128<double, 1>{_mm_unpackhi_pd(v.raw, v.raw)}; +} + +// Partial +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_API Vec128<T, (N + 1) / 2> UpperHalf(Half<Simd<T, N>> /* tag */, + Vec128<T, N> v) { + const Simd<T, N> d; + const auto vu = BitCast(RebindToUnsigned<decltype(d)>(), v); + const auto upper = BitCast(d, ShiftRightBytes<N * sizeof(T) / 2>(vu)); + return Vec128<T, (N + 1) / 2>{upper.raw}; +} + +// ------------------------------ CombineShiftRightBytes + +template <int kBytes, typename T, class V = Vec128<T>> +HWY_API V CombineShiftRightBytes(Full128<T> d, V hi, V lo) { + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, Vec128<uint8_t>{_mm_alignr_epi8( + BitCast(d8, hi).raw, BitCast(d8, lo).raw, kBytes)}); +} + +template <int kBytes, typename T, size_t N, HWY_IF_LE64(T, N), + class V = Vec128<T, N>> +HWY_API V CombineShiftRightBytes(Simd<T, N> d, V hi, V lo) { + constexpr size_t kSize = N * sizeof(T); + static_assert(0 < kBytes && kBytes < kSize, "kBytes invalid"); + const Repartition<uint8_t, decltype(d)> d8; + const Full128<uint8_t> d_full8; + using V8 = VFromD<decltype(d_full8)>; + const V8 hi8{BitCast(d8, hi).raw}; + // Move into most-significant bytes + const V8 lo8 = ShiftLeftBytes<16 - kSize>(V8{BitCast(d8, lo).raw}); + const V8 r = CombineShiftRightBytes<16 - kSize + kBytes>(d_full8, hi8, lo8); + return V{BitCast(Full128<T>(), r).raw}; +} + +// ------------------------------ Broadcast/splat any lane + +// Unsigned +template <int kLane, size_t N> +HWY_API Vec128<uint16_t, N> Broadcast(const Vec128<uint16_t, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + if (kLane < 4) { + const __m128i lo = _mm_shufflelo_epi16(v.raw, (0x55 * kLane) & 0xFF); + return Vec128<uint16_t, N>{_mm_unpacklo_epi64(lo, lo)}; + } else { + const __m128i hi = _mm_shufflehi_epi16(v.raw, (0x55 * (kLane - 4)) & 0xFF); + return Vec128<uint16_t, N>{_mm_unpackhi_epi64(hi, hi)}; + } +} +template <int kLane, size_t N> +HWY_API Vec128<uint32_t, N> Broadcast(const Vec128<uint32_t, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<uint32_t, N>{_mm_shuffle_epi32(v.raw, 0x55 * kLane)}; +} +template <int kLane, size_t N> +HWY_API Vec128<uint64_t, N> Broadcast(const Vec128<uint64_t, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<uint64_t, N>{_mm_shuffle_epi32(v.raw, kLane ? 0xEE : 0x44)}; +} + +// Signed +template <int kLane, size_t N> +HWY_API Vec128<int16_t, N> Broadcast(const Vec128<int16_t, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + if (kLane < 4) { + const __m128i lo = _mm_shufflelo_epi16(v.raw, (0x55 * kLane) & 0xFF); + return Vec128<int16_t, N>{_mm_unpacklo_epi64(lo, lo)}; + } else { + const __m128i hi = _mm_shufflehi_epi16(v.raw, (0x55 * (kLane - 4)) & 0xFF); + return Vec128<int16_t, N>{_mm_unpackhi_epi64(hi, hi)}; + } +} +template <int kLane, size_t N> +HWY_API Vec128<int32_t, N> Broadcast(const Vec128<int32_t, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<int32_t, N>{_mm_shuffle_epi32(v.raw, 0x55 * kLane)}; +} +template <int kLane, size_t N> +HWY_API Vec128<int64_t, N> Broadcast(const Vec128<int64_t, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<int64_t, N>{_mm_shuffle_epi32(v.raw, kLane ? 0xEE : 0x44)}; +} + +// Float +template <int kLane, size_t N> +HWY_API Vec128<float, N> Broadcast(const Vec128<float, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<float, N>{_mm_shuffle_ps(v.raw, v.raw, 0x55 * kLane)}; +} +template <int kLane, size_t N> +HWY_API Vec128<double, N> Broadcast(const Vec128<double, N> v) { + static_assert(0 <= kLane && kLane < N, "Invalid lane"); + return Vec128<double, N>{_mm_shuffle_pd(v.raw, v.raw, 3 * kLane)}; +} + +// ------------------------------ TableLookupBytes +template <typename T, size_t N, typename TI, size_t NI> +HWY_API Vec128<TI, NI> TableLookupBytes(const Vec128<T, N> bytes, + const Vec128<TI, NI> from) { + return Vec128<TI, NI>{_mm_shuffle_epi8(bytes.raw, from.raw)}; +} + +// ------------------------------ TableLookupBytesOr0 +// For all vector widths; x86 anyway zeroes if >= 0x80. +template <class V, class VI> +HWY_API VI TableLookupBytesOr0(const V bytes, const VI from) { + return TableLookupBytes(bytes, from); +} + +// ------------------------------ TableLookupLanes (Shuffle01) + +// Returned by SetTableIndices/IndicesFromVec for use by TableLookupLanes. +template <typename T, size_t N = 16 / sizeof(T)> +struct Indices128 { + __m128i raw; +}; + +template <typename T, size_t N, typename TI, HWY_IF_LE128(T, N), + HWY_IF_LANE_SIZE(T, 4)> +HWY_API Indices128<T, N> IndicesFromVec(Simd<T, N> d, Vec128<TI, N> vec) { + static_assert(sizeof(T) == sizeof(TI), "Index size must match lane"); +#if HWY_IS_DEBUG_BUILD + const Simd<TI, N> di; + HWY_DASSERT(AllFalse(di, Lt(vec, Zero(di))) && + AllTrue(di, Lt(vec, Set(di, N)))); +#endif + +#if HWY_TARGET <= HWY_AVX2 + (void)d; + return Indices128<T, N>{vec.raw}; +#else + const Repartition<uint8_t, decltype(d)> d8; + using V8 = VFromD<decltype(d8)>; + alignas(16) constexpr uint8_t kByteOffsets[16] = {0, 1, 2, 3, 0, 1, 2, 3, + 0, 1, 2, 3, 0, 1, 2, 3}; + + // Broadcast each lane index to all 4 bytes of T + alignas(16) constexpr uint8_t kBroadcastLaneBytes[16] = { + 0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12}; + const V8 lane_indices = TableLookupBytes(vec, Load(d8, kBroadcastLaneBytes)); + + // Shift to bytes + const Repartition<uint16_t, decltype(d)> d16; + const V8 byte_indices = BitCast(d8, ShiftLeft<2>(BitCast(d16, lane_indices))); + + return Indices128<T, N>{Add(byte_indices, Load(d8, kByteOffsets)).raw}; +#endif +} + +template <typename T, size_t N, typename TI, HWY_IF_LE128(T, N), + HWY_IF_LANE_SIZE(T, 8)> +HWY_API Indices128<T, N> IndicesFromVec(Simd<T, N> /* tag */, + Vec128<TI, N> vec) { + static_assert(sizeof(T) == sizeof(TI), "Index size must match lane"); +#if HWY_IS_DEBUG_BUILD + const Simd<TI, N> di; + HWY_DASSERT(AllFalse(di, Lt(vec, Zero(di))) && + AllTrue(di, Lt(vec, Set(di, static_cast<TI>(N))))); +#endif + + // No change - even without AVX3, we can shuffle+blend. + return Indices128<T, N>{vec.raw}; +} + +template <typename T, size_t N, typename TI, HWY_IF_LE128(T, N)> +HWY_API Indices128<T, N> SetTableIndices(Simd<T, N> d, const TI* idx) { + const Rebind<TI, decltype(d)> di; + return IndicesFromVec(d, LoadU(di, idx)); +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T, N> TableLookupLanes(Vec128<T, N> v, Indices128<T, N> idx) { +#if HWY_TARGET <= HWY_AVX2 + const Simd<T, N> d; + const Simd<float, N> df; + const Vec128<float, N> perm{_mm_permutevar_ps(BitCast(df, v).raw, idx.raw)}; + return BitCast(d, perm); +#else + return TableLookupBytes(v, Vec128<T, N>{idx.raw}); +#endif +} + +template <size_t N, HWY_IF_GE64(float, N)> +HWY_API Vec128<float, N> TableLookupLanes(Vec128<float, N> v, + Indices128<float, N> idx) { +#if HWY_TARGET <= HWY_AVX2 + return Vec128<float, N>{_mm_permutevar_ps(v.raw, idx.raw)}; +#else + const Simd<int32_t, N> di; + const Simd<float, N> df; + return BitCast(df, + TableLookupBytes(BitCast(di, v), Vec128<int32_t, N>{idx.raw})); +#endif +} + +// Single lane: no change +template <typename T> +HWY_API Vec128<T, 1> TableLookupLanes(Vec128<T, 1> v, + Indices128<T, 1> /* idx */) { + return v; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec128<T> TableLookupLanes(Vec128<T> v, Indices128<T> idx) { + const Full128<T> d; + Vec128<int64_t> vidx{idx.raw}; +#if HWY_TARGET <= HWY_AVX2 + // There is no _mm_permute[x]var_epi64. + vidx += vidx; // bit1 is the decider (unusual) + const Full128<double> df; + return BitCast( + d, Vec128<double>{_mm_permutevar_pd(BitCast(df, v).raw, vidx.raw)}); +#else + // Only 2 lanes: can swap+blend. Choose v if vidx == iota. To avoid a 64-bit + // comparison (expensive on SSSE3), just invert the upper lane and subtract 1 + // to obtain an all-zero or all-one mask. + const Full128<int64_t> di; + const Vec128<int64_t> same = (vidx ^ Iota(di, 0)) - Set(di, 1); + const Mask128<T> mask_same = RebindMask(d, MaskFromVec(same)); + return IfThenElse(mask_same, v, Shuffle01(v)); +#endif +} + +HWY_API Vec128<double> TableLookupLanes(Vec128<double> v, + Indices128<double> idx) { + Vec128<int64_t> vidx{idx.raw}; +#if HWY_TARGET <= HWY_AVX2 + vidx += vidx; // bit1 is the decider (unusual) + return Vec128<double>{_mm_permutevar_pd(v.raw, vidx.raw)}; +#else + // Only 2 lanes: can swap+blend. Choose v if vidx == iota. To avoid a 64-bit + // comparison (expensive on SSSE3), just invert the upper lane and subtract 1 + // to obtain an all-zero or all-one mask. + const Full128<double> d; + const Full128<int64_t> di; + const Vec128<int64_t> same = (vidx ^ Iota(di, 0)) - Set(di, 1); + const Mask128<double> mask_same = RebindMask(d, MaskFromVec(same)); + return IfThenElse(mask_same, v, Shuffle01(v)); +#endif +} + +// ------------------------------ Reverse (Shuffle0123, Shuffle2301) + +// Single lane: no change +template <typename T> +HWY_API Vec128<T, 1> Reverse(Simd<T, 1> /* tag */, const Vec128<T, 1> v) { + return v; +} + +// Two lanes: shuffle +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T, 2> Reverse(Simd<T, 2> /* tag */, const Vec128<T, 2> v) { + return Vec128<T, 2>{Shuffle2301(Vec128<T>{v.raw}).raw}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec128<T> Reverse(Full128<T> /* tag */, const Vec128<T> v) { + return Shuffle01(v); +} + +// Four lanes: shuffle +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T> Reverse(Full128<T> /* tag */, const Vec128<T> v) { + return Shuffle0123(v); +} + +// 16-bit +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Vec128<T, N> Reverse(Simd<T, N> d, const Vec128<T, N> v) { +#if HWY_TARGET <= HWY_AVX3 + if (N == 1) return v; + if (N == 2) { + const Repartition<uint32_t, decltype(d)> du32; + return BitCast(d, RotateRight<16>(BitCast(du32, v))); + } + const RebindToSigned<decltype(d)> di; + alignas(16) constexpr int16_t kReverse[8] = {7, 6, 5, 4, 3, 2, 1, 0}; + const Vec128<int16_t, N> idx = Load(di, kReverse + (N == 8 ? 0 : 4)); + return BitCast(d, Vec128<int16_t, N>{ + _mm_permutexvar_epi16(idx.raw, BitCast(di, v).raw)}); +#else + const RepartitionToWide<RebindToUnsigned<decltype(d)>> du32; + return BitCast(d, RotateRight<16>(Reverse(du32, BitCast(du32, v)))); +#endif +} + +// ------------------------------ InterleaveLower + +// Interleaves lanes from halves of the 128-bit blocks of "a" (which provides +// the least-significant lane) and "b". To concatenate two half-width integers +// into one, use ZipLower/Upper instead (also works with scalar). + +template <size_t N, HWY_IF_LE128(uint8_t, N)> +HWY_API Vec128<uint8_t, N> InterleaveLower(const Vec128<uint8_t, N> a, + const Vec128<uint8_t, N> b) { + return Vec128<uint8_t, N>{_mm_unpacklo_epi8(a.raw, b.raw)}; +} +template <size_t N, HWY_IF_LE128(uint16_t, N)> +HWY_API Vec128<uint16_t, N> InterleaveLower(const Vec128<uint16_t, N> a, + const Vec128<uint16_t, N> b) { + return Vec128<uint16_t, N>{_mm_unpacklo_epi16(a.raw, b.raw)}; +} +template <size_t N, HWY_IF_LE128(uint32_t, N)> +HWY_API Vec128<uint32_t, N> InterleaveLower(const Vec128<uint32_t, N> a, + const Vec128<uint32_t, N> b) { + return Vec128<uint32_t, N>{_mm_unpacklo_epi32(a.raw, b.raw)}; +} +template <size_t N, HWY_IF_LE128(uint64_t, N)> +HWY_API Vec128<uint64_t, N> InterleaveLower(const Vec128<uint64_t, N> a, + const Vec128<uint64_t, N> b) { + return Vec128<uint64_t, N>{_mm_unpacklo_epi64(a.raw, b.raw)}; +} + +template <size_t N, HWY_IF_LE128(int8_t, N)> +HWY_API Vec128<int8_t, N> InterleaveLower(const Vec128<int8_t, N> a, + const Vec128<int8_t, N> b) { + return Vec128<int8_t, N>{_mm_unpacklo_epi8(a.raw, b.raw)}; +} +template <size_t N, HWY_IF_LE128(int16_t, N)> +HWY_API Vec128<int16_t, N> InterleaveLower(const Vec128<int16_t, N> a, + const Vec128<int16_t, N> b) { + return Vec128<int16_t, N>{_mm_unpacklo_epi16(a.raw, b.raw)}; +} +template <size_t N, HWY_IF_LE128(int32_t, N)> +HWY_API Vec128<int32_t, N> InterleaveLower(const Vec128<int32_t, N> a, + const Vec128<int32_t, N> b) { + return Vec128<int32_t, N>{_mm_unpacklo_epi32(a.raw, b.raw)}; +} +template <size_t N, HWY_IF_LE128(int64_t, N)> +HWY_API Vec128<int64_t, N> InterleaveLower(const Vec128<int64_t, N> a, + const Vec128<int64_t, N> b) { + return Vec128<int64_t, N>{_mm_unpacklo_epi64(a.raw, b.raw)}; +} + +template <size_t N, HWY_IF_LE128(float, N)> +HWY_API Vec128<float, N> InterleaveLower(const Vec128<float, N> a, + const Vec128<float, N> b) { + return Vec128<float, N>{_mm_unpacklo_ps(a.raw, b.raw)}; +} +template <size_t N, HWY_IF_LE128(double, N)> +HWY_API Vec128<double, N> InterleaveLower(const Vec128<double, N> a, + const Vec128<double, N> b) { + return Vec128<double, N>{_mm_unpacklo_pd(a.raw, b.raw)}; +} + +// Additional overload for the optional Simd<> tag. +template <typename T, size_t N, HWY_IF_LE128(T, N), class V = Vec128<T, N>> +HWY_API V InterleaveLower(Simd<T, N> /* tag */, V a, V b) { + return InterleaveLower(a, b); +} + +// ------------------------------ InterleaveUpper (UpperHalf) + +// All functions inside detail lack the required D parameter. +namespace detail { + +HWY_API Vec128<uint8_t> InterleaveUpper(const Vec128<uint8_t> a, + const Vec128<uint8_t> b) { + return Vec128<uint8_t>{_mm_unpackhi_epi8(a.raw, b.raw)}; +} +HWY_API Vec128<uint16_t> InterleaveUpper(const Vec128<uint16_t> a, + const Vec128<uint16_t> b) { + return Vec128<uint16_t>{_mm_unpackhi_epi16(a.raw, b.raw)}; +} +HWY_API Vec128<uint32_t> InterleaveUpper(const Vec128<uint32_t> a, + const Vec128<uint32_t> b) { + return Vec128<uint32_t>{_mm_unpackhi_epi32(a.raw, b.raw)}; +} +HWY_API Vec128<uint64_t> InterleaveUpper(const Vec128<uint64_t> a, + const Vec128<uint64_t> b) { + return Vec128<uint64_t>{_mm_unpackhi_epi64(a.raw, b.raw)}; +} + +HWY_API Vec128<int8_t> InterleaveUpper(const Vec128<int8_t> a, + const Vec128<int8_t> b) { + return Vec128<int8_t>{_mm_unpackhi_epi8(a.raw, b.raw)}; +} +HWY_API Vec128<int16_t> InterleaveUpper(const Vec128<int16_t> a, + const Vec128<int16_t> b) { + return Vec128<int16_t>{_mm_unpackhi_epi16(a.raw, b.raw)}; +} +HWY_API Vec128<int32_t> InterleaveUpper(const Vec128<int32_t> a, + const Vec128<int32_t> b) { + return Vec128<int32_t>{_mm_unpackhi_epi32(a.raw, b.raw)}; +} +HWY_API Vec128<int64_t> InterleaveUpper(const Vec128<int64_t> a, + const Vec128<int64_t> b) { + return Vec128<int64_t>{_mm_unpackhi_epi64(a.raw, b.raw)}; +} + +HWY_API Vec128<float> InterleaveUpper(const Vec128<float> a, + const Vec128<float> b) { + return Vec128<float>{_mm_unpackhi_ps(a.raw, b.raw)}; +} +HWY_API Vec128<double> InterleaveUpper(const Vec128<double> a, + const Vec128<double> b) { + return Vec128<double>{_mm_unpackhi_pd(a.raw, b.raw)}; +} + +} // namespace detail + +// Full +template <typename T, class V = Vec128<T>> +HWY_API V InterleaveUpper(Full128<T> /* tag */, V a, V b) { + return detail::InterleaveUpper(a, b); +} + +// Partial +template <typename T, size_t N, HWY_IF_LE64(T, N), class V = Vec128<T, N>> +HWY_API V InterleaveUpper(Simd<T, N> d, V a, V b) { + const Half<decltype(d)> d2; + return InterleaveLower(d, V{UpperHalf(d2, a).raw}, V{UpperHalf(d2, b).raw}); +} + +// ------------------------------ ZipLower/ZipUpper (InterleaveLower) + +// Same as Interleave*, except that the return lanes are double-width integers; +// this is necessary because the single-lane scalar cannot return two values. +template <typename T, size_t N, class DW = RepartitionToWide<Simd<T, N>>> +HWY_API VFromD<DW> ZipLower(Vec128<T, N> a, Vec128<T, N> b) { + return BitCast(DW(), InterleaveLower(a, b)); +} +template <typename T, size_t N, class D = Simd<T, N>, + class DW = RepartitionToWide<D>> +HWY_API VFromD<DW> ZipLower(DW dw, Vec128<T, N> a, Vec128<T, N> b) { + return BitCast(dw, InterleaveLower(D(), a, b)); +} + +template <typename T, size_t N, class D = Simd<T, N>, + class DW = RepartitionToWide<D>> +HWY_API VFromD<DW> ZipUpper(DW dw, Vec128<T, N> a, Vec128<T, N> b) { + return BitCast(dw, InterleaveUpper(D(), a, b)); +} + +// ================================================== COMBINE + +// ------------------------------ Combine (InterleaveLower) + +// N = N/2 + N/2 (upper half undefined) +template <typename T, size_t N, HWY_IF_LE128(T, N)> +HWY_API Vec128<T, N> Combine(Simd<T, N> d, Vec128<T, N / 2> hi_half, + Vec128<T, N / 2> lo_half) { + const Half<decltype(d)> d2; + const RebindToUnsigned<decltype(d2)> du2; + // Treat half-width input as one lane, and expand to two lanes. + using VU = Vec128<UnsignedFromSize<N * sizeof(T) / 2>, 2>; + const VU lo{BitCast(du2, lo_half).raw}; + const VU hi{BitCast(du2, hi_half).raw}; + return BitCast(d, InterleaveLower(lo, hi)); +} + +// ------------------------------ ZeroExtendVector (Combine, IfThenElseZero) + +template <typename T, HWY_IF_NOT_FLOAT(T)> +HWY_API Vec128<T> ZeroExtendVector(Full128<T> /* tag */, + Vec128<T, 8 / sizeof(T)> lo) { + return Vec128<T>{_mm_move_epi64(lo.raw)}; +} + +template <typename T, HWY_IF_FLOAT(T)> +HWY_API Vec128<T> ZeroExtendVector(Full128<T> d, Vec128<T, 8 / sizeof(T)> lo) { + const RebindToUnsigned<decltype(d)> du; + return BitCast(d, ZeroExtendVector(du, BitCast(Half<decltype(du)>(), lo))); +} + +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_API Vec128<T, N> ZeroExtendVector(Simd<T, N> d, Vec128<T, N / 2> lo) { + return IfThenElseZero(FirstN(d, N / 2), Vec128<T, N>{lo.raw}); +} + +// ------------------------------ Concat full (InterleaveLower) + +// hiH,hiL loH,loL |-> hiL,loL (= lower halves) +template <typename T> +HWY_API Vec128<T> ConcatLowerLower(Full128<T> d, Vec128<T> hi, Vec128<T> lo) { + const Repartition<uint64_t, decltype(d)> d64; + return BitCast(d, InterleaveLower(BitCast(d64, lo), BitCast(d64, hi))); +} + +// hiH,hiL loH,loL |-> hiH,loH (= upper halves) +template <typename T> +HWY_API Vec128<T> ConcatUpperUpper(Full128<T> d, Vec128<T> hi, Vec128<T> lo) { + const Repartition<uint64_t, decltype(d)> d64; + return BitCast(d, InterleaveUpper(d64, BitCast(d64, lo), BitCast(d64, hi))); +} + +// hiH,hiL loH,loL |-> hiL,loH (= inner halves) +template <typename T> +HWY_API Vec128<T> ConcatLowerUpper(Full128<T> d, const Vec128<T> hi, + const Vec128<T> lo) { + return CombineShiftRightBytes<8>(d, hi, lo); +} + +// hiH,hiL loH,loL |-> hiH,loL (= outer halves) +template <typename T> +HWY_API Vec128<T> ConcatUpperLower(Full128<T> d, Vec128<T> hi, Vec128<T> lo) { +#if HWY_TARGET == HWY_SSSE3 + const Full128<double> dd; + const __m128d concat = _mm_move_sd(BitCast(dd, hi).raw, BitCast(dd, lo).raw); + return BitCast(d, Vec128<double>{concat}); +#else + (void)d; + return Vec128<T>{_mm_blend_epi16(hi.raw, lo.raw, 0x0F)}; +#endif +} +HWY_API Vec128<float> ConcatUpperLower(Full128<float> /* tag */, + const Vec128<float> hi, + const Vec128<float> lo) { + return Vec128<float>{_mm_shuffle_ps(lo.raw, hi.raw, _MM_SHUFFLE(3, 2, 1, 0))}; +} +HWY_API Vec128<double> ConcatUpperLower(Full128<double> /* tag */, + const Vec128<double> hi, + const Vec128<double> lo) { + return Vec128<double>{_mm_shuffle_pd(lo.raw, hi.raw, _MM_SHUFFLE2(1, 0))}; +} + +// ------------------------------ Concat partial (Combine, LowerHalf) + +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_API Vec128<T, N> ConcatLowerLower(Simd<T, N> d, Vec128<T, N> hi, + Vec128<T, N> lo) { + const Half<decltype(d)> d2; + return Combine(LowerHalf(d2, hi), LowerHalf(d2, lo)); +} + +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_API Vec128<T, N> ConcatUpperUpper(Simd<T, N> d, Vec128<T, N> hi, + Vec128<T, N> lo) { + const Half<decltype(d)> d2; + return Combine(UpperHalf(d2, hi), UpperHalf(d2, lo)); +} + +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_API Vec128<T, N> ConcatLowerUpper(Simd<T, N> d, const Vec128<T, N> hi, + const Vec128<T, N> lo) { + const Half<decltype(d)> d2; + return Combine(LowerHalf(d2, hi), UpperHalf(d2, lo)); +} + +template <typename T, size_t N, HWY_IF_LE64(T, N)> +HWY_API Vec128<T, N> ConcatUpperLower(Simd<T, N> d, Vec128<T, N> hi, + Vec128<T, N> lo) { + const Half<decltype(d)> d2; + return Combine(UpperHalf(d2, hi), LowerHalf(d2, lo)); +} + +// ------------------------------ ConcatOdd + +// 32-bit full +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T> ConcatOdd(Full128<T> d, Vec128<T> hi, Vec128<T> lo) { + const RebindToFloat<decltype(d)> df; + return BitCast( + d, Vec128<float>{_mm_shuffle_ps(BitCast(df, lo).raw, BitCast(df, hi).raw, + _MM_SHUFFLE(3, 1, 3, 1))}); +} +template <size_t N> +HWY_API Vec128<float> ConcatOdd(Full128<float> /* tag */, Vec128<float> hi, + Vec128<float> lo) { + return Vec128<float>{_mm_shuffle_ps(lo.raw, hi.raw, _MM_SHUFFLE(3, 1, 3, 1))}; +} + +// 32-bit partial +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T, 2> ConcatOdd(Simd<T, 2> d, Vec128<T, 2> hi, Vec128<T, 2> lo) { + return InterleaveUpper(d, lo, hi); +} + +// 64-bit full - no partial because we need at least two inputs to have +// even/odd. +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec128<T> ConcatOdd(Full128<T> d, Vec128<T> hi, Vec128<T> lo) { + return InterleaveUpper(d, lo, hi); +} + +// ------------------------------ ConcatEven (InterleaveLower) + +// 32-bit full +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T> ConcatEven(Full128<T> d, Vec128<T> hi, Vec128<T> lo) { + const RebindToFloat<decltype(d)> df; + return BitCast( + d, Vec128<float>{_mm_shuffle_ps(BitCast(df, lo).raw, BitCast(df, hi).raw, + _MM_SHUFFLE(2, 0, 2, 0))}); +} +template <size_t N> +HWY_API Vec128<float> ConcatEven(Full128<float> /* tag */, Vec128<float> hi, + Vec128<float> lo) { + return Vec128<float>{_mm_shuffle_ps(lo.raw, hi.raw, _MM_SHUFFLE(2, 0, 2, 0))}; +} + +// 32-bit partial +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T, 2> ConcatEven(Simd<T, 2> d, Vec128<T, 2> hi, + Vec128<T, 2> lo) { + return InterleaveLower(d, lo, hi); +} + +// 64-bit full - no partial because we need at least two inputs to have +// even/odd. +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec128<T> ConcatEven(Full128<T> d, Vec128<T> hi, Vec128<T> lo) { + return InterleaveLower(d, lo, hi); +} + +// ------------------------------ OddEven (IfThenElse) + +namespace detail { + +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> OddEven(hwy::SizeTag<1> /* tag */, const Vec128<T, N> a, + const Vec128<T, N> b) { + const Simd<T, N> d; + const Repartition<uint8_t, decltype(d)> d8; + alignas(16) constexpr uint8_t mask[16] = {0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, + 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0}; + return IfThenElse(MaskFromVec(BitCast(d, Load(d8, mask))), b, a); +} +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> OddEven(hwy::SizeTag<2> /* tag */, const Vec128<T, N> a, + const Vec128<T, N> b) { +#if HWY_TARGET == HWY_SSSE3 + const Simd<T, N> d; + const Repartition<uint8_t, decltype(d)> d8; + alignas(16) constexpr uint8_t mask[16] = {0xFF, 0xFF, 0, 0, 0xFF, 0xFF, 0, 0, + 0xFF, 0xFF, 0, 0, 0xFF, 0xFF, 0, 0}; + return IfThenElse(MaskFromVec(BitCast(d, Load(d8, mask))), b, a); +#else + return Vec128<T, N>{_mm_blend_epi16(a.raw, b.raw, 0x55)}; +#endif +} +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> OddEven(hwy::SizeTag<4> /* tag */, const Vec128<T, N> a, + const Vec128<T, N> b) { +#if HWY_TARGET == HWY_SSSE3 + const __m128i odd = _mm_shuffle_epi32(a.raw, _MM_SHUFFLE(3, 1, 3, 1)); + const __m128i even = _mm_shuffle_epi32(b.raw, _MM_SHUFFLE(2, 0, 2, 0)); + return Vec128<T, N>{_mm_unpacklo_epi32(even, odd)}; +#else + return Vec128<T, N>{_mm_blend_epi16(a.raw, b.raw, 0x33)}; +#endif +} +template <typename T, size_t N> +HWY_INLINE Vec128<T, N> OddEven(hwy::SizeTag<8> /* tag */, const Vec128<T, N> a, + const Vec128<T, N> b) { +#if HWY_TARGET == HWY_SSSE3 + const Full128<double> dd; + const __m128d concat = _mm_move_sd(BitCast(dd, a).raw, BitCast(dd, b).raw); + return BitCast(Full128<T>(), Vec128<double>{concat}); +#else + return Vec128<T, N>{_mm_blend_epi16(a.raw, b.raw, 0x0F)}; +#endif +} + +} // namespace detail + +template <typename T, size_t N> +HWY_API Vec128<T, N> OddEven(const Vec128<T, N> a, const Vec128<T, N> b) { + return detail::OddEven(hwy::SizeTag<sizeof(T)>(), a, b); +} +template <size_t N> +HWY_API Vec128<float, N> OddEven(const Vec128<float, N> a, + const Vec128<float, N> b) { +#if HWY_TARGET == HWY_SSSE3 + // SHUFPS must fill the lower half of the output from one register, so we + // need another shuffle. Unpack avoids another immediate byte. + const __m128 odd = _mm_shuffle_ps(a.raw, a.raw, _MM_SHUFFLE(3, 1, 3, 1)); + const __m128 even = _mm_shuffle_ps(b.raw, b.raw, _MM_SHUFFLE(2, 0, 2, 0)); + return Vec128<float, N>{_mm_unpacklo_ps(even, odd)}; +#else + return Vec128<float, N>{_mm_blend_ps(a.raw, b.raw, 5)}; +#endif +} + +template <size_t N> +HWY_API Vec128<double, N> OddEven(const Vec128<double, N> a, + const Vec128<double, N> b) { + return Vec128<double>{_mm_shuffle_pd(b.raw, a.raw, _MM_SHUFFLE2(1, 0))}; +} + +// ------------------------------ OddEvenBlocks +template <typename T, size_t N> +HWY_API Vec128<T, N> OddEvenBlocks(Vec128<T, N> /* odd */, Vec128<T, N> even) { + return even; +} + +// ------------------------------ SwapAdjacentBlocks + +template <typename T, size_t N> +HWY_API Vec128<T, N> SwapAdjacentBlocks(Vec128<T, N> v) { + return v; +} + +// ------------------------------ Shl (ZipLower, Mul) + +// Use AVX2/3 variable shifts where available, otherwise multiply by powers of +// two from loading float exponents, which is considerably faster (according +// to LLVM-MCA) than scalar or testing bits: https://gcc.godbolt.org/z/9G7Y9v. + +#if HWY_TARGET > HWY_AVX3 // AVX2 or older +namespace detail { + +// Returns 2^v for use as per-lane multipliers to emulate 16-bit shifts. +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 2)> +HWY_INLINE Vec128<MakeUnsigned<T>, N> Pow2(const Vec128<T, N> v) { + const Simd<T, N> d; + const RepartitionToWide<decltype(d)> dw; + const Rebind<float, decltype(dw)> df; + const auto zero = Zero(d); + // Move into exponent (this u16 will become the upper half of an f32) + const auto exp = ShiftLeft<23 - 16>(v); + const auto upper = exp + Set(d, 0x3F80); // upper half of 1.0f + // Insert 0 into lower halves for reinterpreting as binary32. + const auto f0 = ZipLower(dw, zero, upper); + const auto f1 = ZipUpper(dw, zero, upper); + // See comment below. + const Vec128<int32_t, N> bits0{_mm_cvtps_epi32(BitCast(df, f0).raw)}; + const Vec128<int32_t, N> bits1{_mm_cvtps_epi32(BitCast(df, f1).raw)}; + return Vec128<MakeUnsigned<T>, N>{_mm_packus_epi32(bits0.raw, bits1.raw)}; +} + +// Same, for 32-bit shifts. +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 4)> +HWY_INLINE Vec128<MakeUnsigned<T>, N> Pow2(const Vec128<T, N> v) { + const Simd<T, N> d; + const auto exp = ShiftLeft<23>(v); + const auto f = exp + Set(d, 0x3F800000); // 1.0f + // Do not use ConvertTo because we rely on the native 0x80..00 overflow + // behavior. cvt instead of cvtt should be equivalent, but avoids test + // failure under GCC 10.2.1. + return Vec128<MakeUnsigned<T>, N>{_mm_cvtps_epi32(_mm_castsi128_ps(f.raw))}; +} + +} // namespace detail +#endif // HWY_TARGET > HWY_AVX3 + +template <size_t N> +HWY_API Vec128<uint16_t, N> operator<<(const Vec128<uint16_t, N> v, + const Vec128<uint16_t, N> bits) { +#if HWY_TARGET <= HWY_AVX3 + return Vec128<uint16_t, N>{_mm_sllv_epi16(v.raw, bits.raw)}; +#else + return v * detail::Pow2(bits); +#endif +} +HWY_API Vec128<uint16_t, 1> operator<<(const Vec128<uint16_t, 1> v, + const Vec128<uint16_t, 1> bits) { + return Vec128<uint16_t, 1>{_mm_sll_epi16(v.raw, bits.raw)}; +} + +template <size_t N> +HWY_API Vec128<uint32_t, N> operator<<(const Vec128<uint32_t, N> v, + const Vec128<uint32_t, N> bits) { +#if HWY_TARGET == HWY_SSSE3 || HWY_TARGET == HWY_SSE4 + return v * detail::Pow2(bits); +#else + return Vec128<uint32_t, N>{_mm_sllv_epi32(v.raw, bits.raw)}; +#endif +} +HWY_API Vec128<uint32_t, 1> operator<<(const Vec128<uint32_t, 1> v, + const Vec128<uint32_t, 1> bits) { + return Vec128<uint32_t, 1>{_mm_sll_epi32(v.raw, bits.raw)}; +} + +HWY_API Vec128<uint64_t> operator<<(const Vec128<uint64_t> v, + const Vec128<uint64_t> bits) { +#if HWY_TARGET == HWY_SSSE3 || HWY_TARGET == HWY_SSE4 + // Individual shifts and combine + const Vec128<uint64_t> out0{_mm_sll_epi64(v.raw, bits.raw)}; + const __m128i bits1 = _mm_unpackhi_epi64(bits.raw, bits.raw); + const Vec128<uint64_t> out1{_mm_sll_epi64(v.raw, bits1)}; + return ConcatUpperLower(Full128<uint64_t>(), out1, out0); +#else + return Vec128<uint64_t>{_mm_sllv_epi64(v.raw, bits.raw)}; +#endif +} +HWY_API Vec128<uint64_t, 1> operator<<(const Vec128<uint64_t, 1> v, + const Vec128<uint64_t, 1> bits) { + return Vec128<uint64_t, 1>{_mm_sll_epi64(v.raw, bits.raw)}; +} + +// Signed left shift is the same as unsigned. +template <typename T, size_t N, HWY_IF_SIGNED(T)> +HWY_API Vec128<T, N> operator<<(const Vec128<T, N> v, const Vec128<T, N> bits) { + const Simd<T, N> di; + const Simd<MakeUnsigned<T>, N> du; + return BitCast(di, BitCast(du, v) << BitCast(du, bits)); +} + +// ------------------------------ Shr (mul, mask, BroadcastSignBit) + +// Use AVX2+ variable shifts except for SSSE3/SSE4 or 16-bit. There, we use +// widening multiplication by powers of two obtained by loading float exponents, +// followed by a constant right-shift. This is still faster than a scalar or +// bit-test approach: https://gcc.godbolt.org/z/9G7Y9v. + +template <size_t N> +HWY_API Vec128<uint16_t, N> operator>>(const Vec128<uint16_t, N> in, + const Vec128<uint16_t, N> bits) { +#if HWY_TARGET <= HWY_AVX3 + return Vec128<uint16_t, N>{_mm_srlv_epi16(in.raw, bits.raw)}; +#else + const Simd<uint16_t, N> d; + // For bits=0, we cannot mul by 2^16, so fix the result later. + const auto out = MulHigh(in, detail::Pow2(Set(d, 16) - bits)); + // Replace output with input where bits == 0. + return IfThenElse(bits == Zero(d), in, out); +#endif +} +HWY_API Vec128<uint16_t, 1> operator>>(const Vec128<uint16_t, 1> in, + const Vec128<uint16_t, 1> bits) { + return Vec128<uint16_t, 1>{_mm_srl_epi16(in.raw, bits.raw)}; +} + +template <size_t N> +HWY_API Vec128<uint32_t, N> operator>>(const Vec128<uint32_t, N> in, + const Vec128<uint32_t, N> bits) { +#if HWY_TARGET == HWY_SSSE3 || HWY_TARGET == HWY_SSE4 + // 32x32 -> 64 bit mul, then shift right by 32. + const Simd<uint32_t, N> d32; + // Move odd lanes into position for the second mul. Shuffle more gracefully + // handles N=1 than repartitioning to u64 and shifting 32 bits right. + const Vec128<uint32_t, N> in31{_mm_shuffle_epi32(in.raw, 0x31)}; + // For bits=0, we cannot mul by 2^32, so fix the result later. + const auto mul = detail::Pow2(Set(d32, 32) - bits); + const auto out20 = ShiftRight<32>(MulEven(in, mul)); // z 2 z 0 + const Vec128<uint32_t, N> mul31{_mm_shuffle_epi32(mul.raw, 0x31)}; + // No need to shift right, already in the correct position. + const auto out31 = BitCast(d32, MulEven(in31, mul31)); // 3 ? 1 ? + const Vec128<uint32_t, N> out = OddEven(out31, BitCast(d32, out20)); + // Replace output with input where bits == 0. + return IfThenElse(bits == Zero(d32), in, out); +#else + return Vec128<uint32_t, N>{_mm_srlv_epi32(in.raw, bits.raw)}; +#endif +} +HWY_API Vec128<uint32_t, 1> operator>>(const Vec128<uint32_t, 1> in, + const Vec128<uint32_t, 1> bits) { + return Vec128<uint32_t, 1>{_mm_srl_epi32(in.raw, bits.raw)}; +} + +HWY_API Vec128<uint64_t> operator>>(const Vec128<uint64_t> v, + const Vec128<uint64_t> bits) { +#if HWY_TARGET == HWY_SSSE3 || HWY_TARGET == HWY_SSE4 + // Individual shifts and combine + const Vec128<uint64_t> out0{_mm_srl_epi64(v.raw, bits.raw)}; + const __m128i bits1 = _mm_unpackhi_epi64(bits.raw, bits.raw); + const Vec128<uint64_t> out1{_mm_srl_epi64(v.raw, bits1)}; + return ConcatUpperLower(Full128<uint64_t>(), out1, out0); +#else + return Vec128<uint64_t>{_mm_srlv_epi64(v.raw, bits.raw)}; +#endif +} +HWY_API Vec128<uint64_t, 1> operator>>(const Vec128<uint64_t, 1> v, + const Vec128<uint64_t, 1> bits) { + return Vec128<uint64_t, 1>{_mm_srl_epi64(v.raw, bits.raw)}; +} + +#if HWY_TARGET > HWY_AVX3 // AVX2 or older +namespace detail { + +// Also used in x86_256-inl.h. +template <class DI, class V> +HWY_INLINE V SignedShr(const DI di, const V v, const V count_i) { + const RebindToUnsigned<DI> du; + const auto count = BitCast(du, count_i); // same type as value to shift + // Clear sign and restore afterwards. This is preferable to shifting the MSB + // downwards because Shr is somewhat more expensive than Shl. + const auto sign = BroadcastSignBit(v); + const auto abs = BitCast(du, v ^ sign); // off by one, but fixed below + return BitCast(di, abs >> count) ^ sign; +} + +} // namespace detail +#endif // HWY_TARGET > HWY_AVX3 + +template <size_t N> +HWY_API Vec128<int16_t, N> operator>>(const Vec128<int16_t, N> v, + const Vec128<int16_t, N> bits) { +#if HWY_TARGET <= HWY_AVX3 + return Vec128<int16_t, N>{_mm_srav_epi16(v.raw, bits.raw)}; +#else + return detail::SignedShr(Simd<int16_t, N>(), v, bits); +#endif +} +HWY_API Vec128<int16_t, 1> operator>>(const Vec128<int16_t, 1> v, + const Vec128<int16_t, 1> bits) { + return Vec128<int16_t, 1>{_mm_sra_epi16(v.raw, bits.raw)}; +} + +template <size_t N> +HWY_API Vec128<int32_t, N> operator>>(const Vec128<int32_t, N> v, + const Vec128<int32_t, N> bits) { +#if HWY_TARGET <= HWY_AVX3 + return Vec128<int32_t, N>{_mm_srav_epi32(v.raw, bits.raw)}; +#else + return detail::SignedShr(Simd<int32_t, N>(), v, bits); +#endif +} +HWY_API Vec128<int32_t, 1> operator>>(const Vec128<int32_t, 1> v, + const Vec128<int32_t, 1> bits) { + return Vec128<int32_t, 1>{_mm_sra_epi32(v.raw, bits.raw)}; +} + +template <size_t N> +HWY_API Vec128<int64_t, N> operator>>(const Vec128<int64_t, N> v, + const Vec128<int64_t, N> bits) { +#if HWY_TARGET <= HWY_AVX3 + return Vec128<int64_t, N>{_mm_srav_epi64(v.raw, bits.raw)}; +#else + return detail::SignedShr(Simd<int64_t, N>(), v, bits); +#endif +} + +// ------------------------------ MulEven/Odd 64x64 (UpperHalf) + +HWY_INLINE Vec128<uint64_t> MulEven(const Vec128<uint64_t> a, + const Vec128<uint64_t> b) { + alignas(16) uint64_t mul[2]; + mul[0] = Mul128(GetLane(a), GetLane(b), &mul[1]); + return Load(Full128<uint64_t>(), mul); +} + +HWY_INLINE Vec128<uint64_t> MulOdd(const Vec128<uint64_t> a, + const Vec128<uint64_t> b) { + alignas(16) uint64_t mul[2]; + const Half<Full128<uint64_t>> d2; + mul[0] = + Mul128(GetLane(UpperHalf(d2, a)), GetLane(UpperHalf(d2, b)), &mul[1]); + return Load(Full128<uint64_t>(), mul); +} + +// ------------------------------ ReorderWidenMulAccumulate (MulAdd, ZipLower) + +template <size_t N> +HWY_API Vec128<float, N> ReorderWidenMulAccumulate(Simd<float, N> df32, + Vec128<bfloat16_t, 2 * N> a, + Vec128<bfloat16_t, 2 * N> b, + const Vec128<float, N> sum0, + Vec128<float, N>& sum1) { + // TODO(janwas): _mm_dpbf16_ps when available + const Repartition<uint16_t, decltype(df32)> du16; + const RebindToUnsigned<decltype(df32)> du32; + const Vec128<uint16_t, 2 * N> zero = Zero(du16); + // Lane order within sum0/1 is undefined, hence we can avoid the + // longer-latency lane-crossing PromoteTo. + const Vec128<uint32_t, N> a0 = ZipLower(du32, zero, BitCast(du16, a)); + const Vec128<uint32_t, N> a1 = ZipUpper(du32, zero, BitCast(du16, a)); + const Vec128<uint32_t, N> b0 = ZipLower(du32, zero, BitCast(du16, b)); + const Vec128<uint32_t, N> b1 = ZipUpper(du32, zero, BitCast(du16, b)); + sum1 = MulAdd(BitCast(df32, a1), BitCast(df32, b1), sum1); + return MulAdd(BitCast(df32, a0), BitCast(df32, b0), sum0); +} + +// ================================================== CONVERT + +// ------------------------------ Promotions (part w/ narrow lanes -> full) + +// Unsigned: zero-extend. +template <size_t N> +HWY_API Vec128<uint16_t, N> PromoteTo(Simd<uint16_t, N> /* tag */, + const Vec128<uint8_t, N> v) { +#if HWY_TARGET == HWY_SSSE3 + const __m128i zero = _mm_setzero_si128(); + return Vec128<uint16_t, N>{_mm_unpacklo_epi8(v.raw, zero)}; +#else + return Vec128<uint16_t, N>{_mm_cvtepu8_epi16(v.raw)}; +#endif +} +template <size_t N> +HWY_API Vec128<uint32_t, N> PromoteTo(Simd<uint32_t, N> /* tag */, + const Vec128<uint16_t, N> v) { +#if HWY_TARGET == HWY_SSSE3 + return Vec128<uint32_t, N>{_mm_unpacklo_epi16(v.raw, _mm_setzero_si128())}; +#else + return Vec128<uint32_t, N>{_mm_cvtepu16_epi32(v.raw)}; +#endif +} +template <size_t N> +HWY_API Vec128<uint64_t, N> PromoteTo(Simd<uint64_t, N> /* tag */, + const Vec128<uint32_t, N> v) { +#if HWY_TARGET == HWY_SSSE3 + return Vec128<uint64_t, N>{_mm_unpacklo_epi32(v.raw, _mm_setzero_si128())}; +#else + return Vec128<uint64_t, N>{_mm_cvtepu32_epi64(v.raw)}; +#endif +} +template <size_t N> +HWY_API Vec128<uint32_t, N> PromoteTo(Simd<uint32_t, N> /* tag */, + const Vec128<uint8_t, N> v) { +#if HWY_TARGET == HWY_SSSE3 + const __m128i zero = _mm_setzero_si128(); + const __m128i u16 = _mm_unpacklo_epi8(v.raw, zero); + return Vec128<uint32_t, N>{_mm_unpacklo_epi16(u16, zero)}; +#else + return Vec128<uint32_t, N>{_mm_cvtepu8_epi32(v.raw)}; +#endif +} + +// Unsigned to signed: same plus cast. +template <size_t N> +HWY_API Vec128<int16_t, N> PromoteTo(Simd<int16_t, N> di, + const Vec128<uint8_t, N> v) { + return BitCast(di, PromoteTo(Simd<uint16_t, N>(), v)); +} +template <size_t N> +HWY_API Vec128<int32_t, N> PromoteTo(Simd<int32_t, N> di, + const Vec128<uint16_t, N> v) { + return BitCast(di, PromoteTo(Simd<uint32_t, N>(), v)); +} +template <size_t N> +HWY_API Vec128<int32_t, N> PromoteTo(Simd<int32_t, N> di, + const Vec128<uint8_t, N> v) { + return BitCast(di, PromoteTo(Simd<uint32_t, N>(), v)); +} + +// Signed: replicate sign bit. +template <size_t N> +HWY_API Vec128<int16_t, N> PromoteTo(Simd<int16_t, N> /* tag */, + const Vec128<int8_t, N> v) { +#if HWY_TARGET == HWY_SSSE3 + return ShiftRight<8>(Vec128<int16_t, N>{_mm_unpacklo_epi8(v.raw, v.raw)}); +#else + return Vec128<int16_t, N>{_mm_cvtepi8_epi16(v.raw)}; +#endif +} +template <size_t N> +HWY_API Vec128<int32_t, N> PromoteTo(Simd<int32_t, N> /* tag */, + const Vec128<int16_t, N> v) { +#if HWY_TARGET == HWY_SSSE3 + return ShiftRight<16>(Vec128<int32_t, N>{_mm_unpacklo_epi16(v.raw, v.raw)}); +#else + return Vec128<int32_t, N>{_mm_cvtepi16_epi32(v.raw)}; +#endif +} +template <size_t N> +HWY_API Vec128<int64_t, N> PromoteTo(Simd<int64_t, N> /* tag */, + const Vec128<int32_t, N> v) { +#if HWY_TARGET == HWY_SSSE3 + return ShiftRight<32>(Vec128<int64_t, N>{_mm_unpacklo_epi32(v.raw, v.raw)}); +#else + return Vec128<int64_t, N>{_mm_cvtepi32_epi64(v.raw)}; +#endif +} +template <size_t N> +HWY_API Vec128<int32_t, N> PromoteTo(Simd<int32_t, N> /* tag */, + const Vec128<int8_t, N> v) { +#if HWY_TARGET == HWY_SSSE3 + const __m128i x2 = _mm_unpacklo_epi8(v.raw, v.raw); + const __m128i x4 = _mm_unpacklo_epi16(x2, x2); + return ShiftRight<24>(Vec128<int32_t, N>{x4}); +#else + return Vec128<int32_t, N>{_mm_cvtepi8_epi32(v.raw)}; +#endif +} + +// Workaround for origin tracking bug in Clang msan prior to 11.0 +// (spurious "uninitialized memory" for TestF16 with "ORIGIN: invalid") +#if defined(MEMORY_SANITIZER) && \ + (HWY_COMPILER_CLANG != 0 && HWY_COMPILER_CLANG < 1100) +#define HWY_INLINE_F16 HWY_NOINLINE +#else +#define HWY_INLINE_F16 HWY_INLINE +#endif +template <size_t N> +HWY_INLINE_F16 Vec128<float, N> PromoteTo(Simd<float, N> df32, + const Vec128<float16_t, N> v) { +#if HWY_TARGET >= HWY_SSE4 || defined(HWY_DISABLE_F16C) + const RebindToSigned<decltype(df32)> di32; + const RebindToUnsigned<decltype(df32)> du32; + // Expand to u32 so we can shift. + const auto bits16 = PromoteTo(du32, Vec128<uint16_t, N>{v.raw}); + const auto sign = ShiftRight<15>(bits16); + const auto biased_exp = ShiftRight<10>(bits16) & Set(du32, 0x1F); + const auto mantissa = bits16 & Set(du32, 0x3FF); + const auto subnormal = + BitCast(du32, ConvertTo(df32, BitCast(di32, mantissa)) * + Set(df32, 1.0f / 16384 / 1024)); + + const auto biased_exp32 = biased_exp + Set(du32, 127 - 15); + const auto mantissa32 = ShiftLeft<23 - 10>(mantissa); + const auto normal = ShiftLeft<23>(biased_exp32) | mantissa32; + const auto bits32 = IfThenElse(biased_exp == Zero(du32), subnormal, normal); + return BitCast(df32, ShiftLeft<31>(sign) | bits32); +#else + (void)df32; + return Vec128<float, N>{_mm_cvtph_ps(v.raw)}; +#endif +} + +template <size_t N> +HWY_API Vec128<float, N> PromoteTo(Simd<float, N> df32, + const Vec128<bfloat16_t, N> v) { + const Rebind<uint16_t, decltype(df32)> du16; + const RebindToSigned<decltype(df32)> di32; + return BitCast(df32, ShiftLeft<16>(PromoteTo(di32, BitCast(du16, v)))); +} + +template <size_t N> +HWY_API Vec128<double, N> PromoteTo(Simd<double, N> /* tag */, + const Vec128<float, N> v) { + return Vec128<double, N>{_mm_cvtps_pd(v.raw)}; +} + +template <size_t N> +HWY_API Vec128<double, N> PromoteTo(Simd<double, N> /* tag */, + const Vec128<int32_t, N> v) { + return Vec128<double, N>{_mm_cvtepi32_pd(v.raw)}; +} + +// ------------------------------ Demotions (full -> part w/ narrow lanes) + +template <size_t N> +HWY_API Vec128<uint16_t, N> DemoteTo(Simd<uint16_t, N> /* tag */, + const Vec128<int32_t, N> v) { +#if HWY_TARGET == HWY_SSSE3 + const Simd<int32_t, N> di32; + const Simd<uint16_t, N * 2> du16; + const auto zero_if_neg = AndNot(ShiftRight<31>(v), v); + const auto too_big = VecFromMask(di32, Gt(v, Set(di32, 0xFFFF))); + const auto clamped = Or(zero_if_neg, too_big); + // Lower 2 bytes from each 32-bit lane; same as return type for fewer casts. + alignas(16) constexpr uint16_t kLower2Bytes[16] = { + 0x0100, 0x0504, 0x0908, 0x0D0C, 0x8080, 0x8080, 0x8080, 0x8080}; + const auto lo2 = Load(du16, kLower2Bytes); + return Vec128<uint16_t, N>{TableLookupBytes(BitCast(du16, clamped), lo2).raw}; +#else + return Vec128<uint16_t, N>{_mm_packus_epi32(v.raw, v.raw)}; +#endif +} + +template <size_t N> +HWY_API Vec128<int16_t, N> DemoteTo(Simd<int16_t, N> /* tag */, + const Vec128<int32_t, N> v) { + return Vec128<int16_t, N>{_mm_packs_epi32(v.raw, v.raw)}; +} + +template <size_t N> +HWY_API Vec128<uint8_t, N> DemoteTo(Simd<uint8_t, N> /* tag */, + const Vec128<int32_t, N> v) { + const __m128i i16 = _mm_packs_epi32(v.raw, v.raw); + return Vec128<uint8_t, N>{_mm_packus_epi16(i16, i16)}; +} + +template <size_t N> +HWY_API Vec128<uint8_t, N> DemoteTo(Simd<uint8_t, N> /* tag */, + const Vec128<int16_t, N> v) { + return Vec128<uint8_t, N>{_mm_packus_epi16(v.raw, v.raw)}; +} + +template <size_t N> +HWY_API Vec128<int8_t, N> DemoteTo(Simd<int8_t, N> /* tag */, + const Vec128<int32_t, N> v) { + const __m128i i16 = _mm_packs_epi32(v.raw, v.raw); + return Vec128<int8_t, N>{_mm_packs_epi16(i16, i16)}; +} + +template <size_t N> +HWY_API Vec128<int8_t, N> DemoteTo(Simd<int8_t, N> /* tag */, + const Vec128<int16_t, N> v) { + return Vec128<int8_t, N>{_mm_packs_epi16(v.raw, v.raw)}; +} + +template <size_t N> +HWY_API Vec128<float16_t, N> DemoteTo(Simd<float16_t, N> df16, + const Vec128<float, N> v) { +#if HWY_TARGET >= HWY_SSE4 || defined(HWY_DISABLE_F16C) + const RebindToUnsigned<decltype(df16)> du16; + const Rebind<uint32_t, decltype(df16)> du; + const RebindToSigned<decltype(du)> di; + const auto bits32 = BitCast(du, v); + const auto sign = ShiftRight<31>(bits32); + const auto biased_exp32 = ShiftRight<23>(bits32) & Set(du, 0xFF); + const auto mantissa32 = bits32 & Set(du, 0x7FFFFF); + + const auto k15 = Set(di, 15); + const auto exp = Min(BitCast(di, biased_exp32) - Set(di, 127), k15); + const auto is_tiny = exp < Set(di, -24); + + const auto is_subnormal = exp < Set(di, -14); + const auto biased_exp16 = + BitCast(du, IfThenZeroElse(is_subnormal, exp + k15)); + const auto sub_exp = BitCast(du, Set(di, -14) - exp); // [1, 11) + const auto sub_m = (Set(du, 1) << (Set(du, 10) - sub_exp)) + + (mantissa32 >> (Set(du, 13) + sub_exp)); + const auto mantissa16 = IfThenElse(RebindMask(du, is_subnormal), sub_m, + ShiftRight<13>(mantissa32)); // <1024 + + const auto sign16 = ShiftLeft<15>(sign); + const auto normal16 = sign16 | ShiftLeft<10>(biased_exp16) | mantissa16; + const auto bits16 = IfThenZeroElse(is_tiny, BitCast(di, normal16)); + return BitCast(df16, DemoteTo(du16, bits16)); +#else + (void)df16; + return Vec128<float16_t, N>{_mm_cvtps_ph(v.raw, _MM_FROUND_NO_EXC)}; +#endif +} + +template <size_t N> +HWY_API Vec128<bfloat16_t, N> DemoteTo(Simd<bfloat16_t, N> dbf16, + const Vec128<float, N> v) { + // TODO(janwas): _mm_cvtneps_pbh once we have avx512bf16. + const Rebind<int32_t, decltype(dbf16)> di32; + const Rebind<uint32_t, decltype(dbf16)> du32; // for logical shift right + const Rebind<uint16_t, decltype(dbf16)> du16; + const auto bits_in_32 = BitCast(di32, ShiftRight<16>(BitCast(du32, v))); + return BitCast(dbf16, DemoteTo(du16, bits_in_32)); +} + +template <size_t N> +HWY_API Vec128<bfloat16_t, 2 * N> ReorderDemote2To( + Simd<bfloat16_t, 2 * N> dbf16, Vec128<float, N> a, Vec128<float, N> b) { + // TODO(janwas): _mm_cvtne2ps_pbh once we have avx512bf16. + const RebindToUnsigned<decltype(dbf16)> du16; + const Repartition<uint32_t, decltype(dbf16)> du32; + const Vec128<uint32_t, N> b_in_even = ShiftRight<16>(BitCast(du32, b)); + return BitCast(dbf16, OddEven(BitCast(du16, a), BitCast(du16, b_in_even))); +} + +template <size_t N> +HWY_API Vec128<float, N> DemoteTo(Simd<float, N> /* tag */, + const Vec128<double, N> v) { + return Vec128<float, N>{_mm_cvtpd_ps(v.raw)}; +} + +namespace detail { + +// For well-defined float->int demotion in all x86_*-inl.h. + +template <size_t N> +HWY_INLINE auto ClampF64ToI32Max(Simd<double, N> d, decltype(Zero(d)) v) + -> decltype(Zero(d)) { + // The max can be exactly represented in binary64, so clamping beforehand + // prevents x86 conversion from raising an exception and returning 80..00. + return Min(v, Set(d, 2147483647.0)); +} + +// For ConvertTo float->int of same size, clamping before conversion would +// change the result because the max integer value is not exactly representable. +// Instead detect the overflow result after conversion and fix it. +template <typename TI, size_t N, class DF = Simd<MakeFloat<TI>, N>> +HWY_INLINE auto FixConversionOverflow(Simd<TI, N> di, + decltype(Zero(DF())) original, + decltype(Zero(di).raw) converted_raw) + -> decltype(Zero(di)) { + // Combinations of original and output sign: + // --: normal <0 or -huge_val to 80..00: OK + // -+: -0 to 0 : OK + // +-: +huge_val to 80..00 : xor with FF..FF to get 7F..FF + // ++: normal >0 : OK + const auto converted = decltype(Zero(di)){converted_raw}; + const auto sign_wrong = AndNot(BitCast(di, original), converted); + return BitCast(di, Xor(converted, BroadcastSignBit(sign_wrong))); +} + +} // namespace detail + +template <size_t N> +HWY_API Vec128<int32_t, N> DemoteTo(Simd<int32_t, N> /* tag */, + const Vec128<double, N> v) { + const auto clamped = detail::ClampF64ToI32Max(Simd<double, N>(), v); + return Vec128<int32_t, N>{_mm_cvttpd_epi32(clamped.raw)}; +} + +// For already range-limited input [0, 255]. +template <size_t N> +HWY_API Vec128<uint8_t, N> U8FromU32(const Vec128<uint32_t, N> v) { + const Simd<uint32_t, N> d32; + const Simd<uint8_t, N * 4> d8; + alignas(16) static constexpr uint32_t k8From32[4] = { + 0x0C080400u, 0x0C080400u, 0x0C080400u, 0x0C080400u}; + // Also replicate bytes into all 32 bit lanes for safety. + const auto quad = TableLookupBytes(v, Load(d32, k8From32)); + return LowerHalf(LowerHalf(BitCast(d8, quad))); +} + +// ------------------------------ Integer <=> fp (ShiftRight, OddEven) + +template <size_t N> +HWY_API Vec128<float, N> ConvertTo(Simd<float, N> /* tag */, + const Vec128<int32_t, N> v) { + return Vec128<float, N>{_mm_cvtepi32_ps(v.raw)}; +} + +template <size_t N> +HWY_API Vec128<double, N> ConvertTo(Simd<double, N> dd, + const Vec128<int64_t, N> v) { +#if HWY_TARGET <= HWY_AVX3 + (void)dd; + return Vec128<double, N>{_mm_cvtepi64_pd(v.raw)}; +#else + // Based on wim's approach (https://stackoverflow.com/questions/41144668/) + const Repartition<uint32_t, decltype(dd)> d32; + const Repartition<uint64_t, decltype(dd)> d64; + + // Toggle MSB of lower 32-bits and insert exponent for 2^84 + 2^63 + const auto k84_63 = Set(d64, 0x4530000080000000ULL); + const auto v_upper = BitCast(dd, ShiftRight<32>(BitCast(d64, v)) ^ k84_63); + + // Exponent is 2^52, lower 32 bits from v (=> 32-bit OddEven) + const auto k52 = Set(d32, 0x43300000); + const auto v_lower = BitCast(dd, OddEven(k52, BitCast(d32, v))); + + const auto k84_63_52 = BitCast(dd, Set(d64, 0x4530000080100000ULL)); + return (v_upper - k84_63_52) + v_lower; // order matters! +#endif +} + +// Truncates (rounds toward zero). +template <size_t N> +HWY_API Vec128<int32_t, N> ConvertTo(const Simd<int32_t, N> di, + const Vec128<float, N> v) { + return detail::FixConversionOverflow(di, v, _mm_cvttps_epi32(v.raw)); +} + +// Full (partial handled below) +HWY_API Vec128<int64_t> ConvertTo(Full128<int64_t> di, const Vec128<double> v) { +#if HWY_TARGET <= HWY_AVX3 && HWY_ARCH_X86_64 + return detail::FixConversionOverflow(di, v, _mm_cvttpd_epi64(v.raw)); +#elif HWY_ARCH_X86_64 + const __m128i i0 = _mm_cvtsi64_si128(_mm_cvttsd_si64(v.raw)); + const Half<Full128<double>> dd2; + const __m128i i1 = _mm_cvtsi64_si128(_mm_cvttsd_si64(UpperHalf(dd2, v).raw)); + return detail::FixConversionOverflow(di, v, _mm_unpacklo_epi64(i0, i1)); +#else + using VI = decltype(Zero(di)); + const VI k0 = Zero(di); + const VI k1 = Set(di, 1); + const VI k51 = Set(di, 51); + + // Exponent indicates whether the number can be represented as int64_t. + const VI biased_exp = ShiftRight<52>(BitCast(di, v)) & Set(di, 0x7FF); + const VI exp = biased_exp - Set(di, 0x3FF); + const auto in_range = exp < Set(di, 63); + + // If we were to cap the exponent at 51 and add 2^52, the number would be in + // [2^52, 2^53) and mantissa bits could be read out directly. We need to + // round-to-0 (truncate), but changing rounding mode in MXCSR hits a + // compiler reordering bug: https://gcc.godbolt.org/z/4hKj6c6qc . We instead + // manually shift the mantissa into place (we already have many of the + // inputs anyway). + const VI shift_mnt = Max(k51 - exp, k0); + const VI shift_int = Max(exp - k51, k0); + const VI mantissa = BitCast(di, v) & Set(di, (1ULL << 52) - 1); + // Include implicit 1-bit; shift by one more to ensure it's in the mantissa. + const VI int52 = (mantissa | Set(di, 1ULL << 52)) >> (shift_mnt + k1); + // For inputs larger than 2^52, insert zeros at the bottom. + const VI shifted = int52 << shift_int; + // Restore the one bit lost when shifting in the implicit 1-bit. + const VI restored = shifted | ((mantissa & k1) << (shift_int - k1)); + + // Saturate to LimitsMin (unchanged when negating below) or LimitsMax. + const VI sign_mask = BroadcastSignBit(BitCast(di, v)); + const VI limit = Set(di, LimitsMax<int64_t>()) - sign_mask; + const VI magnitude = IfThenElse(in_range, restored, limit); + + // If the input was negative, negate the integer (two's complement). + return (magnitude ^ sign_mask) - sign_mask; +#endif +} +HWY_API Vec128<int64_t, 1> ConvertTo(Simd<int64_t, 1> di, + const Vec128<double, 1> v) { + // Only need to specialize for non-AVX3, 64-bit (single scalar op) +#if HWY_TARGET > HWY_AVX3 && HWY_ARCH_X86_64 + const Vec128<int64_t, 1> i0{_mm_cvtsi64_si128(_mm_cvttsd_si64(v.raw))}; + return detail::FixConversionOverflow(di, v, i0.raw); +#else + (void)di; + const auto full = ConvertTo(Full128<int64_t>(), Vec128<double>{v.raw}); + return Vec128<int64_t, 1>{full.raw}; +#endif +} + +template <size_t N> +HWY_API Vec128<int32_t, N> NearestInt(const Vec128<float, N> v) { + const Simd<int32_t, N> di; + return detail::FixConversionOverflow(di, v, _mm_cvtps_epi32(v.raw)); +} + +// ------------------------------ Floating-point rounding (ConvertTo) + +#if HWY_TARGET == HWY_SSSE3 + +// Toward nearest integer, ties to even +template <typename T, size_t N, HWY_IF_FLOAT(T)> +HWY_API Vec128<T, N> Round(const Vec128<T, N> v) { + // Rely on rounding after addition with a large value such that no mantissa + // bits remain (assuming the current mode is nearest-even). We may need a + // compiler flag for precise floating-point to prevent "optimizing" this out. + const Simd<T, N> df; + const auto max = Set(df, MantissaEnd<T>()); + const auto large = CopySignToAbs(max, v); + const auto added = large + v; + const auto rounded = added - large; + // Keep original if NaN or the magnitude is large (already an int). + return IfThenElse(Abs(v) < max, rounded, v); +} + +namespace detail { + +// Truncating to integer and converting back to float is correct except when the +// input magnitude is large, in which case the input was already an integer +// (because mantissa >> exponent is zero). +template <typename T, size_t N, HWY_IF_FLOAT(T)> +HWY_INLINE Mask128<T, N> UseInt(const Vec128<T, N> v) { + return Abs(v) < Set(Simd<T, N>(), MantissaEnd<T>()); +} + +} // namespace detail + +// Toward zero, aka truncate +template <typename T, size_t N, HWY_IF_FLOAT(T)> +HWY_API Vec128<T, N> Trunc(const Vec128<T, N> v) { + const Simd<T, N> df; + const RebindToSigned<decltype(df)> di; + + const auto integer = ConvertTo(di, v); // round toward 0 + const auto int_f = ConvertTo(df, integer); + + return IfThenElse(detail::UseInt(v), CopySign(int_f, v), v); +} + +// Toward +infinity, aka ceiling +template <typename T, size_t N, HWY_IF_FLOAT(T)> +HWY_API Vec128<T, N> Ceil(const Vec128<T, N> v) { + const Simd<T, N> df; + const RebindToSigned<decltype(df)> di; + + const auto integer = ConvertTo(di, v); // round toward 0 + const auto int_f = ConvertTo(df, integer); + + // Truncating a positive non-integer ends up smaller; if so, add 1. + const auto neg1 = ConvertTo(df, VecFromMask(di, RebindMask(di, int_f < v))); + + return IfThenElse(detail::UseInt(v), int_f - neg1, v); +} + +// Toward -infinity, aka floor +template <typename T, size_t N, HWY_IF_FLOAT(T)> +HWY_API Vec128<T, N> Floor(const Vec128<T, N> v) { + const Simd<T, N> df; + const RebindToSigned<decltype(df)> di; + + const auto integer = ConvertTo(di, v); // round toward 0 + const auto int_f = ConvertTo(df, integer); + + // Truncating a negative non-integer ends up larger; if so, subtract 1. + const auto neg1 = ConvertTo(df, VecFromMask(di, RebindMask(di, int_f > v))); + + return IfThenElse(detail::UseInt(v), int_f + neg1, v); +} + +#else + +// Toward nearest integer, ties to even +template <size_t N> +HWY_API Vec128<float, N> Round(const Vec128<float, N> v) { + return Vec128<float, N>{ + _mm_round_ps(v.raw, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)}; +} +template <size_t N> +HWY_API Vec128<double, N> Round(const Vec128<double, N> v) { + return Vec128<double, N>{ + _mm_round_pd(v.raw, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)}; +} + +// Toward zero, aka truncate +template <size_t N> +HWY_API Vec128<float, N> Trunc(const Vec128<float, N> v) { + return Vec128<float, N>{ + _mm_round_ps(v.raw, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC)}; +} +template <size_t N> +HWY_API Vec128<double, N> Trunc(const Vec128<double, N> v) { + return Vec128<double, N>{ + _mm_round_pd(v.raw, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC)}; +} + +// Toward +infinity, aka ceiling +template <size_t N> +HWY_API Vec128<float, N> Ceil(const Vec128<float, N> v) { + return Vec128<float, N>{ + _mm_round_ps(v.raw, _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC)}; +} +template <size_t N> +HWY_API Vec128<double, N> Ceil(const Vec128<double, N> v) { + return Vec128<double, N>{ + _mm_round_pd(v.raw, _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC)}; +} + +// Toward -infinity, aka floor +template <size_t N> +HWY_API Vec128<float, N> Floor(const Vec128<float, N> v) { + return Vec128<float, N>{ + _mm_round_ps(v.raw, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC)}; +} +template <size_t N> +HWY_API Vec128<double, N> Floor(const Vec128<double, N> v) { + return Vec128<double, N>{ + _mm_round_pd(v.raw, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC)}; +} + +#endif // !HWY_SSSE3 + +// ================================================== CRYPTO + +#if !defined(HWY_DISABLE_PCLMUL_AES) && HWY_TARGET != HWY_SSSE3 + +// Per-target flag to prevent generic_ops-inl.h from defining AESRound. +#ifdef HWY_NATIVE_AES +#undef HWY_NATIVE_AES +#else +#define HWY_NATIVE_AES +#endif + +HWY_API Vec128<uint8_t> AESRound(Vec128<uint8_t> state, + Vec128<uint8_t> round_key) { + return Vec128<uint8_t>{_mm_aesenc_si128(state.raw, round_key.raw)}; +} + +template <size_t N, HWY_IF_LE128(uint64_t, N)> +HWY_API Vec128<uint64_t, N> CLMulLower(Vec128<uint64_t, N> a, + Vec128<uint64_t, N> b) { + return Vec128<uint64_t, N>{_mm_clmulepi64_si128(a.raw, b.raw, 0x00)}; +} + +template <size_t N, HWY_IF_LE128(uint64_t, N)> +HWY_API Vec128<uint64_t, N> CLMulUpper(Vec128<uint64_t, N> a, + Vec128<uint64_t, N> b) { + return Vec128<uint64_t, N>{_mm_clmulepi64_si128(a.raw, b.raw, 0x11)}; +} + +#endif // !defined(HWY_DISABLE_PCLMUL_AES) && HWY_TARGET != HWY_SSSE3 + +// ================================================== MISC + +#if HWY_TARGET <= HWY_AVX3 + +// ------------------------------ LoadMaskBits + +// `p` points to at least 8 readable bytes, not all of which need be valid. +template <typename T, size_t N, HWY_IF_LE128(T, N)> +HWY_API Mask128<T, N> LoadMaskBits(Simd<T, N> /* tag */, + const uint8_t* HWY_RESTRICT bits) { + uint64_t mask_bits = 0; + constexpr size_t kNumBytes = (N + 7) / 8; + CopyBytes<kNumBytes>(bits, &mask_bits); + if (N < 8) { + mask_bits &= (1ull << N) - 1; + } + + return Mask128<T, N>::FromBits(mask_bits); +} + +// ------------------------------ StoreMaskBits + +// `p` points to at least 8 writable bytes. +template <typename T, size_t N> +HWY_API size_t StoreMaskBits(const Simd<T, N> /* tag */, + const Mask128<T, N> mask, uint8_t* bits) { + constexpr size_t kNumBytes = (N + 7) / 8; + CopyBytes<kNumBytes>(&mask.raw, bits); + + // Non-full byte, need to clear the undefined upper bits. + if (N < 8) { + const int mask = (1 << N) - 1; + bits[0] = static_cast<uint8_t>(bits[0] & mask); + } + + return kNumBytes; +} + +// ------------------------------ Mask testing + +// Beware: the suffix indicates the number of mask bits, not lane size! + +template <typename T, size_t N> +HWY_API size_t CountTrue(const Simd<T, N> /* tag */, const Mask128<T, N> mask) { + const uint64_t mask_bits = static_cast<uint64_t>(mask.raw) & ((1u << N) - 1); + return PopCount(mask_bits); +} + +template <typename T, size_t N> +HWY_API intptr_t FindFirstTrue(const Simd<T, N> /* tag */, + const Mask128<T, N> mask) { + const uint32_t mask_bits = static_cast<uint32_t>(mask.raw) & ((1u << N) - 1); + return mask.raw ? intptr_t(Num0BitsBelowLS1Bit_Nonzero32(mask_bits)) : -1; +} + +template <typename T, size_t N> +HWY_API bool AllFalse(const Simd<T, N> /* tag */, const Mask128<T, N> mask) { + const uint64_t mask_bits = static_cast<uint64_t>(mask.raw) & ((1u << N) - 1); + return mask_bits == 0; +} + +template <typename T, size_t N> +HWY_API bool AllTrue(const Simd<T, N> /* tag */, const Mask128<T, N> mask) { + const uint64_t mask_bits = static_cast<uint64_t>(mask.raw) & ((1u << N) - 1); + // Cannot use _kortestc because we may have less than 8 mask bits. + return mask_bits == (1u << N) - 1; +} + +// ------------------------------ Compress + +#if HWY_TARGET != HWY_AVX3_DL +namespace detail { + +// Returns permutevar_epi16 indices for 16-bit Compress. Also used by x86_256. +HWY_INLINE Vec128<uint16_t, 8> IndicesForCompress16(uint64_t mask_bits) { + Full128<uint16_t> du16; + // Table of u16 indices packed into bytes to reduce L1 usage. Will be unpacked + // to u16. Ideally we would broadcast 8*3 (half of the 8 bytes currently used) + // bits into each lane and then varshift, but that does not fit in 16 bits. + Rebind<uint8_t, decltype(du16)> du8; + alignas(16) constexpr uint8_t tbl[2048] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 2, + 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, + 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 2, 3, 0, 0, + 0, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 1, 2, 3, 0, + 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, + 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, + 0, 1, 2, 4, 0, 0, 0, 0, 0, 0, 1, 2, 4, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, + 0, 3, 4, 0, 0, 0, 0, 0, 1, 3, 4, 0, 0, 0, 0, 0, 0, 1, 3, 4, 0, 0, 0, 0, 2, + 3, 4, 0, 0, 0, 0, 0, 0, 2, 3, 4, 0, 0, 0, 0, 1, 2, 3, 4, 0, 0, 0, 0, 0, 1, + 2, 3, 4, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 1, 5, 0, + 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 2, 5, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, + 0, 0, 0, 0, 1, 2, 5, 0, 0, 0, 0, 0, 0, 1, 2, 5, 0, 0, 0, 0, 3, 5, 0, 0, 0, + 0, 0, 0, 0, 3, 5, 0, 0, 0, 0, 0, 1, 3, 5, 0, 0, 0, 0, 0, 0, 1, 3, 5, 0, 0, + 0, 0, 2, 3, 5, 0, 0, 0, 0, 0, 0, 2, 3, 5, 0, 0, 0, 0, 1, 2, 3, 5, 0, 0, 0, + 0, 0, 1, 2, 3, 5, 0, 0, 0, 4, 5, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 0, 0, 0, 0, + 1, 4, 5, 0, 0, 0, 0, 0, 0, 1, 4, 5, 0, 0, 0, 0, 2, 4, 5, 0, 0, 0, 0, 0, 0, + 2, 4, 5, 0, 0, 0, 0, 1, 2, 4, 5, 0, 0, 0, 0, 0, 1, 2, 4, 5, 0, 0, 0, 3, 4, + 5, 0, 0, 0, 0, 0, 0, 3, 4, 5, 0, 0, 0, 0, 1, 3, 4, 5, 0, 0, 0, 0, 0, 1, 3, + 4, 5, 0, 0, 0, 2, 3, 4, 5, 0, 0, 0, 0, 0, 2, 3, 4, 5, 0, 0, 0, 1, 2, 3, 4, + 5, 0, 0, 0, 0, 1, 2, 3, 4, 5, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, + 0, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 0, 0, 0, 2, 6, 0, 0, 0, 0, + 0, 0, 0, 2, 6, 0, 0, 0, 0, 0, 1, 2, 6, 0, 0, 0, 0, 0, 0, 1, 2, 6, 0, 0, 0, + 0, 3, 6, 0, 0, 0, 0, 0, 0, 0, 3, 6, 0, 0, 0, 0, 0, 1, 3, 6, 0, 0, 0, 0, 0, + 0, 1, 3, 6, 0, 0, 0, 0, 2, 3, 6, 0, 0, 0, 0, 0, 0, 2, 3, 6, 0, 0, 0, 0, 1, + 2, 3, 6, 0, 0, 0, 0, 0, 1, 2, 3, 6, 0, 0, 0, 4, 6, 0, 0, 0, 0, 0, 0, 0, 4, + 6, 0, 0, 0, 0, 0, 1, 4, 6, 0, 0, 0, 0, 0, 0, 1, 4, 6, 0, 0, 0, 0, 2, 4, 6, + 0, 0, 0, 0, 0, 0, 2, 4, 6, 0, 0, 0, 0, 1, 2, 4, 6, 0, 0, 0, 0, 0, 1, 2, 4, + 6, 0, 0, 0, 3, 4, 6, 0, 0, 0, 0, 0, 0, 3, 4, 6, 0, 0, 0, 0, 1, 3, 4, 6, 0, + 0, 0, 0, 0, 1, 3, 4, 6, 0, 0, 0, 2, 3, 4, 6, 0, 0, 0, 0, 0, 2, 3, 4, 6, 0, + 0, 0, 1, 2, 3, 4, 6, 0, 0, 0, 0, 1, 2, 3, 4, 6, 0, 0, 5, 6, 0, 0, 0, 0, 0, + 0, 0, 5, 6, 0, 0, 0, 0, 0, 1, 5, 6, 0, 0, 0, 0, 0, 0, 1, 5, 6, 0, 0, 0, 0, + 2, 5, 6, 0, 0, 0, 0, 0, 0, 2, 5, 6, 0, 0, 0, 0, 1, 2, 5, 6, 0, 0, 0, 0, 0, + 1, 2, 5, 6, 0, 0, 0, 3, 5, 6, 0, 0, 0, 0, 0, 0, 3, 5, 6, 0, 0, 0, 0, 1, 3, + 5, 6, 0, 0, 0, 0, 0, 1, 3, 5, 6, 0, 0, 0, 2, 3, 5, 6, 0, 0, 0, 0, 0, 2, 3, + 5, 6, 0, 0, 0, 1, 2, 3, 5, 6, 0, 0, 0, 0, 1, 2, 3, 5, 6, 0, 0, 4, 5, 6, 0, + 0, 0, 0, 0, 0, 4, 5, 6, 0, 0, 0, 0, 1, 4, 5, 6, 0, 0, 0, 0, 0, 1, 4, 5, 6, + 0, 0, 0, 2, 4, 5, 6, 0, 0, 0, 0, 0, 2, 4, 5, 6, 0, 0, 0, 1, 2, 4, 5, 6, 0, + 0, 0, 0, 1, 2, 4, 5, 6, 0, 0, 3, 4, 5, 6, 0, 0, 0, 0, 0, 3, 4, 5, 6, 0, 0, + 0, 1, 3, 4, 5, 6, 0, 0, 0, 0, 1, 3, 4, 5, 6, 0, 0, 2, 3, 4, 5, 6, 0, 0, 0, + 0, 2, 3, 4, 5, 6, 0, 0, 1, 2, 3, 4, 5, 6, 0, 0, 0, 1, 2, 3, 4, 5, 6, 0, 7, + 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 7, 0, 0, 0, 0, 0, 0, 0, 1, + 7, 0, 0, 0, 0, 0, 2, 7, 0, 0, 0, 0, 0, 0, 0, 2, 7, 0, 0, 0, 0, 0, 1, 2, 7, + 0, 0, 0, 0, 0, 0, 1, 2, 7, 0, 0, 0, 0, 3, 7, 0, 0, 0, 0, 0, 0, 0, 3, 7, 0, + 0, 0, 0, 0, 1, 3, 7, 0, 0, 0, 0, 0, 0, 1, 3, 7, 0, 0, 0, 0, 2, 3, 7, 0, 0, + 0, 0, 0, 0, 2, 3, 7, 0, 0, 0, 0, 1, 2, 3, 7, 0, 0, 0, 0, 0, 1, 2, 3, 7, 0, + 0, 0, 4, 7, 0, 0, 0, 0, 0, 0, 0, 4, 7, 0, 0, 0, 0, 0, 1, 4, 7, 0, 0, 0, 0, + 0, 0, 1, 4, 7, 0, 0, 0, 0, 2, 4, 7, 0, 0, 0, 0, 0, 0, 2, 4, 7, 0, 0, 0, 0, + 1, 2, 4, 7, 0, 0, 0, 0, 0, 1, 2, 4, 7, 0, 0, 0, 3, 4, 7, 0, 0, 0, 0, 0, 0, + 3, 4, 7, 0, 0, 0, 0, 1, 3, 4, 7, 0, 0, 0, 0, 0, 1, 3, 4, 7, 0, 0, 0, 2, 3, + 4, 7, 0, 0, 0, 0, 0, 2, 3, 4, 7, 0, 0, 0, 1, 2, 3, 4, 7, 0, 0, 0, 0, 1, 2, + 3, 4, 7, 0, 0, 5, 7, 0, 0, 0, 0, 0, 0, 0, 5, 7, 0, 0, 0, 0, 0, 1, 5, 7, 0, + 0, 0, 0, 0, 0, 1, 5, 7, 0, 0, 0, 0, 2, 5, 7, 0, 0, 0, 0, 0, 0, 2, 5, 7, 0, + 0, 0, 0, 1, 2, 5, 7, 0, 0, 0, 0, 0, 1, 2, 5, 7, 0, 0, 0, 3, 5, 7, 0, 0, 0, + 0, 0, 0, 3, 5, 7, 0, 0, 0, 0, 1, 3, 5, 7, 0, 0, 0, 0, 0, 1, 3, 5, 7, 0, 0, + 0, 2, 3, 5, 7, 0, 0, 0, 0, 0, 2, 3, 5, 7, 0, 0, 0, 1, 2, 3, 5, 7, 0, 0, 0, + 0, 1, 2, 3, 5, 7, 0, 0, 4, 5, 7, 0, 0, 0, 0, 0, 0, 4, 5, 7, 0, 0, 0, 0, 1, + 4, 5, 7, 0, 0, 0, 0, 0, 1, 4, 5, 7, 0, 0, 0, 2, 4, 5, 7, 0, 0, 0, 0, 0, 2, + 4, 5, 7, 0, 0, 0, 1, 2, 4, 5, 7, 0, 0, 0, 0, 1, 2, 4, 5, 7, 0, 0, 3, 4, 5, + 7, 0, 0, 0, 0, 0, 3, 4, 5, 7, 0, 0, 0, 1, 3, 4, 5, 7, 0, 0, 0, 0, 1, 3, 4, + 5, 7, 0, 0, 2, 3, 4, 5, 7, 0, 0, 0, 0, 2, 3, 4, 5, 7, 0, 0, 1, 2, 3, 4, 5, + 7, 0, 0, 0, 1, 2, 3, 4, 5, 7, 0, 6, 7, 0, 0, 0, 0, 0, 0, 0, 6, 7, 0, 0, 0, + 0, 0, 1, 6, 7, 0, 0, 0, 0, 0, 0, 1, 6, 7, 0, 0, 0, 0, 2, 6, 7, 0, 0, 0, 0, + 0, 0, 2, 6, 7, 0, 0, 0, 0, 1, 2, 6, 7, 0, 0, 0, 0, 0, 1, 2, 6, 7, 0, 0, 0, + 3, 6, 7, 0, 0, 0, 0, 0, 0, 3, 6, 7, 0, 0, 0, 0, 1, 3, 6, 7, 0, 0, 0, 0, 0, + 1, 3, 6, 7, 0, 0, 0, 2, 3, 6, 7, 0, 0, 0, 0, 0, 2, 3, 6, 7, 0, 0, 0, 1, 2, + 3, 6, 7, 0, 0, 0, 0, 1, 2, 3, 6, 7, 0, 0, 4, 6, 7, 0, 0, 0, 0, 0, 0, 4, 6, + 7, 0, 0, 0, 0, 1, 4, 6, 7, 0, 0, 0, 0, 0, 1, 4, 6, 7, 0, 0, 0, 2, 4, 6, 7, + 0, 0, 0, 0, 0, 2, 4, 6, 7, 0, 0, 0, 1, 2, 4, 6, 7, 0, 0, 0, 0, 1, 2, 4, 6, + 7, 0, 0, 3, 4, 6, 7, 0, 0, 0, 0, 0, 3, 4, 6, 7, 0, 0, 0, 1, 3, 4, 6, 7, 0, + 0, 0, 0, 1, 3, 4, 6, 7, 0, 0, 2, 3, 4, 6, 7, 0, 0, 0, 0, 2, 3, 4, 6, 7, 0, + 0, 1, 2, 3, 4, 6, 7, 0, 0, 0, 1, 2, 3, 4, 6, 7, 0, 5, 6, 7, 0, 0, 0, 0, 0, + 0, 5, 6, 7, 0, 0, 0, 0, 1, 5, 6, 7, 0, 0, 0, 0, 0, 1, 5, 6, 7, 0, 0, 0, 2, + 5, 6, 7, 0, 0, 0, 0, 0, 2, 5, 6, 7, 0, 0, 0, 1, 2, 5, 6, 7, 0, 0, 0, 0, 1, + 2, 5, 6, 7, 0, 0, 3, 5, 6, 7, 0, 0, 0, 0, 0, 3, 5, 6, 7, 0, 0, 0, 1, 3, 5, + 6, 7, 0, 0, 0, 0, 1, 3, 5, 6, 7, 0, 0, 2, 3, 5, 6, 7, 0, 0, 0, 0, 2, 3, 5, + 6, 7, 0, 0, 1, 2, 3, 5, 6, 7, 0, 0, 0, 1, 2, 3, 5, 6, 7, 0, 4, 5, 6, 7, 0, + 0, 0, 0, 0, 4, 5, 6, 7, 0, 0, 0, 1, 4, 5, 6, 7, 0, 0, 0, 0, 1, 4, 5, 6, 7, + 0, 0, 2, 4, 5, 6, 7, 0, 0, 0, 0, 2, 4, 5, 6, 7, 0, 0, 1, 2, 4, 5, 6, 7, 0, + 0, 0, 1, 2, 4, 5, 6, 7, 0, 3, 4, 5, 6, 7, 0, 0, 0, 0, 3, 4, 5, 6, 7, 0, 0, + 1, 3, 4, 5, 6, 7, 0, 0, 0, 1, 3, 4, 5, 6, 7, 0, 2, 3, 4, 5, 6, 7, 0, 0, 0, + 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, 0, 0, 1, 2, 3, 4, 5, 6, 7}; + return PromoteTo(du16, Load(du8, tbl + mask_bits * 8)); +} + +} // namespace detail +#endif // HWY_TARGET != HWY_AVX3_DL + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Vec128<T, N> Compress(Vec128<T, N> v, Mask128<T, N> mask) { + const Simd<T, N> d; + const Rebind<uint16_t, decltype(d)> du; + const auto vu = BitCast(du, v); // (required for float16_t inputs) + +#if HWY_TARGET == HWY_AVX3_DL // VBMI2 + const Vec128<uint16_t, N> cu{_mm_maskz_compress_epi16(mask.raw, vu.raw)}; +#else + const auto idx = detail::IndicesForCompress16(uint64_t{mask.raw}); + const Vec128<uint16_t, N> cu{_mm_permutexvar_epi16(idx.raw, vu.raw)}; +#endif // HWY_TARGET != HWY_AVX3_DL + return BitCast(d, cu); +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec128<T, N> Compress(Vec128<T, N> v, Mask128<T, N> mask) { + return Vec128<T, N>{_mm_maskz_compress_epi32(mask.raw, v.raw)}; +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec128<T, N> Compress(Vec128<T, N> v, Mask128<T, N> mask) { + return Vec128<T, N>{_mm_maskz_compress_epi64(mask.raw, v.raw)}; +} + +template <size_t N> +HWY_API Vec128<float, N> Compress(Vec128<float, N> v, Mask128<float, N> mask) { + return Vec128<float, N>{_mm_maskz_compress_ps(mask.raw, v.raw)}; +} + +template <size_t N> +HWY_API Vec128<double, N> Compress(Vec128<double, N> v, + Mask128<double, N> mask) { + return Vec128<double, N>{_mm_maskz_compress_pd(mask.raw, v.raw)}; +} + +// ------------------------------ CompressBits (LoadMaskBits) + +template <typename T, size_t N> +HWY_API Vec128<T, N> CompressBits(Vec128<T, N> v, + const uint8_t* HWY_RESTRICT bits) { + return Compress(v, LoadMaskBits(Simd<T, N>(), bits)); +} + +// ------------------------------ CompressStore + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 2)> +HWY_API size_t CompressStore(Vec128<T, N> v, Mask128<T, N> mask, Simd<T, N> d, + T* HWY_RESTRICT unaligned) { + const Rebind<uint16_t, decltype(d)> du; + const auto vu = BitCast(du, v); // (required for float16_t inputs) + + const uint64_t mask_bits{mask.raw}; + +#if HWY_TARGET == HWY_AVX3_DL // VBMI2 + _mm_mask_compressstoreu_epi16(unaligned, mask.raw, vu.raw); +#else + const auto idx = detail::IndicesForCompress16(mask_bits); + const Vec128<uint16_t, N> cu{_mm_permutexvar_epi16(idx.raw, vu.raw)}; + StoreU(BitCast(d, cu), d, unaligned); +#endif // HWY_TARGET == HWY_AVX3_DL + return PopCount(uint64_t{mask.raw} & ((1ull << N) - 1)); +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 4)> +HWY_API size_t CompressStore(Vec128<T, N> v, Mask128<T, N> mask, + Simd<T, N> /* tag */, T* HWY_RESTRICT unaligned) { + _mm_mask_compressstoreu_epi32(unaligned, mask.raw, v.raw); + return PopCount(uint64_t{mask.raw} & ((1ull << N) - 1)); +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 8)> +HWY_API size_t CompressStore(Vec128<T, N> v, Mask128<T, N> mask, + Simd<T, N> /* tag */, T* HWY_RESTRICT unaligned) { + _mm_mask_compressstoreu_epi64(unaligned, mask.raw, v.raw); + return PopCount(uint64_t{mask.raw} & ((1ull << N) - 1)); +} + +template <size_t N, HWY_IF_LE128(float, N)> +HWY_API size_t CompressStore(Vec128<float, N> v, Mask128<float, N> mask, + Simd<float, N> /* tag */, + float* HWY_RESTRICT unaligned) { + _mm_mask_compressstoreu_ps(unaligned, mask.raw, v.raw); + return PopCount(uint64_t{mask.raw} & ((1ull << N) - 1)); +} + +template <size_t N, HWY_IF_LE128(double, N)> +HWY_API size_t CompressStore(Vec128<double, N> v, Mask128<double, N> mask, + Simd<double, N> /* tag */, + double* HWY_RESTRICT unaligned) { + _mm_mask_compressstoreu_pd(unaligned, mask.raw, v.raw); + return PopCount(uint64_t{mask.raw} & ((1ull << N) - 1)); +} + +// ------------------------------ CompressBlendedStore (CompressStore) +template <typename T, size_t N> +HWY_API size_t CompressBlendedStore(Vec128<T, N> v, Mask128<T, N> m, + Simd<T, N> d, T* HWY_RESTRICT unaligned) { + // AVX-512 already does the blending at no extra cost (latency 11, + // rthroughput 2 - same as compress plus store). + if (HWY_TARGET == HWY_AVX3_DL || sizeof(T) != 2) { + // We're relying on the mask to blend. Clear the undefined upper bits. + if (N != 16 / sizeof(T)) { + m = And(m, FirstN(d, N)); + } + return CompressStore(v, m, d, unaligned); + } else { + const size_t count = CountTrue(m); + const Vec128<T, N> compressed = Compress(v, m); + const Vec128<T, N> prev = LoadU(d, unaligned); + StoreU(IfThenElse(FirstN(d, count), compressed, prev), d, unaligned); + return count; + } +} + +// ------------------------------ CompressBitsStore (LoadMaskBits) + +template <typename T, size_t N> +HWY_API size_t CompressBitsStore(Vec128<T, N> v, + const uint8_t* HWY_RESTRICT bits, Simd<T, N> d, + T* HWY_RESTRICT unaligned) { + return CompressStore(v, LoadMaskBits(d, bits), d, unaligned); +} + +#else // AVX2 or below + +// ------------------------------ LoadMaskBits (TestBit) + +namespace detail { + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 1)> +HWY_INLINE Mask128<T, N> LoadMaskBits(Simd<T, N> d, uint64_t mask_bits) { + const RebindToUnsigned<decltype(d)> du; + // Easier than Set(), which would require an >8-bit type, which would not + // compile for T=uint8_t, N=1. + const Vec128<T, N> vbits{_mm_cvtsi32_si128(static_cast<int>(mask_bits))}; + + // Replicate bytes 8x such that each byte contains the bit that governs it. + alignas(16) constexpr uint8_t kRep8[16] = {0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, 1, 1, 1}; + const auto rep8 = TableLookupBytes(vbits, Load(du, kRep8)); + + alignas(16) constexpr uint8_t kBit[16] = {1, 2, 4, 8, 16, 32, 64, 128, + 1, 2, 4, 8, 16, 32, 64, 128}; + return RebindMask(d, TestBit(rep8, LoadDup128(du, kBit))); +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 2)> +HWY_INLINE Mask128<T, N> LoadMaskBits(Simd<T, N> d, uint64_t mask_bits) { + const RebindToUnsigned<decltype(d)> du; + alignas(16) constexpr uint16_t kBit[8] = {1, 2, 4, 8, 16, 32, 64, 128}; + const auto vmask_bits = Set(du, static_cast<uint16_t>(mask_bits)); + return RebindMask(d, TestBit(vmask_bits, Load(du, kBit))); +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 4)> +HWY_INLINE Mask128<T, N> LoadMaskBits(Simd<T, N> d, uint64_t mask_bits) { + const RebindToUnsigned<decltype(d)> du; + alignas(16) constexpr uint32_t kBit[8] = {1, 2, 4, 8}; + const auto vmask_bits = Set(du, static_cast<uint32_t>(mask_bits)); + return RebindMask(d, TestBit(vmask_bits, Load(du, kBit))); +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 8)> +HWY_INLINE Mask128<T, N> LoadMaskBits(Simd<T, N> d, uint64_t mask_bits) { + const RebindToUnsigned<decltype(d)> du; + alignas(16) constexpr uint64_t kBit[8] = {1, 2}; + return RebindMask(d, TestBit(Set(du, mask_bits), Load(du, kBit))); +} + +} // namespace detail + +// `p` points to at least 8 readable bytes, not all of which need be valid. +template <typename T, size_t N, HWY_IF_LE128(T, N)> +HWY_API Mask128<T, N> LoadMaskBits(Simd<T, N> d, + const uint8_t* HWY_RESTRICT bits) { + uint64_t mask_bits = 0; + constexpr size_t kNumBytes = (N + 7) / 8; + CopyBytes<kNumBytes>(bits, &mask_bits); + if (N < 8) { + mask_bits &= (1ull << N) - 1; + } + + return detail::LoadMaskBits(d, mask_bits); +} + +// ------------------------------ StoreMaskBits + +namespace detail { + +constexpr HWY_INLINE uint64_t U64FromInt(int mask_bits) { + return static_cast<uint64_t>(static_cast<unsigned>(mask_bits)); +} + +template <typename T, size_t N> +HWY_INLINE uint64_t BitsFromMask(hwy::SizeTag<1> /*tag*/, + const Mask128<T, N> mask) { + const Simd<T, N> d; + const auto sign_bits = BitCast(d, VecFromMask(d, mask)).raw; + return U64FromInt(_mm_movemask_epi8(sign_bits)); +} + +template <typename T, size_t N> +HWY_INLINE uint64_t BitsFromMask(hwy::SizeTag<2> /*tag*/, + const Mask128<T, N> mask) { + // Remove useless lower half of each u16 while preserving the sign bit. + const auto sign_bits = _mm_packs_epi16(mask.raw, _mm_setzero_si128()); + return U64FromInt(_mm_movemask_epi8(sign_bits)); +} + +template <typename T, size_t N> +HWY_INLINE uint64_t BitsFromMask(hwy::SizeTag<4> /*tag*/, + const Mask128<T, N> mask) { + const Simd<T, N> d; + const Simd<float, N> df; + const auto sign_bits = BitCast(df, VecFromMask(d, mask)); + return U64FromInt(_mm_movemask_ps(sign_bits.raw)); +} + +template <typename T, size_t N> +HWY_INLINE uint64_t BitsFromMask(hwy::SizeTag<8> /*tag*/, + const Mask128<T, N> mask) { + const Simd<T, N> d; + const Simd<double, N> df; + const auto sign_bits = BitCast(df, VecFromMask(d, mask)); + return U64FromInt(_mm_movemask_pd(sign_bits.raw)); +} + +// Returns the lowest N of the _mm_movemask* bits. +template <typename T, size_t N> +constexpr uint64_t OnlyActive(uint64_t mask_bits) { + return ((N * sizeof(T)) == 16) ? mask_bits : mask_bits & ((1ull << N) - 1); +} + +template <typename T, size_t N> +HWY_INLINE uint64_t BitsFromMask(const Mask128<T, N> mask) { + return OnlyActive<T, N>(BitsFromMask(hwy::SizeTag<sizeof(T)>(), mask)); +} + +} // namespace detail + +// `p` points to at least 8 writable bytes. +template <typename T, size_t N> +HWY_API size_t StoreMaskBits(const Simd<T, N> /* tag */, + const Mask128<T, N> mask, uint8_t* bits) { + constexpr size_t kNumBytes = (N + 7) / 8; + const uint64_t mask_bits = detail::BitsFromMask(mask); + CopyBytes<kNumBytes>(&mask_bits, bits); + return kNumBytes; +} + +// ------------------------------ Mask testing + +template <typename T, size_t N> +HWY_API bool AllFalse(const Simd<T, N> /* tag */, const Mask128<T, N> mask) { + // Cheaper than PTEST, which is 2 uop / 3L. + return detail::BitsFromMask(mask) == 0; +} + +template <typename T, size_t N> +HWY_API bool AllTrue(const Simd<T, N> /* tag */, const Mask128<T, N> mask) { + constexpr uint64_t kAllBits = + detail::OnlyActive<T, N>((1ull << (16 / sizeof(T))) - 1); + return detail::BitsFromMask(mask) == kAllBits; +} + +template <typename T, size_t N> +HWY_API size_t CountTrue(const Simd<T, N> /* tag */, const Mask128<T, N> mask) { + return PopCount(detail::BitsFromMask(mask)); +} + +template <typename T, size_t N> +HWY_API intptr_t FindFirstTrue(const Simd<T, N> /* tag */, + const Mask128<T, N> mask) { + const uint64_t mask_bits = detail::BitsFromMask(mask); + return mask_bits ? intptr_t(Num0BitsBelowLS1Bit_Nonzero64(mask_bits)) : -1; +} + +// ------------------------------ Compress, CompressBits + +namespace detail { + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 2)> +HWY_INLINE Vec128<T, N> IndicesFromBits(Simd<T, N> d, uint64_t mask_bits) { + HWY_DASSERT(mask_bits < 256); + const Rebind<uint8_t, decltype(d)> d8; + const Simd<uint16_t, N> du; + + // compress_epi16 requires VBMI2 and there is no permutevar_epi16, so we need + // byte indices for PSHUFB (one vector's worth for each of 256 combinations of + // 8 mask bits). Loading them directly would require 4 KiB. We can instead + // store lane indices and convert to byte indices (2*lane + 0..1), with the + // doubling baked into the table. AVX2 Compress32 stores eight 4-bit lane + // indices (total 1 KiB), broadcasts them into each 32-bit lane and shifts. + // Here, 16-bit lanes are too narrow to hold all bits, and unpacking nibbles + // is likely more costly than the higher cache footprint from storing bytes. + alignas(16) constexpr uint8_t table[2048] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, + 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, + 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, + 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, + 0, 6, 0, 0, 0, 0, 0, 0, 2, 6, 0, 0, 0, 0, 0, 0, 0, 2, + 6, 0, 0, 0, 0, 0, 4, 6, 0, 0, 0, 0, 0, 0, 0, 4, 6, 0, + 0, 0, 0, 0, 2, 4, 6, 0, 0, 0, 0, 0, 0, 2, 4, 6, 0, 0, + 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, + 2, 8, 0, 0, 0, 0, 0, 0, 0, 2, 8, 0, 0, 0, 0, 0, 4, 8, + 0, 0, 0, 0, 0, 0, 0, 4, 8, 0, 0, 0, 0, 0, 2, 4, 8, 0, + 0, 0, 0, 0, 0, 2, 4, 8, 0, 0, 0, 0, 6, 8, 0, 0, 0, 0, + 0, 0, 0, 6, 8, 0, 0, 0, 0, 0, 2, 6, 8, 0, 0, 0, 0, 0, + 0, 2, 6, 8, 0, 0, 0, 0, 4, 6, 8, 0, 0, 0, 0, 0, 0, 4, + 6, 8, 0, 0, 0, 0, 2, 4, 6, 8, 0, 0, 0, 0, 0, 2, 4, 6, + 8, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, + 0, 0, 2, 10, 0, 0, 0, 0, 0, 0, 0, 2, 10, 0, 0, 0, 0, 0, + 4, 10, 0, 0, 0, 0, 0, 0, 0, 4, 10, 0, 0, 0, 0, 0, 2, 4, + 10, 0, 0, 0, 0, 0, 0, 2, 4, 10, 0, 0, 0, 0, 6, 10, 0, 0, + 0, 0, 0, 0, 0, 6, 10, 0, 0, 0, 0, 0, 2, 6, 10, 0, 0, 0, + 0, 0, 0, 2, 6, 10, 0, 0, 0, 0, 4, 6, 10, 0, 0, 0, 0, 0, + 0, 4, 6, 10, 0, 0, 0, 0, 2, 4, 6, 10, 0, 0, 0, 0, 0, 2, + 4, 6, 10, 0, 0, 0, 8, 10, 0, 0, 0, 0, 0, 0, 0, 8, 10, 0, + 0, 0, 0, 0, 2, 8, 10, 0, 0, 0, 0, 0, 0, 2, 8, 10, 0, 0, + 0, 0, 4, 8, 10, 0, 0, 0, 0, 0, 0, 4, 8, 10, 0, 0, 0, 0, + 2, 4, 8, 10, 0, 0, 0, 0, 0, 2, 4, 8, 10, 0, 0, 0, 6, 8, + 10, 0, 0, 0, 0, 0, 0, 6, 8, 10, 0, 0, 0, 0, 2, 6, 8, 10, + 0, 0, 0, 0, 0, 2, 6, 8, 10, 0, 0, 0, 4, 6, 8, 10, 0, 0, + 0, 0, 0, 4, 6, 8, 10, 0, 0, 0, 2, 4, 6, 8, 10, 0, 0, 0, + 0, 2, 4, 6, 8, 10, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 12, + 0, 0, 0, 0, 0, 0, 2, 12, 0, 0, 0, 0, 0, 0, 0, 2, 12, 0, + 0, 0, 0, 0, 4, 12, 0, 0, 0, 0, 0, 0, 0, 4, 12, 0, 0, 0, + 0, 0, 2, 4, 12, 0, 0, 0, 0, 0, 0, 2, 4, 12, 0, 0, 0, 0, + 6, 12, 0, 0, 0, 0, 0, 0, 0, 6, 12, 0, 0, 0, 0, 0, 2, 6, + 12, 0, 0, 0, 0, 0, 0, 2, 6, 12, 0, 0, 0, 0, 4, 6, 12, 0, + 0, 0, 0, 0, 0, 4, 6, 12, 0, 0, 0, 0, 2, 4, 6, 12, 0, 0, + 0, 0, 0, 2, 4, 6, 12, 0, 0, 0, 8, 12, 0, 0, 0, 0, 0, 0, + 0, 8, 12, 0, 0, 0, 0, 0, 2, 8, 12, 0, 0, 0, 0, 0, 0, 2, + 8, 12, 0, 0, 0, 0, 4, 8, 12, 0, 0, 0, 0, 0, 0, 4, 8, 12, + 0, 0, 0, 0, 2, 4, 8, 12, 0, 0, 0, 0, 0, 2, 4, 8, 12, 0, + 0, 0, 6, 8, 12, 0, 0, 0, 0, 0, 0, 6, 8, 12, 0, 0, 0, 0, + 2, 6, 8, 12, 0, 0, 0, 0, 0, 2, 6, 8, 12, 0, 0, 0, 4, 6, + 8, 12, 0, 0, 0, 0, 0, 4, 6, 8, 12, 0, 0, 0, 2, 4, 6, 8, + 12, 0, 0, 0, 0, 2, 4, 6, 8, 12, 0, 0, 10, 12, 0, 0, 0, 0, + 0, 0, 0, 10, 12, 0, 0, 0, 0, 0, 2, 10, 12, 0, 0, 0, 0, 0, + 0, 2, 10, 12, 0, 0, 0, 0, 4, 10, 12, 0, 0, 0, 0, 0, 0, 4, + 10, 12, 0, 0, 0, 0, 2, 4, 10, 12, 0, 0, 0, 0, 0, 2, 4, 10, + 12, 0, 0, 0, 6, 10, 12, 0, 0, 0, 0, 0, 0, 6, 10, 12, 0, 0, + 0, 0, 2, 6, 10, 12, 0, 0, 0, 0, 0, 2, 6, 10, 12, 0, 0, 0, + 4, 6, 10, 12, 0, 0, 0, 0, 0, 4, 6, 10, 12, 0, 0, 0, 2, 4, + 6, 10, 12, 0, 0, 0, 0, 2, 4, 6, 10, 12, 0, 0, 8, 10, 12, 0, + 0, 0, 0, 0, 0, 8, 10, 12, 0, 0, 0, 0, 2, 8, 10, 12, 0, 0, + 0, 0, 0, 2, 8, 10, 12, 0, 0, 0, 4, 8, 10, 12, 0, 0, 0, 0, + 0, 4, 8, 10, 12, 0, 0, 0, 2, 4, 8, 10, 12, 0, 0, 0, 0, 2, + 4, 8, 10, 12, 0, 0, 6, 8, 10, 12, 0, 0, 0, 0, 0, 6, 8, 10, + 12, 0, 0, 0, 2, 6, 8, 10, 12, 0, 0, 0, 0, 2, 6, 8, 10, 12, + 0, 0, 4, 6, 8, 10, 12, 0, 0, 0, 0, 4, 6, 8, 10, 12, 0, 0, + 2, 4, 6, 8, 10, 12, 0, 0, 0, 2, 4, 6, 8, 10, 12, 0, 14, 0, + 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 2, 14, 0, 0, + 0, 0, 0, 0, 0, 2, 14, 0, 0, 0, 0, 0, 4, 14, 0, 0, 0, 0, + 0, 0, 0, 4, 14, 0, 0, 0, 0, 0, 2, 4, 14, 0, 0, 0, 0, 0, + 0, 2, 4, 14, 0, 0, 0, 0, 6, 14, 0, 0, 0, 0, 0, 0, 0, 6, + 14, 0, 0, 0, 0, 0, 2, 6, 14, 0, 0, 0, 0, 0, 0, 2, 6, 14, + 0, 0, 0, 0, 4, 6, 14, 0, 0, 0, 0, 0, 0, 4, 6, 14, 0, 0, + 0, 0, 2, 4, 6, 14, 0, 0, 0, 0, 0, 2, 4, 6, 14, 0, 0, 0, + 8, 14, 0, 0, 0, 0, 0, 0, 0, 8, 14, 0, 0, 0, 0, 0, 2, 8, + 14, 0, 0, 0, 0, 0, 0, 2, 8, 14, 0, 0, 0, 0, 4, 8, 14, 0, + 0, 0, 0, 0, 0, 4, 8, 14, 0, 0, 0, 0, 2, 4, 8, 14, 0, 0, + 0, 0, 0, 2, 4, 8, 14, 0, 0, 0, 6, 8, 14, 0, 0, 0, 0, 0, + 0, 6, 8, 14, 0, 0, 0, 0, 2, 6, 8, 14, 0, 0, 0, 0, 0, 2, + 6, 8, 14, 0, 0, 0, 4, 6, 8, 14, 0, 0, 0, 0, 0, 4, 6, 8, + 14, 0, 0, 0, 2, 4, 6, 8, 14, 0, 0, 0, 0, 2, 4, 6, 8, 14, + 0, 0, 10, 14, 0, 0, 0, 0, 0, 0, 0, 10, 14, 0, 0, 0, 0, 0, + 2, 10, 14, 0, 0, 0, 0, 0, 0, 2, 10, 14, 0, 0, 0, 0, 4, 10, + 14, 0, 0, 0, 0, 0, 0, 4, 10, 14, 0, 0, 0, 0, 2, 4, 10, 14, + 0, 0, 0, 0, 0, 2, 4, 10, 14, 0, 0, 0, 6, 10, 14, 0, 0, 0, + 0, 0, 0, 6, 10, 14, 0, 0, 0, 0, 2, 6, 10, 14, 0, 0, 0, 0, + 0, 2, 6, 10, 14, 0, 0, 0, 4, 6, 10, 14, 0, 0, 0, 0, 0, 4, + 6, 10, 14, 0, 0, 0, 2, 4, 6, 10, 14, 0, 0, 0, 0, 2, 4, 6, + 10, 14, 0, 0, 8, 10, 14, 0, 0, 0, 0, 0, 0, 8, 10, 14, 0, 0, + 0, 0, 2, 8, 10, 14, 0, 0, 0, 0, 0, 2, 8, 10, 14, 0, 0, 0, + 4, 8, 10, 14, 0, 0, 0, 0, 0, 4, 8, 10, 14, 0, 0, 0, 2, 4, + 8, 10, 14, 0, 0, 0, 0, 2, 4, 8, 10, 14, 0, 0, 6, 8, 10, 14, + 0, 0, 0, 0, 0, 6, 8, 10, 14, 0, 0, 0, 2, 6, 8, 10, 14, 0, + 0, 0, 0, 2, 6, 8, 10, 14, 0, 0, 4, 6, 8, 10, 14, 0, 0, 0, + 0, 4, 6, 8, 10, 14, 0, 0, 2, 4, 6, 8, 10, 14, 0, 0, 0, 2, + 4, 6, 8, 10, 14, 0, 12, 14, 0, 0, 0, 0, 0, 0, 0, 12, 14, 0, + 0, 0, 0, 0, 2, 12, 14, 0, 0, 0, 0, 0, 0, 2, 12, 14, 0, 0, + 0, 0, 4, 12, 14, 0, 0, 0, 0, 0, 0, 4, 12, 14, 0, 0, 0, 0, + 2, 4, 12, 14, 0, 0, 0, 0, 0, 2, 4, 12, 14, 0, 0, 0, 6, 12, + 14, 0, 0, 0, 0, 0, 0, 6, 12, 14, 0, 0, 0, 0, 2, 6, 12, 14, + 0, 0, 0, 0, 0, 2, 6, 12, 14, 0, 0, 0, 4, 6, 12, 14, 0, 0, + 0, 0, 0, 4, 6, 12, 14, 0, 0, 0, 2, 4, 6, 12, 14, 0, 0, 0, + 0, 2, 4, 6, 12, 14, 0, 0, 8, 12, 14, 0, 0, 0, 0, 0, 0, 8, + 12, 14, 0, 0, 0, 0, 2, 8, 12, 14, 0, 0, 0, 0, 0, 2, 8, 12, + 14, 0, 0, 0, 4, 8, 12, 14, 0, 0, 0, 0, 0, 4, 8, 12, 14, 0, + 0, 0, 2, 4, 8, 12, 14, 0, 0, 0, 0, 2, 4, 8, 12, 14, 0, 0, + 6, 8, 12, 14, 0, 0, 0, 0, 0, 6, 8, 12, 14, 0, 0, 0, 2, 6, + 8, 12, 14, 0, 0, 0, 0, 2, 6, 8, 12, 14, 0, 0, 4, 6, 8, 12, + 14, 0, 0, 0, 0, 4, 6, 8, 12, 14, 0, 0, 2, 4, 6, 8, 12, 14, + 0, 0, 0, 2, 4, 6, 8, 12, 14, 0, 10, 12, 14, 0, 0, 0, 0, 0, + 0, 10, 12, 14, 0, 0, 0, 0, 2, 10, 12, 14, 0, 0, 0, 0, 0, 2, + 10, 12, 14, 0, 0, 0, 4, 10, 12, 14, 0, 0, 0, 0, 0, 4, 10, 12, + 14, 0, 0, 0, 2, 4, 10, 12, 14, 0, 0, 0, 0, 2, 4, 10, 12, 14, + 0, 0, 6, 10, 12, 14, 0, 0, 0, 0, 0, 6, 10, 12, 14, 0, 0, 0, + 2, 6, 10, 12, 14, 0, 0, 0, 0, 2, 6, 10, 12, 14, 0, 0, 4, 6, + 10, 12, 14, 0, 0, 0, 0, 4, 6, 10, 12, 14, 0, 0, 2, 4, 6, 10, + 12, 14, 0, 0, 0, 2, 4, 6, 10, 12, 14, 0, 8, 10, 12, 14, 0, 0, + 0, 0, 0, 8, 10, 12, 14, 0, 0, 0, 2, 8, 10, 12, 14, 0, 0, 0, + 0, 2, 8, 10, 12, 14, 0, 0, 4, 8, 10, 12, 14, 0, 0, 0, 0, 4, + 8, 10, 12, 14, 0, 0, 2, 4, 8, 10, 12, 14, 0, 0, 0, 2, 4, 8, + 10, 12, 14, 0, 6, 8, 10, 12, 14, 0, 0, 0, 0, 6, 8, 10, 12, 14, + 0, 0, 2, 6, 8, 10, 12, 14, 0, 0, 0, 2, 6, 8, 10, 12, 14, 0, + 4, 6, 8, 10, 12, 14, 0, 0, 0, 4, 6, 8, 10, 12, 14, 0, 2, 4, + 6, 8, 10, 12, 14, 0, 0, 2, 4, 6, 8, 10, 12, 14}; + + const Vec128<uint8_t, 2 * N> byte_idx{Load(d8, table + mask_bits * 8).raw}; + const Vec128<uint16_t, N> pairs = ZipLower(byte_idx, byte_idx); + return BitCast(d, pairs + Set(du, 0x0100)); +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 4)> +HWY_INLINE Vec128<T, N> IndicesFromBits(Simd<T, N> d, uint64_t mask_bits) { + HWY_DASSERT(mask_bits < 16); + + // There are only 4 lanes, so we can afford to load the index vector directly. + alignas(16) constexpr uint8_t packed_array[256] = { + 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, // + 4, 5, 6, 7, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 0, 1, 2, 3, // + 8, 9, 10, 11, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 8, 9, 10, 11, 0, 1, 2, 3, 0, 1, 2, 3, // + 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, // + 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3, // + 4, 5, 6, 7, 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 4, 5, 6, 7, 12, 13, 14, 15, 0, 1, 2, 3, // + 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 0, 1, 2, 3, // + 0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, // + 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, // + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, Load(d8, packed_array + 16 * mask_bits)); +} + +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 8)> +HWY_INLINE Vec128<T, N> IndicesFromBits(Simd<T, N> d, uint64_t mask_bits) { + HWY_DASSERT(mask_bits < 4); + + // There are only 2 lanes, so we can afford to load the index vector directly. + alignas(16) constexpr uint8_t packed_array[64] = { + 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, // + 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, // + 8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7, // + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, Load(d8, packed_array + 16 * mask_bits)); +} + +} // namespace detail + +template <typename T, size_t N> +HWY_API Vec128<T, N> Compress(Vec128<T, N> v, Mask128<T, N> m) { + const Simd<T, N> d; + const RebindToUnsigned<decltype(d)> du; + + const uint64_t mask_bits = detail::BitsFromMask(m); + HWY_DASSERT(mask_bits < (1ull << N)); + + const auto indices = BitCast(du, detail::IndicesFromBits(d, mask_bits)); + return BitCast(d, TableLookupBytes(BitCast(du, v), indices)); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> CompressBits(Vec128<T, N> v, + const uint8_t* HWY_RESTRICT bits) { + const Simd<T, N> d; + const RebindToUnsigned<decltype(d)> du; + + uint64_t mask_bits = 0; + constexpr size_t kNumBytes = (N + 7) / 8; + CopyBytes<kNumBytes>(bits, &mask_bits); + if (N < 8) { + mask_bits &= (1ull << N) - 1; + } + + const auto indices = BitCast(du, detail::IndicesFromBits(d, mask_bits)); + return BitCast(d, TableLookupBytes(BitCast(du, v), indices)); +} + +// ------------------------------ CompressStore, CompressBitsStore + +template <typename T, size_t N> +HWY_API size_t CompressStore(Vec128<T, N> v, Mask128<T, N> m, Simd<T, N> d, + T* HWY_RESTRICT unaligned) { + const RebindToUnsigned<decltype(d)> du; + + const uint64_t mask_bits = detail::BitsFromMask(m); + HWY_DASSERT(mask_bits < (1ull << N)); + + // Avoid _mm_maskmoveu_si128 (>500 cycle latency because it bypasses caches). + const auto indices = BitCast(du, detail::IndicesFromBits(d, mask_bits)); + const auto compressed = BitCast(d, TableLookupBytes(BitCast(du, v), indices)); + StoreU(compressed, d, unaligned); + return PopCount(mask_bits); +} + +template <typename T, size_t N> +HWY_API size_t CompressBlendedStore(Vec128<T, N> v, Mask128<T, N> m, + Simd<T, N> d, T* HWY_RESTRICT unaligned) { + const RebindToUnsigned<decltype(d)> du; + + const uint64_t mask_bits = detail::BitsFromMask(m); + HWY_DASSERT(mask_bits < (1ull << N)); + const size_t count = PopCount(mask_bits); + + // Avoid _mm_maskmoveu_si128 (>500 cycle latency because it bypasses caches). + const auto indices = BitCast(du, detail::IndicesFromBits(d, mask_bits)); + const auto compressed = BitCast(d, TableLookupBytes(BitCast(du, v), indices)); + + const Vec128<T, N> prev = LoadU(d, unaligned); + StoreU(IfThenElse(FirstN(d, count), compressed, prev), d, unaligned); + return count; +} + +template <typename T, size_t N> +HWY_API size_t CompressBitsStore(Vec128<T, N> v, + const uint8_t* HWY_RESTRICT bits, Simd<T, N> d, + T* HWY_RESTRICT unaligned) { + const RebindToUnsigned<decltype(d)> du; + + uint64_t mask_bits = 0; + constexpr size_t kNumBytes = (N + 7) / 8; + CopyBytes<kNumBytes>(bits, &mask_bits); + if (N < 8) { + mask_bits &= (1ull << N) - 1; + } + + // Avoid _mm_maskmoveu_si128 (>500 cycle latency because it bypasses caches). + const auto indices = BitCast(du, detail::IndicesFromBits(d, mask_bits)); + const auto compressed = BitCast(d, TableLookupBytes(BitCast(du, v), indices)); + StoreU(compressed, d, unaligned); + return PopCount(mask_bits); +} + +#endif // HWY_TARGET <= HWY_AVX3 + +// ------------------------------ StoreInterleaved3 (CombineShiftRightBytes, +// TableLookupBytes) + +// 128 bits +HWY_API void StoreInterleaved3(const Vec128<uint8_t> v0, + const Vec128<uint8_t> v1, + const Vec128<uint8_t> v2, Full128<uint8_t> d, + uint8_t* HWY_RESTRICT unaligned) { + const auto k5 = Set(d, 5); + const auto k6 = Set(d, 6); + + // Shuffle (v0,v1,v2) vector bytes to (MSB on left): r5, bgr[4:0]. + // 0x80 so lanes to be filled from other vectors are 0 for blending. + alignas(16) static constexpr uint8_t tbl_r0[16] = { + 0, 0x80, 0x80, 1, 0x80, 0x80, 2, 0x80, 0x80, // + 3, 0x80, 0x80, 4, 0x80, 0x80, 5}; + alignas(16) static constexpr uint8_t tbl_g0[16] = { + 0x80, 0, 0x80, 0x80, 1, 0x80, // + 0x80, 2, 0x80, 0x80, 3, 0x80, 0x80, 4, 0x80, 0x80}; + const auto shuf_r0 = Load(d, tbl_r0); + const auto shuf_g0 = Load(d, tbl_g0); // cannot reuse r0 due to 5 in MSB + const auto shuf_b0 = CombineShiftRightBytes<15>(d, shuf_g0, shuf_g0); + const auto r0 = TableLookupBytes(v0, shuf_r0); // 5..4..3..2..1..0 + const auto g0 = TableLookupBytes(v1, shuf_g0); // ..4..3..2..1..0. + const auto b0 = TableLookupBytes(v2, shuf_b0); // .4..3..2..1..0.. + const auto int0 = r0 | g0 | b0; + StoreU(int0, d, unaligned + 0 * 16); + + // Second vector: g10,r10, bgr[9:6], b5,g5 + const auto shuf_r1 = shuf_b0 + k6; // .A..9..8..7..6.. + const auto shuf_g1 = shuf_r0 + k5; // A..9..8..7..6..5 + const auto shuf_b1 = shuf_g0 + k5; // ..9..8..7..6..5. + const auto r1 = TableLookupBytes(v0, shuf_r1); + const auto g1 = TableLookupBytes(v1, shuf_g1); + const auto b1 = TableLookupBytes(v2, shuf_b1); + const auto int1 = r1 | g1 | b1; + StoreU(int1, d, unaligned + 1 * 16); + + // Third vector: bgr[15:11], b10 + const auto shuf_r2 = shuf_b1 + k6; // ..F..E..D..C..B. + const auto shuf_g2 = shuf_r1 + k5; // .F..E..D..C..B.. + const auto shuf_b2 = shuf_g1 + k5; // F..E..D..C..B..A + const auto r2 = TableLookupBytes(v0, shuf_r2); + const auto g2 = TableLookupBytes(v1, shuf_g2); + const auto b2 = TableLookupBytes(v2, shuf_b2); + const auto int2 = r2 | g2 | b2; + StoreU(int2, d, unaligned + 2 * 16); +} + +// 64 bits +HWY_API void StoreInterleaved3(const Vec128<uint8_t, 8> v0, + const Vec128<uint8_t, 8> v1, + const Vec128<uint8_t, 8> v2, Simd<uint8_t, 8> d, + uint8_t* HWY_RESTRICT unaligned) { + // Use full vectors for the shuffles and first result. + const Full128<uint8_t> d_full; + const auto k5 = Set(d_full, 5); + const auto k6 = Set(d_full, 6); + + const Vec128<uint8_t> full_a{v0.raw}; + const Vec128<uint8_t> full_b{v1.raw}; + const Vec128<uint8_t> full_c{v2.raw}; + + // Shuffle (v0,v1,v2) vector bytes to (MSB on left): r5, bgr[4:0]. + // 0x80 so lanes to be filled from other vectors are 0 for blending. + alignas(16) static constexpr uint8_t tbl_r0[16] = { + 0, 0x80, 0x80, 1, 0x80, 0x80, 2, 0x80, 0x80, // + 3, 0x80, 0x80, 4, 0x80, 0x80, 5}; + alignas(16) static constexpr uint8_t tbl_g0[16] = { + 0x80, 0, 0x80, 0x80, 1, 0x80, // + 0x80, 2, 0x80, 0x80, 3, 0x80, 0x80, 4, 0x80, 0x80}; + const auto shuf_r0 = Load(d_full, tbl_r0); + const auto shuf_g0 = Load(d_full, tbl_g0); // cannot reuse r0 due to 5 in MSB + const auto shuf_b0 = CombineShiftRightBytes<15>(d_full, shuf_g0, shuf_g0); + const auto r0 = TableLookupBytes(full_a, shuf_r0); // 5..4..3..2..1..0 + const auto g0 = TableLookupBytes(full_b, shuf_g0); // ..4..3..2..1..0. + const auto b0 = TableLookupBytes(full_c, shuf_b0); // .4..3..2..1..0.. + const auto int0 = r0 | g0 | b0; + StoreU(int0, d_full, unaligned + 0 * 16); + + // Second (HALF) vector: bgr[7:6], b5,g5 + const auto shuf_r1 = shuf_b0 + k6; // ..7..6.. + const auto shuf_g1 = shuf_r0 + k5; // .7..6..5 + const auto shuf_b1 = shuf_g0 + k5; // 7..6..5. + const auto r1 = TableLookupBytes(full_a, shuf_r1); + const auto g1 = TableLookupBytes(full_b, shuf_g1); + const auto b1 = TableLookupBytes(full_c, shuf_b1); + const decltype(Zero(d)) int1{(r1 | g1 | b1).raw}; + StoreU(int1, d, unaligned + 1 * 16); +} + +// <= 32 bits +template <size_t N, HWY_IF_LE32(uint8_t, N)> +HWY_API void StoreInterleaved3(const Vec128<uint8_t, N> v0, + const Vec128<uint8_t, N> v1, + const Vec128<uint8_t, N> v2, + Simd<uint8_t, N> /*tag*/, + uint8_t* HWY_RESTRICT unaligned) { + // Use full vectors for the shuffles and result. + const Full128<uint8_t> d_full; + + const Vec128<uint8_t> full_a{v0.raw}; + const Vec128<uint8_t> full_b{v1.raw}; + const Vec128<uint8_t> full_c{v2.raw}; + + // Shuffle (v0,v1,v2) vector bytes to bgr[3:0]. + // 0x80 so lanes to be filled from other vectors are 0 for blending. + alignas(16) static constexpr uint8_t tbl_r0[16] = { + 0, 0x80, 0x80, 1, 0x80, 0x80, 2, 0x80, 0x80, 3, 0x80, 0x80, // + 0x80, 0x80, 0x80, 0x80}; + const auto shuf_r0 = Load(d_full, tbl_r0); + const auto shuf_g0 = CombineShiftRightBytes<15>(d_full, shuf_r0, shuf_r0); + const auto shuf_b0 = CombineShiftRightBytes<14>(d_full, shuf_r0, shuf_r0); + const auto r0 = TableLookupBytes(full_a, shuf_r0); // ......3..2..1..0 + const auto g0 = TableLookupBytes(full_b, shuf_g0); // .....3..2..1..0. + const auto b0 = TableLookupBytes(full_c, shuf_b0); // ....3..2..1..0.. + const auto int0 = r0 | g0 | b0; + alignas(16) uint8_t buf[16]; + StoreU(int0, d_full, buf); + CopyBytes<N * 3>(buf, unaligned); +} + +// ------------------------------ StoreInterleaved4 + +// 128 bits +HWY_API void StoreInterleaved4(const Vec128<uint8_t> v0, + const Vec128<uint8_t> v1, + const Vec128<uint8_t> v2, + const Vec128<uint8_t> v3, Full128<uint8_t> d8, + uint8_t* HWY_RESTRICT unaligned) { + const RepartitionToWide<decltype(d8)> d16; + const RepartitionToWide<decltype(d16)> d32; + // let a,b,c,d denote v0..3. + const auto ba0 = ZipLower(d16, v0, v1); // b7 a7 .. b0 a0 + const auto dc0 = ZipLower(d16, v2, v3); // d7 c7 .. d0 c0 + const auto ba8 = ZipUpper(d16, v0, v1); + const auto dc8 = ZipUpper(d16, v2, v3); + const auto dcba_0 = ZipLower(d32, ba0, dc0); // d..a3 d..a0 + const auto dcba_4 = ZipUpper(d32, ba0, dc0); // d..a7 d..a4 + const auto dcba_8 = ZipLower(d32, ba8, dc8); // d..aB d..a8 + const auto dcba_C = ZipUpper(d32, ba8, dc8); // d..aF d..aC + StoreU(BitCast(d8, dcba_0), d8, unaligned + 0 * 16); + StoreU(BitCast(d8, dcba_4), d8, unaligned + 1 * 16); + StoreU(BitCast(d8, dcba_8), d8, unaligned + 2 * 16); + StoreU(BitCast(d8, dcba_C), d8, unaligned + 3 * 16); +} + +// 64 bits +HWY_API void StoreInterleaved4(const Vec128<uint8_t, 8> in0, + const Vec128<uint8_t, 8> in1, + const Vec128<uint8_t, 8> in2, + const Vec128<uint8_t, 8> in3, + Simd<uint8_t, 8> /*tag*/, + uint8_t* HWY_RESTRICT unaligned) { + // Use full vectors to reduce the number of stores. + const Full128<uint8_t> d_full8; + const RepartitionToWide<decltype(d_full8)> d16; + const RepartitionToWide<decltype(d16)> d32; + const Vec128<uint8_t> v0{in0.raw}; + const Vec128<uint8_t> v1{in1.raw}; + const Vec128<uint8_t> v2{in2.raw}; + const Vec128<uint8_t> v3{in3.raw}; + // let a,b,c,d denote v0..3. + const auto ba0 = ZipLower(d16, v0, v1); // b7 a7 .. b0 a0 + const auto dc0 = ZipLower(d16, v2, v3); // d7 c7 .. d0 c0 + const auto dcba_0 = ZipLower(d32, ba0, dc0); // d..a3 d..a0 + const auto dcba_4 = ZipUpper(d32, ba0, dc0); // d..a7 d..a4 + StoreU(BitCast(d_full8, dcba_0), d_full8, unaligned + 0 * 16); + StoreU(BitCast(d_full8, dcba_4), d_full8, unaligned + 1 * 16); +} + +// <= 32 bits +template <size_t N, HWY_IF_LE32(uint8_t, N)> +HWY_API void StoreInterleaved4(const Vec128<uint8_t, N> in0, + const Vec128<uint8_t, N> in1, + const Vec128<uint8_t, N> in2, + const Vec128<uint8_t, N> in3, + Simd<uint8_t, N> /*tag*/, + uint8_t* HWY_RESTRICT unaligned) { + // Use full vectors to reduce the number of stores. + const Full128<uint8_t> d_full8; + const RepartitionToWide<decltype(d_full8)> d16; + const RepartitionToWide<decltype(d16)> d32; + const Vec128<uint8_t> v0{in0.raw}; + const Vec128<uint8_t> v1{in1.raw}; + const Vec128<uint8_t> v2{in2.raw}; + const Vec128<uint8_t> v3{in3.raw}; + // let a,b,c,d denote v0..3. + const auto ba0 = ZipLower(d16, v0, v1); // b3 a3 .. b0 a0 + const auto dc0 = ZipLower(d16, v2, v3); // d3 c3 .. d0 c0 + const auto dcba_0 = ZipLower(d32, ba0, dc0); // d..a3 d..a0 + alignas(16) uint8_t buf[16]; + StoreU(BitCast(d_full8, dcba_0), d_full8, buf); + CopyBytes<4 * N>(buf, unaligned); +} + +// ------------------------------ Reductions + +namespace detail { + +// N=1 for any T: no-op +template <typename T> +HWY_INLINE Vec128<T, 1> SumOfLanes(hwy::SizeTag<sizeof(T)> /* tag */, + const Vec128<T, 1> v) { + return v; +} +template <typename T> +HWY_INLINE Vec128<T, 1> MinOfLanes(hwy::SizeTag<sizeof(T)> /* tag */, + const Vec128<T, 1> v) { + return v; +} +template <typename T> +HWY_INLINE Vec128<T, 1> MaxOfLanes(hwy::SizeTag<sizeof(T)> /* tag */, + const Vec128<T, 1> v) { + return v; +} + +// u32/i32/f32: + +// N=2 +template <typename T> +HWY_INLINE Vec128<T, 2> SumOfLanes(hwy::SizeTag<4> /* tag */, + const Vec128<T, 2> v10) { + return v10 + Shuffle2301(v10); +} +template <typename T> +HWY_INLINE Vec128<T, 2> MinOfLanes(hwy::SizeTag<4> /* tag */, + const Vec128<T, 2> v10) { + return Min(v10, Shuffle2301(v10)); +} +template <typename T> +HWY_INLINE Vec128<T, 2> MaxOfLanes(hwy::SizeTag<4> /* tag */, + const Vec128<T, 2> v10) { + return Max(v10, Shuffle2301(v10)); +} + +// N=4 (full) +template <typename T> +HWY_INLINE Vec128<T> SumOfLanes(hwy::SizeTag<4> /* tag */, + const Vec128<T> v3210) { + const Vec128<T> v1032 = Shuffle1032(v3210); + const Vec128<T> v31_20_31_20 = v3210 + v1032; + const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20); + return v20_31_20_31 + v31_20_31_20; +} +template <typename T> +HWY_INLINE Vec128<T> MinOfLanes(hwy::SizeTag<4> /* tag */, + const Vec128<T> v3210) { + const Vec128<T> v1032 = Shuffle1032(v3210); + const Vec128<T> v31_20_31_20 = Min(v3210, v1032); + const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20); + return Min(v20_31_20_31, v31_20_31_20); +} +template <typename T> +HWY_INLINE Vec128<T> MaxOfLanes(hwy::SizeTag<4> /* tag */, + const Vec128<T> v3210) { + const Vec128<T> v1032 = Shuffle1032(v3210); + const Vec128<T> v31_20_31_20 = Max(v3210, v1032); + const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20); + return Max(v20_31_20_31, v31_20_31_20); +} + +// u64/i64/f64: + +// N=2 (full) +template <typename T> +HWY_INLINE Vec128<T> SumOfLanes(hwy::SizeTag<8> /* tag */, + const Vec128<T> v10) { + const Vec128<T> v01 = Shuffle01(v10); + return v10 + v01; +} +template <typename T> +HWY_INLINE Vec128<T> MinOfLanes(hwy::SizeTag<8> /* tag */, + const Vec128<T> v10) { + const Vec128<T> v01 = Shuffle01(v10); + return Min(v10, v01); +} +template <typename T> +HWY_INLINE Vec128<T> MaxOfLanes(hwy::SizeTag<8> /* tag */, + const Vec128<T> v10) { + const Vec128<T> v01 = Shuffle01(v10); + return Max(v10, v01); +} + +// u16/i16 +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 2), HWY_IF_GE32(T, N)> +HWY_API Vec128<T, N> MinOfLanes(hwy::SizeTag<2> /* tag */, Vec128<T, N> v) { + const Repartition<int32_t, Simd<T, N>> d32; + const auto even = And(BitCast(d32, v), Set(d32, 0xFFFF)); + const auto odd = ShiftRight<16>(BitCast(d32, v)); + const auto min = MinOfLanes(d32, Min(even, odd)); + // Also broadcast into odd lanes. + return BitCast(Simd<T, N>(), Or(min, ShiftLeft<16>(min))); +} +template <typename T, size_t N, HWY_IF_LANE_SIZE(T, 2), HWY_IF_GE32(T, N)> +HWY_API Vec128<T, N> MaxOfLanes(hwy::SizeTag<2> /* tag */, Vec128<T, N> v) { + const Repartition<int32_t, Simd<T, N>> d32; + const auto even = And(BitCast(d32, v), Set(d32, 0xFFFF)); + const auto odd = ShiftRight<16>(BitCast(d32, v)); + const auto min = MaxOfLanes(d32, Max(even, odd)); + // Also broadcast into odd lanes. + return BitCast(Simd<T, N>(), Or(min, ShiftLeft<16>(min))); +} + +} // namespace detail + +// Supported for u/i/f 32/64. Returns the same value in each lane. +template <typename T, size_t N> +HWY_API Vec128<T, N> SumOfLanes(Simd<T, N> /* tag */, const Vec128<T, N> v) { + return detail::SumOfLanes(hwy::SizeTag<sizeof(T)>(), v); +} +template <typename T, size_t N> +HWY_API Vec128<T, N> MinOfLanes(Simd<T, N> /* tag */, const Vec128<T, N> v) { + return detail::MinOfLanes(hwy::SizeTag<sizeof(T)>(), v); +} +template <typename T, size_t N> +HWY_API Vec128<T, N> MaxOfLanes(Simd<T, N> /* tag */, const Vec128<T, N> v) { + return detail::MaxOfLanes(hwy::SizeTag<sizeof(T)>(), v); +} + +// ================================================== DEPRECATED + +template <typename T, size_t N> +HWY_API size_t StoreMaskBits(const Mask128<T, N> mask, uint8_t* bits) { + return StoreMaskBits(Simd<T, N>(), mask, bits); +} + +template <typename T, size_t N> +HWY_API bool AllTrue(const Mask128<T, N> mask) { + return AllTrue(Simd<T, N>(), mask); +} + +template <typename T, size_t N> +HWY_API bool AllFalse(const Mask128<T, N> mask) { + return AllFalse(Simd<T, N>(), mask); +} + +template <typename T, size_t N> +HWY_API size_t CountTrue(const Mask128<T, N> mask) { + return CountTrue(Simd<T, N>(), mask); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> SumOfLanes(const Vec128<T, N> v) { + return SumOfLanes(Simd<T, N>(), v); +} +template <typename T, size_t N> +HWY_API Vec128<T, N> MinOfLanes(const Vec128<T, N> v) { + return MinOfLanes(Simd<T, N>(), v); +} +template <typename T, size_t N> +HWY_API Vec128<T, N> MaxOfLanes(const Vec128<T, N> v) { + return MaxOfLanes(Simd<T, N>(), v); +} + +template <typename T, size_t N> +HWY_API Vec128<T, (N + 1) / 2> UpperHalf(Vec128<T, N> v) { + return UpperHalf(Half<Simd<T, N>>(), v); +} + +template <int kBytes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftRightBytes(const Vec128<T, N> v) { + return ShiftRightBytes<kBytes>(Simd<T, N>(), v); +} + +template <int kLanes, typename T, size_t N> +HWY_API Vec128<T, N> ShiftRightLanes(const Vec128<T, N> v) { + return ShiftRightLanes<kLanes>(Simd<T, N>(), v); +} + +template <size_t kBytes, typename T, size_t N> +HWY_API Vec128<T, N> CombineShiftRightBytes(Vec128<T, N> hi, Vec128<T, N> lo) { + return CombineShiftRightBytes<kBytes>(Simd<T, N>(), hi, lo); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> InterleaveUpper(Vec128<T, N> a, Vec128<T, N> b) { + return InterleaveUpper(Simd<T, N>(), a, b); +} + +template <typename T, size_t N, class D = Simd<T, N>> +HWY_API VFromD<RepartitionToWide<D>> ZipUpper(Vec128<T, N> a, Vec128<T, N> b) { + return InterleaveUpper(RepartitionToWide<D>(), a, b); +} + +template <typename T, size_t N2> +HWY_API Vec128<T, N2 * 2> Combine(Vec128<T, N2> hi2, Vec128<T, N2> lo2) { + return Combine(Simd<T, N2 * 2>(), hi2, lo2); +} + +template <typename T, size_t N2, HWY_IF_LE64(T, N2)> +HWY_API Vec128<T, N2 * 2> ZeroExtendVector(Vec128<T, N2> lo) { + return ZeroExtendVector(Simd<T, N2 * 2>(), lo); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> ConcatLowerLower(Vec128<T, N> hi, Vec128<T, N> lo) { + return ConcatLowerLower(Simd<T, N>(), hi, lo); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> ConcatUpperUpper(Vec128<T, N> hi, Vec128<T, N> lo) { + return ConcatUpperUpper(Simd<T, N>(), hi, lo); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> ConcatLowerUpper(const Vec128<T, N> hi, + const Vec128<T, N> lo) { + return ConcatLowerUpper(Simd<T, N>(), hi, lo); +} + +template <typename T, size_t N> +HWY_API Vec128<T, N> ConcatUpperLower(Vec128<T, N> hi, Vec128<T, N> lo) { + return ConcatUpperLower(Simd<T, N>(), hi, lo); +} + +// ================================================== Operator wrapper + +// These apply to all x86_*-inl.h because there are no restrictions on V. + +template <class V> +HWY_API V Add(V a, V b) { + return a + b; +} +template <class V> +HWY_API V Sub(V a, V b) { + return a - b; +} + +template <class V> +HWY_API V Mul(V a, V b) { + return a * b; +} +template <class V> +HWY_API V Div(V a, V b) { + return a / b; +} + +template <class V> +V Shl(V a, V b) { + return a << b; +} +template <class V> +V Shr(V a, V b) { + return a >> b; +} + +template <class V> +HWY_API auto Eq(V a, V b) -> decltype(a == b) { + return a == b; +} +template <class V> +HWY_API auto Ne(V a, V b) -> decltype(a == b) { + return a != b; +} +template <class V> +HWY_API auto Lt(V a, V b) -> decltype(a == b) { + return a < b; +} + +template <class V> +HWY_API auto Gt(V a, V b) -> decltype(a == b) { + return a > b; +} +template <class V> +HWY_API auto Ge(V a, V b) -> decltype(a == b) { + return a >= b; +} + +template <class V> +HWY_API auto Le(V a, V b) -> decltype(a == b) { + return a <= b; +} + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); diff --git a/third_party/highway/hwy/ops/x86_256-inl.h b/third_party/highway/hwy/ops/x86_256-inl.h new file mode 100644 index 0000000..2a5315a --- /dev/null +++ b/third_party/highway/hwy/ops/x86_256-inl.h @@ -0,0 +1,4726 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// 256-bit vectors and AVX2 instructions, plus some AVX512-VL operations when +// compiling for that target. +// External include guard in highway.h - see comment there. + +// WARNING: most operations do not cross 128-bit block boundaries. In +// particular, "Broadcast", pack and zip behavior may be surprising. + +#include <immintrin.h> // AVX2+ + +#include "hwy/base.h" +#if defined(_MSC_VER) && defined(__clang__) +// Including <immintrin.h> should be enough, but Clang's headers helpfully skip +// including these headers when _MSC_VER is defined, like when using clang-cl. +// Include these directly here. +#include <avxintrin.h> +// avxintrin defines __m256i and must come before avx2intrin. +#include <avx2intrin.h> +#include <bmi2intrin.h> // _pext_u64 +#include <f16cintrin.h> +#include <fmaintrin.h> +#include <smmintrin.h> +#endif + +#include <stddef.h> +#include <stdint.h> + +// For half-width vectors. Already includes base.h and shared-inl.h. +#include "hwy/ops/x86_128-inl.h" + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +template <typename T> +using Full256 = Simd<T, 32 / sizeof(T)>; + +namespace detail { + +template <typename T> +struct Raw256 { + using type = __m256i; +}; +template <> +struct Raw256<float> { + using type = __m256; +}; +template <> +struct Raw256<double> { + using type = __m256d; +}; + +} // namespace detail + +template <typename T> +class Vec256 { + using Raw = typename detail::Raw256<T>::type; + + public: + // Compound assignment. Only usable if there is a corresponding non-member + // binary operator overload. For example, only f32 and f64 support division. + HWY_INLINE Vec256& operator*=(const Vec256 other) { + return *this = (*this * other); + } + HWY_INLINE Vec256& operator/=(const Vec256 other) { + return *this = (*this / other); + } + HWY_INLINE Vec256& operator+=(const Vec256 other) { + return *this = (*this + other); + } + HWY_INLINE Vec256& operator-=(const Vec256 other) { + return *this = (*this - other); + } + HWY_INLINE Vec256& operator&=(const Vec256 other) { + return *this = (*this & other); + } + HWY_INLINE Vec256& operator|=(const Vec256 other) { + return *this = (*this | other); + } + HWY_INLINE Vec256& operator^=(const Vec256 other) { + return *this = (*this ^ other); + } + + Raw raw; +}; + +#if HWY_TARGET <= HWY_AVX3 + +namespace detail { + +// Template arg: sizeof(lane type) +template <size_t size> +struct RawMask256 {}; +template <> +struct RawMask256<1> { + using type = __mmask32; +}; +template <> +struct RawMask256<2> { + using type = __mmask16; +}; +template <> +struct RawMask256<4> { + using type = __mmask8; +}; +template <> +struct RawMask256<8> { + using type = __mmask8; +}; + +} // namespace detail + +template <typename T> +struct Mask256 { + using Raw = typename detail::RawMask256<sizeof(T)>::type; + + static Mask256<T> FromBits(uint64_t mask_bits) { + return Mask256<T>{static_cast<Raw>(mask_bits)}; + } + + Raw raw; +}; + +#else // AVX2 + +// FF..FF or 0. +template <typename T> +struct Mask256 { + typename detail::Raw256<T>::type raw; +}; + +#endif // HWY_TARGET <= HWY_AVX3 + +// ------------------------------ BitCast + +namespace detail { + +HWY_INLINE __m256i BitCastToInteger(__m256i v) { return v; } +HWY_INLINE __m256i BitCastToInteger(__m256 v) { return _mm256_castps_si256(v); } +HWY_INLINE __m256i BitCastToInteger(__m256d v) { + return _mm256_castpd_si256(v); +} + +template <typename T> +HWY_INLINE Vec256<uint8_t> BitCastToByte(Vec256<T> v) { + return Vec256<uint8_t>{BitCastToInteger(v.raw)}; +} + +// Cannot rely on function overloading because return types differ. +template <typename T> +struct BitCastFromInteger256 { + HWY_INLINE __m256i operator()(__m256i v) { return v; } +}; +template <> +struct BitCastFromInteger256<float> { + HWY_INLINE __m256 operator()(__m256i v) { return _mm256_castsi256_ps(v); } +}; +template <> +struct BitCastFromInteger256<double> { + HWY_INLINE __m256d operator()(__m256i v) { return _mm256_castsi256_pd(v); } +}; + +template <typename T> +HWY_INLINE Vec256<T> BitCastFromByte(Full256<T> /* tag */, Vec256<uint8_t> v) { + return Vec256<T>{BitCastFromInteger256<T>()(v.raw)}; +} + +} // namespace detail + +template <typename T, typename FromT> +HWY_API Vec256<T> BitCast(Full256<T> d, Vec256<FromT> v) { + return detail::BitCastFromByte(d, detail::BitCastToByte(v)); +} + +// ------------------------------ Set + +// Returns an all-zero vector. +template <typename T> +HWY_API Vec256<T> Zero(Full256<T> /* tag */) { + return Vec256<T>{_mm256_setzero_si256()}; +} +HWY_API Vec256<float> Zero(Full256<float> /* tag */) { + return Vec256<float>{_mm256_setzero_ps()}; +} +HWY_API Vec256<double> Zero(Full256<double> /* tag */) { + return Vec256<double>{_mm256_setzero_pd()}; +} + +// Returns a vector with all lanes set to "t". +HWY_API Vec256<uint8_t> Set(Full256<uint8_t> /* tag */, const uint8_t t) { + return Vec256<uint8_t>{_mm256_set1_epi8(static_cast<char>(t))}; // NOLINT +} +HWY_API Vec256<uint16_t> Set(Full256<uint16_t> /* tag */, const uint16_t t) { + return Vec256<uint16_t>{_mm256_set1_epi16(static_cast<short>(t))}; // NOLINT +} +HWY_API Vec256<uint32_t> Set(Full256<uint32_t> /* tag */, const uint32_t t) { + return Vec256<uint32_t>{_mm256_set1_epi32(static_cast<int>(t))}; +} +HWY_API Vec256<uint64_t> Set(Full256<uint64_t> /* tag */, const uint64_t t) { + return Vec256<uint64_t>{ + _mm256_set1_epi64x(static_cast<long long>(t))}; // NOLINT +} +HWY_API Vec256<int8_t> Set(Full256<int8_t> /* tag */, const int8_t t) { + return Vec256<int8_t>{_mm256_set1_epi8(static_cast<char>(t))}; // NOLINT +} +HWY_API Vec256<int16_t> Set(Full256<int16_t> /* tag */, const int16_t t) { + return Vec256<int16_t>{_mm256_set1_epi16(static_cast<short>(t))}; // NOLINT +} +HWY_API Vec256<int32_t> Set(Full256<int32_t> /* tag */, const int32_t t) { + return Vec256<int32_t>{_mm256_set1_epi32(t)}; +} +HWY_API Vec256<int64_t> Set(Full256<int64_t> /* tag */, const int64_t t) { + return Vec256<int64_t>{ + _mm256_set1_epi64x(static_cast<long long>(t))}; // NOLINT +} +HWY_API Vec256<float> Set(Full256<float> /* tag */, const float t) { + return Vec256<float>{_mm256_set1_ps(t)}; +} +HWY_API Vec256<double> Set(Full256<double> /* tag */, const double t) { + return Vec256<double>{_mm256_set1_pd(t)}; +} + +HWY_DIAGNOSTICS(push) +HWY_DIAGNOSTICS_OFF(disable : 4700, ignored "-Wuninitialized") + +// Returns a vector with uninitialized elements. +template <typename T> +HWY_API Vec256<T> Undefined(Full256<T> /* tag */) { + // Available on Clang 6.0, GCC 6.2, ICC 16.03, MSVC 19.14. All but ICC + // generate an XOR instruction. + return Vec256<T>{_mm256_undefined_si256()}; +} +HWY_API Vec256<float> Undefined(Full256<float> /* tag */) { + return Vec256<float>{_mm256_undefined_ps()}; +} +HWY_API Vec256<double> Undefined(Full256<double> /* tag */) { + return Vec256<double>{_mm256_undefined_pd()}; +} + +HWY_DIAGNOSTICS(pop) + +// ================================================== LOGICAL + +// ------------------------------ And + +template <typename T> +HWY_API Vec256<T> And(Vec256<T> a, Vec256<T> b) { + return Vec256<T>{_mm256_and_si256(a.raw, b.raw)}; +} + +HWY_API Vec256<float> And(const Vec256<float> a, const Vec256<float> b) { + return Vec256<float>{_mm256_and_ps(a.raw, b.raw)}; +} +HWY_API Vec256<double> And(const Vec256<double> a, const Vec256<double> b) { + return Vec256<double>{_mm256_and_pd(a.raw, b.raw)}; +} + +// ------------------------------ AndNot + +// Returns ~not_mask & mask. +template <typename T> +HWY_API Vec256<T> AndNot(Vec256<T> not_mask, Vec256<T> mask) { + return Vec256<T>{_mm256_andnot_si256(not_mask.raw, mask.raw)}; +} +HWY_API Vec256<float> AndNot(const Vec256<float> not_mask, + const Vec256<float> mask) { + return Vec256<float>{_mm256_andnot_ps(not_mask.raw, mask.raw)}; +} +HWY_API Vec256<double> AndNot(const Vec256<double> not_mask, + const Vec256<double> mask) { + return Vec256<double>{_mm256_andnot_pd(not_mask.raw, mask.raw)}; +} + +// ------------------------------ Or + +template <typename T> +HWY_API Vec256<T> Or(Vec256<T> a, Vec256<T> b) { + return Vec256<T>{_mm256_or_si256(a.raw, b.raw)}; +} + +HWY_API Vec256<float> Or(const Vec256<float> a, const Vec256<float> b) { + return Vec256<float>{_mm256_or_ps(a.raw, b.raw)}; +} +HWY_API Vec256<double> Or(const Vec256<double> a, const Vec256<double> b) { + return Vec256<double>{_mm256_or_pd(a.raw, b.raw)}; +} + +// ------------------------------ Xor + +template <typename T> +HWY_API Vec256<T> Xor(Vec256<T> a, Vec256<T> b) { + return Vec256<T>{_mm256_xor_si256(a.raw, b.raw)}; +} + +HWY_API Vec256<float> Xor(const Vec256<float> a, const Vec256<float> b) { + return Vec256<float>{_mm256_xor_ps(a.raw, b.raw)}; +} +HWY_API Vec256<double> Xor(const Vec256<double> a, const Vec256<double> b) { + return Vec256<double>{_mm256_xor_pd(a.raw, b.raw)}; +} + +// ------------------------------ Not + +template <typename T> +HWY_API Vec256<T> Not(const Vec256<T> v) { + using TU = MakeUnsigned<T>; +#if HWY_TARGET <= HWY_AVX3 + const __m256i vu = BitCast(Full256<TU>(), v).raw; + return BitCast(Full256<T>(), + Vec256<TU>{_mm256_ternarylogic_epi32(vu, vu, vu, 0x55)}); +#else + return Xor(v, BitCast(Full256<T>(), Vec256<TU>{_mm256_set1_epi32(-1)})); +#endif +} + +// ------------------------------ Operator overloads (internal-only if float) + +template <typename T> +HWY_API Vec256<T> operator&(const Vec256<T> a, const Vec256<T> b) { + return And(a, b); +} + +template <typename T> +HWY_API Vec256<T> operator|(const Vec256<T> a, const Vec256<T> b) { + return Or(a, b); +} + +template <typename T> +HWY_API Vec256<T> operator^(const Vec256<T> a, const Vec256<T> b) { + return Xor(a, b); +} + +// ------------------------------ PopulationCount + +// 8/16 require BITALG, 32/64 require VPOPCNTDQ. +#if HWY_TARGET == HWY_AVX3_DL + +#ifdef HWY_NATIVE_POPCNT +#undef HWY_NATIVE_POPCNT +#else +#define HWY_NATIVE_POPCNT +#endif + +namespace detail { + +template <typename T> +HWY_INLINE Vec256<T> PopulationCount(hwy::SizeTag<1> /* tag */, Vec256<T> v) { + return Vec256<T>{_mm256_popcnt_epi8(v.raw)}; +} +template <typename T> +HWY_INLINE Vec256<T> PopulationCount(hwy::SizeTag<2> /* tag */, Vec256<T> v) { + return Vec256<T>{_mm256_popcnt_epi16(v.raw)}; +} +template <typename T> +HWY_INLINE Vec256<T> PopulationCount(hwy::SizeTag<4> /* tag */, Vec256<T> v) { + return Vec256<T>{_mm256_popcnt_epi32(v.raw)}; +} +template <typename T> +HWY_INLINE Vec256<T> PopulationCount(hwy::SizeTag<8> /* tag */, Vec256<T> v) { + return Vec256<T>{_mm256_popcnt_epi64(v.raw)}; +} + +} // namespace detail + +template <typename T> +HWY_API Vec256<T> PopulationCount(Vec256<T> v) { + return detail::PopulationCount(hwy::SizeTag<sizeof(T)>(), v); +} + +#endif // HWY_TARGET == HWY_AVX3_DL + +// ================================================== SIGN + +// ------------------------------ CopySign + +template <typename T> +HWY_API Vec256<T> CopySign(const Vec256<T> magn, const Vec256<T> sign) { + static_assert(IsFloat<T>(), "Only makes sense for floating-point"); + + const Full256<T> d; + const auto msb = SignBit(d); + +#if HWY_TARGET <= HWY_AVX3 + const Rebind<MakeUnsigned<T>, decltype(d)> du; + // Truth table for msb, magn, sign | bitwise msb ? sign : mag + // 0 0 0 | 0 + // 0 0 1 | 0 + // 0 1 0 | 1 + // 0 1 1 | 1 + // 1 0 0 | 0 + // 1 0 1 | 1 + // 1 1 0 | 0 + // 1 1 1 | 1 + // The lane size does not matter because we are not using predication. + const __m256i out = _mm256_ternarylogic_epi32( + BitCast(du, msb).raw, BitCast(du, magn).raw, BitCast(du, sign).raw, 0xAC); + return BitCast(d, decltype(Zero(du)){out}); +#else + return Or(AndNot(msb, magn), And(msb, sign)); +#endif +} + +template <typename T> +HWY_API Vec256<T> CopySignToAbs(const Vec256<T> abs, const Vec256<T> sign) { +#if HWY_TARGET <= HWY_AVX3 + // AVX3 can also handle abs < 0, so no extra action needed. + return CopySign(abs, sign); +#else + return Or(abs, And(SignBit(Full256<T>()), sign)); +#endif +} + +// ================================================== MASK + +#if HWY_TARGET <= HWY_AVX3 + +// ------------------------------ IfThenElse + +// Returns mask ? b : a. + +namespace detail { + +// Templates for signed/unsigned integer of a particular size. +template <typename T> +HWY_INLINE Vec256<T> IfThenElse(hwy::SizeTag<1> /* tag */, Mask256<T> mask, + Vec256<T> yes, Vec256<T> no) { + return Vec256<T>{_mm256_mask_mov_epi8(no.raw, mask.raw, yes.raw)}; +} +template <typename T> +HWY_INLINE Vec256<T> IfThenElse(hwy::SizeTag<2> /* tag */, Mask256<T> mask, + Vec256<T> yes, Vec256<T> no) { + return Vec256<T>{_mm256_mask_mov_epi16(no.raw, mask.raw, yes.raw)}; +} +template <typename T> +HWY_INLINE Vec256<T> IfThenElse(hwy::SizeTag<4> /* tag */, Mask256<T> mask, + Vec256<T> yes, Vec256<T> no) { + return Vec256<T>{_mm256_mask_mov_epi32(no.raw, mask.raw, yes.raw)}; +} +template <typename T> +HWY_INLINE Vec256<T> IfThenElse(hwy::SizeTag<8> /* tag */, Mask256<T> mask, + Vec256<T> yes, Vec256<T> no) { + return Vec256<T>{_mm256_mask_mov_epi64(no.raw, mask.raw, yes.raw)}; +} + +} // namespace detail + +template <typename T> +HWY_API Vec256<T> IfThenElse(Mask256<T> mask, Vec256<T> yes, Vec256<T> no) { + return detail::IfThenElse(hwy::SizeTag<sizeof(T)>(), mask, yes, no); +} +HWY_API Vec256<float> IfThenElse(Mask256<float> mask, Vec256<float> yes, + Vec256<float> no) { + return Vec256<float>{_mm256_mask_mov_ps(no.raw, mask.raw, yes.raw)}; +} +HWY_API Vec256<double> IfThenElse(Mask256<double> mask, Vec256<double> yes, + Vec256<double> no) { + return Vec256<double>{_mm256_mask_mov_pd(no.raw, mask.raw, yes.raw)}; +} + +namespace detail { + +template <typename T> +HWY_INLINE Vec256<T> IfThenElseZero(hwy::SizeTag<1> /* tag */, Mask256<T> mask, + Vec256<T> yes) { + return Vec256<T>{_mm256_maskz_mov_epi8(mask.raw, yes.raw)}; +} +template <typename T> +HWY_INLINE Vec256<T> IfThenElseZero(hwy::SizeTag<2> /* tag */, Mask256<T> mask, + Vec256<T> yes) { + return Vec256<T>{_mm256_maskz_mov_epi16(mask.raw, yes.raw)}; +} +template <typename T> +HWY_INLINE Vec256<T> IfThenElseZero(hwy::SizeTag<4> /* tag */, Mask256<T> mask, + Vec256<T> yes) { + return Vec256<T>{_mm256_maskz_mov_epi32(mask.raw, yes.raw)}; +} +template <typename T> +HWY_INLINE Vec256<T> IfThenElseZero(hwy::SizeTag<8> /* tag */, Mask256<T> mask, + Vec256<T> yes) { + return Vec256<T>{_mm256_maskz_mov_epi64(mask.raw, yes.raw)}; +} + +} // namespace detail + +template <typename T> +HWY_API Vec256<T> IfThenElseZero(Mask256<T> mask, Vec256<T> yes) { + return detail::IfThenElseZero(hwy::SizeTag<sizeof(T)>(), mask, yes); +} +HWY_API Vec256<float> IfThenElseZero(Mask256<float> mask, Vec256<float> yes) { + return Vec256<float>{_mm256_maskz_mov_ps(mask.raw, yes.raw)}; +} +HWY_API Vec256<double> IfThenElseZero(Mask256<double> mask, + Vec256<double> yes) { + return Vec256<double>{_mm256_maskz_mov_pd(mask.raw, yes.raw)}; +} + +namespace detail { + +template <typename T> +HWY_INLINE Vec256<T> IfThenZeroElse(hwy::SizeTag<1> /* tag */, Mask256<T> mask, + Vec256<T> no) { + // xor_epi8/16 are missing, but we have sub, which is just as fast for u8/16. + return Vec256<T>{_mm256_mask_sub_epi8(no.raw, mask.raw, no.raw, no.raw)}; +} +template <typename T> +HWY_INLINE Vec256<T> IfThenZeroElse(hwy::SizeTag<2> /* tag */, Mask256<T> mask, + Vec256<T> no) { + return Vec256<T>{_mm256_mask_sub_epi16(no.raw, mask.raw, no.raw, no.raw)}; +} +template <typename T> +HWY_INLINE Vec256<T> IfThenZeroElse(hwy::SizeTag<4> /* tag */, Mask256<T> mask, + Vec256<T> no) { + return Vec256<T>{_mm256_mask_xor_epi32(no.raw, mask.raw, no.raw, no.raw)}; +} +template <typename T> +HWY_INLINE Vec256<T> IfThenZeroElse(hwy::SizeTag<8> /* tag */, Mask256<T> mask, + Vec256<T> no) { + return Vec256<T>{_mm256_mask_xor_epi64(no.raw, mask.raw, no.raw, no.raw)}; +} + +} // namespace detail + +template <typename T> +HWY_API Vec256<T> IfThenZeroElse(Mask256<T> mask, Vec256<T> no) { + return detail::IfThenZeroElse(hwy::SizeTag<sizeof(T)>(), mask, no); +} +HWY_API Vec256<float> IfThenZeroElse(Mask256<float> mask, Vec256<float> no) { + return Vec256<float>{_mm256_mask_xor_ps(no.raw, mask.raw, no.raw, no.raw)}; +} +HWY_API Vec256<double> IfThenZeroElse(Mask256<double> mask, Vec256<double> no) { + return Vec256<double>{_mm256_mask_xor_pd(no.raw, mask.raw, no.raw, no.raw)}; +} + +template <typename T, HWY_IF_FLOAT(T)> +HWY_API Vec256<T> ZeroIfNegative(const Vec256<T> v) { + // AVX3 MaskFromVec only looks at the MSB + return IfThenZeroElse(MaskFromVec(v), v); +} + +// ------------------------------ Mask logical + +namespace detail { + +template <typename T> +HWY_INLINE Mask256<T> And(hwy::SizeTag<1> /*tag*/, const Mask256<T> a, + const Mask256<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask256<T>{_kand_mask32(a.raw, b.raw)}; +#else + return Mask256<T>{static_cast<__mmask32>(a.raw & b.raw)}; +#endif +} +template <typename T> +HWY_INLINE Mask256<T> And(hwy::SizeTag<2> /*tag*/, const Mask256<T> a, + const Mask256<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask256<T>{_kand_mask16(a.raw, b.raw)}; +#else + return Mask256<T>{static_cast<__mmask16>(a.raw & b.raw)}; +#endif +} +template <typename T> +HWY_INLINE Mask256<T> And(hwy::SizeTag<4> /*tag*/, const Mask256<T> a, + const Mask256<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask256<T>{_kand_mask8(a.raw, b.raw)}; +#else + return Mask256<T>{static_cast<__mmask8>(a.raw & b.raw)}; +#endif +} +template <typename T> +HWY_INLINE Mask256<T> And(hwy::SizeTag<8> /*tag*/, const Mask256<T> a, + const Mask256<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask256<T>{_kand_mask8(a.raw, b.raw)}; +#else + return Mask256<T>{static_cast<__mmask8>(a.raw & b.raw)}; +#endif +} + +template <typename T> +HWY_INLINE Mask256<T> AndNot(hwy::SizeTag<1> /*tag*/, const Mask256<T> a, + const Mask256<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask256<T>{_kandn_mask32(a.raw, b.raw)}; +#else + return Mask256<T>{static_cast<__mmask32>(~a.raw & b.raw)}; +#endif +} +template <typename T> +HWY_INLINE Mask256<T> AndNot(hwy::SizeTag<2> /*tag*/, const Mask256<T> a, + const Mask256<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask256<T>{_kandn_mask16(a.raw, b.raw)}; +#else + return Mask256<T>{static_cast<__mmask16>(~a.raw & b.raw)}; +#endif +} +template <typename T> +HWY_INLINE Mask256<T> AndNot(hwy::SizeTag<4> /*tag*/, const Mask256<T> a, + const Mask256<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask256<T>{_kandn_mask8(a.raw, b.raw)}; +#else + return Mask256<T>{static_cast<__mmask8>(~a.raw & b.raw)}; +#endif +} +template <typename T> +HWY_INLINE Mask256<T> AndNot(hwy::SizeTag<8> /*tag*/, const Mask256<T> a, + const Mask256<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask256<T>{_kandn_mask8(a.raw, b.raw)}; +#else + return Mask256<T>{static_cast<__mmask8>(~a.raw & b.raw)}; +#endif +} + +template <typename T> +HWY_INLINE Mask256<T> Or(hwy::SizeTag<1> /*tag*/, const Mask256<T> a, + const Mask256<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask256<T>{_kor_mask32(a.raw, b.raw)}; +#else + return Mask256<T>{static_cast<__mmask32>(a.raw | b.raw)}; +#endif +} +template <typename T> +HWY_INLINE Mask256<T> Or(hwy::SizeTag<2> /*tag*/, const Mask256<T> a, + const Mask256<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask256<T>{_kor_mask16(a.raw, b.raw)}; +#else + return Mask256<T>{static_cast<__mmask16>(a.raw | b.raw)}; +#endif +} +template <typename T> +HWY_INLINE Mask256<T> Or(hwy::SizeTag<4> /*tag*/, const Mask256<T> a, + const Mask256<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask256<T>{_kor_mask8(a.raw, b.raw)}; +#else + return Mask256<T>{static_cast<__mmask8>(a.raw | b.raw)}; +#endif +} +template <typename T> +HWY_INLINE Mask256<T> Or(hwy::SizeTag<8> /*tag*/, const Mask256<T> a, + const Mask256<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask256<T>{_kor_mask8(a.raw, b.raw)}; +#else + return Mask256<T>{static_cast<__mmask8>(a.raw | b.raw)}; +#endif +} + +template <typename T> +HWY_INLINE Mask256<T> Xor(hwy::SizeTag<1> /*tag*/, const Mask256<T> a, + const Mask256<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask256<T>{_kxor_mask32(a.raw, b.raw)}; +#else + return Mask256<T>{static_cast<__mmask32>(a.raw ^ b.raw)}; +#endif +} +template <typename T> +HWY_INLINE Mask256<T> Xor(hwy::SizeTag<2> /*tag*/, const Mask256<T> a, + const Mask256<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask256<T>{_kxor_mask16(a.raw, b.raw)}; +#else + return Mask256<T>{static_cast<__mmask16>(a.raw ^ b.raw)}; +#endif +} +template <typename T> +HWY_INLINE Mask256<T> Xor(hwy::SizeTag<4> /*tag*/, const Mask256<T> a, + const Mask256<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask256<T>{_kxor_mask8(a.raw, b.raw)}; +#else + return Mask256<T>{static_cast<__mmask8>(a.raw ^ b.raw)}; +#endif +} +template <typename T> +HWY_INLINE Mask256<T> Xor(hwy::SizeTag<8> /*tag*/, const Mask256<T> a, + const Mask256<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask256<T>{_kxor_mask8(a.raw, b.raw)}; +#else + return Mask256<T>{static_cast<__mmask8>(a.raw ^ b.raw)}; +#endif +} + +} // namespace detail + +template <typename T> +HWY_API Mask256<T> And(const Mask256<T> a, Mask256<T> b) { + return detail::And(hwy::SizeTag<sizeof(T)>(), a, b); +} + +template <typename T> +HWY_API Mask256<T> AndNot(const Mask256<T> a, Mask256<T> b) { + return detail::AndNot(hwy::SizeTag<sizeof(T)>(), a, b); +} + +template <typename T> +HWY_API Mask256<T> Or(const Mask256<T> a, Mask256<T> b) { + return detail::Or(hwy::SizeTag<sizeof(T)>(), a, b); +} + +template <typename T> +HWY_API Mask256<T> Xor(const Mask256<T> a, Mask256<T> b) { + return detail::Xor(hwy::SizeTag<sizeof(T)>(), a, b); +} + +template <typename T> +HWY_API Mask256<T> Not(const Mask256<T> m) { + // Flip only the valid bits. + constexpr size_t N = 32 / sizeof(T); + return Xor(m, Mask256<T>::FromBits((1ull << N) - 1)); +} + +#else // AVX2 + +// ------------------------------ Mask + +// Mask and Vec are the same (true = FF..FF). +template <typename T> +HWY_API Mask256<T> MaskFromVec(const Vec256<T> v) { + return Mask256<T>{v.raw}; +} + +template <typename T> +HWY_API Vec256<T> VecFromMask(const Mask256<T> v) { + return Vec256<T>{v.raw}; +} + +template <typename T> +HWY_API Vec256<T> VecFromMask(Full256<T> /* tag */, const Mask256<T> v) { + return Vec256<T>{v.raw}; +} + +// ------------------------------ IfThenElse + +// mask ? yes : no +template <typename T> +HWY_API Vec256<T> IfThenElse(const Mask256<T> mask, const Vec256<T> yes, + const Vec256<T> no) { + return Vec256<T>{_mm256_blendv_epi8(no.raw, yes.raw, mask.raw)}; +} +HWY_API Vec256<float> IfThenElse(const Mask256<float> mask, + const Vec256<float> yes, + const Vec256<float> no) { + return Vec256<float>{_mm256_blendv_ps(no.raw, yes.raw, mask.raw)}; +} +HWY_API Vec256<double> IfThenElse(const Mask256<double> mask, + const Vec256<double> yes, + const Vec256<double> no) { + return Vec256<double>{_mm256_blendv_pd(no.raw, yes.raw, mask.raw)}; +} + +// mask ? yes : 0 +template <typename T> +HWY_API Vec256<T> IfThenElseZero(Mask256<T> mask, Vec256<T> yes) { + return yes & VecFromMask(Full256<T>(), mask); +} + +// mask ? 0 : no +template <typename T> +HWY_API Vec256<T> IfThenZeroElse(Mask256<T> mask, Vec256<T> no) { + return AndNot(VecFromMask(Full256<T>(), mask), no); +} + +template <typename T, HWY_IF_FLOAT(T)> +HWY_API Vec256<T> ZeroIfNegative(Vec256<T> v) { + const auto zero = Zero(Full256<T>()); + return IfThenElse(MaskFromVec(v), zero, v); +} + +// ------------------------------ Mask logical + +template <typename T> +HWY_API Mask256<T> Not(const Mask256<T> m) { + return MaskFromVec(Not(VecFromMask(Full256<T>(), m))); +} + +template <typename T> +HWY_API Mask256<T> And(const Mask256<T> a, Mask256<T> b) { + const Full256<T> d; + return MaskFromVec(And(VecFromMask(d, a), VecFromMask(d, b))); +} + +template <typename T> +HWY_API Mask256<T> AndNot(const Mask256<T> a, Mask256<T> b) { + const Full256<T> d; + return MaskFromVec(AndNot(VecFromMask(d, a), VecFromMask(d, b))); +} + +template <typename T> +HWY_API Mask256<T> Or(const Mask256<T> a, Mask256<T> b) { + const Full256<T> d; + return MaskFromVec(Or(VecFromMask(d, a), VecFromMask(d, b))); +} + +template <typename T> +HWY_API Mask256<T> Xor(const Mask256<T> a, Mask256<T> b) { + const Full256<T> d; + return MaskFromVec(Xor(VecFromMask(d, a), VecFromMask(d, b))); +} + +#endif // HWY_TARGET <= HWY_AVX3 + +// ================================================== COMPARE + +#if HWY_TARGET <= HWY_AVX3 + +// Comparisons set a mask bit to 1 if the condition is true, else 0. + +template <typename TFrom, typename TTo> +HWY_API Mask256<TTo> RebindMask(Full256<TTo> /*tag*/, Mask256<TFrom> m) { + static_assert(sizeof(TFrom) == sizeof(TTo), "Must have same size"); + return Mask256<TTo>{m.raw}; +} + +namespace detail { + +template <typename T> +HWY_INLINE Mask256<T> TestBit(hwy::SizeTag<1> /*tag*/, const Vec256<T> v, + const Vec256<T> bit) { + return Mask256<T>{_mm256_test_epi8_mask(v.raw, bit.raw)}; +} +template <typename T> +HWY_INLINE Mask256<T> TestBit(hwy::SizeTag<2> /*tag*/, const Vec256<T> v, + const Vec256<T> bit) { + return Mask256<T>{_mm256_test_epi16_mask(v.raw, bit.raw)}; +} +template <typename T> +HWY_INLINE Mask256<T> TestBit(hwy::SizeTag<4> /*tag*/, const Vec256<T> v, + const Vec256<T> bit) { + return Mask256<T>{_mm256_test_epi32_mask(v.raw, bit.raw)}; +} +template <typename T> +HWY_INLINE Mask256<T> TestBit(hwy::SizeTag<8> /*tag*/, const Vec256<T> v, + const Vec256<T> bit) { + return Mask256<T>{_mm256_test_epi64_mask(v.raw, bit.raw)}; +} + +} // namespace detail + +template <typename T> +HWY_API Mask256<T> TestBit(const Vec256<T> v, const Vec256<T> bit) { + static_assert(!hwy::IsFloat<T>(), "Only integer vectors supported"); + return detail::TestBit(hwy::SizeTag<sizeof(T)>(), v, bit); +} + +// ------------------------------ Equality + +template <typename T, HWY_IF_LANE_SIZE(T, 1)> +HWY_API Mask256<T> operator==(const Vec256<T> a, const Vec256<T> b) { + return Mask256<T>{_mm256_cmpeq_epi8_mask(a.raw, b.raw)}; +} +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Mask256<T> operator==(const Vec256<T> a, const Vec256<T> b) { + return Mask256<T>{_mm256_cmpeq_epi16_mask(a.raw, b.raw)}; +} +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Mask256<T> operator==(const Vec256<T> a, const Vec256<T> b) { + return Mask256<T>{_mm256_cmpeq_epi32_mask(a.raw, b.raw)}; +} +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Mask256<T> operator==(const Vec256<T> a, const Vec256<T> b) { + return Mask256<T>{_mm256_cmpeq_epi64_mask(a.raw, b.raw)}; +} + +HWY_API Mask256<float> operator==(Vec256<float> a, Vec256<float> b) { + return Mask256<float>{_mm256_cmp_ps_mask(a.raw, b.raw, _CMP_EQ_OQ)}; +} + +HWY_API Mask256<double> operator==(Vec256<double> a, Vec256<double> b) { + return Mask256<double>{_mm256_cmp_pd_mask(a.raw, b.raw, _CMP_EQ_OQ)}; +} + +// ------------------------------ Inequality + +template <typename T, HWY_IF_LANE_SIZE(T, 1)> +HWY_API Mask256<T> operator!=(const Vec256<T> a, const Vec256<T> b) { + return Mask256<T>{_mm256_cmpneq_epi8_mask(a.raw, b.raw)}; +} +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Mask256<T> operator!=(const Vec256<T> a, const Vec256<T> b) { + return Mask256<T>{_mm256_cmpneq_epi16_mask(a.raw, b.raw)}; +} +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Mask256<T> operator!=(const Vec256<T> a, const Vec256<T> b) { + return Mask256<T>{_mm256_cmpneq_epi32_mask(a.raw, b.raw)}; +} +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Mask256<T> operator!=(const Vec256<T> a, const Vec256<T> b) { + return Mask256<T>{_mm256_cmpneq_epi64_mask(a.raw, b.raw)}; +} + +HWY_API Mask256<float> operator!=(Vec256<float> a, Vec256<float> b) { + return Mask256<float>{_mm256_cmp_ps_mask(a.raw, b.raw, _CMP_NEQ_OQ)}; +} + +HWY_API Mask256<double> operator!=(Vec256<double> a, Vec256<double> b) { + return Mask256<double>{_mm256_cmp_pd_mask(a.raw, b.raw, _CMP_NEQ_OQ)}; +} + +// ------------------------------ Strict inequality + +HWY_API Mask256<int8_t> operator>(Vec256<int8_t> a, Vec256<int8_t> b) { + return Mask256<int8_t>{_mm256_cmpgt_epi8_mask(a.raw, b.raw)}; +} +HWY_API Mask256<int16_t> operator>(Vec256<int16_t> a, Vec256<int16_t> b) { + return Mask256<int16_t>{_mm256_cmpgt_epi16_mask(a.raw, b.raw)}; +} +HWY_API Mask256<int32_t> operator>(Vec256<int32_t> a, Vec256<int32_t> b) { + return Mask256<int32_t>{_mm256_cmpgt_epi32_mask(a.raw, b.raw)}; +} +HWY_API Mask256<int64_t> operator>(Vec256<int64_t> a, Vec256<int64_t> b) { + return Mask256<int64_t>{_mm256_cmpgt_epi64_mask(a.raw, b.raw)}; +} + +HWY_API Mask256<uint8_t> operator>(Vec256<uint8_t> a, Vec256<uint8_t> b) { + return Mask256<uint8_t>{_mm256_cmpgt_epu8_mask(a.raw, b.raw)}; +} +HWY_API Mask256<uint16_t> operator>(const Vec256<uint16_t> a, + const Vec256<uint16_t> b) { + return Mask256<uint16_t>{_mm256_cmpgt_epu16_mask(a.raw, b.raw)}; +} +HWY_API Mask256<uint32_t> operator>(const Vec256<uint32_t> a, + const Vec256<uint32_t> b) { + return Mask256<uint32_t>{_mm256_cmpgt_epu32_mask(a.raw, b.raw)}; +} +HWY_API Mask256<uint64_t> operator>(const Vec256<uint64_t> a, + const Vec256<uint64_t> b) { + return Mask256<uint64_t>{_mm256_cmpgt_epu64_mask(a.raw, b.raw)}; +} + +HWY_API Mask256<float> operator>(Vec256<float> a, Vec256<float> b) { + return Mask256<float>{_mm256_cmp_ps_mask(a.raw, b.raw, _CMP_GT_OQ)}; +} +HWY_API Mask256<double> operator>(Vec256<double> a, Vec256<double> b) { + return Mask256<double>{_mm256_cmp_pd_mask(a.raw, b.raw, _CMP_GT_OQ)}; +} + +// ------------------------------ Weak inequality + +HWY_API Mask256<float> operator>=(Vec256<float> a, Vec256<float> b) { + return Mask256<float>{_mm256_cmp_ps_mask(a.raw, b.raw, _CMP_GE_OQ)}; +} +HWY_API Mask256<double> operator>=(Vec256<double> a, Vec256<double> b) { + return Mask256<double>{_mm256_cmp_pd_mask(a.raw, b.raw, _CMP_GE_OQ)}; +} + +// ------------------------------ Mask + +namespace detail { + +template <typename T> +HWY_INLINE Mask256<T> MaskFromVec(hwy::SizeTag<1> /*tag*/, const Vec256<T> v) { + return Mask256<T>{_mm256_movepi8_mask(v.raw)}; +} +template <typename T> +HWY_INLINE Mask256<T> MaskFromVec(hwy::SizeTag<2> /*tag*/, const Vec256<T> v) { + return Mask256<T>{_mm256_movepi16_mask(v.raw)}; +} +template <typename T> +HWY_INLINE Mask256<T> MaskFromVec(hwy::SizeTag<4> /*tag*/, const Vec256<T> v) { + return Mask256<T>{_mm256_movepi32_mask(v.raw)}; +} +template <typename T> +HWY_INLINE Mask256<T> MaskFromVec(hwy::SizeTag<8> /*tag*/, const Vec256<T> v) { + return Mask256<T>{_mm256_movepi64_mask(v.raw)}; +} + +} // namespace detail + +template <typename T> +HWY_API Mask256<T> MaskFromVec(const Vec256<T> v) { + return detail::MaskFromVec(hwy::SizeTag<sizeof(T)>(), v); +} +// There do not seem to be native floating-point versions of these instructions. +HWY_API Mask256<float> MaskFromVec(const Vec256<float> v) { + return Mask256<float>{MaskFromVec(BitCast(Full256<int32_t>(), v)).raw}; +} +HWY_API Mask256<double> MaskFromVec(const Vec256<double> v) { + return Mask256<double>{MaskFromVec(BitCast(Full256<int64_t>(), v)).raw}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 1)> +HWY_API Vec256<T> VecFromMask(const Mask256<T> v) { + return Vec256<T>{_mm256_movm_epi8(v.raw)}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Vec256<T> VecFromMask(const Mask256<T> v) { + return Vec256<T>{_mm256_movm_epi16(v.raw)}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec256<T> VecFromMask(const Mask256<T> v) { + return Vec256<T>{_mm256_movm_epi32(v.raw)}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec256<T> VecFromMask(const Mask256<T> v) { + return Vec256<T>{_mm256_movm_epi64(v.raw)}; +} + +HWY_API Vec256<float> VecFromMask(const Mask256<float> v) { + return Vec256<float>{_mm256_castsi256_ps(_mm256_movm_epi32(v.raw))}; +} + +HWY_API Vec256<double> VecFromMask(const Mask256<double> v) { + return Vec256<double>{_mm256_castsi256_pd(_mm256_movm_epi64(v.raw))}; +} + +template <typename T> +HWY_API Vec256<T> VecFromMask(Full256<T> /* tag */, const Mask256<T> v) { + return VecFromMask(v); +} + +#else // AVX2 + +// Comparisons fill a lane with 1-bits if the condition is true, else 0. + +template <typename TFrom, typename TTo> +HWY_API Mask256<TTo> RebindMask(Full256<TTo> d_to, Mask256<TFrom> m) { + static_assert(sizeof(TFrom) == sizeof(TTo), "Must have same size"); + return MaskFromVec(BitCast(d_to, VecFromMask(Full256<TFrom>(), m))); +} + +template <typename T> +HWY_API Mask256<T> TestBit(const Vec256<T> v, const Vec256<T> bit) { + static_assert(!hwy::IsFloat<T>(), "Only integer vectors supported"); + return (v & bit) == bit; +} + +// ------------------------------ Equality + +template <typename T, HWY_IF_LANE_SIZE(T, 1)> +HWY_API Mask256<T> operator==(const Vec256<T> a, const Vec256<T> b) { + return Mask256<T>{_mm256_cmpeq_epi8(a.raw, b.raw)}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Mask256<T> operator==(const Vec256<T> a, const Vec256<T> b) { + return Mask256<T>{_mm256_cmpeq_epi16(a.raw, b.raw)}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Mask256<T> operator==(const Vec256<T> a, const Vec256<T> b) { + return Mask256<T>{_mm256_cmpeq_epi32(a.raw, b.raw)}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Mask256<T> operator==(const Vec256<T> a, const Vec256<T> b) { + return Mask256<T>{_mm256_cmpeq_epi64(a.raw, b.raw)}; +} + +HWY_API Mask256<float> operator==(const Vec256<float> a, + const Vec256<float> b) { + return Mask256<float>{_mm256_cmp_ps(a.raw, b.raw, _CMP_EQ_OQ)}; +} + +HWY_API Mask256<double> operator==(const Vec256<double> a, + const Vec256<double> b) { + return Mask256<double>{_mm256_cmp_pd(a.raw, b.raw, _CMP_EQ_OQ)}; +} + +// ------------------------------ Inequality + +template <typename T, HWY_IF_NOT_FLOAT(T)> +HWY_API Mask256<T> operator!=(const Vec256<T> a, const Vec256<T> b) { + return Not(a == b); +} + +HWY_API Mask256<float> operator!=(const Vec256<float> a, + const Vec256<float> b) { + return Mask256<float>{_mm256_cmp_ps(a.raw, b.raw, _CMP_NEQ_OQ)}; +} +HWY_API Mask256<double> operator!=(const Vec256<double> a, + const Vec256<double> b) { + return Mask256<double>{_mm256_cmp_pd(a.raw, b.raw, _CMP_NEQ_OQ)}; +} + +// ------------------------------ Strict inequality + +// Pre-9.3 GCC immintrin.h uses char, which may be unsigned, causing cmpgt_epi8 +// to perform an unsigned comparison instead of the intended signed. Workaround +// is to cast to an explicitly signed type. See https://godbolt.org/z/PL7Ujy +#if HWY_COMPILER_GCC != 0 && HWY_COMPILER_GCC < 930 +#define HWY_AVX2_GCC_CMPGT8_WORKAROUND 1 +#else +#define HWY_AVX2_GCC_CMPGT8_WORKAROUND 0 +#endif + +HWY_API Mask256<int8_t> operator>(Vec256<int8_t> a, Vec256<int8_t> b) { +#if HWY_AVX2_GCC_CMPGT8_WORKAROUND + using i8x32 = signed char __attribute__((__vector_size__(32))); + return Mask256<int8_t>{static_cast<__m256i>(reinterpret_cast<i8x32>(a.raw) > + reinterpret_cast<i8x32>(b.raw))}; +#else + return Mask256<int8_t>{_mm256_cmpgt_epi8(a.raw, b.raw)}; +#endif +} +HWY_API Mask256<int16_t> operator>(const Vec256<int16_t> a, + const Vec256<int16_t> b) { + return Mask256<int16_t>{_mm256_cmpgt_epi16(a.raw, b.raw)}; +} +HWY_API Mask256<int32_t> operator>(const Vec256<int32_t> a, + const Vec256<int32_t> b) { + return Mask256<int32_t>{_mm256_cmpgt_epi32(a.raw, b.raw)}; +} +HWY_API Mask256<int64_t> operator>(const Vec256<int64_t> a, + const Vec256<int64_t> b) { + return Mask256<int64_t>{_mm256_cmpgt_epi64(a.raw, b.raw)}; +} + +template <typename T, HWY_IF_UNSIGNED(T)> +HWY_API Mask256<T> operator>(const Vec256<T> a, const Vec256<T> b) { + const Full256<T> du; + const RebindToSigned<decltype(du)> di; + const Vec256<T> msb = Set(du, (LimitsMax<T>() >> 1) + 1); + return RebindMask(du, BitCast(di, Xor(a, msb)) > BitCast(di, Xor(b, msb))); +} + +HWY_API Mask256<float> operator>(const Vec256<float> a, const Vec256<float> b) { + return Mask256<float>{_mm256_cmp_ps(a.raw, b.raw, _CMP_GT_OQ)}; +} +HWY_API Mask256<double> operator>(Vec256<double> a, Vec256<double> b) { + return Mask256<double>{_mm256_cmp_pd(a.raw, b.raw, _CMP_GT_OQ)}; +} + +// ------------------------------ Weak inequality + +HWY_API Mask256<float> operator>=(const Vec256<float> a, + const Vec256<float> b) { + return Mask256<float>{_mm256_cmp_ps(a.raw, b.raw, _CMP_GE_OQ)}; +} +HWY_API Mask256<double> operator>=(const Vec256<double> a, + const Vec256<double> b) { + return Mask256<double>{_mm256_cmp_pd(a.raw, b.raw, _CMP_GE_OQ)}; +} + +#endif // HWY_TARGET <= HWY_AVX3 + +// ------------------------------ Reversed comparisons + +template <typename T> +HWY_API Mask256<T> operator<(const Vec256<T> a, const Vec256<T> b) { + return b > a; +} + +template <typename T> +HWY_API Mask256<T> operator<=(const Vec256<T> a, const Vec256<T> b) { + return b >= a; +} + +// ------------------------------ Min (Gt, IfThenElse) + +// Unsigned +HWY_API Vec256<uint8_t> Min(const Vec256<uint8_t> a, const Vec256<uint8_t> b) { + return Vec256<uint8_t>{_mm256_min_epu8(a.raw, b.raw)}; +} +HWY_API Vec256<uint16_t> Min(const Vec256<uint16_t> a, + const Vec256<uint16_t> b) { + return Vec256<uint16_t>{_mm256_min_epu16(a.raw, b.raw)}; +} +HWY_API Vec256<uint32_t> Min(const Vec256<uint32_t> a, + const Vec256<uint32_t> b) { + return Vec256<uint32_t>{_mm256_min_epu32(a.raw, b.raw)}; +} +HWY_API Vec256<uint64_t> Min(const Vec256<uint64_t> a, + const Vec256<uint64_t> b) { +#if HWY_TARGET <= HWY_AVX3 + return Vec256<uint64_t>{_mm256_min_epu64(a.raw, b.raw)}; +#else + const Full256<uint64_t> du; + const Full256<int64_t> di; + const auto msb = Set(du, 1ull << 63); + const auto gt = RebindMask(du, BitCast(di, a ^ msb) > BitCast(di, b ^ msb)); + return IfThenElse(gt, b, a); +#endif +} + +// Signed +HWY_API Vec256<int8_t> Min(const Vec256<int8_t> a, const Vec256<int8_t> b) { + return Vec256<int8_t>{_mm256_min_epi8(a.raw, b.raw)}; +} +HWY_API Vec256<int16_t> Min(const Vec256<int16_t> a, const Vec256<int16_t> b) { + return Vec256<int16_t>{_mm256_min_epi16(a.raw, b.raw)}; +} +HWY_API Vec256<int32_t> Min(const Vec256<int32_t> a, const Vec256<int32_t> b) { + return Vec256<int32_t>{_mm256_min_epi32(a.raw, b.raw)}; +} +HWY_API Vec256<int64_t> Min(const Vec256<int64_t> a, const Vec256<int64_t> b) { +#if HWY_TARGET <= HWY_AVX3 + return Vec256<int64_t>{_mm256_min_epi64(a.raw, b.raw)}; +#else + return IfThenElse(a < b, a, b); +#endif +} + +// Float +HWY_API Vec256<float> Min(const Vec256<float> a, const Vec256<float> b) { + return Vec256<float>{_mm256_min_ps(a.raw, b.raw)}; +} +HWY_API Vec256<double> Min(const Vec256<double> a, const Vec256<double> b) { + return Vec256<double>{_mm256_min_pd(a.raw, b.raw)}; +} + +// ------------------------------ Max (Gt, IfThenElse) + +// Unsigned +HWY_API Vec256<uint8_t> Max(const Vec256<uint8_t> a, const Vec256<uint8_t> b) { + return Vec256<uint8_t>{_mm256_max_epu8(a.raw, b.raw)}; +} +HWY_API Vec256<uint16_t> Max(const Vec256<uint16_t> a, + const Vec256<uint16_t> b) { + return Vec256<uint16_t>{_mm256_max_epu16(a.raw, b.raw)}; +} +HWY_API Vec256<uint32_t> Max(const Vec256<uint32_t> a, + const Vec256<uint32_t> b) { + return Vec256<uint32_t>{_mm256_max_epu32(a.raw, b.raw)}; +} +HWY_API Vec256<uint64_t> Max(const Vec256<uint64_t> a, + const Vec256<uint64_t> b) { +#if HWY_TARGET <= HWY_AVX3 + return Vec256<uint64_t>{_mm256_max_epu64(a.raw, b.raw)}; +#else + const Full256<uint64_t> du; + const Full256<int64_t> di; + const auto msb = Set(du, 1ull << 63); + const auto gt = RebindMask(du, BitCast(di, a ^ msb) > BitCast(di, b ^ msb)); + return IfThenElse(gt, a, b); +#endif +} + +// Signed +HWY_API Vec256<int8_t> Max(const Vec256<int8_t> a, const Vec256<int8_t> b) { + return Vec256<int8_t>{_mm256_max_epi8(a.raw, b.raw)}; +} +HWY_API Vec256<int16_t> Max(const Vec256<int16_t> a, const Vec256<int16_t> b) { + return Vec256<int16_t>{_mm256_max_epi16(a.raw, b.raw)}; +} +HWY_API Vec256<int32_t> Max(const Vec256<int32_t> a, const Vec256<int32_t> b) { + return Vec256<int32_t>{_mm256_max_epi32(a.raw, b.raw)}; +} +HWY_API Vec256<int64_t> Max(const Vec256<int64_t> a, const Vec256<int64_t> b) { +#if HWY_TARGET <= HWY_AVX3 + return Vec256<int64_t>{_mm256_max_epi64(a.raw, b.raw)}; +#else + return IfThenElse(a < b, b, a); +#endif +} + +// Float +HWY_API Vec256<float> Max(const Vec256<float> a, const Vec256<float> b) { + return Vec256<float>{_mm256_max_ps(a.raw, b.raw)}; +} +HWY_API Vec256<double> Max(const Vec256<double> a, const Vec256<double> b) { + return Vec256<double>{_mm256_max_pd(a.raw, b.raw)}; +} + +// ------------------------------ FirstN (Iota, Lt) + +template <typename T> +HWY_API Mask256<T> FirstN(const Full256<T> d, size_t n) { +#if HWY_TARGET <= HWY_AVX3 + (void)d; + constexpr size_t N = 32 / sizeof(T); +#if HWY_ARCH_X86_64 + const uint64_t all = (1ull << N) - 1; + // BZHI only looks at the lower 8 bits of n! + return Mask256<T>::FromBits((n > 255) ? all : _bzhi_u64(all, n)); +#else + const uint32_t all = static_cast<uint32_t>((1ull << N) - 1); + // BZHI only looks at the lower 8 bits of n! + return Mask256<T>::FromBits( + (n > 255) ? all : _bzhi_u32(all, static_cast<uint32_t>(n))); +#endif // HWY_ARCH_X86_64 +#else + const RebindToSigned<decltype(d)> di; // Signed comparisons are cheaper. + return RebindMask(d, Iota(di, 0) < Set(di, static_cast<MakeSigned<T>>(n))); +#endif +} + +// ================================================== ARITHMETIC + +// ------------------------------ Addition + +// Unsigned +HWY_API Vec256<uint8_t> operator+(const Vec256<uint8_t> a, + const Vec256<uint8_t> b) { + return Vec256<uint8_t>{_mm256_add_epi8(a.raw, b.raw)}; +} +HWY_API Vec256<uint16_t> operator+(const Vec256<uint16_t> a, + const Vec256<uint16_t> b) { + return Vec256<uint16_t>{_mm256_add_epi16(a.raw, b.raw)}; +} +HWY_API Vec256<uint32_t> operator+(const Vec256<uint32_t> a, + const Vec256<uint32_t> b) { + return Vec256<uint32_t>{_mm256_add_epi32(a.raw, b.raw)}; +} +HWY_API Vec256<uint64_t> operator+(const Vec256<uint64_t> a, + const Vec256<uint64_t> b) { + return Vec256<uint64_t>{_mm256_add_epi64(a.raw, b.raw)}; +} + +// Signed +HWY_API Vec256<int8_t> operator+(const Vec256<int8_t> a, + const Vec256<int8_t> b) { + return Vec256<int8_t>{_mm256_add_epi8(a.raw, b.raw)}; +} +HWY_API Vec256<int16_t> operator+(const Vec256<int16_t> a, + const Vec256<int16_t> b) { + return Vec256<int16_t>{_mm256_add_epi16(a.raw, b.raw)}; +} +HWY_API Vec256<int32_t> operator+(const Vec256<int32_t> a, + const Vec256<int32_t> b) { + return Vec256<int32_t>{_mm256_add_epi32(a.raw, b.raw)}; +} +HWY_API Vec256<int64_t> operator+(const Vec256<int64_t> a, + const Vec256<int64_t> b) { + return Vec256<int64_t>{_mm256_add_epi64(a.raw, b.raw)}; +} + +// Float +HWY_API Vec256<float> operator+(const Vec256<float> a, const Vec256<float> b) { + return Vec256<float>{_mm256_add_ps(a.raw, b.raw)}; +} +HWY_API Vec256<double> operator+(const Vec256<double> a, + const Vec256<double> b) { + return Vec256<double>{_mm256_add_pd(a.raw, b.raw)}; +} + +// ------------------------------ Subtraction + +// Unsigned +HWY_API Vec256<uint8_t> operator-(const Vec256<uint8_t> a, + const Vec256<uint8_t> b) { + return Vec256<uint8_t>{_mm256_sub_epi8(a.raw, b.raw)}; +} +HWY_API Vec256<uint16_t> operator-(const Vec256<uint16_t> a, + const Vec256<uint16_t> b) { + return Vec256<uint16_t>{_mm256_sub_epi16(a.raw, b.raw)}; +} +HWY_API Vec256<uint32_t> operator-(const Vec256<uint32_t> a, + const Vec256<uint32_t> b) { + return Vec256<uint32_t>{_mm256_sub_epi32(a.raw, b.raw)}; +} +HWY_API Vec256<uint64_t> operator-(const Vec256<uint64_t> a, + const Vec256<uint64_t> b) { + return Vec256<uint64_t>{_mm256_sub_epi64(a.raw, b.raw)}; +} + +// Signed +HWY_API Vec256<int8_t> operator-(const Vec256<int8_t> a, + const Vec256<int8_t> b) { + return Vec256<int8_t>{_mm256_sub_epi8(a.raw, b.raw)}; +} +HWY_API Vec256<int16_t> operator-(const Vec256<int16_t> a, + const Vec256<int16_t> b) { + return Vec256<int16_t>{_mm256_sub_epi16(a.raw, b.raw)}; +} +HWY_API Vec256<int32_t> operator-(const Vec256<int32_t> a, + const Vec256<int32_t> b) { + return Vec256<int32_t>{_mm256_sub_epi32(a.raw, b.raw)}; +} +HWY_API Vec256<int64_t> operator-(const Vec256<int64_t> a, + const Vec256<int64_t> b) { + return Vec256<int64_t>{_mm256_sub_epi64(a.raw, b.raw)}; +} + +// Float +HWY_API Vec256<float> operator-(const Vec256<float> a, const Vec256<float> b) { + return Vec256<float>{_mm256_sub_ps(a.raw, b.raw)}; +} +HWY_API Vec256<double> operator-(const Vec256<double> a, + const Vec256<double> b) { + return Vec256<double>{_mm256_sub_pd(a.raw, b.raw)}; +} + +// ------------------------------ Saturating addition + +// Returns a + b clamped to the destination range. + +// Unsigned +HWY_API Vec256<uint8_t> SaturatedAdd(const Vec256<uint8_t> a, + const Vec256<uint8_t> b) { + return Vec256<uint8_t>{_mm256_adds_epu8(a.raw, b.raw)}; +} +HWY_API Vec256<uint16_t> SaturatedAdd(const Vec256<uint16_t> a, + const Vec256<uint16_t> b) { + return Vec256<uint16_t>{_mm256_adds_epu16(a.raw, b.raw)}; +} + +// Signed +HWY_API Vec256<int8_t> SaturatedAdd(const Vec256<int8_t> a, + const Vec256<int8_t> b) { + return Vec256<int8_t>{_mm256_adds_epi8(a.raw, b.raw)}; +} +HWY_API Vec256<int16_t> SaturatedAdd(const Vec256<int16_t> a, + const Vec256<int16_t> b) { + return Vec256<int16_t>{_mm256_adds_epi16(a.raw, b.raw)}; +} + +// ------------------------------ Saturating subtraction + +// Returns a - b clamped to the destination range. + +// Unsigned +HWY_API Vec256<uint8_t> SaturatedSub(const Vec256<uint8_t> a, + const Vec256<uint8_t> b) { + return Vec256<uint8_t>{_mm256_subs_epu8(a.raw, b.raw)}; +} +HWY_API Vec256<uint16_t> SaturatedSub(const Vec256<uint16_t> a, + const Vec256<uint16_t> b) { + return Vec256<uint16_t>{_mm256_subs_epu16(a.raw, b.raw)}; +} + +// Signed +HWY_API Vec256<int8_t> SaturatedSub(const Vec256<int8_t> a, + const Vec256<int8_t> b) { + return Vec256<int8_t>{_mm256_subs_epi8(a.raw, b.raw)}; +} +HWY_API Vec256<int16_t> SaturatedSub(const Vec256<int16_t> a, + const Vec256<int16_t> b) { + return Vec256<int16_t>{_mm256_subs_epi16(a.raw, b.raw)}; +} + +// ------------------------------ Average + +// Returns (a + b + 1) / 2 + +// Unsigned +HWY_API Vec256<uint8_t> AverageRound(const Vec256<uint8_t> a, + const Vec256<uint8_t> b) { + return Vec256<uint8_t>{_mm256_avg_epu8(a.raw, b.raw)}; +} +HWY_API Vec256<uint16_t> AverageRound(const Vec256<uint16_t> a, + const Vec256<uint16_t> b) { + return Vec256<uint16_t>{_mm256_avg_epu16(a.raw, b.raw)}; +} + +// ------------------------------ Abs (Sub) + +// Returns absolute value, except that LimitsMin() maps to LimitsMax() + 1. +HWY_API Vec256<int8_t> Abs(const Vec256<int8_t> v) { +#if HWY_COMPILER_MSVC + // Workaround for incorrect codegen? (wrong result) + const auto zero = Zero(Full256<int8_t>()); + return Vec256<int8_t>{_mm256_max_epi8(v.raw, (zero - v).raw)}; +#else + return Vec256<int8_t>{_mm256_abs_epi8(v.raw)}; +#endif +} +HWY_API Vec256<int16_t> Abs(const Vec256<int16_t> v) { + return Vec256<int16_t>{_mm256_abs_epi16(v.raw)}; +} +HWY_API Vec256<int32_t> Abs(const Vec256<int32_t> v) { + return Vec256<int32_t>{_mm256_abs_epi32(v.raw)}; +} +// i64 is implemented after BroadcastSignBit. + +HWY_API Vec256<float> Abs(const Vec256<float> v) { + const Vec256<int32_t> mask{_mm256_set1_epi32(0x7FFFFFFF)}; + return v & BitCast(Full256<float>(), mask); +} +HWY_API Vec256<double> Abs(const Vec256<double> v) { + const Vec256<int64_t> mask{_mm256_set1_epi64x(0x7FFFFFFFFFFFFFFFLL)}; + return v & BitCast(Full256<double>(), mask); +} + +// ------------------------------ Integer multiplication + +// Unsigned +HWY_API Vec256<uint16_t> operator*(const Vec256<uint16_t> a, + const Vec256<uint16_t> b) { + return Vec256<uint16_t>{_mm256_mullo_epi16(a.raw, b.raw)}; +} +HWY_API Vec256<uint32_t> operator*(const Vec256<uint32_t> a, + const Vec256<uint32_t> b) { + return Vec256<uint32_t>{_mm256_mullo_epi32(a.raw, b.raw)}; +} + +// Signed +HWY_API Vec256<int16_t> operator*(const Vec256<int16_t> a, + const Vec256<int16_t> b) { + return Vec256<int16_t>{_mm256_mullo_epi16(a.raw, b.raw)}; +} +HWY_API Vec256<int32_t> operator*(const Vec256<int32_t> a, + const Vec256<int32_t> b) { + return Vec256<int32_t>{_mm256_mullo_epi32(a.raw, b.raw)}; +} + +// Returns the upper 16 bits of a * b in each lane. +HWY_API Vec256<uint16_t> MulHigh(const Vec256<uint16_t> a, + const Vec256<uint16_t> b) { + return Vec256<uint16_t>{_mm256_mulhi_epu16(a.raw, b.raw)}; +} +HWY_API Vec256<int16_t> MulHigh(const Vec256<int16_t> a, + const Vec256<int16_t> b) { + return Vec256<int16_t>{_mm256_mulhi_epi16(a.raw, b.raw)}; +} + +// Multiplies even lanes (0, 2 ..) and places the double-wide result into +// even and the upper half into its odd neighbor lane. +HWY_API Vec256<int64_t> MulEven(const Vec256<int32_t> a, + const Vec256<int32_t> b) { + return Vec256<int64_t>{_mm256_mul_epi32(a.raw, b.raw)}; +} +HWY_API Vec256<uint64_t> MulEven(const Vec256<uint32_t> a, + const Vec256<uint32_t> b) { + return Vec256<uint64_t>{_mm256_mul_epu32(a.raw, b.raw)}; +} + +// ------------------------------ ShiftLeft + +template <int kBits> +HWY_API Vec256<uint16_t> ShiftLeft(const Vec256<uint16_t> v) { + return Vec256<uint16_t>{_mm256_slli_epi16(v.raw, kBits)}; +} + +template <int kBits> +HWY_API Vec256<uint32_t> ShiftLeft(const Vec256<uint32_t> v) { + return Vec256<uint32_t>{_mm256_slli_epi32(v.raw, kBits)}; +} + +template <int kBits> +HWY_API Vec256<uint64_t> ShiftLeft(const Vec256<uint64_t> v) { + return Vec256<uint64_t>{_mm256_slli_epi64(v.raw, kBits)}; +} + +template <int kBits> +HWY_API Vec256<int16_t> ShiftLeft(const Vec256<int16_t> v) { + return Vec256<int16_t>{_mm256_slli_epi16(v.raw, kBits)}; +} + +template <int kBits> +HWY_API Vec256<int32_t> ShiftLeft(const Vec256<int32_t> v) { + return Vec256<int32_t>{_mm256_slli_epi32(v.raw, kBits)}; +} + +template <int kBits> +HWY_API Vec256<int64_t> ShiftLeft(const Vec256<int64_t> v) { + return Vec256<int64_t>{_mm256_slli_epi64(v.raw, kBits)}; +} + +template <int kBits, typename T, HWY_IF_LANE_SIZE(T, 1)> +HWY_API Vec256<T> ShiftLeft(const Vec256<T> v) { + const Full256<T> d8; + const RepartitionToWide<decltype(d8)> d16; + const auto shifted = BitCast(d8, ShiftLeft<kBits>(BitCast(d16, v))); + return kBits == 1 + ? (v + v) + : (shifted & Set(d8, static_cast<T>((0xFF << kBits) & 0xFF))); +} + +// ------------------------------ ShiftRight + +template <int kBits> +HWY_API Vec256<uint16_t> ShiftRight(const Vec256<uint16_t> v) { + return Vec256<uint16_t>{_mm256_srli_epi16(v.raw, kBits)}; +} + +template <int kBits> +HWY_API Vec256<uint32_t> ShiftRight(const Vec256<uint32_t> v) { + return Vec256<uint32_t>{_mm256_srli_epi32(v.raw, kBits)}; +} + +template <int kBits> +HWY_API Vec256<uint64_t> ShiftRight(const Vec256<uint64_t> v) { + return Vec256<uint64_t>{_mm256_srli_epi64(v.raw, kBits)}; +} + +template <int kBits> +HWY_API Vec256<uint8_t> ShiftRight(const Vec256<uint8_t> v) { + const Full256<uint8_t> d8; + // Use raw instead of BitCast to support N=1. + const Vec256<uint8_t> shifted{ShiftRight<kBits>(Vec256<uint16_t>{v.raw}).raw}; + return shifted & Set(d8, 0xFF >> kBits); +} + +template <int kBits> +HWY_API Vec256<int16_t> ShiftRight(const Vec256<int16_t> v) { + return Vec256<int16_t>{_mm256_srai_epi16(v.raw, kBits)}; +} + +template <int kBits> +HWY_API Vec256<int32_t> ShiftRight(const Vec256<int32_t> v) { + return Vec256<int32_t>{_mm256_srai_epi32(v.raw, kBits)}; +} + +template <int kBits> +HWY_API Vec256<int8_t> ShiftRight(const Vec256<int8_t> v) { + const Full256<int8_t> di; + const Full256<uint8_t> du; + const auto shifted = BitCast(di, ShiftRight<kBits>(BitCast(du, v))); + const auto shifted_sign = BitCast(di, Set(du, 0x80 >> kBits)); + return (shifted ^ shifted_sign) - shifted_sign; +} + +// i64 is implemented after BroadcastSignBit. + +// ------------------------------ RotateRight + +template <int kBits> +HWY_API Vec256<uint32_t> RotateRight(const Vec256<uint32_t> v) { + static_assert(0 <= kBits && kBits < 32, "Invalid shift count"); +#if HWY_TARGET <= HWY_AVX3 + return Vec256<uint32_t>{_mm256_ror_epi32(v.raw, kBits)}; +#else + if (kBits == 0) return v; + return Or(ShiftRight<kBits>(v), ShiftLeft<HWY_MIN(31, 32 - kBits)>(v)); +#endif +} + +template <int kBits> +HWY_API Vec256<uint64_t> RotateRight(const Vec256<uint64_t> v) { + static_assert(0 <= kBits && kBits < 64, "Invalid shift count"); +#if HWY_TARGET <= HWY_AVX3 + return Vec256<uint64_t>{_mm256_ror_epi64(v.raw, kBits)}; +#else + if (kBits == 0) return v; + return Or(ShiftRight<kBits>(v), ShiftLeft<HWY_MIN(63, 64 - kBits)>(v)); +#endif +} + +// ------------------------------ BroadcastSignBit (ShiftRight, compare, mask) + +HWY_API Vec256<int8_t> BroadcastSignBit(const Vec256<int8_t> v) { + return VecFromMask(v < Zero(Full256<int8_t>())); +} + +HWY_API Vec256<int16_t> BroadcastSignBit(const Vec256<int16_t> v) { + return ShiftRight<15>(v); +} + +HWY_API Vec256<int32_t> BroadcastSignBit(const Vec256<int32_t> v) { + return ShiftRight<31>(v); +} + +HWY_API Vec256<int64_t> BroadcastSignBit(const Vec256<int64_t> v) { +#if HWY_TARGET == HWY_AVX2 + return VecFromMask(v < Zero(Full256<int64_t>())); +#else + return Vec256<int64_t>{_mm256_srai_epi64(v.raw, 63)}; +#endif +} + +template <int kBits> +HWY_API Vec256<int64_t> ShiftRight(const Vec256<int64_t> v) { +#if HWY_TARGET <= HWY_AVX3 + return Vec256<int64_t>{_mm256_srai_epi64(v.raw, kBits)}; +#else + const Full256<int64_t> di; + const Full256<uint64_t> du; + const auto right = BitCast(di, ShiftRight<kBits>(BitCast(du, v))); + const auto sign = ShiftLeft<64 - kBits>(BroadcastSignBit(v)); + return right | sign; +#endif +} + +HWY_API Vec256<int64_t> Abs(const Vec256<int64_t> v) { +#if HWY_TARGET <= HWY_AVX3 + return Vec256<int64_t>{_mm256_abs_epi64(v.raw)}; +#else + const auto zero = Zero(Full256<int64_t>()); + return IfThenElse(MaskFromVec(BroadcastSignBit(v)), zero - v, v); +#endif +} + +// ------------------------------ ShiftLeftSame + +HWY_API Vec256<uint16_t> ShiftLeftSame(const Vec256<uint16_t> v, + const int bits) { + return Vec256<uint16_t>{_mm256_sll_epi16(v.raw, _mm_cvtsi32_si128(bits))}; +} +HWY_API Vec256<uint32_t> ShiftLeftSame(const Vec256<uint32_t> v, + const int bits) { + return Vec256<uint32_t>{_mm256_sll_epi32(v.raw, _mm_cvtsi32_si128(bits))}; +} +HWY_API Vec256<uint64_t> ShiftLeftSame(const Vec256<uint64_t> v, + const int bits) { + return Vec256<uint64_t>{_mm256_sll_epi64(v.raw, _mm_cvtsi32_si128(bits))}; +} + +HWY_API Vec256<int16_t> ShiftLeftSame(const Vec256<int16_t> v, const int bits) { + return Vec256<int16_t>{_mm256_sll_epi16(v.raw, _mm_cvtsi32_si128(bits))}; +} + +HWY_API Vec256<int32_t> ShiftLeftSame(const Vec256<int32_t> v, const int bits) { + return Vec256<int32_t>{_mm256_sll_epi32(v.raw, _mm_cvtsi32_si128(bits))}; +} + +HWY_API Vec256<int64_t> ShiftLeftSame(const Vec256<int64_t> v, const int bits) { + return Vec256<int64_t>{_mm256_sll_epi64(v.raw, _mm_cvtsi32_si128(bits))}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 1)> +HWY_API Vec256<T> ShiftLeftSame(const Vec256<T> v, const int bits) { + const Full256<T> d8; + const RepartitionToWide<decltype(d8)> d16; + const auto shifted = BitCast(d8, ShiftLeftSame(BitCast(d16, v), bits)); + return shifted & Set(d8, static_cast<T>((0xFF << bits) & 0xFF)); +} + +// ------------------------------ ShiftRightSame (BroadcastSignBit) + +HWY_API Vec256<uint16_t> ShiftRightSame(const Vec256<uint16_t> v, + const int bits) { + return Vec256<uint16_t>{_mm256_srl_epi16(v.raw, _mm_cvtsi32_si128(bits))}; +} +HWY_API Vec256<uint32_t> ShiftRightSame(const Vec256<uint32_t> v, + const int bits) { + return Vec256<uint32_t>{_mm256_srl_epi32(v.raw, _mm_cvtsi32_si128(bits))}; +} +HWY_API Vec256<uint64_t> ShiftRightSame(const Vec256<uint64_t> v, + const int bits) { + return Vec256<uint64_t>{_mm256_srl_epi64(v.raw, _mm_cvtsi32_si128(bits))}; +} + +HWY_API Vec256<uint8_t> ShiftRightSame(Vec256<uint8_t> v, const int bits) { + const Full256<uint8_t> d8; + const RepartitionToWide<decltype(d8)> d16; + const auto shifted = BitCast(d8, ShiftRightSame(BitCast(d16, v), bits)); + return shifted & Set(d8, static_cast<uint8_t>(0xFF >> bits)); +} + +HWY_API Vec256<int16_t> ShiftRightSame(const Vec256<int16_t> v, + const int bits) { + return Vec256<int16_t>{_mm256_sra_epi16(v.raw, _mm_cvtsi32_si128(bits))}; +} + +HWY_API Vec256<int32_t> ShiftRightSame(const Vec256<int32_t> v, + const int bits) { + return Vec256<int32_t>{_mm256_sra_epi32(v.raw, _mm_cvtsi32_si128(bits))}; +} +HWY_API Vec256<int64_t> ShiftRightSame(const Vec256<int64_t> v, + const int bits) { +#if HWY_TARGET <= HWY_AVX3 + return Vec256<int64_t>{_mm256_sra_epi64(v.raw, _mm_cvtsi32_si128(bits))}; +#else + const Full256<int64_t> di; + const Full256<uint64_t> du; + const auto right = BitCast(di, ShiftRightSame(BitCast(du, v), bits)); + const auto sign = ShiftLeftSame(BroadcastSignBit(v), 64 - bits); + return right | sign; +#endif +} + +HWY_API Vec256<int8_t> ShiftRightSame(Vec256<int8_t> v, const int bits) { + const Full256<int8_t> di; + const Full256<uint8_t> du; + const auto shifted = BitCast(di, ShiftRightSame(BitCast(du, v), bits)); + const auto shifted_sign = + BitCast(di, Set(du, static_cast<uint8_t>(0x80 >> bits))); + return (shifted ^ shifted_sign) - shifted_sign; +} + +// ------------------------------ Neg (Xor, Sub) + +template <typename T, HWY_IF_FLOAT(T)> +HWY_API Vec256<T> Neg(const Vec256<T> v) { + return Xor(v, SignBit(Full256<T>())); +} + +template <typename T, HWY_IF_NOT_FLOAT(T)> +HWY_API Vec256<T> Neg(const Vec256<T> v) { + return Zero(Full256<T>()) - v; +} + +// ------------------------------ Floating-point mul / div + +HWY_API Vec256<float> operator*(const Vec256<float> a, const Vec256<float> b) { + return Vec256<float>{_mm256_mul_ps(a.raw, b.raw)}; +} +HWY_API Vec256<double> operator*(const Vec256<double> a, + const Vec256<double> b) { + return Vec256<double>{_mm256_mul_pd(a.raw, b.raw)}; +} + +HWY_API Vec256<float> operator/(const Vec256<float> a, const Vec256<float> b) { + return Vec256<float>{_mm256_div_ps(a.raw, b.raw)}; +} +HWY_API Vec256<double> operator/(const Vec256<double> a, + const Vec256<double> b) { + return Vec256<double>{_mm256_div_pd(a.raw, b.raw)}; +} + +// Approximate reciprocal +HWY_API Vec256<float> ApproximateReciprocal(const Vec256<float> v) { + return Vec256<float>{_mm256_rcp_ps(v.raw)}; +} + +// Absolute value of difference. +HWY_API Vec256<float> AbsDiff(const Vec256<float> a, const Vec256<float> b) { + return Abs(a - b); +} + +// ------------------------------ Floating-point multiply-add variants + +// Returns mul * x + add +HWY_API Vec256<float> MulAdd(const Vec256<float> mul, const Vec256<float> x, + const Vec256<float> add) { +#ifdef HWY_DISABLE_BMI2_FMA + return mul * x + add; +#else + return Vec256<float>{_mm256_fmadd_ps(mul.raw, x.raw, add.raw)}; +#endif +} +HWY_API Vec256<double> MulAdd(const Vec256<double> mul, const Vec256<double> x, + const Vec256<double> add) { +#ifdef HWY_DISABLE_BMI2_FMA + return mul * x + add; +#else + return Vec256<double>{_mm256_fmadd_pd(mul.raw, x.raw, add.raw)}; +#endif +} + +// Returns add - mul * x +HWY_API Vec256<float> NegMulAdd(const Vec256<float> mul, const Vec256<float> x, + const Vec256<float> add) { +#ifdef HWY_DISABLE_BMI2_FMA + return add - mul * x; +#else + return Vec256<float>{_mm256_fnmadd_ps(mul.raw, x.raw, add.raw)}; +#endif +} +HWY_API Vec256<double> NegMulAdd(const Vec256<double> mul, + const Vec256<double> x, + const Vec256<double> add) { +#ifdef HWY_DISABLE_BMI2_FMA + return add - mul * x; +#else + return Vec256<double>{_mm256_fnmadd_pd(mul.raw, x.raw, add.raw)}; +#endif +} + +// Returns mul * x - sub +HWY_API Vec256<float> MulSub(const Vec256<float> mul, const Vec256<float> x, + const Vec256<float> sub) { +#ifdef HWY_DISABLE_BMI2_FMA + return mul * x - sub; +#else + return Vec256<float>{_mm256_fmsub_ps(mul.raw, x.raw, sub.raw)}; +#endif +} +HWY_API Vec256<double> MulSub(const Vec256<double> mul, const Vec256<double> x, + const Vec256<double> sub) { +#ifdef HWY_DISABLE_BMI2_FMA + return mul * x - sub; +#else + return Vec256<double>{_mm256_fmsub_pd(mul.raw, x.raw, sub.raw)}; +#endif +} + +// Returns -mul * x - sub +HWY_API Vec256<float> NegMulSub(const Vec256<float> mul, const Vec256<float> x, + const Vec256<float> sub) { +#ifdef HWY_DISABLE_BMI2_FMA + return Neg(mul * x) - sub; +#else + return Vec256<float>{_mm256_fnmsub_ps(mul.raw, x.raw, sub.raw)}; +#endif +} +HWY_API Vec256<double> NegMulSub(const Vec256<double> mul, + const Vec256<double> x, + const Vec256<double> sub) { +#ifdef HWY_DISABLE_BMI2_FMA + return Neg(mul * x) - sub; +#else + return Vec256<double>{_mm256_fnmsub_pd(mul.raw, x.raw, sub.raw)}; +#endif +} + +// ------------------------------ Floating-point square root + +// Full precision square root +HWY_API Vec256<float> Sqrt(const Vec256<float> v) { + return Vec256<float>{_mm256_sqrt_ps(v.raw)}; +} +HWY_API Vec256<double> Sqrt(const Vec256<double> v) { + return Vec256<double>{_mm256_sqrt_pd(v.raw)}; +} + +// Approximate reciprocal square root +HWY_API Vec256<float> ApproximateReciprocalSqrt(const Vec256<float> v) { + return Vec256<float>{_mm256_rsqrt_ps(v.raw)}; +} + +// ------------------------------ Floating-point rounding + +// Toward nearest integer, tie to even +HWY_API Vec256<float> Round(const Vec256<float> v) { + return Vec256<float>{ + _mm256_round_ps(v.raw, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)}; +} +HWY_API Vec256<double> Round(const Vec256<double> v) { + return Vec256<double>{ + _mm256_round_pd(v.raw, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)}; +} + +// Toward zero, aka truncate +HWY_API Vec256<float> Trunc(const Vec256<float> v) { + return Vec256<float>{ + _mm256_round_ps(v.raw, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC)}; +} +HWY_API Vec256<double> Trunc(const Vec256<double> v) { + return Vec256<double>{ + _mm256_round_pd(v.raw, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC)}; +} + +// Toward +infinity, aka ceiling +HWY_API Vec256<float> Ceil(const Vec256<float> v) { + return Vec256<float>{ + _mm256_round_ps(v.raw, _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC)}; +} +HWY_API Vec256<double> Ceil(const Vec256<double> v) { + return Vec256<double>{ + _mm256_round_pd(v.raw, _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC)}; +} + +// Toward -infinity, aka floor +HWY_API Vec256<float> Floor(const Vec256<float> v) { + return Vec256<float>{ + _mm256_round_ps(v.raw, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC)}; +} +HWY_API Vec256<double> Floor(const Vec256<double> v) { + return Vec256<double>{ + _mm256_round_pd(v.raw, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC)}; +} + +// ================================================== MEMORY + +// ------------------------------ Load + +template <typename T> +HWY_API Vec256<T> Load(Full256<T> /* tag */, const T* HWY_RESTRICT aligned) { + return Vec256<T>{ + _mm256_load_si256(reinterpret_cast<const __m256i*>(aligned))}; +} +HWY_API Vec256<float> Load(Full256<float> /* tag */, + const float* HWY_RESTRICT aligned) { + return Vec256<float>{_mm256_load_ps(aligned)}; +} +HWY_API Vec256<double> Load(Full256<double> /* tag */, + const double* HWY_RESTRICT aligned) { + return Vec256<double>{_mm256_load_pd(aligned)}; +} + +template <typename T> +HWY_API Vec256<T> LoadU(Full256<T> /* tag */, const T* HWY_RESTRICT p) { + return Vec256<T>{_mm256_loadu_si256(reinterpret_cast<const __m256i*>(p))}; +} +HWY_API Vec256<float> LoadU(Full256<float> /* tag */, + const float* HWY_RESTRICT p) { + return Vec256<float>{_mm256_loadu_ps(p)}; +} +HWY_API Vec256<double> LoadU(Full256<double> /* tag */, + const double* HWY_RESTRICT p) { + return Vec256<double>{_mm256_loadu_pd(p)}; +} + +// ------------------------------ MaskedLoad + +#if HWY_TARGET <= HWY_AVX3 + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec256<T> MaskedLoad(Mask256<T> m, Full256<T> /* tag */, + const T* HWY_RESTRICT aligned) { + return Vec256<T>{_mm256_maskz_load_epi32(m.raw, aligned)}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec256<T> MaskedLoad(Mask256<T> m, Full256<T> /* tag */, + const T* HWY_RESTRICT aligned) { + return Vec256<T>{_mm256_maskz_load_epi64(m.raw, aligned)}; +} + +HWY_API Vec256<float> MaskedLoad(Mask256<float> m, Full256<float> /* tag */, + const float* HWY_RESTRICT aligned) { + return Vec256<float>{_mm256_maskz_load_ps(m.raw, aligned)}; +} + +HWY_API Vec256<double> MaskedLoad(Mask256<double> m, Full256<double> /* tag */, + const double* HWY_RESTRICT aligned) { + return Vec256<double>{_mm256_maskz_load_pd(m.raw, aligned)}; +} + +// There is no load_epi8/16, so use loadu instead. +template <typename T, HWY_IF_LANE_SIZE(T, 1)> +HWY_API Vec256<T> MaskedLoad(Mask256<T> m, Full256<T> /* tag */, + const T* HWY_RESTRICT aligned) { + return Vec256<T>{_mm256_maskz_loadu_epi8(m.raw, aligned)}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Vec256<T> MaskedLoad(Mask256<T> m, Full256<T> /* tag */, + const T* HWY_RESTRICT aligned) { + return Vec256<T>{_mm256_maskz_loadu_epi16(m.raw, aligned)}; +} + +#else // AVX2 + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec256<T> MaskedLoad(Mask256<T> m, Full256<T> /* tag */, + const T* HWY_RESTRICT aligned) { + auto aligned_p = reinterpret_cast<const int*>(aligned); // NOLINT + return Vec256<T>{_mm256_maskload_epi32(aligned_p, m.raw)}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec256<T> MaskedLoad(Mask256<T> m, Full256<T> /* tag */, + const T* HWY_RESTRICT aligned) { + auto aligned_p = reinterpret_cast<const long long*>(aligned); // NOLINT + return Vec256<T>{_mm256_maskload_epi64(aligned_p, m.raw)}; +} + +HWY_API Vec256<float> MaskedLoad(Mask256<float> m, Full256<float> d, + const float* HWY_RESTRICT aligned) { + const Vec256<int32_t> mi = + BitCast(RebindToSigned<decltype(d)>(), VecFromMask(d, m)); + return Vec256<float>{_mm256_maskload_ps(aligned, mi.raw)}; +} + +HWY_API Vec256<double> MaskedLoad(Mask256<double> m, Full256<double> d, + const double* HWY_RESTRICT aligned) { + const Vec256<int64_t> mi = + BitCast(RebindToSigned<decltype(d)>(), VecFromMask(d, m)); + return Vec256<double>{_mm256_maskload_pd(aligned, mi.raw)}; +} + +// There is no maskload_epi8/16, so blend instead. +template <typename T, hwy::EnableIf<sizeof(T) <= 2>* = nullptr> +HWY_API Vec256<T> MaskedLoad(Mask256<T> m, Full256<T> d, + const T* HWY_RESTRICT aligned) { + return IfThenElseZero(m, Load(d, aligned)); +} + +#endif + +// ------------------------------ LoadDup128 + +// Loads 128 bit and duplicates into both 128-bit halves. This avoids the +// 3-cycle cost of moving data between 128-bit halves and avoids port 5. +template <typename T> +HWY_API Vec256<T> LoadDup128(Full256<T> /* tag */, const T* HWY_RESTRICT p) { +#if HWY_LOADDUP_ASM + __m256i out; + asm("vbroadcasti128 %1, %[reg]" : [ reg ] "=x"(out) : "m"(p[0])); + return Vec256<T>{out}; +#elif HWY_COMPILER_MSVC && !HWY_COMPILER_CLANG + // Workaround for incorrect results with _mm256_broadcastsi128_si256. Note + // that MSVC also lacks _mm256_zextsi128_si256, but cast (which leaves the + // upper half undefined) is fine because we're overwriting that anyway. + const __m128i v128 = LoadU(Full128<T>(), p).raw; + return Vec256<T>{ + _mm256_inserti128_si256(_mm256_castsi128_si256(v128), v128, 1)}; +#else + return Vec256<T>{_mm256_broadcastsi128_si256(LoadU(Full128<T>(), p).raw)}; +#endif +} +HWY_API Vec256<float> LoadDup128(Full256<float> /* tag */, + const float* const HWY_RESTRICT p) { +#if HWY_LOADDUP_ASM + __m256 out; + asm("vbroadcastf128 %1, %[reg]" : [ reg ] "=x"(out) : "m"(p[0])); + return Vec256<float>{out}; +#elif HWY_COMPILER_MSVC && !HWY_COMPILER_CLANG + const __m128 v128 = LoadU(Full128<float>(), p).raw; + return Vec256<float>{ + _mm256_insertf128_ps(_mm256_castps128_ps256(v128), v128, 1)}; +#else + return Vec256<float>{_mm256_broadcast_ps(reinterpret_cast<const __m128*>(p))}; +#endif +} +HWY_API Vec256<double> LoadDup128(Full256<double> /* tag */, + const double* const HWY_RESTRICT p) { +#if HWY_LOADDUP_ASM + __m256d out; + asm("vbroadcastf128 %1, %[reg]" : [ reg ] "=x"(out) : "m"(p[0])); + return Vec256<double>{out}; +#elif HWY_COMPILER_MSVC && !HWY_COMPILER_CLANG + const __m128d v128 = LoadU(Full128<double>(), p).raw; + return Vec256<double>{ + _mm256_insertf128_pd(_mm256_castpd128_pd256(v128), v128, 1)}; +#else + return Vec256<double>{ + _mm256_broadcast_pd(reinterpret_cast<const __m128d*>(p))}; +#endif +} + +// ------------------------------ Store + +template <typename T> +HWY_API void Store(Vec256<T> v, Full256<T> /* tag */, T* HWY_RESTRICT aligned) { + _mm256_store_si256(reinterpret_cast<__m256i*>(aligned), v.raw); +} +HWY_API void Store(const Vec256<float> v, Full256<float> /* tag */, + float* HWY_RESTRICT aligned) { + _mm256_store_ps(aligned, v.raw); +} +HWY_API void Store(const Vec256<double> v, Full256<double> /* tag */, + double* HWY_RESTRICT aligned) { + _mm256_store_pd(aligned, v.raw); +} + +template <typename T> +HWY_API void StoreU(Vec256<T> v, Full256<T> /* tag */, T* HWY_RESTRICT p) { + _mm256_storeu_si256(reinterpret_cast<__m256i*>(p), v.raw); +} +HWY_API void StoreU(const Vec256<float> v, Full256<float> /* tag */, + float* HWY_RESTRICT p) { + _mm256_storeu_ps(p, v.raw); +} +HWY_API void StoreU(const Vec256<double> v, Full256<double> /* tag */, + double* HWY_RESTRICT p) { + _mm256_storeu_pd(p, v.raw); +} + +// ------------------------------ Non-temporal stores + +template <typename T> +HWY_API void Stream(Vec256<T> v, Full256<T> /* tag */, + T* HWY_RESTRICT aligned) { + _mm256_stream_si256(reinterpret_cast<__m256i*>(aligned), v.raw); +} +HWY_API void Stream(const Vec256<float> v, Full256<float> /* tag */, + float* HWY_RESTRICT aligned) { + _mm256_stream_ps(aligned, v.raw); +} +HWY_API void Stream(const Vec256<double> v, Full256<double> /* tag */, + double* HWY_RESTRICT aligned) { + _mm256_stream_pd(aligned, v.raw); +} + +// ------------------------------ Scatter + +// Work around warnings in the intrinsic definitions (passing -1 as a mask). +HWY_DIAGNOSTICS(push) +HWY_DIAGNOSTICS_OFF(disable : 4245 4365, ignored "-Wsign-conversion") + +#if HWY_TARGET <= HWY_AVX3 +namespace detail { + +template <typename T> +HWY_INLINE void ScatterOffset(hwy::SizeTag<4> /* tag */, Vec256<T> v, + Full256<T> /* tag */, T* HWY_RESTRICT base, + const Vec256<int32_t> offset) { + _mm256_i32scatter_epi32(base, offset.raw, v.raw, 1); +} +template <typename T> +HWY_INLINE void ScatterIndex(hwy::SizeTag<4> /* tag */, Vec256<T> v, + Full256<T> /* tag */, T* HWY_RESTRICT base, + const Vec256<int32_t> index) { + _mm256_i32scatter_epi32(base, index.raw, v.raw, 4); +} + +template <typename T> +HWY_INLINE void ScatterOffset(hwy::SizeTag<8> /* tag */, Vec256<T> v, + Full256<T> /* tag */, T* HWY_RESTRICT base, + const Vec256<int64_t> offset) { + _mm256_i64scatter_epi64(base, offset.raw, v.raw, 1); +} +template <typename T> +HWY_INLINE void ScatterIndex(hwy::SizeTag<8> /* tag */, Vec256<T> v, + Full256<T> /* tag */, T* HWY_RESTRICT base, + const Vec256<int64_t> index) { + _mm256_i64scatter_epi64(base, index.raw, v.raw, 8); +} + +} // namespace detail + +template <typename T, typename Offset> +HWY_API void ScatterOffset(Vec256<T> v, Full256<T> d, T* HWY_RESTRICT base, + const Vec256<Offset> offset) { + static_assert(sizeof(T) == sizeof(Offset), "Must match for portability"); + return detail::ScatterOffset(hwy::SizeTag<sizeof(T)>(), v, d, base, offset); +} +template <typename T, typename Index> +HWY_API void ScatterIndex(Vec256<T> v, Full256<T> d, T* HWY_RESTRICT base, + const Vec256<Index> index) { + static_assert(sizeof(T) == sizeof(Index), "Must match for portability"); + return detail::ScatterIndex(hwy::SizeTag<sizeof(T)>(), v, d, base, index); +} + +HWY_API void ScatterOffset(Vec256<float> v, Full256<float> /* tag */, + float* HWY_RESTRICT base, + const Vec256<int32_t> offset) { + _mm256_i32scatter_ps(base, offset.raw, v.raw, 1); +} +HWY_API void ScatterIndex(Vec256<float> v, Full256<float> /* tag */, + float* HWY_RESTRICT base, + const Vec256<int32_t> index) { + _mm256_i32scatter_ps(base, index.raw, v.raw, 4); +} + +HWY_API void ScatterOffset(Vec256<double> v, Full256<double> /* tag */, + double* HWY_RESTRICT base, + const Vec256<int64_t> offset) { + _mm256_i64scatter_pd(base, offset.raw, v.raw, 1); +} +HWY_API void ScatterIndex(Vec256<double> v, Full256<double> /* tag */, + double* HWY_RESTRICT base, + const Vec256<int64_t> index) { + _mm256_i64scatter_pd(base, index.raw, v.raw, 8); +} + +#else + +template <typename T, typename Offset> +HWY_API void ScatterOffset(Vec256<T> v, Full256<T> d, T* HWY_RESTRICT base, + const Vec256<Offset> offset) { + static_assert(sizeof(T) == sizeof(Offset), "Must match for portability"); + + constexpr size_t N = 32 / sizeof(T); + alignas(32) T lanes[N]; + Store(v, d, lanes); + + alignas(32) Offset offset_lanes[N]; + Store(offset, Simd<Offset, N>(), offset_lanes); + + uint8_t* base_bytes = reinterpret_cast<uint8_t*>(base); + for (size_t i = 0; i < N; ++i) { + CopyBytes<sizeof(T)>(&lanes[i], base_bytes + offset_lanes[i]); + } +} + +template <typename T, typename Index> +HWY_API void ScatterIndex(Vec256<T> v, Full256<T> d, T* HWY_RESTRICT base, + const Vec256<Index> index) { + static_assert(sizeof(T) == sizeof(Index), "Must match for portability"); + + constexpr size_t N = 32 / sizeof(T); + alignas(32) T lanes[N]; + Store(v, d, lanes); + + alignas(32) Index index_lanes[N]; + Store(index, Simd<Index, N>(), index_lanes); + + for (size_t i = 0; i < N; ++i) { + base[index_lanes[i]] = lanes[i]; + } +} + +#endif + +// ------------------------------ Gather + +namespace detail { + +template <typename T> +HWY_INLINE Vec256<T> GatherOffset(hwy::SizeTag<4> /* tag */, + Full256<T> /* tag */, + const T* HWY_RESTRICT base, + const Vec256<int32_t> offset) { + return Vec256<T>{_mm256_i32gather_epi32( + reinterpret_cast<const int32_t*>(base), offset.raw, 1)}; +} +template <typename T> +HWY_INLINE Vec256<T> GatherIndex(hwy::SizeTag<4> /* tag */, + Full256<T> /* tag */, + const T* HWY_RESTRICT base, + const Vec256<int32_t> index) { + return Vec256<T>{_mm256_i32gather_epi32( + reinterpret_cast<const int32_t*>(base), index.raw, 4)}; +} + +template <typename T> +HWY_INLINE Vec256<T> GatherOffset(hwy::SizeTag<8> /* tag */, + Full256<T> /* tag */, + const T* HWY_RESTRICT base, + const Vec256<int64_t> offset) { + return Vec256<T>{_mm256_i64gather_epi64( + reinterpret_cast<const GatherIndex64*>(base), offset.raw, 1)}; +} +template <typename T> +HWY_INLINE Vec256<T> GatherIndex(hwy::SizeTag<8> /* tag */, + Full256<T> /* tag */, + const T* HWY_RESTRICT base, + const Vec256<int64_t> index) { + return Vec256<T>{_mm256_i64gather_epi64( + reinterpret_cast<const GatherIndex64*>(base), index.raw, 8)}; +} + +} // namespace detail + +template <typename T, typename Offset> +HWY_API Vec256<T> GatherOffset(Full256<T> d, const T* HWY_RESTRICT base, + const Vec256<Offset> offset) { + static_assert(sizeof(T) == sizeof(Offset), "Must match for portability"); + return detail::GatherOffset(hwy::SizeTag<sizeof(T)>(), d, base, offset); +} +template <typename T, typename Index> +HWY_API Vec256<T> GatherIndex(Full256<T> d, const T* HWY_RESTRICT base, + const Vec256<Index> index) { + static_assert(sizeof(T) == sizeof(Index), "Must match for portability"); + return detail::GatherIndex(hwy::SizeTag<sizeof(T)>(), d, base, index); +} + +HWY_API Vec256<float> GatherOffset(Full256<float> /* tag */, + const float* HWY_RESTRICT base, + const Vec256<int32_t> offset) { + return Vec256<float>{_mm256_i32gather_ps(base, offset.raw, 1)}; +} +HWY_API Vec256<float> GatherIndex(Full256<float> /* tag */, + const float* HWY_RESTRICT base, + const Vec256<int32_t> index) { + return Vec256<float>{_mm256_i32gather_ps(base, index.raw, 4)}; +} + +HWY_API Vec256<double> GatherOffset(Full256<double> /* tag */, + const double* HWY_RESTRICT base, + const Vec256<int64_t> offset) { + return Vec256<double>{_mm256_i64gather_pd(base, offset.raw, 1)}; +} +HWY_API Vec256<double> GatherIndex(Full256<double> /* tag */, + const double* HWY_RESTRICT base, + const Vec256<int64_t> index) { + return Vec256<double>{_mm256_i64gather_pd(base, index.raw, 8)}; +} + +HWY_DIAGNOSTICS(pop) + +// ================================================== SWIZZLE + +// ------------------------------ LowerHalf + +template <typename T> +HWY_API Vec128<T> LowerHalf(Full128<T> /* tag */, Vec256<T> v) { + return Vec128<T>{_mm256_castsi256_si128(v.raw)}; +} +HWY_API Vec128<float> LowerHalf(Full128<float> /* tag */, Vec256<float> v) { + return Vec128<float>{_mm256_castps256_ps128(v.raw)}; +} +HWY_API Vec128<double> LowerHalf(Full128<double> /* tag */, Vec256<double> v) { + return Vec128<double>{_mm256_castpd256_pd128(v.raw)}; +} + +template <typename T> +HWY_API Vec128<T> LowerHalf(Vec256<T> v) { + return LowerHalf(Full128<T>(), v); +} + +// ------------------------------ UpperHalf + +template <typename T> +HWY_API Vec128<T> UpperHalf(Full128<T> /* tag */, Vec256<T> v) { + return Vec128<T>{_mm256_extracti128_si256(v.raw, 1)}; +} +HWY_API Vec128<float> UpperHalf(Full128<float> /* tag */, Vec256<float> v) { + return Vec128<float>{_mm256_extractf128_ps(v.raw, 1)}; +} +HWY_API Vec128<double> UpperHalf(Full128<double> /* tag */, Vec256<double> v) { + return Vec128<double>{_mm256_extractf128_pd(v.raw, 1)}; +} + +// ------------------------------ GetLane (LowerHalf) +template <typename T> +HWY_API T GetLane(const Vec256<T> v) { + return GetLane(LowerHalf(v)); +} + +// ------------------------------ ZeroExtendVector + +// Unfortunately the initial _mm256_castsi128_si256 intrinsic leaves the upper +// bits undefined. Although it makes sense for them to be zero (VEX encoded +// 128-bit instructions zero the upper lanes to avoid large penalties), a +// compiler could decide to optimize out code that relies on this. +// +// The newer _mm256_zextsi128_si256 intrinsic fixes this by specifying the +// zeroing, but it is not available on MSVC nor GCC until 10.1. For older GCC, +// we can still obtain the desired code thanks to pattern recognition; note that +// the expensive insert instruction is not actually generated, see +// https://gcc.godbolt.org/z/1MKGaP. + +template <typename T> +HWY_API Vec256<T> ZeroExtendVector(Full256<T> /* tag */, Vec128<T> lo) { +#if !HWY_COMPILER_CLANG && HWY_COMPILER_GCC && (HWY_COMPILER_GCC < 1000) + return Vec256<T>{_mm256_inserti128_si256(_mm256_setzero_si256(), lo.raw, 0)}; +#else + return Vec256<T>{_mm256_zextsi128_si256(lo.raw)}; +#endif +} +HWY_API Vec256<float> ZeroExtendVector(Full256<float> /* tag */, + Vec128<float> lo) { +#if !HWY_COMPILER_CLANG && HWY_COMPILER_GCC && (HWY_COMPILER_GCC < 1000) + return Vec256<float>{_mm256_insertf128_ps(_mm256_setzero_ps(), lo.raw, 0)}; +#else + return Vec256<float>{_mm256_zextps128_ps256(lo.raw)}; +#endif +} +HWY_API Vec256<double> ZeroExtendVector(Full256<double> /* tag */, + Vec128<double> lo) { +#if !HWY_COMPILER_CLANG && HWY_COMPILER_GCC && (HWY_COMPILER_GCC < 1000) + return Vec256<double>{_mm256_insertf128_pd(_mm256_setzero_pd(), lo.raw, 0)}; +#else + return Vec256<double>{_mm256_zextpd128_pd256(lo.raw)}; +#endif +} + +// ------------------------------ Combine + +template <typename T> +HWY_API Vec256<T> Combine(Full256<T> d, Vec128<T> hi, Vec128<T> lo) { + const auto lo256 = ZeroExtendVector(d, lo); + return Vec256<T>{_mm256_inserti128_si256(lo256.raw, hi.raw, 1)}; +} +HWY_API Vec256<float> Combine(Full256<float> d, Vec128<float> hi, + Vec128<float> lo) { + const auto lo256 = ZeroExtendVector(d, lo); + return Vec256<float>{_mm256_insertf128_ps(lo256.raw, hi.raw, 1)}; +} +HWY_API Vec256<double> Combine(Full256<double> d, Vec128<double> hi, + Vec128<double> lo) { + const auto lo256 = ZeroExtendVector(d, lo); + return Vec256<double>{_mm256_insertf128_pd(lo256.raw, hi.raw, 1)}; +} + +// ------------------------------ ShiftLeftBytes + +template <int kBytes, typename T> +HWY_API Vec256<T> ShiftLeftBytes(Full256<T> /* tag */, const Vec256<T> v) { + static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes"); + // This is the same operation as _mm256_bslli_epi128. + return Vec256<T>{_mm256_slli_si256(v.raw, kBytes)}; +} + +template <int kBytes, typename T> +HWY_API Vec256<T> ShiftLeftBytes(const Vec256<T> v) { + return ShiftLeftBytes<kBytes>(Full256<T>(), v); +} + +// ------------------------------ ShiftLeftLanes + +template <int kLanes, typename T> +HWY_API Vec256<T> ShiftLeftLanes(Full256<T> d, const Vec256<T> v) { + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, ShiftLeftBytes<kLanes * sizeof(T)>(BitCast(d8, v))); +} + +template <int kLanes, typename T> +HWY_API Vec256<T> ShiftLeftLanes(const Vec256<T> v) { + return ShiftLeftLanes<kLanes>(Full256<T>(), v); +} + +// ------------------------------ ShiftRightBytes + +template <int kBytes, typename T> +HWY_API Vec256<T> ShiftRightBytes(Full256<T> /* tag */, const Vec256<T> v) { + static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes"); + // This is the same operation as _mm256_bsrli_epi128. + return Vec256<T>{_mm256_srli_si256(v.raw, kBytes)}; +} + +// ------------------------------ ShiftRightLanes +template <int kLanes, typename T> +HWY_API Vec256<T> ShiftRightLanes(Full256<T> d, const Vec256<T> v) { + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, ShiftRightBytes<kLanes * sizeof(T)>(BitCast(d8, v))); +} + +// ------------------------------ CombineShiftRightBytes + +// Extracts 128 bits from <hi, lo> by skipping the least-significant kBytes. +template <int kBytes, typename T, class V = Vec256<T>> +HWY_API V CombineShiftRightBytes(Full256<T> d, V hi, V lo) { + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, Vec256<uint8_t>{_mm256_alignr_epi8( + BitCast(d8, hi).raw, BitCast(d8, lo).raw, kBytes)}); +} + +// ------------------------------ Broadcast/splat any lane + +// Unsigned +template <int kLane> +HWY_API Vec256<uint16_t> Broadcast(const Vec256<uint16_t> v) { + static_assert(0 <= kLane && kLane < 8, "Invalid lane"); + if (kLane < 4) { + const __m256i lo = _mm256_shufflelo_epi16(v.raw, (0x55 * kLane) & 0xFF); + return Vec256<uint16_t>{_mm256_unpacklo_epi64(lo, lo)}; + } else { + const __m256i hi = + _mm256_shufflehi_epi16(v.raw, (0x55 * (kLane - 4)) & 0xFF); + return Vec256<uint16_t>{_mm256_unpackhi_epi64(hi, hi)}; + } +} +template <int kLane> +HWY_API Vec256<uint32_t> Broadcast(const Vec256<uint32_t> v) { + static_assert(0 <= kLane && kLane < 4, "Invalid lane"); + return Vec256<uint32_t>{_mm256_shuffle_epi32(v.raw, 0x55 * kLane)}; +} +template <int kLane> +HWY_API Vec256<uint64_t> Broadcast(const Vec256<uint64_t> v) { + static_assert(0 <= kLane && kLane < 2, "Invalid lane"); + return Vec256<uint64_t>{_mm256_shuffle_epi32(v.raw, kLane ? 0xEE : 0x44)}; +} + +// Signed +template <int kLane> +HWY_API Vec256<int16_t> Broadcast(const Vec256<int16_t> v) { + static_assert(0 <= kLane && kLane < 8, "Invalid lane"); + if (kLane < 4) { + const __m256i lo = _mm256_shufflelo_epi16(v.raw, (0x55 * kLane) & 0xFF); + return Vec256<int16_t>{_mm256_unpacklo_epi64(lo, lo)}; + } else { + const __m256i hi = + _mm256_shufflehi_epi16(v.raw, (0x55 * (kLane - 4)) & 0xFF); + return Vec256<int16_t>{_mm256_unpackhi_epi64(hi, hi)}; + } +} +template <int kLane> +HWY_API Vec256<int32_t> Broadcast(const Vec256<int32_t> v) { + static_assert(0 <= kLane && kLane < 4, "Invalid lane"); + return Vec256<int32_t>{_mm256_shuffle_epi32(v.raw, 0x55 * kLane)}; +} +template <int kLane> +HWY_API Vec256<int64_t> Broadcast(const Vec256<int64_t> v) { + static_assert(0 <= kLane && kLane < 2, "Invalid lane"); + return Vec256<int64_t>{_mm256_shuffle_epi32(v.raw, kLane ? 0xEE : 0x44)}; +} + +// Float +template <int kLane> +HWY_API Vec256<float> Broadcast(Vec256<float> v) { + static_assert(0 <= kLane && kLane < 4, "Invalid lane"); + return Vec256<float>{_mm256_shuffle_ps(v.raw, v.raw, 0x55 * kLane)}; +} +template <int kLane> +HWY_API Vec256<double> Broadcast(const Vec256<double> v) { + static_assert(0 <= kLane && kLane < 2, "Invalid lane"); + return Vec256<double>{_mm256_shuffle_pd(v.raw, v.raw, 15 * kLane)}; +} + +// ------------------------------ Hard-coded shuffles + +// Notation: let Vec256<int32_t> have lanes 7,6,5,4,3,2,1,0 (0 is +// least-significant). Shuffle0321 rotates four-lane blocks one lane to the +// right (the previous least-significant lane is now most-significant => +// 47650321). These could also be implemented via CombineShiftRightBytes but +// the shuffle_abcd notation is more convenient. + +// Swap 32-bit halves in 64-bit halves. +HWY_API Vec256<uint32_t> Shuffle2301(const Vec256<uint32_t> v) { + return Vec256<uint32_t>{_mm256_shuffle_epi32(v.raw, 0xB1)}; +} +HWY_API Vec256<int32_t> Shuffle2301(const Vec256<int32_t> v) { + return Vec256<int32_t>{_mm256_shuffle_epi32(v.raw, 0xB1)}; +} +HWY_API Vec256<float> Shuffle2301(const Vec256<float> v) { + return Vec256<float>{_mm256_shuffle_ps(v.raw, v.raw, 0xB1)}; +} + +// Swap 64-bit halves +HWY_API Vec256<uint32_t> Shuffle1032(const Vec256<uint32_t> v) { + return Vec256<uint32_t>{_mm256_shuffle_epi32(v.raw, 0x4E)}; +} +HWY_API Vec256<int32_t> Shuffle1032(const Vec256<int32_t> v) { + return Vec256<int32_t>{_mm256_shuffle_epi32(v.raw, 0x4E)}; +} +HWY_API Vec256<float> Shuffle1032(const Vec256<float> v) { + // Shorter encoding than _mm256_permute_ps. + return Vec256<float>{_mm256_shuffle_ps(v.raw, v.raw, 0x4E)}; +} +HWY_API Vec256<uint64_t> Shuffle01(const Vec256<uint64_t> v) { + return Vec256<uint64_t>{_mm256_shuffle_epi32(v.raw, 0x4E)}; +} +HWY_API Vec256<int64_t> Shuffle01(const Vec256<int64_t> v) { + return Vec256<int64_t>{_mm256_shuffle_epi32(v.raw, 0x4E)}; +} +HWY_API Vec256<double> Shuffle01(const Vec256<double> v) { + // Shorter encoding than _mm256_permute_pd. + return Vec256<double>{_mm256_shuffle_pd(v.raw, v.raw, 5)}; +} + +// Rotate right 32 bits +HWY_API Vec256<uint32_t> Shuffle0321(const Vec256<uint32_t> v) { + return Vec256<uint32_t>{_mm256_shuffle_epi32(v.raw, 0x39)}; +} +HWY_API Vec256<int32_t> Shuffle0321(const Vec256<int32_t> v) { + return Vec256<int32_t>{_mm256_shuffle_epi32(v.raw, 0x39)}; +} +HWY_API Vec256<float> Shuffle0321(const Vec256<float> v) { + return Vec256<float>{_mm256_shuffle_ps(v.raw, v.raw, 0x39)}; +} +// Rotate left 32 bits +HWY_API Vec256<uint32_t> Shuffle2103(const Vec256<uint32_t> v) { + return Vec256<uint32_t>{_mm256_shuffle_epi32(v.raw, 0x93)}; +} +HWY_API Vec256<int32_t> Shuffle2103(const Vec256<int32_t> v) { + return Vec256<int32_t>{_mm256_shuffle_epi32(v.raw, 0x93)}; +} +HWY_API Vec256<float> Shuffle2103(const Vec256<float> v) { + return Vec256<float>{_mm256_shuffle_ps(v.raw, v.raw, 0x93)}; +} + +// Reverse +HWY_API Vec256<uint32_t> Shuffle0123(const Vec256<uint32_t> v) { + return Vec256<uint32_t>{_mm256_shuffle_epi32(v.raw, 0x1B)}; +} +HWY_API Vec256<int32_t> Shuffle0123(const Vec256<int32_t> v) { + return Vec256<int32_t>{_mm256_shuffle_epi32(v.raw, 0x1B)}; +} +HWY_API Vec256<float> Shuffle0123(const Vec256<float> v) { + return Vec256<float>{_mm256_shuffle_ps(v.raw, v.raw, 0x1B)}; +} + +// ------------------------------ TableLookupLanes + +// Returned by SetTableIndices/IndicesFromVec for use by TableLookupLanes. +template <typename T> +struct Indices256 { + __m256i raw; +}; + +// Native 8x32 instruction: indices remain unchanged +template <typename T, typename TI, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Indices256<T> IndicesFromVec(Full256<T> /* tag */, Vec256<TI> vec) { + static_assert(sizeof(T) == sizeof(TI), "Index size must match lane"); +#if HWY_IS_DEBUG_BUILD + const Full256<TI> di; + HWY_DASSERT(AllFalse(di, Lt(vec, Zero(di))) && + AllTrue(di, Lt(vec, Set(di, static_cast<TI>(32 / sizeof(T)))))); +#endif + return Indices256<T>{vec.raw}; +} + +// 64-bit lanes: convert indices to 8x32 unless AVX3 is available +template <typename T, typename TI, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Indices256<T> IndicesFromVec(Full256<T> d, Vec256<TI> idx64) { + static_assert(sizeof(T) == sizeof(TI), "Index size must match lane"); + const Rebind<TI, decltype(d)> di; + (void)di; // potentially unused +#if HWY_IS_DEBUG_BUILD + HWY_DASSERT(AllFalse(di, Lt(idx64, Zero(di))) && + AllTrue(di, Lt(idx64, Set(di, static_cast<TI>(32 / sizeof(T)))))); +#endif + +#if HWY_TARGET <= HWY_AVX3 + (void)d; + return Indices256<T>{idx64.raw}; +#else + const Repartition<float, decltype(d)> df; // 32-bit! + // Replicate 64-bit index into upper 32 bits + const Vec256<TI> dup = + BitCast(di, Vec256<float>{_mm256_moveldup_ps(BitCast(df, idx64).raw)}); + // For each idx64 i, idx32 are 2*i and 2*i+1. + const Vec256<TI> idx32 = dup + dup + Set(di, TI(1) << 32); + return Indices256<T>{idx32.raw}; +#endif +} + +template <typename T, typename TI> +HWY_API Indices256<T> SetTableIndices(const Full256<T> d, const TI* idx) { + const Rebind<TI, decltype(d)> di; + return IndicesFromVec(d, LoadU(di, idx)); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec256<T> TableLookupLanes(Vec256<T> v, Indices256<T> idx) { + return Vec256<T>{_mm256_permutevar8x32_epi32(v.raw, idx.raw)}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec256<T> TableLookupLanes(Vec256<T> v, Indices256<T> idx) { +#if HWY_TARGET <= HWY_AVX3 + return Vec256<T>{_mm256_permutexvar_epi64(idx.raw, v.raw)}; +#else + return Vec256<T>{_mm256_permutevar8x32_epi32(v.raw, idx.raw)}; +#endif +} + +HWY_API Vec256<float> TableLookupLanes(const Vec256<float> v, + const Indices256<float> idx) { + return Vec256<float>{_mm256_permutevar8x32_ps(v.raw, idx.raw)}; +} + +HWY_API Vec256<double> TableLookupLanes(const Vec256<double> v, + const Indices256<double> idx) { +#if HWY_TARGET <= HWY_AVX3 + return Vec256<double>{_mm256_permutexvar_pd(idx.raw, v.raw)}; +#else + const Full256<double> df; + const Full256<uint64_t> du; + return BitCast(df, Vec256<uint64_t>{_mm256_permutevar8x32_epi32( + BitCast(du, v).raw, idx.raw)}); +#endif +} + +// ------------------------------ Reverse (RotateRight) + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec256<T> Reverse(Full256<T> d, const Vec256<T> v) { + alignas(32) constexpr int32_t kReverse[8] = {7, 6, 5, 4, 3, 2, 1, 0}; + return TableLookupLanes(v, SetTableIndices(d, kReverse)); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec256<T> Reverse(Full256<T> d, const Vec256<T> v) { + alignas(32) constexpr int64_t kReverse[4] = {3, 2, 1, 0}; + return TableLookupLanes(v, SetTableIndices(d, kReverse)); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Vec256<T> Reverse(Full256<T> d, const Vec256<T> v) { +#if HWY_TARGET <= HWY_AVX3 + const RebindToSigned<decltype(d)> di; + alignas(32) constexpr int16_t kReverse[16] = {15, 14, 13, 12, 11, 10, 9, 8, + 7, 6, 5, 4, 3, 2, 1, 0}; + const Vec256<int16_t> idx = Load(di, kReverse); + return BitCast(d, Vec256<int16_t>{ + _mm256_permutexvar_epi16(idx.raw, BitCast(di, v).raw)}); +#else + const RepartitionToWide<RebindToUnsigned<decltype(d)>> du32; + const Vec256<uint32_t> rev32 = Reverse(du32, BitCast(du32, v)); + return BitCast(d, RotateRight<16>(rev32)); +#endif +} + +// ------------------------------ InterleaveLower + +// Interleaves lanes from halves of the 128-bit blocks of "a" (which provides +// the least-significant lane) and "b". To concatenate two half-width integers +// into one, use ZipLower/Upper instead (also works with scalar). + +HWY_API Vec256<uint8_t> InterleaveLower(const Vec256<uint8_t> a, + const Vec256<uint8_t> b) { + return Vec256<uint8_t>{_mm256_unpacklo_epi8(a.raw, b.raw)}; +} +HWY_API Vec256<uint16_t> InterleaveLower(const Vec256<uint16_t> a, + const Vec256<uint16_t> b) { + return Vec256<uint16_t>{_mm256_unpacklo_epi16(a.raw, b.raw)}; +} +HWY_API Vec256<uint32_t> InterleaveLower(const Vec256<uint32_t> a, + const Vec256<uint32_t> b) { + return Vec256<uint32_t>{_mm256_unpacklo_epi32(a.raw, b.raw)}; +} +HWY_API Vec256<uint64_t> InterleaveLower(const Vec256<uint64_t> a, + const Vec256<uint64_t> b) { + return Vec256<uint64_t>{_mm256_unpacklo_epi64(a.raw, b.raw)}; +} + +HWY_API Vec256<int8_t> InterleaveLower(const Vec256<int8_t> a, + const Vec256<int8_t> b) { + return Vec256<int8_t>{_mm256_unpacklo_epi8(a.raw, b.raw)}; +} +HWY_API Vec256<int16_t> InterleaveLower(const Vec256<int16_t> a, + const Vec256<int16_t> b) { + return Vec256<int16_t>{_mm256_unpacklo_epi16(a.raw, b.raw)}; +} +HWY_API Vec256<int32_t> InterleaveLower(const Vec256<int32_t> a, + const Vec256<int32_t> b) { + return Vec256<int32_t>{_mm256_unpacklo_epi32(a.raw, b.raw)}; +} +HWY_API Vec256<int64_t> InterleaveLower(const Vec256<int64_t> a, + const Vec256<int64_t> b) { + return Vec256<int64_t>{_mm256_unpacklo_epi64(a.raw, b.raw)}; +} + +HWY_API Vec256<float> InterleaveLower(const Vec256<float> a, + const Vec256<float> b) { + return Vec256<float>{_mm256_unpacklo_ps(a.raw, b.raw)}; +} +HWY_API Vec256<double> InterleaveLower(const Vec256<double> a, + const Vec256<double> b) { + return Vec256<double>{_mm256_unpacklo_pd(a.raw, b.raw)}; +} + +// Additional overload for the optional Simd<> tag. +template <typename T, class V = Vec256<T>> +HWY_API V InterleaveLower(Full256<T> /* tag */, V a, V b) { + return InterleaveLower(a, b); +} + +// ------------------------------ InterleaveUpper + +// All functions inside detail lack the required D parameter. +namespace detail { + +HWY_API Vec256<uint8_t> InterleaveUpper(const Vec256<uint8_t> a, + const Vec256<uint8_t> b) { + return Vec256<uint8_t>{_mm256_unpackhi_epi8(a.raw, b.raw)}; +} +HWY_API Vec256<uint16_t> InterleaveUpper(const Vec256<uint16_t> a, + const Vec256<uint16_t> b) { + return Vec256<uint16_t>{_mm256_unpackhi_epi16(a.raw, b.raw)}; +} +HWY_API Vec256<uint32_t> InterleaveUpper(const Vec256<uint32_t> a, + const Vec256<uint32_t> b) { + return Vec256<uint32_t>{_mm256_unpackhi_epi32(a.raw, b.raw)}; +} +HWY_API Vec256<uint64_t> InterleaveUpper(const Vec256<uint64_t> a, + const Vec256<uint64_t> b) { + return Vec256<uint64_t>{_mm256_unpackhi_epi64(a.raw, b.raw)}; +} + +HWY_API Vec256<int8_t> InterleaveUpper(const Vec256<int8_t> a, + const Vec256<int8_t> b) { + return Vec256<int8_t>{_mm256_unpackhi_epi8(a.raw, b.raw)}; +} +HWY_API Vec256<int16_t> InterleaveUpper(const Vec256<int16_t> a, + const Vec256<int16_t> b) { + return Vec256<int16_t>{_mm256_unpackhi_epi16(a.raw, b.raw)}; +} +HWY_API Vec256<int32_t> InterleaveUpper(const Vec256<int32_t> a, + const Vec256<int32_t> b) { + return Vec256<int32_t>{_mm256_unpackhi_epi32(a.raw, b.raw)}; +} +HWY_API Vec256<int64_t> InterleaveUpper(const Vec256<int64_t> a, + const Vec256<int64_t> b) { + return Vec256<int64_t>{_mm256_unpackhi_epi64(a.raw, b.raw)}; +} + +HWY_API Vec256<float> InterleaveUpper(const Vec256<float> a, + const Vec256<float> b) { + return Vec256<float>{_mm256_unpackhi_ps(a.raw, b.raw)}; +} +HWY_API Vec256<double> InterleaveUpper(const Vec256<double> a, + const Vec256<double> b) { + return Vec256<double>{_mm256_unpackhi_pd(a.raw, b.raw)}; +} + +} // namespace detail + +template <typename T, class V = Vec256<T>> +HWY_API V InterleaveUpper(Full256<T> /* tag */, V a, V b) { + return detail::InterleaveUpper(a, b); +} + +// ------------------------------ ZipLower/ZipUpper (InterleaveLower) + +// Same as Interleave*, except that the return lanes are double-width integers; +// this is necessary because the single-lane scalar cannot return two values. +template <typename T, typename TW = MakeWide<T>> +HWY_API Vec256<TW> ZipLower(Vec256<T> a, Vec256<T> b) { + return BitCast(Full256<TW>(), InterleaveLower(Full256<T>(), a, b)); +} +template <typename T, typename TW = MakeWide<T>> +HWY_API Vec256<TW> ZipLower(Full256<TW> dw, Vec256<T> a, Vec256<T> b) { + return BitCast(dw, InterleaveLower(Full256<T>(), a, b)); +} + +template <typename T, typename TW = MakeWide<T>> +HWY_API Vec256<TW> ZipUpper(Full256<TW> dw, Vec256<T> a, Vec256<T> b) { + return BitCast(dw, InterleaveUpper(Full256<T>(), a, b)); +} + +// ------------------------------ Blocks (LowerHalf, ZeroExtendVector) + +// _mm256_broadcastsi128_si256 has 7 cycle latency. _mm256_permute2x128_si256 is +// slow on Zen1 (8 uops); we can avoid it for LowerLower and UpperLower, and on +// UpperUpper at the cost of one extra cycle/instruction. + +// hiH,hiL loH,loL |-> hiL,loL (= lower halves) +template <typename T> +HWY_API Vec256<T> ConcatLowerLower(Full256<T> d, const Vec256<T> hi, + const Vec256<T> lo) { + const Half<decltype(d)> d2; + return Vec256<T>{_mm256_inserti128_si256(lo.raw, LowerHalf(d2, hi).raw, 1)}; +} +HWY_API Vec256<float> ConcatLowerLower(Full256<float> d, const Vec256<float> hi, + const Vec256<float> lo) { + const Half<decltype(d)> d2; + return Vec256<float>{_mm256_insertf128_ps(lo.raw, LowerHalf(d2, hi).raw, 1)}; +} +HWY_API Vec256<double> ConcatLowerLower(Full256<double> d, + const Vec256<double> hi, + const Vec256<double> lo) { + const Half<decltype(d)> d2; + return Vec256<double>{_mm256_insertf128_pd(lo.raw, LowerHalf(d2, hi).raw, 1)}; +} + +// hiH,hiL loH,loL |-> hiL,loH (= inner halves / swap blocks) +template <typename T> +HWY_API Vec256<T> ConcatLowerUpper(Full256<T> /* tag */, const Vec256<T> hi, + const Vec256<T> lo) { + return Vec256<T>{_mm256_permute2x128_si256(lo.raw, hi.raw, 0x21)}; +} +HWY_API Vec256<float> ConcatLowerUpper(Full256<float> /* tag */, + const Vec256<float> hi, + const Vec256<float> lo) { + return Vec256<float>{_mm256_permute2f128_ps(lo.raw, hi.raw, 0x21)}; +} +HWY_API Vec256<double> ConcatLowerUpper(Full256<double> /* tag */, + const Vec256<double> hi, + const Vec256<double> lo) { + return Vec256<double>{_mm256_permute2f128_pd(lo.raw, hi.raw, 0x21)}; +} + +// hiH,hiL loH,loL |-> hiH,loL (= outer halves) +template <typename T> +HWY_API Vec256<T> ConcatUpperLower(Full256<T> /* tag */, const Vec256<T> hi, + const Vec256<T> lo) { + return Vec256<T>{_mm256_blend_epi32(hi.raw, lo.raw, 0x0F)}; +} +HWY_API Vec256<float> ConcatUpperLower(Full256<float> /* tag */, + const Vec256<float> hi, + const Vec256<float> lo) { + return Vec256<float>{_mm256_blend_ps(hi.raw, lo.raw, 0x0F)}; +} +HWY_API Vec256<double> ConcatUpperLower(Full256<double> /* tag */, + const Vec256<double> hi, + const Vec256<double> lo) { + return Vec256<double>{_mm256_blend_pd(hi.raw, lo.raw, 3)}; +} + +// hiH,hiL loH,loL |-> hiH,loH (= upper halves) +template <typename T> +HWY_API Vec256<T> ConcatUpperUpper(Full256<T> d, const Vec256<T> hi, + const Vec256<T> lo) { + const Half<decltype(d)> d2; + return ConcatUpperLower(d, hi, ZeroExtendVector(d, UpperHalf(d2, lo))); +} + +// ------------------------------ ConcatOdd + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec256<T> ConcatOdd(Full256<T> d, Vec256<T> hi, Vec256<T> lo) { + const RebindToUnsigned<decltype(d)> du; +#if HWY_TARGET <= HWY_AVX3 + alignas(32) constexpr uint32_t kIdx[8] = {1, 3, 5, 7, 9, 11, 13, 15}; + return BitCast(d, Vec256<uint32_t>{_mm256_mask2_permutex2var_epi32( + BitCast(du, lo).raw, Load(du, kIdx).raw, __mmask8{0xFF}, + BitCast(du, hi).raw)}); +#else + const RebindToFloat<decltype(d)> df; + const Vec256<float> v3131{_mm256_shuffle_ps( + BitCast(df, lo).raw, BitCast(df, hi).raw, _MM_SHUFFLE(3, 1, 3, 1))}; + return Vec256<T>{_mm256_permute4x64_epi64(BitCast(du, v3131).raw, + _MM_SHUFFLE(3, 1, 2, 0))}; +#endif +} + +HWY_API Vec256<float> ConcatOdd(Full256<float> d, Vec256<float> hi, + Vec256<float> lo) { + const RebindToUnsigned<decltype(d)> du; +#if HWY_TARGET <= HWY_AVX3 + alignas(32) constexpr uint32_t kIdx[8] = {1, 3, 5, 7, 9, 11, 13, 15}; + return Vec256<float>{_mm256_mask2_permutex2var_ps(lo.raw, Load(du, kIdx).raw, + __mmask8{0xFF}, hi.raw)}; +#else + const Vec256<float> v3131{ + _mm256_shuffle_ps(lo.raw, hi.raw, _MM_SHUFFLE(3, 1, 3, 1))}; + return BitCast(d, Vec256<uint32_t>{_mm256_permute4x64_epi64( + BitCast(du, v3131).raw, _MM_SHUFFLE(3, 1, 2, 0))}); +#endif +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec256<T> ConcatOdd(Full256<T> d, Vec256<T> hi, Vec256<T> lo) { + const RebindToUnsigned<decltype(d)> du; +#if HWY_TARGET <= HWY_AVX3 + alignas(64) constexpr uint64_t kIdx[4] = {1, 3, 5, 7}; + return BitCast(d, Vec256<uint64_t>{_mm256_mask2_permutex2var_epi64( + BitCast(du, lo).raw, Load(du, kIdx).raw, __mmask8{0xFF}, + BitCast(du, hi).raw)}); +#else + const RebindToFloat<decltype(d)> df; + const Vec256<double> v31{ + _mm256_shuffle_pd(BitCast(df, lo).raw, BitCast(df, hi).raw, 15)}; + return Vec256<T>{ + _mm256_permute4x64_epi64(BitCast(du, v31).raw, _MM_SHUFFLE(3, 1, 2, 0))}; +#endif +} + +HWY_API Vec256<double> ConcatOdd(Full256<double> d, Vec256<double> hi, + Vec256<double> lo) { +#if HWY_TARGET <= HWY_AVX3 + const RebindToUnsigned<decltype(d)> du; + alignas(64) constexpr uint64_t kIdx[4] = {1, 3, 5, 7}; + return Vec256<double>{_mm256_mask2_permutex2var_pd(lo.raw, Load(du, kIdx).raw, + __mmask8{0xFF}, hi.raw)}; +#else + (void)d; + const Vec256<double> v31{_mm256_shuffle_pd(lo.raw, hi.raw, 15)}; + return Vec256<double>{ + _mm256_permute4x64_pd(v31.raw, _MM_SHUFFLE(3, 1, 2, 0))}; +#endif +} + +// ------------------------------ ConcatEven + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec256<T> ConcatEven(Full256<T> d, Vec256<T> hi, Vec256<T> lo) { + const RebindToUnsigned<decltype(d)> du; +#if HWY_TARGET <= HWY_AVX3 + alignas(64) constexpr uint32_t kIdx[8] = {0, 2, 4, 6, 8, 10, 12, 14}; + return BitCast(d, Vec256<uint32_t>{_mm256_mask2_permutex2var_epi32( + BitCast(du, lo).raw, Load(du, kIdx).raw, __mmask8{0xFF}, + BitCast(du, hi).raw)}); +#else + const RebindToFloat<decltype(d)> df; + const Vec256<float> v2020{_mm256_shuffle_ps( + BitCast(df, lo).raw, BitCast(df, hi).raw, _MM_SHUFFLE(2, 0, 2, 0))}; + return Vec256<T>{_mm256_permute4x64_epi64(BitCast(du, v2020).raw, + _MM_SHUFFLE(3, 1, 2, 0))}; + +#endif +} + +HWY_API Vec256<float> ConcatEven(Full256<float> d, Vec256<float> hi, + Vec256<float> lo) { + const RebindToUnsigned<decltype(d)> du; +#if HWY_TARGET <= HWY_AVX3 + alignas(64) constexpr uint32_t kIdx[8] = {0, 2, 4, 6, 8, 10, 12, 14}; + return Vec256<float>{_mm256_mask2_permutex2var_ps(lo.raw, Load(du, kIdx).raw, + __mmask8{0xFF}, hi.raw)}; +#else + const Vec256<float> v2020{ + _mm256_shuffle_ps(lo.raw, hi.raw, _MM_SHUFFLE(2, 0, 2, 0))}; + return BitCast(d, Vec256<uint32_t>{_mm256_permute4x64_epi64( + BitCast(du, v2020).raw, _MM_SHUFFLE(3, 1, 2, 0))}); + +#endif +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec256<T> ConcatEven(Full256<T> d, Vec256<T> hi, Vec256<T> lo) { + const RebindToUnsigned<decltype(d)> du; +#if HWY_TARGET <= HWY_AVX3 + alignas(64) constexpr uint64_t kIdx[4] = {0, 2, 4, 6}; + return BitCast(d, Vec256<uint64_t>{_mm256_mask2_permutex2var_epi64( + BitCast(du, lo).raw, Load(du, kIdx).raw, __mmask8{0xFF}, + BitCast(du, hi).raw)}); +#else + const RebindToFloat<decltype(d)> df; + const Vec256<double> v20{ + _mm256_shuffle_pd(BitCast(df, lo).raw, BitCast(df, hi).raw, 0)}; + return Vec256<T>{ + _mm256_permute4x64_epi64(BitCast(du, v20).raw, _MM_SHUFFLE(3, 1, 2, 0))}; + +#endif +} + +HWY_API Vec256<double> ConcatEven(Full256<double> d, Vec256<double> hi, + Vec256<double> lo) { +#if HWY_TARGET <= HWY_AVX3 + const RebindToUnsigned<decltype(d)> du; + alignas(64) constexpr uint64_t kIdx[4] = {0, 2, 4, 6}; + return Vec256<double>{_mm256_mask2_permutex2var_pd(lo.raw, Load(du, kIdx).raw, + __mmask8{0xFF}, hi.raw)}; +#else + (void)d; + const Vec256<double> v20{_mm256_shuffle_pd(lo.raw, hi.raw, 0)}; + return Vec256<double>{ + _mm256_permute4x64_pd(v20.raw, _MM_SHUFFLE(3, 1, 2, 0))}; +#endif +} + +// ------------------------------ OddEven + +namespace detail { + +template <typename T> +HWY_INLINE Vec256<T> OddEven(hwy::SizeTag<1> /* tag */, const Vec256<T> a, + const Vec256<T> b) { + const Full256<T> d; + const Full256<uint8_t> d8; + alignas(32) constexpr uint8_t mask[16] = {0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0, + 0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0}; + return IfThenElse(MaskFromVec(BitCast(d, LoadDup128(d8, mask))), b, a); +} +template <typename T> +HWY_INLINE Vec256<T> OddEven(hwy::SizeTag<2> /* tag */, const Vec256<T> a, + const Vec256<T> b) { + return Vec256<T>{_mm256_blend_epi16(a.raw, b.raw, 0x55)}; +} +template <typename T> +HWY_INLINE Vec256<T> OddEven(hwy::SizeTag<4> /* tag */, const Vec256<T> a, + const Vec256<T> b) { + return Vec256<T>{_mm256_blend_epi32(a.raw, b.raw, 0x55)}; +} +template <typename T> +HWY_INLINE Vec256<T> OddEven(hwy::SizeTag<8> /* tag */, const Vec256<T> a, + const Vec256<T> b) { + return Vec256<T>{_mm256_blend_epi32(a.raw, b.raw, 0x33)}; +} + +} // namespace detail + +template <typename T> +HWY_API Vec256<T> OddEven(const Vec256<T> a, const Vec256<T> b) { + return detail::OddEven(hwy::SizeTag<sizeof(T)>(), a, b); +} +HWY_API Vec256<float> OddEven(const Vec256<float> a, const Vec256<float> b) { + return Vec256<float>{_mm256_blend_ps(a.raw, b.raw, 0x55)}; +} + +HWY_API Vec256<double> OddEven(const Vec256<double> a, const Vec256<double> b) { + return Vec256<double>{_mm256_blend_pd(a.raw, b.raw, 5)}; +} + +// ------------------------------ OddEvenBlocks + +template <typename T> +Vec256<T> OddEvenBlocks(Vec256<T> odd, Vec256<T> even) { + return Vec256<T>{_mm256_blend_epi32(odd.raw, even.raw, 0xFu)}; +} + +HWY_API Vec256<float> OddEvenBlocks(Vec256<float> odd, Vec256<float> even) { + return Vec256<float>{_mm256_blend_ps(odd.raw, even.raw, 0xFu)}; +} + +HWY_API Vec256<double> OddEvenBlocks(Vec256<double> odd, Vec256<double> even) { + return Vec256<double>{_mm256_blend_pd(odd.raw, even.raw, 0x3u)}; +} + +// ------------------------------ SwapAdjacentBlocks + +template <typename T> +HWY_API Vec256<T> SwapAdjacentBlocks(Vec256<T> v) { + return Vec256<T>{_mm256_permute4x64_epi64(v.raw, _MM_SHUFFLE(1, 0, 3, 2))}; +} + +HWY_API Vec256<float> SwapAdjacentBlocks(Vec256<float> v) { + const Full256<float> df; + const Full256<int32_t> di; + // Avoid _mm256_permute2f128_ps - slow on AMD. + return BitCast(df, Vec256<int32_t>{_mm256_permute4x64_epi64( + BitCast(di, v).raw, _MM_SHUFFLE(1, 0, 3, 2))}); +} + +HWY_API Vec256<double> SwapAdjacentBlocks(Vec256<double> v) { + return Vec256<double>{_mm256_permute4x64_pd(v.raw, _MM_SHUFFLE(1, 0, 3, 2))}; +} + +// ------------------------------ TableLookupBytes (ZeroExtendVector) + +// Both full +template <typename T, typename TI> +HWY_API Vec256<TI> TableLookupBytes(const Vec256<T> bytes, + const Vec256<TI> from) { + return Vec256<TI>{_mm256_shuffle_epi8(bytes.raw, from.raw)}; +} + +// Partial index vector +template <typename T, typename TI, size_t NI> +HWY_API Vec128<TI, NI> TableLookupBytes(const Vec256<T> bytes, + const Vec128<TI, NI> from) { + // First expand to full 128, then 256. + const auto from_256 = ZeroExtendVector(Full256<TI>(), Vec128<TI>{from.raw}); + const auto tbl_full = TableLookupBytes(bytes, from_256); + // Shrink to 128, then partial. + return Vec128<TI, NI>{LowerHalf(Full128<TI>(), tbl_full).raw}; +} + +// Partial table vector +template <typename T, size_t N, typename TI> +HWY_API Vec256<TI> TableLookupBytes(const Vec128<T, N> bytes, + const Vec256<TI> from) { + // First expand to full 128, then 256. + const auto bytes_256 = ZeroExtendVector(Full256<T>(), Vec128<T>{bytes.raw}); + return TableLookupBytes(bytes_256, from); +} + +// Partial both are handled by x86_128. + +// ------------------------------ Shl (Mul, ZipLower) + +#if HWY_TARGET > HWY_AVX3 // AVX2 or older +namespace detail { + +// Returns 2^v for use as per-lane multipliers to emulate 16-bit shifts. +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_INLINE Vec256<MakeUnsigned<T>> Pow2(const Vec256<T> v) { + const Full256<T> d; + const RepartitionToWide<decltype(d)> dw; + const Rebind<float, decltype(dw)> df; + const auto zero = Zero(d); + // Move into exponent (this u16 will become the upper half of an f32) + const auto exp = ShiftLeft<23 - 16>(v); + const auto upper = exp + Set(d, 0x3F80); // upper half of 1.0f + // Insert 0 into lower halves for reinterpreting as binary32. + const auto f0 = ZipLower(dw, zero, upper); + const auto f1 = ZipUpper(dw, zero, upper); + // Do not use ConvertTo because it checks for overflow, which is redundant + // because we only care about v in [0, 16). + const Vec256<int32_t> bits0{_mm256_cvttps_epi32(BitCast(df, f0).raw)}; + const Vec256<int32_t> bits1{_mm256_cvttps_epi32(BitCast(df, f1).raw)}; + return Vec256<MakeUnsigned<T>>{_mm256_packus_epi32(bits0.raw, bits1.raw)}; +} + +} // namespace detail +#endif // HWY_TARGET > HWY_AVX3 + +HWY_API Vec256<uint16_t> operator<<(const Vec256<uint16_t> v, + const Vec256<uint16_t> bits) { +#if HWY_TARGET <= HWY_AVX3 + return Vec256<uint16_t>{_mm256_sllv_epi16(v.raw, bits.raw)}; +#else + return v * detail::Pow2(bits); +#endif +} + +HWY_API Vec256<uint32_t> operator<<(const Vec256<uint32_t> v, + const Vec256<uint32_t> bits) { + return Vec256<uint32_t>{_mm256_sllv_epi32(v.raw, bits.raw)}; +} + +HWY_API Vec256<uint64_t> operator<<(const Vec256<uint64_t> v, + const Vec256<uint64_t> bits) { + return Vec256<uint64_t>{_mm256_sllv_epi64(v.raw, bits.raw)}; +} + +// Signed left shift is the same as unsigned. +template <typename T, HWY_IF_SIGNED(T)> +HWY_API Vec256<T> operator<<(const Vec256<T> v, const Vec256<T> bits) { + const Full256<T> di; + const Full256<MakeUnsigned<T>> du; + return BitCast(di, BitCast(du, v) << BitCast(du, bits)); +} + +// ------------------------------ Shr (MulHigh, IfThenElse, Not) + +HWY_API Vec256<uint16_t> operator>>(const Vec256<uint16_t> v, + const Vec256<uint16_t> bits) { +#if HWY_TARGET <= HWY_AVX3 + return Vec256<uint16_t>{_mm256_srlv_epi16(v.raw, bits.raw)}; +#else + const Full256<uint16_t> d; + // For bits=0, we cannot mul by 2^16, so fix the result later. + const auto out = MulHigh(v, detail::Pow2(Set(d, 16) - bits)); + // Replace output with input where bits == 0. + return IfThenElse(bits == Zero(d), v, out); +#endif +} + +HWY_API Vec256<uint32_t> operator>>(const Vec256<uint32_t> v, + const Vec256<uint32_t> bits) { + return Vec256<uint32_t>{_mm256_srlv_epi32(v.raw, bits.raw)}; +} + +HWY_API Vec256<uint64_t> operator>>(const Vec256<uint64_t> v, + const Vec256<uint64_t> bits) { + return Vec256<uint64_t>{_mm256_srlv_epi64(v.raw, bits.raw)}; +} + +HWY_API Vec256<int16_t> operator>>(const Vec256<int16_t> v, + const Vec256<int16_t> bits) { +#if HWY_TARGET <= HWY_AVX3 + return Vec256<int16_t>{_mm256_srav_epi16(v.raw, bits.raw)}; +#else + return detail::SignedShr(Full256<int16_t>(), v, bits); +#endif +} + +HWY_API Vec256<int32_t> operator>>(const Vec256<int32_t> v, + const Vec256<int32_t> bits) { + return Vec256<int32_t>{_mm256_srav_epi32(v.raw, bits.raw)}; +} + +HWY_API Vec256<int64_t> operator>>(const Vec256<int64_t> v, + const Vec256<int64_t> bits) { +#if HWY_TARGET <= HWY_AVX3 + return Vec256<int64_t>{_mm256_srav_epi64(v.raw, bits.raw)}; +#else + return detail::SignedShr(Full256<int64_t>(), v, bits); +#endif +} + +HWY_INLINE Vec256<uint64_t> MulEven(const Vec256<uint64_t> a, + const Vec256<uint64_t> b) { + const DFromV<decltype(a)> du64; + const RepartitionToNarrow<decltype(du64)> du32; + const auto maskL = Set(du64, 0xFFFFFFFFULL); + const auto a32 = BitCast(du32, a); + const auto b32 = BitCast(du32, b); + // Inputs for MulEven: we only need the lower 32 bits + const auto aH = Shuffle2301(a32); + const auto bH = Shuffle2301(b32); + + // Knuth double-word multiplication. We use 32x32 = 64 MulEven and only need + // the even (lower 64 bits of every 128-bit block) results. See + // https://github.com/hcs0/Hackers-Delight/blob/master/muldwu.c.tat + const auto aLbL = MulEven(a32, b32); + const auto w3 = aLbL & maskL; + + const auto t2 = MulEven(aH, b32) + ShiftRight<32>(aLbL); + const auto w2 = t2 & maskL; + const auto w1 = ShiftRight<32>(t2); + + const auto t = MulEven(a32, bH) + w2; + const auto k = ShiftRight<32>(t); + + const auto mulH = MulEven(aH, bH) + w1 + k; + const auto mulL = ShiftLeft<32>(t) + w3; + return InterleaveLower(mulL, mulH); +} + +HWY_INLINE Vec256<uint64_t> MulOdd(const Vec256<uint64_t> a, + const Vec256<uint64_t> b) { + const DFromV<decltype(a)> du64; + const RepartitionToNarrow<decltype(du64)> du32; + const auto maskL = Set(du64, 0xFFFFFFFFULL); + const auto a32 = BitCast(du32, a); + const auto b32 = BitCast(du32, b); + // Inputs for MulEven: we only need bits [95:64] (= upper half of input) + const auto aH = Shuffle2301(a32); + const auto bH = Shuffle2301(b32); + + // Same as above, but we're using the odd results (upper 64 bits per block). + const auto aLbL = MulEven(a32, b32); + const auto w3 = aLbL & maskL; + + const auto t2 = MulEven(aH, b32) + ShiftRight<32>(aLbL); + const auto w2 = t2 & maskL; + const auto w1 = ShiftRight<32>(t2); + + const auto t = MulEven(a32, bH) + w2; + const auto k = ShiftRight<32>(t); + + const auto mulH = MulEven(aH, bH) + w1 + k; + const auto mulL = ShiftLeft<32>(t) + w3; + return InterleaveUpper(du64, mulL, mulH); +} + +// ------------------------------ ReorderWidenMulAccumulate (MulAdd, ZipLower) + +HWY_API Vec256<float> ReorderWidenMulAccumulate(Full256<float> df32, + Vec256<bfloat16_t> a, + Vec256<bfloat16_t> b, + const Vec256<float> sum0, + Vec256<float>& sum1) { + // TODO(janwas): _mm256_dpbf16_ps when available + const Repartition<uint16_t, decltype(df32)> du16; + const RebindToUnsigned<decltype(df32)> du32; + const Vec256<uint16_t> zero = Zero(du16); + // Lane order within sum0/1 is undefined, hence we can avoid the + // longer-latency lane-crossing PromoteTo. + const Vec256<uint32_t> a0 = ZipLower(du32, zero, BitCast(du16, a)); + const Vec256<uint32_t> a1 = ZipUpper(du32, zero, BitCast(du16, a)); + const Vec256<uint32_t> b0 = ZipLower(du32, zero, BitCast(du16, b)); + const Vec256<uint32_t> b1 = ZipUpper(du32, zero, BitCast(du16, b)); + sum1 = MulAdd(BitCast(df32, a1), BitCast(df32, b1), sum1); + return MulAdd(BitCast(df32, a0), BitCast(df32, b0), sum0); +} + +// ================================================== CONVERT + +// ------------------------------ Promotions (part w/ narrow lanes -> full) + +HWY_API Vec256<double> PromoteTo(Full256<double> /* tag */, + const Vec128<float, 4> v) { + return Vec256<double>{_mm256_cvtps_pd(v.raw)}; +} + +HWY_API Vec256<double> PromoteTo(Full256<double> /* tag */, + const Vec128<int32_t, 4> v) { + return Vec256<double>{_mm256_cvtepi32_pd(v.raw)}; +} + +// Unsigned: zero-extend. +// Note: these have 3 cycle latency; if inputs are already split across the +// 128 bit blocks (in their upper/lower halves), then Zip* would be faster. +HWY_API Vec256<uint16_t> PromoteTo(Full256<uint16_t> /* tag */, + Vec128<uint8_t> v) { + return Vec256<uint16_t>{_mm256_cvtepu8_epi16(v.raw)}; +} +HWY_API Vec256<uint32_t> PromoteTo(Full256<uint32_t> /* tag */, + Vec128<uint8_t, 8> v) { + return Vec256<uint32_t>{_mm256_cvtepu8_epi32(v.raw)}; +} +HWY_API Vec256<int16_t> PromoteTo(Full256<int16_t> /* tag */, + Vec128<uint8_t> v) { + return Vec256<int16_t>{_mm256_cvtepu8_epi16(v.raw)}; +} +HWY_API Vec256<int32_t> PromoteTo(Full256<int32_t> /* tag */, + Vec128<uint8_t, 8> v) { + return Vec256<int32_t>{_mm256_cvtepu8_epi32(v.raw)}; +} +HWY_API Vec256<uint32_t> PromoteTo(Full256<uint32_t> /* tag */, + Vec128<uint16_t> v) { + return Vec256<uint32_t>{_mm256_cvtepu16_epi32(v.raw)}; +} +HWY_API Vec256<int32_t> PromoteTo(Full256<int32_t> /* tag */, + Vec128<uint16_t> v) { + return Vec256<int32_t>{_mm256_cvtepu16_epi32(v.raw)}; +} +HWY_API Vec256<uint64_t> PromoteTo(Full256<uint64_t> /* tag */, + Vec128<uint32_t> v) { + return Vec256<uint64_t>{_mm256_cvtepu32_epi64(v.raw)}; +} + +// Signed: replicate sign bit. +// Note: these have 3 cycle latency; if inputs are already split across the +// 128 bit blocks (in their upper/lower halves), then ZipUpper/lo followed by +// signed shift would be faster. +HWY_API Vec256<int16_t> PromoteTo(Full256<int16_t> /* tag */, + Vec128<int8_t> v) { + return Vec256<int16_t>{_mm256_cvtepi8_epi16(v.raw)}; +} +HWY_API Vec256<int32_t> PromoteTo(Full256<int32_t> /* tag */, + Vec128<int8_t, 8> v) { + return Vec256<int32_t>{_mm256_cvtepi8_epi32(v.raw)}; +} +HWY_API Vec256<int32_t> PromoteTo(Full256<int32_t> /* tag */, + Vec128<int16_t> v) { + return Vec256<int32_t>{_mm256_cvtepi16_epi32(v.raw)}; +} +HWY_API Vec256<int64_t> PromoteTo(Full256<int64_t> /* tag */, + Vec128<int32_t> v) { + return Vec256<int64_t>{_mm256_cvtepi32_epi64(v.raw)}; +} + +// ------------------------------ Demotions (full -> part w/ narrow lanes) + +HWY_API Vec128<uint16_t> DemoteTo(Full128<uint16_t> /* tag */, + const Vec256<int32_t> v) { + const __m256i u16 = _mm256_packus_epi32(v.raw, v.raw); + // Concatenating lower halves of both 128-bit blocks afterward is more + // efficient than an extra input with low block = high block of v. + return Vec128<uint16_t>{ + _mm256_castsi256_si128(_mm256_permute4x64_epi64(u16, 0x88))}; +} + +HWY_API Vec128<int16_t> DemoteTo(Full128<int16_t> /* tag */, + const Vec256<int32_t> v) { + const __m256i i16 = _mm256_packs_epi32(v.raw, v.raw); + return Vec128<int16_t>{ + _mm256_castsi256_si128(_mm256_permute4x64_epi64(i16, 0x88))}; +} + +HWY_API Vec128<uint8_t, 8> DemoteTo(Simd<uint8_t, 8> /* tag */, + const Vec256<int32_t> v) { + const __m256i u16_blocks = _mm256_packus_epi32(v.raw, v.raw); + // Concatenate lower 64 bits of each 128-bit block + const __m256i u16_concat = _mm256_permute4x64_epi64(u16_blocks, 0x88); + const __m128i u16 = _mm256_castsi256_si128(u16_concat); + // packus treats the input as signed; we want unsigned. Clear the MSB to get + // unsigned saturation to u8. + const __m128i i16 = _mm_and_si128(u16, _mm_set1_epi16(0x7FFF)); + return Vec128<uint8_t, 8>{_mm_packus_epi16(i16, i16)}; +} + +HWY_API Vec128<uint8_t> DemoteTo(Full128<uint8_t> /* tag */, + const Vec256<int16_t> v) { + const __m256i u8 = _mm256_packus_epi16(v.raw, v.raw); + return Vec128<uint8_t>{ + _mm256_castsi256_si128(_mm256_permute4x64_epi64(u8, 0x88))}; +} + +HWY_API Vec128<int8_t, 8> DemoteTo(Simd<int8_t, 8> /* tag */, + const Vec256<int32_t> v) { + const __m256i i16_blocks = _mm256_packs_epi32(v.raw, v.raw); + // Concatenate lower 64 bits of each 128-bit block + const __m256i i16_concat = _mm256_permute4x64_epi64(i16_blocks, 0x88); + const __m128i i16 = _mm256_castsi256_si128(i16_concat); + return Vec128<int8_t, 8>{_mm_packs_epi16(i16, i16)}; +} + +HWY_API Vec128<int8_t> DemoteTo(Full128<int8_t> /* tag */, + const Vec256<int16_t> v) { + const __m256i i8 = _mm256_packs_epi16(v.raw, v.raw); + return Vec128<int8_t>{ + _mm256_castsi256_si128(_mm256_permute4x64_epi64(i8, 0x88))}; +} + + // Avoid "value of intrinsic immediate argument '8' is out of range '0 - 7'". + // 8 is the correct value of _MM_FROUND_NO_EXC, which is allowed here. +HWY_DIAGNOSTICS(push) +HWY_DIAGNOSTICS_OFF(disable : 4556, ignored "-Wsign-conversion") + +HWY_API Vec128<float16_t> DemoteTo(Full128<float16_t> df16, + const Vec256<float> v) { +#ifdef HWY_DISABLE_F16C + const RebindToUnsigned<decltype(df16)> du16; + const Rebind<uint32_t, decltype(df16)> du; + const RebindToSigned<decltype(du)> di; + const auto bits32 = BitCast(du, v); + const auto sign = ShiftRight<31>(bits32); + const auto biased_exp32 = ShiftRight<23>(bits32) & Set(du, 0xFF); + const auto mantissa32 = bits32 & Set(du, 0x7FFFFF); + + const auto k15 = Set(di, 15); + const auto exp = Min(BitCast(di, biased_exp32) - Set(di, 127), k15); + const auto is_tiny = exp < Set(di, -24); + + const auto is_subnormal = exp < Set(di, -14); + const auto biased_exp16 = + BitCast(du, IfThenZeroElse(is_subnormal, exp + k15)); + const auto sub_exp = BitCast(du, Set(di, -14) - exp); // [1, 11) + const auto sub_m = (Set(du, 1) << (Set(du, 10) - sub_exp)) + + (mantissa32 >> (Set(du, 13) + sub_exp)); + const auto mantissa16 = IfThenElse(RebindMask(du, is_subnormal), sub_m, + ShiftRight<13>(mantissa32)); // <1024 + + const auto sign16 = ShiftLeft<15>(sign); + const auto normal16 = sign16 | ShiftLeft<10>(biased_exp16) | mantissa16; + const auto bits16 = IfThenZeroElse(is_tiny, BitCast(di, normal16)); + return BitCast(df16, DemoteTo(du16, bits16)); +#else + (void)df16; + return Vec128<float16_t>{_mm256_cvtps_ph(v.raw, _MM_FROUND_NO_EXC)}; +#endif +} + +HWY_DIAGNOSTICS(pop) + +HWY_API Vec128<bfloat16_t> DemoteTo(Full128<bfloat16_t> dbf16, + const Vec256<float> v) { + // TODO(janwas): _mm256_cvtneps_pbh once we have avx512bf16. + const Rebind<int32_t, decltype(dbf16)> di32; + const Rebind<uint32_t, decltype(dbf16)> du32; // for logical shift right + const Rebind<uint16_t, decltype(dbf16)> du16; + const auto bits_in_32 = BitCast(di32, ShiftRight<16>(BitCast(du32, v))); + return BitCast(dbf16, DemoteTo(du16, bits_in_32)); +} + +HWY_API Vec256<bfloat16_t> ReorderDemote2To(Full256<bfloat16_t> dbf16, + Vec256<float> a, Vec256<float> b) { + // TODO(janwas): _mm256_cvtne2ps_pbh once we have avx512bf16. + const RebindToUnsigned<decltype(dbf16)> du16; + const Repartition<uint32_t, decltype(dbf16)> du32; + const Vec256<uint32_t> b_in_even = ShiftRight<16>(BitCast(du32, b)); + return BitCast(dbf16, OddEven(BitCast(du16, a), BitCast(du16, b_in_even))); +} + +HWY_API Vec128<float> DemoteTo(Full128<float> /* tag */, + const Vec256<double> v) { + return Vec128<float>{_mm256_cvtpd_ps(v.raw)}; +} + +HWY_API Vec128<int32_t> DemoteTo(Full128<int32_t> /* tag */, + const Vec256<double> v) { + const auto clamped = detail::ClampF64ToI32Max(Full256<double>(), v); + return Vec128<int32_t>{_mm256_cvttpd_epi32(clamped.raw)}; +} + +// For already range-limited input [0, 255]. +HWY_API Vec128<uint8_t, 8> U8FromU32(const Vec256<uint32_t> v) { + const Full256<uint32_t> d32; + alignas(32) static constexpr uint32_t k8From32[8] = { + 0x0C080400u, ~0u, ~0u, ~0u, ~0u, 0x0C080400u, ~0u, ~0u}; + // Place first four bytes in lo[0], remaining 4 in hi[1]. + const auto quad = TableLookupBytes(v, Load(d32, k8From32)); + // Interleave both quadruplets - OR instead of unpack reduces port5 pressure. + const auto lo = LowerHalf(quad); + const auto hi = UpperHalf(Full128<uint32_t>(), quad); + const auto pair = LowerHalf(lo | hi); + return BitCast(Simd<uint8_t, 8>(), pair); +} + +// ------------------------------ Integer <=> fp (ShiftRight, OddEven) + +HWY_API Vec256<float> ConvertTo(Full256<float> /* tag */, + const Vec256<int32_t> v) { + return Vec256<float>{_mm256_cvtepi32_ps(v.raw)}; +} + +HWY_API Vec256<double> ConvertTo(Full256<double> dd, const Vec256<int64_t> v) { +#if HWY_TARGET <= HWY_AVX3 + (void)dd; + return Vec256<double>{_mm256_cvtepi64_pd(v.raw)}; +#else + // Based on wim's approach (https://stackoverflow.com/questions/41144668/) + const Repartition<uint32_t, decltype(dd)> d32; + const Repartition<uint64_t, decltype(dd)> d64; + + // Toggle MSB of lower 32-bits and insert exponent for 2^84 + 2^63 + const auto k84_63 = Set(d64, 0x4530000080000000ULL); + const auto v_upper = BitCast(dd, ShiftRight<32>(BitCast(d64, v)) ^ k84_63); + + // Exponent is 2^52, lower 32 bits from v (=> 32-bit OddEven) + const auto k52 = Set(d32, 0x43300000); + const auto v_lower = BitCast(dd, OddEven(k52, BitCast(d32, v))); + + const auto k84_63_52 = BitCast(dd, Set(d64, 0x4530000080100000ULL)); + return (v_upper - k84_63_52) + v_lower; // order matters! +#endif +} + +// Truncates (rounds toward zero). +HWY_API Vec256<int32_t> ConvertTo(Full256<int32_t> d, const Vec256<float> v) { + return detail::FixConversionOverflow(d, v, _mm256_cvttps_epi32(v.raw)); +} + +HWY_API Vec256<int64_t> ConvertTo(Full256<int64_t> di, const Vec256<double> v) { +#if HWY_TARGET <= HWY_AVX3 + return detail::FixConversionOverflow(di, v, _mm256_cvttpd_epi64(v.raw)); +#else + using VI = decltype(Zero(di)); + const VI k0 = Zero(di); + const VI k1 = Set(di, 1); + const VI k51 = Set(di, 51); + + // Exponent indicates whether the number can be represented as int64_t. + const VI biased_exp = ShiftRight<52>(BitCast(di, v)) & Set(di, 0x7FF); + const VI exp = biased_exp - Set(di, 0x3FF); + const auto in_range = exp < Set(di, 63); + + // If we were to cap the exponent at 51 and add 2^52, the number would be in + // [2^52, 2^53) and mantissa bits could be read out directly. We need to + // round-to-0 (truncate), but changing rounding mode in MXCSR hits a + // compiler reordering bug: https://gcc.godbolt.org/z/4hKj6c6qc . We instead + // manually shift the mantissa into place (we already have many of the + // inputs anyway). + const VI shift_mnt = Max(k51 - exp, k0); + const VI shift_int = Max(exp - k51, k0); + const VI mantissa = BitCast(di, v) & Set(di, (1ULL << 52) - 1); + // Include implicit 1-bit; shift by one more to ensure it's in the mantissa. + const VI int52 = (mantissa | Set(di, 1ULL << 52)) >> (shift_mnt + k1); + // For inputs larger than 2^52, insert zeros at the bottom. + const VI shifted = int52 << shift_int; + // Restore the one bit lost when shifting in the implicit 1-bit. + const VI restored = shifted | ((mantissa & k1) << (shift_int - k1)); + + // Saturate to LimitsMin (unchanged when negating below) or LimitsMax. + const VI sign_mask = BroadcastSignBit(BitCast(di, v)); + const VI limit = Set(di, LimitsMax<int64_t>()) - sign_mask; + const VI magnitude = IfThenElse(in_range, restored, limit); + + // If the input was negative, negate the integer (two's complement). + return (magnitude ^ sign_mask) - sign_mask; +#endif +} + +HWY_API Vec256<int32_t> NearestInt(const Vec256<float> v) { + const Full256<int32_t> di; + return detail::FixConversionOverflow(di, v, _mm256_cvtps_epi32(v.raw)); +} + + +HWY_API Vec256<float> PromoteTo(Full256<float> df32, + const Vec128<float16_t> v) { +#ifdef HWY_DISABLE_F16C + const RebindToSigned<decltype(df32)> di32; + const RebindToUnsigned<decltype(df32)> du32; + // Expand to u32 so we can shift. + const auto bits16 = PromoteTo(du32, Vec128<uint16_t>{v.raw}); + const auto sign = ShiftRight<15>(bits16); + const auto biased_exp = ShiftRight<10>(bits16) & Set(du32, 0x1F); + const auto mantissa = bits16 & Set(du32, 0x3FF); + const auto subnormal = + BitCast(du32, ConvertTo(df32, BitCast(di32, mantissa)) * + Set(df32, 1.0f / 16384 / 1024)); + + const auto biased_exp32 = biased_exp + Set(du32, 127 - 15); + const auto mantissa32 = ShiftLeft<23 - 10>(mantissa); + const auto normal = ShiftLeft<23>(biased_exp32) | mantissa32; + const auto bits32 = IfThenElse(biased_exp == Zero(du32), subnormal, normal); + return BitCast(df32, ShiftLeft<31>(sign) | bits32); +#else + (void)df32; + return Vec256<float>{_mm256_cvtph_ps(v.raw)}; +#endif +} + +HWY_API Vec256<float> PromoteTo(Full256<float> df32, + const Vec128<bfloat16_t> v) { + const Rebind<uint16_t, decltype(df32)> du16; + const RebindToSigned<decltype(df32)> di32; + return BitCast(df32, ShiftLeft<16>(PromoteTo(di32, BitCast(du16, v)))); +} + +// ================================================== CRYPTO + +#if !defined(HWY_DISABLE_PCLMUL_AES) + +// Per-target flag to prevent generic_ops-inl.h from defining AESRound. +#ifdef HWY_NATIVE_AES +#undef HWY_NATIVE_AES +#else +#define HWY_NATIVE_AES +#endif + +HWY_API Vec256<uint8_t> AESRound(Vec256<uint8_t> state, + Vec256<uint8_t> round_key) { +#if HWY_TARGET == HWY_AVX3_DL + return Vec256<uint8_t>{_mm256_aesenc_epi128(state.raw, round_key.raw)}; +#else + const Full256<uint8_t> d; + const Half<decltype(d)> d2; + return Combine(d, AESRound(UpperHalf(d2, state), UpperHalf(d2, round_key)), + AESRound(LowerHalf(state), LowerHalf(round_key))); +#endif +} + +HWY_API Vec256<uint64_t> CLMulLower(Vec256<uint64_t> a, Vec256<uint64_t> b) { +#if HWY_TARGET == HWY_AVX3_DL + return Vec256<uint64_t>{_mm256_clmulepi64_epi128(a.raw, b.raw, 0x00)}; +#else + const Full256<uint64_t> d; + const Half<decltype(d)> d2; + return Combine(d, CLMulLower(UpperHalf(d2, a), UpperHalf(d2, b)), + CLMulLower(LowerHalf(a), LowerHalf(b))); +#endif +} + +HWY_API Vec256<uint64_t> CLMulUpper(Vec256<uint64_t> a, Vec256<uint64_t> b) { +#if HWY_TARGET == HWY_AVX3_DL + return Vec256<uint64_t>{_mm256_clmulepi64_epi128(a.raw, b.raw, 0x11)}; +#else + const Full256<uint64_t> d; + const Half<decltype(d)> d2; + return Combine(d, CLMulUpper(UpperHalf(d2, a), UpperHalf(d2, b)), + CLMulUpper(LowerHalf(a), LowerHalf(b))); +#endif +} + +#endif // HWY_DISABLE_PCLMUL_AES + +// ================================================== MISC + +// Returns a vector with lane i=[0, N) set to "first" + i. +template <typename T, typename T2> +HWY_API Vec256<T> Iota(const Full256<T> d, const T2 first) { + HWY_ALIGN T lanes[32 / sizeof(T)]; + for (size_t i = 0; i < 32 / sizeof(T); ++i) { + lanes[i] = static_cast<T>(first + static_cast<T2>(i)); + } + return Load(d, lanes); +} + +#if HWY_TARGET <= HWY_AVX3 + +// ------------------------------ LoadMaskBits + +// `p` points to at least 8 readable bytes, not all of which need be valid. +template <typename T> +HWY_API Mask256<T> LoadMaskBits(const Full256<T> /* tag */, + const uint8_t* HWY_RESTRICT bits) { + constexpr size_t N = 32 / sizeof(T); + constexpr size_t kNumBytes = (N + 7) / 8; + + uint64_t mask_bits = 0; + CopyBytes<kNumBytes>(bits, &mask_bits); + + if (N < 8) { + mask_bits &= (1ull << N) - 1; + } + + return Mask256<T>::FromBits(mask_bits); +} + +// ------------------------------ StoreMaskBits + +// `p` points to at least 8 writable bytes. +template <typename T> +HWY_API size_t StoreMaskBits(const Full256<T> /* tag */, const Mask256<T> mask, + uint8_t* bits) { + constexpr size_t N = 32 / sizeof(T); + constexpr size_t kNumBytes = (N + 7) / 8; + + CopyBytes<kNumBytes>(&mask.raw, bits); + + // Non-full byte, need to clear the undefined upper bits. + if (N < 8) { + const int mask = static_cast<int>((1ull << N) - 1); + bits[0] = static_cast<uint8_t>(bits[0] & mask); + } + return kNumBytes; +} + +// ------------------------------ Mask testing + +template <typename T> +HWY_API size_t CountTrue(const Full256<T> /* tag */, const Mask256<T> mask) { + return PopCount(static_cast<uint64_t>(mask.raw)); +} + +template <typename T> +HWY_API intptr_t FindFirstTrue(const Full256<T> /* tag */, + const Mask256<T> mask) { + return mask.raw ? intptr_t(Num0BitsBelowLS1Bit_Nonzero32(mask.raw)) : -1; +} + +// Beware: the suffix indicates the number of mask bits, not lane size! + +namespace detail { + +template <typename T> +HWY_INLINE bool AllFalse(hwy::SizeTag<1> /*tag*/, const Mask256<T> mask) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return _kortestz_mask32_u8(mask.raw, mask.raw); +#else + return mask.raw == 0; +#endif +} +template <typename T> +HWY_INLINE bool AllFalse(hwy::SizeTag<2> /*tag*/, const Mask256<T> mask) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return _kortestz_mask16_u8(mask.raw, mask.raw); +#else + return mask.raw == 0; +#endif +} +template <typename T> +HWY_INLINE bool AllFalse(hwy::SizeTag<4> /*tag*/, const Mask256<T> mask) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return _kortestz_mask8_u8(mask.raw, mask.raw); +#else + return mask.raw == 0; +#endif +} +template <typename T> +HWY_INLINE bool AllFalse(hwy::SizeTag<8> /*tag*/, const Mask256<T> mask) { + return (uint64_t{mask.raw} & 0xF) == 0; +} + +} // namespace detail + +template <typename T> +HWY_API bool AllFalse(const Full256<T> /* tag */, const Mask256<T> mask) { + return detail::AllFalse(hwy::SizeTag<sizeof(T)>(), mask); +} + +namespace detail { + +template <typename T> +HWY_INLINE bool AllTrue(hwy::SizeTag<1> /*tag*/, const Mask256<T> mask) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return _kortestc_mask32_u8(mask.raw, mask.raw); +#else + return mask.raw == 0xFFFFFFFFu; +#endif +} +template <typename T> +HWY_INLINE bool AllTrue(hwy::SizeTag<2> /*tag*/, const Mask256<T> mask) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return _kortestc_mask16_u8(mask.raw, mask.raw); +#else + return mask.raw == 0xFFFFu; +#endif +} +template <typename T> +HWY_INLINE bool AllTrue(hwy::SizeTag<4> /*tag*/, const Mask256<T> mask) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return _kortestc_mask8_u8(mask.raw, mask.raw); +#else + return mask.raw == 0xFFu; +#endif +} +template <typename T> +HWY_INLINE bool AllTrue(hwy::SizeTag<8> /*tag*/, const Mask256<T> mask) { + // Cannot use _kortestc because we have less than 8 mask bits. + return mask.raw == 0xFu; +} + +} // namespace detail + +template <typename T> +HWY_API bool AllTrue(const Full256<T> /* tag */, const Mask256<T> mask) { + return detail::AllTrue(hwy::SizeTag<sizeof(T)>(), mask); +} + +// ------------------------------ Compress + +// 16-bit is defined in x86_512 so we can use 512-bit vectors. + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec256<T> Compress(Vec256<T> v, Mask256<T> mask) { + return Vec256<T>{_mm256_maskz_compress_epi32(mask.raw, v.raw)}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec256<T> Compress(Vec256<T> v, Mask256<T> mask) { + return Vec256<T>{_mm256_maskz_compress_epi64(mask.raw, v.raw)}; +} + +HWY_API Vec256<float> Compress(Vec256<float> v, Mask256<float> mask) { + return Vec256<float>{_mm256_maskz_compress_ps(mask.raw, v.raw)}; +} + +HWY_API Vec256<double> Compress(Vec256<double> v, Mask256<double> mask) { + return Vec256<double>{_mm256_maskz_compress_pd(mask.raw, v.raw)}; +} + +// ------------------------------ CompressBits (LoadMaskBits) + +template <typename T> +HWY_API Vec256<T> CompressBits(Vec256<T> v, const uint8_t* HWY_RESTRICT bits) { + return Compress(v, LoadMaskBits(Full256<T>(), bits)); +} + +// ------------------------------ CompressStore + +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API size_t CompressStore(Vec256<T> v, Mask256<T> mask, Full256<T> d, + T* HWY_RESTRICT unaligned) { + const Rebind<uint16_t, decltype(d)> du; + const auto vu = BitCast(du, v); // (required for float16_t inputs) + + const uint64_t mask_bits{mask.raw}; + +#if HWY_TARGET == HWY_AVX3_DL // VBMI2 + _mm256_mask_compressstoreu_epi16(unaligned, mask.raw, v.raw); +#else + // Split into halves to keep the table size manageable. + const Half<decltype(du)> duh; + const auto vL = LowerHalf(duh, vu); + const auto vH = UpperHalf(duh, vu); + + const uint64_t mask_bitsL = mask_bits & 0xFF; + const uint64_t mask_bitsH = mask_bits >> 8; + + const auto idxL = detail::IndicesForCompress16(mask_bitsL); + const auto idxH = detail::IndicesForCompress16(mask_bitsH); + + // Compress and 128-bit halves. + const Vec128<uint16_t> cL{_mm_permutexvar_epi16(idxL.raw, vL.raw)}; + const Vec128<uint16_t> cH{_mm_permutexvar_epi16(idxH.raw, vH.raw)}; + const Half<decltype(d)> dh; + StoreU(BitCast(dh, cL), dh, unaligned); + StoreU(BitCast(dh, cH), dh, unaligned + PopCount(mask_bitsL)); +#endif // HWY_TARGET == HWY_AVX3_DL + + return PopCount(mask_bits); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API size_t CompressStore(Vec256<T> v, Mask256<T> mask, Full256<T> /* tag */, + T* HWY_RESTRICT unaligned) { + _mm256_mask_compressstoreu_epi32(unaligned, mask.raw, v.raw); + return PopCount(uint64_t{mask.raw}); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API size_t CompressStore(Vec256<T> v, Mask256<T> mask, Full256<T> /* tag */, + T* HWY_RESTRICT unaligned) { + _mm256_mask_compressstoreu_epi64(unaligned, mask.raw, v.raw); + return PopCount(uint64_t{mask.raw} & 0xFull); +} + +HWY_API size_t CompressStore(Vec256<float> v, Mask256<float> mask, + Full256<float> /* tag */, + float* HWY_RESTRICT unaligned) { + _mm256_mask_compressstoreu_ps(unaligned, mask.raw, v.raw); + return PopCount(uint64_t{mask.raw}); +} + +HWY_API size_t CompressStore(Vec256<double> v, Mask256<double> mask, + Full256<double> /* tag */, + double* HWY_RESTRICT unaligned) { + _mm256_mask_compressstoreu_pd(unaligned, mask.raw, v.raw); + return PopCount(uint64_t{mask.raw} & 0xFull); +} + +// ------------------------------ CompressBlendedStore (CompressStore) + +#if HWY_TARGET == HWY_AVX2 +namespace detail { + +// Intel SDM says "No AC# reported for any mask bit combinations". However, AMD +// allows AC# if "Alignment checking enabled and: 256-bit memory operand not +// 32-byte aligned". Fortunately AC# is not enabled by default and requires both +// OS support (CR0) and the application to set rflags.AC. We assume these remain +// disabled because x86/x64 code and compiler output often contain misaligned +// scalar accesses, which would also fault. +// +// Caveat: these are slow on AMD Jaguar/Bulldozer. + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API void MaskedStore(Mask256<T> m, Vec256<T> v, Full256<T> /* tag */, + T* HWY_RESTRICT unaligned) { + auto unaligned_p = reinterpret_cast<int*>(aligned); // NOLINT + _mm256_maskstore_epi32(unaligned_p, m.raw, v.raw); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API void MaskedStore(Mask256<T> m, Vec256<T> v, Full256<T> /* tag */, + T* HWY_RESTRICT unaligned) { + auto unaligned_p = reinterpret_cast<long long*>(aligned); // NOLINT + _mm256_maskstore_epi64(unaligned_p, m.raw, v.raw); +} + +HWY_API void MaskedStore(Mask256<float> m, Vec256<float> v, Full256<float> d, + float* HWY_RESTRICT unaligned) { + const Vec256<int32_t> mi = + BitCast(RebindToSigned<decltype(d)>(), VecFromMask(d, m)); + _mm256_maskstore_ps(unaligned, mi.raw, v.raw); +} + +HWY_API void MaskedStore(Mask256<double> m, Vec256<double> v, Full256<double> d, + double* HWY_RESTRICT unaligned) { + const Vec256<int64_t> mi = + BitCast(RebindToSigned<decltype(d)>(), VecFromMask(d, m)); + _mm256_maskstore_pd(unaligned, mi.raw, v.raw); +} + +// There is no maskstore_epi8/16, so blend instead. +template <typename T, hwy::EnableIf<sizeof(T) <= 2>* = nullptr> +HWY_API void MaskedStore(Mask256<T> m, Vec256<T> v, Full256<T> d, + T* HWY_RESTRICT unaligned) { + StoreU(IfThenElse(m, v, LoadU(d, unaligned)), d, unaligned); +} + +} // namespace detail +#endif // HWY_TARGET == HWY_AVX2 + +#if HWY_TARGET <= HWY_AVX3 + +template <typename T, HWY_IF_NOT_LANE_SIZE(T, 2)> +HWY_API size_t CompressBlendedStore(Vec256<T> v, Mask256<T> m, Full256<T> d, + T* HWY_RESTRICT unaligned) { + // Native (32 or 64-bit) AVX-512 instruction already does the blending at no + // extra cost (latency 11, rthroughput 2 - same as compress plus store). + return CompressStore(v, m, d, unaligned); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API size_t CompressBlendedStore(Vec256<T> v, Mask256<T> m, Full256<T> d, + T* HWY_RESTRICT unaligned) { +#if HWY_TARGET <= HWY_AVX3_DL + return CompressStore(v, m, d, unaligned); // also native +#else + const size_t count = CountTrue(m); + const Vec256<T> compressed = Compress(v, m); + // There is no 16-bit MaskedStore, so blend. + const Vec256<T> prev = LoadU(d, unaligned); + StoreU(IfThenElse(FirstN(d, count), compressed, prev), d, unaligned); + return count; +#endif +} + +#else // AVX2 + +template <typename T, HWY_IF_NOT_LANE_SIZE(T, 2)> +HWY_API size_t CompressBlendedStore(Vec256<T> v, Mask256<T> m, Full256<T> d, + T* HWY_RESTRICT unaligned) { + const size_t count = CountTrue(m); + detail::MaskedStore(FirstN(d, count), d, Compress(v, m)); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API size_t CompressBlendedStore(Vec256<T> v, Mask256<T> m, Full256<T> d, + T* HWY_RESTRICT unaligned) { + // There is no 16-bit MaskedStore, so blend. + const size_t count = CountTrue(m); + const Vec256<T> compressed = Compress(v, m); + const Vec256<T> prev = LoadU(d, unaligned); + StoreU(IfThenElse(FirstN(d, count), compressed, prev), d, unaligned); + return count; +} + +#endif // AVX2 + +// ------------------------------ CompressBitsStore (LoadMaskBits) + +template <typename T> +HWY_API size_t CompressBitsStore(Vec256<T> v, const uint8_t* HWY_RESTRICT bits, + Full256<T> d, T* HWY_RESTRICT unaligned) { + return CompressStore(v, LoadMaskBits(d, bits), d, unaligned); +} + +#else // AVX2 + +// ------------------------------ LoadMaskBits (TestBit) + +namespace detail { + +// 256 suffix avoids ambiguity with x86_128 without needing HWY_IF_LE128 there. +template <typename T, HWY_IF_LANE_SIZE(T, 1)> +HWY_INLINE Mask256<T> LoadMaskBits256(Full256<T> d, uint64_t mask_bits) { + const RebindToUnsigned<decltype(d)> du; + const Repartition<uint32_t, decltype(d)> du32; + const auto vbits = BitCast(du, Set(du32, static_cast<uint32_t>(mask_bits))); + + // Replicate bytes 8x such that each byte contains the bit that governs it. + const Repartition<uint64_t, decltype(d)> du64; + alignas(32) constexpr uint64_t kRep8[4] = { + 0x0000000000000000ull, 0x0101010101010101ull, 0x0202020202020202ull, + 0x0303030303030303ull}; + const auto rep8 = TableLookupBytes(vbits, BitCast(du, Load(du64, kRep8))); + + alignas(32) constexpr uint8_t kBit[16] = {1, 2, 4, 8, 16, 32, 64, 128, + 1, 2, 4, 8, 16, 32, 64, 128}; + return RebindMask(d, TestBit(rep8, LoadDup128(du, kBit))); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_INLINE Mask256<T> LoadMaskBits256(Full256<T> d, uint64_t mask_bits) { + const RebindToUnsigned<decltype(d)> du; + alignas(32) constexpr uint16_t kBit[16] = { + 1, 2, 4, 8, 16, 32, 64, 128, + 0x100, 0x200, 0x400, 0x800, 0x1000, 0x2000, 0x4000, 0x8000}; + const auto vmask_bits = Set(du, static_cast<uint16_t>(mask_bits)); + return RebindMask(d, TestBit(vmask_bits, Load(du, kBit))); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_INLINE Mask256<T> LoadMaskBits256(Full256<T> d, uint64_t mask_bits) { + const RebindToUnsigned<decltype(d)> du; + alignas(32) constexpr uint32_t kBit[8] = {1, 2, 4, 8, 16, 32, 64, 128}; + const auto vmask_bits = Set(du, static_cast<uint32_t>(mask_bits)); + return RebindMask(d, TestBit(vmask_bits, Load(du, kBit))); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_INLINE Mask256<T> LoadMaskBits256(Full256<T> d, uint64_t mask_bits) { + const RebindToUnsigned<decltype(d)> du; + alignas(32) constexpr uint64_t kBit[8] = {1, 2, 4, 8}; + return RebindMask(d, TestBit(Set(du, mask_bits), Load(du, kBit))); +} + +} // namespace detail + +// `p` points to at least 8 readable bytes, not all of which need be valid. +template <typename T> +HWY_API Mask256<T> LoadMaskBits(Full256<T> d, + const uint8_t* HWY_RESTRICT bits) { + constexpr size_t N = 32 / sizeof(T); + constexpr size_t kNumBytes = (N + 7) / 8; + + uint64_t mask_bits = 0; + CopyBytes<kNumBytes>(bits, &mask_bits); + + if (N < 8) { + mask_bits &= (1ull << N) - 1; + } + + return detail::LoadMaskBits256(d, mask_bits); +} + +// ------------------------------ StoreMaskBits + +namespace detail { + +template <typename T, HWY_IF_LANE_SIZE(T, 1)> +HWY_INLINE uint64_t BitsFromMask(const Mask256<T> mask) { + const Full256<T> d; + const Full256<uint8_t> d8; + const auto sign_bits = BitCast(d8, VecFromMask(d, mask)).raw; + // Prevent sign-extension of 32-bit masks because the intrinsic returns int. + return static_cast<uint32_t>(_mm256_movemask_epi8(sign_bits)); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_INLINE uint64_t BitsFromMask(const Mask256<T> mask) { +#if HWY_ARCH_X86_64 + const Full256<T> d; + const Full256<uint8_t> d8; + const Mask256<uint8_t> mask8 = MaskFromVec(BitCast(d8, VecFromMask(d, mask))); + const uint64_t sign_bits8 = BitsFromMask(mask8); + // Skip the bits from the lower byte of each u16 (better not to use the + // same packs_epi16 as SSE4, because that requires an extra swizzle here). + return _pext_u64(sign_bits8, 0xAAAAAAAAull); +#else + // Slow workaround for 32-bit builds, which lack _pext_u64. + // Remove useless lower half of each u16 while preserving the sign bit. + // Bytes [0, 8) and [16, 24) have the same sign bits as the input lanes. + const auto sign_bits = _mm256_packs_epi16(mask.raw, _mm256_setzero_si256()); + // Move odd qwords (value zero) to top so they don't affect the mask value. + const auto compressed = + _mm256_permute4x64_epi64(sign_bits, _MM_SHUFFLE(3, 1, 2, 0)); + return static_cast<unsigned>(_mm256_movemask_epi8(compressed)); +#endif // HWY_ARCH_X86_64 +} + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_INLINE uint64_t BitsFromMask(const Mask256<T> mask) { + const Full256<T> d; + const Full256<float> df; + const auto sign_bits = BitCast(df, VecFromMask(d, mask)).raw; + return static_cast<unsigned>(_mm256_movemask_ps(sign_bits)); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_INLINE uint64_t BitsFromMask(const Mask256<T> mask) { + const Full256<T> d; + const Full256<double> df; + const auto sign_bits = BitCast(df, VecFromMask(d, mask)).raw; + return static_cast<unsigned>(_mm256_movemask_pd(sign_bits)); +} + +} // namespace detail + +// `p` points to at least 8 writable bytes. +template <typename T> +HWY_API size_t StoreMaskBits(const Full256<T> /* tag */, const Mask256<T> mask, + uint8_t* bits) { + constexpr size_t N = 32 / sizeof(T); + constexpr size_t kNumBytes = (N + 7) / 8; + + const uint64_t mask_bits = detail::BitsFromMask(mask); + CopyBytes<kNumBytes>(&mask_bits, bits); + return kNumBytes; +} + +// ------------------------------ Mask testing + +// Specialize for 16-bit lanes to avoid unnecessary pext. This assumes each mask +// lane is 0 or ~0. +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API bool AllFalse(const Full256<T> d, const Mask256<T> mask) { + const Repartition<uint8_t, decltype(d)> d8; + const Mask256<uint8_t> mask8 = MaskFromVec(BitCast(d8, VecFromMask(d, mask))); + return detail::BitsFromMask(mask8) == 0; +} + +template <typename T, HWY_IF_NOT_LANE_SIZE(T, 2)> +HWY_API bool AllFalse(const Full256<T> /* tag */, const Mask256<T> mask) { + // Cheaper than PTEST, which is 2 uop / 3L. + return detail::BitsFromMask(mask) == 0; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API bool AllTrue(const Full256<T> d, const Mask256<T> mask) { + const Repartition<uint8_t, decltype(d)> d8; + const Mask256<uint8_t> mask8 = MaskFromVec(BitCast(d8, VecFromMask(d, mask))); + return detail::BitsFromMask(mask8) == (1ull << 32) - 1; +} +template <typename T, HWY_IF_NOT_LANE_SIZE(T, 2)> +HWY_API bool AllTrue(const Full256<T> /* tag */, const Mask256<T> mask) { + constexpr uint64_t kAllBits = (1ull << (32 / sizeof(T))) - 1; + return detail::BitsFromMask(mask) == kAllBits; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API size_t CountTrue(const Full256<T> d, const Mask256<T> mask) { + const Repartition<uint8_t, decltype(d)> d8; + const Mask256<uint8_t> mask8 = MaskFromVec(BitCast(d8, VecFromMask(d, mask))); + return PopCount(detail::BitsFromMask(mask8)) >> 1; +} +template <typename T, HWY_IF_NOT_LANE_SIZE(T, 2)> +HWY_API size_t CountTrue(const Full256<T> /* tag */, const Mask256<T> mask) { + return PopCount(detail::BitsFromMask(mask)); +} + +template <typename T> +HWY_API intptr_t FindFirstTrue(const Full256<T> /* tag */, + const Mask256<T> mask) { + const uint64_t mask_bits = detail::BitsFromMask(mask); + return mask_bits ? intptr_t(Num0BitsBelowLS1Bit_Nonzero64(mask_bits)) : -1; +} + +// ------------------------------ Compress, CompressBits + +namespace detail { + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_INLINE Indices256<uint32_t> IndicesFromBits(Simd<T, 8> d, + uint64_t mask_bits) { + const RebindToUnsigned<decltype(d)> d32; + // We need a masked Iota(). With 8 lanes, there are 256 combinations and a LUT + // of SetTableIndices would require 8 KiB, a large part of L1D. The other + // alternative is _pext_u64, but this is extremely slow on Zen2 (18 cycles) + // and unavailable in 32-bit builds. We instead compress each index into 4 + // bits, for a total of 1 KiB. + alignas(16) constexpr uint32_t packed_array[256] = { + 0x00000000, 0x00000000, 0x00000001, 0x00000010, 0x00000002, 0x00000020, + 0x00000021, 0x00000210, 0x00000003, 0x00000030, 0x00000031, 0x00000310, + 0x00000032, 0x00000320, 0x00000321, 0x00003210, 0x00000004, 0x00000040, + 0x00000041, 0x00000410, 0x00000042, 0x00000420, 0x00000421, 0x00004210, + 0x00000043, 0x00000430, 0x00000431, 0x00004310, 0x00000432, 0x00004320, + 0x00004321, 0x00043210, 0x00000005, 0x00000050, 0x00000051, 0x00000510, + 0x00000052, 0x00000520, 0x00000521, 0x00005210, 0x00000053, 0x00000530, + 0x00000531, 0x00005310, 0x00000532, 0x00005320, 0x00005321, 0x00053210, + 0x00000054, 0x00000540, 0x00000541, 0x00005410, 0x00000542, 0x00005420, + 0x00005421, 0x00054210, 0x00000543, 0x00005430, 0x00005431, 0x00054310, + 0x00005432, 0x00054320, 0x00054321, 0x00543210, 0x00000006, 0x00000060, + 0x00000061, 0x00000610, 0x00000062, 0x00000620, 0x00000621, 0x00006210, + 0x00000063, 0x00000630, 0x00000631, 0x00006310, 0x00000632, 0x00006320, + 0x00006321, 0x00063210, 0x00000064, 0x00000640, 0x00000641, 0x00006410, + 0x00000642, 0x00006420, 0x00006421, 0x00064210, 0x00000643, 0x00006430, + 0x00006431, 0x00064310, 0x00006432, 0x00064320, 0x00064321, 0x00643210, + 0x00000065, 0x00000650, 0x00000651, 0x00006510, 0x00000652, 0x00006520, + 0x00006521, 0x00065210, 0x00000653, 0x00006530, 0x00006531, 0x00065310, + 0x00006532, 0x00065320, 0x00065321, 0x00653210, 0x00000654, 0x00006540, + 0x00006541, 0x00065410, 0x00006542, 0x00065420, 0x00065421, 0x00654210, + 0x00006543, 0x00065430, 0x00065431, 0x00654310, 0x00065432, 0x00654320, + 0x00654321, 0x06543210, 0x00000007, 0x00000070, 0x00000071, 0x00000710, + 0x00000072, 0x00000720, 0x00000721, 0x00007210, 0x00000073, 0x00000730, + 0x00000731, 0x00007310, 0x00000732, 0x00007320, 0x00007321, 0x00073210, + 0x00000074, 0x00000740, 0x00000741, 0x00007410, 0x00000742, 0x00007420, + 0x00007421, 0x00074210, 0x00000743, 0x00007430, 0x00007431, 0x00074310, + 0x00007432, 0x00074320, 0x00074321, 0x00743210, 0x00000075, 0x00000750, + 0x00000751, 0x00007510, 0x00000752, 0x00007520, 0x00007521, 0x00075210, + 0x00000753, 0x00007530, 0x00007531, 0x00075310, 0x00007532, 0x00075320, + 0x00075321, 0x00753210, 0x00000754, 0x00007540, 0x00007541, 0x00075410, + 0x00007542, 0x00075420, 0x00075421, 0x00754210, 0x00007543, 0x00075430, + 0x00075431, 0x00754310, 0x00075432, 0x00754320, 0x00754321, 0x07543210, + 0x00000076, 0x00000760, 0x00000761, 0x00007610, 0x00000762, 0x00007620, + 0x00007621, 0x00076210, 0x00000763, 0x00007630, 0x00007631, 0x00076310, + 0x00007632, 0x00076320, 0x00076321, 0x00763210, 0x00000764, 0x00007640, + 0x00007641, 0x00076410, 0x00007642, 0x00076420, 0x00076421, 0x00764210, + 0x00007643, 0x00076430, 0x00076431, 0x00764310, 0x00076432, 0x00764320, + 0x00764321, 0x07643210, 0x00000765, 0x00007650, 0x00007651, 0x00076510, + 0x00007652, 0x00076520, 0x00076521, 0x00765210, 0x00007653, 0x00076530, + 0x00076531, 0x00765310, 0x00076532, 0x00765320, 0x00765321, 0x07653210, + 0x00007654, 0x00076540, 0x00076541, 0x00765410, 0x00076542, 0x00765420, + 0x00765421, 0x07654210, 0x00076543, 0x00765430, 0x00765431, 0x07654310, + 0x00765432, 0x07654320, 0x07654321, 0x76543210}; + + // No need to mask because _mm256_permutevar8x32_epi32 ignores bits 3..31. + // Just shift each copy of the 32 bit LUT to extract its 4-bit fields. + // If broadcasting 32-bit from memory incurs the 3-cycle block-crossing + // latency, it may be faster to use LoadDup128 and PSHUFB. + const auto packed = Set(d32, packed_array[mask_bits]); + alignas(32) constexpr uint32_t shifts[8] = {0, 4, 8, 12, 16, 20, 24, 28}; + return Indices256<uint32_t>{(packed >> Load(d32, shifts)).raw}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_INLINE Indices256<uint32_t> IndicesFromBits(Simd<T, 4> d, + uint64_t mask_bits) { + const Repartition<uint32_t, decltype(d)> d32; + + // For 64-bit, we still need 32-bit indices because there is no 64-bit + // permutevar, but there are only 4 lanes, so we can afford to skip the + // unpacking and load the entire index vector directly. + alignas(32) constexpr uint32_t packed_array[128] = { + 0, 1, 0, 1, 0, 1, 0, 1, /**/ 0, 1, 0, 1, 0, 1, 0, 1, // + 2, 3, 0, 1, 0, 1, 0, 1, /**/ 0, 1, 2, 3, 0, 1, 0, 1, // + 4, 5, 0, 1, 0, 1, 0, 1, /**/ 0, 1, 4, 5, 0, 1, 0, 1, // + 2, 3, 4, 5, 0, 1, 0, 1, /**/ 0, 1, 2, 3, 4, 5, 0, 1, // + 6, 7, 0, 1, 0, 1, 0, 1, /**/ 0, 1, 6, 7, 0, 1, 0, 1, // + 2, 3, 6, 7, 0, 1, 0, 1, /**/ 0, 1, 2, 3, 6, 7, 0, 1, // + 4, 5, 6, 7, 0, 1, 0, 1, /**/ 0, 1, 4, 5, 6, 7, 0, 1, + 2, 3, 4, 5, 6, 7, 0, 1, /**/ 0, 1, 2, 3, 4, 5, 6, 7}; + return Indices256<uint32_t>{Load(d32, packed_array + 8 * mask_bits).raw}; +} + +template <typename T, HWY_IF_NOT_LANE_SIZE(T, 2)> +HWY_INLINE Vec256<T> Compress(Vec256<T> v, const uint64_t mask_bits) { + const Full256<T> d; + const Repartition<uint32_t, decltype(d)> du32; + + HWY_DASSERT(mask_bits < (1ull << (32 / sizeof(T)))); + const auto indices = IndicesFromBits(d, mask_bits); + return BitCast(d, TableLookupLanes(BitCast(du32, v), indices)); +} + +// LUTs are infeasible for 2^16 possible masks. Promoting to 32-bit and using +// the native Compress is probably more efficient than 2 LUTs. +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_INLINE Vec256<T> Compress(Vec256<T> v, const uint64_t mask_bits) { + using D = Full256<T>; + const Rebind<uint16_t, D> du; + const Repartition<int32_t, D> dw; + const auto vu16 = BitCast(du, v); // (required for float16_t inputs) + const auto promoted0 = PromoteTo(dw, LowerHalf(vu16)); + const auto promoted1 = PromoteTo(dw, UpperHalf(Half<decltype(du)>(), vu16)); + + const uint64_t mask_bits0 = mask_bits & 0xFF; + const uint64_t mask_bits1 = mask_bits >> 8; + const auto compressed0 = Compress(promoted0, mask_bits0); + const auto compressed1 = Compress(promoted1, mask_bits1); + + const Half<decltype(du)> dh; + const auto demoted0 = ZeroExtendVector(DemoteTo(dh, compressed0)); + const auto demoted1 = ZeroExtendVector(DemoteTo(dh, compressed1)); + + const size_t count0 = PopCount(mask_bits0); + // Now combine by shifting demoted1 up. AVX2 lacks VPERMW, so start with + // VPERMD for shifting at 4 byte granularity. + alignas(32) constexpr int32_t iota4[16] = {0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 2, 3, 4, 5, 6, 7}; + const auto indices = SetTableIndices(dw, iota4 + 8 - count0 / 2); + const auto shift1_multiple4 = + BitCast(du, TableLookupLanes(BitCast(dw, demoted1), indices)); + + // Whole-register unconditional shift by 2 bytes. + // TODO(janwas): slow on AMD, use 2 shifts + permq + OR instead? + const __m256i lo_zz = _mm256_permute2x128_si256(shift1_multiple4.raw, + shift1_multiple4.raw, 0x08); + const auto shift1_multiple2 = + Vec256<uint16_t>{_mm256_alignr_epi8(shift1_multiple4.raw, lo_zz, 14)}; + + // Make the shift conditional on the lower bit of count0. + const auto m_odd = + TestBit(Set(du, static_cast<uint16_t>(count0)), Set(du, 1)); + const auto shifted1 = IfThenElse(m_odd, shift1_multiple2, shift1_multiple4); + + // Blend the lower and shifted upper parts. + constexpr uint16_t on = 0xFFFF; + alignas(32) constexpr uint16_t lower_lanes[32] = {HWY_REP4(on), HWY_REP4(on), + HWY_REP4(on), HWY_REP4(on)}; + const auto m_lower = MaskFromVec(LoadU(du, lower_lanes + 16 - count0)); + return BitCast(D(), IfThenElse(m_lower, demoted0, shifted1)); +} + +} // namespace detail + +template <typename T> +HWY_API Vec256<T> Compress(Vec256<T> v, Mask256<T> m) { + const uint64_t mask_bits = detail::BitsFromMask(m); + return detail::Compress(v, mask_bits); +} + +template <typename T> +HWY_API Vec256<T> CompressBits(Vec256<T> v, const uint8_t* HWY_RESTRICT bits) { + constexpr size_t N = 32 / sizeof(T); + constexpr size_t kNumBytes = (N + 7) / 8; + + uint64_t mask_bits = 0; + CopyBytes<kNumBytes>(bits, &mask_bits); + + if (N < 8) { + mask_bits &= (1ull << N) - 1; + } + + return detail::Compress(v, mask_bits); +} + +// ------------------------------ CompressStore, CompressBitsStore + +template <typename T> +HWY_API size_t CompressStore(Vec256<T> v, Mask256<T> m, Full256<T> d, + T* HWY_RESTRICT unaligned) { + const uint64_t mask_bits = detail::BitsFromMask(m); + StoreU(detail::Compress(v, mask_bits), d, unaligned); + return PopCount(mask_bits); +} + +template <typename T> +HWY_API size_t CompressBlendedStore(Vec256<T> v, Mask256<T> m, Full256<T> d, + T* HWY_RESTRICT unaligned) { + const uint64_t mask_bits = detail::BitsFromMask(m); + const size_t count = PopCount(mask_bits); + const Vec256<T> compress = detail::Compress(v, mask_bits); + const Vec256<T> prev = LoadU(d, unaligned); + StoreU(IfThenElse(FirstN(d, count), compress, prev), d, unaligned); + return count; +} + +template <typename T> +HWY_API size_t CompressBitsStore(Vec256<T> v, const uint8_t* HWY_RESTRICT bits, + Full256<T> d, T* HWY_RESTRICT unaligned) { + constexpr size_t N = 32 / sizeof(T); + constexpr size_t kNumBytes = (N + 7) / 8; + + uint64_t mask_bits = 0; + CopyBytes<kNumBytes>(bits, &mask_bits); + + if (N < 8) { + mask_bits &= (1ull << N) - 1; + } + + StoreU(detail::Compress(v, mask_bits), d, unaligned); + return PopCount(mask_bits); +} + +#endif // HWY_TARGET <= HWY_AVX3 + +// ------------------------------ StoreInterleaved3 (CombineShiftRightBytes, +// TableLookupBytes, ConcatUpperLower) + +HWY_API void StoreInterleaved3(const Vec256<uint8_t> v0, + const Vec256<uint8_t> v1, + const Vec256<uint8_t> v2, Full256<uint8_t> d, + uint8_t* HWY_RESTRICT unaligned) { + const auto k5 = Set(d, 5); + const auto k6 = Set(d, 6); + + // Shuffle (v0,v1,v2) vector bytes to (MSB on left): r5, bgr[4:0]. + // 0x80 so lanes to be filled from other vectors are 0 for blending. + alignas(16) static constexpr uint8_t tbl_r0[16] = { + 0, 0x80, 0x80, 1, 0x80, 0x80, 2, 0x80, 0x80, // + 3, 0x80, 0x80, 4, 0x80, 0x80, 5}; + alignas(16) static constexpr uint8_t tbl_g0[16] = { + 0x80, 0, 0x80, 0x80, 1, 0x80, // + 0x80, 2, 0x80, 0x80, 3, 0x80, 0x80, 4, 0x80, 0x80}; + const auto shuf_r0 = LoadDup128(d, tbl_r0); + const auto shuf_g0 = LoadDup128(d, tbl_g0); // cannot reuse r0 due to 5 + const auto shuf_b0 = CombineShiftRightBytes<15>(d, shuf_g0, shuf_g0); + const auto r0 = TableLookupBytes(v0, shuf_r0); // 5..4..3..2..1..0 + const auto g0 = TableLookupBytes(v1, shuf_g0); // ..4..3..2..1..0. + const auto b0 = TableLookupBytes(v2, shuf_b0); // .4..3..2..1..0.. + const auto interleaved_10_00 = r0 | g0 | b0; + + // Second vector: g10,r10, bgr[9:6], b5,g5 + const auto shuf_r1 = shuf_b0 + k6; // .A..9..8..7..6.. + const auto shuf_g1 = shuf_r0 + k5; // A..9..8..7..6..5 + const auto shuf_b1 = shuf_g0 + k5; // ..9..8..7..6..5. + const auto r1 = TableLookupBytes(v0, shuf_r1); + const auto g1 = TableLookupBytes(v1, shuf_g1); + const auto b1 = TableLookupBytes(v2, shuf_b1); + const auto interleaved_15_05 = r1 | g1 | b1; + + // We want to write the lower halves of the interleaved vectors, then the + // upper halves. We could obtain 10_05 and 15_0A via ConcatUpperLower, but + // that would require two ununaligned stores. For the lower halves, we can + // merge two 128-bit stores for the same swizzling cost: + const auto out0 = ConcatLowerLower(d, interleaved_15_05, interleaved_10_00); + StoreU(out0, d, unaligned + 0 * 32); + + // Third vector: bgr[15:11], b10 + const auto shuf_r2 = shuf_b1 + k6; // ..F..E..D..C..B. + const auto shuf_g2 = shuf_r1 + k5; // .F..E..D..C..B.. + const auto shuf_b2 = shuf_g1 + k5; // F..E..D..C..B..A + const auto r2 = TableLookupBytes(v0, shuf_r2); + const auto g2 = TableLookupBytes(v1, shuf_g2); + const auto b2 = TableLookupBytes(v2, shuf_b2); + const auto interleaved_1A_0A = r2 | g2 | b2; + + const auto out1 = ConcatUpperLower(d, interleaved_10_00, interleaved_1A_0A); + StoreU(out1, d, unaligned + 1 * 32); + + const auto out2 = ConcatUpperUpper(d, interleaved_1A_0A, interleaved_15_05); + StoreU(out2, d, unaligned + 2 * 32); +} + +// ------------------------------ StoreInterleaved4 + +HWY_API void StoreInterleaved4(const Vec256<uint8_t> v0, + const Vec256<uint8_t> v1, + const Vec256<uint8_t> v2, + const Vec256<uint8_t> v3, Full256<uint8_t> d8, + uint8_t* HWY_RESTRICT unaligned) { + const RepartitionToWide<decltype(d8)> d16; + const RepartitionToWide<decltype(d16)> d32; + // let a,b,c,d denote v0..3. + const auto ba0 = ZipLower(d16, v0, v1); // b7 a7 .. b0 a0 + const auto dc0 = ZipLower(d16, v2, v3); // d7 c7 .. d0 c0 + const auto ba8 = ZipUpper(d16, v0, v1); + const auto dc8 = ZipUpper(d16, v2, v3); + const auto dcba_0 = ZipLower(d32, ba0, dc0); // d..a13 d..a10 | d..a03 d..a00 + const auto dcba_4 = ZipUpper(d32, ba0, dc0); // d..a17 d..a14 | d..a07 d..a04 + const auto dcba_8 = ZipLower(d32, ba8, dc8); // d..a1B d..a18 | d..a0B d..a08 + const auto dcba_C = ZipUpper(d32, ba8, dc8); // d..a1F d..a1C | d..a0F d..a0C + // Write lower halves, then upper. vperm2i128 is slow on Zen1 but we can + // efficiently combine two lower halves into 256 bits: + const auto out0 = BitCast(d8, ConcatLowerLower(d32, dcba_4, dcba_0)); + const auto out1 = BitCast(d8, ConcatLowerLower(d32, dcba_C, dcba_8)); + StoreU(out0, d8, unaligned + 0 * 32); + StoreU(out1, d8, unaligned + 1 * 32); + const auto out2 = BitCast(d8, ConcatUpperUpper(d32, dcba_4, dcba_0)); + const auto out3 = BitCast(d8, ConcatUpperUpper(d32, dcba_C, dcba_8)); + StoreU(out2, d8, unaligned + 2 * 32); + StoreU(out3, d8, unaligned + 3 * 32); +} + +// ------------------------------ Reductions + +namespace detail { + +// Returns sum{lane[i]} in each lane. "v3210" is a replicated 128-bit block. +// Same logic as x86/128.h, but with Vec256 arguments. +template <typename T> +HWY_INLINE Vec256<T> SumOfLanes(hwy::SizeTag<4> /* tag */, + const Vec256<T> v3210) { + const auto v1032 = Shuffle1032(v3210); + const auto v31_20_31_20 = v3210 + v1032; + const auto v20_31_20_31 = Shuffle0321(v31_20_31_20); + return v20_31_20_31 + v31_20_31_20; +} +template <typename T> +HWY_INLINE Vec256<T> MinOfLanes(hwy::SizeTag<4> /* tag */, + const Vec256<T> v3210) { + const auto v1032 = Shuffle1032(v3210); + const auto v31_20_31_20 = Min(v3210, v1032); + const auto v20_31_20_31 = Shuffle0321(v31_20_31_20); + return Min(v20_31_20_31, v31_20_31_20); +} +template <typename T> +HWY_INLINE Vec256<T> MaxOfLanes(hwy::SizeTag<4> /* tag */, + const Vec256<T> v3210) { + const auto v1032 = Shuffle1032(v3210); + const auto v31_20_31_20 = Max(v3210, v1032); + const auto v20_31_20_31 = Shuffle0321(v31_20_31_20); + return Max(v20_31_20_31, v31_20_31_20); +} + +template <typename T> +HWY_INLINE Vec256<T> SumOfLanes(hwy::SizeTag<8> /* tag */, + const Vec256<T> v10) { + const auto v01 = Shuffle01(v10); + return v10 + v01; +} +template <typename T> +HWY_INLINE Vec256<T> MinOfLanes(hwy::SizeTag<8> /* tag */, + const Vec256<T> v10) { + const auto v01 = Shuffle01(v10); + return Min(v10, v01); +} +template <typename T> +HWY_INLINE Vec256<T> MaxOfLanes(hwy::SizeTag<8> /* tag */, + const Vec256<T> v10) { + const auto v01 = Shuffle01(v10); + return Max(v10, v01); +} + +// u16/i16 +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Vec256<T> MinOfLanes(hwy::SizeTag<2> /* tag */, Vec256<T> v) { + const Repartition<int32_t, Full256<T>> d32; + const auto even = And(BitCast(d32, v), Set(d32, 0xFFFF)); + const auto odd = ShiftRight<16>(BitCast(d32, v)); + const auto min = MinOfLanes(d32, Min(even, odd)); + // Also broadcast into odd lanes. + return BitCast(Full256<T>(), Or(min, ShiftLeft<16>(min))); +} +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Vec256<T> MaxOfLanes(hwy::SizeTag<2> /* tag */, Vec256<T> v) { + const Repartition<int32_t, Full256<T>> d32; + const auto even = And(BitCast(d32, v), Set(d32, 0xFFFF)); + const auto odd = ShiftRight<16>(BitCast(d32, v)); + const auto min = MaxOfLanes(d32, Max(even, odd)); + // Also broadcast into odd lanes. + return BitCast(Full256<T>(), Or(min, ShiftLeft<16>(min))); +} + +} // namespace detail + +// Supported for {uif}32x8, {uif}64x4. Returns the sum in each lane. +template <typename T> +HWY_API Vec256<T> SumOfLanes(Full256<T> d, const Vec256<T> vHL) { + const Vec256<T> vLH = ConcatLowerUpper(d, vHL, vHL); + return detail::SumOfLanes(hwy::SizeTag<sizeof(T)>(), vLH + vHL); +} +template <typename T> +HWY_API Vec256<T> MinOfLanes(Full256<T> d, const Vec256<T> vHL) { + const Vec256<T> vLH = ConcatLowerUpper(d, vHL, vHL); + return detail::MinOfLanes(hwy::SizeTag<sizeof(T)>(), Min(vLH, vHL)); +} +template <typename T> +HWY_API Vec256<T> MaxOfLanes(Full256<T> d, const Vec256<T> vHL) { + const Vec256<T> vLH = ConcatLowerUpper(d, vHL, vHL); + return detail::MaxOfLanes(hwy::SizeTag<sizeof(T)>(), Max(vLH, vHL)); +} + +// ================================================== DEPRECATED + +template <typename T> +HWY_API size_t StoreMaskBits(const Mask256<T> mask, uint8_t* bits) { + return StoreMaskBits(Full256<T>(), mask, bits); +} + +template <typename T> +HWY_API bool AllTrue(const Mask256<T> mask) { + return AllTrue(Full256<T>(), mask); +} + +template <typename T> +HWY_API bool AllFalse(const Mask256<T> mask) { + return AllFalse(Full256<T>(), mask); +} + +template <typename T> +HWY_API size_t CountTrue(const Mask256<T> mask) { + return CountTrue(Full256<T>(), mask); +} + +template <typename T> +HWY_API Vec256<T> SumOfLanes(const Vec256<T> vHL) { + return SumOfLanes(Full256<T>(), vHL); +} +template <typename T> +HWY_API Vec256<T> MinOfLanes(const Vec256<T> vHL) { + return MinOfLanes(Full256<T>(), vHL); +} +template <typename T> +HWY_API Vec256<T> MaxOfLanes(const Vec256<T> vHL) { + return MaxOfLanes(Full256<T>(), vHL); +} + +template <typename T> +HWY_API Vec128<T> UpperHalf(Vec256<T> v) { + return UpperHalf(Full128<T>(), v); +} + +template <int kBytes, typename T> +HWY_API Vec256<T> ShiftRightBytes(const Vec256<T> v) { + return ShiftRightBytes<kBytes>(Full256<T>(), v); +} + +template <int kLanes, typename T> +HWY_API Vec256<T> ShiftRightLanes(const Vec256<T> v) { + return ShiftRightLanes<kLanes>(Full256<T>(), v); +} + +template <size_t kBytes, typename T> +HWY_API Vec256<T> CombineShiftRightBytes(Vec256<T> hi, Vec256<T> lo) { + return CombineShiftRightBytes<kBytes>(Full256<T>(), hi, lo); +} + +template <typename T> +HWY_API Vec256<T> InterleaveUpper(Vec256<T> a, Vec256<T> b) { + return InterleaveUpper(Full256<T>(), a, b); +} + +template <typename T> +HWY_API Vec256<MakeWide<T>> ZipUpper(Vec256<T> a, Vec256<T> b) { + return InterleaveUpper(Full256<MakeWide<T>>(), a, b); +} + +template <typename T> +HWY_API Vec256<T> Combine(Vec128<T> hi, Vec128<T> lo) { + return Combine(Full256<T>(), hi, lo); +} + +template <typename T> +HWY_API Vec256<T> ZeroExtendVector(Vec128<T> lo) { + return ZeroExtendVector(Full256<T>(), lo); +} + +template <typename T> +HWY_API Vec256<T> ConcatLowerLower(Vec256<T> hi, Vec256<T> lo) { + return ConcatLowerLower(Full256<T>(), hi, lo); +} + +template <typename T> +HWY_API Vec256<T> ConcatLowerUpper(Vec256<T> hi, Vec256<T> lo) { + return ConcatLowerUpper(Full256<T>(), hi, lo); +} + +template <typename T> +HWY_API Vec256<T> ConcatUpperLower(Vec256<T> hi, Vec256<T> lo) { + return ConcatUpperLower(Full256<T>(), hi, lo); +} + +template <typename T> +HWY_API Vec256<T> ConcatUpperUpper(Vec256<T> hi, Vec256<T> lo) { + return ConcatUpperUpper(Full256<T>(), hi, lo); +} + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); diff --git a/third_party/highway/hwy/ops/x86_512-inl.h b/third_party/highway/hwy/ops/x86_512-inl.h new file mode 100644 index 0000000..10bad0d --- /dev/null +++ b/third_party/highway/hwy/ops/x86_512-inl.h @@ -0,0 +1,3734 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// 512-bit AVX512 vectors and operations. +// External include guard in highway.h - see comment there. + +// WARNING: most operations do not cross 128-bit block boundaries. In +// particular, "Broadcast", pack and zip behavior may be surprising. + +#include <immintrin.h> // AVX2+ + +#include "hwy/base.h" + +#if defined(_MSC_VER) && defined(__clang__) +// Including <immintrin.h> should be enough, but Clang's headers helpfully skip +// including these headers when _MSC_VER is defined, like when using clang-cl. +// Include these directly here. +// clang-format off +#include <smmintrin.h> + +#include <avxintrin.h> +#include <avx2intrin.h> +#include <f16cintrin.h> +#include <fmaintrin.h> + +#include <avx512fintrin.h> +#include <avx512vlintrin.h> +#include <avx512bwintrin.h> +#include <avx512dqintrin.h> +#include <avx512vlbwintrin.h> +#include <avx512vldqintrin.h> +#include <avx512bitalgintrin.h> +#include <avx512vlbitalgintrin.h> +#include <avx512vpopcntdqintrin.h> +#include <avx512vpopcntdqvlintrin.h> +// clang-format on +#endif + +#include <stddef.h> +#include <stdint.h> + +// For half-width vectors. Already includes base.h and shared-inl.h. +#include "hwy/ops/x86_256-inl.h" + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +template <typename T> +using Full512 = Simd<T, 64 / sizeof(T)>; + +namespace detail { + +template <typename T> +struct Raw512 { + using type = __m512i; +}; +template <> +struct Raw512<float> { + using type = __m512; +}; +template <> +struct Raw512<double> { + using type = __m512d; +}; + +// Template arg: sizeof(lane type) +template <size_t size> +struct RawMask512 {}; +template <> +struct RawMask512<1> { + using type = __mmask64; +}; +template <> +struct RawMask512<2> { + using type = __mmask32; +}; +template <> +struct RawMask512<4> { + using type = __mmask16; +}; +template <> +struct RawMask512<8> { + using type = __mmask8; +}; + +} // namespace detail + +template <typename T> +class Vec512 { + using Raw = typename detail::Raw512<T>::type; + + public: + // Compound assignment. Only usable if there is a corresponding non-member + // binary operator overload. For example, only f32 and f64 support division. + HWY_INLINE Vec512& operator*=(const Vec512 other) { + return *this = (*this * other); + } + HWY_INLINE Vec512& operator/=(const Vec512 other) { + return *this = (*this / other); + } + HWY_INLINE Vec512& operator+=(const Vec512 other) { + return *this = (*this + other); + } + HWY_INLINE Vec512& operator-=(const Vec512 other) { + return *this = (*this - other); + } + HWY_INLINE Vec512& operator&=(const Vec512 other) { + return *this = (*this & other); + } + HWY_INLINE Vec512& operator|=(const Vec512 other) { + return *this = (*this | other); + } + HWY_INLINE Vec512& operator^=(const Vec512 other) { + return *this = (*this ^ other); + } + + Raw raw; +}; + +// Mask register: one bit per lane. +template <typename T> +struct Mask512 { + typename detail::RawMask512<sizeof(T)>::type raw; +}; + +// ------------------------------ BitCast + +namespace detail { + +HWY_INLINE __m512i BitCastToInteger(__m512i v) { return v; } +HWY_INLINE __m512i BitCastToInteger(__m512 v) { return _mm512_castps_si512(v); } +HWY_INLINE __m512i BitCastToInteger(__m512d v) { + return _mm512_castpd_si512(v); +} + +template <typename T> +HWY_INLINE Vec512<uint8_t> BitCastToByte(Vec512<T> v) { + return Vec512<uint8_t>{BitCastToInteger(v.raw)}; +} + +// Cannot rely on function overloading because return types differ. +template <typename T> +struct BitCastFromInteger512 { + HWY_INLINE __m512i operator()(__m512i v) { return v; } +}; +template <> +struct BitCastFromInteger512<float> { + HWY_INLINE __m512 operator()(__m512i v) { return _mm512_castsi512_ps(v); } +}; +template <> +struct BitCastFromInteger512<double> { + HWY_INLINE __m512d operator()(__m512i v) { return _mm512_castsi512_pd(v); } +}; + +template <typename T> +HWY_INLINE Vec512<T> BitCastFromByte(Full512<T> /* tag */, Vec512<uint8_t> v) { + return Vec512<T>{BitCastFromInteger512<T>()(v.raw)}; +} + +} // namespace detail + +template <typename T, typename FromT> +HWY_API Vec512<T> BitCast(Full512<T> d, Vec512<FromT> v) { + return detail::BitCastFromByte(d, detail::BitCastToByte(v)); +} + +// ------------------------------ Set + +// Returns an all-zero vector. +template <typename T> +HWY_API Vec512<T> Zero(Full512<T> /* tag */) { + return Vec512<T>{_mm512_setzero_si512()}; +} +HWY_API Vec512<float> Zero(Full512<float> /* tag */) { + return Vec512<float>{_mm512_setzero_ps()}; +} +HWY_API Vec512<double> Zero(Full512<double> /* tag */) { + return Vec512<double>{_mm512_setzero_pd()}; +} + +// Returns a vector with all lanes set to "t". +HWY_API Vec512<uint8_t> Set(Full512<uint8_t> /* tag */, const uint8_t t) { + return Vec512<uint8_t>{_mm512_set1_epi8(static_cast<char>(t))}; // NOLINT +} +HWY_API Vec512<uint16_t> Set(Full512<uint16_t> /* tag */, const uint16_t t) { + return Vec512<uint16_t>{_mm512_set1_epi16(static_cast<short>(t))}; // NOLINT +} +HWY_API Vec512<uint32_t> Set(Full512<uint32_t> /* tag */, const uint32_t t) { + return Vec512<uint32_t>{_mm512_set1_epi32(static_cast<int>(t))}; +} +HWY_API Vec512<uint64_t> Set(Full512<uint64_t> /* tag */, const uint64_t t) { + return Vec512<uint64_t>{ + _mm512_set1_epi64(static_cast<long long>(t))}; // NOLINT +} +HWY_API Vec512<int8_t> Set(Full512<int8_t> /* tag */, const int8_t t) { + return Vec512<int8_t>{_mm512_set1_epi8(static_cast<char>(t))}; // NOLINT +} +HWY_API Vec512<int16_t> Set(Full512<int16_t> /* tag */, const int16_t t) { + return Vec512<int16_t>{_mm512_set1_epi16(static_cast<short>(t))}; // NOLINT +} +HWY_API Vec512<int32_t> Set(Full512<int32_t> /* tag */, const int32_t t) { + return Vec512<int32_t>{_mm512_set1_epi32(t)}; +} +HWY_API Vec512<int64_t> Set(Full512<int64_t> /* tag */, const int64_t t) { + return Vec512<int64_t>{ + _mm512_set1_epi64(static_cast<long long>(t))}; // NOLINT +} +HWY_API Vec512<float> Set(Full512<float> /* tag */, const float t) { + return Vec512<float>{_mm512_set1_ps(t)}; +} +HWY_API Vec512<double> Set(Full512<double> /* tag */, const double t) { + return Vec512<double>{_mm512_set1_pd(t)}; +} + +HWY_DIAGNOSTICS(push) +HWY_DIAGNOSTICS_OFF(disable : 4700, ignored "-Wuninitialized") + +// Returns a vector with uninitialized elements. +template <typename T> +HWY_API Vec512<T> Undefined(Full512<T> /* tag */) { + // Available on Clang 6.0, GCC 6.2, ICC 16.03, MSVC 19.14. All but ICC + // generate an XOR instruction. + return Vec512<T>{_mm512_undefined_epi32()}; +} +HWY_API Vec512<float> Undefined(Full512<float> /* tag */) { + return Vec512<float>{_mm512_undefined_ps()}; +} +HWY_API Vec512<double> Undefined(Full512<double> /* tag */) { + return Vec512<double>{_mm512_undefined_pd()}; +} + +HWY_DIAGNOSTICS(pop) + +// ================================================== LOGICAL + +// ------------------------------ Not + +template <typename T> +HWY_API Vec512<T> Not(const Vec512<T> v) { + using TU = MakeUnsigned<T>; + const __m512i vu = BitCast(Full512<TU>(), v).raw; + return BitCast(Full512<T>(), + Vec512<TU>{_mm512_ternarylogic_epi32(vu, vu, vu, 0x55)}); +} + +// ------------------------------ And + +template <typename T> +HWY_API Vec512<T> And(const Vec512<T> a, const Vec512<T> b) { + return Vec512<T>{_mm512_and_si512(a.raw, b.raw)}; +} + +HWY_API Vec512<float> And(const Vec512<float> a, const Vec512<float> b) { + return Vec512<float>{_mm512_and_ps(a.raw, b.raw)}; +} +HWY_API Vec512<double> And(const Vec512<double> a, const Vec512<double> b) { + return Vec512<double>{_mm512_and_pd(a.raw, b.raw)}; +} + +// ------------------------------ AndNot + +// Returns ~not_mask & mask. +template <typename T> +HWY_API Vec512<T> AndNot(const Vec512<T> not_mask, const Vec512<T> mask) { + return Vec512<T>{_mm512_andnot_si512(not_mask.raw, mask.raw)}; +} +HWY_API Vec512<float> AndNot(const Vec512<float> not_mask, + const Vec512<float> mask) { + return Vec512<float>{_mm512_andnot_ps(not_mask.raw, mask.raw)}; +} +HWY_API Vec512<double> AndNot(const Vec512<double> not_mask, + const Vec512<double> mask) { + return Vec512<double>{_mm512_andnot_pd(not_mask.raw, mask.raw)}; +} + +// ------------------------------ Or + +template <typename T> +HWY_API Vec512<T> Or(const Vec512<T> a, const Vec512<T> b) { + return Vec512<T>{_mm512_or_si512(a.raw, b.raw)}; +} + +HWY_API Vec512<float> Or(const Vec512<float> a, const Vec512<float> b) { + return Vec512<float>{_mm512_or_ps(a.raw, b.raw)}; +} +HWY_API Vec512<double> Or(const Vec512<double> a, const Vec512<double> b) { + return Vec512<double>{_mm512_or_pd(a.raw, b.raw)}; +} + +// ------------------------------ Xor + +template <typename T> +HWY_API Vec512<T> Xor(const Vec512<T> a, const Vec512<T> b) { + return Vec512<T>{_mm512_xor_si512(a.raw, b.raw)}; +} + +HWY_API Vec512<float> Xor(const Vec512<float> a, const Vec512<float> b) { + return Vec512<float>{_mm512_xor_ps(a.raw, b.raw)}; +} +HWY_API Vec512<double> Xor(const Vec512<double> a, const Vec512<double> b) { + return Vec512<double>{_mm512_xor_pd(a.raw, b.raw)}; +} + +// ------------------------------ Operator overloads (internal-only if float) + +template <typename T> +HWY_API Vec512<T> operator&(const Vec512<T> a, const Vec512<T> b) { + return And(a, b); +} + +template <typename T> +HWY_API Vec512<T> operator|(const Vec512<T> a, const Vec512<T> b) { + return Or(a, b); +} + +template <typename T> +HWY_API Vec512<T> operator^(const Vec512<T> a, const Vec512<T> b) { + return Xor(a, b); +} + +// ------------------------------ PopulationCount + +// 8/16 require BITALG, 32/64 require VPOPCNTDQ. +#if HWY_TARGET == HWY_AVX3_DL + +#ifdef HWY_NATIVE_POPCNT +#undef HWY_NATIVE_POPCNT +#else +#define HWY_NATIVE_POPCNT +#endif + +namespace detail { + +template <typename T> +HWY_INLINE Vec512<T> PopulationCount(hwy::SizeTag<1> /* tag */, Vec512<T> v) { + return Vec512<T>{_mm512_popcnt_epi8(v.raw)}; +} +template <typename T> +HWY_INLINE Vec512<T> PopulationCount(hwy::SizeTag<2> /* tag */, Vec512<T> v) { + return Vec512<T>{_mm512_popcnt_epi16(v.raw)}; +} +template <typename T> +HWY_INLINE Vec512<T> PopulationCount(hwy::SizeTag<4> /* tag */, Vec512<T> v) { + return Vec512<T>{_mm512_popcnt_epi32(v.raw)}; +} +template <typename T> +HWY_INLINE Vec512<T> PopulationCount(hwy::SizeTag<8> /* tag */, Vec512<T> v) { + return Vec512<T>{_mm512_popcnt_epi64(v.raw)}; +} + +} // namespace detail + +template <typename T> +HWY_API Vec512<T> PopulationCount(Vec512<T> v) { + return detail::PopulationCount(hwy::SizeTag<sizeof(T)>(), v); +} + +#endif // HWY_TARGET == HWY_AVX3_DL + +// ================================================== SIGN + +// ------------------------------ CopySign + +template <typename T> +HWY_API Vec512<T> CopySign(const Vec512<T> magn, const Vec512<T> sign) { + static_assert(IsFloat<T>(), "Only makes sense for floating-point"); + + const Full512<T> d; + const auto msb = SignBit(d); + + const Rebind<MakeUnsigned<T>, decltype(d)> du; + // Truth table for msb, magn, sign | bitwise msb ? sign : mag + // 0 0 0 | 0 + // 0 0 1 | 0 + // 0 1 0 | 1 + // 0 1 1 | 1 + // 1 0 0 | 0 + // 1 0 1 | 1 + // 1 1 0 | 0 + // 1 1 1 | 1 + // The lane size does not matter because we are not using predication. + const __m512i out = _mm512_ternarylogic_epi32( + BitCast(du, msb).raw, BitCast(du, magn).raw, BitCast(du, sign).raw, 0xAC); + return BitCast(d, decltype(Zero(du)){out}); +} + +template <typename T> +HWY_API Vec512<T> CopySignToAbs(const Vec512<T> abs, const Vec512<T> sign) { + // AVX3 can also handle abs < 0, so no extra action needed. + return CopySign(abs, sign); +} + +// ================================================== MASK + +// ------------------------------ FirstN + +// Possibilities for constructing a bitmask of N ones: +// - kshift* only consider the lowest byte of the shift count, so they would +// not correctly handle large n. +// - Scalar shifts >= 64 are UB. +// - BZHI has the desired semantics; we assume AVX-512 implies BMI2. However, +// we need 64-bit masks for sizeof(T) == 1, so special-case 32-bit builds. + +#if HWY_ARCH_X86_32 +namespace detail { + +// 32 bit mask is sufficient for lane size >= 2. +template <typename T, HWY_IF_NOT_LANE_SIZE(T, 1)> +HWY_INLINE Mask512<T> FirstN(size_t n) { + Mask512<T> m; + const uint32_t all = ~uint32_t(0); + // BZHI only looks at the lower 8 bits of n! + m.raw = static_cast<decltype(m.raw)>((n > 255) ? all : _bzhi_u32(all, n)); + return m; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 1)> +HWY_INLINE Mask512<T> FirstN(size_t n) { + const uint64_t bits = n < 64 ? ((1ULL << n) - 1) : ~uint64_t(0); + return Mask512<T>{static_cast<__mmask64>(bits)}; +} + +} // namespace detail +#endif // HWY_ARCH_X86_32 + +template <typename T> +HWY_API Mask512<T> FirstN(const Full512<T> /*tag*/, size_t n) { +#if HWY_ARCH_X86_64 + Mask512<T> m; + const uint64_t all = ~uint64_t(0); + // BZHI only looks at the lower 8 bits of n! + m.raw = static_cast<decltype(m.raw)>((n > 255) ? all : _bzhi_u64(all, n)); + return m; +#else + return detail::FirstN<T>(n); +#endif // HWY_ARCH_X86_64 +} + +// ------------------------------ IfThenElse + +// Returns mask ? b : a. + +namespace detail { + +// Templates for signed/unsigned integer of a particular size. +template <typename T> +HWY_INLINE Vec512<T> IfThenElse(hwy::SizeTag<1> /* tag */, + const Mask512<T> mask, const Vec512<T> yes, + const Vec512<T> no) { + return Vec512<T>{_mm512_mask_mov_epi8(no.raw, mask.raw, yes.raw)}; +} +template <typename T> +HWY_INLINE Vec512<T> IfThenElse(hwy::SizeTag<2> /* tag */, + const Mask512<T> mask, const Vec512<T> yes, + const Vec512<T> no) { + return Vec512<T>{_mm512_mask_mov_epi16(no.raw, mask.raw, yes.raw)}; +} +template <typename T> +HWY_INLINE Vec512<T> IfThenElse(hwy::SizeTag<4> /* tag */, + const Mask512<T> mask, const Vec512<T> yes, + const Vec512<T> no) { + return Vec512<T>{_mm512_mask_mov_epi32(no.raw, mask.raw, yes.raw)}; +} +template <typename T> +HWY_INLINE Vec512<T> IfThenElse(hwy::SizeTag<8> /* tag */, + const Mask512<T> mask, const Vec512<T> yes, + const Vec512<T> no) { + return Vec512<T>{_mm512_mask_mov_epi64(no.raw, mask.raw, yes.raw)}; +} + +} // namespace detail + +template <typename T> +HWY_API Vec512<T> IfThenElse(const Mask512<T> mask, const Vec512<T> yes, + const Vec512<T> no) { + return detail::IfThenElse(hwy::SizeTag<sizeof(T)>(), mask, yes, no); +} +HWY_API Vec512<float> IfThenElse(const Mask512<float> mask, + const Vec512<float> yes, + const Vec512<float> no) { + return Vec512<float>{_mm512_mask_mov_ps(no.raw, mask.raw, yes.raw)}; +} +HWY_API Vec512<double> IfThenElse(const Mask512<double> mask, + const Vec512<double> yes, + const Vec512<double> no) { + return Vec512<double>{_mm512_mask_mov_pd(no.raw, mask.raw, yes.raw)}; +} + +namespace detail { + +template <typename T> +HWY_INLINE Vec512<T> IfThenElseZero(hwy::SizeTag<1> /* tag */, + const Mask512<T> mask, + const Vec512<T> yes) { + return Vec512<T>{_mm512_maskz_mov_epi8(mask.raw, yes.raw)}; +} +template <typename T> +HWY_INLINE Vec512<T> IfThenElseZero(hwy::SizeTag<2> /* tag */, + const Mask512<T> mask, + const Vec512<T> yes) { + return Vec512<T>{_mm512_maskz_mov_epi16(mask.raw, yes.raw)}; +} +template <typename T> +HWY_INLINE Vec512<T> IfThenElseZero(hwy::SizeTag<4> /* tag */, + const Mask512<T> mask, + const Vec512<T> yes) { + return Vec512<T>{_mm512_maskz_mov_epi32(mask.raw, yes.raw)}; +} +template <typename T> +HWY_INLINE Vec512<T> IfThenElseZero(hwy::SizeTag<8> /* tag */, + const Mask512<T> mask, + const Vec512<T> yes) { + return Vec512<T>{_mm512_maskz_mov_epi64(mask.raw, yes.raw)}; +} + +} // namespace detail + +template <typename T> +HWY_API Vec512<T> IfThenElseZero(const Mask512<T> mask, const Vec512<T> yes) { + return detail::IfThenElseZero(hwy::SizeTag<sizeof(T)>(), mask, yes); +} +HWY_API Vec512<float> IfThenElseZero(const Mask512<float> mask, + const Vec512<float> yes) { + return Vec512<float>{_mm512_maskz_mov_ps(mask.raw, yes.raw)}; +} +HWY_API Vec512<double> IfThenElseZero(const Mask512<double> mask, + const Vec512<double> yes) { + return Vec512<double>{_mm512_maskz_mov_pd(mask.raw, yes.raw)}; +} + +namespace detail { + +template <typename T> +HWY_INLINE Vec512<T> IfThenZeroElse(hwy::SizeTag<1> /* tag */, + const Mask512<T> mask, const Vec512<T> no) { + // xor_epi8/16 are missing, but we have sub, which is just as fast for u8/16. + return Vec512<T>{_mm512_mask_sub_epi8(no.raw, mask.raw, no.raw, no.raw)}; +} +template <typename T> +HWY_INLINE Vec512<T> IfThenZeroElse(hwy::SizeTag<2> /* tag */, + const Mask512<T> mask, const Vec512<T> no) { + return Vec512<T>{_mm512_mask_sub_epi16(no.raw, mask.raw, no.raw, no.raw)}; +} +template <typename T> +HWY_INLINE Vec512<T> IfThenZeroElse(hwy::SizeTag<4> /* tag */, + const Mask512<T> mask, const Vec512<T> no) { + return Vec512<T>{_mm512_mask_xor_epi32(no.raw, mask.raw, no.raw, no.raw)}; +} +template <typename T> +HWY_INLINE Vec512<T> IfThenZeroElse(hwy::SizeTag<8> /* tag */, + const Mask512<T> mask, const Vec512<T> no) { + return Vec512<T>{_mm512_mask_xor_epi64(no.raw, mask.raw, no.raw, no.raw)}; +} + +} // namespace detail + +template <typename T> +HWY_API Vec512<T> IfThenZeroElse(const Mask512<T> mask, const Vec512<T> no) { + return detail::IfThenZeroElse(hwy::SizeTag<sizeof(T)>(), mask, no); +} +HWY_API Vec512<float> IfThenZeroElse(const Mask512<float> mask, + const Vec512<float> no) { + return Vec512<float>{_mm512_mask_xor_ps(no.raw, mask.raw, no.raw, no.raw)}; +} +HWY_API Vec512<double> IfThenZeroElse(const Mask512<double> mask, + const Vec512<double> no) { + return Vec512<double>{_mm512_mask_xor_pd(no.raw, mask.raw, no.raw, no.raw)}; +} + +template <typename T, HWY_IF_FLOAT(T)> +HWY_API Vec512<T> ZeroIfNegative(const Vec512<T> v) { + // AVX3 MaskFromVec only looks at the MSB + return IfThenZeroElse(MaskFromVec(v), v); +} + +// ================================================== ARITHMETIC + +// ------------------------------ Addition + +// Unsigned +HWY_API Vec512<uint8_t> operator+(const Vec512<uint8_t> a, + const Vec512<uint8_t> b) { + return Vec512<uint8_t>{_mm512_add_epi8(a.raw, b.raw)}; +} +HWY_API Vec512<uint16_t> operator+(const Vec512<uint16_t> a, + const Vec512<uint16_t> b) { + return Vec512<uint16_t>{_mm512_add_epi16(a.raw, b.raw)}; +} +HWY_API Vec512<uint32_t> operator+(const Vec512<uint32_t> a, + const Vec512<uint32_t> b) { + return Vec512<uint32_t>{_mm512_add_epi32(a.raw, b.raw)}; +} +HWY_API Vec512<uint64_t> operator+(const Vec512<uint64_t> a, + const Vec512<uint64_t> b) { + return Vec512<uint64_t>{_mm512_add_epi64(a.raw, b.raw)}; +} + +// Signed +HWY_API Vec512<int8_t> operator+(const Vec512<int8_t> a, + const Vec512<int8_t> b) { + return Vec512<int8_t>{_mm512_add_epi8(a.raw, b.raw)}; +} +HWY_API Vec512<int16_t> operator+(const Vec512<int16_t> a, + const Vec512<int16_t> b) { + return Vec512<int16_t>{_mm512_add_epi16(a.raw, b.raw)}; +} +HWY_API Vec512<int32_t> operator+(const Vec512<int32_t> a, + const Vec512<int32_t> b) { + return Vec512<int32_t>{_mm512_add_epi32(a.raw, b.raw)}; +} +HWY_API Vec512<int64_t> operator+(const Vec512<int64_t> a, + const Vec512<int64_t> b) { + return Vec512<int64_t>{_mm512_add_epi64(a.raw, b.raw)}; +} + +// Float +HWY_API Vec512<float> operator+(const Vec512<float> a, const Vec512<float> b) { + return Vec512<float>{_mm512_add_ps(a.raw, b.raw)}; +} +HWY_API Vec512<double> operator+(const Vec512<double> a, + const Vec512<double> b) { + return Vec512<double>{_mm512_add_pd(a.raw, b.raw)}; +} + +// ------------------------------ Subtraction + +// Unsigned +HWY_API Vec512<uint8_t> operator-(const Vec512<uint8_t> a, + const Vec512<uint8_t> b) { + return Vec512<uint8_t>{_mm512_sub_epi8(a.raw, b.raw)}; +} +HWY_API Vec512<uint16_t> operator-(const Vec512<uint16_t> a, + const Vec512<uint16_t> b) { + return Vec512<uint16_t>{_mm512_sub_epi16(a.raw, b.raw)}; +} +HWY_API Vec512<uint32_t> operator-(const Vec512<uint32_t> a, + const Vec512<uint32_t> b) { + return Vec512<uint32_t>{_mm512_sub_epi32(a.raw, b.raw)}; +} +HWY_API Vec512<uint64_t> operator-(const Vec512<uint64_t> a, + const Vec512<uint64_t> b) { + return Vec512<uint64_t>{_mm512_sub_epi64(a.raw, b.raw)}; +} + +// Signed +HWY_API Vec512<int8_t> operator-(const Vec512<int8_t> a, + const Vec512<int8_t> b) { + return Vec512<int8_t>{_mm512_sub_epi8(a.raw, b.raw)}; +} +HWY_API Vec512<int16_t> operator-(const Vec512<int16_t> a, + const Vec512<int16_t> b) { + return Vec512<int16_t>{_mm512_sub_epi16(a.raw, b.raw)}; +} +HWY_API Vec512<int32_t> operator-(const Vec512<int32_t> a, + const Vec512<int32_t> b) { + return Vec512<int32_t>{_mm512_sub_epi32(a.raw, b.raw)}; +} +HWY_API Vec512<int64_t> operator-(const Vec512<int64_t> a, + const Vec512<int64_t> b) { + return Vec512<int64_t>{_mm512_sub_epi64(a.raw, b.raw)}; +} + +// Float +HWY_API Vec512<float> operator-(const Vec512<float> a, const Vec512<float> b) { + return Vec512<float>{_mm512_sub_ps(a.raw, b.raw)}; +} +HWY_API Vec512<double> operator-(const Vec512<double> a, + const Vec512<double> b) { + return Vec512<double>{_mm512_sub_pd(a.raw, b.raw)}; +} + +// ------------------------------ Saturating addition + +// Returns a + b clamped to the destination range. + +// Unsigned +HWY_API Vec512<uint8_t> SaturatedAdd(const Vec512<uint8_t> a, + const Vec512<uint8_t> b) { + return Vec512<uint8_t>{_mm512_adds_epu8(a.raw, b.raw)}; +} +HWY_API Vec512<uint16_t> SaturatedAdd(const Vec512<uint16_t> a, + const Vec512<uint16_t> b) { + return Vec512<uint16_t>{_mm512_adds_epu16(a.raw, b.raw)}; +} + +// Signed +HWY_API Vec512<int8_t> SaturatedAdd(const Vec512<int8_t> a, + const Vec512<int8_t> b) { + return Vec512<int8_t>{_mm512_adds_epi8(a.raw, b.raw)}; +} +HWY_API Vec512<int16_t> SaturatedAdd(const Vec512<int16_t> a, + const Vec512<int16_t> b) { + return Vec512<int16_t>{_mm512_adds_epi16(a.raw, b.raw)}; +} + +// ------------------------------ Saturating subtraction + +// Returns a - b clamped to the destination range. + +// Unsigned +HWY_API Vec512<uint8_t> SaturatedSub(const Vec512<uint8_t> a, + const Vec512<uint8_t> b) { + return Vec512<uint8_t>{_mm512_subs_epu8(a.raw, b.raw)}; +} +HWY_API Vec512<uint16_t> SaturatedSub(const Vec512<uint16_t> a, + const Vec512<uint16_t> b) { + return Vec512<uint16_t>{_mm512_subs_epu16(a.raw, b.raw)}; +} + +// Signed +HWY_API Vec512<int8_t> SaturatedSub(const Vec512<int8_t> a, + const Vec512<int8_t> b) { + return Vec512<int8_t>{_mm512_subs_epi8(a.raw, b.raw)}; +} +HWY_API Vec512<int16_t> SaturatedSub(const Vec512<int16_t> a, + const Vec512<int16_t> b) { + return Vec512<int16_t>{_mm512_subs_epi16(a.raw, b.raw)}; +} + +// ------------------------------ Average + +// Returns (a + b + 1) / 2 + +// Unsigned +HWY_API Vec512<uint8_t> AverageRound(const Vec512<uint8_t> a, + const Vec512<uint8_t> b) { + return Vec512<uint8_t>{_mm512_avg_epu8(a.raw, b.raw)}; +} +HWY_API Vec512<uint16_t> AverageRound(const Vec512<uint16_t> a, + const Vec512<uint16_t> b) { + return Vec512<uint16_t>{_mm512_avg_epu16(a.raw, b.raw)}; +} + +// ------------------------------ Abs (Sub) + +// Returns absolute value, except that LimitsMin() maps to LimitsMax() + 1. +HWY_API Vec512<int8_t> Abs(const Vec512<int8_t> v) { +#if HWY_COMPILER_MSVC + // Workaround for incorrect codegen? (untested due to internal compiler error) + const auto zero = Zero(Full512<int8_t>()); + return Vec512<int8_t>{_mm512_max_epi8(v.raw, (zero - v).raw)}; +#else + return Vec512<int8_t>{_mm512_abs_epi8(v.raw)}; +#endif +} +HWY_API Vec512<int16_t> Abs(const Vec512<int16_t> v) { + return Vec512<int16_t>{_mm512_abs_epi16(v.raw)}; +} +HWY_API Vec512<int32_t> Abs(const Vec512<int32_t> v) { + return Vec512<int32_t>{_mm512_abs_epi32(v.raw)}; +} +HWY_API Vec512<int64_t> Abs(const Vec512<int64_t> v) { + return Vec512<int64_t>{_mm512_abs_epi64(v.raw)}; +} + +// These aren't native instructions, they also involve AND with constant. +HWY_API Vec512<float> Abs(const Vec512<float> v) { + return Vec512<float>{_mm512_abs_ps(v.raw)}; +} +HWY_API Vec512<double> Abs(const Vec512<double> v) { + return Vec512<double>{_mm512_abs_pd(v.raw)}; +} +// ------------------------------ ShiftLeft + +template <int kBits> +HWY_API Vec512<uint16_t> ShiftLeft(const Vec512<uint16_t> v) { + return Vec512<uint16_t>{_mm512_slli_epi16(v.raw, kBits)}; +} + +template <int kBits> +HWY_API Vec512<uint32_t> ShiftLeft(const Vec512<uint32_t> v) { + return Vec512<uint32_t>{_mm512_slli_epi32(v.raw, kBits)}; +} + +template <int kBits> +HWY_API Vec512<uint64_t> ShiftLeft(const Vec512<uint64_t> v) { + return Vec512<uint64_t>{_mm512_slli_epi64(v.raw, kBits)}; +} + +template <int kBits> +HWY_API Vec512<int16_t> ShiftLeft(const Vec512<int16_t> v) { + return Vec512<int16_t>{_mm512_slli_epi16(v.raw, kBits)}; +} + +template <int kBits> +HWY_API Vec512<int32_t> ShiftLeft(const Vec512<int32_t> v) { + return Vec512<int32_t>{_mm512_slli_epi32(v.raw, kBits)}; +} + +template <int kBits> +HWY_API Vec512<int64_t> ShiftLeft(const Vec512<int64_t> v) { + return Vec512<int64_t>{_mm512_slli_epi64(v.raw, kBits)}; +} + +template <int kBits, typename T, HWY_IF_LANE_SIZE(T, 1)> +HWY_API Vec512<T> ShiftLeft(const Vec512<T> v) { + const Full512<T> d8; + const RepartitionToWide<decltype(d8)> d16; + const auto shifted = BitCast(d8, ShiftLeft<kBits>(BitCast(d16, v))); + return kBits == 1 + ? (v + v) + : (shifted & Set(d8, static_cast<T>((0xFF << kBits) & 0xFF))); +} + +// ------------------------------ ShiftRight + +template <int kBits> +HWY_API Vec512<uint16_t> ShiftRight(const Vec512<uint16_t> v) { + return Vec512<uint16_t>{_mm512_srli_epi16(v.raw, kBits)}; +} + +template <int kBits> +HWY_API Vec512<uint32_t> ShiftRight(const Vec512<uint32_t> v) { + return Vec512<uint32_t>{_mm512_srli_epi32(v.raw, kBits)}; +} + +template <int kBits> +HWY_API Vec512<uint64_t> ShiftRight(const Vec512<uint64_t> v) { + return Vec512<uint64_t>{_mm512_srli_epi64(v.raw, kBits)}; +} + +template <int kBits> +HWY_API Vec512<uint8_t> ShiftRight(const Vec512<uint8_t> v) { + const Full512<uint8_t> d8; + // Use raw instead of BitCast to support N=1. + const Vec512<uint8_t> shifted{ShiftRight<kBits>(Vec512<uint16_t>{v.raw}).raw}; + return shifted & Set(d8, 0xFF >> kBits); +} + +template <int kBits> +HWY_API Vec512<int16_t> ShiftRight(const Vec512<int16_t> v) { + return Vec512<int16_t>{_mm512_srai_epi16(v.raw, kBits)}; +} + +template <int kBits> +HWY_API Vec512<int32_t> ShiftRight(const Vec512<int32_t> v) { + return Vec512<int32_t>{_mm512_srai_epi32(v.raw, kBits)}; +} + +template <int kBits> +HWY_API Vec512<int64_t> ShiftRight(const Vec512<int64_t> v) { + return Vec512<int64_t>{_mm512_srai_epi64(v.raw, kBits)}; +} + +template <int kBits> +HWY_API Vec512<int8_t> ShiftRight(const Vec512<int8_t> v) { + const Full512<int8_t> di; + const Full512<uint8_t> du; + const auto shifted = BitCast(di, ShiftRight<kBits>(BitCast(du, v))); + const auto shifted_sign = BitCast(di, Set(du, 0x80 >> kBits)); + return (shifted ^ shifted_sign) - shifted_sign; +} + +// ------------------------------ RotateRight + +template <int kBits> +HWY_API Vec512<uint32_t> RotateRight(const Vec512<uint32_t> v) { + static_assert(0 <= kBits && kBits < 32, "Invalid shift count"); + return Vec512<uint32_t>{_mm512_ror_epi32(v.raw, kBits)}; +} + +template <int kBits> +HWY_API Vec512<uint64_t> RotateRight(const Vec512<uint64_t> v) { + static_assert(0 <= kBits && kBits < 64, "Invalid shift count"); + return Vec512<uint64_t>{_mm512_ror_epi64(v.raw, kBits)}; +} + +// ------------------------------ ShiftLeftSame + +HWY_API Vec512<uint16_t> ShiftLeftSame(const Vec512<uint16_t> v, + const int bits) { + return Vec512<uint16_t>{_mm512_sll_epi16(v.raw, _mm_cvtsi32_si128(bits))}; +} +HWY_API Vec512<uint32_t> ShiftLeftSame(const Vec512<uint32_t> v, + const int bits) { + return Vec512<uint32_t>{_mm512_sll_epi32(v.raw, _mm_cvtsi32_si128(bits))}; +} +HWY_API Vec512<uint64_t> ShiftLeftSame(const Vec512<uint64_t> v, + const int bits) { + return Vec512<uint64_t>{_mm512_sll_epi64(v.raw, _mm_cvtsi32_si128(bits))}; +} + +HWY_API Vec512<int16_t> ShiftLeftSame(const Vec512<int16_t> v, const int bits) { + return Vec512<int16_t>{_mm512_sll_epi16(v.raw, _mm_cvtsi32_si128(bits))}; +} + +HWY_API Vec512<int32_t> ShiftLeftSame(const Vec512<int32_t> v, const int bits) { + return Vec512<int32_t>{_mm512_sll_epi32(v.raw, _mm_cvtsi32_si128(bits))}; +} + +HWY_API Vec512<int64_t> ShiftLeftSame(const Vec512<int64_t> v, const int bits) { + return Vec512<int64_t>{_mm512_sll_epi64(v.raw, _mm_cvtsi32_si128(bits))}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 1)> +HWY_API Vec512<T> ShiftLeftSame(const Vec512<T> v, const int bits) { + const Full512<T> d8; + const RepartitionToWide<decltype(d8)> d16; + const auto shifted = BitCast(d8, ShiftLeftSame(BitCast(d16, v), bits)); + return shifted & Set(d8, static_cast<T>((0xFF << bits) & 0xFF)); +} + +// ------------------------------ ShiftRightSame + +HWY_API Vec512<uint16_t> ShiftRightSame(const Vec512<uint16_t> v, + const int bits) { + return Vec512<uint16_t>{_mm512_srl_epi16(v.raw, _mm_cvtsi32_si128(bits))}; +} +HWY_API Vec512<uint32_t> ShiftRightSame(const Vec512<uint32_t> v, + const int bits) { + return Vec512<uint32_t>{_mm512_srl_epi32(v.raw, _mm_cvtsi32_si128(bits))}; +} +HWY_API Vec512<uint64_t> ShiftRightSame(const Vec512<uint64_t> v, + const int bits) { + return Vec512<uint64_t>{_mm512_srl_epi64(v.raw, _mm_cvtsi32_si128(bits))}; +} + +HWY_API Vec512<uint8_t> ShiftRightSame(Vec512<uint8_t> v, const int bits) { + const Full512<uint8_t> d8; + const RepartitionToWide<decltype(d8)> d16; + const auto shifted = BitCast(d8, ShiftRightSame(BitCast(d16, v), bits)); + return shifted & Set(d8, static_cast<uint8_t>(0xFF >> bits)); +} + +HWY_API Vec512<int16_t> ShiftRightSame(const Vec512<int16_t> v, + const int bits) { + return Vec512<int16_t>{_mm512_sra_epi16(v.raw, _mm_cvtsi32_si128(bits))}; +} + +HWY_API Vec512<int32_t> ShiftRightSame(const Vec512<int32_t> v, + const int bits) { + return Vec512<int32_t>{_mm512_sra_epi32(v.raw, _mm_cvtsi32_si128(bits))}; +} +HWY_API Vec512<int64_t> ShiftRightSame(const Vec512<int64_t> v, + const int bits) { + return Vec512<int64_t>{_mm512_sra_epi64(v.raw, _mm_cvtsi32_si128(bits))}; +} + +HWY_API Vec512<int8_t> ShiftRightSame(Vec512<int8_t> v, const int bits) { + const Full512<int8_t> di; + const Full512<uint8_t> du; + const auto shifted = BitCast(di, ShiftRightSame(BitCast(du, v), bits)); + const auto shifted_sign = + BitCast(di, Set(du, static_cast<uint8_t>(0x80 >> bits))); + return (shifted ^ shifted_sign) - shifted_sign; +} + +// ------------------------------ Shl + +HWY_API Vec512<uint16_t> operator<<(const Vec512<uint16_t> v, + const Vec512<uint16_t> bits) { + return Vec512<uint16_t>{_mm512_sllv_epi16(v.raw, bits.raw)}; +} + +HWY_API Vec512<uint32_t> operator<<(const Vec512<uint32_t> v, + const Vec512<uint32_t> bits) { + return Vec512<uint32_t>{_mm512_sllv_epi32(v.raw, bits.raw)}; +} + +HWY_API Vec512<uint64_t> operator<<(const Vec512<uint64_t> v, + const Vec512<uint64_t> bits) { + return Vec512<uint64_t>{_mm512_sllv_epi64(v.raw, bits.raw)}; +} + +// Signed left shift is the same as unsigned. +template <typename T, HWY_IF_SIGNED(T)> +HWY_API Vec512<T> operator<<(const Vec512<T> v, const Vec512<T> bits) { + const Full512<T> di; + const Full512<MakeUnsigned<T>> du; + return BitCast(di, BitCast(du, v) << BitCast(du, bits)); +} + +// ------------------------------ Shr + +HWY_API Vec512<uint16_t> operator>>(const Vec512<uint16_t> v, + const Vec512<uint16_t> bits) { + return Vec512<uint16_t>{_mm512_srlv_epi16(v.raw, bits.raw)}; +} + +HWY_API Vec512<uint32_t> operator>>(const Vec512<uint32_t> v, + const Vec512<uint32_t> bits) { + return Vec512<uint32_t>{_mm512_srlv_epi32(v.raw, bits.raw)}; +} + +HWY_API Vec512<uint64_t> operator>>(const Vec512<uint64_t> v, + const Vec512<uint64_t> bits) { + return Vec512<uint64_t>{_mm512_srlv_epi64(v.raw, bits.raw)}; +} + +HWY_API Vec512<int16_t> operator>>(const Vec512<int16_t> v, + const Vec512<int16_t> bits) { + return Vec512<int16_t>{_mm512_srav_epi16(v.raw, bits.raw)}; +} + +HWY_API Vec512<int32_t> operator>>(const Vec512<int32_t> v, + const Vec512<int32_t> bits) { + return Vec512<int32_t>{_mm512_srav_epi32(v.raw, bits.raw)}; +} + +HWY_API Vec512<int64_t> operator>>(const Vec512<int64_t> v, + const Vec512<int64_t> bits) { + return Vec512<int64_t>{_mm512_srav_epi64(v.raw, bits.raw)}; +} + +// ------------------------------ Minimum + +// Unsigned +HWY_API Vec512<uint8_t> Min(const Vec512<uint8_t> a, const Vec512<uint8_t> b) { + return Vec512<uint8_t>{_mm512_min_epu8(a.raw, b.raw)}; +} +HWY_API Vec512<uint16_t> Min(const Vec512<uint16_t> a, + const Vec512<uint16_t> b) { + return Vec512<uint16_t>{_mm512_min_epu16(a.raw, b.raw)}; +} +HWY_API Vec512<uint32_t> Min(const Vec512<uint32_t> a, + const Vec512<uint32_t> b) { + return Vec512<uint32_t>{_mm512_min_epu32(a.raw, b.raw)}; +} +HWY_API Vec512<uint64_t> Min(const Vec512<uint64_t> a, + const Vec512<uint64_t> b) { + return Vec512<uint64_t>{_mm512_min_epu64(a.raw, b.raw)}; +} + +// Signed +HWY_API Vec512<int8_t> Min(const Vec512<int8_t> a, const Vec512<int8_t> b) { + return Vec512<int8_t>{_mm512_min_epi8(a.raw, b.raw)}; +} +HWY_API Vec512<int16_t> Min(const Vec512<int16_t> a, const Vec512<int16_t> b) { + return Vec512<int16_t>{_mm512_min_epi16(a.raw, b.raw)}; +} +HWY_API Vec512<int32_t> Min(const Vec512<int32_t> a, const Vec512<int32_t> b) { + return Vec512<int32_t>{_mm512_min_epi32(a.raw, b.raw)}; +} +HWY_API Vec512<int64_t> Min(const Vec512<int64_t> a, const Vec512<int64_t> b) { + return Vec512<int64_t>{_mm512_min_epi64(a.raw, b.raw)}; +} + +// Float +HWY_API Vec512<float> Min(const Vec512<float> a, const Vec512<float> b) { + return Vec512<float>{_mm512_min_ps(a.raw, b.raw)}; +} +HWY_API Vec512<double> Min(const Vec512<double> a, const Vec512<double> b) { + return Vec512<double>{_mm512_min_pd(a.raw, b.raw)}; +} + +// ------------------------------ Maximum + +// Unsigned +HWY_API Vec512<uint8_t> Max(const Vec512<uint8_t> a, const Vec512<uint8_t> b) { + return Vec512<uint8_t>{_mm512_max_epu8(a.raw, b.raw)}; +} +HWY_API Vec512<uint16_t> Max(const Vec512<uint16_t> a, + const Vec512<uint16_t> b) { + return Vec512<uint16_t>{_mm512_max_epu16(a.raw, b.raw)}; +} +HWY_API Vec512<uint32_t> Max(const Vec512<uint32_t> a, + const Vec512<uint32_t> b) { + return Vec512<uint32_t>{_mm512_max_epu32(a.raw, b.raw)}; +} +HWY_API Vec512<uint64_t> Max(const Vec512<uint64_t> a, + const Vec512<uint64_t> b) { + return Vec512<uint64_t>{_mm512_max_epu64(a.raw, b.raw)}; +} + +// Signed +HWY_API Vec512<int8_t> Max(const Vec512<int8_t> a, const Vec512<int8_t> b) { + return Vec512<int8_t>{_mm512_max_epi8(a.raw, b.raw)}; +} +HWY_API Vec512<int16_t> Max(const Vec512<int16_t> a, const Vec512<int16_t> b) { + return Vec512<int16_t>{_mm512_max_epi16(a.raw, b.raw)}; +} +HWY_API Vec512<int32_t> Max(const Vec512<int32_t> a, const Vec512<int32_t> b) { + return Vec512<int32_t>{_mm512_max_epi32(a.raw, b.raw)}; +} +HWY_API Vec512<int64_t> Max(const Vec512<int64_t> a, const Vec512<int64_t> b) { + return Vec512<int64_t>{_mm512_max_epi64(a.raw, b.raw)}; +} + +// Float +HWY_API Vec512<float> Max(const Vec512<float> a, const Vec512<float> b) { + return Vec512<float>{_mm512_max_ps(a.raw, b.raw)}; +} +HWY_API Vec512<double> Max(const Vec512<double> a, const Vec512<double> b) { + return Vec512<double>{_mm512_max_pd(a.raw, b.raw)}; +} + +// ------------------------------ Integer multiplication + +// Unsigned +HWY_API Vec512<uint16_t> operator*(const Vec512<uint16_t> a, + const Vec512<uint16_t> b) { + return Vec512<uint16_t>{_mm512_mullo_epi16(a.raw, b.raw)}; +} +HWY_API Vec512<uint32_t> operator*(const Vec512<uint32_t> a, + const Vec512<uint32_t> b) { + return Vec512<uint32_t>{_mm512_mullo_epi32(a.raw, b.raw)}; +} + +// Signed +HWY_API Vec512<int16_t> operator*(const Vec512<int16_t> a, + const Vec512<int16_t> b) { + return Vec512<int16_t>{_mm512_mullo_epi16(a.raw, b.raw)}; +} +HWY_API Vec512<int32_t> operator*(const Vec512<int32_t> a, + const Vec512<int32_t> b) { + return Vec512<int32_t>{_mm512_mullo_epi32(a.raw, b.raw)}; +} + +// Returns the upper 16 bits of a * b in each lane. +HWY_API Vec512<uint16_t> MulHigh(const Vec512<uint16_t> a, + const Vec512<uint16_t> b) { + return Vec512<uint16_t>{_mm512_mulhi_epu16(a.raw, b.raw)}; +} +HWY_API Vec512<int16_t> MulHigh(const Vec512<int16_t> a, + const Vec512<int16_t> b) { + return Vec512<int16_t>{_mm512_mulhi_epi16(a.raw, b.raw)}; +} + +// Multiplies even lanes (0, 2 ..) and places the double-wide result into +// even and the upper half into its odd neighbor lane. +HWY_API Vec512<int64_t> MulEven(const Vec512<int32_t> a, + const Vec512<int32_t> b) { + return Vec512<int64_t>{_mm512_mul_epi32(a.raw, b.raw)}; +} +HWY_API Vec512<uint64_t> MulEven(const Vec512<uint32_t> a, + const Vec512<uint32_t> b) { + return Vec512<uint64_t>{_mm512_mul_epu32(a.raw, b.raw)}; +} + +// ------------------------------ Neg (Sub) + +template <typename T, HWY_IF_FLOAT(T)> +HWY_API Vec512<T> Neg(const Vec512<T> v) { + return Xor(v, SignBit(Full512<T>())); +} + +template <typename T, HWY_IF_NOT_FLOAT(T)> +HWY_API Vec512<T> Neg(const Vec512<T> v) { + return Zero(Full512<T>()) - v; +} + +// ------------------------------ Floating-point mul / div + +HWY_API Vec512<float> operator*(const Vec512<float> a, const Vec512<float> b) { + return Vec512<float>{_mm512_mul_ps(a.raw, b.raw)}; +} +HWY_API Vec512<double> operator*(const Vec512<double> a, + const Vec512<double> b) { + return Vec512<double>{_mm512_mul_pd(a.raw, b.raw)}; +} + +HWY_API Vec512<float> operator/(const Vec512<float> a, const Vec512<float> b) { + return Vec512<float>{_mm512_div_ps(a.raw, b.raw)}; +} +HWY_API Vec512<double> operator/(const Vec512<double> a, + const Vec512<double> b) { + return Vec512<double>{_mm512_div_pd(a.raw, b.raw)}; +} + +// Approximate reciprocal +HWY_API Vec512<float> ApproximateReciprocal(const Vec512<float> v) { + return Vec512<float>{_mm512_rcp14_ps(v.raw)}; +} + +// Absolute value of difference. +HWY_API Vec512<float> AbsDiff(const Vec512<float> a, const Vec512<float> b) { + return Abs(a - b); +} + +// ------------------------------ Floating-point multiply-add variants + +// Returns mul * x + add +HWY_API Vec512<float> MulAdd(const Vec512<float> mul, const Vec512<float> x, + const Vec512<float> add) { + return Vec512<float>{_mm512_fmadd_ps(mul.raw, x.raw, add.raw)}; +} +HWY_API Vec512<double> MulAdd(const Vec512<double> mul, const Vec512<double> x, + const Vec512<double> add) { + return Vec512<double>{_mm512_fmadd_pd(mul.raw, x.raw, add.raw)}; +} + +// Returns add - mul * x +HWY_API Vec512<float> NegMulAdd(const Vec512<float> mul, const Vec512<float> x, + const Vec512<float> add) { + return Vec512<float>{_mm512_fnmadd_ps(mul.raw, x.raw, add.raw)}; +} +HWY_API Vec512<double> NegMulAdd(const Vec512<double> mul, + const Vec512<double> x, + const Vec512<double> add) { + return Vec512<double>{_mm512_fnmadd_pd(mul.raw, x.raw, add.raw)}; +} + +// Returns mul * x - sub +HWY_API Vec512<float> MulSub(const Vec512<float> mul, const Vec512<float> x, + const Vec512<float> sub) { + return Vec512<float>{_mm512_fmsub_ps(mul.raw, x.raw, sub.raw)}; +} +HWY_API Vec512<double> MulSub(const Vec512<double> mul, const Vec512<double> x, + const Vec512<double> sub) { + return Vec512<double>{_mm512_fmsub_pd(mul.raw, x.raw, sub.raw)}; +} + +// Returns -mul * x - sub +HWY_API Vec512<float> NegMulSub(const Vec512<float> mul, const Vec512<float> x, + const Vec512<float> sub) { + return Vec512<float>{_mm512_fnmsub_ps(mul.raw, x.raw, sub.raw)}; +} +HWY_API Vec512<double> NegMulSub(const Vec512<double> mul, + const Vec512<double> x, + const Vec512<double> sub) { + return Vec512<double>{_mm512_fnmsub_pd(mul.raw, x.raw, sub.raw)}; +} + +// ------------------------------ Floating-point square root + +// Full precision square root +HWY_API Vec512<float> Sqrt(const Vec512<float> v) { + return Vec512<float>{_mm512_sqrt_ps(v.raw)}; +} +HWY_API Vec512<double> Sqrt(const Vec512<double> v) { + return Vec512<double>{_mm512_sqrt_pd(v.raw)}; +} + +// Approximate reciprocal square root +HWY_API Vec512<float> ApproximateReciprocalSqrt(const Vec512<float> v) { + return Vec512<float>{_mm512_rsqrt14_ps(v.raw)}; +} + +// ------------------------------ Floating-point rounding + +// Work around warnings in the intrinsic definitions (passing -1 as a mask). +HWY_DIAGNOSTICS(push) +HWY_DIAGNOSTICS_OFF(disable : 4245 4365, ignored "-Wsign-conversion") + +// Toward nearest integer, tie to even +HWY_API Vec512<float> Round(const Vec512<float> v) { + return Vec512<float>{_mm512_roundscale_ps( + v.raw, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)}; +} +HWY_API Vec512<double> Round(const Vec512<double> v) { + return Vec512<double>{_mm512_roundscale_pd( + v.raw, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)}; +} + +// Toward zero, aka truncate +HWY_API Vec512<float> Trunc(const Vec512<float> v) { + return Vec512<float>{ + _mm512_roundscale_ps(v.raw, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC)}; +} +HWY_API Vec512<double> Trunc(const Vec512<double> v) { + return Vec512<double>{ + _mm512_roundscale_pd(v.raw, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC)}; +} + +// Toward +infinity, aka ceiling +HWY_API Vec512<float> Ceil(const Vec512<float> v) { + return Vec512<float>{ + _mm512_roundscale_ps(v.raw, _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC)}; +} +HWY_API Vec512<double> Ceil(const Vec512<double> v) { + return Vec512<double>{ + _mm512_roundscale_pd(v.raw, _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC)}; +} + +// Toward -infinity, aka floor +HWY_API Vec512<float> Floor(const Vec512<float> v) { + return Vec512<float>{ + _mm512_roundscale_ps(v.raw, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC)}; +} +HWY_API Vec512<double> Floor(const Vec512<double> v) { + return Vec512<double>{ + _mm512_roundscale_pd(v.raw, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC)}; +} + +HWY_DIAGNOSTICS(pop) + +// ================================================== COMPARE + +// Comparisons set a mask bit to 1 if the condition is true, else 0. + +template <typename TFrom, typename TTo> +HWY_API Mask512<TTo> RebindMask(Full512<TTo> /*tag*/, Mask512<TFrom> m) { + static_assert(sizeof(TFrom) == sizeof(TTo), "Must have same size"); + return Mask512<TTo>{m.raw}; +} + +namespace detail { + +template <typename T> +HWY_INLINE Mask512<T> TestBit(hwy::SizeTag<1> /*tag*/, const Vec512<T> v, + const Vec512<T> bit) { + return Mask512<T>{_mm512_test_epi8_mask(v.raw, bit.raw)}; +} +template <typename T> +HWY_INLINE Mask512<T> TestBit(hwy::SizeTag<2> /*tag*/, const Vec512<T> v, + const Vec512<T> bit) { + return Mask512<T>{_mm512_test_epi16_mask(v.raw, bit.raw)}; +} +template <typename T> +HWY_INLINE Mask512<T> TestBit(hwy::SizeTag<4> /*tag*/, const Vec512<T> v, + const Vec512<T> bit) { + return Mask512<T>{_mm512_test_epi32_mask(v.raw, bit.raw)}; +} +template <typename T> +HWY_INLINE Mask512<T> TestBit(hwy::SizeTag<8> /*tag*/, const Vec512<T> v, + const Vec512<T> bit) { + return Mask512<T>{_mm512_test_epi64_mask(v.raw, bit.raw)}; +} + +} // namespace detail + +template <typename T> +HWY_API Mask512<T> TestBit(const Vec512<T> v, const Vec512<T> bit) { + static_assert(!hwy::IsFloat<T>(), "Only integer vectors supported"); + return detail::TestBit(hwy::SizeTag<sizeof(T)>(), v, bit); +} + +// ------------------------------ Equality + +template <typename T, HWY_IF_LANE_SIZE(T, 1)> +HWY_API Mask512<T> operator==(Vec512<T> a, Vec512<T> b) { + return Mask512<T>{_mm512_cmpeq_epi8_mask(a.raw, b.raw)}; +} +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Mask512<T> operator==(Vec512<T> a, Vec512<T> b) { + return Mask512<T>{_mm512_cmpeq_epi16_mask(a.raw, b.raw)}; +} +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Mask512<T> operator==(Vec512<T> a, Vec512<T> b) { + return Mask512<T>{_mm512_cmpeq_epi32_mask(a.raw, b.raw)}; +} +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Mask512<T> operator==(Vec512<T> a, Vec512<T> b) { + return Mask512<T>{_mm512_cmpeq_epi64_mask(a.raw, b.raw)}; +} + +HWY_API Mask512<float> operator==(Vec512<float> a, Vec512<float> b) { + return Mask512<float>{_mm512_cmp_ps_mask(a.raw, b.raw, _CMP_EQ_OQ)}; +} + +HWY_API Mask512<double> operator==(Vec512<double> a, Vec512<double> b) { + return Mask512<double>{_mm512_cmp_pd_mask(a.raw, b.raw, _CMP_EQ_OQ)}; +} + +// ------------------------------ Inequality + +template <typename T, HWY_IF_LANE_SIZE(T, 1)> +HWY_API Mask512<T> operator!=(Vec512<T> a, Vec512<T> b) { + return Mask512<T>{_mm512_cmpneq_epi8_mask(a.raw, b.raw)}; +} +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Mask512<T> operator!=(Vec512<T> a, Vec512<T> b) { + return Mask512<T>{_mm512_cmpneq_epi16_mask(a.raw, b.raw)}; +} +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Mask512<T> operator!=(Vec512<T> a, Vec512<T> b) { + return Mask512<T>{_mm512_cmpneq_epi32_mask(a.raw, b.raw)}; +} +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Mask512<T> operator!=(Vec512<T> a, Vec512<T> b) { + return Mask512<T>{_mm512_cmpneq_epi64_mask(a.raw, b.raw)}; +} + +HWY_API Mask512<float> operator!=(Vec512<float> a, Vec512<float> b) { + return Mask512<float>{_mm512_cmp_ps_mask(a.raw, b.raw, _CMP_NEQ_OQ)}; +} + +HWY_API Mask512<double> operator!=(Vec512<double> a, Vec512<double> b) { + return Mask512<double>{_mm512_cmp_pd_mask(a.raw, b.raw, _CMP_NEQ_OQ)}; +} + +// ------------------------------ Strict inequality + +HWY_API Mask512<uint8_t> operator>(Vec512<uint8_t> a, Vec512<uint8_t> b) { + return Mask512<uint8_t>{_mm512_cmpgt_epu8_mask(a.raw, b.raw)}; +} +HWY_API Mask512<uint16_t> operator>(Vec512<uint16_t> a, Vec512<uint16_t> b) { + return Mask512<uint16_t>{_mm512_cmpgt_epu16_mask(a.raw, b.raw)}; +} +HWY_API Mask512<uint32_t> operator>(Vec512<uint32_t> a, Vec512<uint32_t> b) { + return Mask512<uint32_t>{_mm512_cmpgt_epu32_mask(a.raw, b.raw)}; +} +HWY_API Mask512<uint64_t> operator>(Vec512<uint64_t> a, Vec512<uint64_t> b) { + return Mask512<uint64_t>{_mm512_cmpgt_epu64_mask(a.raw, b.raw)}; +} + +HWY_API Mask512<int8_t> operator>(Vec512<int8_t> a, Vec512<int8_t> b) { + return Mask512<int8_t>{_mm512_cmpgt_epi8_mask(a.raw, b.raw)}; +} +HWY_API Mask512<int16_t> operator>(Vec512<int16_t> a, Vec512<int16_t> b) { + return Mask512<int16_t>{_mm512_cmpgt_epi16_mask(a.raw, b.raw)}; +} +HWY_API Mask512<int32_t> operator>(Vec512<int32_t> a, Vec512<int32_t> b) { + return Mask512<int32_t>{_mm512_cmpgt_epi32_mask(a.raw, b.raw)}; +} +HWY_API Mask512<int64_t> operator>(Vec512<int64_t> a, Vec512<int64_t> b) { + return Mask512<int64_t>{_mm512_cmpgt_epi64_mask(a.raw, b.raw)}; +} + +HWY_API Mask512<float> operator>(Vec512<float> a, Vec512<float> b) { + return Mask512<float>{_mm512_cmp_ps_mask(a.raw, b.raw, _CMP_GT_OQ)}; +} +HWY_API Mask512<double> operator>(Vec512<double> a, Vec512<double> b) { + return Mask512<double>{_mm512_cmp_pd_mask(a.raw, b.raw, _CMP_GT_OQ)}; +} + +// ------------------------------ Weak inequality + +HWY_API Mask512<float> operator>=(Vec512<float> a, Vec512<float> b) { + return Mask512<float>{_mm512_cmp_ps_mask(a.raw, b.raw, _CMP_GE_OQ)}; +} +HWY_API Mask512<double> operator>=(Vec512<double> a, Vec512<double> b) { + return Mask512<double>{_mm512_cmp_pd_mask(a.raw, b.raw, _CMP_GE_OQ)}; +} + +// ------------------------------ Reversed comparisons + +template <typename T> +HWY_API Mask512<T> operator<(Vec512<T> a, Vec512<T> b) { + return b > a; +} + +template <typename T> +HWY_API Mask512<T> operator<=(Vec512<T> a, Vec512<T> b) { + return b >= a; +} + +// ------------------------------ Mask + +namespace detail { + +template <typename T> +HWY_INLINE Mask512<T> MaskFromVec(hwy::SizeTag<1> /*tag*/, const Vec512<T> v) { + return Mask512<T>{_mm512_movepi8_mask(v.raw)}; +} +template <typename T> +HWY_INLINE Mask512<T> MaskFromVec(hwy::SizeTag<2> /*tag*/, const Vec512<T> v) { + return Mask512<T>{_mm512_movepi16_mask(v.raw)}; +} +template <typename T> +HWY_INLINE Mask512<T> MaskFromVec(hwy::SizeTag<4> /*tag*/, const Vec512<T> v) { + return Mask512<T>{_mm512_movepi32_mask(v.raw)}; +} +template <typename T> +HWY_INLINE Mask512<T> MaskFromVec(hwy::SizeTag<8> /*tag*/, const Vec512<T> v) { + return Mask512<T>{_mm512_movepi64_mask(v.raw)}; +} + +} // namespace detail + +template <typename T> +HWY_API Mask512<T> MaskFromVec(const Vec512<T> v) { + return detail::MaskFromVec(hwy::SizeTag<sizeof(T)>(), v); +} +// There do not seem to be native floating-point versions of these instructions. +HWY_API Mask512<float> MaskFromVec(const Vec512<float> v) { + return Mask512<float>{MaskFromVec(BitCast(Full512<int32_t>(), v)).raw}; +} +HWY_API Mask512<double> MaskFromVec(const Vec512<double> v) { + return Mask512<double>{MaskFromVec(BitCast(Full512<int64_t>(), v)).raw}; +} + +HWY_API Vec512<uint8_t> VecFromMask(const Mask512<uint8_t> v) { + return Vec512<uint8_t>{_mm512_movm_epi8(v.raw)}; +} +HWY_API Vec512<int8_t> VecFromMask(const Mask512<int8_t> v) { + return Vec512<int8_t>{_mm512_movm_epi8(v.raw)}; +} + +HWY_API Vec512<uint16_t> VecFromMask(const Mask512<uint16_t> v) { + return Vec512<uint16_t>{_mm512_movm_epi16(v.raw)}; +} +HWY_API Vec512<int16_t> VecFromMask(const Mask512<int16_t> v) { + return Vec512<int16_t>{_mm512_movm_epi16(v.raw)}; +} + +HWY_API Vec512<uint32_t> VecFromMask(const Mask512<uint32_t> v) { + return Vec512<uint32_t>{_mm512_movm_epi32(v.raw)}; +} +HWY_API Vec512<int32_t> VecFromMask(const Mask512<int32_t> v) { + return Vec512<int32_t>{_mm512_movm_epi32(v.raw)}; +} +HWY_API Vec512<float> VecFromMask(const Mask512<float> v) { + return Vec512<float>{_mm512_castsi512_ps(_mm512_movm_epi32(v.raw))}; +} + +HWY_API Vec512<uint64_t> VecFromMask(const Mask512<uint64_t> v) { + return Vec512<uint64_t>{_mm512_movm_epi64(v.raw)}; +} +HWY_API Vec512<int64_t> VecFromMask(const Mask512<int64_t> v) { + return Vec512<int64_t>{_mm512_movm_epi64(v.raw)}; +} +HWY_API Vec512<double> VecFromMask(const Mask512<double> v) { + return Vec512<double>{_mm512_castsi512_pd(_mm512_movm_epi64(v.raw))}; +} + +template <typename T> +HWY_API Vec512<T> VecFromMask(Full512<T> /* tag */, const Mask512<T> v) { + return VecFromMask(v); +} + +// ------------------------------ Mask logical + +namespace detail { + +template <typename T> +HWY_INLINE Mask512<T> Not(hwy::SizeTag<1> /*tag*/, const Mask512<T> m) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask512<T>{_knot_mask64(m.raw)}; +#else + return Mask512<T>{~m.raw}; +#endif +} +template <typename T> +HWY_INLINE Mask512<T> Not(hwy::SizeTag<2> /*tag*/, const Mask512<T> m) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask512<T>{_knot_mask32(m.raw)}; +#else + return Mask512<T>{~m.raw}; +#endif +} +template <typename T> +HWY_INLINE Mask512<T> Not(hwy::SizeTag<4> /*tag*/, const Mask512<T> m) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask512<T>{_knot_mask16(m.raw)}; +#else + return Mask512<T>{static_cast<uint16_t>(~m.raw & 0xFFFF)}; +#endif +} +template <typename T> +HWY_INLINE Mask512<T> Not(hwy::SizeTag<8> /*tag*/, const Mask512<T> m) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask512<T>{_knot_mask8(m.raw)}; +#else + return Mask512<T>{static_cast<uint8_t>(~m.raw & 0xFF)}; +#endif +} + +template <typename T> +HWY_INLINE Mask512<T> And(hwy::SizeTag<1> /*tag*/, const Mask512<T> a, + const Mask512<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask512<T>{_kand_mask64(a.raw, b.raw)}; +#else + return Mask512<T>{a.raw & b.raw}; +#endif +} +template <typename T> +HWY_INLINE Mask512<T> And(hwy::SizeTag<2> /*tag*/, const Mask512<T> a, + const Mask512<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask512<T>{_kand_mask32(a.raw, b.raw)}; +#else + return Mask512<T>{a.raw & b.raw}; +#endif +} +template <typename T> +HWY_INLINE Mask512<T> And(hwy::SizeTag<4> /*tag*/, const Mask512<T> a, + const Mask512<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask512<T>{_kand_mask16(a.raw, b.raw)}; +#else + return Mask512<T>{static_cast<uint16_t>(a.raw & b.raw)}; +#endif +} +template <typename T> +HWY_INLINE Mask512<T> And(hwy::SizeTag<8> /*tag*/, const Mask512<T> a, + const Mask512<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask512<T>{_kand_mask8(a.raw, b.raw)}; +#else + return Mask512<T>{static_cast<uint8_t>(a.raw & b.raw)}; +#endif +} + +template <typename T> +HWY_INLINE Mask512<T> AndNot(hwy::SizeTag<1> /*tag*/, const Mask512<T> a, + const Mask512<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask512<T>{_kandn_mask64(a.raw, b.raw)}; +#else + return Mask512<T>{~a.raw & b.raw}; +#endif +} +template <typename T> +HWY_INLINE Mask512<T> AndNot(hwy::SizeTag<2> /*tag*/, const Mask512<T> a, + const Mask512<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask512<T>{_kandn_mask32(a.raw, b.raw)}; +#else + return Mask512<T>{~a.raw & b.raw}; +#endif +} +template <typename T> +HWY_INLINE Mask512<T> AndNot(hwy::SizeTag<4> /*tag*/, const Mask512<T> a, + const Mask512<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask512<T>{_kandn_mask16(a.raw, b.raw)}; +#else + return Mask512<T>{static_cast<uint16_t>(~a.raw & b.raw)}; +#endif +} +template <typename T> +HWY_INLINE Mask512<T> AndNot(hwy::SizeTag<8> /*tag*/, const Mask512<T> a, + const Mask512<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask512<T>{_kandn_mask8(a.raw, b.raw)}; +#else + return Mask512<T>{static_cast<uint8_t>(~a.raw & b.raw)}; +#endif +} + +template <typename T> +HWY_INLINE Mask512<T> Or(hwy::SizeTag<1> /*tag*/, const Mask512<T> a, + const Mask512<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask512<T>{_kor_mask64(a.raw, b.raw)}; +#else + return Mask512<T>{a.raw | b.raw}; +#endif +} +template <typename T> +HWY_INLINE Mask512<T> Or(hwy::SizeTag<2> /*tag*/, const Mask512<T> a, + const Mask512<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask512<T>{_kor_mask32(a.raw, b.raw)}; +#else + return Mask512<T>{a.raw | b.raw}; +#endif +} +template <typename T> +HWY_INLINE Mask512<T> Or(hwy::SizeTag<4> /*tag*/, const Mask512<T> a, + const Mask512<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask512<T>{_kor_mask16(a.raw, b.raw)}; +#else + return Mask512<T>{static_cast<uint16_t>(a.raw | b.raw)}; +#endif +} +template <typename T> +HWY_INLINE Mask512<T> Or(hwy::SizeTag<8> /*tag*/, const Mask512<T> a, + const Mask512<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask512<T>{_kor_mask8(a.raw, b.raw)}; +#else + return Mask512<T>{static_cast<uint8_t>(a.raw | b.raw)}; +#endif +} + +template <typename T> +HWY_INLINE Mask512<T> Xor(hwy::SizeTag<1> /*tag*/, const Mask512<T> a, + const Mask512<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask512<T>{_kxor_mask64(a.raw, b.raw)}; +#else + return Mask512<T>{a.raw ^ b.raw}; +#endif +} +template <typename T> +HWY_INLINE Mask512<T> Xor(hwy::SizeTag<2> /*tag*/, const Mask512<T> a, + const Mask512<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask512<T>{_kxor_mask32(a.raw, b.raw)}; +#else + return Mask512<T>{a.raw ^ b.raw}; +#endif +} +template <typename T> +HWY_INLINE Mask512<T> Xor(hwy::SizeTag<4> /*tag*/, const Mask512<T> a, + const Mask512<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask512<T>{_kxor_mask16(a.raw, b.raw)}; +#else + return Mask512<T>{static_cast<uint16_t>(a.raw ^ b.raw)}; +#endif +} +template <typename T> +HWY_INLINE Mask512<T> Xor(hwy::SizeTag<8> /*tag*/, const Mask512<T> a, + const Mask512<T> b) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return Mask512<T>{_kxor_mask8(a.raw, b.raw)}; +#else + return Mask512<T>{static_cast<uint8_t>(a.raw ^ b.raw)}; +#endif +} + +} // namespace detail + +template <typename T> +HWY_API Mask512<T> Not(const Mask512<T> m) { + return detail::Not(hwy::SizeTag<sizeof(T)>(), m); +} + +template <typename T> +HWY_API Mask512<T> And(const Mask512<T> a, Mask512<T> b) { + return detail::And(hwy::SizeTag<sizeof(T)>(), a, b); +} + +template <typename T> +HWY_API Mask512<T> AndNot(const Mask512<T> a, Mask512<T> b) { + return detail::AndNot(hwy::SizeTag<sizeof(T)>(), a, b); +} + +template <typename T> +HWY_API Mask512<T> Or(const Mask512<T> a, Mask512<T> b) { + return detail::Or(hwy::SizeTag<sizeof(T)>(), a, b); +} + +template <typename T> +HWY_API Mask512<T> Xor(const Mask512<T> a, Mask512<T> b) { + return detail::Xor(hwy::SizeTag<sizeof(T)>(), a, b); +} + +// ------------------------------ BroadcastSignBit (ShiftRight, compare, mask) + +HWY_API Vec512<int8_t> BroadcastSignBit(const Vec512<int8_t> v) { + return VecFromMask(v < Zero(Full512<int8_t>())); +} + +HWY_API Vec512<int16_t> BroadcastSignBit(const Vec512<int16_t> v) { + return ShiftRight<15>(v); +} + +HWY_API Vec512<int32_t> BroadcastSignBit(const Vec512<int32_t> v) { + return ShiftRight<31>(v); +} + +HWY_API Vec512<int64_t> BroadcastSignBit(const Vec512<int64_t> v) { + return Vec512<int64_t>{_mm512_srai_epi64(v.raw, 63)}; +} + +// ================================================== MEMORY + +// ------------------------------ Load + +template <typename T> +HWY_API Vec512<T> Load(Full512<T> /* tag */, const T* HWY_RESTRICT aligned) { + return Vec512<T>{_mm512_load_si512(aligned)}; +} +HWY_API Vec512<float> Load(Full512<float> /* tag */, + const float* HWY_RESTRICT aligned) { + return Vec512<float>{_mm512_load_ps(aligned)}; +} +HWY_API Vec512<double> Load(Full512<double> /* tag */, + const double* HWY_RESTRICT aligned) { + return Vec512<double>{_mm512_load_pd(aligned)}; +} + +template <typename T> +HWY_API Vec512<T> LoadU(Full512<T> /* tag */, const T* HWY_RESTRICT p) { + return Vec512<T>{_mm512_loadu_si512(p)}; +} +HWY_API Vec512<float> LoadU(Full512<float> /* tag */, + const float* HWY_RESTRICT p) { + return Vec512<float>{_mm512_loadu_ps(p)}; +} +HWY_API Vec512<double> LoadU(Full512<double> /* tag */, + const double* HWY_RESTRICT p) { + return Vec512<double>{_mm512_loadu_pd(p)}; +} + +// ------------------------------ MaskedLoad + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec512<T> MaskedLoad(Mask512<T> m, Full512<T> /* tag */, + const T* HWY_RESTRICT aligned) { + return Vec512<T>{_mm512_maskz_load_epi32(m.raw, aligned)}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec512<T> MaskedLoad(Mask512<T> m, Full512<T> /* tag */, + const T* HWY_RESTRICT aligned) { + return Vec512<T>{_mm512_maskz_load_epi64(m.raw, aligned)}; +} + +HWY_API Vec512<float> MaskedLoad(Mask512<float> m, Full512<float> /* tag */, + const float* HWY_RESTRICT aligned) { + return Vec512<float>{_mm512_maskz_load_ps(m.raw, aligned)}; +} + +HWY_API Vec512<double> MaskedLoad(Mask512<double> m, Full512<double> /* tag */, + const double* HWY_RESTRICT aligned) { + return Vec512<double>{_mm512_maskz_load_pd(m.raw, aligned)}; +} + +// There is no load_epi8/16, so use loadu instead. +template <typename T, HWY_IF_LANE_SIZE(T, 1)> +HWY_API Vec512<T> MaskedLoad(Mask512<T> m, Full512<T> /* tag */, + const T* HWY_RESTRICT aligned) { + return Vec512<T>{_mm512_maskz_loadu_epi8(m.raw, aligned)}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Vec512<T> MaskedLoad(Mask512<T> m, Full512<T> /* tag */, + const T* HWY_RESTRICT aligned) { + return Vec512<T>{_mm512_maskz_loadu_epi16(m.raw, aligned)}; +} + +// ------------------------------ LoadDup128 + +// Loads 128 bit and duplicates into both 128-bit halves. This avoids the +// 3-cycle cost of moving data between 128-bit halves and avoids port 5. +template <typename T> +HWY_API Vec512<T> LoadDup128(Full512<T> /* tag */, + const T* const HWY_RESTRICT p) { + // Clang 3.9 generates VINSERTF128 which is slower, but inline assembly leads + // to "invalid output size for constraint" without -mavx512: + // https://gcc.godbolt.org/z/-Jt_-F +#if HWY_LOADDUP_ASM + __m512i out; + asm("vbroadcasti128 %1, %[reg]" : [ reg ] "=x"(out) : "m"(p[0])); + return Vec512<T>{out}; +#else + const auto x4 = LoadU(Full128<T>(), p); + return Vec512<T>{_mm512_broadcast_i32x4(x4.raw)}; +#endif +} +HWY_API Vec512<float> LoadDup128(Full512<float> /* tag */, + const float* const HWY_RESTRICT p) { +#if HWY_LOADDUP_ASM + __m512 out; + asm("vbroadcastf128 %1, %[reg]" : [ reg ] "=x"(out) : "m"(p[0])); + return Vec512<float>{out}; +#else + const __m128 x4 = _mm_loadu_ps(p); + return Vec512<float>{_mm512_broadcast_f32x4(x4)}; +#endif +} + +HWY_API Vec512<double> LoadDup128(Full512<double> /* tag */, + const double* const HWY_RESTRICT p) { +#if HWY_LOADDUP_ASM + __m512d out; + asm("vbroadcastf128 %1, %[reg]" : [ reg ] "=x"(out) : "m"(p[0])); + return Vec512<double>{out}; +#else + const __m128d x2 = _mm_loadu_pd(p); + return Vec512<double>{_mm512_broadcast_f64x2(x2)}; +#endif +} + +// ------------------------------ Store + +template <typename T> +HWY_API void Store(const Vec512<T> v, Full512<T> /* tag */, + T* HWY_RESTRICT aligned) { + _mm512_store_si512(reinterpret_cast<__m512i*>(aligned), v.raw); +} +HWY_API void Store(const Vec512<float> v, Full512<float> /* tag */, + float* HWY_RESTRICT aligned) { + _mm512_store_ps(aligned, v.raw); +} +HWY_API void Store(const Vec512<double> v, Full512<double> /* tag */, + double* HWY_RESTRICT aligned) { + _mm512_store_pd(aligned, v.raw); +} + +template <typename T> +HWY_API void StoreU(const Vec512<T> v, Full512<T> /* tag */, + T* HWY_RESTRICT p) { + _mm512_storeu_si512(reinterpret_cast<__m512i*>(p), v.raw); +} +HWY_API void StoreU(const Vec512<float> v, Full512<float> /* tag */, + float* HWY_RESTRICT p) { + _mm512_storeu_ps(p, v.raw); +} +HWY_API void StoreU(const Vec512<double> v, Full512<double>, + double* HWY_RESTRICT p) { + _mm512_storeu_pd(p, v.raw); +} + +// ------------------------------ Non-temporal stores + +template <typename T> +HWY_API void Stream(const Vec512<T> v, Full512<T> /* tag */, + T* HWY_RESTRICT aligned) { + _mm512_stream_si512(reinterpret_cast<__m512i*>(aligned), v.raw); +} +HWY_API void Stream(const Vec512<float> v, Full512<float> /* tag */, + float* HWY_RESTRICT aligned) { + _mm512_stream_ps(aligned, v.raw); +} +HWY_API void Stream(const Vec512<double> v, Full512<double>, + double* HWY_RESTRICT aligned) { + _mm512_stream_pd(aligned, v.raw); +} + +// ------------------------------ Scatter + +// Work around warnings in the intrinsic definitions (passing -1 as a mask). +HWY_DIAGNOSTICS(push) +HWY_DIAGNOSTICS_OFF(disable : 4245 4365, ignored "-Wsign-conversion") + +namespace detail { + +template <typename T> +HWY_INLINE void ScatterOffset(hwy::SizeTag<4> /* tag */, Vec512<T> v, + Full512<T> /* tag */, T* HWY_RESTRICT base, + const Vec512<int32_t> offset) { + _mm512_i32scatter_epi32(base, offset.raw, v.raw, 1); +} +template <typename T> +HWY_INLINE void ScatterIndex(hwy::SizeTag<4> /* tag */, Vec512<T> v, + Full512<T> /* tag */, T* HWY_RESTRICT base, + const Vec512<int32_t> index) { + _mm512_i32scatter_epi32(base, index.raw, v.raw, 4); +} + +template <typename T> +HWY_INLINE void ScatterOffset(hwy::SizeTag<8> /* tag */, Vec512<T> v, + Full512<T> /* tag */, T* HWY_RESTRICT base, + const Vec512<int64_t> offset) { + _mm512_i64scatter_epi64(base, offset.raw, v.raw, 1); +} +template <typename T> +HWY_INLINE void ScatterIndex(hwy::SizeTag<8> /* tag */, Vec512<T> v, + Full512<T> /* tag */, T* HWY_RESTRICT base, + const Vec512<int64_t> index) { + _mm512_i64scatter_epi64(base, index.raw, v.raw, 8); +} + +} // namespace detail + +template <typename T, typename Offset> +HWY_API void ScatterOffset(Vec512<T> v, Full512<T> d, T* HWY_RESTRICT base, + const Vec512<Offset> offset) { + static_assert(sizeof(T) == sizeof(Offset), "Must match for portability"); + return detail::ScatterOffset(hwy::SizeTag<sizeof(T)>(), v, d, base, offset); +} +template <typename T, typename Index> +HWY_API void ScatterIndex(Vec512<T> v, Full512<T> d, T* HWY_RESTRICT base, + const Vec512<Index> index) { + static_assert(sizeof(T) == sizeof(Index), "Must match for portability"); + return detail::ScatterIndex(hwy::SizeTag<sizeof(T)>(), v, d, base, index); +} + +HWY_API void ScatterOffset(Vec512<float> v, Full512<float> /* tag */, + float* HWY_RESTRICT base, + const Vec512<int32_t> offset) { + _mm512_i32scatter_ps(base, offset.raw, v.raw, 1); +} +HWY_API void ScatterIndex(Vec512<float> v, Full512<float> /* tag */, + float* HWY_RESTRICT base, + const Vec512<int32_t> index) { + _mm512_i32scatter_ps(base, index.raw, v.raw, 4); +} + +HWY_API void ScatterOffset(Vec512<double> v, Full512<double> /* tag */, + double* HWY_RESTRICT base, + const Vec512<int64_t> offset) { + _mm512_i64scatter_pd(base, offset.raw, v.raw, 1); +} +HWY_API void ScatterIndex(Vec512<double> v, Full512<double> /* tag */, + double* HWY_RESTRICT base, + const Vec512<int64_t> index) { + _mm512_i64scatter_pd(base, index.raw, v.raw, 8); +} + +// ------------------------------ Gather + +namespace detail { + +template <typename T> +HWY_INLINE Vec512<T> GatherOffset(hwy::SizeTag<4> /* tag */, + Full512<T> /* tag */, + const T* HWY_RESTRICT base, + const Vec512<int32_t> offset) { + return Vec512<T>{_mm512_i32gather_epi32(offset.raw, base, 1)}; +} +template <typename T> +HWY_INLINE Vec512<T> GatherIndex(hwy::SizeTag<4> /* tag */, + Full512<T> /* tag */, + const T* HWY_RESTRICT base, + const Vec512<int32_t> index) { + return Vec512<T>{_mm512_i32gather_epi32(index.raw, base, 4)}; +} + +template <typename T> +HWY_INLINE Vec512<T> GatherOffset(hwy::SizeTag<8> /* tag */, + Full512<T> /* tag */, + const T* HWY_RESTRICT base, + const Vec512<int64_t> offset) { + return Vec512<T>{_mm512_i64gather_epi64(offset.raw, base, 1)}; +} +template <typename T> +HWY_INLINE Vec512<T> GatherIndex(hwy::SizeTag<8> /* tag */, + Full512<T> /* tag */, + const T* HWY_RESTRICT base, + const Vec512<int64_t> index) { + return Vec512<T>{_mm512_i64gather_epi64(index.raw, base, 8)}; +} + +} // namespace detail + +template <typename T, typename Offset> +HWY_API Vec512<T> GatherOffset(Full512<T> d, const T* HWY_RESTRICT base, + const Vec512<Offset> offset) { +static_assert(sizeof(T) == sizeof(Offset), "Must match for portability"); + return detail::GatherOffset(hwy::SizeTag<sizeof(T)>(), d, base, offset); +} +template <typename T, typename Index> +HWY_API Vec512<T> GatherIndex(Full512<T> d, const T* HWY_RESTRICT base, + const Vec512<Index> index) { + static_assert(sizeof(T) == sizeof(Index), "Must match for portability"); + return detail::GatherIndex(hwy::SizeTag<sizeof(T)>(), d, base, index); +} + +HWY_API Vec512<float> GatherOffset(Full512<float> /* tag */, + const float* HWY_RESTRICT base, + const Vec512<int32_t> offset) { + return Vec512<float>{_mm512_i32gather_ps(offset.raw, base, 1)}; +} +HWY_API Vec512<float> GatherIndex(Full512<float> /* tag */, + const float* HWY_RESTRICT base, + const Vec512<int32_t> index) { + return Vec512<float>{_mm512_i32gather_ps(index.raw, base, 4)}; +} + +HWY_API Vec512<double> GatherOffset(Full512<double> /* tag */, + const double* HWY_RESTRICT base, + const Vec512<int64_t> offset) { + return Vec512<double>{_mm512_i64gather_pd(offset.raw, base, 1)}; +} +HWY_API Vec512<double> GatherIndex(Full512<double> /* tag */, + const double* HWY_RESTRICT base, + const Vec512<int64_t> index) { + return Vec512<double>{_mm512_i64gather_pd(index.raw, base, 8)}; +} + +HWY_DIAGNOSTICS(pop) + +// ================================================== SWIZZLE + +// ------------------------------ LowerHalf + +template <typename T> +HWY_API Vec256<T> LowerHalf(Full256<T> /* tag */, Vec512<T> v) { + return Vec256<T>{_mm512_castsi512_si256(v.raw)}; +} +HWY_API Vec256<float> LowerHalf(Full256<float> /* tag */, Vec512<float> v) { + return Vec256<float>{_mm512_castps512_ps256(v.raw)}; +} +HWY_API Vec256<double> LowerHalf(Full256<double> /* tag */, Vec512<double> v) { + return Vec256<double>{_mm512_castpd512_pd256(v.raw)}; +} + +template <typename T> +HWY_API Vec256<T> LowerHalf(Vec512<T> v) { + return LowerHalf(Full256<T>(), v); +} + +// ------------------------------ UpperHalf + +template <typename T> +HWY_API Vec256<T> UpperHalf(Full256<T> /* tag */, Vec512<T> v) { + return Vec256<T>{_mm512_extracti32x8_epi32(v.raw, 1)}; +} +HWY_API Vec256<float> UpperHalf(Full256<float> /* tag */, Vec512<float> v) { + return Vec256<float>{_mm512_extractf32x8_ps(v.raw, 1)}; +} +HWY_API Vec256<double> UpperHalf(Full256<double> /* tag */, Vec512<double> v) { + return Vec256<double>{_mm512_extractf64x4_pd(v.raw, 1)}; +} + +// ------------------------------ GetLane (LowerHalf) +template <typename T> +HWY_API T GetLane(const Vec512<T> v) { + return GetLane(LowerHalf(v)); +} + +// ------------------------------ ZeroExtendVector + +// Unfortunately the initial _mm512_castsi256_si512 intrinsic leaves the upper +// bits undefined. Although it makes sense for them to be zero (EVEX encoded +// instructions have that effect), a compiler could decide to optimize out code +// that relies on this. +// +// The newer _mm512_zextsi256_si512 intrinsic fixes this by specifying the +// zeroing, but it is not available on GCC until 10.1. For older GCC, we can +// still obtain the desired code thanks to pattern recognition; note that the +// expensive insert instruction is not actually generated, see +// https://gcc.godbolt.org/z/1MKGaP. + +template <typename T> +HWY_API Vec512<T> ZeroExtendVector(Full512<T> /* tag */, Vec256<T> lo) { +#if !HWY_COMPILER_CLANG && HWY_COMPILER_GCC && (HWY_COMPILER_GCC < 1000) + return Vec512<T>{_mm512_inserti32x8(_mm512_setzero_si512(), lo.raw, 0)}; +#else + return Vec512<T>{_mm512_zextsi256_si512(lo.raw)}; +#endif +} +HWY_API Vec512<float> ZeroExtendVector(Full512<float> /* tag */, + Vec256<float> lo) { +#if !HWY_COMPILER_CLANG && HWY_COMPILER_GCC && (HWY_COMPILER_GCC < 1000) + return Vec512<float>{_mm512_insertf32x8(_mm512_setzero_ps(), lo.raw, 0)}; +#else + return Vec512<float>{_mm512_zextps256_ps512(lo.raw)}; +#endif +} +HWY_API Vec512<double> ZeroExtendVector(Full512<double> /* tag */, + Vec256<double> lo) { +#if !HWY_COMPILER_CLANG && HWY_COMPILER_GCC && (HWY_COMPILER_GCC < 1000) + return Vec512<double>{_mm512_insertf64x4(_mm512_setzero_pd(), lo.raw, 0)}; +#else + return Vec512<double>{_mm512_zextpd256_pd512(lo.raw)}; +#endif +} + +// ------------------------------ Combine + +template <typename T> +HWY_API Vec512<T> Combine(Full512<T> d, Vec256<T> hi, Vec256<T> lo) { + const auto lo512 = ZeroExtendVector(d, lo); + return Vec512<T>{_mm512_inserti32x8(lo512.raw, hi.raw, 1)}; +} +HWY_API Vec512<float> Combine(Full512<float> d, Vec256<float> hi, + Vec256<float> lo) { + const auto lo512 = ZeroExtendVector(d, lo); + return Vec512<float>{_mm512_insertf32x8(lo512.raw, hi.raw, 1)}; +} +HWY_API Vec512<double> Combine(Full512<double> d, Vec256<double> hi, + Vec256<double> lo) { + const auto lo512 = ZeroExtendVector(d, lo); + return Vec512<double>{_mm512_insertf64x4(lo512.raw, hi.raw, 1)}; +} + +// ------------------------------ ShiftLeftBytes + +template <int kBytes, typename T> +HWY_API Vec512<T> ShiftLeftBytes(Full512<T> /* tag */, const Vec512<T> v) { + static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes"); + return Vec512<T>{_mm512_bslli_epi128(v.raw, kBytes)}; +} + +template <int kBytes, typename T> +HWY_API Vec512<T> ShiftLeftBytes(const Vec512<T> v) { + return ShiftLeftBytes<kBytes>(Full512<T>(), v); +} + +// ------------------------------ ShiftLeftLanes + +template <int kLanes, typename T> +HWY_API Vec512<T> ShiftLeftLanes(Full512<T> d, const Vec512<T> v) { + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, ShiftLeftBytes<kLanes * sizeof(T)>(BitCast(d8, v))); +} + +template <int kLanes, typename T> +HWY_API Vec512<T> ShiftLeftLanes(const Vec512<T> v) { + return ShiftLeftLanes<kLanes>(Full512<T>(), v); +} + +// ------------------------------ ShiftRightBytes +template <int kBytes, typename T> +HWY_API Vec512<T> ShiftRightBytes(Full512<T> /* tag */, const Vec512<T> v) { + static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes"); + return Vec512<T>{_mm512_bsrli_epi128(v.raw, kBytes)}; +} + +// ------------------------------ ShiftRightLanes +template <int kLanes, typename T> +HWY_API Vec512<T> ShiftRightLanes(Full512<T> d, const Vec512<T> v) { + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, ShiftRightBytes<kLanes * sizeof(T)>(BitCast(d8, v))); +} + +// ------------------------------ CombineShiftRightBytes + +template <int kBytes, typename T, class V = Vec512<T>> +HWY_API V CombineShiftRightBytes(Full512<T> d, V hi, V lo) { + const Repartition<uint8_t, decltype(d)> d8; + return BitCast(d, Vec512<uint8_t>{_mm512_alignr_epi8( + BitCast(d8, hi).raw, BitCast(d8, lo).raw, kBytes)}); +} + +// ------------------------------ Broadcast/splat any lane + +// Unsigned +template <int kLane> +HWY_API Vec512<uint16_t> Broadcast(const Vec512<uint16_t> v) { + static_assert(0 <= kLane && kLane < 8, "Invalid lane"); + if (kLane < 4) { + const __m512i lo = _mm512_shufflelo_epi16(v.raw, (0x55 * kLane) & 0xFF); + return Vec512<uint16_t>{_mm512_unpacklo_epi64(lo, lo)}; + } else { + const __m512i hi = + _mm512_shufflehi_epi16(v.raw, (0x55 * (kLane - 4)) & 0xFF); + return Vec512<uint16_t>{_mm512_unpackhi_epi64(hi, hi)}; + } +} +template <int kLane> +HWY_API Vec512<uint32_t> Broadcast(const Vec512<uint32_t> v) { + static_assert(0 <= kLane && kLane < 4, "Invalid lane"); + constexpr _MM_PERM_ENUM perm = static_cast<_MM_PERM_ENUM>(0x55 * kLane); + return Vec512<uint32_t>{_mm512_shuffle_epi32(v.raw, perm)}; +} +template <int kLane> +HWY_API Vec512<uint64_t> Broadcast(const Vec512<uint64_t> v) { + static_assert(0 <= kLane && kLane < 2, "Invalid lane"); + constexpr _MM_PERM_ENUM perm = kLane ? _MM_PERM_DCDC : _MM_PERM_BABA; + return Vec512<uint64_t>{_mm512_shuffle_epi32(v.raw, perm)}; +} + +// Signed +template <int kLane> +HWY_API Vec512<int16_t> Broadcast(const Vec512<int16_t> v) { + static_assert(0 <= kLane && kLane < 8, "Invalid lane"); + if (kLane < 4) { + const __m512i lo = _mm512_shufflelo_epi16(v.raw, (0x55 * kLane) & 0xFF); + return Vec512<int16_t>{_mm512_unpacklo_epi64(lo, lo)}; + } else { + const __m512i hi = + _mm512_shufflehi_epi16(v.raw, (0x55 * (kLane - 4)) & 0xFF); + return Vec512<int16_t>{_mm512_unpackhi_epi64(hi, hi)}; + } +} +template <int kLane> +HWY_API Vec512<int32_t> Broadcast(const Vec512<int32_t> v) { + static_assert(0 <= kLane && kLane < 4, "Invalid lane"); + constexpr _MM_PERM_ENUM perm = static_cast<_MM_PERM_ENUM>(0x55 * kLane); + return Vec512<int32_t>{_mm512_shuffle_epi32(v.raw, perm)}; +} +template <int kLane> +HWY_API Vec512<int64_t> Broadcast(const Vec512<int64_t> v) { + static_assert(0 <= kLane && kLane < 2, "Invalid lane"); + constexpr _MM_PERM_ENUM perm = kLane ? _MM_PERM_DCDC : _MM_PERM_BABA; + return Vec512<int64_t>{_mm512_shuffle_epi32(v.raw, perm)}; +} + +// Float +template <int kLane> +HWY_API Vec512<float> Broadcast(const Vec512<float> v) { + static_assert(0 <= kLane && kLane < 4, "Invalid lane"); + constexpr _MM_PERM_ENUM perm = static_cast<_MM_PERM_ENUM>(0x55 * kLane); + return Vec512<float>{_mm512_shuffle_ps(v.raw, v.raw, perm)}; +} +template <int kLane> +HWY_API Vec512<double> Broadcast(const Vec512<double> v) { + static_assert(0 <= kLane && kLane < 2, "Invalid lane"); + constexpr _MM_PERM_ENUM perm = static_cast<_MM_PERM_ENUM>(0xFF * kLane); + return Vec512<double>{_mm512_shuffle_pd(v.raw, v.raw, perm)}; +} + +// ------------------------------ Hard-coded shuffles + +// Notation: let Vec512<int32_t> have lanes 7,6,5,4,3,2,1,0 (0 is +// least-significant). Shuffle0321 rotates four-lane blocks one lane to the +// right (the previous least-significant lane is now most-significant => +// 47650321). These could also be implemented via CombineShiftRightBytes but +// the shuffle_abcd notation is more convenient. + +// Swap 32-bit halves in 64-bit halves. +HWY_API Vec512<uint32_t> Shuffle2301(const Vec512<uint32_t> v) { + return Vec512<uint32_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_CDAB)}; +} +HWY_API Vec512<int32_t> Shuffle2301(const Vec512<int32_t> v) { + return Vec512<int32_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_CDAB)}; +} +HWY_API Vec512<float> Shuffle2301(const Vec512<float> v) { + return Vec512<float>{_mm512_shuffle_ps(v.raw, v.raw, _MM_PERM_CDAB)}; +} + +// Swap 64-bit halves +HWY_API Vec512<uint32_t> Shuffle1032(const Vec512<uint32_t> v) { + return Vec512<uint32_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_BADC)}; +} +HWY_API Vec512<int32_t> Shuffle1032(const Vec512<int32_t> v) { + return Vec512<int32_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_BADC)}; +} +HWY_API Vec512<float> Shuffle1032(const Vec512<float> v) { + // Shorter encoding than _mm512_permute_ps. + return Vec512<float>{_mm512_shuffle_ps(v.raw, v.raw, _MM_PERM_BADC)}; +} +HWY_API Vec512<uint64_t> Shuffle01(const Vec512<uint64_t> v) { + return Vec512<uint64_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_BADC)}; +} +HWY_API Vec512<int64_t> Shuffle01(const Vec512<int64_t> v) { + return Vec512<int64_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_BADC)}; +} +HWY_API Vec512<double> Shuffle01(const Vec512<double> v) { + // Shorter encoding than _mm512_permute_pd. + return Vec512<double>{_mm512_shuffle_pd(v.raw, v.raw, _MM_PERM_BBBB)}; +} + +// Rotate right 32 bits +HWY_API Vec512<uint32_t> Shuffle0321(const Vec512<uint32_t> v) { + return Vec512<uint32_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_ADCB)}; +} +HWY_API Vec512<int32_t> Shuffle0321(const Vec512<int32_t> v) { + return Vec512<int32_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_ADCB)}; +} +HWY_API Vec512<float> Shuffle0321(const Vec512<float> v) { + return Vec512<float>{_mm512_shuffle_ps(v.raw, v.raw, _MM_PERM_ADCB)}; +} +// Rotate left 32 bits +HWY_API Vec512<uint32_t> Shuffle2103(const Vec512<uint32_t> v) { + return Vec512<uint32_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_CBAD)}; +} +HWY_API Vec512<int32_t> Shuffle2103(const Vec512<int32_t> v) { + return Vec512<int32_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_CBAD)}; +} +HWY_API Vec512<float> Shuffle2103(const Vec512<float> v) { + return Vec512<float>{_mm512_shuffle_ps(v.raw, v.raw, _MM_PERM_CBAD)}; +} + +// Reverse +HWY_API Vec512<uint32_t> Shuffle0123(const Vec512<uint32_t> v) { + return Vec512<uint32_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_ABCD)}; +} +HWY_API Vec512<int32_t> Shuffle0123(const Vec512<int32_t> v) { + return Vec512<int32_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_ABCD)}; +} +HWY_API Vec512<float> Shuffle0123(const Vec512<float> v) { + return Vec512<float>{_mm512_shuffle_ps(v.raw, v.raw, _MM_PERM_ABCD)}; +} + +// ------------------------------ TableLookupLanes + +// Returned by SetTableIndices/IndicesFromVec for use by TableLookupLanes. +template <typename T> +struct Indices512 { + __m512i raw; +}; + +template <typename T, typename TI> +HWY_API Indices512<T> IndicesFromVec(Full512<T> /* tag */, Vec512<TI> vec) { + static_assert(sizeof(T) == sizeof(TI), "Index size must match lane"); +#if HWY_IS_DEBUG_BUILD + const Full512<TI> di; + HWY_DASSERT(AllFalse(di, Lt(vec, Zero(di))) && + AllTrue(di, Lt(vec, Set(di, static_cast<TI>(64 / sizeof(T)))))); +#endif + return Indices512<T>{vec.raw}; +} + +template <typename T, typename TI> +HWY_API Indices512<T> SetTableIndices(const Full512<T> d, const TI* idx) { + const Rebind<TI, decltype(d)> di; + return IndicesFromVec(d, LoadU(di, idx)); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec512<T> TableLookupLanes(Vec512<T> v, Indices512<T> idx) { + return Vec512<T>{_mm512_permutexvar_epi32(idx.raw, v.raw)}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec512<T> TableLookupLanes(Vec512<T> v, Indices512<T> idx) { + return Vec512<T>{_mm512_permutexvar_epi64(idx.raw, v.raw)}; +} + +HWY_API Vec512<float> TableLookupLanes(Vec512<float> v, Indices512<float> idx) { + return Vec512<float>{_mm512_permutexvar_ps(idx.raw, v.raw)}; +} + +HWY_API Vec512<double> TableLookupLanes(Vec512<double> v, + Indices512<double> idx) { + return Vec512<double>{_mm512_permutexvar_pd(idx.raw, v.raw)}; +} + +// ------------------------------ Reverse + +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Vec512<T> Reverse(Full512<T> d, const Vec512<T> v) { + const RebindToSigned<decltype(d)> di; + alignas(64) constexpr int16_t kReverse[32] = { + 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, + 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0}; + const Vec512<int16_t> idx = Load(di, kReverse); + return BitCast(d, Vec512<int16_t>{ + _mm512_permutexvar_epi16(idx.raw, BitCast(di, v).raw)}); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec512<T> Reverse(Full512<T> d, const Vec512<T> v) { + alignas(64) constexpr int32_t kReverse[16] = {15, 14, 13, 12, 11, 10, 9, 8, + 7, 6, 5, 4, 3, 2, 1, 0}; + return TableLookupLanes(v, SetTableIndices(d, kReverse)); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec512<T> Reverse(Full512<T> d, const Vec512<T> v) { + alignas(64) constexpr int64_t kReverse[8] = {7, 6, 5, 4, 3, 2, 1, 0}; + return TableLookupLanes(v, SetTableIndices(d, kReverse)); +} + +// ------------------------------ InterleaveLower + +// Interleaves lanes from halves of the 128-bit blocks of "a" (which provides +// the least-significant lane) and "b". To concatenate two half-width integers +// into one, use ZipLower/Upper instead (also works with scalar). + +HWY_API Vec512<uint8_t> InterleaveLower(const Vec512<uint8_t> a, + const Vec512<uint8_t> b) { + return Vec512<uint8_t>{_mm512_unpacklo_epi8(a.raw, b.raw)}; +} +HWY_API Vec512<uint16_t> InterleaveLower(const Vec512<uint16_t> a, + const Vec512<uint16_t> b) { + return Vec512<uint16_t>{_mm512_unpacklo_epi16(a.raw, b.raw)}; +} +HWY_API Vec512<uint32_t> InterleaveLower(const Vec512<uint32_t> a, + const Vec512<uint32_t> b) { + return Vec512<uint32_t>{_mm512_unpacklo_epi32(a.raw, b.raw)}; +} +HWY_API Vec512<uint64_t> InterleaveLower(const Vec512<uint64_t> a, + const Vec512<uint64_t> b) { + return Vec512<uint64_t>{_mm512_unpacklo_epi64(a.raw, b.raw)}; +} + +HWY_API Vec512<int8_t> InterleaveLower(const Vec512<int8_t> a, + const Vec512<int8_t> b) { + return Vec512<int8_t>{_mm512_unpacklo_epi8(a.raw, b.raw)}; +} +HWY_API Vec512<int16_t> InterleaveLower(const Vec512<int16_t> a, + const Vec512<int16_t> b) { + return Vec512<int16_t>{_mm512_unpacklo_epi16(a.raw, b.raw)}; +} +HWY_API Vec512<int32_t> InterleaveLower(const Vec512<int32_t> a, + const Vec512<int32_t> b) { + return Vec512<int32_t>{_mm512_unpacklo_epi32(a.raw, b.raw)}; +} +HWY_API Vec512<int64_t> InterleaveLower(const Vec512<int64_t> a, + const Vec512<int64_t> b) { + return Vec512<int64_t>{_mm512_unpacklo_epi64(a.raw, b.raw)}; +} + +HWY_API Vec512<float> InterleaveLower(const Vec512<float> a, + const Vec512<float> b) { + return Vec512<float>{_mm512_unpacklo_ps(a.raw, b.raw)}; +} +HWY_API Vec512<double> InterleaveLower(const Vec512<double> a, + const Vec512<double> b) { + return Vec512<double>{_mm512_unpacklo_pd(a.raw, b.raw)}; +} + +// Additional overload for the optional Simd<> tag. +template <typename T, class V = Vec512<T>> +HWY_API V InterleaveLower(Full512<T> /* tag */, V a, V b) { + return InterleaveLower(a, b); +} + +// ------------------------------ InterleaveUpper + +// All functions inside detail lack the required D parameter. +namespace detail { + +HWY_API Vec512<uint8_t> InterleaveUpper(const Vec512<uint8_t> a, + const Vec512<uint8_t> b) { + return Vec512<uint8_t>{_mm512_unpackhi_epi8(a.raw, b.raw)}; +} +HWY_API Vec512<uint16_t> InterleaveUpper(const Vec512<uint16_t> a, + const Vec512<uint16_t> b) { + return Vec512<uint16_t>{_mm512_unpackhi_epi16(a.raw, b.raw)}; +} +HWY_API Vec512<uint32_t> InterleaveUpper(const Vec512<uint32_t> a, + const Vec512<uint32_t> b) { + return Vec512<uint32_t>{_mm512_unpackhi_epi32(a.raw, b.raw)}; +} +HWY_API Vec512<uint64_t> InterleaveUpper(const Vec512<uint64_t> a, + const Vec512<uint64_t> b) { + return Vec512<uint64_t>{_mm512_unpackhi_epi64(a.raw, b.raw)}; +} + +HWY_API Vec512<int8_t> InterleaveUpper(const Vec512<int8_t> a, + const Vec512<int8_t> b) { + return Vec512<int8_t>{_mm512_unpackhi_epi8(a.raw, b.raw)}; +} +HWY_API Vec512<int16_t> InterleaveUpper(const Vec512<int16_t> a, + const Vec512<int16_t> b) { + return Vec512<int16_t>{_mm512_unpackhi_epi16(a.raw, b.raw)}; +} +HWY_API Vec512<int32_t> InterleaveUpper(const Vec512<int32_t> a, + const Vec512<int32_t> b) { + return Vec512<int32_t>{_mm512_unpackhi_epi32(a.raw, b.raw)}; +} +HWY_API Vec512<int64_t> InterleaveUpper(const Vec512<int64_t> a, + const Vec512<int64_t> b) { + return Vec512<int64_t>{_mm512_unpackhi_epi64(a.raw, b.raw)}; +} + +HWY_API Vec512<float> InterleaveUpper(const Vec512<float> a, + const Vec512<float> b) { + return Vec512<float>{_mm512_unpackhi_ps(a.raw, b.raw)}; +} +HWY_API Vec512<double> InterleaveUpper(const Vec512<double> a, + const Vec512<double> b) { + return Vec512<double>{_mm512_unpackhi_pd(a.raw, b.raw)}; +} + +} // namespace detail + +template <typename T, class V = Vec512<T>> +HWY_API V InterleaveUpper(Full512<T> /* tag */, V a, V b) { + return detail::InterleaveUpper(a, b); +} + +// ------------------------------ ZipLower/ZipUpper (InterleaveLower) + +// Same as Interleave*, except that the return lanes are double-width integers; +// this is necessary because the single-lane scalar cannot return two values. +template <typename T, typename TW = MakeWide<T>> +HWY_API Vec512<TW> ZipLower(Vec512<T> a, Vec512<T> b) { + return BitCast(Full512<TW>(), InterleaveLower(a, b)); +} +template <typename T, typename TW = MakeWide<T>> +HWY_API Vec512<TW> ZipLower(Full512<TW> d, Vec512<T> a, Vec512<T> b) { + return BitCast(Full512<TW>(), InterleaveLower(d, a, b)); +} + +template <typename T, typename TW = MakeWide<T>> +HWY_API Vec512<TW> ZipUpper(Full512<TW> d, Vec512<T> a, Vec512<T> b) { + return BitCast(Full512<TW>(), InterleaveUpper(d, a, b)); +} + +// ------------------------------ Concat* halves + +// hiH,hiL loH,loL |-> hiL,loL (= lower halves) +template <typename T> +HWY_API Vec512<T> ConcatLowerLower(Full512<T> /* tag */, const Vec512<T> hi, + const Vec512<T> lo) { + return Vec512<T>{_mm512_shuffle_i32x4(lo.raw, hi.raw, _MM_PERM_BABA)}; +} +HWY_API Vec512<float> ConcatLowerLower(Full512<float> /* tag */, + const Vec512<float> hi, + const Vec512<float> lo) { + return Vec512<float>{_mm512_shuffle_f32x4(lo.raw, hi.raw, _MM_PERM_BABA)}; +} +HWY_API Vec512<double> ConcatLowerLower(Full512<double> /* tag */, + const Vec512<double> hi, + const Vec512<double> lo) { + return Vec512<double>{_mm512_shuffle_f64x2(lo.raw, hi.raw, _MM_PERM_BABA)}; +} + +// hiH,hiL loH,loL |-> hiH,loH (= upper halves) +template <typename T> +HWY_API Vec512<T> ConcatUpperUpper(Full512<T> /* tag */, const Vec512<T> hi, + const Vec512<T> lo) { + return Vec512<T>{_mm512_shuffle_i32x4(lo.raw, hi.raw, _MM_PERM_DCDC)}; +} +HWY_API Vec512<float> ConcatUpperUpper(Full512<float> /* tag */, + const Vec512<float> hi, + const Vec512<float> lo) { + return Vec512<float>{_mm512_shuffle_f32x4(lo.raw, hi.raw, _MM_PERM_DCDC)}; +} +HWY_API Vec512<double> ConcatUpperUpper(Full512<double> /* tag */, + const Vec512<double> hi, + const Vec512<double> lo) { + return Vec512<double>{_mm512_shuffle_f64x2(lo.raw, hi.raw, _MM_PERM_DCDC)}; +} + +// hiH,hiL loH,loL |-> hiL,loH (= inner halves / swap blocks) +template <typename T> +HWY_API Vec512<T> ConcatLowerUpper(Full512<T> /* tag */, const Vec512<T> hi, + const Vec512<T> lo) { + return Vec512<T>{_mm512_shuffle_i32x4(lo.raw, hi.raw, 0x4E)}; +} +HWY_API Vec512<float> ConcatLowerUpper(Full512<float> /* tag */, + const Vec512<float> hi, + const Vec512<float> lo) { + return Vec512<float>{_mm512_shuffle_f32x4(lo.raw, hi.raw, 0x4E)}; +} +HWY_API Vec512<double> ConcatLowerUpper(Full512<double> /* tag */, + const Vec512<double> hi, + const Vec512<double> lo) { + return Vec512<double>{_mm512_shuffle_f64x2(lo.raw, hi.raw, 0x4E)}; +} + +// hiH,hiL loH,loL |-> hiH,loL (= outer halves) +template <typename T> +HWY_API Vec512<T> ConcatUpperLower(Full512<T> /* tag */, const Vec512<T> hi, + const Vec512<T> lo) { + // There are no imm8 blend in AVX512. Use blend16 because 32-bit masks + // are efficiently loaded from 32-bit regs. + const __mmask32 mask = /*_cvtu32_mask32 */ (0x0000FFFF); + return Vec512<T>{_mm512_mask_blend_epi16(mask, hi.raw, lo.raw)}; +} +HWY_API Vec512<float> ConcatUpperLower(Full512<float> /* tag */, + const Vec512<float> hi, + const Vec512<float> lo) { + const __mmask16 mask = /*_cvtu32_mask16 */ (0x00FF); + return Vec512<float>{_mm512_mask_blend_ps(mask, hi.raw, lo.raw)}; +} +HWY_API Vec512<double> ConcatUpperLower(Full512<double> /* tag */, + const Vec512<double> hi, + const Vec512<double> lo) { + const __mmask8 mask = /*_cvtu32_mask8 */ (0x0F); + return Vec512<double>{_mm512_mask_blend_pd(mask, hi.raw, lo.raw)}; +} + +// ------------------------------ ConcatOdd + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec512<T> ConcatOdd(Full512<T> d, Vec512<T> hi, Vec512<T> lo) { + const RebindToUnsigned<decltype(d)> du; + alignas(64) constexpr uint32_t kIdx[16] = {1, 3, 5, 7, 9, 11, 13, 15, + 17, 19, 21, 23, 25, 27, 29, 31}; + return BitCast(d, Vec512<uint32_t>{_mm512_mask2_permutex2var_epi32( + BitCast(du, lo).raw, Load(du, kIdx).raw, + __mmask16{0xFFFF}, BitCast(du, hi).raw)}); +} + +HWY_API Vec512<float> ConcatOdd(Full512<float> d, Vec512<float> hi, + Vec512<float> lo) { + const RebindToUnsigned<decltype(d)> du; + alignas(64) constexpr uint32_t kIdx[16] = {1, 3, 5, 7, 9, 11, 13, 15, + 17, 19, 21, 23, 25, 27, 29, 31}; + return Vec512<float>{_mm512_mask2_permutex2var_ps(lo.raw, Load(du, kIdx).raw, + __mmask16{0xFFFF}, hi.raw)}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec512<T> ConcatOdd(Full512<T> d, Vec512<T> hi, Vec512<T> lo) { + const RebindToUnsigned<decltype(d)> du; + alignas(64) constexpr uint64_t kIdx[8] = {1, 3, 5, 7, 9, 11, 13, 15}; + return BitCast(d, Vec512<uint64_t>{_mm512_mask2_permutex2var_epi64( + BitCast(du, lo).raw, Load(du, kIdx).raw, __mmask8{0xFF}, + BitCast(du, hi).raw)}); +} + +HWY_API Vec512<double> ConcatOdd(Full512<double> d, Vec512<double> hi, + Vec512<double> lo) { + const RebindToUnsigned<decltype(d)> du; + alignas(64) constexpr uint64_t kIdx[8] = {1, 3, 5, 7, 9, 11, 13, 15}; + return Vec512<double>{_mm512_mask2_permutex2var_pd(lo.raw, Load(du, kIdx).raw, + __mmask8{0xFF}, hi.raw)}; +} + +// ------------------------------ ConcatEven + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec512<T> ConcatEven(Full512<T> d, Vec512<T> hi, Vec512<T> lo) { + const RebindToUnsigned<decltype(d)> du; + alignas(64) constexpr uint32_t kIdx[16] = {0, 2, 4, 6, 8, 10, 12, 14, + 16, 18, 20, 22, 24, 26, 28, 30}; + return BitCast(d, Vec512<uint32_t>{_mm512_mask2_permutex2var_epi32( + BitCast(du, lo).raw, Load(du, kIdx).raw, + __mmask16{0xFFFF}, BitCast(du, hi).raw)}); +} + +HWY_API Vec512<float> ConcatEven(Full512<float> d, Vec512<float> hi, + Vec512<float> lo) { + const RebindToUnsigned<decltype(d)> du; + alignas(64) constexpr uint32_t kIdx[16] = {0, 2, 4, 6, 8, 10, 12, 14, + 16, 18, 20, 22, 24, 26, 28, 30}; + return Vec512<float>{_mm512_mask2_permutex2var_ps(lo.raw, Load(du, kIdx).raw, + __mmask16{0xFFFF}, hi.raw)}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec512<T> ConcatEven(Full512<T> d, Vec512<T> hi, Vec512<T> lo) { + const RebindToUnsigned<decltype(d)> du; + alignas(64) constexpr uint64_t kIdx[8] = {0, 2, 4, 6, 8, 10, 12, 14}; + return BitCast(d, Vec512<uint64_t>{_mm512_mask2_permutex2var_epi64( + BitCast(du, lo).raw, Load(du, kIdx).raw, __mmask8{0xFF}, + BitCast(du, hi).raw)}); +} + +HWY_API Vec512<double> ConcatEven(Full512<double> d, Vec512<double> hi, + Vec512<double> lo) { + const RebindToUnsigned<decltype(d)> du; + alignas(64) constexpr uint64_t kIdx[8] = {0, 2, 4, 6, 8, 10, 12, 14}; + return Vec512<double>{_mm512_mask2_permutex2var_pd(lo.raw, Load(du, kIdx).raw, + __mmask8{0xFF}, hi.raw)}; +} + +// ------------------------------ OddEven + +template <typename T> +HWY_API Vec512<T> OddEven(const Vec512<T> a, const Vec512<T> b) { + constexpr size_t s = sizeof(T); + constexpr int shift = s == 1 ? 0 : s == 2 ? 32 : s == 4 ? 48 : 56; + return IfThenElse(Mask512<T>{0x5555555555555555ull >> shift}, b, a); +} + +// ------------------------------ OddEvenBlocks + +template <typename T> +HWY_API Vec512<T> OddEvenBlocks(Vec512<T> odd, Vec512<T> even) { + return Vec512<T>{_mm512_mask_blend_epi64(__mmask8{0x33u}, odd.raw, even.raw)}; +} + +HWY_API Vec512<float> OddEvenBlocks(Vec512<float> odd, Vec512<float> even) { + return Vec512<float>{ + _mm512_mask_blend_ps(__mmask16{0x0F0Fu}, odd.raw, even.raw)}; +} + +HWY_API Vec512<double> OddEvenBlocks(Vec512<double> odd, Vec512<double> even) { + return Vec512<double>{ + _mm512_mask_blend_pd(__mmask8{0x33u}, odd.raw, even.raw)}; +} + +// ------------------------------ SwapAdjacentBlocks + +template <typename T> +HWY_API Vec512<T> SwapAdjacentBlocks(Vec512<T> v) { + return Vec512<T>{_mm512_shuffle_i32x4(v.raw, v.raw, _MM_SHUFFLE(2, 3, 0, 1))}; +} + +HWY_API Vec512<float> SwapAdjacentBlocks(Vec512<float> v) { + return Vec512<float>{ + _mm512_shuffle_f32x4(v.raw, v.raw, _MM_SHUFFLE(2, 3, 0, 1))}; +} + +HWY_API Vec512<double> SwapAdjacentBlocks(Vec512<double> v) { + return Vec512<double>{ + _mm512_shuffle_f64x2(v.raw, v.raw, _MM_SHUFFLE(2, 3, 0, 1))}; +} + +// ------------------------------ TableLookupBytes (ZeroExtendVector) + +// Both full +template <typename T, typename TI> +HWY_API Vec512<TI> TableLookupBytes(Vec512<T> bytes, Vec512<TI> indices) { + return Vec512<TI>{_mm512_shuffle_epi8(bytes.raw, indices.raw)}; +} + +// Partial index vector +template <typename T, typename TI, size_t NI> +HWY_API Vec128<TI, NI> TableLookupBytes(Vec512<T> bytes, Vec128<TI, NI> from) { + const Full512<TI> d512; + const Half<decltype(d512)> d256; + const Half<decltype(d256)> d128; + // First expand to full 128, then 256, then 512. + const Vec128<TI> from_full{from.raw}; + const auto from_512 = + ZeroExtendVector(d512, ZeroExtendVector(d256, from_full)); + const auto tbl_full = TableLookupBytes(bytes, from_512); + // Shrink to 256, then 128, then partial. + return Vec128<TI, NI>{LowerHalf(d128, LowerHalf(d256, tbl_full)).raw}; +} +template <typename T, typename TI> +HWY_API Vec256<TI> TableLookupBytes(Vec512<T> bytes, Vec256<TI> from) { + const auto from_512 = ZeroExtendVector(Full512<TI>(), from); + return LowerHalf(Full256<TI>(), TableLookupBytes(bytes, from_512)); +} + +// Partial table vector +template <typename T, size_t N, typename TI> +HWY_API Vec512<TI> TableLookupBytes(Vec128<T, N> bytes, Vec512<TI> from) { + const Full512<TI> d512; + const Half<decltype(d512)> d256; + const Half<decltype(d256)> d128; + // First expand to full 128, then 256, then 512. + const Vec128<T> bytes_full{bytes.raw}; + const auto bytes_512 = + ZeroExtendVector(d512, ZeroExtendVector(d256, bytes_full)); + return TableLookupBytes(bytes_512, from); +} +template <typename T, typename TI> +HWY_API Vec512<TI> TableLookupBytes(Vec256<T> bytes, Vec512<TI> from) { + const auto bytes_512 = ZeroExtendVector(Full512<T>(), bytes); + return TableLookupBytes(bytes_512, from); +} + +// Partial both are handled by x86_128/256. + +// ================================================== CONVERT + +// ------------------------------ Promotions (part w/ narrow lanes -> full) + +// Unsigned: zero-extend. +// Note: these have 3 cycle latency; if inputs are already split across the +// 128 bit blocks (in their upper/lower halves), then Zip* would be faster. +HWY_API Vec512<uint16_t> PromoteTo(Full512<uint16_t> /* tag */, + Vec256<uint8_t> v) { + return Vec512<uint16_t>{_mm512_cvtepu8_epi16(v.raw)}; +} +HWY_API Vec512<uint32_t> PromoteTo(Full512<uint32_t> /* tag */, + Vec128<uint8_t> v) { + return Vec512<uint32_t>{_mm512_cvtepu8_epi32(v.raw)}; +} +HWY_API Vec512<int16_t> PromoteTo(Full512<int16_t> /* tag */, + Vec256<uint8_t> v) { + return Vec512<int16_t>{_mm512_cvtepu8_epi16(v.raw)}; +} +HWY_API Vec512<int32_t> PromoteTo(Full512<int32_t> /* tag */, + Vec128<uint8_t> v) { + return Vec512<int32_t>{_mm512_cvtepu8_epi32(v.raw)}; +} +HWY_API Vec512<uint32_t> PromoteTo(Full512<uint32_t> /* tag */, + Vec256<uint16_t> v) { + return Vec512<uint32_t>{_mm512_cvtepu16_epi32(v.raw)}; +} +HWY_API Vec512<int32_t> PromoteTo(Full512<int32_t> /* tag */, + Vec256<uint16_t> v) { + return Vec512<int32_t>{_mm512_cvtepu16_epi32(v.raw)}; +} +HWY_API Vec512<uint64_t> PromoteTo(Full512<uint64_t> /* tag */, + Vec256<uint32_t> v) { + return Vec512<uint64_t>{_mm512_cvtepu32_epi64(v.raw)}; +} + +// Signed: replicate sign bit. +// Note: these have 3 cycle latency; if inputs are already split across the +// 128 bit blocks (in their upper/lower halves), then ZipUpper/lo followed by +// signed shift would be faster. +HWY_API Vec512<int16_t> PromoteTo(Full512<int16_t> /* tag */, + Vec256<int8_t> v) { + return Vec512<int16_t>{_mm512_cvtepi8_epi16(v.raw)}; +} +HWY_API Vec512<int32_t> PromoteTo(Full512<int32_t> /* tag */, + Vec128<int8_t> v) { + return Vec512<int32_t>{_mm512_cvtepi8_epi32(v.raw)}; +} +HWY_API Vec512<int32_t> PromoteTo(Full512<int32_t> /* tag */, + Vec256<int16_t> v) { + return Vec512<int32_t>{_mm512_cvtepi16_epi32(v.raw)}; +} +HWY_API Vec512<int64_t> PromoteTo(Full512<int64_t> /* tag */, + Vec256<int32_t> v) { + return Vec512<int64_t>{_mm512_cvtepi32_epi64(v.raw)}; +} + +// Float +HWY_API Vec512<float> PromoteTo(Full512<float> /* tag */, + const Vec256<float16_t> v) { + return Vec512<float>{_mm512_cvtph_ps(v.raw)}; +} + +HWY_API Vec512<float> PromoteTo(Full512<float> df32, + const Vec256<bfloat16_t> v) { + const Rebind<uint16_t, decltype(df32)> du16; + const RebindToSigned<decltype(df32)> di32; + return BitCast(df32, ShiftLeft<16>(PromoteTo(di32, BitCast(du16, v)))); +} + +HWY_API Vec512<double> PromoteTo(Full512<double> /* tag */, Vec256<float> v) { + return Vec512<double>{_mm512_cvtps_pd(v.raw)}; +} + +HWY_API Vec512<double> PromoteTo(Full512<double> /* tag */, Vec256<int32_t> v) { + return Vec512<double>{_mm512_cvtepi32_pd(v.raw)}; +} + +// ------------------------------ Demotions (full -> part w/ narrow lanes) + +HWY_API Vec256<uint16_t> DemoteTo(Full256<uint16_t> /* tag */, + const Vec512<int32_t> v) { + const Vec512<uint16_t> u16{_mm512_packus_epi32(v.raw, v.raw)}; + + // Compress even u64 lanes into 256 bit. + alignas(64) static constexpr uint64_t kLanes[8] = {0, 2, 4, 6, 0, 2, 4, 6}; + const auto idx64 = Load(Full512<uint64_t>(), kLanes); + const Vec512<uint16_t> even{_mm512_permutexvar_epi64(idx64.raw, u16.raw)}; + return LowerHalf(even); +} + +HWY_API Vec256<int16_t> DemoteTo(Full256<int16_t> /* tag */, + const Vec512<int32_t> v) { + const Vec512<int16_t> i16{_mm512_packs_epi32(v.raw, v.raw)}; + + // Compress even u64 lanes into 256 bit. + alignas(64) static constexpr uint64_t kLanes[8] = {0, 2, 4, 6, 0, 2, 4, 6}; + const auto idx64 = Load(Full512<uint64_t>(), kLanes); + const Vec512<int16_t> even{_mm512_permutexvar_epi64(idx64.raw, i16.raw)}; + return LowerHalf(even); +} + +HWY_API Vec128<uint8_t, 16> DemoteTo(Full128<uint8_t> /* tag */, + const Vec512<int32_t> v) { + const Vec512<uint16_t> u16{_mm512_packus_epi32(v.raw, v.raw)}; + // packus treats the input as signed; we want unsigned. Clear the MSB to get + // unsigned saturation to u8. + const Vec512<int16_t> i16{ + _mm512_and_si512(u16.raw, _mm512_set1_epi16(0x7FFF))}; + const Vec512<uint8_t> u8{_mm512_packus_epi16(i16.raw, i16.raw)}; + + alignas(16) static constexpr uint32_t kLanes[4] = {0, 4, 8, 12}; + const auto idx32 = LoadDup128(Full512<uint32_t>(), kLanes); + const Vec512<uint8_t> fixed{_mm512_permutexvar_epi32(idx32.raw, u8.raw)}; + return LowerHalf(LowerHalf(fixed)); +} + +HWY_API Vec256<uint8_t> DemoteTo(Full256<uint8_t> /* tag */, + const Vec512<int16_t> v) { + const Vec512<uint8_t> u8{_mm512_packus_epi16(v.raw, v.raw)}; + + // Compress even u64 lanes into 256 bit. + alignas(64) static constexpr uint64_t kLanes[8] = {0, 2, 4, 6, 0, 2, 4, 6}; + const auto idx64 = Load(Full512<uint64_t>(), kLanes); + const Vec512<uint8_t> even{_mm512_permutexvar_epi64(idx64.raw, u8.raw)}; + return LowerHalf(even); +} + +HWY_API Vec128<int8_t, 16> DemoteTo(Full128<int8_t> /* tag */, + const Vec512<int32_t> v) { + const Vec512<int16_t> i16{_mm512_packs_epi32(v.raw, v.raw)}; + const Vec512<int8_t> i8{_mm512_packs_epi16(i16.raw, i16.raw)}; + + alignas(16) static constexpr uint32_t kLanes[16] = {0, 4, 8, 12, 0, 4, 8, 12, + 0, 4, 8, 12, 0, 4, 8, 12}; + const auto idx32 = LoadDup128(Full512<uint32_t>(), kLanes); + const Vec512<int8_t> fixed{_mm512_permutexvar_epi32(idx32.raw, i8.raw)}; + return LowerHalf(LowerHalf(fixed)); +} + +HWY_API Vec256<int8_t> DemoteTo(Full256<int8_t> /* tag */, + const Vec512<int16_t> v) { + const Vec512<int8_t> u8{_mm512_packs_epi16(v.raw, v.raw)}; + + // Compress even u64 lanes into 256 bit. + alignas(64) static constexpr uint64_t kLanes[8] = {0, 2, 4, 6, 0, 2, 4, 6}; + const auto idx64 = Load(Full512<uint64_t>(), kLanes); + const Vec512<int8_t> even{_mm512_permutexvar_epi64(idx64.raw, u8.raw)}; + return LowerHalf(even); +} + +HWY_API Vec256<float16_t> DemoteTo(Full256<float16_t> /* tag */, + const Vec512<float> v) { + // Work around warnings in the intrinsic definitions (passing -1 as a mask). + HWY_DIAGNOSTICS(push) + HWY_DIAGNOSTICS_OFF(disable : 4245 4365, ignored "-Wsign-conversion") + return Vec256<float16_t>{_mm512_cvtps_ph(v.raw, _MM_FROUND_NO_EXC)}; + HWY_DIAGNOSTICS(pop) +} + +HWY_API Vec256<bfloat16_t> DemoteTo(Full256<bfloat16_t> dbf16, + const Vec512<float> v) { + // TODO(janwas): _mm512_cvtneps_pbh once we have avx512bf16. + const Rebind<int32_t, decltype(dbf16)> di32; + const Rebind<uint32_t, decltype(dbf16)> du32; // for logical shift right + const Rebind<uint16_t, decltype(dbf16)> du16; + const auto bits_in_32 = BitCast(di32, ShiftRight<16>(BitCast(du32, v))); + return BitCast(dbf16, DemoteTo(du16, bits_in_32)); +} + +HWY_API Vec512<bfloat16_t> ReorderDemote2To(Full512<bfloat16_t> dbf16, + Vec512<float> a, Vec512<float> b) { + // TODO(janwas): _mm512_cvtne2ps_pbh once we have avx512bf16. + const RebindToUnsigned<decltype(dbf16)> du16; + const Repartition<uint32_t, decltype(dbf16)> du32; + const Vec512<uint32_t> b_in_even = ShiftRight<16>(BitCast(du32, b)); + return BitCast(dbf16, OddEven(BitCast(du16, a), BitCast(du16, b_in_even))); +} + +HWY_API Vec256<float> DemoteTo(Full256<float> /* tag */, + const Vec512<double> v) { + return Vec256<float>{_mm512_cvtpd_ps(v.raw)}; +} + +HWY_API Vec256<int32_t> DemoteTo(Full256<int32_t> /* tag */, + const Vec512<double> v) { + const auto clamped = detail::ClampF64ToI32Max(Full512<double>(), v); + return Vec256<int32_t>{_mm512_cvttpd_epi32(clamped.raw)}; +} + +// For already range-limited input [0, 255]. +HWY_API Vec128<uint8_t, 16> U8FromU32(const Vec512<uint32_t> v) { + const Full512<uint32_t> d32; + // In each 128 bit block, gather the lower byte of 4 uint32_t lanes into the + // lowest 4 bytes. + alignas(16) static constexpr uint32_t k8From32[4] = {0x0C080400u, ~0u, ~0u, + ~0u}; + const auto quads = TableLookupBytes(v, LoadDup128(d32, k8From32)); + // Gather the lowest 4 bytes of 4 128-bit blocks. + alignas(16) static constexpr uint32_t kIndex32[4] = {0, 4, 8, 12}; + const Vec512<uint8_t> bytes{ + _mm512_permutexvar_epi32(LoadDup128(d32, kIndex32).raw, quads.raw)}; + return LowerHalf(LowerHalf(bytes)); +} + +// ------------------------------ Convert integer <=> floating point + +HWY_API Vec512<float> ConvertTo(Full512<float> /* tag */, + const Vec512<int32_t> v) { + return Vec512<float>{_mm512_cvtepi32_ps(v.raw)}; +} + +HWY_API Vec512<double> ConvertTo(Full512<double> /* tag */, + const Vec512<int64_t> v) { + return Vec512<double>{_mm512_cvtepi64_pd(v.raw)}; +} + +// Truncates (rounds toward zero). +HWY_API Vec512<int32_t> ConvertTo(Full512<int32_t> d, const Vec512<float> v) { + return detail::FixConversionOverflow(d, v, _mm512_cvttps_epi32(v.raw)); +} +HWY_API Vec512<int64_t> ConvertTo(Full512<int64_t> di, const Vec512<double> v) { + return detail::FixConversionOverflow(di, v, _mm512_cvttpd_epi64(v.raw)); +} + +HWY_API Vec512<int32_t> NearestInt(const Vec512<float> v) { + const Full512<int32_t> di; + return detail::FixConversionOverflow(di, v, _mm512_cvtps_epi32(v.raw)); +} + +// ================================================== CRYPTO + +#if !defined(HWY_DISABLE_PCLMUL_AES) + +// Per-target flag to prevent generic_ops-inl.h from defining AESRound. +#ifdef HWY_NATIVE_AES +#undef HWY_NATIVE_AES +#else +#define HWY_NATIVE_AES +#endif + +HWY_API Vec512<uint8_t> AESRound(Vec512<uint8_t> state, + Vec512<uint8_t> round_key) { +#if HWY_TARGET == HWY_AVX3_DL + return Vec512<uint8_t>{_mm512_aesenc_epi128(state.raw, round_key.raw)}; +#else + alignas(64) uint8_t a[64]; + alignas(64) uint8_t b[64]; + const Full512<uint8_t> d; + const Full128<uint8_t> d128; + Store(state, d, a); + Store(round_key, d, b); + for (size_t i = 0; i < 64; i += 16) { + const auto enc = AESRound(Load(d128, a + i), Load(d128, b + i)); + Store(enc, d128, a + i); + } + return Load(d, a); +#endif +} + +HWY_API Vec512<uint64_t> CLMulLower(Vec512<uint64_t> va, Vec512<uint64_t> vb) { +#if HWY_TARGET == HWY_AVX3_DL + return Vec512<uint64_t>{_mm512_clmulepi64_epi128(va.raw, vb.raw, 0x00)}; +#else + alignas(64) uint64_t a[8]; + alignas(64) uint64_t b[8]; + const Full512<uint64_t> d; + const Full128<uint64_t> d128; + Store(va, d, a); + Store(vb, d, b); + for (size_t i = 0; i < 8; i += 2) { + const auto mul = CLMulLower(Load(d128, a + i), Load(d128, b + i)); + Store(mul, d128, a + i); + } + return Load(d, a); +#endif +} + +HWY_API Vec512<uint64_t> CLMulUpper(Vec512<uint64_t> va, Vec512<uint64_t> vb) { +#if HWY_TARGET == HWY_AVX3_DL + return Vec512<uint64_t>{_mm512_clmulepi64_epi128(va.raw, vb.raw, 0x11)}; +#else + alignas(64) uint64_t a[8]; + alignas(64) uint64_t b[8]; + const Full512<uint64_t> d; + const Full128<uint64_t> d128; + Store(va, d, a); + Store(vb, d, b); + for (size_t i = 0; i < 8; i += 2) { + const auto mul = CLMulUpper(Load(d128, a + i), Load(d128, b + i)); + Store(mul, d128, a + i); + } + return Load(d, a); +#endif +} + +#endif // HWY_DISABLE_PCLMUL_AES + +// ================================================== MISC + +// Returns a vector with lane i=[0, N) set to "first" + i. +template <typename T, typename T2> +Vec512<T> Iota(const Full512<T> d, const T2 first) { + HWY_ALIGN T lanes[64 / sizeof(T)]; + for (size_t i = 0; i < 64 / sizeof(T); ++i) { + lanes[i] = static_cast<T>(first + static_cast<T2>(i)); + } + return Load(d, lanes); +} + +// ------------------------------ Mask testing + +// Beware: the suffix indicates the number of mask bits, not lane size! + +namespace detail { + +template <typename T> +HWY_INLINE bool AllFalse(hwy::SizeTag<1> /*tag*/, const Mask512<T> mask) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return _kortestz_mask64_u8(mask.raw, mask.raw); +#else + return mask.raw == 0; +#endif +} +template <typename T> +HWY_INLINE bool AllFalse(hwy::SizeTag<2> /*tag*/, const Mask512<T> mask) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return _kortestz_mask32_u8(mask.raw, mask.raw); +#else + return mask.raw == 0; +#endif +} +template <typename T> +HWY_INLINE bool AllFalse(hwy::SizeTag<4> /*tag*/, const Mask512<T> mask) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return _kortestz_mask16_u8(mask.raw, mask.raw); +#else + return mask.raw == 0; +#endif +} +template <typename T> +HWY_INLINE bool AllFalse(hwy::SizeTag<8> /*tag*/, const Mask512<T> mask) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return _kortestz_mask8_u8(mask.raw, mask.raw); +#else + return mask.raw == 0; +#endif +} + +} // namespace detail + +template <typename T> +HWY_API bool AllFalse(const Full512<T> /* tag */, const Mask512<T> mask) { + return detail::AllFalse(hwy::SizeTag<sizeof(T)>(), mask); +} + +namespace detail { + +template <typename T> +HWY_INLINE bool AllTrue(hwy::SizeTag<1> /*tag*/, const Mask512<T> mask) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return _kortestc_mask64_u8(mask.raw, mask.raw); +#else + return mask.raw == 0xFFFFFFFFFFFFFFFFull; +#endif +} +template <typename T> +HWY_INLINE bool AllTrue(hwy::SizeTag<2> /*tag*/, const Mask512<T> mask) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return _kortestc_mask32_u8(mask.raw, mask.raw); +#else + return mask.raw == 0xFFFFFFFFull; +#endif +} +template <typename T> +HWY_INLINE bool AllTrue(hwy::SizeTag<4> /*tag*/, const Mask512<T> mask) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return _kortestc_mask16_u8(mask.raw, mask.raw); +#else + return mask.raw == 0xFFFFull; +#endif +} +template <typename T> +HWY_INLINE bool AllTrue(hwy::SizeTag<8> /*tag*/, const Mask512<T> mask) { +#if HWY_COMPILER_HAS_MASK_INTRINSICS + return _kortestc_mask8_u8(mask.raw, mask.raw); +#else + return mask.raw == 0xFFull; +#endif +} + +} // namespace detail + +template <typename T> +HWY_API bool AllTrue(const Full512<T> /* tag */, const Mask512<T> mask) { + return detail::AllTrue(hwy::SizeTag<sizeof(T)>(), mask); +} + +// `p` points to at least 8 readable bytes, not all of which need be valid. +template <typename T> +HWY_API Mask512<T> LoadMaskBits(const Full512<T> /* tag */, + const uint8_t* HWY_RESTRICT bits) { + Mask512<T> mask; + CopyBytes<8 / sizeof(T)>(bits, &mask.raw); + // N >= 8 (= 512 / 64), so no need to mask invalid bits. + return mask; +} + +// `p` points to at least 8 writable bytes. +template <typename T> +HWY_API size_t StoreMaskBits(const Full512<T> /* tag */, const Mask512<T> mask, + uint8_t* bits) { + const size_t kNumBytes = 8 / sizeof(T); + CopyBytes<kNumBytes>(&mask.raw, bits); + // N >= 8 (= 512 / 64), so no need to mask invalid bits. + return kNumBytes; +} + +template <typename T> +HWY_API size_t CountTrue(const Full512<T> /* tag */, const Mask512<T> mask) { + return PopCount(static_cast<uint64_t>(mask.raw)); +} + +template <typename T, HWY_IF_NOT_LANE_SIZE(T, 1)> +HWY_API intptr_t FindFirstTrue(const Full512<T> /* tag */, + const Mask512<T> mask) { + return mask.raw ? intptr_t(Num0BitsBelowLS1Bit_Nonzero32(mask.raw)) : -1; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 1)> +HWY_API intptr_t FindFirstTrue(const Full512<T> /* tag */, + const Mask512<T> mask) { + return mask.raw ? intptr_t(Num0BitsBelowLS1Bit_Nonzero64(mask.raw)) : -1; +} + +// ------------------------------ Compress + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API Vec512<T> Compress(Vec512<T> v, Mask512<T> mask) { + return Vec512<T>{_mm512_maskz_compress_epi32(mask.raw, v.raw)}; +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API Vec512<T> Compress(Vec512<T> v, Mask512<T> mask) { + return Vec512<T>{_mm512_maskz_compress_epi64(mask.raw, v.raw)}; +} + +HWY_API Vec512<float> Compress(Vec512<float> v, Mask512<float> mask) { + return Vec512<float>{_mm512_maskz_compress_ps(mask.raw, v.raw)}; +} + +HWY_API Vec512<double> Compress(Vec512<double> v, Mask512<double> mask) { + return Vec512<double>{_mm512_maskz_compress_pd(mask.raw, v.raw)}; +} + +// 16-bit may use the 32-bit Compress and must be defined after it. +// +// Ignore IDE redefinition error - this is not actually defined in x86_256 if +// we are including x86_512-inl.h. +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Vec256<T> Compress(Vec256<T> v, Mask256<T> mask) { + const Full256<T> d; + const Rebind<uint16_t, decltype(d)> du; + const auto vu = BitCast(du, v); // (required for float16_t inputs) + +#if HWY_TARGET == HWY_AVX3_DL // VBMI2 + const Vec256<uint16_t> cu{_mm256_maskz_compress_epi16(mask.raw, vu.raw)}; +#else + // Promote to i32 (512-bit vector!) so we can use the native Compress. + const auto vw = PromoteTo(Rebind<int32_t, decltype(d)>(), vu); + const Mask512<int32_t> mask32{static_cast<__mmask16>(mask.raw)}; + const auto cu = DemoteTo(du, Compress(vw, mask32)); +#endif // HWY_TARGET == HWY_AVX3_DL + + return BitCast(d, cu); +} + +// Expands to 32-bit, compresses, concatenate demoted halves. +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Vec512<T> Compress(Vec512<T> v, const Mask512<T> mask) { + const Full512<T> d; + const Rebind<uint16_t, decltype(d)> du; + const auto vu = BitCast(du, v); // (required for float16_t inputs) + +#if HWY_TARGET == HWY_AVX3_DL // VBMI2 + const Vec512<uint16_t> cu{_mm512_maskz_compress_epi16(mask.raw, v.raw)}; +#else + const Repartition<int32_t, decltype(d)> dw; + const Half<decltype(du)> duh; + const auto promoted0 = PromoteTo(dw, LowerHalf(duh, vu)); + const auto promoted1 = PromoteTo(dw, UpperHalf(duh, vu)); + + const uint32_t mask_bits{mask.raw}; + const Mask512<int32_t> mask0{static_cast<__mmask16>(mask_bits & 0xFFFF)}; + const Mask512<int32_t> mask1{static_cast<__mmask16>(mask_bits >> 16)}; + const auto compressed0 = Compress(promoted0, mask0); + const auto compressed1 = Compress(promoted1, mask1); + + const auto demoted0 = ZeroExtendVector(DemoteTo(duh, compressed0)); + const auto demoted1 = ZeroExtendVector(DemoteTo(duh, compressed1)); + + // Concatenate into single vector by shifting upper with writemask. + const size_t num0 = CountTrue(dw, mask0); + const __mmask32 m_upper = ~((1u << num0) - 1); + alignas(64) uint16_t iota[64] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}; + const auto idx = LoadU(du, iota + 32 - num0); + const Vec512<uint16_t> cu{_mm512_mask_permutexvar_epi16( + demoted0.raw, m_upper, idx.raw, demoted1.raw)}; +#endif // HWY_TARGET == HWY_AVX3_DL + + return BitCast(d, cu); +} + +// ------------------------------ CompressBits +template <typename T> +HWY_API Vec512<T> CompressBits(Vec512<T> v, const uint8_t* HWY_RESTRICT bits) { + return Compress(v, LoadMaskBits(Full512<T>(), bits)); +} + +// ------------------------------ CompressStore + +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API size_t CompressStore(Vec512<T> v, Mask512<T> mask, Full512<T> d, + T* HWY_RESTRICT unaligned) { + const Rebind<uint16_t, decltype(d)> du; + const auto vu = BitCast(du, v); // (required for float16_t inputs) + + const uint64_t mask_bits{mask.raw}; + +#if HWY_TARGET == HWY_AVX3_DL // VBMI2 + _mm512_mask_compressstoreu_epi16(unaligned, mask.raw, v.raw); +#else + const Repartition<int32_t, decltype(d)> dw; + const Half<decltype(du)> duh; + const auto promoted0 = PromoteTo(dw, LowerHalf(duh, vu)); + const auto promoted1 = PromoteTo(dw, UpperHalf(duh, vu)); + + const uint64_t maskL = mask_bits & 0xFFFF; + const uint64_t maskH = mask_bits >> 16; + const Mask512<int32_t> mask0{static_cast<__mmask16>(maskL)}; + const Mask512<int32_t> mask1{static_cast<__mmask16>(maskH)}; + const auto compressed0 = Compress(promoted0, mask0); + const auto compressed1 = Compress(promoted1, mask1); + + const Half<decltype(d)> dh; + const auto demoted0 = BitCast(dh, DemoteTo(duh, compressed0)); + const auto demoted1 = BitCast(dh, DemoteTo(duh, compressed1)); + + // Store 256-bit halves + StoreU(demoted0, dh, unaligned); + StoreU(demoted1, dh, unaligned + PopCount(maskL)); +#endif + + return PopCount(mask_bits); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 4)> +HWY_API size_t CompressStore(Vec512<T> v, Mask512<T> mask, Full512<T> /* tag */, + T* HWY_RESTRICT unaligned) { + _mm512_mask_compressstoreu_epi32(unaligned, mask.raw, v.raw); + return PopCount(uint64_t{mask.raw}); +} + +template <typename T, HWY_IF_LANE_SIZE(T, 8)> +HWY_API size_t CompressStore(Vec512<T> v, Mask512<T> mask, Full512<T> /* tag */, + T* HWY_RESTRICT unaligned) { + _mm512_mask_compressstoreu_epi64(unaligned, mask.raw, v.raw); + return PopCount(uint64_t{mask.raw}); +} + +HWY_API size_t CompressStore(Vec512<float> v, Mask512<float> mask, + Full512<float> /* tag */, + float* HWY_RESTRICT unaligned) { + _mm512_mask_compressstoreu_ps(unaligned, mask.raw, v.raw); + return PopCount(uint64_t{mask.raw}); +} + +HWY_API size_t CompressStore(Vec512<double> v, Mask512<double> mask, + Full512<double> /* tag */, + double* HWY_RESTRICT unaligned) { + _mm512_mask_compressstoreu_pd(unaligned, mask.raw, v.raw); + return PopCount(uint64_t{mask.raw}); +} + +// ------------------------------ CompressBlendedStore +template <typename T> +HWY_API size_t CompressBlendedStore(Vec512<T> v, Mask512<T> m, Full512<T> d, + T* HWY_RESTRICT unaligned) { + // AVX-512 already does the blending at no extra cost (latency 11, + // rthroughput 2 - same as compress plus store). + if (HWY_TARGET == HWY_AVX3_DL || sizeof(T) != 2) { + return CompressStore(v, m, d, unaligned); + } else { + const size_t count = CountTrue(m); + const Vec512<T> compressed = Compress(v, m); + const Vec512<T> prev = LoadU(d, unaligned); + StoreU(IfThenElse(FirstN(d, count), compressed, prev), d, unaligned); + return count; + } +} + +// ------------------------------ CompressBitsStore +template <typename T> +HWY_API size_t CompressBitsStore(Vec512<T> v, const uint8_t* HWY_RESTRICT bits, + Full512<T> d, T* HWY_RESTRICT unaligned) { + return CompressStore(v, LoadMaskBits(d, bits), d, unaligned); +} + +// ------------------------------ StoreInterleaved3 (CombineShiftRightBytes, +// TableLookupBytes) + +HWY_API void StoreInterleaved3(const Vec512<uint8_t> a, const Vec512<uint8_t> b, + const Vec512<uint8_t> c, Full512<uint8_t> d, + uint8_t* HWY_RESTRICT unaligned) { + const auto k5 = Set(d, 5); + const auto k6 = Set(d, 6); + + // Shuffle (a,b,c) vector bytes to (MSB on left): r5, bgr[4:0]. + // 0x80 so lanes to be filled from other vectors are 0 for blending. + alignas(16) static constexpr uint8_t tbl_r0[16] = { + 0, 0x80, 0x80, 1, 0x80, 0x80, 2, 0x80, 0x80, // + 3, 0x80, 0x80, 4, 0x80, 0x80, 5}; + alignas(16) static constexpr uint8_t tbl_g0[16] = { + 0x80, 0, 0x80, 0x80, 1, 0x80, // + 0x80, 2, 0x80, 0x80, 3, 0x80, 0x80, 4, 0x80, 0x80}; + const auto shuf_r0 = LoadDup128(d, tbl_r0); + const auto shuf_g0 = LoadDup128(d, tbl_g0); // cannot reuse r0 due to 5 + const auto shuf_b0 = CombineShiftRightBytes<15>(d, shuf_g0, shuf_g0); + const auto r0 = TableLookupBytes(a, shuf_r0); // 5..4..3..2..1..0 + const auto g0 = TableLookupBytes(b, shuf_g0); // ..4..3..2..1..0. + const auto b0 = TableLookupBytes(c, shuf_b0); // .4..3..2..1..0.. + const auto i = (r0 | g0 | b0).raw; // low byte in each 128bit: 30 20 10 00 + + // Second vector: g10,r10, bgr[9:6], b5,g5 + const auto shuf_r1 = shuf_b0 + k6; // .A..9..8..7..6.. + const auto shuf_g1 = shuf_r0 + k5; // A..9..8..7..6..5 + const auto shuf_b1 = shuf_g0 + k5; // ..9..8..7..6..5. + const auto r1 = TableLookupBytes(a, shuf_r1); + const auto g1 = TableLookupBytes(b, shuf_g1); + const auto b1 = TableLookupBytes(c, shuf_b1); + const auto j = (r1 | g1 | b1).raw; // low byte in each 128bit: 35 25 15 05 + + // Third vector: bgr[15:11], b10 + const auto shuf_r2 = shuf_b1 + k6; // ..F..E..D..C..B. + const auto shuf_g2 = shuf_r1 + k5; // .F..E..D..C..B.. + const auto shuf_b2 = shuf_g1 + k5; // F..E..D..C..B..A + const auto r2 = TableLookupBytes(a, shuf_r2); + const auto g2 = TableLookupBytes(b, shuf_g2); + const auto b2 = TableLookupBytes(c, shuf_b2); + const auto k = (r2 | g2 | b2).raw; // low byte in each 128bit: 3A 2A 1A 0A + + // To obtain 10 0A 05 00 in one vector, transpose "rows" into "columns". + const auto k3_k0_i3_i0 = _mm512_shuffle_i64x2(i, k, _MM_SHUFFLE(3, 0, 3, 0)); + const auto i1_i2_j0_j1 = _mm512_shuffle_i64x2(j, i, _MM_SHUFFLE(1, 2, 0, 1)); + const auto j2_j3_k1_k2 = _mm512_shuffle_i64x2(k, j, _MM_SHUFFLE(2, 3, 1, 2)); + + // Alternating order, most-significant 128 bits from the second arg. + const __mmask8 m = 0xCC; + const auto i1_k0_j0_i0 = _mm512_mask_blend_epi64(m, k3_k0_i3_i0, i1_i2_j0_j1); + const auto j2_i2_k1_j1 = _mm512_mask_blend_epi64(m, i1_i2_j0_j1, j2_j3_k1_k2); + const auto k3_j3_i3_k2 = _mm512_mask_blend_epi64(m, j2_j3_k1_k2, k3_k0_i3_i0); + + StoreU(Vec512<uint8_t>{i1_k0_j0_i0}, d, unaligned + 0 * 64); // 10 0A 05 00 + StoreU(Vec512<uint8_t>{j2_i2_k1_j1}, d, unaligned + 1 * 64); // 25 20 1A 15 + StoreU(Vec512<uint8_t>{k3_j3_i3_k2}, d, unaligned + 2 * 64); // 3A 35 30 2A +} + +// ------------------------------ StoreInterleaved4 + +HWY_API void StoreInterleaved4(const Vec512<uint8_t> v0, + const Vec512<uint8_t> v1, + const Vec512<uint8_t> v2, + const Vec512<uint8_t> v3, Full512<uint8_t> d8, + uint8_t* HWY_RESTRICT unaligned) { + const RepartitionToWide<decltype(d8)> d16; + const RepartitionToWide<decltype(d16)> d32; + // let a,b,c,d denote v0..3. + const auto ba0 = ZipLower(d16, v0, v1); // b7 a7 .. b0 a0 + const auto dc0 = ZipLower(d16, v2, v3); // d7 c7 .. d0 c0 + const auto ba8 = ZipUpper(d16, v0, v1); + const auto dc8 = ZipUpper(d16, v2, v3); + const auto i = ZipLower(d32, ba0, dc0).raw; // 4x128bit: d..a3 d..a0 + const auto j = ZipUpper(d32, ba0, dc0).raw; // 4x128bit: d..a7 d..a4 + const auto k = ZipLower(d32, ba8, dc8).raw; // 4x128bit: d..aB d..a8 + const auto l = ZipUpper(d32, ba8, dc8).raw; // 4x128bit: d..aF d..aC + // 128-bit blocks were independent until now; transpose 4x4. + const auto j1_j0_i1_i0 = _mm512_shuffle_i64x2(i, j, _MM_SHUFFLE(1, 0, 1, 0)); + const auto l1_l0_k1_k0 = _mm512_shuffle_i64x2(k, l, _MM_SHUFFLE(1, 0, 1, 0)); + const auto j3_j2_i3_i2 = _mm512_shuffle_i64x2(i, j, _MM_SHUFFLE(3, 2, 3, 2)); + const auto l3_l2_k3_k2 = _mm512_shuffle_i64x2(k, l, _MM_SHUFFLE(3, 2, 3, 2)); + constexpr int k20 = _MM_SHUFFLE(2, 0, 2, 0); + constexpr int k31 = _MM_SHUFFLE(3, 1, 3, 1); + const auto l0_k0_j0_i0 = _mm512_shuffle_i64x2(j1_j0_i1_i0, l1_l0_k1_k0, k20); + const auto l1_k1_j1_i1 = _mm512_shuffle_i64x2(j1_j0_i1_i0, l1_l0_k1_k0, k31); + const auto l2_k2_j2_i2 = _mm512_shuffle_i64x2(j3_j2_i3_i2, l3_l2_k3_k2, k20); + const auto l3_k3_j3_i3 = _mm512_shuffle_i64x2(j3_j2_i3_i2, l3_l2_k3_k2, k31); + StoreU(Vec512<uint8_t>{l0_k0_j0_i0}, d8, unaligned + 0 * 64); + StoreU(Vec512<uint8_t>{l1_k1_j1_i1}, d8, unaligned + 1 * 64); + StoreU(Vec512<uint8_t>{l2_k2_j2_i2}, d8, unaligned + 2 * 64); + StoreU(Vec512<uint8_t>{l3_k3_j3_i3}, d8, unaligned + 3 * 64); +} + +// ------------------------------ MulEven/Odd (Shuffle2301, InterleaveLower) + +HWY_INLINE Vec512<uint64_t> MulEven(const Vec512<uint64_t> a, + const Vec512<uint64_t> b) { + const DFromV<decltype(a)> du64; + const RepartitionToNarrow<decltype(du64)> du32; + const auto maskL = Set(du64, 0xFFFFFFFFULL); + const auto a32 = BitCast(du32, a); + const auto b32 = BitCast(du32, b); + // Inputs for MulEven: we only need the lower 32 bits + const auto aH = Shuffle2301(a32); + const auto bH = Shuffle2301(b32); + + // Knuth double-word multiplication. We use 32x32 = 64 MulEven and only need + // the even (lower 64 bits of every 128-bit block) results. See + // https://github.com/hcs0/Hackers-Delight/blob/master/muldwu.c.tat + const auto aLbL = MulEven(a32, b32); + const auto w3 = aLbL & maskL; + + const auto t2 = MulEven(aH, b32) + ShiftRight<32>(aLbL); + const auto w2 = t2 & maskL; + const auto w1 = ShiftRight<32>(t2); + + const auto t = MulEven(a32, bH) + w2; + const auto k = ShiftRight<32>(t); + + const auto mulH = MulEven(aH, bH) + w1 + k; + const auto mulL = ShiftLeft<32>(t) + w3; + return InterleaveLower(mulL, mulH); +} + +HWY_INLINE Vec512<uint64_t> MulOdd(const Vec512<uint64_t> a, + const Vec512<uint64_t> b) { + const DFromV<decltype(a)> du64; + const RepartitionToNarrow<decltype(du64)> du32; + const auto maskL = Set(du64, 0xFFFFFFFFULL); + const auto a32 = BitCast(du32, a); + const auto b32 = BitCast(du32, b); + // Inputs for MulEven: we only need bits [95:64] (= upper half of input) + const auto aH = Shuffle2301(a32); + const auto bH = Shuffle2301(b32); + + // Same as above, but we're using the odd results (upper 64 bits per block). + const auto aLbL = MulEven(a32, b32); + const auto w3 = aLbL & maskL; + + const auto t2 = MulEven(aH, b32) + ShiftRight<32>(aLbL); + const auto w2 = t2 & maskL; + const auto w1 = ShiftRight<32>(t2); + + const auto t = MulEven(a32, bH) + w2; + const auto k = ShiftRight<32>(t); + + const auto mulH = MulEven(aH, bH) + w1 + k; + const auto mulL = ShiftLeft<32>(t) + w3; + return InterleaveUpper(du64, mulL, mulH); +} + +// ------------------------------ ReorderWidenMulAccumulate (MulAdd, ZipLower) + +HWY_API Vec512<float> ReorderWidenMulAccumulate(Full512<float> df32, + Vec512<bfloat16_t> a, + Vec512<bfloat16_t> b, + const Vec512<float> sum0, + Vec512<float>& sum1) { + // TODO(janwas): _mm512_dpbf16_ps when available + const Repartition<uint16_t, decltype(df32)> du16; + const RebindToUnsigned<decltype(df32)> du32; + const Vec512<uint16_t> zero = Zero(du16); + // Lane order within sum0/1 is undefined, hence we can avoid the + // longer-latency lane-crossing PromoteTo. + const Vec512<uint32_t> a0 = ZipLower(du32, zero, BitCast(du16, a)); + const Vec512<uint32_t> a1 = ZipUpper(du32, zero, BitCast(du16, a)); + const Vec512<uint32_t> b0 = ZipLower(du32, zero, BitCast(du16, b)); + const Vec512<uint32_t> b1 = ZipUpper(du32, zero, BitCast(du16, b)); + sum1 = MulAdd(BitCast(df32, a1), BitCast(df32, b1), sum1); + return MulAdd(BitCast(df32, a0), BitCast(df32, b0), sum0); +} + +// ------------------------------ Reductions + +// Returns the sum in each lane. +HWY_API Vec512<int32_t> SumOfLanes(Full512<int32_t> d, Vec512<int32_t> v) { + return Set(d, _mm512_reduce_add_epi32(v.raw)); +} +HWY_API Vec512<int64_t> SumOfLanes(Full512<int64_t> d, Vec512<int64_t> v) { + return Set(d, _mm512_reduce_add_epi64(v.raw)); +} +HWY_API Vec512<uint32_t> SumOfLanes(Full512<uint32_t> d, Vec512<uint32_t> v) { + return Set(d, static_cast<uint32_t>(_mm512_reduce_add_epi32(v.raw))); +} +HWY_API Vec512<uint64_t> SumOfLanes(Full512<uint64_t> d, Vec512<uint64_t> v) { + return Set(d, static_cast<uint64_t>(_mm512_reduce_add_epi64(v.raw))); +} +HWY_API Vec512<float> SumOfLanes(Full512<float> d, Vec512<float> v) { + return Set(d, _mm512_reduce_add_ps(v.raw)); +} +HWY_API Vec512<double> SumOfLanes(Full512<double> d, Vec512<double> v) { + return Set(d, _mm512_reduce_add_pd(v.raw)); +} + +// Returns the minimum in each lane. +HWY_API Vec512<int32_t> MinOfLanes(Full512<int32_t> d, Vec512<int32_t> v) { + return Set(d, _mm512_reduce_min_epi32(v.raw)); +} +HWY_API Vec512<int64_t> MinOfLanes(Full512<int64_t> d, Vec512<int64_t> v) { + return Set(d, _mm512_reduce_min_epi64(v.raw)); +} +HWY_API Vec512<uint32_t> MinOfLanes(Full512<uint32_t> d, Vec512<uint32_t> v) { + return Set(d, _mm512_reduce_min_epu32(v.raw)); +} +HWY_API Vec512<uint64_t> MinOfLanes(Full512<uint64_t> d, Vec512<uint64_t> v) { + return Set(d, _mm512_reduce_min_epu64(v.raw)); +} +HWY_API Vec512<float> MinOfLanes(Full512<float> d, Vec512<float> v) { + return Set(d, _mm512_reduce_min_ps(v.raw)); +} +HWY_API Vec512<double> MinOfLanes(Full512<double> d, Vec512<double> v) { + return Set(d, _mm512_reduce_min_pd(v.raw)); +} +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Vec512<T> MinOfLanes(Full512<T> d, Vec512<T> v) { + const Repartition<int32_t, decltype(d)> d32; + const auto even = And(BitCast(d32, v), Set(d32, 0xFFFF)); + const auto odd = ShiftRight<16>(BitCast(d32, v)); + const auto min = MinOfLanes(d32, Min(even, odd)); + // Also broadcast into odd lanes. + return BitCast(d, Or(min, ShiftLeft<16>(min))); +} + +// Returns the maximum in each lane. +HWY_API Vec512<int32_t> MaxOfLanes(Full512<int32_t> d, Vec512<int32_t> v) { + return Set(d, _mm512_reduce_max_epi32(v.raw)); +} +HWY_API Vec512<int64_t> MaxOfLanes(Full512<int64_t> d, Vec512<int64_t> v) { + return Set(d, _mm512_reduce_max_epi64(v.raw)); +} +HWY_API Vec512<uint32_t> MaxOfLanes(Full512<uint32_t> d, Vec512<uint32_t> v) { + return Set(d, _mm512_reduce_max_epu32(v.raw)); +} +HWY_API Vec512<uint64_t> MaxOfLanes(Full512<uint64_t> d, Vec512<uint64_t> v) { + return Set(d, _mm512_reduce_max_epu64(v.raw)); +} +HWY_API Vec512<float> MaxOfLanes(Full512<float> d, Vec512<float> v) { + return Set(d, _mm512_reduce_max_ps(v.raw)); +} +HWY_API Vec512<double> MaxOfLanes(Full512<double> d, Vec512<double> v) { + return Set(d, _mm512_reduce_max_pd(v.raw)); +} +template <typename T, HWY_IF_LANE_SIZE(T, 2)> +HWY_API Vec512<T> MaxOfLanes(Full512<T> d, Vec512<T> v) { + const Repartition<int32_t, decltype(d)> d32; + const auto even = And(BitCast(d32, v), Set(d32, 0xFFFF)); + const auto odd = ShiftRight<16>(BitCast(d32, v)); + const auto min = MaxOfLanes(d32, Max(even, odd)); + // Also broadcast into odd lanes. + return BitCast(d, Or(min, ShiftLeft<16>(min))); +} + +// ================================================== DEPRECATED + +template <typename T> +HWY_API size_t StoreMaskBits(const Mask512<T> mask, uint8_t* bits) { + return StoreMaskBits(Full512<T>(), mask, bits); +} + +template <typename T> +HWY_API bool AllTrue(const Mask512<T> mask) { + return AllTrue(Full512<T>(), mask); +} + +template <typename T> +HWY_API bool AllFalse(const Mask512<T> mask) { + return AllFalse(Full512<T>(), mask); +} + +template <typename T> +HWY_API size_t CountTrue(const Mask512<T> mask) { + return CountTrue(Full512<T>(), mask); +} + +template <typename T> +HWY_API Vec512<T> SumOfLanes(Vec512<T> v) { + return SumOfLanes(Full512<T>(), v); +} + +template <typename T> +HWY_API Vec512<T> MinOfLanes(Vec512<T> v) { + return MinOfLanes(Full512<T>(), v); +} + +template <typename T> +HWY_API Vec512<T> MaxOfLanes(Vec512<T> v) { + return MaxOfLanes(Full512<T>(), v); +} + +template <typename T> +HWY_API Vec256<T> UpperHalf(Vec512<T> v) { + return UpperHalf(Full256<T>(), v); +} + +template <int kBytes, typename T> +HWY_API Vec512<T> ShiftRightBytes(const Vec512<T> v) { + return ShiftRightBytes<kBytes>(Full512<T>(), v); +} + +template <int kLanes, typename T> +HWY_API Vec512<T> ShiftRightLanes(const Vec512<T> v) { + return ShiftRightBytes<kLanes>(Full512<T>(), v); +} + +template <size_t kBytes, typename T> +HWY_API Vec512<T> CombineShiftRightBytes(Vec512<T> hi, Vec512<T> lo) { + return CombineShiftRightBytes<kBytes>(Full512<T>(), hi, lo); +} + +template <typename T> +HWY_API Vec512<T> InterleaveUpper(Vec512<T> a, Vec512<T> b) { + return InterleaveUpper(Full512<T>(), a, b); +} + +template <typename T> +HWY_API Vec512<MakeWide<T>> ZipUpper(Vec512<T> a, Vec512<T> b) { + return InterleaveUpper(Full512<MakeWide<T>>(), a, b); +} + +template <typename T> +HWY_API Vec512<T> Combine(Vec256<T> hi, Vec256<T> lo) { + return Combine(Full512<T>(), hi, lo); +} + +template <typename T> +HWY_API Vec512<T> ZeroExtendVector(Vec256<T> lo) { + return ZeroExtendVector(Full512<T>(), lo); +} + +template <typename T> +HWY_API Vec512<T> ConcatLowerLower(Vec512<T> hi, Vec512<T> lo) { + return ConcatLowerLower(Full512<T>(), hi, lo); +} + +template <typename T> +HWY_API Vec512<T> ConcatLowerUpper(Vec512<T> hi, Vec512<T> lo) { + return ConcatLowerUpper(Full512<T>(), hi, lo); +} + +template <typename T> +HWY_API Vec512<T> ConcatUpperLower(Vec512<T> hi, Vec512<T> lo) { + return ConcatUpperLower(Full512<T>(), hi, lo); +} + +template <typename T> +HWY_API Vec512<T> ConcatUpperUpper(Vec512<T> hi, Vec512<T> lo) { + return ConcatUpperUpper(Full512<T>(), hi, lo); +} + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); diff --git a/third_party/highway/hwy/targets.cc b/third_party/highway/hwy/targets.cc new file mode 100644 index 0000000..daab3a6 --- /dev/null +++ b/third_party/highway/hwy/targets.cc @@ -0,0 +1,360 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "hwy/targets.h" + +#include <stdarg.h> +#include <stdint.h> +#include <stdio.h> + +#include <atomic> +#include <cstddef> +#include <limits> + +#if defined(ADDRESS_SANITIZER) || defined(MEMORY_SANITIZER) || \ + defined(THREAD_SANITIZER) +#include "sanitizer/common_interface_defs.h" // __sanitizer_print_stack_trace +#endif // defined(*_SANITIZER) + +#if HWY_ARCH_X86 +#include <xmmintrin.h> +#if HWY_COMPILER_MSVC +#include <intrin.h> +#else // HWY_COMPILER_MSVC +#include <cpuid.h> +#endif // HWY_COMPILER_MSVC +#endif // HWY_ARCH_X86 + +namespace hwy { +namespace { + +#if HWY_ARCH_X86 + +HWY_INLINE bool IsBitSet(const uint32_t reg, const int index) { + return (reg & (1U << index)) != 0; +} + +// Calls CPUID instruction with eax=level and ecx=count and returns the result +// in abcd array where abcd = {eax, ebx, ecx, edx} (hence the name abcd). +HWY_INLINE void Cpuid(const uint32_t level, const uint32_t count, + uint32_t* HWY_RESTRICT abcd) { +#if HWY_COMPILER_MSVC + int regs[4]; + __cpuidex(regs, level, count); + for (int i = 0; i < 4; ++i) { + abcd[i] = regs[i]; + } +#else // HWY_COMPILER_MSVC + uint32_t a; + uint32_t b; + uint32_t c; + uint32_t d; + __cpuid_count(level, count, a, b, c, d); + abcd[0] = a; + abcd[1] = b; + abcd[2] = c; + abcd[3] = d; +#endif // HWY_COMPILER_MSVC +} + +// Returns the lower 32 bits of extended control register 0. +// Requires CPU support for "OSXSAVE" (see below). +uint32_t ReadXCR0() { +#if HWY_COMPILER_MSVC + return static_cast<uint32_t>(_xgetbv(0)); +#else // HWY_COMPILER_MSVC + uint32_t xcr0, xcr0_high; + const uint32_t index = 0; + asm volatile(".byte 0x0F, 0x01, 0xD0" + : "=a"(xcr0), "=d"(xcr0_high) + : "c"(index)); + return xcr0; +#endif // HWY_COMPILER_MSVC +} + +#endif // HWY_ARCH_X86 + +// Not function-local => no compiler-generated locking. +std::atomic<uint32_t> supported_{0}; // Not yet initialized + +// When running tests, this value can be set to the mocked supported targets +// mask. Only written to from a single thread before the test starts. +uint32_t supported_targets_for_test_ = 0; + +// Mask of targets disabled at runtime with DisableTargets. +uint32_t supported_mask_{std::numeric_limits<uint32_t>::max()}; + +#if HWY_ARCH_X86 +// Arbritrary bit indices indicating which instruction set extensions are +// supported. Use enum to ensure values are distinct. +enum class FeatureIndex : uint32_t { + kSSE = 0, + kSSE2, + kSSE3, + kSSSE3, + + kSSE41, + kSSE42, + kCLMUL, + kAES, + + kAVX, + kAVX2, + kF16C, + kFMA, + kLZCNT, + kBMI, + kBMI2, + + kAVX512F, + kAVX512VL, + kAVX512DQ, + kAVX512BW, + + kVNNI, + kVPCLMULQDQ, + kVBMI2, + kVAES, + kPOPCNTDQ, + kBITALG, + + kSentinel +}; +static_assert(static_cast<size_t>(FeatureIndex::kSentinel) < 64, + "Too many bits for u64"); + +HWY_INLINE constexpr uint64_t Bit(FeatureIndex index) { + return 1ull << static_cast<size_t>(index); +} + +constexpr uint64_t kGroupSSSE3 = + Bit(FeatureIndex::kSSE) | Bit(FeatureIndex::kSSE2) | + Bit(FeatureIndex::kSSE3) | Bit(FeatureIndex::kSSSE3); + +constexpr uint64_t kGroupSSE4 = + Bit(FeatureIndex::kSSE41) | Bit(FeatureIndex::kSSE42) | + Bit(FeatureIndex::kCLMUL) | Bit(FeatureIndex::kAES) | kGroupSSSE3; + +// We normally assume BMI/BMI2/FMA are available if AVX2 is. This allows us to +// use BZHI and (compiler-generated) MULX. However, VirtualBox lacks them +// [https://www.virtualbox.org/ticket/15471]. Thus we provide the option of +// avoiding using and requiring these so AVX2 can still be used. +#ifdef HWY_DISABLE_BMI2_FMA +constexpr uint64_t kGroupBMI2_FMA = 0; +#else +constexpr uint64_t kGroupBMI2_FMA = Bit(FeatureIndex::kBMI) | + Bit(FeatureIndex::kBMI2) | + Bit(FeatureIndex::kFMA); +#endif + +#ifdef HWY_DISABLE_F16C +constexpr uint64_t kGroupF16C = 0; +#else +constexpr uint64_t kGroupF16C = Bit(FeatureIndex::kF16C); +#endif + +constexpr uint64_t kGroupAVX2 = + Bit(FeatureIndex::kAVX) | Bit(FeatureIndex::kAVX2) | + Bit(FeatureIndex::kLZCNT) | kGroupBMI2_FMA | kGroupF16C | kGroupSSE4; + +constexpr uint64_t kGroupAVX3 = + Bit(FeatureIndex::kAVX512F) | Bit(FeatureIndex::kAVX512VL) | + Bit(FeatureIndex::kAVX512DQ) | Bit(FeatureIndex::kAVX512BW) | kGroupAVX2; + +constexpr uint64_t kGroupAVX3_DL = + Bit(FeatureIndex::kVNNI) | Bit(FeatureIndex::kVPCLMULQDQ) | + Bit(FeatureIndex::kVBMI2) | Bit(FeatureIndex::kVAES) | + Bit(FeatureIndex::kPOPCNTDQ) | Bit(FeatureIndex::kBITALG) | kGroupAVX3; + +#endif // HWY_ARCH_X86 + +} // namespace + +HWY_NORETURN void HWY_FORMAT(3, 4) + Abort(const char* file, int line, const char* format, ...) { + char buf[2000]; + va_list args; + va_start(args, format); + vsnprintf(buf, sizeof(buf), format, args); + va_end(args); + + fprintf(stderr, "Abort at %s:%d: %s\n", file, line, buf); +#if defined(ADDRESS_SANITIZER) || defined(MEMORY_SANITIZER) || \ + defined(THREAD_SANITIZER) + // If compiled with any sanitizer print a stack trace. This call doesn't crash + // the program, instead the trap below will crash it also allowing gdb to + // break there. + __sanitizer_print_stack_trace(); +#endif // defined(*_SANITIZER) + fflush(stderr); + +#if HWY_COMPILER_MSVC + abort(); // Compile error without this due to HWY_NORETURN. +#elif HWY_ARCH_RVV + exit(1); // trap/abort just freeze Spike +#else + __builtin_trap(); +#endif +} + +void DisableTargets(uint32_t disabled_targets) { + supported_mask_ = ~(disabled_targets & ~uint32_t(HWY_ENABLED_BASELINE)); + // We can call Update() here to initialize the mask but that will trigger a + // call to SupportedTargets() which we use in tests to tell whether any of the + // highway dynamic dispatch functions were used. + chosen_target.DeInit(); +} + +void SetSupportedTargetsForTest(uint32_t targets) { + // Reset the cached supported_ value to 0 to force a re-evaluation in the + // next call to SupportedTargets() which will use the mocked value set here + // if not zero. + supported_.store(0, std::memory_order_release); + supported_targets_for_test_ = targets; + chosen_target.DeInit(); +} + +bool SupportedTargetsCalledForTest() { + return supported_.load(std::memory_order_acquire) != 0; +} + +uint32_t SupportedTargets() { + uint32_t bits = supported_.load(std::memory_order_acquire); + // Already initialized? + if (HWY_LIKELY(bits != 0)) { + return bits & supported_mask_; + } + + // When running tests, this allows to mock the current supported targets. + if (HWY_UNLIKELY(supported_targets_for_test_ != 0)) { + // Store the value to signal that this was used. + supported_.store(supported_targets_for_test_, std::memory_order_release); + return supported_targets_for_test_ & supported_mask_; + } + + bits = HWY_SCALAR; + +#if HWY_ARCH_X86 + bool has_osxsave = false; + { // ensures we do not accidentally use flags outside this block + uint64_t flags = 0; + uint32_t abcd[4]; + + Cpuid(0, 0, abcd); + const uint32_t max_level = abcd[0]; + + // Standard feature flags + Cpuid(1, 0, abcd); + flags |= IsBitSet(abcd[3], 25) ? Bit(FeatureIndex::kSSE) : 0; + flags |= IsBitSet(abcd[3], 26) ? Bit(FeatureIndex::kSSE2) : 0; + flags |= IsBitSet(abcd[2], 0) ? Bit(FeatureIndex::kSSE3) : 0; + flags |= IsBitSet(abcd[2], 1) ? Bit(FeatureIndex::kCLMUL) : 0; + flags |= IsBitSet(abcd[2], 9) ? Bit(FeatureIndex::kSSSE3) : 0; + flags |= IsBitSet(abcd[2], 12) ? Bit(FeatureIndex::kFMA) : 0; + flags |= IsBitSet(abcd[2], 19) ? Bit(FeatureIndex::kSSE41) : 0; + flags |= IsBitSet(abcd[2], 20) ? Bit(FeatureIndex::kSSE42) : 0; + flags |= IsBitSet(abcd[2], 25) ? Bit(FeatureIndex::kAES) : 0; + flags |= IsBitSet(abcd[2], 28) ? Bit(FeatureIndex::kAVX) : 0; + flags |= IsBitSet(abcd[2], 29) ? Bit(FeatureIndex::kF16C) : 0; + has_osxsave = IsBitSet(abcd[2], 27); + + // Extended feature flags + Cpuid(0x80000001U, 0, abcd); + flags |= IsBitSet(abcd[2], 5) ? Bit(FeatureIndex::kLZCNT) : 0; + + // Extended features + if (max_level >= 7) { + Cpuid(7, 0, abcd); + flags |= IsBitSet(abcd[1], 3) ? Bit(FeatureIndex::kBMI) : 0; + flags |= IsBitSet(abcd[1], 5) ? Bit(FeatureIndex::kAVX2) : 0; + flags |= IsBitSet(abcd[1], 8) ? Bit(FeatureIndex::kBMI2) : 0; + + flags |= IsBitSet(abcd[1], 16) ? Bit(FeatureIndex::kAVX512F) : 0; + flags |= IsBitSet(abcd[1], 17) ? Bit(FeatureIndex::kAVX512DQ) : 0; + flags |= IsBitSet(abcd[1], 30) ? Bit(FeatureIndex::kAVX512BW) : 0; + flags |= IsBitSet(abcd[1], 31) ? Bit(FeatureIndex::kAVX512VL) : 0; + + flags |= IsBitSet(abcd[2], 6) ? Bit(FeatureIndex::kVBMI2) : 0; + flags |= IsBitSet(abcd[2], 9) ? Bit(FeatureIndex::kVAES) : 0; + flags |= IsBitSet(abcd[2], 10) ? Bit(FeatureIndex::kVPCLMULQDQ) : 0; + flags |= IsBitSet(abcd[2], 11) ? Bit(FeatureIndex::kVNNI) : 0; + flags |= IsBitSet(abcd[2], 12) ? Bit(FeatureIndex::kBITALG) : 0; + flags |= IsBitSet(abcd[2], 14) ? Bit(FeatureIndex::kPOPCNTDQ) : 0; + } + + // Set target bit(s) if all their group's flags are all set. + if ((flags & kGroupAVX3_DL) == kGroupAVX3_DL) { + bits |= HWY_AVX3_DL; + } + if ((flags & kGroupAVX3) == kGroupAVX3) { + bits |= HWY_AVX3; + } + if ((flags & kGroupAVX2) == kGroupAVX2) { + bits |= HWY_AVX2; + } + if ((flags & kGroupSSE4) == kGroupSSE4) { + bits |= HWY_SSE4; + } + if ((flags & kGroupSSSE3) == kGroupSSSE3) { + bits |= HWY_SSSE3; + } + } + + // Clear bits if the OS does not support XSAVE - otherwise, registers + // are not preserved across context switches. + if (has_osxsave) { + const uint32_t xcr0 = ReadXCR0(); + // XMM + if (!IsBitSet(xcr0, 1)) { + bits &= + ~uint32_t(HWY_SSSE3 | HWY_SSE4 | HWY_AVX2 | HWY_AVX3 | HWY_AVX3_DL); + } + // YMM + if (!IsBitSet(xcr0, 2)) { + bits &= ~uint32_t(HWY_AVX2 | HWY_AVX3 | HWY_AVX3_DL); + } + // ZMM + opmask + if ((xcr0 & 0x70) != 0x70) { + bits &= ~uint32_t(HWY_AVX3 | HWY_AVX3_DL); + } + } + +#else + // TODO(janwas): detect for other platforms + bits = HWY_ENABLED_BASELINE; +#endif // HWY_ARCH_X86 + + if ((bits & HWY_ENABLED_BASELINE) != HWY_ENABLED_BASELINE) { + fprintf(stderr, "WARNING: CPU supports %zx but software requires %x\n", + size_t(bits), HWY_ENABLED_BASELINE); + } + + supported_.store(bits, std::memory_order_release); + return bits & supported_mask_; +} + +// Declared in targets.h +ChosenTarget chosen_target; + +void ChosenTarget::Update() { + // The supported variable contains the current CPU supported targets shifted + // to the location expected by the ChosenTarget mask. We enabled SCALAR + // regardless of whether it was compiled since it is also used as the + // fallback mechanism to the baseline target. + uint32_t supported = HWY_CHOSEN_TARGET_SHIFT(hwy::SupportedTargets()) | + HWY_CHOSEN_TARGET_MASK_SCALAR; + mask_.store(supported); +} + +} // namespace hwy diff --git a/third_party/highway/hwy/targets.h b/third_party/highway/hwy/targets.h new file mode 100644 index 0000000..95381e4 --- /dev/null +++ b/third_party/highway/hwy/targets.h @@ -0,0 +1,256 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef HIGHWAY_HWY_TARGETS_H_ +#define HIGHWAY_HWY_TARGETS_H_ + +#include <vector> + +// For SIMD module implementations and their callers. Defines which targets to +// generate and call. + +#include "hwy/base.h" +#include "hwy/detect_targets.h" + +namespace hwy { + +// Returns (cached) bitfield of enabled targets that are supported on this CPU. +// Implemented in targets.cc; unconditionally compiled to support the use case +// of binary-only distributions. The HWY_SUPPORTED_TARGETS wrapper may allow +// eliding calls to this function. +uint32_t SupportedTargets(); + +// Evaluates to a function call, or literal if there is a single target. +#if (HWY_TARGETS & (HWY_TARGETS - 1)) == 0 +#define HWY_SUPPORTED_TARGETS HWY_TARGETS +#else +#define HWY_SUPPORTED_TARGETS hwy::SupportedTargets() +#endif + +// Disable from runtime dispatch the mask of compiled in targets. Targets that +// were not enabled at compile time are ignored. This function is useful to +// disable a target supported by the CPU that is known to have bugs or when a +// lower target is desired. For this reason, attempts to disable targets which +// are in HWY_ENABLED_BASELINE have no effect so SupportedTargets() always +// returns at least the baseline target. +void DisableTargets(uint32_t disabled_targets); + +// Set the mock mask of CPU supported targets instead of the actual CPU +// supported targets computed in SupportedTargets(). The return value of +// SupportedTargets() will still be affected by the DisableTargets() mask +// regardless of this mock, to prevent accidentally adding targets that are +// known to be buggy in the current CPU. Call with a mask of 0 to disable the +// mock and use the actual CPU supported targets instead. +void SetSupportedTargetsForTest(uint32_t targets); + +// Returns whether the SupportedTargets() function was called since the last +// SetSupportedTargetsForTest() call. +bool SupportedTargetsCalledForTest(); + +// Return the list of targets in HWY_TARGETS supported by the CPU as a list of +// individual HWY_* target macros such as HWY_SCALAR or HWY_NEON. This list +// is affected by the current SetSupportedTargetsForTest() mock if any. +HWY_INLINE std::vector<uint32_t> SupportedAndGeneratedTargets() { + std::vector<uint32_t> ret; + for (uint32_t targets = SupportedTargets() & HWY_TARGETS; targets != 0; + targets = targets & (targets - 1)) { + uint32_t current_target = targets & ~(targets - 1); + ret.push_back(current_target); + } + return ret; +} + +static inline HWY_MAYBE_UNUSED const char* TargetName(uint32_t target) { + switch (target) { +#if HWY_ARCH_X86 + case HWY_SSSE3: + return "SSSE3"; + case HWY_SSE4: + return "SSE4"; + case HWY_AVX2: + return "AVX2"; + case HWY_AVX3: + return "AVX3"; + case HWY_AVX3_DL: + return "AVX3_DL"; +#endif + +#if HWY_ARCH_ARM + case HWY_SVE2: + return "SVE2"; + case HWY_SVE: + return "SVE"; + case HWY_NEON: + return "Neon"; +#endif + +#if HWY_ARCH_PPC + case HWY_PPC8: + return "Power8"; +#endif + +#if HWY_ARCH_WASM + case HWY_WASM: + return "Wasm"; +#endif + +#if HWY_ARCH_RVV + case HWY_RVV: + return "RVV"; +#endif + + case HWY_SCALAR: + return "Scalar"; + + default: + return "Unknown"; // must satisfy gtest IsValidParamName() + } +} + +// The maximum number of dynamic targets on any architecture is defined by +// HWY_MAX_DYNAMIC_TARGETS and depends on the arch. + +// For the ChosenTarget mask and index we use a different bit arrangement than +// in the HWY_TARGETS mask. Only the targets involved in the current +// architecture are used in this mask, and therefore only the least significant +// (HWY_MAX_DYNAMIC_TARGETS + 2) bits of the uint32_t mask are used. The least +// significant bit is set when the mask is not initialized, the next +// HWY_MAX_DYNAMIC_TARGETS more significant bits are a range of bits from the +// HWY_TARGETS or SupportedTargets() mask for the given architecture shifted to +// that position and the next more significant bit is used for the scalar +// target. Because of this we need to define equivalent values for HWY_TARGETS +// in this representation. +// This mask representation allows to use ctz() on this mask and obtain a small +// number that's used as an index of the table for dynamic dispatch. In this +// way the first entry is used when the mask is uninitialized, the following +// HWY_MAX_DYNAMIC_TARGETS are for dynamic dispatch and the last one is for +// scalar. + +// The HWY_SCALAR bit in the ChosenTarget mask format. +#define HWY_CHOSEN_TARGET_MASK_SCALAR (1u << (HWY_MAX_DYNAMIC_TARGETS + 1)) + +// Converts from a HWY_TARGETS mask to a ChosenTarget mask format for the +// current architecture. +#define HWY_CHOSEN_TARGET_SHIFT(X) \ + ((((X) >> (HWY_HIGHEST_TARGET_BIT + 1 - HWY_MAX_DYNAMIC_TARGETS)) & \ + ((1u << HWY_MAX_DYNAMIC_TARGETS) - 1)) \ + << 1) + +// The HWY_TARGETS mask in the ChosenTarget mask format. +#define HWY_CHOSEN_TARGET_MASK_TARGETS \ + (HWY_CHOSEN_TARGET_SHIFT(HWY_TARGETS) | HWY_CHOSEN_TARGET_MASK_SCALAR | 1u) + +#if HWY_ARCH_X86 +// Maximum number of dynamic targets, changing this value is an ABI incompatible +// change +#define HWY_MAX_DYNAMIC_TARGETS 10 +#define HWY_HIGHEST_TARGET_BIT HWY_HIGHEST_TARGET_BIT_X86 +// These must match the order in which the HWY_TARGETS are defined +// starting by the least significant (HWY_HIGHEST_TARGET_BIT + 1 - +// HWY_MAX_DYNAMIC_TARGETS) bit. This list must contain exactly +// HWY_MAX_DYNAMIC_TARGETS elements and does not include SCALAR. The first entry +// corresponds to the best target. Don't include a "," at the end of the list. +#define HWY_CHOOSE_TARGET_LIST(func_name) \ + nullptr, /* reserved */ \ + nullptr, /* reserved */ \ + HWY_CHOOSE_AVX3_DL(func_name), /* AVX3_DL */ \ + HWY_CHOOSE_AVX3(func_name), /* AVX3 */ \ + HWY_CHOOSE_AVX2(func_name), /* AVX2 */ \ + nullptr, /* AVX */ \ + HWY_CHOOSE_SSE4(func_name), /* SSE4 */ \ + HWY_CHOOSE_SSSE3(func_name), /* SSSE3 */ \ + nullptr, /* SSE3 */ \ + nullptr /* SSE2 */ + +#elif HWY_ARCH_ARM +// See HWY_ARCH_X86 above for details. +#define HWY_MAX_DYNAMIC_TARGETS 4 +#define HWY_HIGHEST_TARGET_BIT HWY_HIGHEST_TARGET_BIT_ARM +#define HWY_CHOOSE_TARGET_LIST(func_name) \ + HWY_CHOOSE_SVE2(func_name), /* SVE2 */ \ + HWY_CHOOSE_SVE(func_name), /* SVE */ \ + nullptr, /* reserved */ \ + HWY_CHOOSE_NEON(func_name) /* NEON */ + +#elif HWY_ARCH_PPC +// See HWY_ARCH_X86 above for details. +#define HWY_MAX_DYNAMIC_TARGETS 5 +#define HWY_HIGHEST_TARGET_BIT HWY_HIGHEST_TARGET_BIT_PPC +#define HWY_CHOOSE_TARGET_LIST(func_name) \ + nullptr, /* reserved */ \ + nullptr, /* reserved */ \ + HWY_CHOOSE_PPC8(func_name), /* PPC8 */ \ + nullptr, /* VSX */ \ + nullptr /* AltiVec */ + +#elif HWY_ARCH_WASM +// See HWY_ARCH_X86 above for details. +#define HWY_MAX_DYNAMIC_TARGETS 4 +#define HWY_HIGHEST_TARGET_BIT HWY_HIGHEST_TARGET_BIT_WASM +#define HWY_CHOOSE_TARGET_LIST(func_name) \ + nullptr, /* reserved */ \ + nullptr, /* reserved */ \ + HWY_CHOOSE_WASM2(func_name), /* WASM2 */ \ + HWY_CHOOSE_WASM(func_name) /* WASM */ + +#elif HWY_ARCH_RVV +// See HWY_ARCH_X86 above for details. +#define HWY_MAX_DYNAMIC_TARGETS 4 +#define HWY_HIGHEST_TARGET_BIT HWY_HIGHEST_TARGET_BIT_RVV +#define HWY_CHOOSE_TARGET_LIST(func_name) \ + nullptr, /* reserved */ \ + nullptr, /* reserved */ \ + nullptr, /* reserved */ \ + HWY_CHOOSE_RVV(func_name) /* RVV */ + +#else +// Unknown architecture, will use HWY_SCALAR without dynamic dispatch, though +// still creating single-entry tables in HWY_EXPORT to ensure portability. +#define HWY_MAX_DYNAMIC_TARGETS 1 +#define HWY_HIGHEST_TARGET_BIT HWY_HIGHEST_TARGET_BIT_SCALAR +#endif + +struct ChosenTarget { + public: + // Update the ChosenTarget mask based on the current CPU supported + // targets. + void Update(); + + // Reset the ChosenTarget to the uninitialized state. + void DeInit() { mask_.store(1); } + + // Whether the ChosenTarget was initialized. This is useful to know whether + // any HWY_DYNAMIC_DISPATCH function was called. + bool IsInitialized() const { return mask_.load() != 1; } + + // Return the index in the dynamic dispatch table to be used by the current + // CPU. Note that this method must be in the header file so it uses the value + // of HWY_CHOSEN_TARGET_MASK_TARGETS defined in the translation unit that + // calls it, which may be different from others. This allows to only consider + // those targets that were actually compiled in this module. + size_t HWY_INLINE GetIndex() const { + return hwy::Num0BitsBelowLS1Bit_Nonzero32(mask_.load() & + HWY_CHOSEN_TARGET_MASK_TARGETS); + } + + private: + // Initialized to 1 so GetChosenTargetIndex() returns 0. + std::atomic<uint32_t> mask_{1}; +}; + +extern ChosenTarget chosen_target; + +} // namespace hwy + +#endif // HIGHWAY_HWY_TARGETS_H_ diff --git a/third_party/highway/hwy/targets_test.cc b/third_party/highway/hwy/targets_test.cc new file mode 100644 index 0000000..5e6b443 --- /dev/null +++ b/third_party/highway/hwy/targets_test.cc @@ -0,0 +1,116 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "hwy/targets.h" + +#include "hwy/tests/test_util-inl.h" + +namespace fake { + +#define DECLARE_FUNCTION(TGT) \ + namespace N_##TGT { \ + uint32_t FakeFunction(int) { return HWY_##TGT; } \ + } + +DECLARE_FUNCTION(AVX3_DL) +DECLARE_FUNCTION(AVX3) +DECLARE_FUNCTION(AVX2) +DECLARE_FUNCTION(SSE4) +DECLARE_FUNCTION(SSSE3) +DECLARE_FUNCTION(NEON) +DECLARE_FUNCTION(SVE) +DECLARE_FUNCTION(SVE2) +DECLARE_FUNCTION(PPC8) +DECLARE_FUNCTION(WASM) +DECLARE_FUNCTION(RVV) +DECLARE_FUNCTION(SCALAR) + +HWY_EXPORT(FakeFunction); + +void CheckFakeFunction() { +#define CHECK_ARRAY_ENTRY(TGT) \ + if ((HWY_TARGETS & HWY_##TGT) != 0) { \ + hwy::SetSupportedTargetsForTest(HWY_##TGT); \ + /* Calling Update() first to make &HWY_DYNAMIC_DISPATCH() return */ \ + /* the pointer to the already cached function. */ \ + hwy::chosen_target.Update(); \ + EXPECT_EQ(uint32_t(HWY_##TGT), HWY_DYNAMIC_DISPATCH(FakeFunction)(42)); \ + /* Calling DeInit() will test that the initializer function */ \ + /* also calls the right function. */ \ + hwy::chosen_target.DeInit(); \ + EXPECT_EQ(uint32_t(HWY_##TGT), HWY_DYNAMIC_DISPATCH(FakeFunction)(42)); \ + /* Second call uses the cached value from the previous call. */ \ + EXPECT_EQ(uint32_t(HWY_##TGT), HWY_DYNAMIC_DISPATCH(FakeFunction)(42)); \ + } + CHECK_ARRAY_ENTRY(AVX3_DL) + CHECK_ARRAY_ENTRY(AVX3) + CHECK_ARRAY_ENTRY(AVX2) + CHECK_ARRAY_ENTRY(SSE4) + CHECK_ARRAY_ENTRY(SSSE3) + CHECK_ARRAY_ENTRY(NEON) + CHECK_ARRAY_ENTRY(SVE) + CHECK_ARRAY_ENTRY(SVE2) + CHECK_ARRAY_ENTRY(PPC8) + CHECK_ARRAY_ENTRY(WASM) + CHECK_ARRAY_ENTRY(RVV) + CHECK_ARRAY_ENTRY(SCALAR) +#undef CHECK_ARRAY_ENTRY +} + +} // namespace fake + +namespace hwy { + +class HwyTargetsTest : public testing::Test { + protected: + void TearDown() override { + SetSupportedTargetsForTest(0); + DisableTargets(0); // Reset the mask. + } +}; + +// Test that the order in the HWY_EXPORT static array matches the expected +// value of the target bits. This is only checked for the targets that are +// enabled in the current compilation. +TEST_F(HwyTargetsTest, ChosenTargetOrderTest) { fake::CheckFakeFunction(); } + +TEST_F(HwyTargetsTest, DisabledTargetsTest) { + DisableTargets(~0u); + // Check that the baseline can't be disabled. + HWY_ASSERT(HWY_ENABLED_BASELINE == SupportedTargets()); + + DisableTargets(0); // Reset the mask. + uint32_t current_targets = SupportedTargets(); + if ((current_targets & ~uint32_t(HWY_ENABLED_BASELINE)) == 0) { + // We can't test anything else if the only compiled target is the baseline. + return; + } + // Get the lowest bit in the mask (the best target) and disable that one. + uint32_t lowest_target = current_targets & (~current_targets + 1); + // The lowest target shouldn't be one in the baseline. + HWY_ASSERT((lowest_target & ~uint32_t(HWY_ENABLED_BASELINE)) != 0); + DisableTargets(lowest_target); + + // Check that the other targets are still enabled. + HWY_ASSERT((lowest_target ^ current_targets) == SupportedTargets()); + DisableTargets(0); // Reset the mask. +} + +} // namespace hwy + +// Ought not to be necessary, but without this, no tests run on RVV. +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/third_party/highway/hwy/tests/arithmetic_test.cc b/third_party/highway/hwy/tests/arithmetic_test.cc new file mode 100644 index 0000000..6408acb --- /dev/null +++ b/third_party/highway/hwy/tests/arithmetic_test.cc @@ -0,0 +1,1432 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <inttypes.h> +#include <stddef.h> +#include <stdint.h> + +#include <algorithm> +#include <limits> + +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "tests/arithmetic_test.cc" +#include "hwy/foreach_target.h" +#include "hwy/highway.h" +#include "hwy/tests/test_util-inl.h" + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +struct TestPlusMinus { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v2 = Iota(d, T(2)); + const auto v3 = Iota(d, T(3)); + const auto v4 = Iota(d, T(4)); + + const size_t N = Lanes(d); + auto lanes = AllocateAligned<T>(N); + for (size_t i = 0; i < N; ++i) { + lanes[i] = static_cast<T>((2 + i) + (3 + i)); + } + HWY_ASSERT_VEC_EQ(d, lanes.get(), Add(v2, v3)); + HWY_ASSERT_VEC_EQ(d, Set(d, 2), Sub(v4, v2)); + + for (size_t i = 0; i < N; ++i) { + lanes[i] = static_cast<T>((2 + i) + (4 + i)); + } + auto sum = v2; + sum = Add(sum, v4); // sum == 6,8.. + HWY_ASSERT_VEC_EQ(d, Load(d, lanes.get()), sum); + + sum = Sub(sum, v4); + HWY_ASSERT_VEC_EQ(d, v2, sum); + } +}; + +HWY_NOINLINE void TestAllPlusMinus() { + ForAllTypes(ForPartialVectors<TestPlusMinus>()); +} + +struct TestUnsignedSaturatingArithmetic { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v0 = Zero(d); + const auto vi = Iota(d, 1); + const auto vm = Set(d, LimitsMax<T>()); + + HWY_ASSERT_VEC_EQ(d, Add(v0, v0), SaturatedAdd(v0, v0)); + HWY_ASSERT_VEC_EQ(d, Add(v0, vi), SaturatedAdd(v0, vi)); + HWY_ASSERT_VEC_EQ(d, Add(v0, vm), SaturatedAdd(v0, vm)); + HWY_ASSERT_VEC_EQ(d, vm, SaturatedAdd(vi, vm)); + HWY_ASSERT_VEC_EQ(d, vm, SaturatedAdd(vm, vm)); + + HWY_ASSERT_VEC_EQ(d, v0, SaturatedSub(v0, v0)); + HWY_ASSERT_VEC_EQ(d, v0, SaturatedSub(v0, vi)); + HWY_ASSERT_VEC_EQ(d, v0, SaturatedSub(vi, vi)); + HWY_ASSERT_VEC_EQ(d, v0, SaturatedSub(vi, vm)); + HWY_ASSERT_VEC_EQ(d, Sub(vm, vi), SaturatedSub(vm, vi)); + } +}; + +struct TestSignedSaturatingArithmetic { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v0 = Zero(d); + const auto vpm = Set(d, LimitsMax<T>()); + // Ensure all lanes are positive, even if Iota wraps around + const auto vi = Or(And(Iota(d, 0), vpm), Set(d, 1)); + const auto vn = Sub(v0, vi); + const auto vnm = Set(d, LimitsMin<T>()); + HWY_ASSERT_MASK_EQ(d, MaskTrue(d), Gt(vi, v0)); + HWY_ASSERT_MASK_EQ(d, MaskTrue(d), Lt(vn, v0)); + + HWY_ASSERT_VEC_EQ(d, v0, SaturatedAdd(v0, v0)); + HWY_ASSERT_VEC_EQ(d, vi, SaturatedAdd(v0, vi)); + HWY_ASSERT_VEC_EQ(d, vpm, SaturatedAdd(v0, vpm)); + HWY_ASSERT_VEC_EQ(d, vpm, SaturatedAdd(vi, vpm)); + HWY_ASSERT_VEC_EQ(d, vpm, SaturatedAdd(vpm, vpm)); + + HWY_ASSERT_VEC_EQ(d, v0, SaturatedSub(v0, v0)); + HWY_ASSERT_VEC_EQ(d, Sub(v0, vi), SaturatedSub(v0, vi)); + HWY_ASSERT_VEC_EQ(d, vn, SaturatedSub(vn, v0)); + HWY_ASSERT_VEC_EQ(d, vnm, SaturatedSub(vnm, vi)); + HWY_ASSERT_VEC_EQ(d, vnm, SaturatedSub(vnm, vpm)); + } +}; + +HWY_NOINLINE void TestAllSaturatingArithmetic() { + const ForPartialVectors<TestUnsignedSaturatingArithmetic> test_unsigned; + test_unsigned(uint8_t()); + test_unsigned(uint16_t()); + + const ForPartialVectors<TestSignedSaturatingArithmetic> test_signed; + test_signed(int8_t()); + test_signed(int16_t()); +} + +struct TestAverage { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v0 = Zero(d); + const auto v1 = Set(d, T(1)); + const auto v2 = Set(d, T(2)); + + HWY_ASSERT_VEC_EQ(d, v0, AverageRound(v0, v0)); + HWY_ASSERT_VEC_EQ(d, v1, AverageRound(v0, v1)); + HWY_ASSERT_VEC_EQ(d, v1, AverageRound(v1, v1)); + HWY_ASSERT_VEC_EQ(d, v2, AverageRound(v1, v2)); + HWY_ASSERT_VEC_EQ(d, v2, AverageRound(v2, v2)); + } +}; + +HWY_NOINLINE void TestAllAverage() { + const ForPartialVectors<TestAverage> test; + test(uint8_t()); + test(uint16_t()); +} + +struct TestAbs { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v0 = Zero(d); + const auto vp1 = Set(d, T(1)); + const auto vn1 = Set(d, T(-1)); + const auto vpm = Set(d, LimitsMax<T>()); + const auto vnm = Set(d, LimitsMin<T>()); + + HWY_ASSERT_VEC_EQ(d, v0, Abs(v0)); + HWY_ASSERT_VEC_EQ(d, vp1, Abs(vp1)); + HWY_ASSERT_VEC_EQ(d, vp1, Abs(vn1)); + HWY_ASSERT_VEC_EQ(d, vpm, Abs(vpm)); + HWY_ASSERT_VEC_EQ(d, vnm, Abs(vnm)); + } +}; + +struct TestFloatAbs { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v0 = Zero(d); + const auto vp1 = Set(d, T(1)); + const auto vn1 = Set(d, T(-1)); + const auto vp2 = Set(d, T(0.01)); + const auto vn2 = Set(d, T(-0.01)); + + HWY_ASSERT_VEC_EQ(d, v0, Abs(v0)); + HWY_ASSERT_VEC_EQ(d, vp1, Abs(vp1)); + HWY_ASSERT_VEC_EQ(d, vp1, Abs(vn1)); + HWY_ASSERT_VEC_EQ(d, vp2, Abs(vp2)); + HWY_ASSERT_VEC_EQ(d, vp2, Abs(vn2)); + } +}; + +HWY_NOINLINE void TestAllAbs() { + ForSignedTypes(ForPartialVectors<TestAbs>()); + ForFloatTypes(ForPartialVectors<TestFloatAbs>()); +} + +template <bool kSigned> +struct TestLeftShifts { + template <typename T, class D> + HWY_NOINLINE void operator()(T t, D d) { + if (kSigned) { + // Also test positive values + TestLeftShifts</*kSigned=*/false>()(t, d); + } + + using TI = MakeSigned<T>; + using TU = MakeUnsigned<T>; + const size_t N = Lanes(d); + auto expected = AllocateAligned<T>(N); + + const auto values = Iota(d, kSigned ? -TI(N) : TI(0)); // value to shift + constexpr size_t kMaxShift = (sizeof(T) * 8) - 1; + + // 0 + HWY_ASSERT_VEC_EQ(d, values, ShiftLeft<0>(values)); + HWY_ASSERT_VEC_EQ(d, values, ShiftLeftSame(values, 0)); + + // 1 + for (size_t i = 0; i < N; ++i) { + const T value = kSigned ? T(T(i) - T(N)) : T(i); + expected[i] = T(TU(value) << 1); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftLeft<1>(values)); + HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftLeftSame(values, 1)); + + // max + for (size_t i = 0; i < N; ++i) { + const T value = kSigned ? T(T(i) - T(N)) : T(i); + expected[i] = T(TU(value) << kMaxShift); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftLeft<kMaxShift>(values)); + HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftLeftSame(values, kMaxShift)); + } +}; + +template <bool kSigned> +struct TestVariableLeftShifts { + template <typename T, class D> + HWY_NOINLINE void operator()(T t, D d) { + if (kSigned) { + // Also test positive values + TestVariableLeftShifts</*kSigned=*/false>()(t, d); + } + + using TI = MakeSigned<T>; + using TU = MakeUnsigned<T>; + const size_t N = Lanes(d); + auto expected = AllocateAligned<T>(N); + + const auto v0 = Zero(d); + const auto v1 = Set(d, 1); + const auto values = Iota(d, kSigned ? -TI(N) : TI(0)); // value to shift + + constexpr size_t kMaxShift = (sizeof(T) * 8) - 1; + const auto max_shift = Set(d, kMaxShift); + const auto small_shifts = And(Iota(d, 0), max_shift); + const auto large_shifts = max_shift - small_shifts; + + // Same: 0 + HWY_ASSERT_VEC_EQ(d, values, Shl(values, v0)); + + // Same: 1 + for (size_t i = 0; i < N; ++i) { + const T value = kSigned ? T(i) - T(N) : T(i); + expected[i] = T(TU(value) << 1); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), Shl(values, v1)); + + // Same: max + for (size_t i = 0; i < N; ++i) { + const T value = kSigned ? T(i) - T(N) : T(i); + expected[i] = T(TU(value) << kMaxShift); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), Shl(values, max_shift)); + + // Variable: small + for (size_t i = 0; i < N; ++i) { + const T value = kSigned ? T(i) - T(N) : T(i); + expected[i] = T(TU(value) << (i & kMaxShift)); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), Shl(values, small_shifts)); + + // Variable: large + for (size_t i = 0; i < N; ++i) { + expected[i] = T(TU(1) << (kMaxShift - (i & kMaxShift))); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), Shl(v1, large_shifts)); + } +}; + +struct TestUnsignedRightShifts { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const size_t N = Lanes(d); + auto expected = AllocateAligned<T>(N); + + const auto values = Iota(d, 0); + + const T kMax = LimitsMax<T>(); + constexpr size_t kMaxShift = (sizeof(T) * 8) - 1; + + // Shift by 0 + HWY_ASSERT_VEC_EQ(d, values, ShiftRight<0>(values)); + HWY_ASSERT_VEC_EQ(d, values, ShiftRightSame(values, 0)); + + // Shift by 1 + for (size_t i = 0; i < N; ++i) { + expected[i] = T(T(i & kMax) >> 1); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRight<1>(values)); + HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRightSame(values, 1)); + + // max + for (size_t i = 0; i < N; ++i) { + expected[i] = T(T(i & kMax) >> kMaxShift); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRight<kMaxShift>(values)); + HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRightSame(values, kMaxShift)); + } +}; + +struct TestRotateRight { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const size_t N = Lanes(d); + auto expected = AllocateAligned<T>(N); + + constexpr size_t kBits = sizeof(T) * 8; + const auto mask_shift = Set(d, T{kBits}); + // Cover as many bit positions as possible to test shifting out + const auto values = Shl(Set(d, T{1}), And(Iota(d, 0), mask_shift)); + + // Rotate by 0 + HWY_ASSERT_VEC_EQ(d, values, RotateRight<0>(values)); + + // Rotate by 1 + Store(values, d, expected.get()); + for (size_t i = 0; i < N; ++i) { + expected[i] = (expected[i] >> 1) | (expected[i] << (kBits - 1)); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), RotateRight<1>(values)); + + // Rotate by half + Store(values, d, expected.get()); + for (size_t i = 0; i < N; ++i) { + expected[i] = (expected[i] >> (kBits / 2)) | (expected[i] << (kBits / 2)); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), RotateRight<kBits / 2>(values)); + + // Rotate by max + Store(values, d, expected.get()); + for (size_t i = 0; i < N; ++i) { + expected[i] = (expected[i] >> (kBits - 1)) | (expected[i] << 1); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), RotateRight<kBits - 1>(values)); + } +}; + +struct TestVariableUnsignedRightShifts { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const size_t N = Lanes(d); + auto expected = AllocateAligned<T>(N); + + const auto v0 = Zero(d); + const auto v1 = Set(d, 1); + const auto values = Iota(d, 0); + + const T kMax = LimitsMax<T>(); + const auto max = Set(d, kMax); + + constexpr size_t kMaxShift = (sizeof(T) * 8) - 1; + const auto max_shift = Set(d, kMaxShift); + const auto small_shifts = And(Iota(d, 0), max_shift); + const auto large_shifts = max_shift - small_shifts; + + // Same: 0 + HWY_ASSERT_VEC_EQ(d, values, Shr(values, v0)); + + // Same: 1 + for (size_t i = 0; i < N; ++i) { + expected[i] = T(T(i & kMax) >> 1); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), Shr(values, v1)); + + // Same: max + HWY_ASSERT_VEC_EQ(d, v0, Shr(values, max_shift)); + + // Variable: small + for (size_t i = 0; i < N; ++i) { + expected[i] = T(i) >> (i & kMaxShift); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), Shr(values, small_shifts)); + + // Variable: Large + for (size_t i = 0; i < N; ++i) { + expected[i] = kMax >> (kMaxShift - (i & kMaxShift)); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), Shr(max, large_shifts)); + } +}; + +template <int kAmount, typename T> +T RightShiftNegative(T val) { + // C++ shifts are implementation-defined for negative numbers, and we have + // seen divisions replaced with shifts, so resort to bit operations. + using TU = hwy::MakeUnsigned<T>; + TU bits; + CopyBytes<sizeof(T)>(&val, &bits); + + const TU shifted = TU(bits >> kAmount); + + const TU all = TU(~TU(0)); + const size_t num_zero = sizeof(TU) * 8 - 1 - kAmount; + const TU sign_extended = static_cast<TU>((all << num_zero) & LimitsMax<TU>()); + + bits = shifted | sign_extended; + CopyBytes<sizeof(T)>(&bits, &val); + return val; +} + +class TestSignedRightShifts { + public: + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const size_t N = Lanes(d); + auto expected = AllocateAligned<T>(N); + constexpr T kMin = LimitsMin<T>(); + constexpr T kMax = LimitsMax<T>(); + constexpr size_t kMaxShift = (sizeof(T) * 8) - 1; + + // First test positive values, negative are checked below. + const auto v0 = Zero(d); + const auto values = And(Iota(d, 0), Set(d, kMax)); + + // Shift by 0 + HWY_ASSERT_VEC_EQ(d, values, ShiftRight<0>(values)); + HWY_ASSERT_VEC_EQ(d, values, ShiftRightSame(values, 0)); + + // Shift by 1 + for (size_t i = 0; i < N; ++i) { + expected[i] = T(T(i & kMax) >> 1); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRight<1>(values)); + HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRightSame(values, 1)); + + // max + HWY_ASSERT_VEC_EQ(d, v0, ShiftRight<kMaxShift>(values)); + HWY_ASSERT_VEC_EQ(d, v0, ShiftRightSame(values, kMaxShift)); + + // Even negative value + Test<0>(kMin, d, __LINE__); + Test<1>(kMin, d, __LINE__); + Test<2>(kMin, d, __LINE__); + Test<kMaxShift>(kMin, d, __LINE__); + + const T odd = static_cast<T>(kMin + 1); + Test<0>(odd, d, __LINE__); + Test<1>(odd, d, __LINE__); + Test<2>(odd, d, __LINE__); + Test<kMaxShift>(odd, d, __LINE__); + } + + private: + template <int kAmount, typename T, class D> + void Test(T val, D d, int line) { + const auto expected = Set(d, RightShiftNegative<kAmount>(val)); + const auto in = Set(d, val); + const char* file = __FILE__; + AssertVecEqual(d, expected, ShiftRight<kAmount>(in), file, line); + AssertVecEqual(d, expected, ShiftRightSame(in, kAmount), file, line); + } +}; + +struct TestVariableSignedRightShifts { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + using TU = MakeUnsigned<T>; + const size_t N = Lanes(d); + auto expected = AllocateAligned<T>(N); + + constexpr T kMin = LimitsMin<T>(); + constexpr T kMax = LimitsMax<T>(); + + constexpr size_t kMaxShift = (sizeof(T) * 8) - 1; + + // First test positive values, negative are checked below. + const auto v0 = Zero(d); + const auto positive = Iota(d, 0) & Set(d, kMax); + + // Shift by 0 + HWY_ASSERT_VEC_EQ(d, positive, ShiftRight<0>(positive)); + HWY_ASSERT_VEC_EQ(d, positive, ShiftRightSame(positive, 0)); + + // Shift by 1 + for (size_t i = 0; i < N; ++i) { + expected[i] = T(T(i & kMax) >> 1); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRight<1>(positive)); + HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRightSame(positive, 1)); + + // max + HWY_ASSERT_VEC_EQ(d, v0, ShiftRight<kMaxShift>(positive)); + HWY_ASSERT_VEC_EQ(d, v0, ShiftRightSame(positive, kMaxShift)); + + const auto max_shift = Set(d, kMaxShift); + const auto small_shifts = And(Iota(d, 0), max_shift); + const auto large_shifts = max_shift - small_shifts; + + const auto negative = Iota(d, kMin); + + // Test varying negative to shift + for (size_t i = 0; i < N; ++i) { + expected[i] = RightShiftNegative<1>(static_cast<T>(kMin + i)); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), Shr(negative, Set(d, 1))); + + // Shift MSB right by small amounts + for (size_t i = 0; i < N; ++i) { + const size_t amount = i & kMaxShift; + const TU shifted = ~((1ull << (kMaxShift - amount)) - 1); + CopyBytes<sizeof(T)>(&shifted, &expected[i]); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), Shr(Set(d, kMin), small_shifts)); + + // Shift MSB right by large amounts + for (size_t i = 0; i < N; ++i) { + const size_t amount = kMaxShift - (i & kMaxShift); + const TU shifted = ~((1ull << (kMaxShift - amount)) - 1); + CopyBytes<sizeof(T)>(&shifted, &expected[i]); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), Shr(Set(d, kMin), large_shifts)); + } +}; + +HWY_NOINLINE void TestAllShifts() { + ForUnsignedTypes(ForPartialVectors<TestLeftShifts</*kSigned=*/false>>()); + ForSignedTypes(ForPartialVectors<TestLeftShifts</*kSigned=*/true>>()); + ForUnsignedTypes(ForPartialVectors<TestUnsignedRightShifts>()); + ForSignedTypes(ForPartialVectors<TestSignedRightShifts>()); +} + +HWY_NOINLINE void TestAllVariableShifts() { + const ForPartialVectors<TestLeftShifts</*kSigned=*/false>> shl_u; + const ForPartialVectors<TestLeftShifts</*kSigned=*/true>> shl_s; + const ForPartialVectors<TestUnsignedRightShifts> shr_u; + const ForPartialVectors<TestSignedRightShifts> shr_s; + + shl_u(uint16_t()); + shr_u(uint16_t()); + + shl_u(uint32_t()); + shr_u(uint32_t()); + + shl_s(int16_t()); + shr_s(int16_t()); + + shl_s(int32_t()); + shr_s(int32_t()); + +#if HWY_CAP_INTEGER64 + shl_u(uint64_t()); + shr_u(uint64_t()); + + shl_s(int64_t()); + shr_s(int64_t()); +#endif +} + +HWY_NOINLINE void TestAllRotateRight() { + const ForPartialVectors<TestRotateRight> test; + test(uint32_t()); +#if HWY_CAP_INTEGER64 + test(uint64_t()); +#endif +} + +struct TestUnsignedMinMax { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v0 = Zero(d); + // Leave headroom such that v1 < v2 even after wraparound. + const auto mod = And(Iota(d, 0), Set(d, LimitsMax<T>() >> 1)); + const auto v1 = Add(mod, Set(d, 1)); + const auto v2 = Add(mod, Set(d, 2)); + HWY_ASSERT_VEC_EQ(d, v1, Min(v1, v2)); + HWY_ASSERT_VEC_EQ(d, v2, Max(v1, v2)); + HWY_ASSERT_VEC_EQ(d, v0, Min(v1, v0)); + HWY_ASSERT_VEC_EQ(d, v1, Max(v1, v0)); + + const auto vmin = Set(d, LimitsMin<T>()); + const auto vmax = Set(d, LimitsMax<T>()); + + HWY_ASSERT_VEC_EQ(d, vmin, Min(vmin, vmax)); + HWY_ASSERT_VEC_EQ(d, vmin, Min(vmax, vmin)); + + HWY_ASSERT_VEC_EQ(d, vmax, Max(vmin, vmax)); + HWY_ASSERT_VEC_EQ(d, vmax, Max(vmax, vmin)); + } +}; + +struct TestSignedMinMax { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + // Leave headroom such that v1 < v2 even after wraparound. + const auto mod = And(Iota(d, 0), Set(d, LimitsMax<T>() >> 1)); + const auto v1 = Add(mod, Set(d, 1)); + const auto v2 = Add(mod, Set(d, 2)); + const auto v_neg = Sub(Zero(d), v1); + HWY_ASSERT_VEC_EQ(d, v1, Min(v1, v2)); + HWY_ASSERT_VEC_EQ(d, v2, Max(v1, v2)); + HWY_ASSERT_VEC_EQ(d, v_neg, Min(v1, v_neg)); + HWY_ASSERT_VEC_EQ(d, v1, Max(v1, v_neg)); + + const auto v0 = Zero(d); + const auto vmin = Set(d, LimitsMin<T>()); + const auto vmax = Set(d, LimitsMax<T>()); + HWY_ASSERT_VEC_EQ(d, vmin, Min(v0, vmin)); + HWY_ASSERT_VEC_EQ(d, vmin, Min(vmin, v0)); + HWY_ASSERT_VEC_EQ(d, v0, Max(v0, vmin)); + HWY_ASSERT_VEC_EQ(d, v0, Max(vmin, v0)); + + HWY_ASSERT_VEC_EQ(d, vmin, Min(vmin, vmax)); + HWY_ASSERT_VEC_EQ(d, vmin, Min(vmax, vmin)); + + HWY_ASSERT_VEC_EQ(d, vmax, Max(vmin, vmax)); + HWY_ASSERT_VEC_EQ(d, vmax, Max(vmax, vmin)); + } +}; + +struct TestFloatMinMax { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v1 = Iota(d, 1); + const auto v2 = Iota(d, 2); + const auto v_neg = Iota(d, -T(Lanes(d))); + HWY_ASSERT_VEC_EQ(d, v1, Min(v1, v2)); + HWY_ASSERT_VEC_EQ(d, v2, Max(v1, v2)); + HWY_ASSERT_VEC_EQ(d, v_neg, Min(v1, v_neg)); + HWY_ASSERT_VEC_EQ(d, v1, Max(v1, v_neg)); + + const auto v0 = Zero(d); + const auto vmin = Set(d, T(-1E30)); + const auto vmax = Set(d, T(1E30)); + HWY_ASSERT_VEC_EQ(d, vmin, Min(v0, vmin)); + HWY_ASSERT_VEC_EQ(d, vmin, Min(vmin, v0)); + HWY_ASSERT_VEC_EQ(d, v0, Max(v0, vmin)); + HWY_ASSERT_VEC_EQ(d, v0, Max(vmin, v0)); + + HWY_ASSERT_VEC_EQ(d, vmin, Min(vmin, vmax)); + HWY_ASSERT_VEC_EQ(d, vmin, Min(vmax, vmin)); + + HWY_ASSERT_VEC_EQ(d, vmax, Max(vmin, vmax)); + HWY_ASSERT_VEC_EQ(d, vmax, Max(vmax, vmin)); + } +}; + +HWY_NOINLINE void TestAllMinMax() { + ForUnsignedTypes(ForPartialVectors<TestUnsignedMinMax>()); + ForSignedTypes(ForPartialVectors<TestSignedMinMax>()); + ForFloatTypes(ForPartialVectors<TestFloatMinMax>()); +} + +struct TestUnsignedMul { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v0 = Zero(d); + const auto v1 = Set(d, T(1)); + const auto vi = Iota(d, 1); + const auto vj = Iota(d, 3); + const size_t N = Lanes(d); + auto expected = AllocateAligned<T>(N); + + HWY_ASSERT_VEC_EQ(d, v0, Mul(v0, v0)); + HWY_ASSERT_VEC_EQ(d, v1, Mul(v1, v1)); + HWY_ASSERT_VEC_EQ(d, vi, Mul(v1, vi)); + HWY_ASSERT_VEC_EQ(d, vi, Mul(vi, v1)); + + for (size_t i = 0; i < N; ++i) { + expected[i] = static_cast<T>((1 + i) * (1 + i)); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), Mul(vi, vi)); + + for (size_t i = 0; i < N; ++i) { + expected[i] = static_cast<T>((1 + i) * (3 + i)); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), Mul(vi, vj)); + + const T max = LimitsMax<T>(); + const auto vmax = Set(d, max); + HWY_ASSERT_VEC_EQ(d, vmax, Mul(vmax, v1)); + HWY_ASSERT_VEC_EQ(d, vmax, Mul(v1, vmax)); + + const size_t bits = sizeof(T) * 8; + const uint64_t mask = (1ull << bits) - 1; + const T max2 = (uint64_t(max) * max) & mask; + HWY_ASSERT_VEC_EQ(d, Set(d, max2), Mul(vmax, vmax)); + } +}; + +struct TestSignedMul { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const size_t N = Lanes(d); + auto expected = AllocateAligned<T>(N); + + const auto v0 = Zero(d); + const auto v1 = Set(d, T(1)); + const auto vi = Iota(d, 1); + const auto vn = Iota(d, -T(N)); // no i8 supported, so no wraparound + HWY_ASSERT_VEC_EQ(d, v0, Mul(v0, v0)); + HWY_ASSERT_VEC_EQ(d, v1, Mul(v1, v1)); + HWY_ASSERT_VEC_EQ(d, vi, Mul(v1, vi)); + HWY_ASSERT_VEC_EQ(d, vi, Mul(vi, v1)); + + for (size_t i = 0; i < N; ++i) { + expected[i] = static_cast<T>((1 + i) * (1 + i)); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), Mul(vi, vi)); + + for (size_t i = 0; i < N; ++i) { + expected[i] = static_cast<T>((-T(N) + T(i)) * T(1u + i)); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), Mul(vn, vi)); + HWY_ASSERT_VEC_EQ(d, expected.get(), Mul(vi, vn)); + } +}; + +HWY_NOINLINE void TestAllMul() { + const ForPartialVectors<TestUnsignedMul> test_unsigned; + // No u8. + test_unsigned(uint16_t()); + test_unsigned(uint32_t()); + // No u64. + + const ForPartialVectors<TestSignedMul> test_signed; + // No i8. + test_signed(int16_t()); + test_signed(int32_t()); + // No i64. +} + +struct TestMulHigh { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + using Wide = MakeWide<T>; + const size_t N = Lanes(d); + auto in_lanes = AllocateAligned<T>(N); + auto expected_lanes = AllocateAligned<T>(N); + + const auto vi = Iota(d, 1); + const auto vni = Iota(d, -T(N)); // no i8 supported, so no wraparound + + const auto v0 = Zero(d); + HWY_ASSERT_VEC_EQ(d, v0, MulHigh(v0, v0)); + HWY_ASSERT_VEC_EQ(d, v0, MulHigh(v0, vi)); + HWY_ASSERT_VEC_EQ(d, v0, MulHigh(vi, v0)); + + // Large positive squared + for (size_t i = 0; i < N; ++i) { + in_lanes[i] = T(LimitsMax<T>() >> i); + expected_lanes[i] = T((Wide(in_lanes[i]) * in_lanes[i]) >> 16); + } + auto v = Load(d, in_lanes.get()); + HWY_ASSERT_VEC_EQ(d, expected_lanes.get(), MulHigh(v, v)); + + // Large positive * small positive + for (size_t i = 0; i < N; ++i) { + expected_lanes[i] = T((Wide(in_lanes[i]) * T(1u + i)) >> 16); + } + HWY_ASSERT_VEC_EQ(d, expected_lanes.get(), MulHigh(v, vi)); + HWY_ASSERT_VEC_EQ(d, expected_lanes.get(), MulHigh(vi, v)); + + // Large positive * small negative + for (size_t i = 0; i < N; ++i) { + expected_lanes[i] = T((Wide(in_lanes[i]) * T(i - N)) >> 16); + } + HWY_ASSERT_VEC_EQ(d, expected_lanes.get(), MulHigh(v, vni)); + HWY_ASSERT_VEC_EQ(d, expected_lanes.get(), MulHigh(vni, v)); + } +}; + +HWY_NOINLINE void TestAllMulHigh() { + ForPartialVectors<TestMulHigh> test; + test(int16_t()); + test(uint16_t()); +} + +struct TestMulEven { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + using Wide = MakeWide<T>; + const Repartition<Wide, D> d2; + const auto v0 = Zero(d); + HWY_ASSERT_VEC_EQ(d2, Zero(d2), MulEven(v0, v0)); + + const size_t N = Lanes(d); + auto in_lanes = AllocateAligned<T>(N); + auto expected = AllocateAligned<Wide>(Lanes(d2)); + for (size_t i = 0; i < N; i += 2) { + in_lanes[i + 0] = LimitsMax<T>() >> i; + if (N != 1) { + in_lanes[i + 1] = 1; // unused + } + expected[i / 2] = Wide(in_lanes[i + 0]) * in_lanes[i + 0]; + } + + const auto v = Load(d, in_lanes.get()); + HWY_ASSERT_VEC_EQ(d2, expected.get(), MulEven(v, v)); + } +}; + +struct TestMulEvenOdd64 { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { +#if HWY_TARGET != HWY_SCALAR + const auto v0 = Zero(d); + HWY_ASSERT_VEC_EQ(d, Zero(d), MulEven(v0, v0)); + HWY_ASSERT_VEC_EQ(d, Zero(d), MulOdd(v0, v0)); + + const size_t N = Lanes(d); + if (N == 1) return; + + auto in1 = AllocateAligned<T>(N); + auto in2 = AllocateAligned<T>(N); + auto expected_even = AllocateAligned<T>(N); + auto expected_odd = AllocateAligned<T>(N); + + // Random inputs in each lane + RandomState rng; + for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) { + for (size_t i = 0; i < N; ++i) { + in1[i] = Random64(&rng); + in2[i] = Random64(&rng); + } + + for (size_t i = 0; i < N; i += 2) { + expected_even[i] = Mul128(in1[i], in2[i], &expected_even[i + 1]); + expected_odd[i] = Mul128(in1[i + 1], in2[i + 1], &expected_odd[i + 1]); + } + + const auto a = Load(d, in1.get()); + const auto b = Load(d, in2.get()); + HWY_ASSERT_VEC_EQ(d, expected_even.get(), MulEven(a, b)); + HWY_ASSERT_VEC_EQ(d, expected_odd.get(), MulOdd(a, b)); + } +#else + (void)d; +#endif // HWY_TARGET != HWY_SCALAR + } +}; + +HWY_NOINLINE void TestAllMulEven() { + ForExtendableVectors<TestMulEven> test; + test(int32_t()); + test(uint32_t()); + + ForGE128Vectors<TestMulEvenOdd64>()(uint64_t()); +} + +struct TestMulAdd { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto k0 = Zero(d); + const auto kNeg0 = Set(d, T(-0.0)); + const auto v1 = Iota(d, 1); + const auto v2 = Iota(d, 2); + const size_t N = Lanes(d); + auto expected = AllocateAligned<T>(N); + HWY_ASSERT_VEC_EQ(d, k0, MulAdd(k0, k0, k0)); + HWY_ASSERT_VEC_EQ(d, v2, MulAdd(k0, v1, v2)); + HWY_ASSERT_VEC_EQ(d, v2, MulAdd(v1, k0, v2)); + HWY_ASSERT_VEC_EQ(d, k0, NegMulAdd(k0, k0, k0)); + HWY_ASSERT_VEC_EQ(d, v2, NegMulAdd(k0, v1, v2)); + HWY_ASSERT_VEC_EQ(d, v2, NegMulAdd(v1, k0, v2)); + + for (size_t i = 0; i < N; ++i) { + expected[i] = static_cast<T>((i + 1) * (i + 2)); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), MulAdd(v2, v1, k0)); + HWY_ASSERT_VEC_EQ(d, expected.get(), MulAdd(v1, v2, k0)); + HWY_ASSERT_VEC_EQ(d, expected.get(), NegMulAdd(Neg(v2), v1, k0)); + HWY_ASSERT_VEC_EQ(d, expected.get(), NegMulAdd(v1, Neg(v2), k0)); + + for (size_t i = 0; i < N; ++i) { + expected[i] = static_cast<T>((i + 2) * (i + 2) + (i + 1)); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), MulAdd(v2, v2, v1)); + HWY_ASSERT_VEC_EQ(d, expected.get(), NegMulAdd(Neg(v2), v2, v1)); + + for (size_t i = 0; i < N; ++i) { + expected[i] = + T(-T(i + 2u) * static_cast<T>(i + 2) + static_cast<T>(1 + i)); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), NegMulAdd(v2, v2, v1)); + + HWY_ASSERT_VEC_EQ(d, k0, MulSub(k0, k0, k0)); + HWY_ASSERT_VEC_EQ(d, kNeg0, NegMulSub(k0, k0, k0)); + + for (size_t i = 0; i < N; ++i) { + expected[i] = -T(i + 2); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), MulSub(k0, v1, v2)); + HWY_ASSERT_VEC_EQ(d, expected.get(), MulSub(v1, k0, v2)); + HWY_ASSERT_VEC_EQ(d, expected.get(), NegMulSub(Neg(k0), v1, v2)); + HWY_ASSERT_VEC_EQ(d, expected.get(), NegMulSub(v1, Neg(k0), v2)); + + for (size_t i = 0; i < N; ++i) { + expected[i] = static_cast<T>((i + 1) * (i + 2)); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), MulSub(v1, v2, k0)); + HWY_ASSERT_VEC_EQ(d, expected.get(), MulSub(v2, v1, k0)); + HWY_ASSERT_VEC_EQ(d, expected.get(), NegMulSub(Neg(v1), v2, k0)); + HWY_ASSERT_VEC_EQ(d, expected.get(), NegMulSub(v2, Neg(v1), k0)); + + for (size_t i = 0; i < N; ++i) { + expected[i] = static_cast<T>((i + 2) * (i + 2) - (1 + i)); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), MulSub(v2, v2, v1)); + HWY_ASSERT_VEC_EQ(d, expected.get(), NegMulSub(Neg(v2), v2, v1)); + } +}; + +HWY_NOINLINE void TestAllMulAdd() { + ForFloatTypes(ForPartialVectors<TestMulAdd>()); +} + +struct TestReorderWidenMulAccumulate { + template <typename TN, class DN> + HWY_NOINLINE void operator()(TN /*unused*/, DN dn) { + using TW = MakeWide<TN>; + const RepartitionToWide<DN> dw; + const auto f0 = Zero(dw); + const auto f1 = Set(dw, 1.0f); + const auto fi = Iota(dw, 1); + const auto bf0 = ReorderDemote2To(dn, f0, f0); + const auto bf1 = ReorderDemote2To(dn, f1, f1); + const auto bfi = ReorderDemote2To(dn, fi, fi); + const size_t NW = Lanes(dw); + auto delta = AllocateAligned<TW>(2 * NW); + for (size_t i = 0; i < 2 * NW; ++i) { + delta[i] = 0.0f; + } + + // Any input zero => both outputs zero + auto sum1 = f0; + HWY_ASSERT_VEC_EQ(dw, f0, + ReorderWidenMulAccumulate(dw, bf0, bf0, f0, sum1)); + HWY_ASSERT_VEC_EQ(dw, f0, sum1); + HWY_ASSERT_VEC_EQ(dw, f0, + ReorderWidenMulAccumulate(dw, bf0, bfi, f0, sum1)); + HWY_ASSERT_VEC_EQ(dw, f0, sum1); + HWY_ASSERT_VEC_EQ(dw, f0, + ReorderWidenMulAccumulate(dw, bfi, bf0, f0, sum1)); + HWY_ASSERT_VEC_EQ(dw, f0, sum1); + + // delta[p] := 1.0, all others zero. For each p: Dot(delta, all-ones) == 1. + for (size_t p = 0; p < 2 * NW; ++p) { + delta[p] = 1.0f; + const auto delta0 = Load(dw, delta.get() + 0); + const auto delta1 = Load(dw, delta.get() + NW); + delta[p] = 0.0f; + const auto bf_delta = ReorderDemote2To(dn, delta0, delta1); + + { + sum1 = f0; + const auto sum0 = + ReorderWidenMulAccumulate(dw, bf_delta, bf1, f0, sum1); + HWY_ASSERT_EQ(1.0f, GetLane(SumOfLanes(dw, Add(sum0, sum1)))); + } + // Swapped arg order + { + sum1 = f0; + const auto sum0 = + ReorderWidenMulAccumulate(dw, bf1, bf_delta, f0, sum1); + HWY_ASSERT_EQ(1.0f, GetLane(SumOfLanes(dw, Add(sum0, sum1)))); + } + // Start with nonzero sum0 or sum1 + { + sum1 = delta1; + const auto sum0 = + ReorderWidenMulAccumulate(dw, bf_delta, bf1, delta0, sum1); + HWY_ASSERT_EQ(2.0f, GetLane(SumOfLanes(dw, Add(sum0, sum1)))); + } + // Start with nonzero sum0 or sum1, and swap arg order + { + sum1 = delta1; + const auto sum0 = + ReorderWidenMulAccumulate(dw, bf1, bf_delta, delta0, sum1); + HWY_ASSERT_EQ(2.0f, GetLane(SumOfLanes(dw, Add(sum0, sum1)))); + } + } + } +}; + +HWY_NOINLINE void TestAllReorderWidenMulAccumulate() { + ForShrinkableVectors<TestReorderWidenMulAccumulate>()(bfloat16_t()); +} + +struct TestDiv { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v = Iota(d, T(-2)); + const auto v1 = Set(d, T(1)); + + // Unchanged after division by 1. + HWY_ASSERT_VEC_EQ(d, v, Div(v, v1)); + + const size_t N = Lanes(d); + auto expected = AllocateAligned<T>(N); + for (size_t i = 0; i < N; ++i) { + expected[i] = (T(i) - 2) / T(2); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), Div(v, Set(d, T(2)))); + } +}; + +HWY_NOINLINE void TestAllDiv() { ForFloatTypes(ForPartialVectors<TestDiv>()); } + +struct TestApproximateReciprocal { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v = Iota(d, T(-2)); + const auto nonzero = IfThenElse(Eq(v, Zero(d)), Set(d, T(1)), v); + const size_t N = Lanes(d); + auto input = AllocateAligned<T>(N); + Store(nonzero, d, input.get()); + + auto actual = AllocateAligned<T>(N); + Store(ApproximateReciprocal(nonzero), d, actual.get()); + + double max_l1 = 0.0; + double worst_expected = 0.0; + double worst_actual = 0.0; + for (size_t i = 0; i < N; ++i) { + const double expected = 1.0 / input[i]; + const double l1 = std::abs(expected - actual[i]); + if (l1 > max_l1) { + max_l1 = l1; + worst_expected = expected; + worst_actual = actual[i]; + } + } + const double abs_worst_expected = std::abs(worst_expected); + if (abs_worst_expected > 1E-5) { + const double max_rel = max_l1 / abs_worst_expected; + fprintf(stderr, "max l1 %f rel %f (%f vs %f)\n", max_l1, max_rel, + worst_expected, worst_actual); + HWY_ASSERT(max_rel < 0.004); + } + } +}; + +HWY_NOINLINE void TestAllApproximateReciprocal() { + ForPartialVectors<TestApproximateReciprocal>()(float()); +} + +struct TestSquareRoot { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto vi = Iota(d, 0); + HWY_ASSERT_VEC_EQ(d, vi, Sqrt(Mul(vi, vi))); + } +}; + +HWY_NOINLINE void TestAllSquareRoot() { + ForFloatTypes(ForPartialVectors<TestSquareRoot>()); +} + +struct TestReciprocalSquareRoot { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v = Set(d, 123.0f); + const size_t N = Lanes(d); + auto lanes = AllocateAligned<T>(N); + Store(ApproximateReciprocalSqrt(v), d, lanes.get()); + for (size_t i = 0; i < N; ++i) { + float err = lanes[i] - 0.090166f; + if (err < 0.0f) err = -err; + if (err >= 4E-4f) { + HWY_ABORT("Lane %" PRIu64 "(%" PRIu64 "): actual %f err %f\n", + static_cast<uint64_t>(i), static_cast<uint64_t>(N), lanes[i], + err); + } + } + } +}; + +HWY_NOINLINE void TestAllReciprocalSquareRoot() { + ForPartialVectors<TestReciprocalSquareRoot>()(float()); +} + +template <typename T, class D> +AlignedFreeUniquePtr<T[]> RoundTestCases(T /*unused*/, D d, size_t& padded) { + const T eps = std::numeric_limits<T>::epsilon(); + const T test_cases[] = { + // +/- 1 + T(1), + T(-1), + // +/- 0 + T(0), + T(-0), + // near 0 + T(0.4), + T(-0.4), + // +/- integer + T(4), + T(-32), + // positive near limit + MantissaEnd<T>() - T(1.5), + MantissaEnd<T>() + T(1.5), + // negative near limit + -MantissaEnd<T>() - T(1.5), + -MantissaEnd<T>() + T(1.5), + // positive tiebreak + T(1.5), + T(2.5), + // negative tiebreak + T(-1.5), + T(-2.5), + // positive +/- delta + T(2.0001), + T(3.9999), + // negative +/- delta + T(-999.9999), + T(-998.0001), + // positive +/- epsilon + T(1) + eps, + T(1) - eps, + // negative +/- epsilon + T(-1) + eps, + T(-1) - eps, +#if !defined(HWY_EMULATE_SVE) // these are not safe to just cast to int + // +/- huge (but still fits in float) + T(1E34), + T(-1E35), + // +/- infinity + std::numeric_limits<T>::infinity(), + -std::numeric_limits<T>::infinity(), + // qNaN + GetLane(NaN(d)) +#endif + }; + const size_t kNumTestCases = sizeof(test_cases) / sizeof(test_cases[0]); + const size_t N = Lanes(d); + padded = RoundUpTo(kNumTestCases, N); // allow loading whole vectors + auto in = AllocateAligned<T>(padded); + auto expected = AllocateAligned<T>(padded); + std::copy(test_cases, test_cases + kNumTestCases, in.get()); + std::fill(in.get() + kNumTestCases, in.get() + padded, T(0)); + return in; +} + +struct TestRound { + template <typename T, class D> + HWY_NOINLINE void operator()(T t, D d) { + size_t padded; + auto in = RoundTestCases(t, d, padded); + auto expected = AllocateAligned<T>(padded); + + for (size_t i = 0; i < padded; ++i) { + // Avoid [std::]round, which does not round to nearest *even*. + // NOTE: std:: version from C++11 cmath is not defined in RVV GCC, see + // https://lists.freebsd.org/pipermail/freebsd-current/2014-January/048130.html + expected[i] = static_cast<T>(nearbyint(in[i])); + } + for (size_t i = 0; i < padded; i += Lanes(d)) { + HWY_ASSERT_VEC_EQ(d, &expected[i], Round(Load(d, &in[i]))); + } + } +}; + +HWY_NOINLINE void TestAllRound() { + ForFloatTypes(ForPartialVectors<TestRound>()); +} + +struct TestNearestInt { + template <typename TF, class DF> + HWY_NOINLINE void operator()(TF tf, const DF df) { + using TI = MakeSigned<TF>; + const RebindToSigned<DF> di; + + size_t padded; + auto in = RoundTestCases(tf, df, padded); + auto expected = AllocateAligned<TI>(padded); + + constexpr double max = static_cast<double>(LimitsMax<TI>()); + for (size_t i = 0; i < padded; ++i) { + if (std::isnan(in[i])) { + // We replace NaN with 0 below (no_nan) + expected[i] = 0; + } else if (std::isinf(in[i]) || double(std::abs(in[i])) >= max) { + // Avoid undefined result for lrintf + expected[i] = std::signbit(in[i]) ? LimitsMin<TI>() : LimitsMax<TI>(); + } else { + expected[i] = static_cast<TI>(lrintf(in[i])); + } + } + for (size_t i = 0; i < padded; i += Lanes(df)) { + const auto v = Load(df, &in[i]); + const auto no_nan = IfThenElse(Eq(v, v), v, Zero(df)); + HWY_ASSERT_VEC_EQ(di, &expected[i], NearestInt(no_nan)); + } + } +}; + +HWY_NOINLINE void TestAllNearestInt() { + ForPartialVectors<TestNearestInt>()(float()); +} + +struct TestTrunc { + template <typename T, class D> + HWY_NOINLINE void operator()(T t, D d) { + size_t padded; + auto in = RoundTestCases(t, d, padded); + auto expected = AllocateAligned<T>(padded); + + for (size_t i = 0; i < padded; ++i) { + // NOTE: std:: version from C++11 cmath is not defined in RVV GCC, see + // https://lists.freebsd.org/pipermail/freebsd-current/2014-January/048130.html + expected[i] = static_cast<T>(trunc(in[i])); + } + for (size_t i = 0; i < padded; i += Lanes(d)) { + HWY_ASSERT_VEC_EQ(d, &expected[i], Trunc(Load(d, &in[i]))); + } + } +}; + +HWY_NOINLINE void TestAllTrunc() { + ForFloatTypes(ForPartialVectors<TestTrunc>()); +} + +struct TestCeil { + template <typename T, class D> + HWY_NOINLINE void operator()(T t, D d) { + size_t padded; + auto in = RoundTestCases(t, d, padded); + auto expected = AllocateAligned<T>(padded); + + for (size_t i = 0; i < padded; ++i) { + expected[i] = std::ceil(in[i]); + } + for (size_t i = 0; i < padded; i += Lanes(d)) { + HWY_ASSERT_VEC_EQ(d, &expected[i], Ceil(Load(d, &in[i]))); + } + } +}; + +HWY_NOINLINE void TestAllCeil() { + ForFloatTypes(ForPartialVectors<TestCeil>()); +} + +struct TestFloor { + template <typename T, class D> + HWY_NOINLINE void operator()(T t, D d) { + size_t padded; + auto in = RoundTestCases(t, d, padded); + auto expected = AllocateAligned<T>(padded); + + for (size_t i = 0; i < padded; ++i) { + expected[i] = std::floor(in[i]); + } + for (size_t i = 0; i < padded; i += Lanes(d)) { + HWY_ASSERT_VEC_EQ(d, &expected[i], Floor(Load(d, &in[i]))); + } + } +}; + +HWY_NOINLINE void TestAllFloor() { + ForFloatTypes(ForPartialVectors<TestFloor>()); +} + +struct TestSumOfLanes { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const size_t N = Lanes(d); + auto in_lanes = AllocateAligned<T>(N); + + // Lane i = bit i, higher lanes 0 + double sum = 0.0; + // Avoid setting sign bit and cap at double precision + constexpr size_t kBits = HWY_MIN(sizeof(T) * 8 - 1, 51); + for (size_t i = 0; i < N; ++i) { + in_lanes[i] = i < kBits ? static_cast<T>(1ull << i) : 0; + sum += static_cast<double>(in_lanes[i]); + } + HWY_ASSERT_VEC_EQ(d, Set(d, T(sum)), + SumOfLanes(d, Load(d, in_lanes.get()))); + + // Lane i = i (iota) to include upper lanes + sum = 0.0; + for (size_t i = 0; i < N; ++i) { + sum += static_cast<double>(i); + } + HWY_ASSERT_VEC_EQ(d, Set(d, T(sum)), SumOfLanes(d, Iota(d, 0))); + } +}; + +HWY_NOINLINE void TestAllSumOfLanes() { + ForUIF3264(ForPartialVectors<TestSumOfLanes>()); +} + +struct TestMinOfLanes { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const size_t N = Lanes(d); + auto in_lanes = AllocateAligned<T>(N); + + // Lane i = bit i, higher lanes = 2 (not the minimum) + T min = HighestValue<T>(); + // Avoid setting sign bit and cap at double precision + constexpr size_t kBits = HWY_MIN(sizeof(T) * 8 - 1, 51); + for (size_t i = 0; i < N; ++i) { + in_lanes[i] = i < kBits ? static_cast<T>(1ull << i) : 2; + min = HWY_MIN(min, in_lanes[i]); + } + HWY_ASSERT_VEC_EQ(d, Set(d, min), MinOfLanes(d, Load(d, in_lanes.get()))); + + // Lane i = N - i to include upper lanes + min = HighestValue<T>(); + for (size_t i = 0; i < N; ++i) { + in_lanes[i] = static_cast<T>(N - i); // no 8-bit T so no wraparound + min = HWY_MIN(min, in_lanes[i]); + } + HWY_ASSERT_VEC_EQ(d, Set(d, min), MinOfLanes(d, Load(d, in_lanes.get()))); + } +}; + +struct TestMaxOfLanes { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const size_t N = Lanes(d); + auto in_lanes = AllocateAligned<T>(N); + + T max = LowestValue<T>(); + // Avoid setting sign bit and cap at double precision + constexpr size_t kBits = HWY_MIN(sizeof(T) * 8 - 1, 51); + for (size_t i = 0; i < N; ++i) { + in_lanes[i] = i < kBits ? static_cast<T>(1ull << i) : 0; + max = HWY_MAX(max, in_lanes[i]); + } + HWY_ASSERT_VEC_EQ(d, Set(d, max), MaxOfLanes(d, Load(d, in_lanes.get()))); + + // Lane i = i to include upper lanes + max = LowestValue<T>(); + for (size_t i = 0; i < N; ++i) { + in_lanes[i] = static_cast<T>(i); // no 8-bit T so no wraparound + max = HWY_MAX(max, in_lanes[i]); + } + HWY_ASSERT_VEC_EQ(d, Set(d, max), MaxOfLanes(d, Load(d, in_lanes.get()))); + } +}; + +HWY_NOINLINE void TestAllMinMaxOfLanes() { + const ForPartialVectors<TestMinOfLanes> test_min; + const ForPartialVectors<TestMaxOfLanes> test_max; + ForUIF3264(test_min); + ForUIF3264(test_max); + test_min(uint16_t()); + test_max(uint16_t()); + test_min(int16_t()); + test_max(int16_t()); +} + +struct TestAbsDiff { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const size_t N = Lanes(d); + auto in_lanes_a = AllocateAligned<T>(N); + auto in_lanes_b = AllocateAligned<T>(N); + auto out_lanes = AllocateAligned<T>(N); + for (size_t i = 0; i < N; ++i) { + in_lanes_a[i] = static_cast<T>((i ^ 1u) << i); + in_lanes_b[i] = static_cast<T>(i << i); + out_lanes[i] = std::abs(in_lanes_a[i] - in_lanes_b[i]); + } + const auto a = Load(d, in_lanes_a.get()); + const auto b = Load(d, in_lanes_b.get()); + const auto expected = Load(d, out_lanes.get()); + HWY_ASSERT_VEC_EQ(d, expected, AbsDiff(a, b)); + HWY_ASSERT_VEC_EQ(d, expected, AbsDiff(b, a)); + } +}; + +HWY_NOINLINE void TestAllAbsDiff() { + ForPartialVectors<TestAbsDiff>()(float()); +} + +struct TestNeg { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v0 = Zero(d); + const auto vn = Set(d, T(-3)); + const auto vp = Set(d, T(3)); + HWY_ASSERT_VEC_EQ(d, v0, Neg(v0)); + HWY_ASSERT_VEC_EQ(d, vp, Neg(vn)); + HWY_ASSERT_VEC_EQ(d, vn, Neg(vp)); + } +}; + +HWY_NOINLINE void TestAllNeg() { + ForSignedTypes(ForPartialVectors<TestNeg>()); + ForFloatTypes(ForPartialVectors<TestNeg>()); +} + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE + +namespace hwy { +HWY_BEFORE_TEST(HwyArithmeticTest); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllPlusMinus); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllSaturatingArithmetic); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllShifts); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllVariableShifts); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllRotateRight); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllMinMax); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllAverage); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllAbs); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllMul); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllMulHigh); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllMulEven); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllMulAdd); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllReorderWidenMulAccumulate); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllDiv); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllApproximateReciprocal); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllSquareRoot); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllReciprocalSquareRoot); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllSumOfLanes); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllMinMaxOfLanes); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllRound); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllNearestInt); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllTrunc); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllCeil); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllFloor); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllAbsDiff); +HWY_EXPORT_AND_TEST_P(HwyArithmeticTest, TestAllNeg); +} // namespace hwy + +// Ought not to be necessary, but without this, no tests run on RVV. +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#endif diff --git a/third_party/highway/hwy/tests/blockwise_test.cc b/third_party/highway/hwy/tests/blockwise_test.cc new file mode 100644 index 0000000..eb4e0ee --- /dev/null +++ b/third_party/highway/hwy/tests/blockwise_test.cc @@ -0,0 +1,645 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <stddef.h> +#include <stdint.h> +#include <string.h> + +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "tests/blockwise_test.cc" +#include "hwy/foreach_target.h" +#include "hwy/highway.h" +#include "hwy/tests/test_util-inl.h" + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +struct TestShiftBytes { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + // Scalar does not define Shift*Bytes. +#if HWY_TARGET != HWY_SCALAR || HWY_IDE + const Repartition<uint8_t, D> du8; + const size_t N8 = Lanes(du8); + + // Zero remains zero + const auto v0 = Zero(d); + HWY_ASSERT_VEC_EQ(d, v0, ShiftLeftBytes<1>(v0)); + HWY_ASSERT_VEC_EQ(d, v0, ShiftLeftBytes<1>(d, v0)); + HWY_ASSERT_VEC_EQ(d, v0, ShiftRightBytes<1>(d, v0)); + + // Zero after shifting out the high/low byte + auto bytes = AllocateAligned<uint8_t>(N8); + std::fill(bytes.get(), bytes.get() + N8, 0); + bytes[N8 - 1] = 0x7F; + const auto vhi = BitCast(d, Load(du8, bytes.get())); + bytes[N8 - 1] = 0; + bytes[0] = 0x7F; + const auto vlo = BitCast(d, Load(du8, bytes.get())); + HWY_ASSERT_VEC_EQ(d, v0, ShiftLeftBytes<1>(vhi)); + HWY_ASSERT_VEC_EQ(d, v0, ShiftLeftBytes<1>(d, vhi)); + HWY_ASSERT_VEC_EQ(d, v0, ShiftRightBytes<1>(d, vlo)); + + // Check expected result with Iota + const size_t N = Lanes(d); + auto in = AllocateAligned<T>(N); + const uint8_t* in_bytes = reinterpret_cast<const uint8_t*>(in.get()); + const auto v = BitCast(d, Iota(du8, 1)); + Store(v, d, in.get()); + + auto expected = AllocateAligned<T>(N); + uint8_t* expected_bytes = reinterpret_cast<uint8_t*>(expected.get()); + + const size_t kBlockSize = HWY_MIN(N8, 16); + for (size_t block = 0; block < N8; block += kBlockSize) { + expected_bytes[block] = 0; + memcpy(expected_bytes + block + 1, in_bytes + block, kBlockSize - 1); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftLeftBytes<1>(v)); + HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftLeftBytes<1>(d, v)); + + for (size_t block = 0; block < N8; block += kBlockSize) { + memcpy(expected_bytes + block, in_bytes + block + 1, kBlockSize - 1); + expected_bytes[block + kBlockSize - 1] = 0; + } + HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRightBytes<1>(d, v)); +#else + (void)d; +#endif // #if HWY_TARGET != HWY_SCALAR + } +}; + +HWY_NOINLINE void TestAllShiftBytes() { + ForIntegerTypes(ForPartialVectors<TestShiftBytes>()); +} + +struct TestShiftLanes { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + // Scalar does not define Shift*Lanes. +#if HWY_TARGET != HWY_SCALAR || HWY_IDE + const auto v = Iota(d, T(1)); + const size_t N = Lanes(d); + auto expected = AllocateAligned<T>(N); + + HWY_ASSERT_VEC_EQ(d, v, ShiftLeftLanes<0>(v)); + HWY_ASSERT_VEC_EQ(d, v, ShiftLeftLanes<0>(d, v)); + HWY_ASSERT_VEC_EQ(d, v, ShiftRightLanes<0>(d, v)); + + constexpr size_t kLanesPerBlock = 16 / sizeof(T); + + for (size_t i = 0; i < N; ++i) { + expected[i] = (i % kLanesPerBlock) == 0 ? T(0) : T(i); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftLeftLanes<1>(v)); + HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftLeftLanes<1>(d, v)); + + for (size_t i = 0; i < N; ++i) { + const size_t mod = i % kLanesPerBlock; + expected[i] = mod == (kLanesPerBlock - 1) || i >= N - 1 ? T(0) : T(2 + i); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), ShiftRightLanes<1>(d, v)); +#else + (void)d; +#endif // #if HWY_TARGET != HWY_SCALAR + } +}; + +HWY_NOINLINE void TestAllShiftLanes() { + ForAllTypes(ForPartialVectors<TestShiftLanes>()); +} + +template <typename D, int kLane> +struct TestBroadcastR { + HWY_NOINLINE void operator()() const { + using T = typename D::T; + const D d; + const size_t N = Lanes(d); + if (kLane >= N) return; + auto in_lanes = AllocateAligned<T>(N); + std::fill(in_lanes.get(), in_lanes.get() + N, T(0)); + const size_t blockN = HWY_MIN(N * sizeof(T), 16) / sizeof(T); + // Need to set within each 128-bit block + for (size_t block = 0; block < N; block += blockN) { + in_lanes[block + kLane] = static_cast<T>(block + 1); + } + const auto in = Load(d, in_lanes.get()); + auto expected = AllocateAligned<T>(N); + for (size_t block = 0; block < N; block += blockN) { + for (size_t i = 0; i < blockN; ++i) { + expected[block + i] = T(block + 1); + } + } + HWY_ASSERT_VEC_EQ(d, expected.get(), Broadcast<kLane>(in)); + + TestBroadcastR<D, kLane - 1>()(); + } +}; + +template <class D> +struct TestBroadcastR<D, -1> { + void operator()() const {} +}; + +struct TestBroadcast { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + TestBroadcastR<D, HWY_MIN(MaxLanes(d), 16 / sizeof(T)) - 1>()(); + } +}; + +HWY_NOINLINE void TestAllBroadcast() { + const ForPartialVectors<TestBroadcast> test; + // No u/i8. + test(uint16_t()); + test(int16_t()); + ForUIF3264(test); +} + +template <bool kFull> +struct ChooseTableSize { + template <typename T, typename DIdx> + using type = DIdx; +}; +template <> +struct ChooseTableSize<true> { + template <typename T, typename DIdx> + using type = ScalableTag<T>; +}; + +template <bool kFull> +struct TestTableLookupBytes { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { +#if HWY_TARGET != HWY_SCALAR + RandomState rng; + const typename ChooseTableSize<kFull>::template type<T, D> d_tbl; + const Repartition<uint8_t, decltype(d_tbl)> d_tbl8; + const size_t NT8 = Lanes(d_tbl8); + + const Repartition<uint8_t, D> d8; + const size_t N = Lanes(d); + const size_t N8 = Lanes(d8); + + // Random input bytes + auto in_bytes = AllocateAligned<uint8_t>(NT8); + for (size_t i = 0; i < NT8; ++i) { + in_bytes[i] = Random32(&rng) & 0xFF; + } + const auto in = BitCast(d_tbl, Load(d_tbl8, in_bytes.get())); + + // Enough test data; for larger vectors, upper lanes will be zero. + const uint8_t index_bytes_source[64] = { + // Same index as source, multiple outputs from same input, + // unused input (9), ascending/descending and nonconsecutive neighbors. + 0, 2, 1, 2, 15, 12, 13, 14, 6, 7, 8, 5, 4, 3, 10, 11, + 11, 10, 3, 4, 5, 8, 7, 6, 14, 13, 12, 15, 2, 1, 2, 0, + 4, 3, 2, 2, 5, 6, 7, 7, 15, 15, 15, 15, 15, 15, 0, 1}; + auto index_bytes = AllocateAligned<uint8_t>(N8); + const size_t max_index = HWY_MIN(N8, 16) - 1; + for (size_t i = 0; i < N8; ++i) { + index_bytes[i] = (i < 64) ? index_bytes_source[i] : 0; + // Avoid asan error for partial vectors. + index_bytes[i] = static_cast<uint8_t>(HWY_MIN(index_bytes[i], max_index)); + } + const auto indices = Load(d, reinterpret_cast<const T*>(index_bytes.get())); + + auto expected = AllocateAligned<T>(N); + uint8_t* expected_bytes = reinterpret_cast<uint8_t*>(expected.get()); + + for (size_t block = 0; block < N8; block += 16) { + for (size_t i = 0; i < 16 && (block + i) < N8; ++i) { + const uint8_t index = index_bytes[block + i]; + HWY_ASSERT(block + index < N8); // indices were already capped to N8. + // For large vectors, the lane index may wrap around due to block. + expected_bytes[block + i] = in_bytes[(block & 0xFF) + index]; + } + } + HWY_ASSERT_VEC_EQ(d, expected.get(), TableLookupBytes(in, indices)); + + // Individually test zeroing each byte position. + for (size_t i = 0; i < N8; ++i) { + const uint8_t prev_expected = expected_bytes[i]; + const uint8_t prev_index = index_bytes[i]; + expected_bytes[i] = 0; + + const int idx = 0x80 + (int(Random32(&rng) & 7) << 4); + HWY_ASSERT(0x80 <= idx && idx < 256); + index_bytes[i] = static_cast<uint8_t>(idx); + + const auto indices = + Load(d, reinterpret_cast<const T*>(index_bytes.get())); + HWY_ASSERT_VEC_EQ(d, expected.get(), TableLookupBytesOr0(in, indices)); + expected_bytes[i] = prev_expected; + index_bytes[i] = prev_index; + } +#else + (void)d; +#endif + } +}; + +HWY_NOINLINE void TestAllTableLookupBytes() { + // Partial index, same-sized table. + ForIntegerTypes(ForPartialVectors<TestTableLookupBytes<false>>()); + +// TODO(janwas): requires LMUL trunc/ext, which is not yet implemented. +#if HWY_TARGET != HWY_RVV + // Partial index, full-size table. + ForIntegerTypes(ForPartialVectors<TestTableLookupBytes<true>>()); +#endif +} + +struct TestInterleaveLower { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + using TU = MakeUnsigned<T>; + const size_t N = Lanes(d); + auto even_lanes = AllocateAligned<T>(N); + auto odd_lanes = AllocateAligned<T>(N); + auto expected = AllocateAligned<T>(N); + for (size_t i = 0; i < N; ++i) { + even_lanes[i] = static_cast<T>(2 * i + 0); + odd_lanes[i] = static_cast<T>(2 * i + 1); + } + const auto even = Load(d, even_lanes.get()); + const auto odd = Load(d, odd_lanes.get()); + + const size_t blockN = HWY_MIN(16 / sizeof(T), N); + for (size_t i = 0; i < Lanes(d); ++i) { + const size_t block = i / blockN; + const size_t index = (i % blockN) + block * 2 * blockN; + expected[i] = static_cast<T>(index & LimitsMax<TU>()); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), InterleaveLower(even, odd)); + HWY_ASSERT_VEC_EQ(d, expected.get(), InterleaveLower(d, even, odd)); + } +}; + +struct TestInterleaveUpper { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const size_t N = Lanes(d); + if (N == 1) return; + auto even_lanes = AllocateAligned<T>(N); + auto odd_lanes = AllocateAligned<T>(N); + auto expected = AllocateAligned<T>(N); + for (size_t i = 0; i < N; ++i) { + even_lanes[i] = static_cast<T>(2 * i + 0); + odd_lanes[i] = static_cast<T>(2 * i + 1); + } + const auto even = Load(d, even_lanes.get()); + const auto odd = Load(d, odd_lanes.get()); + + const size_t blockN = HWY_MIN(16 / sizeof(T), N); + for (size_t i = 0; i < Lanes(d); ++i) { + const size_t block = i / blockN; + expected[i] = T((i % blockN) + block * 2 * blockN + blockN); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), InterleaveUpper(d, even, odd)); + } +}; + +HWY_NOINLINE void TestAllInterleave() { + // Not DemoteVectors because this cannot be supported by HWY_SCALAR. + ForAllTypes(ForShrinkableVectors<TestInterleaveLower>()); + ForAllTypes(ForShrinkableVectors<TestInterleaveUpper>()); +} + +struct TestZipLower { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + using WideT = MakeWide<T>; + static_assert(sizeof(T) * 2 == sizeof(WideT), "Must be double-width"); + static_assert(IsSigned<T>() == IsSigned<WideT>(), "Must have same sign"); + const size_t N = Lanes(d); + auto even_lanes = AllocateAligned<T>(N); + auto odd_lanes = AllocateAligned<T>(N); + for (size_t i = 0; i < N; ++i) { + even_lanes[i] = static_cast<T>(2 * i + 0); + odd_lanes[i] = static_cast<T>(2 * i + 1); + } + const auto even = Load(d, even_lanes.get()); + const auto odd = Load(d, odd_lanes.get()); + + const Repartition<WideT, D> dw; + const size_t NW = Lanes(dw); + auto expected = AllocateAligned<WideT>(NW); + const size_t blockN = HWY_MIN(size_t(16) / sizeof(WideT), NW); + + for (size_t i = 0; i < NW; ++i) { + const size_t block = i / blockN; + // Value of least-significant lane in lo-vector. + const size_t lo = 2u * (i % blockN) + 4u * block * blockN; + const size_t kBits = sizeof(T) * 8; + expected[i] = static_cast<WideT>((static_cast<WideT>(lo + 1) << kBits) + + static_cast<WideT>(lo)); + } + HWY_ASSERT_VEC_EQ(dw, expected.get(), ZipLower(even, odd)); + HWY_ASSERT_VEC_EQ(dw, expected.get(), ZipLower(dw, even, odd)); + } +}; + +struct TestZipUpper { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + using WideT = MakeWide<T>; + static_assert(sizeof(T) * 2 == sizeof(WideT), "Must be double-width"); + static_assert(IsSigned<T>() == IsSigned<WideT>(), "Must have same sign"); + const size_t N = Lanes(d); + if (N < 16 / sizeof(T)) return; + auto even_lanes = AllocateAligned<T>(N); + auto odd_lanes = AllocateAligned<T>(N); + for (size_t i = 0; i < Lanes(d); ++i) { + even_lanes[i] = static_cast<T>(2 * i + 0); + odd_lanes[i] = static_cast<T>(2 * i + 1); + } + const auto even = Load(d, even_lanes.get()); + const auto odd = Load(d, odd_lanes.get()); + + const Repartition<WideT, D> dw; + const size_t NW = Lanes(dw); + auto expected = AllocateAligned<WideT>(NW); + const size_t blockN = HWY_MIN(size_t(16) / sizeof(WideT), NW); + + for (size_t i = 0; i < NW; ++i) { + const size_t block = i / blockN; + const size_t lo = 2u * (i % blockN) + 4u * block * blockN; + const size_t kBits = sizeof(T) * 8; + expected[i] = static_cast<WideT>( + (static_cast<WideT>(lo + 2 * blockN + 1) << kBits) + + static_cast<WideT>(lo + 2 * blockN)); + } + HWY_ASSERT_VEC_EQ(dw, expected.get(), ZipUpper(dw, even, odd)); + } +}; + +HWY_NOINLINE void TestAllZip() { + const ForDemoteVectors<TestZipLower> lower_unsigned; + // TODO(janwas): enable after LowerHalf available +#if HWY_TARGET != HWY_RVV + lower_unsigned(uint8_t()); +#endif + lower_unsigned(uint16_t()); +#if HWY_CAP_INTEGER64 + lower_unsigned(uint32_t()); // generates u64 +#endif + + const ForDemoteVectors<TestZipLower> lower_signed; +#if HWY_TARGET != HWY_RVV + lower_signed(int8_t()); +#endif + lower_signed(int16_t()); +#if HWY_CAP_INTEGER64 + lower_signed(int32_t()); // generates i64 +#endif + + const ForShrinkableVectors<TestZipUpper> upper_unsigned; +#if HWY_TARGET != HWY_RVV + upper_unsigned(uint8_t()); +#endif + upper_unsigned(uint16_t()); +#if HWY_CAP_INTEGER64 + upper_unsigned(uint32_t()); // generates u64 +#endif + + const ForShrinkableVectors<TestZipUpper> upper_signed; +#if HWY_TARGET != HWY_RVV + upper_signed(int8_t()); +#endif + upper_signed(int16_t()); +#if HWY_CAP_INTEGER64 + upper_signed(int32_t()); // generates i64 +#endif + + // No float - concatenating f32 does not result in a f64 +} + +template <int kBytes> +struct TestCombineShiftRightBytesR { + template <class T, class D> + HWY_NOINLINE void operator()(T t, D d) { +// Scalar does not define CombineShiftRightBytes. +#if HWY_TARGET != HWY_SCALAR || HWY_IDE + const size_t kBlockSize = 16; + static_assert(kBytes < kBlockSize, "Shift count is per block"); + const Repartition<uint8_t, D> d8; + const size_t N8 = Lanes(d8); + if (N8 < 16) return; + auto hi_bytes = AllocateAligned<uint8_t>(N8); + auto lo_bytes = AllocateAligned<uint8_t>(N8); + auto expected_bytes = AllocateAligned<uint8_t>(N8); + uint8_t combined[2 * kBlockSize]; + + // Random inputs in each lane + RandomState rng; + for (size_t rep = 0; rep < AdjustedReps(100); ++rep) { + for (size_t i = 0; i < N8; ++i) { + hi_bytes[i] = static_cast<uint8_t>(Random64(&rng) & 0xFF); + lo_bytes[i] = static_cast<uint8_t>(Random64(&rng) & 0xFF); + } + for (size_t i = 0; i < N8; i += kBlockSize) { + CopyBytes<kBlockSize>(&lo_bytes[i], combined); + CopyBytes<kBlockSize>(&hi_bytes[i], combined + kBlockSize); + CopyBytes<kBlockSize>(combined + kBytes, &expected_bytes[i]); + } + + const auto hi = BitCast(d, Load(d8, hi_bytes.get())); + const auto lo = BitCast(d, Load(d8, lo_bytes.get())); + const auto expected = BitCast(d, Load(d8, expected_bytes.get())); + HWY_ASSERT_VEC_EQ(d, expected, CombineShiftRightBytes<kBytes>(d, hi, lo)); + } + + TestCombineShiftRightBytesR<kBytes - 1>()(t, d); +#else + (void)t; + (void)d; +#endif // #if HWY_TARGET != HWY_SCALAR + } +}; + +template <int kLanes> +struct TestCombineShiftRightLanesR { + template <class T, class D> + HWY_NOINLINE void operator()(T t, D d) { +// Scalar does not define CombineShiftRightBytes (needed for *Lanes). +#if HWY_TARGET != HWY_SCALAR || HWY_IDE + const Repartition<uint8_t, D> d8; + const size_t N8 = Lanes(d8); + if (N8 < 16) return; + + auto hi_bytes = AllocateAligned<uint8_t>(N8); + auto lo_bytes = AllocateAligned<uint8_t>(N8); + auto expected_bytes = AllocateAligned<uint8_t>(N8); + const size_t kBlockSize = 16; + uint8_t combined[2 * kBlockSize]; + + // Random inputs in each lane + RandomState rng; + for (size_t rep = 0; rep < AdjustedReps(100); ++rep) { + for (size_t i = 0; i < N8; ++i) { + hi_bytes[i] = static_cast<uint8_t>(Random64(&rng) & 0xFF); + lo_bytes[i] = static_cast<uint8_t>(Random64(&rng) & 0xFF); + } + for (size_t i = 0; i < N8; i += kBlockSize) { + CopyBytes<kBlockSize>(&lo_bytes[i], combined); + CopyBytes<kBlockSize>(&hi_bytes[i], combined + kBlockSize); + CopyBytes<kBlockSize>(combined + kLanes * sizeof(T), + &expected_bytes[i]); + } + + const auto hi = BitCast(d, Load(d8, hi_bytes.get())); + const auto lo = BitCast(d, Load(d8, lo_bytes.get())); + const auto expected = BitCast(d, Load(d8, expected_bytes.get())); + HWY_ASSERT_VEC_EQ(d, expected, CombineShiftRightLanes<kLanes>(d, hi, lo)); + } + + TestCombineShiftRightLanesR<kLanes - 1>()(t, d); +#else + (void)t; + (void)d; +#endif // #if HWY_TARGET != HWY_SCALAR + } +}; + +template <> +struct TestCombineShiftRightBytesR<0> { + template <class T, class D> + void operator()(T /*unused*/, D /*unused*/) {} +}; + +template <> +struct TestCombineShiftRightLanesR<0> { + template <class T, class D> + void operator()(T /*unused*/, D /*unused*/) {} +}; + +struct TestCombineShiftRight { + template <class T, class D> + HWY_NOINLINE void operator()(T t, D d) { + constexpr int kMaxBytes = HWY_MIN(16, int(MaxLanes(d) * sizeof(T))); + TestCombineShiftRightBytesR<kMaxBytes - 1>()(t, d); + TestCombineShiftRightLanesR<kMaxBytes / int(sizeof(T)) - 1>()(t, d); + } +}; + +HWY_NOINLINE void TestAllCombineShiftRight() { + // Need at least 2 lanes. + ForAllTypes(ForShrinkableVectors<TestCombineShiftRight>()); +} + +class TestSpecialShuffle32 { + public: + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v = Iota(d, 0); + VerifyLanes32(d, Shuffle2301(v), 2, 3, 0, 1, __FILE__, __LINE__); + VerifyLanes32(d, Shuffle1032(v), 1, 0, 3, 2, __FILE__, __LINE__); + VerifyLanes32(d, Shuffle0321(v), 0, 3, 2, 1, __FILE__, __LINE__); + VerifyLanes32(d, Shuffle2103(v), 2, 1, 0, 3, __FILE__, __LINE__); + VerifyLanes32(d, Shuffle0123(v), 0, 1, 2, 3, __FILE__, __LINE__); + } + + private: + template <class D, class V> + HWY_NOINLINE void VerifyLanes32(D d, VecArg<V> actual, const size_t i3, + const size_t i2, const size_t i1, + const size_t i0, const char* filename, + const int line) { + using T = TFromD<D>; + constexpr size_t kBlockN = 16 / sizeof(T); + const size_t N = Lanes(d); + if (N < 4) return; + auto expected = AllocateAligned<T>(N); + for (size_t block = 0; block < N; block += kBlockN) { + expected[block + 3] = static_cast<T>(block + i3); + expected[block + 2] = static_cast<T>(block + i2); + expected[block + 1] = static_cast<T>(block + i1); + expected[block + 0] = static_cast<T>(block + i0); + } + AssertVecEqual(d, expected.get(), actual, filename, line); + } +}; + +class TestSpecialShuffle64 { + public: + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v = Iota(d, 0); + VerifyLanes64(d, Shuffle01(v), 0, 1, __FILE__, __LINE__); + } + + private: + template <class D, class V> + HWY_NOINLINE void VerifyLanes64(D d, VecArg<V> actual, const size_t i1, + const size_t i0, const char* filename, + const int line) { + using T = TFromD<D>; + constexpr size_t kBlockN = 16 / sizeof(T); + const size_t N = Lanes(d); + if (N < 2) return; + auto expected = AllocateAligned<T>(N); + for (size_t block = 0; block < N; block += kBlockN) { + expected[block + 1] = static_cast<T>(block + i1); + expected[block + 0] = static_cast<T>(block + i0); + } + AssertVecEqual(d, expected.get(), actual, filename, line); + } +}; + +HWY_NOINLINE void TestAllSpecialShuffles() { + const ForGE128Vectors<TestSpecialShuffle32> test32; + test32(uint32_t()); + test32(int32_t()); + test32(float()); + +#if HWY_CAP_INTEGER64 + const ForGE128Vectors<TestSpecialShuffle64> test64; + test64(uint64_t()); + test64(int64_t()); +#endif + +#if HWY_CAP_FLOAT64 + const ForGE128Vectors<TestSpecialShuffle64> test_d; + test_d(double()); +#endif +} + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE + +namespace hwy { +HWY_BEFORE_TEST(HwyBlockwiseTest); +HWY_EXPORT_AND_TEST_P(HwyBlockwiseTest, TestAllShiftBytes); +HWY_EXPORT_AND_TEST_P(HwyBlockwiseTest, TestAllShiftLanes); +HWY_EXPORT_AND_TEST_P(HwyBlockwiseTest, TestAllBroadcast); +HWY_EXPORT_AND_TEST_P(HwyBlockwiseTest, TestAllTableLookupBytes); +HWY_EXPORT_AND_TEST_P(HwyBlockwiseTest, TestAllInterleave); +HWY_EXPORT_AND_TEST_P(HwyBlockwiseTest, TestAllZip); +HWY_EXPORT_AND_TEST_P(HwyBlockwiseTest, TestAllCombineShiftRight); +HWY_EXPORT_AND_TEST_P(HwyBlockwiseTest, TestAllSpecialShuffles); +} // namespace hwy + +// Ought not to be necessary, but without this, no tests run on RVV. +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#endif diff --git a/third_party/highway/hwy/tests/combine_test.cc b/third_party/highway/hwy/tests/combine_test.cc new file mode 100644 index 0000000..ba37f39 --- /dev/null +++ b/third_party/highway/hwy/tests/combine_test.cc @@ -0,0 +1,278 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <stddef.h> +#include <stdint.h> + +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "tests/combine_test.cc" +#include "hwy/foreach_target.h" + +#include "hwy/highway.h" +#include "hwy/tests/test_util-inl.h" + +// Not yet implemented +#if HWY_TARGET != HWY_RVV + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +struct TestLowerHalf { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const Half<D> d2; + + const size_t N = Lanes(d); + auto lanes = AllocateAligned<T>(N); + auto lanes2 = AllocateAligned<T>(N); + std::fill(lanes.get(), lanes.get() + N, T(0)); + std::fill(lanes2.get(), lanes2.get() + N, T(0)); + const auto v = Iota(d, 1); + Store(LowerHalf(d2, v), d2, lanes.get()); + Store(LowerHalf(v), d2, lanes2.get()); // optionally without D + size_t i = 0; + for (; i < Lanes(d2); ++i) { + HWY_ASSERT_EQ(T(1 + i), lanes[i]); + HWY_ASSERT_EQ(T(1 + i), lanes2[i]); + } + // Other half remains unchanged + for (; i < N; ++i) { + HWY_ASSERT_EQ(T(0), lanes[i]); + HWY_ASSERT_EQ(T(0), lanes2[i]); + } + } +}; + +struct TestLowerQuarter { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const Half<D> d2; + const Half<decltype(d2)> d4; + + const size_t N = Lanes(d); + auto lanes = AllocateAligned<T>(N); + auto lanes2 = AllocateAligned<T>(N); + std::fill(lanes.get(), lanes.get() + N, T(0)); + std::fill(lanes2.get(), lanes2.get() + N, T(0)); + const auto v = Iota(d, 1); + const auto lo = LowerHalf(d4, LowerHalf(d2, v)); + const auto lo2 = LowerHalf(LowerHalf(v)); // optionally without D + Store(lo, d4, lanes.get()); + Store(lo2, d4, lanes2.get()); + size_t i = 0; + for (; i < Lanes(d4); ++i) { + HWY_ASSERT_EQ(T(i + 1), lanes[i]); + HWY_ASSERT_EQ(T(i + 1), lanes2[i]); + } + // Upper 3/4 remain unchanged + for (; i < N; ++i) { + HWY_ASSERT_EQ(T(0), lanes[i]); + HWY_ASSERT_EQ(T(0), lanes2[i]); + } + } +}; + +HWY_NOINLINE void TestAllLowerHalf() { + ForAllTypes(ForDemoteVectors<TestLowerHalf>()); + ForAllTypes(ForDemoteVectors<TestLowerQuarter, 4>()); +} + +struct TestUpperHalf { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + // Scalar does not define UpperHalf. +#if HWY_TARGET != HWY_SCALAR + const Half<D> d2; + + const auto v = Iota(d, 1); + const size_t N = Lanes(d); + auto lanes = AllocateAligned<T>(N); + std::fill(lanes.get(), lanes.get() + N, T(0)); + + Store(UpperHalf(d2, v), d2, lanes.get()); + size_t i = 0; + for (; i < Lanes(d2); ++i) { + HWY_ASSERT_EQ(T(Lanes(d2) + 1 + i), lanes[i]); + } + // Other half remains unchanged + for (; i < N; ++i) { + HWY_ASSERT_EQ(T(0), lanes[i]); + } +#else + (void)d; +#endif + } +}; + +HWY_NOINLINE void TestAllUpperHalf() { + ForAllTypes(ForShrinkableVectors<TestUpperHalf>()); +} + +struct TestZeroExtendVector { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const Twice<D> d2; + + const auto v = Iota(d, 1); + const size_t N2 = Lanes(d2); + auto lanes = AllocateAligned<T>(N2); + Store(v, d, &lanes[0]); + Store(v, d, &lanes[N2 / 2]); + + const auto ext = ZeroExtendVector(d2, v); + Store(ext, d2, lanes.get()); + + size_t i = 0; + // Lower half is unchanged + for (; i < N2 / 2; ++i) { + HWY_ASSERT_EQ(T(1 + i), lanes[i]); + } + // Upper half is zero + for (; i < N2; ++i) { + HWY_ASSERT_EQ(T(0), lanes[i]); + } + } +}; + +HWY_NOINLINE void TestAllZeroExtendVector() { + ForAllTypes(ForExtendableVectors<TestZeroExtendVector>()); +} + +struct TestCombine { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const Twice<D> d2; + const size_t N2 = Lanes(d2); + auto lanes = AllocateAligned<T>(N2); + + const auto lo = Iota(d, 1); + const auto hi = Iota(d, N2 / 2 + 1); + const auto combined = Combine(d2, hi, lo); + Store(combined, d2, lanes.get()); + + const auto expected = Iota(d2, 1); + HWY_ASSERT_VEC_EQ(d2, expected, combined); + } +}; + +HWY_NOINLINE void TestAllCombine() { + ForAllTypes(ForExtendableVectors<TestCombine>()); +} + +struct TestConcat { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const size_t N = Lanes(d); + if (N == 1) return; + const size_t half_bytes = N * sizeof(T) / 2; + + auto hi = AllocateAligned<T>(N); + auto lo = AllocateAligned<T>(N); + auto expected = AllocateAligned<T>(N); + RandomState rng; + for (size_t rep = 0; rep < 10; ++rep) { + for (size_t i = 0; i < N; ++i) { + hi[i] = static_cast<T>(Random64(&rng) & 0xFF); + lo[i] = static_cast<T>(Random64(&rng) & 0xFF); + } + + { + memcpy(&expected[N / 2], &hi[N / 2], half_bytes); + memcpy(&expected[0], &lo[0], half_bytes); + const auto vhi = Load(d, hi.get()); + const auto vlo = Load(d, lo.get()); + HWY_ASSERT_VEC_EQ(d, expected.get(), ConcatUpperLower(d, vhi, vlo)); + } + + { + memcpy(&expected[N / 2], &hi[N / 2], half_bytes); + memcpy(&expected[0], &lo[N / 2], half_bytes); + const auto vhi = Load(d, hi.get()); + const auto vlo = Load(d, lo.get()); + HWY_ASSERT_VEC_EQ(d, expected.get(), ConcatUpperUpper(d, vhi, vlo)); + } + + { + memcpy(&expected[N / 2], &hi[0], half_bytes); + memcpy(&expected[0], &lo[N / 2], half_bytes); + const auto vhi = Load(d, hi.get()); + const auto vlo = Load(d, lo.get()); + HWY_ASSERT_VEC_EQ(d, expected.get(), ConcatLowerUpper(d, vhi, vlo)); + } + + { + memcpy(&expected[N / 2], &hi[0], half_bytes); + memcpy(&expected[0], &lo[0], half_bytes); + const auto vhi = Load(d, hi.get()); + const auto vlo = Load(d, lo.get()); + HWY_ASSERT_VEC_EQ(d, expected.get(), ConcatLowerLower(d, vhi, vlo)); + } + } + } +}; + +HWY_NOINLINE void TestAllConcat() { + ForAllTypes(ForShrinkableVectors<TestConcat>()); +} + +struct TestConcatOddEven { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { +#if HWY_TARGET != HWY_RVV && HWY_TARGET != HWY_SCALAR + const size_t N = Lanes(d); + const auto hi = Iota(d, N); + const auto lo = Iota(d, 0); + const auto even = Add(Iota(d, 0), Iota(d, 0)); + const auto odd = Add(even, Set(d, 1)); + HWY_ASSERT_VEC_EQ(d, odd, ConcatOdd(d, hi, lo)); + HWY_ASSERT_VEC_EQ(d, even, ConcatEven(d, hi, lo)); +#else + (void)d; +#endif + } +}; + +HWY_NOINLINE void TestAllConcatOddEven() { + ForUIF3264(ForShrinkableVectors<TestConcatOddEven>()); +} + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE + +namespace hwy { +HWY_BEFORE_TEST(HwyCombineTest); +HWY_EXPORT_AND_TEST_P(HwyCombineTest, TestAllLowerHalf); +HWY_EXPORT_AND_TEST_P(HwyCombineTest, TestAllUpperHalf); +HWY_EXPORT_AND_TEST_P(HwyCombineTest, TestAllZeroExtendVector); +HWY_EXPORT_AND_TEST_P(HwyCombineTest, TestAllCombine); +HWY_EXPORT_AND_TEST_P(HwyCombineTest, TestAllConcat); +HWY_EXPORT_AND_TEST_P(HwyCombineTest, TestAllConcatOddEven); +} // namespace hwy + +// Ought not to be necessary, but without this, no tests run on RVV. +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#endif // HWY_ONCE + +#else +int main(int, char**) { return 0; } +#endif // HWY_TARGET != HWY_RVV diff --git a/third_party/highway/hwy/tests/compare_test.cc b/third_party/highway/hwy/tests/compare_test.cc new file mode 100644 index 0000000..85cc802 --- /dev/null +++ b/third_party/highway/hwy/tests/compare_test.cc @@ -0,0 +1,243 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <stddef.h> +#include <stdint.h> +#include <string.h> // memset + +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "tests/compare_test.cc" +#include "hwy/foreach_target.h" +#include "hwy/highway.h" +#include "hwy/tests/test_util-inl.h" + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +// All types. +struct TestEquality { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v2 = Iota(d, 2); + const auto v2b = Iota(d, 2); + const auto v3 = Iota(d, 3); + + const auto mask_false = MaskFalse(d); + const auto mask_true = MaskTrue(d); + + HWY_ASSERT_MASK_EQ(d, mask_false, Eq(v2, v3)); + HWY_ASSERT_MASK_EQ(d, mask_false, Eq(v3, v2)); + HWY_ASSERT_MASK_EQ(d, mask_true, Eq(v2, v2)); + HWY_ASSERT_MASK_EQ(d, mask_true, Eq(v2, v2b)); + + HWY_ASSERT_MASK_EQ(d, mask_true, Ne(v2, v3)); + HWY_ASSERT_MASK_EQ(d, mask_true, Ne(v3, v2)); + HWY_ASSERT_MASK_EQ(d, mask_false, Ne(v2, v2)); + HWY_ASSERT_MASK_EQ(d, mask_false, Ne(v2, v2b)); + } +}; + +HWY_NOINLINE void TestAllEquality() { + ForAllTypes(ForPartialVectors<TestEquality>()); +} + +// a > b should be true, verify that for Gt/Lt and with swapped args. +template <class D> +void EnsureGreater(D d, TFromD<D> a, TFromD<D> b, const char* file, int line) { + const auto mask_false = MaskFalse(d); + const auto mask_true = MaskTrue(d); + + const auto va = Set(d, a); + const auto vb = Set(d, b); + AssertMaskEqual(d, mask_true, Gt(va, vb), file, line); + AssertMaskEqual(d, mask_false, Lt(va, vb), file, line); + + // Swapped order + AssertMaskEqual(d, mask_false, Gt(vb, va), file, line); + AssertMaskEqual(d, mask_true, Lt(vb, va), file, line); + + // Also ensure irreflexive + AssertMaskEqual(d, mask_false, Gt(va, va), file, line); + AssertMaskEqual(d, mask_false, Gt(vb, vb), file, line); + AssertMaskEqual(d, mask_false, Lt(va, va), file, line); + AssertMaskEqual(d, mask_false, Lt(vb, vb), file, line); +} + +#define HWY_ENSURE_GREATER(d, a, b) EnsureGreater(d, a, b, __FILE__, __LINE__) + +struct TestStrictUnsigned { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const T max = LimitsMax<T>(); + const auto v0 = Zero(d); + const auto v2 = And(Iota(d, T(2)), Set(d, 255)); // 0..255 + + const auto mask_false = MaskFalse(d); + + // Individual values of interest + HWY_ENSURE_GREATER(d, 2, 1); + HWY_ENSURE_GREATER(d, 1, 0); + HWY_ENSURE_GREATER(d, 128, 127); + HWY_ENSURE_GREATER(d, max, max / 2); + HWY_ENSURE_GREATER(d, max, 1); + HWY_ENSURE_GREATER(d, max, 0); + + // Also use Iota to ensure lanes are independent + HWY_ASSERT_MASK_EQ(d, mask_false, Lt(v2, v0)); + HWY_ASSERT_MASK_EQ(d, mask_false, Gt(v0, v2)); + HWY_ASSERT_MASK_EQ(d, mask_false, Lt(v0, v0)); + HWY_ASSERT_MASK_EQ(d, mask_false, Gt(v0, v0)); + HWY_ASSERT_MASK_EQ(d, mask_false, Lt(v2, v2)); + HWY_ASSERT_MASK_EQ(d, mask_false, Gt(v2, v2)); + } +}; + +HWY_NOINLINE void TestAllStrictUnsigned() { + ForUnsignedTypes(ForPartialVectors<TestStrictUnsigned>()); +} + +struct TestStrictInt { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const T min = LimitsMin<T>(); + const T max = LimitsMax<T>(); + const auto v0 = Zero(d); + const auto v2 = And(Iota(d, T(2)), Set(d, 127)); // 0..127 + const auto vn = Sub(Neg(v2), Set(d, 1)); // -1..-128 + + const auto mask_false = MaskFalse(d); + const auto mask_true = MaskTrue(d); + + // Individual values of interest + HWY_ENSURE_GREATER(d, 2, 1); + HWY_ENSURE_GREATER(d, 1, 0); + HWY_ENSURE_GREATER(d, 0, -1); + HWY_ENSURE_GREATER(d, -1, -2); + HWY_ENSURE_GREATER(d, max, max / 2); + HWY_ENSURE_GREATER(d, max, 1); + HWY_ENSURE_GREATER(d, max, 0); + HWY_ENSURE_GREATER(d, max, -1); + HWY_ENSURE_GREATER(d, max, min); + HWY_ENSURE_GREATER(d, 0, min); + HWY_ENSURE_GREATER(d, min / 2, min); + + // Also use Iota to ensure lanes are independent + HWY_ASSERT_MASK_EQ(d, mask_true, Gt(v2, vn)); + HWY_ASSERT_MASK_EQ(d, mask_true, Lt(vn, v2)); + HWY_ASSERT_MASK_EQ(d, mask_false, Lt(v2, vn)); + HWY_ASSERT_MASK_EQ(d, mask_false, Gt(vn, v2)); + + HWY_ASSERT_MASK_EQ(d, mask_false, Lt(v0, v0)); + HWY_ASSERT_MASK_EQ(d, mask_false, Lt(v2, v2)); + HWY_ASSERT_MASK_EQ(d, mask_false, Lt(vn, vn)); + HWY_ASSERT_MASK_EQ(d, mask_false, Gt(v0, v0)); + HWY_ASSERT_MASK_EQ(d, mask_false, Gt(v2, v2)); + HWY_ASSERT_MASK_EQ(d, mask_false, Gt(vn, vn)); + } +}; + +HWY_NOINLINE void TestAllStrictInt() { + ForSignedTypes(ForPartialVectors<TestStrictInt>()); +} + +struct TestStrictFloat { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const T huge_neg = T(-1E35); + const T huge_pos = T(1E36); + const auto v0 = Zero(d); + const auto v2 = Iota(d, T(2)); + const auto vn = Neg(v2); + + const auto mask_false = MaskFalse(d); + const auto mask_true = MaskTrue(d); + + // Individual values of interest + HWY_ENSURE_GREATER(d, 2, 1); + HWY_ENSURE_GREATER(d, 1, 0); + HWY_ENSURE_GREATER(d, 0, -1); + HWY_ENSURE_GREATER(d, -1, -2); + HWY_ENSURE_GREATER(d, huge_pos, 1); + HWY_ENSURE_GREATER(d, huge_pos, 0); + HWY_ENSURE_GREATER(d, huge_pos, -1); + HWY_ENSURE_GREATER(d, huge_pos, huge_neg); + HWY_ENSURE_GREATER(d, 0, huge_neg); + + // Also use Iota to ensure lanes are independent + HWY_ASSERT_MASK_EQ(d, mask_true, Gt(v2, vn)); + HWY_ASSERT_MASK_EQ(d, mask_true, Lt(vn, v2)); + HWY_ASSERT_MASK_EQ(d, mask_false, Lt(v2, vn)); + HWY_ASSERT_MASK_EQ(d, mask_false, Gt(vn, v2)); + + HWY_ASSERT_MASK_EQ(d, mask_false, Lt(v0, v0)); + HWY_ASSERT_MASK_EQ(d, mask_false, Lt(v2, v2)); + HWY_ASSERT_MASK_EQ(d, mask_false, Lt(vn, vn)); + HWY_ASSERT_MASK_EQ(d, mask_false, Gt(v0, v0)); + HWY_ASSERT_MASK_EQ(d, mask_false, Gt(v2, v2)); + HWY_ASSERT_MASK_EQ(d, mask_false, Gt(vn, vn)); + } +}; + +HWY_NOINLINE void TestAllStrictFloat() { + ForFloatTypes(ForPartialVectors<TestStrictFloat>()); +} + +struct TestWeakFloat { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v2 = Iota(d, T(2)); + const auto vn = Iota(d, -T(Lanes(d))); + + const auto mask_false = MaskFalse(d); + const auto mask_true = MaskTrue(d); + + HWY_ASSERT_MASK_EQ(d, mask_true, Ge(v2, v2)); + HWY_ASSERT_MASK_EQ(d, mask_true, Le(vn, vn)); + + HWY_ASSERT_MASK_EQ(d, mask_true, Ge(v2, vn)); + HWY_ASSERT_MASK_EQ(d, mask_true, Le(vn, v2)); + + HWY_ASSERT_MASK_EQ(d, mask_false, Le(v2, vn)); + HWY_ASSERT_MASK_EQ(d, mask_false, Ge(vn, v2)); + } +}; + +HWY_NOINLINE void TestAllWeakFloat() { + ForFloatTypes(ForPartialVectors<TestWeakFloat>()); +} + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE + +namespace hwy { +HWY_BEFORE_TEST(HwyCompareTest); +HWY_EXPORT_AND_TEST_P(HwyCompareTest, TestAllEquality); +HWY_EXPORT_AND_TEST_P(HwyCompareTest, TestAllStrictUnsigned); +HWY_EXPORT_AND_TEST_P(HwyCompareTest, TestAllStrictInt); +HWY_EXPORT_AND_TEST_P(HwyCompareTest, TestAllStrictFloat); +HWY_EXPORT_AND_TEST_P(HwyCompareTest, TestAllWeakFloat); +} // namespace hwy + +// Ought not to be necessary, but without this, no tests run on RVV. +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#endif diff --git a/third_party/highway/hwy/tests/convert_test.cc b/third_party/highway/hwy/tests/convert_test.cc new file mode 100644 index 0000000..aeed5cc --- /dev/null +++ b/third_party/highway/hwy/tests/convert_test.cc @@ -0,0 +1,811 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <stddef.h> +#include <stdint.h> +#include <string.h> + +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "tests/convert_test.cc" +#include "hwy/foreach_target.h" + +#include "hwy/highway.h" +#include "hwy/tests/test_util-inl.h" + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +// Cast and ensure bytes are the same. Called directly from TestAllBitCast or +// via TestBitCastFrom. +template <typename ToT> +struct TestBitCast { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const Repartition<ToT, D> dto; + const size_t N = Lanes(d); + const size_t Nto = Lanes(dto); + if (N == 0 || Nto == 0) return; + HWY_ASSERT_EQ(N * sizeof(T), Nto * sizeof(ToT)); + const auto vf = Iota(d, 1); + const auto vt = BitCast(dto, vf); + // Must return the same bits + auto from_lanes = AllocateAligned<T>(Lanes(d)); + auto to_lanes = AllocateAligned<ToT>(Lanes(dto)); + Store(vf, d, from_lanes.get()); + Store(vt, dto, to_lanes.get()); + HWY_ASSERT( + BytesEqual(from_lanes.get(), to_lanes.get(), Lanes(d) * sizeof(T))); + } +}; + +// From D to all types. +struct TestBitCastFrom { + template <typename T, class D> + HWY_NOINLINE void operator()(T t, D d) { + TestBitCast<uint8_t>()(t, d); + TestBitCast<uint16_t>()(t, d); + TestBitCast<uint32_t>()(t, d); +#if HWY_CAP_INTEGER64 + TestBitCast<uint64_t>()(t, d); +#endif + TestBitCast<int8_t>()(t, d); + TestBitCast<int16_t>()(t, d); + TestBitCast<int32_t>()(t, d); +#if HWY_CAP_INTEGER64 + TestBitCast<int64_t>()(t, d); +#endif + TestBitCast<float>()(t, d); +#if HWY_CAP_FLOAT64 + TestBitCast<double>()(t, d); +#endif + } +}; + +HWY_NOINLINE void TestAllBitCast() { + // For HWY_SCALAR and partial vectors, we can only cast to same-sized types: + // the former can't partition its single lane, and the latter can be smaller + // than a destination type. + const ForPartialVectors<TestBitCast<uint8_t>> to_u8; + to_u8(uint8_t()); + to_u8(int8_t()); + + const ForPartialVectors<TestBitCast<int8_t>> to_i8; + to_i8(uint8_t()); + to_i8(int8_t()); + + const ForPartialVectors<TestBitCast<uint16_t>> to_u16; + to_u16(uint16_t()); + to_u16(int16_t()); + + const ForPartialVectors<TestBitCast<int16_t>> to_i16; + to_i16(uint16_t()); + to_i16(int16_t()); + + const ForPartialVectors<TestBitCast<uint32_t>> to_u32; + to_u32(uint32_t()); + to_u32(int32_t()); + to_u32(float()); + + const ForPartialVectors<TestBitCast<int32_t>> to_i32; + to_i32(uint32_t()); + to_i32(int32_t()); + to_i32(float()); + +#if HWY_CAP_INTEGER64 + const ForPartialVectors<TestBitCast<uint64_t>> to_u64; + to_u64(uint64_t()); + to_u64(int64_t()); +#if HWY_CAP_FLOAT64 + to_u64(double()); +#endif + + const ForPartialVectors<TestBitCast<int64_t>> to_i64; + to_i64(uint64_t()); + to_i64(int64_t()); +#if HWY_CAP_FLOAT64 + to_i64(double()); +#endif +#endif // HWY_CAP_INTEGER64 + + const ForPartialVectors<TestBitCast<float>> to_float; + to_float(uint32_t()); + to_float(int32_t()); + to_float(float()); + +#if HWY_CAP_FLOAT64 + const ForPartialVectors<TestBitCast<double>> to_double; + to_double(double()); +#if HWY_CAP_INTEGER64 + to_double(uint64_t()); + to_double(int64_t()); +#endif // HWY_CAP_INTEGER64 +#endif // HWY_CAP_FLOAT64 + +#if HWY_TARGET != HWY_SCALAR + // For non-scalar vectors, we can cast all types to all. + ForAllTypes(ForGE64Vectors<TestBitCastFrom>()); +#endif +} + +template <typename ToT> +struct TestPromoteTo { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D from_d) { + static_assert(sizeof(T) < sizeof(ToT), "Input type must be narrower"); + const Rebind<ToT, D> to_d; + + const size_t N = Lanes(from_d); + auto from = AllocateAligned<T>(N); + auto expected = AllocateAligned<ToT>(N); + + RandomState rng; + for (size_t rep = 0; rep < AdjustedReps(200); ++rep) { + for (size_t i = 0; i < N; ++i) { + const uint64_t bits = rng(); + memcpy(&from[i], &bits, sizeof(T)); + expected[i] = from[i]; + } + + HWY_ASSERT_VEC_EQ(to_d, expected.get(), + PromoteTo(to_d, Load(from_d, from.get()))); + } + } +}; + +HWY_NOINLINE void TestAllPromoteTo() { + const ForPromoteVectors<TestPromoteTo<uint16_t>, 2> to_u16div2; + to_u16div2(uint8_t()); + + const ForPromoteVectors<TestPromoteTo<uint32_t>, 4> to_u32div4; + to_u32div4(uint8_t()); + + const ForPromoteVectors<TestPromoteTo<uint32_t>, 2> to_u32div2; + to_u32div2(uint16_t()); + + const ForPromoteVectors<TestPromoteTo<int16_t>, 2> to_i16div2; + to_i16div2(uint8_t()); + to_i16div2(int8_t()); + + const ForPromoteVectors<TestPromoteTo<int32_t>, 2> to_i32div2; + to_i32div2(uint16_t()); + to_i32div2(int16_t()); + + const ForPromoteVectors<TestPromoteTo<int32_t>, 4> to_i32div4; + to_i32div4(uint8_t()); + to_i32div4(int8_t()); + + // Must test f16/bf16 separately because we can only load/store/convert them. + +#if HWY_CAP_INTEGER64 + const ForPromoteVectors<TestPromoteTo<uint64_t>, 2> to_u64div2; + to_u64div2(uint32_t()); + + const ForPromoteVectors<TestPromoteTo<int64_t>, 2> to_i64div2; + to_i64div2(int32_t()); +#endif + +#if HWY_CAP_FLOAT64 + const ForPromoteVectors<TestPromoteTo<double>, 2> to_f64div2; + to_f64div2(int32_t()); + to_f64div2(float()); +#endif +} + +template <typename T, HWY_IF_FLOAT(T)> +bool IsFinite(T t) { + return std::isfinite(t); +} +// Wrapper avoids calling std::isfinite for integer types (ambiguous). +template <typename T, HWY_IF_NOT_FLOAT(T)> +bool IsFinite(T /*unused*/) { + return true; +} + +template <typename ToT> +struct TestDemoteTo { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D from_d) { + static_assert(!IsFloat<ToT>(), "Use TestDemoteToFloat for float output"); + static_assert(sizeof(T) > sizeof(ToT), "Input type must be wider"); + const Rebind<ToT, D> to_d; + + const size_t N = Lanes(from_d); + auto from = AllocateAligned<T>(N); + auto expected = AllocateAligned<ToT>(N); + + // Narrower range in the wider type, for clamping before we cast + const T min = LimitsMin<ToT>(); + const T max = LimitsMax<ToT>(); + + const auto value_ok = [&](T& value) { + if (!IsFinite(value)) return false; +#if HWY_EMULATE_SVE + // farm_sve just casts, which is undefined if the value is out of range. + value = HWY_MIN(HWY_MAX(min, value), max); +#endif + return true; + }; + + RandomState rng; + for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) { + for (size_t i = 0; i < N; ++i) { + do { + const uint64_t bits = rng(); + memcpy(&from[i], &bits, sizeof(T)); + } while (!value_ok(from[i])); + expected[i] = static_cast<ToT>(HWY_MIN(HWY_MAX(min, from[i]), max)); + } + + HWY_ASSERT_VEC_EQ(to_d, expected.get(), + DemoteTo(to_d, Load(from_d, from.get()))); + } + } +}; + +HWY_NOINLINE void TestAllDemoteToInt() { + ForDemoteVectors<TestDemoteTo<uint8_t>>()(int16_t()); + ForDemoteVectors<TestDemoteTo<uint8_t>, 4>()(int32_t()); + + ForDemoteVectors<TestDemoteTo<int8_t>>()(int16_t()); + ForDemoteVectors<TestDemoteTo<int8_t>, 4>()(int32_t()); + + const ForDemoteVectors<TestDemoteTo<uint16_t>> to_u16; + to_u16(int32_t()); + + const ForDemoteVectors<TestDemoteTo<int16_t>> to_i16; + to_i16(int32_t()); +} + +HWY_NOINLINE void TestAllDemoteToMixed() { +#if HWY_CAP_FLOAT64 + const ForDemoteVectors<TestDemoteTo<int32_t>> to_i32; + to_i32(double()); +#endif +} + +template <typename ToT> +struct TestDemoteToFloat { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D from_d) { + // For floats, we clamp differently and cannot call LimitsMin. + static_assert(IsFloat<ToT>(), "Use TestDemoteTo for integer output"); + static_assert(sizeof(T) > sizeof(ToT), "Input type must be wider"); + const Rebind<ToT, D> to_d; + + const size_t N = Lanes(from_d); + auto from = AllocateAligned<T>(N); + auto expected = AllocateAligned<ToT>(N); + + RandomState rng; + for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) { + for (size_t i = 0; i < N; ++i) { + do { + const uint64_t bits = rng(); + memcpy(&from[i], &bits, sizeof(T)); + } while (!IsFinite(from[i])); + const T magn = std::abs(from[i]); + const T max_abs = HighestValue<ToT>(); + // NOTE: std:: version from C++11 cmath is not defined in RVV GCC, see + // https://lists.freebsd.org/pipermail/freebsd-current/2014-January/048130.html + const T clipped = copysign(HWY_MIN(magn, max_abs), from[i]); + expected[i] = static_cast<ToT>(clipped); + } + + HWY_ASSERT_VEC_EQ(to_d, expected.get(), + DemoteTo(to_d, Load(from_d, from.get()))); + } + } +}; + +HWY_NOINLINE void TestAllDemoteToFloat() { + // Must test f16 separately because we can only load/store/convert them. + +#if HWY_CAP_FLOAT64 + const ForDemoteVectors<TestDemoteToFloat<float>, 2> to_float; + to_float(double()); +#endif +} + +template <class D> +AlignedFreeUniquePtr<float[]> F16TestCases(D d, size_t& padded) { + const float test_cases[] = { + // +/- 1 + 1.0f, -1.0f, + // +/- 0 + 0.0f, -0.0f, + // near 0 + 0.25f, -0.25f, + // +/- integer + 4.0f, -32.0f, + // positive near limit + 65472.0f, 65504.0f, + // negative near limit + -65472.0f, -65504.0f, + // positive +/- delta + 2.00390625f, 3.99609375f, + // negative +/- delta + -2.00390625f, -3.99609375f, + // No infinity/NaN - implementation-defined due to ARM. + }; + const size_t kNumTestCases = sizeof(test_cases) / sizeof(test_cases[0]); + const size_t N = Lanes(d); + padded = RoundUpTo(kNumTestCases, N); // allow loading whole vectors + auto in = AllocateAligned<float>(padded); + auto expected = AllocateAligned<float>(padded); + std::copy(test_cases, test_cases + kNumTestCases, in.get()); + std::fill(in.get() + kNumTestCases, in.get() + padded, 0.0f); + return in; +} + +struct TestF16 { + template <typename TF32, class DF32> + HWY_NOINLINE void operator()(TF32 /*t*/, DF32 d32) { +#if HWY_CAP_FLOAT16 + size_t padded; + auto in = F16TestCases(d32, padded); + using TF16 = float16_t; + const Rebind<TF16, DF32> d16; + const size_t N = Lanes(d32); // same count for f16 + auto temp16 = AllocateAligned<TF16>(N); + + for (size_t i = 0; i < padded; i += N) { + const auto loaded = Load(d32, &in[i]); + Store(DemoteTo(d16, loaded), d16, temp16.get()); + HWY_ASSERT_VEC_EQ(d32, loaded, PromoteTo(d32, Load(d16, temp16.get()))); + } +#else + (void)d32; +#endif + } +}; + +HWY_NOINLINE void TestAllF16() { ForDemoteVectors<TestF16>()(float()); } + +template <class D> +AlignedFreeUniquePtr<float[]> BF16TestCases(D d, size_t& padded) { + const float test_cases[] = { + // +/- 1 + 1.0f, -1.0f, + // +/- 0 + 0.0f, -0.0f, + // near 0 + 0.25f, -0.25f, + // +/- integer + 4.0f, -32.0f, + // positive near limit + 3.389531389251535E38f, 1.99384199368e+38f, + // negative near limit + -3.389531389251535E38f, -1.99384199368e+38f, + // positive +/- delta + 2.015625f, 3.984375f, + // negative +/- delta + -2.015625f, -3.984375f, + }; + const size_t kNumTestCases = sizeof(test_cases) / sizeof(test_cases[0]); + const size_t N = Lanes(d); + padded = RoundUpTo(kNumTestCases, N); // allow loading whole vectors + auto in = AllocateAligned<float>(padded); + auto expected = AllocateAligned<float>(padded); + std::copy(test_cases, test_cases + kNumTestCases, in.get()); + std::fill(in.get() + kNumTestCases, in.get() + padded, 0.0f); + return in; +} + +struct TestBF16 { + template <typename TF32, class DF32> + HWY_NOINLINE void operator()(TF32 /*t*/, DF32 d32) { +#if HWY_TARGET != HWY_RVV + size_t padded; + auto in = BF16TestCases(d32, padded); + using TBF16 = bfloat16_t; +#if HWY_TARGET == HWY_SCALAR + const Rebind<TBF16, DF32> dbf16; // avoid 4/2 = 2 lanes +#else + const Repartition<TBF16, DF32> dbf16; +#endif + const Half<decltype(dbf16)> dbf16_half; + const size_t N = Lanes(d32); + auto temp16 = AllocateAligned<TBF16>(N); + + for (size_t i = 0; i < padded; i += N) { + const auto loaded = Load(d32, &in[i]); + const auto v16 = DemoteTo(dbf16_half, loaded); + Store(v16, dbf16_half, temp16.get()); + const auto v16_loaded = Load(dbf16_half, temp16.get()); + HWY_ASSERT_VEC_EQ(d32, loaded, PromoteTo(d32, v16_loaded)); + } +#else + (void)d32; +#endif + } +}; + +HWY_NOINLINE void TestAllBF16() { ForShrinkableVectors<TestBF16>()(float()); } + +template <class D> +AlignedFreeUniquePtr<float[]> ReorderBF16TestCases(D d, size_t& padded) { + const float test_cases[] = { + // Same as BF16TestCases: + // +/- 1 + 1.0f, + -1.0f, + // +/- 0 + 0.0f, + -0.0f, + // near 0 + 0.25f, + -0.25f, + // +/- integer + 4.0f, + -32.0f, + // positive +/- delta + 2.015625f, + 3.984375f, + // negative +/- delta + -2.015625f, + -3.984375f, + + // No huge values - would interfere with sum. But add more to fill 2 * N: + -2.0f, + -10.0f, + 0.03125f, + 1.03125f, + 1.5f, + 2.0f, + 4.0f, + 5.0f, + 6.0f, + 8.0f, + 10.0f, + 256.0f, + 448.0f, + 2080.0f, + }; + const size_t kNumTestCases = sizeof(test_cases) / sizeof(test_cases[0]); + const size_t N = Lanes(d); + padded = RoundUpTo(kNumTestCases, 2 * N); // allow loading pairs of vectors + auto in = AllocateAligned<float>(padded); + auto expected = AllocateAligned<float>(padded); + std::copy(test_cases, test_cases + kNumTestCases, in.get()); + std::fill(in.get() + kNumTestCases, in.get() + padded, 0.0f); + return in; +} + +class TestReorderDemote2To { + // In-place N^2 selection sort to avoid dependencies + void Sort(float* p, size_t count) { + for (size_t i = 0; i < count - 1; ++i) { + // Find min_element + size_t idx_min = i; + for (size_t j = i + 1; j < count; j++) { + if (p[j] < p[idx_min]) { + idx_min = j; + } + } + + // Swap with current + const float tmp = p[i]; + p[i] = p[idx_min]; + p[idx_min] = tmp; + } + } + + public: + template <typename TF32, class DF32> + HWY_NOINLINE void operator()(TF32 /*t*/, DF32 d32) { +#if HWY_TARGET != HWY_SCALAR + size_t padded; + auto in = ReorderBF16TestCases(d32, padded); + + using TBF16 = bfloat16_t; + const Repartition<TBF16, DF32> dbf16; + const Half<decltype(dbf16)> dbf16_half; + const size_t N = Lanes(d32); + auto temp16 = AllocateAligned<TBF16>(2 * N); + auto expected = AllocateAligned<float>(2 * N); + auto actual = AllocateAligned<float>(2 * N); + + for (size_t i = 0; i < padded; i += 2 * N) { + const auto f0 = Load(d32, &in[i + 0]); + const auto f1 = Load(d32, &in[i + N]); + const auto v16 = ReorderDemote2To(dbf16, f0, f1); + Store(v16, dbf16, temp16.get()); + const auto promoted0 = PromoteTo(d32, Load(dbf16_half, temp16.get() + 0)); + const auto promoted1 = PromoteTo(d32, Load(dbf16_half, temp16.get() + N)); + + // Smoke test: sum should be same (with tolerance for non-associativity) + const auto sum_expected = + GetLane(SumOfLanes(d32, Add(promoted0, promoted1))); + const auto sum_actual = GetLane(SumOfLanes(d32, Add(f0, f1))); + HWY_ASSERT(sum_actual - 1E-4 <= sum_actual && + sum_expected <= sum_actual + 1E-4); + + // Ensure values are the same after sorting to undo the Reorder + Store(f0, d32, expected.get() + 0); + Store(f1, d32, expected.get() + N); + Store(promoted0, d32, actual.get() + 0); + Store(promoted1, d32, actual.get() + N); + Sort(expected.get(), 2 * N); + Sort(actual.get(), 2 * N); + HWY_ASSERT_VEC_EQ(d32, expected.get() + 0, Load(d32, actual.get() + 0)); + HWY_ASSERT_VEC_EQ(d32, expected.get() + N, Load(d32, actual.get() + N)); + } +#else // HWY_SCALAR + (void)d32; +#endif + } +}; + +HWY_NOINLINE void TestAllReorderDemote2To() { + ForShrinkableVectors<TestReorderDemote2To>()(float()); +} + +struct TestConvertU8 { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, const D du32) { + const Rebind<uint8_t, D> du8; + auto lanes8 = AllocateAligned<uint8_t>(Lanes(du8)); + Store(Iota(du8, 0), du8, lanes8.get()); + HWY_ASSERT_VEC_EQ(du8, Iota(du8, 0), U8FromU32(Iota(du32, 0))); + HWY_ASSERT_VEC_EQ(du8, Iota(du8, 0x7F), U8FromU32(Iota(du32, 0x7F))); + } +}; + +HWY_NOINLINE void TestAllConvertU8() { + ForDemoteVectors<TestConvertU8, 4>()(uint32_t()); +} + +// Separate function to attempt to work around a compiler bug on ARM: when this +// is merged with TestIntFromFloat, outputs match a previous Iota(-(N+1)) input. +struct TestIntFromFloatHuge { + template <typename TF, class DF> + HWY_NOINLINE void operator()(TF /*unused*/, const DF df) { + // Still does not work, although ARMv7 manual says that float->int + // saturates, i.e. chooses the nearest representable value. Also causes + // out-of-memory for MSVC, and unsafe cast in farm_sve. +#if HWY_TARGET != HWY_NEON && !HWY_COMPILER_MSVC && !defined(HWY_EMULATE_SVE) + using TI = MakeSigned<TF>; + const Rebind<TI, DF> di; + + // Huge positive (lvalue works around GCC bug, tested with 10.2.1, where + // the expected i32 value is otherwise 0x80..00). + const auto expected_max = Set(di, LimitsMax<TI>()); + HWY_ASSERT_VEC_EQ(di, expected_max, ConvertTo(di, Set(df, TF(1E20)))); + + // Huge negative (also lvalue for safety, but GCC bug was not triggered) + const auto expected_min = Set(di, LimitsMin<TI>()); + HWY_ASSERT_VEC_EQ(di, expected_min, ConvertTo(di, Set(df, TF(-1E20)))); +#else + (void)df; +#endif + } +}; + +class TestIntFromFloat { + template <typename TF, class DF> + static HWY_NOINLINE void TestPowers(TF /*unused*/, const DF df) { + using TI = MakeSigned<TF>; + const Rebind<TI, DF> di; + constexpr size_t kBits = sizeof(TF) * 8; + + // Powers of two, plus offsets to set some mantissa bits. + const int64_t ofs_table[3] = {0LL, 3LL << (kBits / 2), 1LL << (kBits - 15)}; + for (int sign = 0; sign < 2; ++sign) { + for (size_t shift = 0; shift < kBits - 1; ++shift) { + for (int64_t ofs : ofs_table) { + const int64_t mag = (int64_t(1) << shift) + ofs; + const int64_t val = sign ? mag : -mag; + HWY_ASSERT_VEC_EQ(di, Set(di, static_cast<TI>(val)), + ConvertTo(di, Set(df, static_cast<TF>(val)))); + } + } + } + } + + template <typename TF, class DF> + static HWY_NOINLINE void TestRandom(TF /*unused*/, const DF df) { + using TI = MakeSigned<TF>; + const Rebind<TI, DF> di; + const size_t N = Lanes(df); + + // TF does not have enough precision to represent TI. + const double min = static_cast<double>(LimitsMin<TI>()); + const double max = static_cast<double>(LimitsMax<TI>()); + + // Also check random values. + auto from = AllocateAligned<TF>(N); + auto expected = AllocateAligned<TI>(N); + RandomState rng; + for (size_t rep = 0; rep < AdjustedReps(1000); ++rep) { + for (size_t i = 0; i < N; ++i) { + do { + const uint64_t bits = rng(); + memcpy(&from[i], &bits, sizeof(TF)); + } while (!std::isfinite(from[i])); +#if defined(HWY_EMULATE_SVE) + // farm_sve just casts, which is undefined if the value is out of range. + from[i] = HWY_MIN(HWY_MAX(min / 2, from[i]), max / 2); +#endif + if (from[i] >= max) { + expected[i] = LimitsMax<TI>(); + } else if (from[i] <= min) { + expected[i] = LimitsMin<TI>(); + } else { + expected[i] = static_cast<TI>(from[i]); + } + } + + HWY_ASSERT_VEC_EQ(di, expected.get(), + ConvertTo(di, Load(df, from.get()))); + } + } + + public: + template <typename TF, class DF> + HWY_NOINLINE void operator()(TF tf, const DF df) { + using TI = MakeSigned<TF>; + const Rebind<TI, DF> di; + const size_t N = Lanes(df); + + // Integer positive + HWY_ASSERT_VEC_EQ(di, Iota(di, TI(4)), ConvertTo(di, Iota(df, TF(4.0)))); + + // Integer negative + HWY_ASSERT_VEC_EQ(di, Iota(di, -TI(N)), ConvertTo(di, Iota(df, -TF(N)))); + + // Above positive + HWY_ASSERT_VEC_EQ(di, Iota(di, TI(2)), ConvertTo(di, Iota(df, TF(2.001)))); + + // Below positive + HWY_ASSERT_VEC_EQ(di, Iota(di, TI(3)), ConvertTo(di, Iota(df, TF(3.9999)))); + + const TF eps = static_cast<TF>(0.0001); + // Above negative + HWY_ASSERT_VEC_EQ(di, Iota(di, -TI(N)), + ConvertTo(di, Iota(df, -TF(N + 1) + eps))); + + // Below negative + HWY_ASSERT_VEC_EQ(di, Iota(di, -TI(N + 1)), + ConvertTo(di, Iota(df, -TF(N + 1) - eps))); + + TestPowers(tf, df); + TestRandom(tf, df); + } +}; + +HWY_NOINLINE void TestAllIntFromFloat() { + ForFloatTypes(ForPartialVectors<TestIntFromFloatHuge>()); + ForFloatTypes(ForPartialVectors<TestIntFromFloat>()); +} + +struct TestFloatFromInt { + template <typename TF, class DF> + HWY_NOINLINE void operator()(TF /*unused*/, const DF df) { + using TI = MakeSigned<TF>; + const RebindToSigned<DF> di; + const size_t N = Lanes(df); + + // Integer positive + HWY_ASSERT_VEC_EQ(df, Iota(df, TF(4.0)), ConvertTo(df, Iota(di, TI(4)))); + + // Integer negative + HWY_ASSERT_VEC_EQ(df, Iota(df, -TF(N)), ConvertTo(df, Iota(di, -TI(N)))); + + // Max positive + HWY_ASSERT_VEC_EQ(df, Set(df, TF(LimitsMax<TI>())), + ConvertTo(df, Set(di, LimitsMax<TI>()))); + + // Min negative + HWY_ASSERT_VEC_EQ(df, Set(df, TF(LimitsMin<TI>())), + ConvertTo(df, Set(di, LimitsMin<TI>()))); + } +}; + +HWY_NOINLINE void TestAllFloatFromInt() { + ForFloatTypes(ForPartialVectors<TestFloatFromInt>()); +} + +struct TestI32F64 { + template <typename TF, class DF> + HWY_NOINLINE void operator()(TF /*unused*/, const DF df) { + using TI = int32_t; + const Rebind<TI, DF> di; + const size_t N = Lanes(df); + + // Integer positive + HWY_ASSERT_VEC_EQ(di, Iota(di, TI(4)), DemoteTo(di, Iota(df, TF(4.0)))); + HWY_ASSERT_VEC_EQ(df, Iota(df, TF(4.0)), PromoteTo(df, Iota(di, TI(4)))); + + // Integer negative + HWY_ASSERT_VEC_EQ(di, Iota(di, -TI(N)), DemoteTo(di, Iota(df, -TF(N)))); + HWY_ASSERT_VEC_EQ(df, Iota(df, -TF(N)), PromoteTo(df, Iota(di, -TI(N)))); + + // Above positive + HWY_ASSERT_VEC_EQ(di, Iota(di, TI(2)), DemoteTo(di, Iota(df, TF(2.001)))); + HWY_ASSERT_VEC_EQ(df, Iota(df, TF(2.0)), PromoteTo(df, Iota(di, TI(2)))); + + // Below positive + HWY_ASSERT_VEC_EQ(di, Iota(di, TI(3)), DemoteTo(di, Iota(df, TF(3.9999)))); + HWY_ASSERT_VEC_EQ(df, Iota(df, TF(4.0)), PromoteTo(df, Iota(di, TI(4)))); + + const TF eps = static_cast<TF>(0.0001); + // Above negative + HWY_ASSERT_VEC_EQ(di, Iota(di, -TI(N)), + DemoteTo(di, Iota(df, -TF(N + 1) + eps))); + HWY_ASSERT_VEC_EQ(df, Iota(df, TF(-4.0)), PromoteTo(df, Iota(di, TI(-4)))); + + // Below negative + HWY_ASSERT_VEC_EQ(di, Iota(di, -TI(N + 1)), + DemoteTo(di, Iota(df, -TF(N + 1) - eps))); + HWY_ASSERT_VEC_EQ(df, Iota(df, TF(-2.0)), PromoteTo(df, Iota(di, TI(-2)))); + + // Max positive int + HWY_ASSERT_VEC_EQ(df, Set(df, TF(LimitsMax<TI>())), + PromoteTo(df, Set(di, LimitsMax<TI>()))); + + // Min negative int + HWY_ASSERT_VEC_EQ(df, Set(df, TF(LimitsMin<TI>())), + PromoteTo(df, Set(di, LimitsMin<TI>()))); + + // farm_sve just casts, which is undefined if the value is out of range. +#if !defined(HWY_EMULATE_SVE) + // Huge positive float + HWY_ASSERT_VEC_EQ(di, Set(di, LimitsMax<TI>()), + DemoteTo(di, Set(df, TF(1E12)))); + + // Huge negative float + HWY_ASSERT_VEC_EQ(di, Set(di, LimitsMin<TI>()), + DemoteTo(di, Set(df, TF(-1E12)))); +#endif + } +}; + +HWY_NOINLINE void TestAllI32F64() { +#if HWY_CAP_FLOAT64 + ForDemoteVectors<TestI32F64>()(double()); +#endif +} + + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE + +namespace hwy { +HWY_BEFORE_TEST(HwyConvertTest); +HWY_EXPORT_AND_TEST_P(HwyConvertTest, TestAllBitCast); +HWY_EXPORT_AND_TEST_P(HwyConvertTest, TestAllPromoteTo); +HWY_EXPORT_AND_TEST_P(HwyConvertTest, TestAllDemoteToInt); +HWY_EXPORT_AND_TEST_P(HwyConvertTest, TestAllDemoteToMixed); +HWY_EXPORT_AND_TEST_P(HwyConvertTest, TestAllDemoteToFloat); +HWY_EXPORT_AND_TEST_P(HwyConvertTest, TestAllF16); +HWY_EXPORT_AND_TEST_P(HwyConvertTest, TestAllBF16); +HWY_EXPORT_AND_TEST_P(HwyConvertTest, TestAllReorderDemote2To); +HWY_EXPORT_AND_TEST_P(HwyConvertTest, TestAllConvertU8); +HWY_EXPORT_AND_TEST_P(HwyConvertTest, TestAllIntFromFloat); +HWY_EXPORT_AND_TEST_P(HwyConvertTest, TestAllFloatFromInt); +HWY_EXPORT_AND_TEST_P(HwyConvertTest, TestAllI32F64); +} // namespace hwy + +// Ought not to be necessary, but without this, no tests run on RVV. +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#endif diff --git a/third_party/highway/hwy/tests/crypto_test.cc b/third_party/highway/hwy/tests/crypto_test.cc new file mode 100644 index 0000000..c85d63a --- /dev/null +++ b/third_party/highway/hwy/tests/crypto_test.cc @@ -0,0 +1,549 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <stddef.h> +#include <stdint.h> +#include <string.h> // memcpy + +#include "hwy/aligned_allocator.h" + +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "tests/crypto_test.cc" +#include "hwy/foreach_target.h" +#include "hwy/highway.h" +#include "hwy/tests/test_util-inl.h" + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +#define HWY_PRINT_CLMUL_GOLDEN 0 + +#if HWY_TARGET != HWY_SCALAR + +class TestAES { + template <typename T, class D> + HWY_NOINLINE void TestSBox(T /*unused*/, D d) { + // The generic implementation of the S-box is difficult to verify by + // inspection, so we add a white-box test that verifies it using enumeration + // (outputs for 0..255 vs. https://en.wikipedia.org/wiki/Rijndael_S-box). + const uint8_t sbox[256] = { + 0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, + 0xfe, 0xd7, 0xab, 0x76, 0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, + 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, 0xb7, 0xfd, 0x93, 0x26, + 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, + 0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, + 0xeb, 0x27, 0xb2, 0x75, 0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, + 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, 0x53, 0xd1, 0x00, 0xed, + 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, + 0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, + 0x50, 0x3c, 0x9f, 0xa8, 0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, + 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, 0xcd, 0x0c, 0x13, 0xec, + 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, + 0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, + 0xde, 0x5e, 0x0b, 0xdb, 0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, + 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, 0xe7, 0xc8, 0x37, 0x6d, + 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, + 0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, + 0x4b, 0xbd, 0x8b, 0x8a, 0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, + 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, 0xe1, 0xf8, 0x98, 0x11, + 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, + 0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, + 0xb0, 0x54, 0xbb, 0x16}; + + // Ensure it's safe to load an entire vector by padding. + const size_t N = Lanes(d); + const size_t padded = RoundUpTo(256, N); + auto expected = AllocateAligned<T>(padded); + // Must wrap around to match the input (Iota). + for (size_t pos = 0; pos < padded;) { + const size_t remaining = HWY_MIN(padded - pos, size_t(256)); + memcpy(expected.get() + pos, sbox, remaining); + pos += remaining; + } + + for (size_t i = 0; i < 256; i += N) { + const auto in = Iota(d, i); + HWY_ASSERT_VEC_EQ(d, expected.get() + i, detail::SubBytes(in)); + } + } + + public: + template <typename T, class D> + HWY_NOINLINE void operator()(T t, D d) { + // Test vector (after first KeyAddition) from + // https://csrc.nist.gov/CSRC/media/Projects/Cryptographic-Standards-and-Guidelines/documents/examples/AES_Core128.pdf + alignas(16) constexpr uint8_t test_lanes[16] = { + 0x40, 0xBF, 0xAB, 0xF4, 0x06, 0xEE, 0x4D, 0x30, + 0x42, 0xCA, 0x6B, 0x99, 0x7A, 0x5C, 0x58, 0x16}; + const auto test = LoadDup128(d, test_lanes); + + // = MixColumn result + alignas(16) constexpr uint8_t expected0_lanes[16] = { + 0x52, 0x9F, 0x16, 0xC2, 0x97, 0x86, 0x15, 0xCA, + 0xE0, 0x1A, 0xAE, 0x54, 0xBA, 0x1A, 0x26, 0x59}; + const auto expected0 = LoadDup128(d, expected0_lanes); + + // = KeyAddition result + alignas(16) constexpr uint8_t expected_lanes[16] = { + 0xF2, 0x65, 0xE8, 0xD5, 0x1F, 0xD2, 0x39, 0x7B, + 0xC3, 0xB9, 0x97, 0x6D, 0x90, 0x76, 0x50, 0x5C}; + const auto expected = LoadDup128(d, expected_lanes); + + alignas(16) uint8_t key_lanes[16]; + for (size_t i = 0; i < 16; ++i) { + key_lanes[i] = expected0_lanes[i] ^ expected_lanes[i]; + } + const auto round_key = LoadDup128(d, key_lanes); + + HWY_ASSERT_VEC_EQ(d, expected0, AESRound(test, Zero(d))); + HWY_ASSERT_VEC_EQ(d, expected, AESRound(test, round_key)); + + TestSBox(t, d); + } +}; +HWY_NOINLINE void TestAllAES() { ForGE128Vectors<TestAES>()(uint8_t()); } + +#else +HWY_NOINLINE void TestAllAES() {} +#endif // HWY_TARGET != HWY_SCALAR + +struct TestCLMul { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + // needs 64 bit lanes and 128-bit result +#if HWY_TARGET != HWY_SCALAR && HWY_CAP_INTEGER64 + const size_t N = Lanes(d); + if (N == 1) return; + + auto in1 = AllocateAligned<T>(N); + auto in2 = AllocateAligned<T>(N); + + constexpr size_t kCLMulNum = 512; + // Depends on rng! + static constexpr uint64_t kCLMulLower[kCLMulNum] = { + 0x24511d4ce34d6350ULL, 0x4ca582edde1236bbULL, 0x537e58f72dac25a8ULL, + 0x4e942d5e130b9225ULL, 0x75a906c519257a68ULL, 0x1df9f85126d96c5eULL, + 0x464e7c13f4ad286aULL, 0x138535ee35dabc40ULL, 0xb2f7477b892664ecULL, + 0x01557b077167c25dULL, 0xf32682490ee49624ULL, 0x0025bac603b9e140ULL, + 0xcaa86aca3e3daf40ULL, 0x1fbcfe4af73eb6c4ULL, 0x8ee8064dd0aae5dcULL, + 0x1248cb547858c213ULL, 0x37a55ee5b10fb34cULL, 0x6eb5c97b958f86e2ULL, + 0x4b1ab3eb655ea7cdULL, 0x1d66645a85627520ULL, 0xf8728e96daa36748ULL, + 0x38621043e6ff5e3bULL, 0xd1d28b5da5ffefb4ULL, 0x0a5cd65931546df7ULL, + 0x2a0639be3d844150ULL, 0x0e2d0f18c8d6f045ULL, 0xfacc770b963326c1ULL, + 0x19611b31ca2ef141ULL, 0xabea29510dd87518ULL, 0x18a7dc4b205f2768ULL, + 0x9d3975ea5612dc86ULL, 0x06319c139e374773ULL, 0x6641710400b4c390ULL, + 0x356c29b6001c3670ULL, 0xe9e04d851e040a00ULL, 0x21febe561222d79aULL, + 0xc071eaae6e148090ULL, 0x0eed351a0af94f5bULL, 0x04324eedb3c03688ULL, + 0x39e89b136e0d6ccdULL, 0x07d0fd2777a31600ULL, 0x44b8573827209822ULL, + 0x6d690229ea177d78ULL, 0x1b9749d960ba9f18ULL, 0x190945271c0fbb94ULL, + 0x189aea0e07d2c88eULL, 0xf18eab6b65a6beb2ULL, 0x57744b21c13d0d84ULL, + 0xf63050a613e95c2eULL, 0x12cd20d25f97102fULL, 0x5a5df0678dbcba60ULL, + 0x0b08fb80948bfafcULL, 0x44cf1cbe7c6fc3c8ULL, 0x166a470ef25da288ULL, + 0x2c498a609204e48cULL, 0x261b0a22585697ecULL, 0x737750574af7dde4ULL, + 0x4079959c60b01e0cULL, 0x06ed8aac13f782d6ULL, 0x019d454ba9b5ef20ULL, + 0xea1edbf96d49e858ULL, 0x17c2f3ebde9ac469ULL, 0x5cf72706e3d6f5e4ULL, + 0x16e856aa3c841516ULL, 0x256f7e3cef83368eULL, 0x47e17c8eb2774e77ULL, + 0x9b48ac150a804821ULL, 0x584523f61ccfdf22ULL, 0xedcb6a2a75d9e7f2ULL, + 0x1fe3d1838e537aa7ULL, 0x778872e9f64549caULL, 0x2f1cea6f0d3faf92ULL, + 0x0e8c4b6a9343f326ULL, 0x01902d1ba3048954ULL, 0xc5c1fd5269e91dc0ULL, + 0x0ef8a4707817eb9cULL, 0x1f696f09a5354ca4ULL, 0x369cd9de808b818cULL, + 0xf6917d1dd43fd784ULL, 0x7f4b76bf40dc166fULL, 0x4ce67698724ace12ULL, + 0x02c3bf60e6e9cd92ULL, 0xb8229e45b21458e8ULL, 0x415efd41e91adf49ULL, + 0x5edfcd516bb921cdULL, 0x5ff2c29429fd187eULL, 0x0af666b17103b3e0ULL, + 0x1f5e4ff8f54c9a5bULL, 0x429253d8a5544ba6ULL, 0x19de2fdf9f4d9dcaULL, + 0x29bf3d37ddc19a40ULL, 0x04d4513a879552baULL, 0x5cc7476cf71ee155ULL, + 0x40011f8c238784a5ULL, 0x1a3ae50b0fd2ee2bULL, 0x7db22f432ba462baULL, + 0x417290b0bee2284aULL, 0x055a6bd5bb853db2ULL, 0xaa667daeed8c2a34ULL, + 0x0d6b316bda7f3577ULL, 0x72d35598468e3d5dULL, 0x375b594804bfd33aULL, + 0x16ed3a319b540ae8ULL, 0x093bace4b4695afdULL, 0xc7118754ec2737ceULL, + 0x0fff361f0505c81aULL, 0x996e9e7291321af0ULL, 0x496b1d9b0b89ba8cULL, + 0x65a98b2e9181da9cULL, 0x70759c8dd45575dfULL, 0x3446fe727f5e2cbbULL, + 0x1121ae609d195e74ULL, 0x5ff5d68ce8a21018ULL, 0x0e27eca3825b60d6ULL, + 0x82f628bceca3d1daULL, 0x2756a0914e344047ULL, 0xa460406c1c708d50ULL, + 0x63ce32a0c083e491ULL, 0xc883e5a685c480e0ULL, 0x602c951891e600f9ULL, + 0x02ecb2e3911ca5f8ULL, 0x0d8675f4bb70781aULL, 0x43545cc3c78ea496ULL, + 0x04164b01d6b011c2ULL, 0x3acbb323dcab2c9bULL, 0x31c5ba4e22793082ULL, + 0x5a6484af5f7c2d10ULL, 0x1a929b16194e8078ULL, 0x7a6a75d03b313924ULL, + 0x0553c73a35b1d525ULL, 0xf18628c51142be34ULL, 0x1b51cf80d7efd8f5ULL, + 0x52e0ca4df63ee258ULL, 0x0e977099160650c9ULL, 0x6be1524e92024f70ULL, + 0x0ee2152625438b9dULL, 0xfa32af436f6d8eb4ULL, 0x5ecf49c2154287e5ULL, + 0x6b72f4ae3590569dULL, 0x086c5ee6e87bfb68ULL, 0x737a4f0dc04b6187ULL, + 0x08c3439280edea41ULL, 0x9547944f01636c5cULL, 0x6acfbfc2571cd71fULL, + 0x85d7842972449637ULL, 0x252ea5e5a7fad86aULL, 0x4e41468f99ba1632ULL, + 0x095e0c3ae63b25a2ULL, 0xb005ce88fd1c9425ULL, 0x748e668abbe09f03ULL, + 0xb2cfdf466b187d18ULL, 0x60b11e633d8fe845ULL, 0x07144c4d246db604ULL, + 0x139bcaac55e96125ULL, 0x118679b5a6176327ULL, 0x1cebe90fa4d9f83fULL, + 0x22244f52f0d312acULL, 0x669d4e17c9bfb713ULL, 0x96390e0b834bb0d0ULL, + 0x01f7f0e82ba08071ULL, 0x2dffeee31ca6d284ULL, 0x1f4738745ef039feULL, + 0x4ce0dd2b603b6420ULL, 0x0035fc905910a4d5ULL, 0x07df2b533df6fb04ULL, + 0x1cee2735c9b910ddULL, 0x2bc4af565f7809eaULL, 0x2f876c1f5cb1076cULL, + 0x33e079524099d056ULL, 0x169e0405d2f9efbaULL, 0x018643ab548a358cULL, + 0x1bb6fc4331cffe92ULL, 0x05111d3a04e92faaULL, 0x23c27ecf0d638b73ULL, + 0x1b79071dc1685d68ULL, 0x0662d20aba8e1e0cULL, 0xe7f6440277144c6fULL, + 0x4ca38b64c22196c0ULL, 0x43c05f6d1936fbeeULL, 0x0654199d4d1faf0fULL, + 0xf2014054e71c2d04ULL, 0x0a103e47e96b4c84ULL, 0x7986e691dd35b040ULL, + 0x4e1ebb53c306a341ULL, 0x2775bb3d75d65ba6ULL, 0x0562ab0adeff0f15ULL, + 0x3c2746ad5eba3eacULL, 0x1facdb5765680c60ULL, 0xb802a60027d81d00ULL, + 0x1191d0f6366ae3a9ULL, 0x81a97b5ae0ea5d14ULL, 0x06bee05b6178a770ULL, + 0xc7baeb2fe1d6aeb3ULL, 0x594cb5b867d04fdfULL, 0xf515a80138a4e350ULL, + 0x646417ad8073cf38ULL, 0x4a229a43373fb8d4ULL, 0x10fa6eafff1ca453ULL, + 0x9f060700895cc731ULL, 0x00521133d11d11f4ULL, 0xb940a2bb912a7a5cULL, + 0x3fab180670ad2a3cULL, 0x45a5f0e5b6fdb95dULL, 0x27c1baad6f946b15ULL, + 0x336c6bdbe527cf58ULL, 0x3b83aa602a5baea3ULL, 0xdf749153f9bcc376ULL, + 0x1a05513a6c0b4a90ULL, 0xb81e0b570a075c47ULL, 0x471fabb40bdc27ceULL, + 0x9dec9472f6853f60ULL, 0x361f71b88114193bULL, 0x3b550a8c4feeff00ULL, + 0x0f6cde5a68bc9bc0ULL, 0x3f50121a925703e0ULL, 0x6967ff66d6d343a9ULL, + 0xff6b5bd2ce7bc3ccULL, 0x05474cea08bf6cd8ULL, 0xf76eabbfaf108eb0ULL, + 0x067529be4fc6d981ULL, 0x4d766b137cf8a988ULL, 0x2f09c7395c5cfbbdULL, + 0x388793712da06228ULL, 0x02c9ff342c8f339aULL, 0x152c734139a860a3ULL, + 0x35776eb2b270c04dULL, 0x0f8d8b41f11c4608ULL, 0x0c2071665be6b288ULL, + 0xc034e212b3f71d88ULL, 0x071d961ef3276f99ULL, 0xf98598ee75b60773ULL, + 0x062062c58c6724e4ULL, 0xd156438e2125572cULL, 0x38552d59a7f0f7c8ULL, + 0x1a402178206e413cULL, 0x1f1f996c68293b26ULL, 0x8bce3cafe1730f7eULL, + 0x2d0480a0828f6bf5ULL, 0x6c99cffa171f92f6ULL, 0x0087f842bb0ac681ULL, + 0x11d7ed06e1e7fd3eULL, 0x07cb1186f2385dc6ULL, 0x5d7763ebff1e170fULL, + 0x2dacc870231ac292ULL, 0x8486317a9ffb390cULL, 0x1c3a6dd20c959ac6ULL, + 0x90dc96e3992e06b8ULL, 0x70d60bfa33e72b67ULL, 0x70c9bddd0985ee63ULL, + 0x012c9767b3673093ULL, 0xfcd3bc5580f6a88aULL, 0x0ac80017ef6308c3ULL, + 0xdb67d709ef4bba09ULL, 0x4c63e324f0e247ccULL, 0xa15481d3fe219d60ULL, + 0x094c4279cdccb501ULL, 0x965a28c72575cb82ULL, 0x022869db25e391ebULL, + 0x37f528c146023910ULL, 0x0c1290636917deceULL, 0x9aee25e96251ca9cULL, + 0x728ac5ba853b69c2ULL, 0x9f272c93c4be20c8ULL, 0x06c1aa6319d28124ULL, + 0x4324496b1ca8a4f7ULL, 0x0096ecfe7dfc0189ULL, 0x9e06131b19ae0020ULL, + 0x15278b15902f4597ULL, 0x2a9fece8c13842d8ULL, 0x1d4e6781f0e1355eULL, + 0x6855b712d3dbf7c0ULL, 0x06a07fad99be6f46ULL, 0x3ed9d7957e4d1d7cULL, + 0x0c326f7cbc248bb2ULL, 0xe6363ad2c537cf51ULL, 0x0e12eb1c40723f13ULL, + 0xf5c6ac850afba803ULL, 0x0322a79d615fa9f0ULL, 0x6116696ed97bd5f8ULL, + 0x0d438080fbbdc9f1ULL, 0x2e4dc42c38f1e243ULL, 0x64948e9104f3a5bfULL, + 0x9fd622371bdb5f00ULL, 0x0f12bf082b2a1b6eULL, 0x4b1f8d867d78031cULL, + 0x134392ea9f5ef832ULL, 0xf3d70472321bc23eULL, 0x05fcbe5e9eea268eULL, + 0x136dede7175a22cfULL, 0x1308f8baac2cbcccULL, 0xd691026f0915eb64ULL, + 0x0e49a668345c3a38ULL, 0x24ddbbe8bc96f331ULL, 0x4d2ec9479b640578ULL, + 0x450f0697327b359cULL, 0x32b45360f4488ee0ULL, 0x4f6d9ecec46a105aULL, + 0x5500c63401ae8e80ULL, 0x47dea495cf6f98baULL, 0x13dc9a2dfca80babULL, + 0xe6f8a93f7b24ca92ULL, 0x073f57a6d900a87fULL, 0x9ddb935fd3aa695aULL, + 0x101e98d24b39e8aaULL, 0x6b8d0eb95a507ddcULL, 0x45a908b3903d209bULL, + 0x6c96a3e119e617d4ULL, 0x2442787543d3be48ULL, 0xd3bc055c7544b364ULL, + 0x7693bb042ca8653eULL, 0xb95e3a4ea5d0101eULL, 0x116f0d459bb94a73ULL, + 0x841244b72cdc5e90ULL, 0x1271acced6cb34d3ULL, 0x07d289106524d638ULL, + 0x537c9cf49c01b5bbULL, 0x8a8e16706bb7a5daULL, 0x12e50a9c499dc3a9ULL, + 0x1cade520db2ba830ULL, 0x1add52f000d7db70ULL, 0x12cf15db2ce78e30ULL, + 0x0657eaf606bfc866ULL, 0x4026816d3b05b1d0ULL, 0x1ba0ebdf90128e4aULL, + 0xdfd649375996dd6eULL, 0x0f416e906c23d9aeULL, 0x384273cad0582a24ULL, + 0x2ff27b0378a46189ULL, 0xc4ecd18a2d7a7616ULL, 0x35cef0b5cd51d640ULL, + 0x7d582363643f48b7ULL, 0x0984ad746ad0ab7cULL, 0x2990a999835f9688ULL, + 0x2d4df66a97b19e05ULL, 0x592c79720af99aa2ULL, 0x052863c230602cd3ULL, + 0x5f5e2b15edcf2840ULL, 0x01dff1b694b978b0ULL, 0x14345a48b622025eULL, + 0x028fab3b6407f715ULL, 0x3455d188e6feca50ULL, 0x1d0d40288fb1b5fdULL, + 0x4685c5c2b6a1e5aeULL, 0x3a2077b1e5fe5adeULL, 0x1bc55d611445a0d8ULL, + 0x05480ae95f3f83feULL, 0xbbb59cfcf7e17fb6ULL, 0x13f7f10970bbb990ULL, + 0x6d00ac169425a352ULL, 0x7da0db397ef2d5d3ULL, 0x5b512a247f8d2479ULL, + 0x637eaa6a977c3c32ULL, 0x3720f0ae37cba89cULL, 0x443df6e6aa7f525bULL, + 0x28664c287dcef321ULL, 0x03c267c00cf35e49ULL, 0x690185572d4021deULL, + 0x2707ff2596e321c2ULL, 0xd865f5af7722c380ULL, 0x1ea285658e33aafbULL, + 0xc257c5e88755bef4ULL, 0x066f67275cfcc31eULL, 0xb09931945cc0fed0ULL, + 0x58c1dc38d6e3a03fULL, 0xf99489678fc94ee8ULL, 0x75045bb99be5758aULL, + 0x6c163bc34b40feefULL, 0x0420063ce7bdd3b4ULL, 0xf86ef10582bf2e28ULL, + 0x162c3449ca14858cULL, 0x94106aa61dfe3280ULL, 0x4073ae7a4e7e4941ULL, + 0x32b13fd179c250b4ULL, 0x0178fbb216a7e744ULL, 0xf840ae2f1cf92669ULL, + 0x18fc709acc80243dULL, 0x20ac2ebd69f4d558ULL, 0x6e580ad9c73ad46aULL, + 0x76d2b535b541c19dULL, 0x6c7a3fb9dd0ce0afULL, 0xc3481689b9754f28ULL, + 0x156e813b6557abdbULL, 0x6ee372e31276eb10ULL, 0x19cf37c038c8d381ULL, + 0x00d4d906c9ae3072ULL, 0x09f03cbb6dfbfd40ULL, 0x461ba31c4125f3cfULL, + 0x25b29fc63ad9f05bULL, 0x6808c95c2dddede9ULL, 0x0564224337066d9bULL, + 0xc87eb5f4a4d966f2ULL, 0x66fc66e1701f5847ULL, 0xc553a3559f74da28ULL, + 0x1dfd841be574df43ULL, 0x3ee2f100c3ebc082ULL, 0x1a2c4f9517b56e89ULL, + 0x502f65c4b535c8ffULL, 0x1da5663ab6f96ec0ULL, 0xba1f80b73988152cULL, + 0x364ff12182ac8dc1ULL, 0xe3457a3c4871db31ULL, 0x6ae9cadf92fd7e84ULL, + 0x9621ba3d6ca15186ULL, 0x00ff5af878c144ceULL, 0x918464dc130101a4ULL, + 0x036511e6b187efa6ULL, 0x06667d66550ff260ULL, 0x7fd18913f9b51bc1ULL, + 0x3740e6b27af77aa8ULL, 0x1f546c2fd358ff8aULL, 0x42f1424e3115c891ULL, + 0x03767db4e3a1bb33ULL, 0xa171a1c564345060ULL, 0x0afcf632fd7b1324ULL, + 0xb59508d933ffb7d0ULL, 0x57d766c42071be83ULL, 0x659f0447546114a2ULL, + 0x4070364481c460aeULL, 0xa2b9752280644d52ULL, 0x04ab884bea5771bdULL, + 0x87cd135602a232b4ULL, 0x15e54cd9a8155313ULL, 0x1e8005efaa3e1047ULL, + 0x696b93f4ab15d39fULL, 0x0855a8e540de863aULL, 0x0bb11799e79f9426ULL, + 0xeffa61e5c1b579baULL, 0x1e060a1d11808219ULL, 0x10e219205667c599ULL, + 0x2f7b206091c49498ULL, 0xb48854c820064860ULL, 0x21c4aaa3bfbe4a38ULL, + 0x8f4a032a3fa67e9cULL, 0x3146b3823401e2acULL, 0x3afee26f19d88400ULL, + 0x167087c485791d38ULL, 0xb67a1ed945b0fb4bULL, 0x02436eb17e27f1c0ULL, + 0xe05afce2ce2d2790ULL, 0x49c536fc6224cfebULL, 0x178865b3b862b856ULL, + 0x1ce530de26acde5bULL, 0x87312c0b30a06f38ULL, 0x03e653b578558d76ULL, + 0x4d3663c21d8b3accULL, 0x038003c23626914aULL, 0xd9d5a2c052a09451ULL, + 0x39b5acfe08a49384ULL, 0x40f349956d5800e4ULL, 0x0968b6950b1bd8feULL, + 0xd60b2ca030f3779cULL, 0x7c8bc11a23ce18edULL, 0xcc23374e27630bc2ULL, + 0x2e38fc2a8bb33210ULL, 0xe421357814ee5c44ULL, 0x315fb65ea71ec671ULL, + 0xfb1b0223f70ed290ULL, 0x30556c9f983eaf07ULL, 0x8dd438c3d0cd625aULL, + 0x05a8fd0c7ffde71bULL, 0x764d1313b5aeec7aULL, 0x2036af5de9622f47ULL, + 0x508a5bfadda292feULL, 0x3f77f04ba2830e90ULL, 0x9047cd9c66ca66d2ULL, + 0x1168b5318a54eb21ULL, 0xc93462d221da2e15ULL, 0x4c2c7cc54abc066eULL, + 0x767a56fec478240eULL, 0x095de72546595bd3ULL, 0xc9da535865158558ULL, + 0x1baccf36f33e73fbULL, 0xf3d7dbe64df77f18ULL, 0x1f8ebbb7be4850b8ULL, + 0x043c5ed77bce25a1ULL, 0x07d401041b2a178aULL, 0x9181ebb8bd8d5618ULL, + 0x078b935dc3e4034aULL, 0x7b59c08954214300ULL, 0x03570dc2a4f84421ULL, + 0xdd8715b82f6b4078ULL, 0x2bb49c8bb544163bULL, 0xc9eb125564d59686ULL, + 0x5fdc7a38f80b810aULL, 0x3a4a6d8fff686544ULL, 0x28360e2418627d3aULL, + 0x60874244c95ed992ULL, 0x2115cc1dd9c34ed3ULL, 0xfaa3ef61f55e9efcULL, + 0x27ac9b1ef1adc7e6ULL, 0x95ea00478fec3f54ULL, 0x5aea808b2d99ab43ULL, + 0xc8f79e51fe43a580ULL, 0x5dbccd714236ce25ULL, 0x783fa76ed0753458ULL, + 0x48cb290f19d84655ULL, 0xc86a832f7696099aULL, 0x52f30c6fec0e71d3ULL, + 0x77d4e91e8cdeb886ULL, 0x7169a703c6a79ccdULL, 0x98208145b9596f74ULL, + 0x0945695c761c0796ULL, 0x0be897830d17bae0ULL, 0x033ad3924caeeeb4ULL, + 0xedecb6cfa2d303a8ULL, 0x3f86b074818642e7ULL, 0xeefa7c878a8b03f4ULL, + 0x093c101b80922551ULL, 0xfb3b4e6c26ac0034ULL, 0x162bf87999b94f5eULL, + 0xeaedae76e975b17cULL, 0x1852aa090effe18eULL}; + + static constexpr uint64_t kCLMulUpper[kCLMulNum] = { + 0xbb41199b1d587c69ULL, 0x514d94d55894ee29ULL, 0xebc6cd4d2efd5d16ULL, + 0x042044ad2de477fdULL, 0xb865c8b0fcdf4b15ULL, 0x0724d7e551cc40f3ULL, + 0xb15a16f39edb0bccULL, 0x37d64419ede7a171ULL, 0x2aa01bb80c753401ULL, + 0x06ff3f8a95fdaf4dULL, 0x79898cc0838546deULL, 0x776acbd1b237c60aULL, + 0x4c1753be4f4e0064ULL, 0x0ba9243601206ed3ULL, 0xd567c3b1bf3ec557ULL, + 0x043fac7bcff61fb3ULL, 0x49356232b159fb2fULL, 0x3910c82038102d4dULL, + 0x30592fef753eb300ULL, 0x7b2660e0c92a9e9aULL, 0x8246c9248d671ef0ULL, + 0x5a0dcd95147af5faULL, 0x43fde953909cc0eaULL, 0x06147b972cb96e1bULL, + 0xd84193a6b2411d80ULL, 0x00cd7711b950196fULL, 0x1088f9f4ade7fa64ULL, + 0x05a13096ec113cfbULL, 0x958d816d53b00edcULL, 0x3846154a7cdba9cbULL, + 0x8af516db6b27d1e6ULL, 0x1a1d462ab8a33b13ULL, 0x4040b0ac1b2c754cULL, + 0x05127fe9af2fe1d6ULL, 0x9f96e79374321fa6ULL, 0x06ff64a4d9c326f3ULL, + 0x28709566e158ac15ULL, 0x301701d7111ca51cULL, 0x31e0445d1b9d9544ULL, + 0x0a95aff69bf1d03eULL, 0x7c298c8414ecb879ULL, 0x00801499b4143195ULL, + 0x91521a00dd676a5cULL, 0x2777526a14c2f723ULL, 0xfa26aac6a6357dddULL, + 0x1d265889b0187a4bULL, 0xcd6e70fa8ed283e4ULL, 0x18a815aa50ea92caULL, + 0xc01e082694a263c6ULL, 0x4b40163ba53daf25ULL, 0xbc658caff6501673ULL, + 0x3ba35359586b9652ULL, 0x74f96acc97a4936cULL, 0x3989dfdb0cf1d2cfULL, + 0x358a01eaa50dda32ULL, 0x01109a5ed8f0802bULL, 0x55b84922e63c2958ULL, + 0x55b14843d87551d5ULL, 0x1db8ec61b1b578d8ULL, 0x79a2d49ef8c3658fULL, + 0xa304516816b3fbe0ULL, 0x163ecc09cc7b82f9ULL, 0xab91e8d22aabef00ULL, + 0x0ed6b09262de8354ULL, 0xcfd47d34cf73f6f2ULL, 0x7dbd1db2390bc6c3ULL, + 0x5ae789d3875e7b00ULL, 0x1d60fd0e70fe8fa4ULL, 0x690bc15d5ae4f6f5ULL, + 0x121ef5565104fb44ULL, 0x6e98e89297353b54ULL, 0x42554949249d62edULL, + 0xd6d6d16b12df78d2ULL, 0x320b33549b74975dULL, 0xd2a0618763d22e00ULL, + 0x0808deb93cba2017ULL, 0x01bd3b2302a2cc70ULL, 0x0b7b8dd4d71c8dd6ULL, + 0x34d60a3382a0756cULL, 0x40984584c8219629ULL, 0xf1152cba10093a66ULL, + 0x068001c6b2159ccbULL, 0x3d70f13c6cda0800ULL, 0x0e6b6746a322b956ULL, + 0x83a494319d8c770bULL, 0x0faecf64a8553e9aULL, 0xa34919222c39b1bcULL, + 0x0c63850d89e71c6fULL, 0x585f0bee92e53dc8ULL, 0x10f222b13b4fa5deULL, + 0x61573114f94252f2ULL, 0x09d59c311fba6c27ULL, 0x014effa7da49ed4eULL, + 0x4a400a1bc1c31d26ULL, 0xc9091c047b484972ULL, 0x3989f341ec2230ccULL, + 0xdcb03a98b3aee41eULL, 0x4a54a676a33a95e1ULL, 0xe499b7753951ef7cULL, + 0x2f43b1d1061d8b48ULL, 0xc3313bdc68ceb146ULL, 0x5159f6bc0e99227fULL, + 0x98128e6d9c05efcaULL, 0x15ea32b27f77815bULL, 0xe882c054e2654eecULL, + 0x003d2cdb8faee8c6ULL, 0xb416dd333a9fe1dfULL, 0x73f6746aefcfc98bULL, + 0x93dc114c10a38d70ULL, 0x05055941657845eaULL, 0x2ed7351347349334ULL, + 0x26fb1ee2c69ae690ULL, 0xa4575d10dc5b28e0ULL, 0x3395b11295e485ebULL, + 0xe840f198a224551cULL, 0x78e6e5a431d941d4ULL, 0xa1fee3ceab27f391ULL, + 0x07d35b3c5698d0dcULL, 0x983c67fca9174a29ULL, 0x2bb6bbae72b5144aULL, + 0xa7730b8d13ce58efULL, 0x51b5272883de1998ULL, 0xb334e128bb55e260ULL, + 0x1cacf5fbbe1b9974ULL, 0x71a9df4bb743de60ULL, 0x5176fe545c2d0d7aULL, + 0xbe592ecf1a16d672ULL, 0x27aa8a30c3efe460ULL, 0x4c78a32f47991e06ULL, + 0x383459294312f26aULL, 0x97ba789127f1490cULL, 0x51c9aa8a3abd1ef1ULL, + 0xcc7355188121e50fULL, 0x0ecb3a178ae334c1ULL, 0x84879a5e574b7160ULL, + 0x0765298f6389e8f3ULL, 0x5c6750435539bb22ULL, 0x11a05cf056c937b5ULL, + 0xb5dc2172dbfb7662ULL, 0x3ffc17915d9f40e8ULL, 0xbc7904daf3b431b0ULL, + 0x71f2088490930a7cULL, 0xa89505fd9efb53c4ULL, 0x02e194afd61c5671ULL, + 0x99a97f4abf35fcecULL, 0x26830aad30fae96fULL, 0x4b2abc16b25cf0b0ULL, + 0x07ec6fffa1cafbdbULL, 0xf38188fde97a280cULL, 0x121335701afff64dULL, + 0xea5ef38b4e672a64ULL, 0x477edbcae3eabf03ULL, 0xa32813cc0e0d244dULL, + 0x13346d2af4972eefULL, 0xcbc18357af1cfa9aULL, 0x561b630316e73fa6ULL, + 0xe9dfb53249249305ULL, 0x5d2b9dd1479312eeULL, 0x3458008119b56d04ULL, + 0x50e6790b49801385ULL, 0x5bb9febe2349492bULL, 0x0c2813954299098fULL, + 0xf747b0c890a071d5ULL, 0x417e8f82cc028d77ULL, 0xa134fee611d804f8ULL, + 0x24c99ee9a0408761ULL, 0x3ebb224e727137f3ULL, 0x0686022073ceb846ULL, + 0xa05e901fb82ad7daULL, 0x0ece7dc43ab470fcULL, 0x2d334ecc58f7d6a3ULL, + 0x23166fadacc54e40ULL, 0x9c3a4472f839556eULL, 0x071717ab5267a4adULL, + 0xb6600ac351ba3ea0ULL, 0x30ec748313bb63d4ULL, 0xb5374e39287b23ccULL, + 0x074d75e784238aebULL, 0x77315879243914a4ULL, 0x3bbb1971490865f1ULL, + 0xa355c21f4fbe02d3ULL, 0x0027f4bb38c8f402ULL, 0xeef8708e652bc5f0ULL, + 0x7b9aa56cf9440050ULL, 0x113ac03c16cfc924ULL, 0x395db36d3e4bef9fULL, + 0x5d826fabcaa597aeULL, 0x2a77d3c58786d7e0ULL, 0x85996859a3ba19d4ULL, + 0x01e7e3c904c2d97fULL, 0x34f90b9b98d51fd0ULL, 0x243aa97fd2e99bb7ULL, + 0x40a0cebc4f65c1e8ULL, 0x46d3922ed4a5503eULL, 0x446e7ecaf1f9c0a4ULL, + 0x49dc11558bc2e6aeULL, 0xe7a9f20881793af8ULL, 0x5771cc4bc98103f1ULL, + 0x2446ea6e718fce90ULL, 0x25d14aca7f7da198ULL, 0x4347af186f9af964ULL, + 0x10cb44fc9146363aULL, 0x8a35587afce476b4ULL, 0x575144662fee3d3aULL, + 0x69f41177a6bc7a05ULL, 0x02ff8c38d6b3c898ULL, 0x57c73589a226ca40ULL, + 0x732f6b5baae66683ULL, 0x00c008bbedd4bb34ULL, 0x7412ff09524d6cadULL, + 0xb8fd0b5ad8c145a8ULL, 0x74bd9f94b6cdc7dfULL, 0x68233b317ca6c19cULL, + 0x314b9c2c08b15c54ULL, 0x5bd1ad72072ebd08ULL, 0x6610e6a6c07030e4ULL, + 0xa4fc38e885ead7ceULL, 0x36975d1ca439e034ULL, 0xa358f0fe358ffb1aULL, + 0x38e247ad663acf7dULL, 0x77daed3643b5deb8ULL, 0x5507c2aeae1ec3d0ULL, + 0xfdec226c73acf775ULL, 0x1b87ff5f5033492dULL, 0xa832dee545d9033fULL, + 0x1cee43a61e41783bULL, 0xdff82b2e2d822f69ULL, 0x2bbc9a376cb38cf2ULL, + 0x117b1cdaf765dc02ULL, 0x26a407f5682be270ULL, 0x8eb664cf5634af28ULL, + 0x17cb4513bec68551ULL, 0xb0df6527900cbfd0ULL, 0x335a2dc79c5afdfcULL, + 0xa2f0ca4cd38dca88ULL, 0x1c370713b81a2de1ULL, 0x849d5df654d1adfcULL, + 0x2fd1f7675ae14e44ULL, 0x4ff64dfc02247f7bULL, 0x3a2bcf40e395a48dULL, + 0x436248c821b187c1ULL, 0x29f4337b1c7104c0ULL, 0xfc317c46e6630ec4ULL, + 0x2774bccc4e3264c7ULL, 0x2d03218d9d5bee23ULL, 0x36a0ed04d659058aULL, + 0x452484461573cab6ULL, 0x0708edf87ed6272bULL, 0xf07960a1587446cbULL, + 0x3660167b067d84e0ULL, 0x65990a6993ddf8c4ULL, 0x0b197cd3d0b40b3fULL, + 0x1dcec4ab619f3a05ULL, 0x722ab223a84f9182ULL, 0x0822d61a81e7c38fULL, + 0x3d22ad75da563201ULL, 0x93cef6979fd35e0fULL, 0x05c3c25ae598b14cULL, + 0x1338df97dd496377ULL, 0x15bc324dc9c20acfULL, 0x96397c6127e6e8cfULL, + 0x004d01069ef2050fULL, 0x2fcf2e27893fdcbcULL, 0x072f77c3e44f4a5cULL, + 0x5eb1d80b3fe44918ULL, 0x1f59e7c28cc21f22ULL, 0x3390ce5df055c1f8ULL, + 0x4c0ef11df92cb6bfULL, 0x50f82f9e0848c900ULL, 0x08d0fde3ffc0ae38ULL, + 0xbd8d0089a3fbfb73ULL, 0x118ba5b0f311ef59ULL, 0x9be9a8407b926a61ULL, + 0x4ea04fbb21318f63ULL, 0xa1c8e7bb07b871ffULL, 0x1253a7262d5d3b02ULL, + 0x13e997a0512e5b29ULL, 0x54318460ce9055baULL, 0x4e1d8a4db0054798ULL, + 0x0b235226e2cade32ULL, 0x2588732c1476b315ULL, 0x16a378750ba8ac68ULL, + 0xba0b116c04448731ULL, 0x4dd02bd47694c2f1ULL, 0x16d6797b218b6b25ULL, + 0x769eb3709cfbf936ULL, 0x197746a0ce396f38ULL, 0x7d17ad8465961d6eULL, + 0xfe58f4998ae19bb4ULL, 0x36df24305233ce69ULL, 0xb88a4eb008f4ee72ULL, + 0x302b2eb923334787ULL, 0x15a4e3edbe13d448ULL, 0x39a4bf64dd7730ceULL, + 0xedf25421b31090c4ULL, 0x4d547fc131be3b69ULL, 0x2b316e120ca3b90eULL, + 0x0faf2357bf18a169ULL, 0x71f34b54ee2c1d62ULL, 0x18eaf6e5c93a3824ULL, + 0x7e168ba03c1b4c18ULL, 0x1a534dd586d9e871ULL, 0xa2cccd307f5f8c38ULL, + 0x2999a6fb4dce30f6ULL, 0x8f6d3b02c1d549a6ULL, 0x5cf7f90d817aac5aULL, + 0xd2a4ceefe66c8170ULL, 0x11560edc4ca959feULL, 0x89e517e6f0dc464dULL, + 0x75bb8972dddd2085ULL, 0x13859ed1e459d65aULL, 0x057114653326fa84ULL, + 0xe2e6f465173cc86cULL, 0x0ada4076497d7de4ULL, 0xa856fa10ec6dbf8aULL, + 0x41505d9a7c25d875ULL, 0x3091b6278382eccdULL, 0x055737185b2c3f13ULL, + 0x2f4df8ecd6f9c632ULL, 0x0633e89c33552d98ULL, 0xf7673724d16db440ULL, + 0x7331bd08e636c391ULL, 0x0252f29672fee426ULL, 0x1fc384946b6b9ddeULL, + 0x03460c12c901443aULL, 0x003a0792e10abcdaULL, 0x8dbec31f624e37d0ULL, + 0x667420d5bfe4dcbeULL, 0xfbfa30e874ed7641ULL, 0x46d1ae14db7ecef6ULL, + 0x216bd7e8f5448768ULL, 0x32bcd40d3d69cc88ULL, 0x2e991dbc39b65abeULL, + 0x0e8fb123a502f553ULL, 0x3d2d486b2c7560c0ULL, 0x09aba1db3079fe03ULL, + 0xcb540c59398c9bceULL, 0x363970e5339ed600ULL, 0x2caee457c28af00eULL, + 0x005e7d7ee47f41a0ULL, 0x69fad3eb10f44100ULL, 0x048109388c75beb3ULL, + 0x253dddf96c7a6fb8ULL, 0x4c47f705b9d47d09ULL, 0x6cec894228b5e978ULL, + 0x04044bb9f8ff45c2ULL, 0x079e75704d775caeULL, 0x073bd54d2a9e2c33ULL, + 0xcec7289270a364fbULL, 0x19e7486f19cd9e4eULL, 0xb50ac15b86b76608ULL, + 0x0620cf81f165c812ULL, 0x63eaaf13be7b11d4ULL, 0x0e0cf831948248c2ULL, + 0xf0412df8f46e7957ULL, 0x671c1fe752517e3fULL, 0x8841bfb04dd3f540ULL, + 0x122de4142249f353ULL, 0x40a4959fb0e76870ULL, 0x25cfd3d4b4bbc459ULL, + 0x78a07c82930c60d0ULL, 0x12c2de24d4cbc969ULL, 0x85d44866096ad7f4ULL, + 0x1fd917ca66b2007bULL, 0x01fbbb0751764764ULL, 0x3d2a4953c6fe0fdcULL, + 0xcc1489c5737afd94ULL, 0x1817c5b6a5346f41ULL, 0xe605a6a7e9985644ULL, + 0x3c50412328ff1946ULL, 0xd8c7fd65817f1291ULL, 0x0bd66975ab66339bULL, + 0x2baf8fa1c7d10fa9ULL, 0x24abdf06ddef848dULL, 0x14df0c9b2ea4f6c2ULL, + 0x2be950edfd2cb1f7ULL, 0x21911e21094178b6ULL, 0x0fa54d518a93b379ULL, + 0xb52508e0ac01ab42ULL, 0x0e035b5fd8cb79beULL, 0x1c1c6d1a3b3c8648ULL, + 0x286037b42ea9871cULL, 0xfe67bf311e48a340ULL, 0x02324131e932a472ULL, + 0x2486dc2dd919e2deULL, 0x008aec7f1da1d2ebULL, 0x63269ba0e8d3eb3aULL, + 0x23c0f11154adb62fULL, 0xc6052393ecd4c018ULL, 0x523585b7d2f5b9fcULL, + 0xf7e6f8c1e87564c9ULL, 0x09eb9fe5dd32c1a3ULL, 0x4d4f86886e055472ULL, + 0x67ea17b58a37966bULL, 0x3d3ce8c23b1ed1a8ULL, 0x0df97c5ac48857ceULL, + 0x9b6992623759eb12ULL, 0x275aa9551ae091f2ULL, 0x08855e19ac5e62e5ULL, + 0x1155fffe0ae083ccULL, 0xbc9c78db7c570240ULL, 0x074560c447dd2418ULL, + 0x3bf78d330bcf1e70ULL, 0x49867cd4b7ed134bULL, 0x8e6eee0cb4470accULL, + 0x1dabafdf59233dd6ULL, 0xea3a50d844fc3fb8ULL, 0x4f03f4454764cb87ULL, + 0x1f2f41cc36c9e6ecULL, 0x53cba4df42963441ULL, 0x10883b70a88d91fbULL, + 0x62b1fc77d4eb9481ULL, 0x893d8f2604b362e1ULL, 0x0933b7855368b440ULL, + 0x9351b545703b2fceULL, 0x59c1d489b9bdd3b4ULL, 0xe72a9c4311417b18ULL, + 0x5355df77e88eb226ULL, 0xe802c37aa963d7e1ULL, 0x381c3747bd6c3bc3ULL, + 0x378565573444258cULL, 0x37848b1e52b43c18ULL, 0x5da2cd32bdce12b6ULL, + 0x13166c5da615f6fdULL, 0xa51ef95efcc66ac8ULL, 0x640c95e473f1e541ULL, + 0x6ec68def1f217500ULL, 0x49ce3543c76a4079ULL, 0x5fc6fd3cddc706b5ULL, + 0x05c3c0f0f6a1fb0dULL, 0xe7820c0996ad1bddULL, 0x21f0d752a088f35cULL, + 0x755405b51d6fc4a0ULL, 0x7ec7649ca4b0e351ULL, 0x3d2b6a46a251f790ULL, + 0x23e1176b19f418adULL, 0x06056575efe8ac05ULL, 0x0f75981b6966e477ULL, + 0x06e87ec41ad437e4ULL, 0x43f6c255d5e1cb84ULL, 0xe4e67d1120ceb580ULL, + 0x2cd67b9e12c26d7bULL, 0xcd00b5ff7fd187f1ULL, 0x3f6cd40accdc4106ULL, + 0x3e895c835459b330ULL, 0x0814d53a217c0850ULL, 0xc9111fe78bc3a62dULL, + 0x719967e351473204ULL, 0xe757707d24282aa4ULL, 0x7226b7f5607f98e6ULL, + 0x7b268ffae3c08d96ULL, 0x16d3917c8b86020eULL, 0x5128bca51c49ea64ULL, + 0x345ffea02bb1698dULL, 0x9460f5111fe4fbc8ULL, 0x60dd1aa5762852cbULL, + 0xbb7440ed3c81667cULL, 0x0a4b12affa7f6f5cULL, 0x95cbcb0ae03861b6ULL, + 0x07ab3b0591db6070ULL, 0xc6476a4c3de78982ULL, 0x204e82e8623ad725ULL, + 0x569a5b4e8ac2a5ccULL, 0x425a1d77d72ebae2ULL, 0xcdaad5551ab33830ULL, + 0x0b7c68fd8422939eULL, 0x46d9a01f53ec3020ULL, 0x102871edbb29e852ULL, + 0x7a8e8084039075a5ULL, 0x40eaede8615e376aULL, 0x4dc67d757a1c751fULL, + 0x1176ef33063f9145ULL, 0x4ea230285b1c8156ULL, 0x6b2aa46ce0027392ULL, + 0x32b13230fba1b068ULL, 0x0e69796851bb984fULL, 0xb749f4542db698c0ULL, + 0x19ad0241ffffd49cULL, 0x2f41e92ef6caff52ULL, 0x4d0b068576747439ULL, + 0x14d607aef7463e00ULL, 0x1443d00d85fb440eULL, 0x529b43bf68688780ULL, + 0x21133a6bc3a3e378ULL, 0x865b6436dae0e7e5ULL, 0x6b4fe83dc1d6defcULL, + 0x03a5858a0ca0be46ULL, 0x1e841b187e67f312ULL, 0x61ee22ef40a66940ULL, + 0x0494bd2e9e741ef8ULL, 0x4eb59e323010e72cULL, 0x19f2abcfb749810eULL, + 0xb30f1e4f994ef9bcULL, 0x53cf6cdd51bd2d96ULL, 0x263943036497a514ULL, + 0x0d4b52170aa2edbaULL, 0x0c4758a1c7b4f758ULL, 0x178dadb1b502b51aULL, + 0x1ddbb20a602eb57aULL, 0x1fc2e2564a9f27fdULL, 0xd5f8c50a0e3d6f90ULL, + 0x0081da3bbe72ac09ULL, 0xcf140d002ccdb200ULL, 0x0ae8389f09b017feULL, + 0x17cc9ffdc03f4440ULL, 0x04eb921d704bcdddULL, 0x139a0ce4cdc521abULL, + 0x0bfce00c145cb0f0ULL, 0x99925ff132eff707ULL, 0x063f6e5da50c3d35ULL, + 0xa0c25dea3f0e6e29ULL, 0x0c7a9048cc8e040fULL, + }; + + const size_t padded = RoundUpTo(kCLMulNum, N); + auto expected_lower = AllocateAligned<T>(padded); + auto expected_upper = AllocateAligned<T>(padded); + memcpy(expected_lower.get(), kCLMulLower, kCLMulNum * sizeof(T)); + memcpy(expected_upper.get(), kCLMulUpper, kCLMulNum * sizeof(T)); + const size_t padding_size = (padded - kCLMulNum) * sizeof(T); + memset(expected_lower.get() + kCLMulNum, 0, padding_size); + memset(expected_upper.get() + kCLMulNum, 0, padding_size); + + // Random inputs in each lane + RandomState rng; + for (size_t rep = 0; rep < kCLMulNum / N; ++rep) { + for (size_t i = 0; i < N; ++i) { + in1[i] = Random64(&rng); + in2[i] = Random64(&rng); + } + + const auto a = Load(d, in1.get()); + const auto b = Load(d, in2.get()); +#if HWY_PRINT_CLMUL_GOLDEN + Store(CLMulLower(a, b), d, expected_lower.get() + rep * N); + Store(CLMulUpper(a, b), d, expected_upper.get() + rep * N); +#else + HWY_ASSERT_VEC_EQ(d, expected_lower.get() + rep * N, CLMulLower(a, b)); + HWY_ASSERT_VEC_EQ(d, expected_upper.get() + rep * N, CLMulUpper(a, b)); +#endif + } + +#if HWY_PRINT_CLMUL_GOLDEN + // RVV lacks PRIu64, so print 32-bit halves. + for (size_t i = 0; i < kCLMulNum; ++i) { + printf("0x%08x%08xULL,", static_cast<uint32_t>(expected_lower[i] >> 32), + static_cast<uint32_t>(expected_lower[i] & 0xFFFFFFFFU)); + } + printf("\n"); + for (size_t i = 0; i < kCLMulNum; ++i) { + printf("0x%08x%08xULL,", static_cast<uint32_t>(expected_upper[i] >> 32), + static_cast<uint32_t>(expected_upper[i] & 0xFFFFFFFFU)); + } +#endif // HWY_PRINT_CLMUL_GOLDEN +#else + (void)d; +#endif + } +}; + +HWY_NOINLINE void TestAllCLMul() { ForGE128Vectors<TestCLMul>()(uint64_t()); } + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE + +namespace hwy { +HWY_BEFORE_TEST(HwyCryptoTest); +HWY_EXPORT_AND_TEST_P(HwyCryptoTest, TestAllAES); +HWY_EXPORT_AND_TEST_P(HwyCryptoTest, TestAllCLMul); +} // namespace hwy + +// Ought not to be necessary, but without this, no tests run on RVV. +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#endif diff --git a/third_party/highway/hwy/tests/hwy_gtest.h b/third_party/highway/hwy/tests/hwy_gtest.h new file mode 100644 index 0000000..f80d345 --- /dev/null +++ b/third_party/highway/hwy/tests/hwy_gtest.h @@ -0,0 +1,156 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef HWY_TESTS_HWY_GTEST_H_ +#define HWY_TESTS_HWY_GTEST_H_ + +// Adapters for GUnit to run tests for all targets. + +#include <stddef.h> +#include <stdint.h> + +#include <string> +#include <utility> // std::tuple + +#include "gtest/gtest.h" +#include "hwy/highway.h" + +namespace hwy { + +// googletest before 1.10 didn't define INSTANTIATE_TEST_SUITE_P() but instead +// used INSTANTIATE_TEST_CASE_P which is now deprecated. +#ifdef INSTANTIATE_TEST_SUITE_P +#define HWY_GTEST_INSTANTIATE_TEST_SUITE_P INSTANTIATE_TEST_SUITE_P +#else +#define HWY_GTEST_INSTANTIATE_TEST_SUITE_P INSTANTIATE_TEST_CASE_P +#endif + +// Helper class to run parametric tests using the hwy target as parameter. To +// use this define the following in your test: +// class MyTestSuite : public TestWithParamTarget { +// ... +// }; +// HWY_TARGET_INSTANTIATE_TEST_SUITE_P(MyTestSuite); +// TEST_P(MyTestSuite, MyTest) { ... } +class TestWithParamTarget : public testing::TestWithParam<uint32_t> { + protected: + void SetUp() override { SetSupportedTargetsForTest(GetParam()); } + + void TearDown() override { + // Check that the parametric test calls SupportedTargets() when the source + // was compiled with more than one target. In the single-target case only + // static dispatch will be used anyway. +#if (HWY_TARGETS & (HWY_TARGETS - 1)) != 0 + EXPECT_TRUE(SupportedTargetsCalledForTest()) + << "This hwy target parametric test doesn't use dynamic-dispatch and " + "doesn't need to be parametric."; +#endif + SetSupportedTargetsForTest(0); + } +}; + +// Function to convert the test parameter of a TestWithParamTarget for +// displaying it in the gtest test name. +static inline std::string TestParamTargetName( + const testing::TestParamInfo<uint32_t>& info) { + return TargetName(info.param); +} + +#define HWY_TARGET_INSTANTIATE_TEST_SUITE_P(suite) \ + HWY_GTEST_INSTANTIATE_TEST_SUITE_P( \ + suite##Group, suite, \ + testing::ValuesIn(::hwy::SupportedAndGeneratedTargets()), \ + ::hwy::TestParamTargetName) + +// Helper class similar to TestWithParamTarget to run parametric tests that +// depend on the target and another parametric test. If you need to use multiple +// extra parameters use a std::tuple<> of them and ::testing::Generate(...) as +// the generator. To use this class define the following in your test: +// class MyTestSuite : public TestWithParamTargetT<int> { +// ... +// }; +// HWY_TARGET_INSTANTIATE_TEST_SUITE_P_T(MyTestSuite, ::testing::Range(0, 9)); +// TEST_P(MyTestSuite, MyTest) { ... GetParam() .... } +template <typename T> +class TestWithParamTargetAndT + : public ::testing::TestWithParam<std::tuple<uint32_t, T>> { + public: + // Expose the parametric type here so it can be used by the + // HWY_TARGET_INSTANTIATE_TEST_SUITE_P_T macro. + using HwyParamType = T; + + protected: + void SetUp() override { + SetSupportedTargetsForTest(std::get<0>( + ::testing::TestWithParam<std::tuple<uint32_t, T>>::GetParam())); + } + + void TearDown() override { + // Check that the parametric test calls SupportedTargets() when the source + // was compiled with more than one target. In the single-target case only + // static dispatch will be used anyway. +#if (HWY_TARGETS & (HWY_TARGETS - 1)) != 0 + EXPECT_TRUE(SupportedTargetsCalledForTest()) + << "This hwy target parametric test doesn't use dynamic-dispatch and " + "doesn't need to be parametric."; +#endif + SetSupportedTargetsForTest(0); + } + + T GetParam() { + return std::get<1>( + ::testing::TestWithParam<std::tuple<uint32_t, T>>::GetParam()); + } +}; + +template <typename T> +std::string TestParamTargetNameAndT( + const testing::TestParamInfo<std::tuple<uint32_t, T>>& info) { + return std::string(TargetName(std::get<0>(info.param))) + "_" + + ::testing::PrintToString(std::get<1>(info.param)); +} + +#define HWY_TARGET_INSTANTIATE_TEST_SUITE_P_T(suite, generator) \ + HWY_GTEST_INSTANTIATE_TEST_SUITE_P( \ + suite##Group, suite, \ + ::testing::Combine( \ + testing::ValuesIn(::hwy::SupportedAndGeneratedTargets()), \ + generator), \ + ::hwy::TestParamTargetNameAndT<suite::HwyParamType>) + +// Helper macro to export a function and define a test that tests it. This is +// equivalent to do a HWY_EXPORT of a void(void) function and run it in a test: +// class MyTestSuite : public TestWithParamTarget { +// ... +// }; +// HWY_TARGET_INSTANTIATE_TEST_SUITE_P(MyTestSuite); +// HWY_EXPORT_AND_TEST_P(MyTestSuite, MyTest); +#define HWY_EXPORT_AND_TEST_P(suite, func_name) \ + HWY_EXPORT(func_name); \ + TEST_P(suite, func_name) { HWY_DYNAMIC_DISPATCH(func_name)(); } \ + static_assert(true, "For requiring trailing semicolon") + +#define HWY_EXPORT_AND_TEST_P_T(suite, func_name) \ + HWY_EXPORT(func_name); \ + TEST_P(suite, func_name) { HWY_DYNAMIC_DISPATCH(func_name)(GetParam()); } \ + static_assert(true, "For requiring trailing semicolon") + +#define HWY_BEFORE_TEST(suite) \ + class suite : public hwy::TestWithParamTarget {}; \ + HWY_TARGET_INSTANTIATE_TEST_SUITE_P(suite); \ + static_assert(true, "For requiring trailing semicolon") + +} // namespace hwy + +#endif // HWY_TESTS_HWY_GTEST_H_ diff --git a/third_party/highway/hwy/tests/list_targets.cc b/third_party/highway/hwy/tests/list_targets.cc new file mode 100644 index 0000000..5682590 --- /dev/null +++ b/third_party/highway/hwy/tests/list_targets.cc @@ -0,0 +1,37 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Simple tool to print the list of targets that were compiled in when building +// this tool. + +#include <stdio.h> + +#include "hwy/highway.h" + +void PrintTargets(const char* msg, uint32_t targets) { + fprintf(stderr, "%s", msg); + // For each bit: + for (uint32_t x = targets; x != 0; x = x & (x - 1)) { + // Extract value of least-significant bit. + fprintf(stderr, " %s", hwy::TargetName(x & (~x + 1))); + } + fprintf(stderr, "\n"); +} + +int main() { + PrintTargets("Compiled HWY_TARGETS:", HWY_TARGETS); + PrintTargets("HWY_BASELINE_TARGETS:", HWY_BASELINE_TARGETS); + PrintTargets("Current CPU supports:", hwy::SupportedTargets()); + return 0; +} diff --git a/third_party/highway/hwy/tests/logical_test.cc b/third_party/highway/hwy/tests/logical_test.cc new file mode 100644 index 0000000..bc9835e --- /dev/null +++ b/third_party/highway/hwy/tests/logical_test.cc @@ -0,0 +1,284 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <stddef.h> +#include <stdint.h> +#include <string.h> // memcmp + +#include "hwy/aligned_allocator.h" +#include "hwy/base.h" + +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "tests/logical_test.cc" +#include "hwy/foreach_target.h" +#include "hwy/highway.h" +#include "hwy/tests/test_util-inl.h" + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +struct TestLogicalInteger { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v0 = Zero(d); + const auto vi = Iota(d, 0); + const auto ones = VecFromMask(d, Eq(v0, v0)); + const auto v1 = Set(d, 1); + const auto vnot1 = Set(d, T(~T(1))); + + HWY_ASSERT_VEC_EQ(d, v0, Not(ones)); + HWY_ASSERT_VEC_EQ(d, ones, Not(v0)); + HWY_ASSERT_VEC_EQ(d, v1, Not(vnot1)); + HWY_ASSERT_VEC_EQ(d, vnot1, Not(v1)); + + HWY_ASSERT_VEC_EQ(d, v0, And(v0, vi)); + HWY_ASSERT_VEC_EQ(d, v0, And(vi, v0)); + HWY_ASSERT_VEC_EQ(d, vi, And(vi, vi)); + + HWY_ASSERT_VEC_EQ(d, vi, Or(v0, vi)); + HWY_ASSERT_VEC_EQ(d, vi, Or(vi, v0)); + HWY_ASSERT_VEC_EQ(d, vi, Or(vi, vi)); + + HWY_ASSERT_VEC_EQ(d, vi, Xor(v0, vi)); + HWY_ASSERT_VEC_EQ(d, vi, Xor(vi, v0)); + HWY_ASSERT_VEC_EQ(d, v0, Xor(vi, vi)); + + HWY_ASSERT_VEC_EQ(d, vi, AndNot(v0, vi)); + HWY_ASSERT_VEC_EQ(d, v0, AndNot(vi, v0)); + HWY_ASSERT_VEC_EQ(d, v0, AndNot(vi, vi)); + + auto v = vi; + v = And(v, vi); + HWY_ASSERT_VEC_EQ(d, vi, v); + v = And(v, v0); + HWY_ASSERT_VEC_EQ(d, v0, v); + + v = Or(v, vi); + HWY_ASSERT_VEC_EQ(d, vi, v); + v = Or(v, v0); + HWY_ASSERT_VEC_EQ(d, vi, v); + + v = Xor(v, vi); + HWY_ASSERT_VEC_EQ(d, v0, v); + v = Xor(v, v0); + HWY_ASSERT_VEC_EQ(d, v0, v); + } +}; + +HWY_NOINLINE void TestAllLogicalInteger() { + ForIntegerTypes(ForPartialVectors<TestLogicalInteger>()); +} + +struct TestLogicalFloat { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v0 = Zero(d); + const auto vi = Iota(d, 0); + + HWY_ASSERT_VEC_EQ(d, v0, And(v0, vi)); + HWY_ASSERT_VEC_EQ(d, v0, And(vi, v0)); + HWY_ASSERT_VEC_EQ(d, vi, And(vi, vi)); + + HWY_ASSERT_VEC_EQ(d, vi, Or(v0, vi)); + HWY_ASSERT_VEC_EQ(d, vi, Or(vi, v0)); + HWY_ASSERT_VEC_EQ(d, vi, Or(vi, vi)); + + HWY_ASSERT_VEC_EQ(d, vi, Xor(v0, vi)); + HWY_ASSERT_VEC_EQ(d, vi, Xor(vi, v0)); + HWY_ASSERT_VEC_EQ(d, v0, Xor(vi, vi)); + + HWY_ASSERT_VEC_EQ(d, vi, AndNot(v0, vi)); + HWY_ASSERT_VEC_EQ(d, v0, AndNot(vi, v0)); + HWY_ASSERT_VEC_EQ(d, v0, AndNot(vi, vi)); + + auto v = vi; + v = And(v, vi); + HWY_ASSERT_VEC_EQ(d, vi, v); + v = And(v, v0); + HWY_ASSERT_VEC_EQ(d, v0, v); + + v = Or(v, vi); + HWY_ASSERT_VEC_EQ(d, vi, v); + v = Or(v, v0); + HWY_ASSERT_VEC_EQ(d, vi, v); + + v = Xor(v, vi); + HWY_ASSERT_VEC_EQ(d, v0, v); + v = Xor(v, v0); + HWY_ASSERT_VEC_EQ(d, v0, v); + } +}; + +HWY_NOINLINE void TestAllLogicalFloat() { + ForFloatTypes(ForPartialVectors<TestLogicalFloat>()); +} + +struct TestCopySign { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v0 = Zero(d); + const auto vp = Iota(d, 1); + const auto vn = Iota(d, T(-1E5)); // assumes N < 10^5 + + // Zero remains zero regardless of sign + HWY_ASSERT_VEC_EQ(d, v0, CopySign(v0, v0)); + HWY_ASSERT_VEC_EQ(d, v0, CopySign(v0, vp)); + HWY_ASSERT_VEC_EQ(d, v0, CopySign(v0, vn)); + HWY_ASSERT_VEC_EQ(d, v0, CopySignToAbs(v0, v0)); + HWY_ASSERT_VEC_EQ(d, v0, CopySignToAbs(v0, vp)); + HWY_ASSERT_VEC_EQ(d, v0, CopySignToAbs(v0, vn)); + + // Positive input, positive sign => unchanged + HWY_ASSERT_VEC_EQ(d, vp, CopySign(vp, vp)); + HWY_ASSERT_VEC_EQ(d, vp, CopySignToAbs(vp, vp)); + + // Positive input, negative sign => negated + HWY_ASSERT_VEC_EQ(d, Neg(vp), CopySign(vp, vn)); + HWY_ASSERT_VEC_EQ(d, Neg(vp), CopySignToAbs(vp, vn)); + + // Negative input, negative sign => unchanged + HWY_ASSERT_VEC_EQ(d, vn, CopySign(vn, vn)); + + // Negative input, positive sign => negated + HWY_ASSERT_VEC_EQ(d, Neg(vn), CopySign(vn, vp)); + } +}; + +HWY_NOINLINE void TestAllCopySign() { + ForFloatTypes(ForPartialVectors<TestCopySign>()); +} + +struct TestZeroIfNegative { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v0 = Zero(d); + const auto vp = Iota(d, 1); + const auto vn = Iota(d, T(-1E5)); // assumes N < 10^5 + + // Zero and positive remain unchanged + HWY_ASSERT_VEC_EQ(d, v0, ZeroIfNegative(v0)); + HWY_ASSERT_VEC_EQ(d, vp, ZeroIfNegative(vp)); + + // Negative are all replaced with zero + HWY_ASSERT_VEC_EQ(d, v0, ZeroIfNegative(vn)); + } +}; + +HWY_NOINLINE void TestAllZeroIfNegative() { + ForFloatTypes(ForPartialVectors<TestZeroIfNegative>()); +} + +struct TestBroadcastSignBit { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto s0 = Zero(d); + const auto s1 = Set(d, -1); // all bit set + const auto vpos = And(Iota(d, 0), Set(d, LimitsMax<T>())); + const auto vneg = Sub(s1, vpos); + + HWY_ASSERT_VEC_EQ(d, s0, BroadcastSignBit(vpos)); + HWY_ASSERT_VEC_EQ(d, s0, BroadcastSignBit(Set(d, LimitsMax<T>()))); + + HWY_ASSERT_VEC_EQ(d, s1, BroadcastSignBit(vneg)); + HWY_ASSERT_VEC_EQ(d, s1, BroadcastSignBit(Set(d, LimitsMin<T>()))); + HWY_ASSERT_VEC_EQ(d, s1, BroadcastSignBit(Set(d, LimitsMin<T>() / 2))); + } +}; + +HWY_NOINLINE void TestAllBroadcastSignBit() { + ForSignedTypes(ForPartialVectors<TestBroadcastSignBit>()); +} + +struct TestTestBit { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const size_t kNumBits = sizeof(T) * 8; + for (size_t i = 0; i < kNumBits; ++i) { + const auto bit1 = Set(d, T(1ull << i)); + const auto bit2 = Set(d, T(1ull << ((i + 1) % kNumBits))); + const auto bit3 = Set(d, T(1ull << ((i + 2) % kNumBits))); + const auto bits12 = Or(bit1, bit2); + const auto bits23 = Or(bit2, bit3); + HWY_ASSERT(AllTrue(d, TestBit(bit1, bit1))); + HWY_ASSERT(AllTrue(d, TestBit(bits12, bit1))); + HWY_ASSERT(AllTrue(d, TestBit(bits12, bit2))); + + HWY_ASSERT(AllFalse(d, TestBit(bits12, bit3))); + HWY_ASSERT(AllFalse(d, TestBit(bits23, bit1))); + HWY_ASSERT(AllFalse(d, TestBit(bit1, bit2))); + HWY_ASSERT(AllFalse(d, TestBit(bit2, bit1))); + HWY_ASSERT(AllFalse(d, TestBit(bit1, bit3))); + HWY_ASSERT(AllFalse(d, TestBit(bit3, bit1))); + HWY_ASSERT(AllFalse(d, TestBit(bit2, bit3))); + HWY_ASSERT(AllFalse(d, TestBit(bit3, bit2))); + } + } +}; + +HWY_NOINLINE void TestAllTestBit() { + ForIntegerTypes(ForPartialVectors<TestTestBit>()); +} + +struct TestPopulationCount { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { +#if HWY_TARGET == HWY_RVV || HWY_IS_DEBUG_BUILD + constexpr size_t kNumTests = 1 << 14; +#else + constexpr size_t kNumTests = 1 << 20; +#endif + RandomState rng; + size_t N = Lanes(d); + auto data = AllocateAligned<T>(N); + auto popcnt = AllocateAligned<T>(N); + for (size_t i = 0; i < kNumTests / N; i++) { + for (size_t i = 0; i < N; i++) { + data[i] = static_cast<T>(rng()); + popcnt[i] = static_cast<T>(PopCount(data[i])); + } + HWY_ASSERT_VEC_EQ(d, popcnt.get(), PopulationCount(Load(d, data.get()))); + } + } +}; + +HWY_NOINLINE void TestAllPopulationCount() { + ForUnsignedTypes(ForPartialVectors<TestPopulationCount>()); +} + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE + +namespace hwy { +HWY_BEFORE_TEST(HwyLogicalTest); +HWY_EXPORT_AND_TEST_P(HwyLogicalTest, TestAllLogicalInteger); +HWY_EXPORT_AND_TEST_P(HwyLogicalTest, TestAllLogicalFloat); +HWY_EXPORT_AND_TEST_P(HwyLogicalTest, TestAllCopySign); +HWY_EXPORT_AND_TEST_P(HwyLogicalTest, TestAllZeroIfNegative); +HWY_EXPORT_AND_TEST_P(HwyLogicalTest, TestAllBroadcastSignBit); +HWY_EXPORT_AND_TEST_P(HwyLogicalTest, TestAllTestBit); +HWY_EXPORT_AND_TEST_P(HwyLogicalTest, TestAllPopulationCount); +} // namespace hwy + +// Ought not to be necessary, but without this, no tests run on RVV. +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#endif diff --git a/third_party/highway/hwy/tests/mask_test.cc b/third_party/highway/hwy/tests/mask_test.cc new file mode 100644 index 0000000..569f85b --- /dev/null +++ b/third_party/highway/hwy/tests/mask_test.cc @@ -0,0 +1,465 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <inttypes.h> +#include <stddef.h> +#include <stdint.h> +#include <string.h> // memcmp + +#include "hwy/base.h" + +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "tests/mask_test.cc" +#include "hwy/foreach_target.h" + +#include "hwy/highway.h" +#include "hwy/tests/test_util-inl.h" + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +// All types. +struct TestFromVec { + template <typename T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const size_t N = Lanes(d); + auto lanes = AllocateAligned<T>(N); + + memset(lanes.get(), 0, N * sizeof(T)); + const auto actual_false = MaskFromVec(Load(d, lanes.get())); + HWY_ASSERT_MASK_EQ(d, MaskFalse(d), actual_false); + + memset(lanes.get(), 0xFF, N * sizeof(T)); + const auto actual_true = MaskFromVec(Load(d, lanes.get())); + HWY_ASSERT_MASK_EQ(d, MaskTrue(d), actual_true); + } +}; + +HWY_NOINLINE void TestAllFromVec() { + ForAllTypes(ForPartialVectors<TestFromVec>()); +} + +struct TestFirstN { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const size_t N = Lanes(d); + + const RebindToSigned<D> di; + using TI = TFromD<decltype(di)>; + using TN = SignedFromSize<HWY_MIN(sizeof(size_t), sizeof(TI))>; + const size_t max_len = static_cast<size_t>(LimitsMax<TN>()); + + for (size_t len = 0; len <= HWY_MIN(2 * N, max_len); ++len) { + const auto expected = + RebindMask(d, Lt(Iota(di, 0), Set(di, static_cast<TI>(len)))); + const auto actual = FirstN(d, len); + HWY_ASSERT_MASK_EQ(d, expected, actual); + } + + // Also ensure huge values yield all-true. + HWY_ASSERT_MASK_EQ(d, MaskTrue(d), FirstN(d, max_len)); + } +}; + +HWY_NOINLINE void TestAllFirstN() { + ForAllTypes(ForPartialVectors<TestFirstN>()); +} + +struct TestIfThenElse { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + RandomState rng; + + using TI = MakeSigned<T>; // For mask > 0 comparison + const Rebind<TI, D> di; + const size_t N = Lanes(d); + auto in1 = AllocateAligned<T>(N); + auto in2 = AllocateAligned<T>(N); + auto bool_lanes = AllocateAligned<TI>(N); + auto expected = AllocateAligned<T>(N); + + // Each lane should have a chance of having mask=true. + for (size_t rep = 0; rep < AdjustedReps(200); ++rep) { + for (size_t i = 0; i < N; ++i) { + in1[i] = static_cast<T>(Random32(&rng)); + in2[i] = static_cast<T>(Random32(&rng)); + bool_lanes[i] = (Random32(&rng) & 16) ? TI(1) : TI(0); + } + + const auto v1 = Load(d, in1.get()); + const auto v2 = Load(d, in2.get()); + const auto mask = RebindMask(d, Gt(Load(di, bool_lanes.get()), Zero(di))); + + for (size_t i = 0; i < N; ++i) { + expected[i] = bool_lanes[i] ? in1[i] : in2[i]; + } + HWY_ASSERT_VEC_EQ(d, expected.get(), IfThenElse(mask, v1, v2)); + + for (size_t i = 0; i < N; ++i) { + expected[i] = bool_lanes[i] ? in1[i] : T(0); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), IfThenElseZero(mask, v1)); + + for (size_t i = 0; i < N; ++i) { + expected[i] = bool_lanes[i] ? T(0) : in2[i]; + } + HWY_ASSERT_VEC_EQ(d, expected.get(), IfThenZeroElse(mask, v2)); + } + } +}; + +HWY_NOINLINE void TestAllIfThenElse() { + ForAllTypes(ForPartialVectors<TestIfThenElse>()); +} + +struct TestMaskVec { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + RandomState rng; + + using TI = MakeSigned<T>; // For mask > 0 comparison + const Rebind<TI, D> di; + const size_t N = Lanes(d); + auto bool_lanes = AllocateAligned<TI>(N); + + // Each lane should have a chance of having mask=true. + for (size_t rep = 0; rep < AdjustedReps(200); ++rep) { + for (size_t i = 0; i < N; ++i) { + bool_lanes[i] = (Random32(&rng) & 1024) ? TI(1) : TI(0); + } + + const auto mask = RebindMask(d, Gt(Load(di, bool_lanes.get()), Zero(di))); + HWY_ASSERT_MASK_EQ(d, mask, MaskFromVec(VecFromMask(d, mask))); + } + } +}; + +HWY_NOINLINE void TestAllMaskVec() { + const ForPartialVectors<TestMaskVec> test; + + test(uint16_t()); + test(int16_t()); + // TODO(janwas): float16_t - cannot compare yet + + ForUIF3264(test); +} + +struct TestMaskedLoad { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + RandomState rng; + + using TI = MakeSigned<T>; // For mask > 0 comparison + const Rebind<TI, D> di; + const size_t N = Lanes(d); + auto bool_lanes = AllocateAligned<TI>(N); + + auto lanes = AllocateAligned<T>(N); + Store(Iota(d, T{1}), d, lanes.get()); + + // Each lane should have a chance of having mask=true. + for (size_t rep = 0; rep < AdjustedReps(200); ++rep) { + for (size_t i = 0; i < N; ++i) { + bool_lanes[i] = (Random32(&rng) & 1024) ? TI(1) : TI(0); + } + + const auto mask = RebindMask(d, Gt(Load(di, bool_lanes.get()), Zero(di))); + const auto expected = IfThenElseZero(mask, Load(d, lanes.get())); + const auto actual = MaskedLoad(mask, d, lanes.get()); + HWY_ASSERT_VEC_EQ(d, expected, actual); + } + } +}; + +HWY_NOINLINE void TestAllMaskedLoad() { + ForAllTypes(ForPartialVectors<TestMaskedLoad>()); +} + +struct TestAllTrueFalse { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto zero = Zero(d); + auto v = zero; + + const size_t N = Lanes(d); + auto lanes = AllocateAligned<T>(N); + std::fill(lanes.get(), lanes.get() + N, T(0)); + + auto mask_lanes = AllocateAligned<T>(N); + + HWY_ASSERT(AllTrue(d, Eq(v, zero))); + HWY_ASSERT(!AllFalse(d, Eq(v, zero))); + + // Single lane implies AllFalse = !AllTrue. Otherwise, there are multiple + // lanes and one is nonzero. + const bool expected_all_false = (N != 1); + + // Set each lane to nonzero and back to zero + for (size_t i = 0; i < N; ++i) { + lanes[i] = T(1); + v = Load(d, lanes.get()); + + // GCC 10.2.1 workaround: AllTrue(Eq(v, zero)) is true but should not be. + // Assigning to an lvalue is insufficient but storing to memory prevents + // the bug; so does Print of VecFromMask(d, Eq(v, zero)). + Store(VecFromMask(d, Eq(v, zero)), d, mask_lanes.get()); + HWY_ASSERT(!AllTrue(d, MaskFromVec(Load(d, mask_lanes.get())))); + + HWY_ASSERT(expected_all_false ^ AllFalse(d, Eq(v, zero))); + + lanes[i] = T(-1); + v = Load(d, lanes.get()); + HWY_ASSERT(!AllTrue(d, Eq(v, zero))); + HWY_ASSERT(expected_all_false ^ AllFalse(d, Eq(v, zero))); + + // Reset to all zero + lanes[i] = T(0); + v = Load(d, lanes.get()); + HWY_ASSERT(AllTrue(d, Eq(v, zero))); + HWY_ASSERT(!AllFalse(d, Eq(v, zero))); + } + } +}; + +HWY_NOINLINE void TestAllAllTrueFalse() { + ForAllTypes(ForPartialVectors<TestAllTrueFalse>()); +} + +class TestStoreMaskBits { + public: + template <class T, class D> + HWY_NOINLINE void operator()(T /*t*/, D /*d*/) { + // TODO(janwas): remove once implemented (cast or vse1) +#if HWY_TARGET != HWY_RVV + RandomState rng; + using TI = MakeSigned<T>; // For mask > 0 comparison + const Rebind<TI, D> di; + const size_t N = Lanes(di); + auto bool_lanes = AllocateAligned<TI>(N); + + const ScalableTag<uint8_t, -3> d_bits; + const size_t expected_num_bytes = (N + 7) / 8; + auto expected = AllocateAligned<uint8_t>(expected_num_bytes); + auto actual = AllocateAligned<uint8_t>(HWY_MAX(8, expected_num_bytes)); + + for (size_t rep = 0; rep < AdjustedReps(200); ++rep) { + // Generate random mask pattern. + for (size_t i = 0; i < N; ++i) { + bool_lanes[i] = static_cast<TI>((rng() & 1024) ? 1 : 0); + } + const auto bools = Load(di, bool_lanes.get()); + const auto mask = Gt(bools, Zero(di)); + + // Requires at least 8 bytes, ensured above. + const size_t bytes_written = StoreMaskBits(di, mask, actual.get()); + if (bytes_written != expected_num_bytes) { + fprintf(stderr, "%s expected %" PRIu64 " bytes, actual %" PRIu64 "\n", + TypeName(T(), N).c_str(), + static_cast<uint64_t>(expected_num_bytes), + static_cast<uint64_t>(bytes_written)); + + HWY_ASSERT(false); + } + +// TODO(janwas): enable after implemented +#if HWY_TARGET != HWY_RVV + // Requires at least 8 bytes, ensured above. + const auto mask2 = LoadMaskBits(di, actual.get()); + HWY_ASSERT_MASK_EQ(di, mask, mask2); +#endif + + memset(expected.get(), 0, expected_num_bytes); + for (size_t i = 0; i < N; ++i) { + expected[i / 8] = uint8_t(expected[i / 8] | (bool_lanes[i] << (i % 8))); + } + + size_t i = 0; + // Stored bits must match original mask + for (; i < N; ++i) { + const TI is_set = (actual[i / 8] & (1 << (i % 8))) ? 1 : 0; + if (is_set != bool_lanes[i]) { + fprintf(stderr, "%s lane %" PRIu64 ": expected %d, actual %d\n", + TypeName(T(), N).c_str(), static_cast<uint64_t>(i), + int(bool_lanes[i]), int(is_set)); + Print(di, "bools", bools, 0, N); + Print(d_bits, "expected bytes", Load(d_bits, expected.get()), 0, + expected_num_bytes); + Print(d_bits, "actual bytes", Load(d_bits, actual.get()), 0, + expected_num_bytes); + + HWY_ASSERT(false); + } + } + // Any partial bits in the last byte must be zero + for (; i < 8 * bytes_written; ++i) { + const int bit = (actual[i / 8] & (1 << (i % 8))); + if (bit != 0) { + fprintf(stderr, "%s: bit #%" PRIu64 " should be zero\n", + TypeName(T(), N).c_str(), static_cast<uint64_t>(i)); + Print(di, "bools", bools, 0, N); + Print(d_bits, "expected bytes", Load(d_bits, expected.get()), 0, + expected_num_bytes); + Print(d_bits, "actual bytes", Load(d_bits, actual.get()), 0, + expected_num_bytes); + + HWY_ASSERT(false); + } + } + } +#endif + } +}; + +HWY_NOINLINE void TestAllStoreMaskBits() { + ForAllTypes(ForPartialVectors<TestStoreMaskBits>()); +} + +struct TestCountTrue { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + using TI = MakeSigned<T>; // For mask > 0 comparison + const Rebind<TI, D> di; + const size_t N = Lanes(di); + auto bool_lanes = AllocateAligned<TI>(N); + memset(bool_lanes.get(), 0, N * sizeof(TI)); + + // For all combinations of zero/nonzero state of subset of lanes: + const size_t max_lanes = HWY_MIN(N, size_t(10)); + + for (size_t code = 0; code < (1ull << max_lanes); ++code) { + // Number of zeros written = number of mask lanes that are true. + size_t expected = 0; + for (size_t i = 0; i < max_lanes; ++i) { + const bool is_true = (code & (1ull << i)) != 0; + bool_lanes[i] = is_true ? TI(1) : TI(0); + expected += is_true; + } + + const auto mask = RebindMask(d, Gt(Load(di, bool_lanes.get()), Zero(di))); + const size_t actual = CountTrue(d, mask); + HWY_ASSERT_EQ(expected, actual); + } + } +}; + +HWY_NOINLINE void TestAllCountTrue() { + ForAllTypes(ForPartialVectors<TestCountTrue>()); +} + +struct TestFindFirstTrue { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + using TI = MakeSigned<T>; // For mask > 0 comparison + const Rebind<TI, D> di; + const size_t N = Lanes(di); + auto bool_lanes = AllocateAligned<TI>(N); + memset(bool_lanes.get(), 0, N * sizeof(TI)); + + // For all combinations of zero/nonzero state of subset of lanes: + const size_t max_lanes = HWY_MIN(N, size_t(10)); + + HWY_ASSERT_EQ(intptr_t(-1), FindFirstTrue(d, MaskFalse(d))); + HWY_ASSERT_EQ(intptr_t(0), FindFirstTrue(d, MaskTrue(d))); + + for (size_t code = 1; code < (1ull << max_lanes); ++code) { + for (size_t i = 0; i < max_lanes; ++i) { + bool_lanes[i] = (code & (1ull << i)) ? TI(1) : TI(0); + } + + const intptr_t expected = + static_cast<intptr_t>(Num0BitsBelowLS1Bit_Nonzero32(uint32_t(code))); + const auto mask = RebindMask(d, Gt(Load(di, bool_lanes.get()), Zero(di))); + const intptr_t actual = FindFirstTrue(d, mask); + HWY_ASSERT_EQ(expected, actual); + } + } +}; + +HWY_NOINLINE void TestAllFindFirstTrue() { + ForAllTypes(ForPartialVectors<TestFindFirstTrue>()); +} + +struct TestLogicalMask { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto m0 = MaskFalse(d); + const auto m_all = MaskTrue(d); + + using TI = MakeSigned<T>; // For mask > 0 comparison + const Rebind<TI, D> di; + const size_t N = Lanes(di); + auto bool_lanes = AllocateAligned<TI>(N); + memset(bool_lanes.get(), 0, N * sizeof(TI)); + + HWY_ASSERT_MASK_EQ(d, m0, Not(m_all)); + HWY_ASSERT_MASK_EQ(d, m_all, Not(m0)); + + // For all combinations of zero/nonzero state of subset of lanes: + const size_t max_lanes = HWY_MIN(N, size_t(6)); + for (size_t code = 0; code < (1ull << max_lanes); ++code) { + for (size_t i = 0; i < max_lanes; ++i) { + bool_lanes[i] = (code & (1ull << i)) ? TI(1) : TI(0); + } + + const auto m = RebindMask(d, Gt(Load(di, bool_lanes.get()), Zero(di))); + + HWY_ASSERT_MASK_EQ(d, m0, Xor(m, m)); + HWY_ASSERT_MASK_EQ(d, m0, AndNot(m, m)); + HWY_ASSERT_MASK_EQ(d, m0, AndNot(m_all, m)); + + HWY_ASSERT_MASK_EQ(d, m, Or(m, m)); + HWY_ASSERT_MASK_EQ(d, m, Or(m0, m)); + HWY_ASSERT_MASK_EQ(d, m, Or(m, m0)); + HWY_ASSERT_MASK_EQ(d, m, Xor(m0, m)); + HWY_ASSERT_MASK_EQ(d, m, Xor(m, m0)); + HWY_ASSERT_MASK_EQ(d, m, And(m, m)); + HWY_ASSERT_MASK_EQ(d, m, And(m_all, m)); + HWY_ASSERT_MASK_EQ(d, m, And(m, m_all)); + HWY_ASSERT_MASK_EQ(d, m, AndNot(m0, m)); + } + } +}; + +HWY_NOINLINE void TestAllLogicalMask() { + ForAllTypes(ForPartialVectors<TestLogicalMask>()); +} +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE + +namespace hwy { +HWY_BEFORE_TEST(HwyMaskTest); +HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllFromVec); +HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllFirstN); +HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllIfThenElse); +HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllMaskVec); +HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllMaskedLoad); +HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllAllTrueFalse); +HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllStoreMaskBits); +HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllCountTrue); +HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllFindFirstTrue); +HWY_EXPORT_AND_TEST_P(HwyMaskTest, TestAllLogicalMask); +} // namespace hwy + +// Ought not to be necessary, but without this, no tests run on RVV. +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#endif diff --git a/third_party/highway/hwy/tests/memory_test.cc b/third_party/highway/hwy/tests/memory_test.cc new file mode 100644 index 0000000..3f72809 --- /dev/null +++ b/third_party/highway/hwy/tests/memory_test.cc @@ -0,0 +1,421 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Ensure incompabilities with Windows macros (e.g. #define StoreFence) are +// detected. Must come before Highway headers. +#if defined(_WIN32) || defined(_WIN64) +#include <windows.h> +#endif + +#include <stddef.h> +#include <stdint.h> + +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "tests/memory_test.cc" +#include "hwy/cache_control.h" +#include "hwy/foreach_target.h" +#include "hwy/highway.h" +#include "hwy/tests/test_util-inl.h" + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +struct TestLoadStore { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const size_t N = Lanes(d); + const auto hi = Iota(d, 1 + N); + const auto lo = Iota(d, 1); + auto lanes = AllocateAligned<T>(2 * N); + Store(hi, d, &lanes[N]); + Store(lo, d, &lanes[0]); + + // Aligned load + const auto lo2 = Load(d, &lanes[0]); + HWY_ASSERT_VEC_EQ(d, lo2, lo); + + // Aligned store + auto lanes2 = AllocateAligned<T>(2 * N); + Store(lo2, d, &lanes2[0]); + Store(hi, d, &lanes2[N]); + for (size_t i = 0; i < 2 * N; ++i) { + HWY_ASSERT_EQ(lanes[i], lanes2[i]); + } + + // Unaligned load + const auto vu = LoadU(d, &lanes[1]); + auto lanes3 = AllocateAligned<T>(N); + Store(vu, d, lanes3.get()); + for (size_t i = 0; i < N; ++i) { + HWY_ASSERT_EQ(T(i + 2), lanes3[i]); + } + + // Unaligned store + StoreU(lo2, d, &lanes2[N / 2]); + size_t i = 0; + for (; i < N / 2; ++i) { + HWY_ASSERT_EQ(lanes[i], lanes2[i]); + } + for (; i < 3 * N / 2; ++i) { + HWY_ASSERT_EQ(T(i - N / 2 + 1), lanes2[i]); + } + // Subsequent values remain unchanged. + for (; i < 2 * N; ++i) { + HWY_ASSERT_EQ(T(i + 1), lanes2[i]); + } + } +}; + +HWY_NOINLINE void TestAllLoadStore() { + ForAllTypes(ForPartialVectors<TestLoadStore>()); +} + +struct TestStoreInterleaved3 { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { +// TODO(janwas): restore once segment intrinsics are available +#if HWY_TARGET != HWY_RVV + const size_t N = Lanes(d); + + RandomState rng; + + // Data to be interleaved + auto bytes = AllocateAligned<uint8_t>(3 * N); + for (size_t i = 0; i < 3 * N; ++i) { + bytes[i] = static_cast<uint8_t>(Random32(&rng) & 0xFF); + } + const auto in0 = Load(d, &bytes[0 * N]); + const auto in1 = Load(d, &bytes[1 * N]); + const auto in2 = Load(d, &bytes[2 * N]); + + // Interleave here, ensure vector results match scalar + auto expected = AllocateAligned<T>(4 * N); + auto actual_aligned = AllocateAligned<T>(4 * N + 1); + T* actual = actual_aligned.get() + 1; + + for (size_t rep = 0; rep < 100; ++rep) { + for (size_t i = 0; i < N; ++i) { + expected[3 * i + 0] = bytes[0 * N + i]; + expected[3 * i + 1] = bytes[1 * N + i]; + expected[3 * i + 2] = bytes[2 * N + i]; + // Ensure we do not write more than 3*N bytes + expected[3 * N + i] = actual[3 * N + i] = 0; + } + StoreInterleaved3(in0, in1, in2, d, actual); + size_t pos = 0; + if (!BytesEqual(expected.get(), actual, 4 * N, &pos)) { + Print(d, "in0", in0, pos / 3); + Print(d, "in1", in1, pos / 3); + Print(d, "in2", in2, pos / 3); + const size_t i = pos - pos % 3; + fprintf(stderr, "interleaved %d %d %d %d %d %d\n", actual[i], + actual[i + 1], actual[i + 2], actual[i + 3], actual[i + 4], + actual[i + 5]); + HWY_ASSERT(false); + } + } +#else + (void)d; +#endif + } +}; + +HWY_NOINLINE void TestAllStoreInterleaved3() { +#if HWY_TARGET == HWY_RVV + // Segments are limited to 8 registers, so we can only go up to LMUL=2. + const ForExtendableVectors<TestStoreInterleaved3, 4> test; +#else + const ForPartialVectors<TestStoreInterleaved3> test; +#endif + test(uint8_t()); +} + +struct TestStoreInterleaved4 { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { +// TODO(janwas): restore once segment intrinsics are available +#if HWY_TARGET != HWY_RVV + const size_t N = Lanes(d); + + RandomState rng; + + // Data to be interleaved + auto bytes = AllocateAligned<uint8_t>(4 * N); + for (size_t i = 0; i < 4 * N; ++i) { + bytes[i] = static_cast<uint8_t>(Random32(&rng) & 0xFF); + } + const auto in0 = Load(d, &bytes[0 * N]); + const auto in1 = Load(d, &bytes[1 * N]); + const auto in2 = Load(d, &bytes[2 * N]); + const auto in3 = Load(d, &bytes[3 * N]); + + // Interleave here, ensure vector results match scalar + auto expected = AllocateAligned<T>(5 * N); + auto actual_aligned = AllocateAligned<T>(5 * N + 1); + T* actual = actual_aligned.get() + 1; + + for (size_t rep = 0; rep < 100; ++rep) { + for (size_t i = 0; i < N; ++i) { + expected[4 * i + 0] = bytes[0 * N + i]; + expected[4 * i + 1] = bytes[1 * N + i]; + expected[4 * i + 2] = bytes[2 * N + i]; + expected[4 * i + 3] = bytes[3 * N + i]; + // Ensure we do not write more than 4*N bytes + expected[4 * N + i] = actual[4 * N + i] = 0; + } + StoreInterleaved4(in0, in1, in2, in3, d, actual); + size_t pos = 0; + if (!BytesEqual(expected.get(), actual, 5 * N, &pos)) { + Print(d, "in0", in0, pos / 4); + Print(d, "in1", in1, pos / 4); + Print(d, "in2", in2, pos / 4); + Print(d, "in3", in3, pos / 4); + const size_t i = pos; + fprintf(stderr, "interleaved %d %d %d %d %d %d %d %d\n", actual[i], + actual[i + 1], actual[i + 2], actual[i + 3], actual[i + 4], + actual[i + 5], actual[i + 6], actual[i + 7]); + HWY_ASSERT(false); + } + } +#else + (void)d; +#endif + } +}; + +HWY_NOINLINE void TestAllStoreInterleaved4() { +#if HWY_TARGET == HWY_RVV + // Segments are limited to 8 registers, so we can only go up to LMUL=2. + const ForExtendableVectors<TestStoreInterleaved4, 4> test; +#else + const ForPartialVectors<TestStoreInterleaved4> test; +#endif + test(uint8_t()); +} + +struct TestLoadDup128 { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + // Scalar does not define LoadDup128. +#if HWY_TARGET != HWY_SCALAR || HWY_IDE + constexpr size_t N128 = 16 / sizeof(T); + alignas(16) T lanes[N128]; + for (size_t i = 0; i < N128; ++i) { + lanes[i] = static_cast<T>(1 + i); + } + + const size_t N = Lanes(d); + auto expected = AllocateAligned<T>(N); + for (size_t i = 0; i < N; ++i) { + expected[i] = static_cast<T>(i % N128 + 1); + } + + HWY_ASSERT_VEC_EQ(d, expected.get(), LoadDup128(d, lanes)); +#else + (void)d; +#endif + } +}; + +HWY_NOINLINE void TestAllLoadDup128() { + ForAllTypes(ForGE128Vectors<TestLoadDup128>()); +} + +struct TestStream { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v = Iota(d, T(1)); + const size_t affected_bytes = + (Lanes(d) * sizeof(T) + HWY_STREAM_MULTIPLE - 1) & + ~size_t(HWY_STREAM_MULTIPLE - 1); + const size_t affected_lanes = affected_bytes / sizeof(T); + auto out = AllocateAligned<T>(2 * affected_lanes); + std::fill(out.get(), out.get() + 2 * affected_lanes, T(0)); + + Stream(v, d, out.get()); + StoreFence(); + const auto actual = Load(d, out.get()); + HWY_ASSERT_VEC_EQ(d, v, actual); + // Ensure Stream didn't modify more memory than expected + for (size_t i = affected_lanes; i < 2 * affected_lanes; ++i) { + HWY_ASSERT_EQ(T(0), out[i]); + } + } +}; + +HWY_NOINLINE void TestAllStream() { + const ForPartialVectors<TestStream> test; + // No u8,u16. + test(uint32_t()); + test(uint64_t()); + // No i8,i16. + test(int32_t()); + test(int64_t()); + ForFloatTypes(test); +} + +// Assumes little-endian byte order! +struct TestScatter { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + using Offset = MakeSigned<T>; + + const size_t N = Lanes(d); + const size_t range = 4 * N; // number of items to scatter + const size_t max_bytes = range * sizeof(T); // upper bound on offset + + RandomState rng; + + // Data to be scattered + auto bytes = AllocateAligned<uint8_t>(max_bytes); + for (size_t i = 0; i < max_bytes; ++i) { + bytes[i] = static_cast<uint8_t>(Random32(&rng) & 0xFF); + } + const auto data = Load(d, reinterpret_cast<const T*>(bytes.get())); + + // Scatter into these regions, ensure vector results match scalar + auto expected = AllocateAligned<T>(range); + auto actual = AllocateAligned<T>(range); + + const Rebind<Offset, D> d_offsets; + auto offsets = AllocateAligned<Offset>(N); // or indices + + for (size_t rep = 0; rep < 100; ++rep) { + // Byte offsets + std::fill(expected.get(), expected.get() + range, T(0)); + std::fill(actual.get(), actual.get() + range, T(0)); + for (size_t i = 0; i < N; ++i) { + // Must be aligned + offsets[i] = static_cast<Offset>((Random32(&rng) % range) * sizeof(T)); + CopyBytes<sizeof(T)>( + bytes.get() + i * sizeof(T), + reinterpret_cast<uint8_t*>(expected.get()) + offsets[i]); + } + const auto voffsets = Load(d_offsets, offsets.get()); + ScatterOffset(data, d, actual.get(), voffsets); + if (!BytesEqual(expected.get(), actual.get(), max_bytes)) { + Print(d, "Data", data); + Print(d_offsets, "Offsets", voffsets); + HWY_ASSERT(false); + } + + // Indices + std::fill(expected.get(), expected.get() + range, T(0)); + std::fill(actual.get(), actual.get() + range, T(0)); + for (size_t i = 0; i < N; ++i) { + offsets[i] = static_cast<Offset>(Random32(&rng) % range); + CopyBytes<sizeof(T)>(bytes.get() + i * sizeof(T), + &expected[size_t(offsets[i])]); + } + const auto vindices = Load(d_offsets, offsets.get()); + ScatterIndex(data, d, actual.get(), vindices); + if (!BytesEqual(expected.get(), actual.get(), max_bytes)) { + Print(d, "Data", data); + Print(d_offsets, "Indices", vindices); + HWY_ASSERT(false); + } + } + } +}; + +HWY_NOINLINE void TestAllScatter() { + ForUIF3264(ForPartialVectors<TestScatter>()); +} + +struct TestGather { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + using Offset = MakeSigned<T>; + + const size_t N = Lanes(d); + const size_t range = 4 * N; // number of items to gather + const size_t max_bytes = range * sizeof(T); // upper bound on offset + + RandomState rng; + + // Data to be gathered from + auto bytes = AllocateAligned<uint8_t>(max_bytes); + for (size_t i = 0; i < max_bytes; ++i) { + bytes[i] = static_cast<uint8_t>(Random32(&rng) & 0xFF); + } + + auto expected = AllocateAligned<T>(N); + auto offsets = AllocateAligned<Offset>(N); + auto indices = AllocateAligned<Offset>(N); + + for (size_t rep = 0; rep < 100; ++rep) { + // Offsets + for (size_t i = 0; i < N; ++i) { + // Must be aligned + offsets[i] = static_cast<Offset>((Random32(&rng) % range) * sizeof(T)); + CopyBytes<sizeof(T)>(bytes.get() + offsets[i], &expected[i]); + } + + const Rebind<Offset, D> d_offset; + const T* base = reinterpret_cast<const T*>(bytes.get()); + auto actual = GatherOffset(d, base, Load(d_offset, offsets.get())); + HWY_ASSERT_VEC_EQ(d, expected.get(), actual); + + // Indices + for (size_t i = 0; i < N; ++i) { + indices[i] = + static_cast<Offset>(Random32(&rng) % (max_bytes / sizeof(T))); + CopyBytes<sizeof(T)>(base + indices[i], &expected[i]); + } + actual = GatherIndex(d, base, Load(d_offset, indices.get())); + HWY_ASSERT_VEC_EQ(d, expected.get(), actual); + } + } +}; + +HWY_NOINLINE void TestAllGather() { + ForUIF3264(ForPartialVectors<TestGather>()); +} + +HWY_NOINLINE void TestAllCache() { + LoadFence(); + StoreFence(); + int test = 0; + Prefetch(&test); + FlushCacheline(&test); + Pause(); +} + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE + +namespace hwy { +HWY_BEFORE_TEST(HwyMemoryTest); +HWY_EXPORT_AND_TEST_P(HwyMemoryTest, TestAllLoadStore); +HWY_EXPORT_AND_TEST_P(HwyMemoryTest, TestAllStoreInterleaved3); +HWY_EXPORT_AND_TEST_P(HwyMemoryTest, TestAllStoreInterleaved4); +HWY_EXPORT_AND_TEST_P(HwyMemoryTest, TestAllLoadDup128); +HWY_EXPORT_AND_TEST_P(HwyMemoryTest, TestAllStream); +HWY_EXPORT_AND_TEST_P(HwyMemoryTest, TestAllScatter); +HWY_EXPORT_AND_TEST_P(HwyMemoryTest, TestAllGather); +HWY_EXPORT_AND_TEST_P(HwyMemoryTest, TestAllCache); +} // namespace hwy + +// Ought not to be necessary, but without this, no tests run on RVV. +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#endif diff --git a/third_party/highway/hwy/tests/swizzle_test.cc b/third_party/highway/hwy/tests/swizzle_test.cc new file mode 100644 index 0000000..ea14514 --- /dev/null +++ b/third_party/highway/hwy/tests/swizzle_test.cc @@ -0,0 +1,499 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <inttypes.h> +#include <stddef.h> +#include <stdint.h> +#include <string.h> + +#include <array> // IWYU pragma: keep + +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "tests/swizzle_test.cc" +#include "hwy/foreach_target.h" +#include "hwy/highway.h" +#include "hwy/tests/test_util-inl.h" + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +// For regenerating tables used in the implementation +#define HWY_PRINT_TABLES 0 + +struct TestGetLane { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const auto v = Iota(d, T(1)); + HWY_ASSERT_EQ(T(1), GetLane(v)); + } +}; + +HWY_NOINLINE void TestAllGetLane() { + ForAllTypes(ForPartialVectors<TestGetLane>()); +} + +struct TestOddEven { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const size_t N = Lanes(d); + const auto even = Iota(d, 1); + const auto odd = Iota(d, 1 + N); + auto expected = AllocateAligned<T>(N); + for (size_t i = 0; i < N; ++i) { + expected[i] = static_cast<T>(1 + i + ((i & 1) ? N : 0)); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), OddEven(odd, even)); + } +}; + +HWY_NOINLINE void TestAllOddEven() { + ForAllTypes(ForShrinkableVectors<TestOddEven>()); +} + +struct TestOddEvenBlocks { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const size_t N = Lanes(d); + const auto even = Iota(d, 1); + const auto odd = Iota(d, 1 + N); + auto expected = AllocateAligned<T>(N); + for (size_t i = 0; i < N; ++i) { + const size_t idx_block = i / (16 / sizeof(T)); + expected[i] = static_cast<T>(1 + i + ((idx_block & 1) ? N : 0)); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), OddEvenBlocks(odd, even)); + } +}; + +HWY_NOINLINE void TestAllOddEvenBlocks() { + ForAllTypes(ForShrinkableVectors<TestOddEvenBlocks>()); +} + +struct TestSwapAdjacentBlocks { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const size_t N = Lanes(d); + constexpr size_t kLanesPerBlock = 16 / sizeof(T); + if (N < 2 * kLanesPerBlock) return; + const auto vi = Iota(d, 1); + auto expected = AllocateAligned<T>(N); + for (size_t i = 0; i < N; ++i) { + const size_t idx_block = i / kLanesPerBlock; + const size_t base = (idx_block ^ 1) * kLanesPerBlock; + const size_t mod = i % kLanesPerBlock; + expected[i] = static_cast<T>(1 + base + mod); + } + HWY_ASSERT_VEC_EQ(d, expected.get(), SwapAdjacentBlocks(vi)); + } +}; + +HWY_NOINLINE void TestAllSwapAdjacentBlocks() { + ForAllTypes(ForPartialVectors<TestSwapAdjacentBlocks>()); +} + +struct TestTableLookupLanes { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + using TI = MakeSigned<T>; +#if HWY_TARGET != HWY_SCALAR + const size_t N = Lanes(d); + const Rebind<TI, D> di; + auto idx = AllocateAligned<TI>(N); + memset(idx.get(), 0, N * sizeof(TI)); + auto expected = AllocateAligned<T>(N); + const auto v = Iota(d, 1); + + if (N <= 8) { // Test all permutations + for (size_t i0 = 0; i0 < N; ++i0) { + idx[0] = static_cast<TI>(i0); + + for (size_t i1 = 0; i1 < N; ++i1) { + if (N >= 2) idx[1] = static_cast<TI>(i1); + for (size_t i2 = 0; i2 < N; ++i2) { + if (N >= 4) idx[2] = static_cast<TI>(i2); + for (size_t i3 = 0; i3 < N; ++i3) { + if (N >= 4) idx[3] = static_cast<TI>(i3); + + for (size_t i = 0; i < N; ++i) { + expected[i] = static_cast<T>(idx[i] + 1); // == v[idx[i]] + } + + const auto opaque1 = IndicesFromVec(d, Load(di, idx.get())); + const auto actual1 = TableLookupLanes(v, opaque1); + HWY_ASSERT_VEC_EQ(d, expected.get(), actual1); + + const auto opaque2 = SetTableIndices(d, idx.get()); + const auto actual2 = TableLookupLanes(v, opaque2); + HWY_ASSERT_VEC_EQ(d, expected.get(), actual2); + } + } + } + } + } else { + // Too many permutations to test exhaustively; choose one with repeated + // and cross-block indices and ensure indices do not exceed #lanes. + // For larger vectors, upper lanes will be zero. + HWY_ALIGN TI idx_source[16] = {1, 3, 2, 2, 8, 1, 7, 6, + 15, 14, 14, 15, 4, 9, 8, 5}; + for (size_t i = 0; i < N; ++i) { + idx[i] = (i < 16) ? idx_source[i] : 0; + // Avoid undefined results / asan error for scalar by capping indices. + if (idx[i] >= static_cast<TI>(N)) { + idx[i] = static_cast<TI>(N - 1); + } + expected[i] = static_cast<T>(idx[i] + 1); // == v[idx[i]] + } + + const auto opaque1 = IndicesFromVec(d, Load(di, idx.get())); + const auto actual1 = TableLookupLanes(v, opaque1); + HWY_ASSERT_VEC_EQ(d, expected.get(), actual1); + + const auto opaque2 = SetTableIndices(d, idx.get()); + const auto actual2 = TableLookupLanes(v, opaque2); + HWY_ASSERT_VEC_EQ(d, expected.get(), actual2); + } +#else + const TI index = 0; + const auto v = Set(d, 1); + const auto opaque1 = SetTableIndices(d, &index); + HWY_ASSERT_VEC_EQ(d, v, TableLookupLanes(v, opaque1)); + const auto opaque2 = IndicesFromVec(d, Zero(d)); + HWY_ASSERT_VEC_EQ(d, v, TableLookupLanes(v, opaque2)); +#endif + } +}; + +HWY_NOINLINE void TestAllTableLookupLanes() { + ForUIF3264(ForPartialVectors<TestTableLookupLanes>()); +} + +struct TestReverse { + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + const size_t N = Lanes(d); + const RebindToUnsigned<D> du; // Iota does not support float16_t. + const auto v = BitCast(d, Iota(du, 1)); + auto expected = AllocateAligned<T>(N); + + // Can't set float16_t value directly, need to permute in memory. + auto copy = AllocateAligned<T>(N); + Store(v, d, copy.get()); + for (size_t i = 0; i < N; ++i) { + expected[i] = copy[N - 1 - i]; + } + HWY_ASSERT_VEC_EQ(d, expected.get(), Reverse(d, v)); + } +}; + +HWY_NOINLINE void TestAllReverse() { + // 8-bit is not supported because Risc-V uses rgather of Lanes - Iota, + // which requires 16 bits. + ForUIF163264(ForPartialVectors<TestReverse>()); +} + +class TestCompress { + template <typename T, typename TI, size_t N> + void CheckStored(Simd<T, N> d, Simd<TI, N> di, size_t expected_pos, + size_t actual_pos, const AlignedFreeUniquePtr<T[]>& in, + const AlignedFreeUniquePtr<TI[]>& mask_lanes, + const AlignedFreeUniquePtr<T[]>& expected, const T* actual_u, + int line) { + if (expected_pos != actual_pos) { + hwy::Abort(__FILE__, line, + "Size mismatch for %s: expected %" PRIu64 ", actual %" PRIu64 "\n", + TypeName(T(), N).c_str(), static_cast<uint64_t>(expected_pos), static_cast<uint64_t>(actual_pos)); + } + // Upper lanes are undefined. Modified from AssertVecEqual. + for (size_t i = 0; i < expected_pos; ++i) { + if (!IsEqual(expected[i], actual_u[i])) { + fprintf(stderr, + "Mismatch at i=%" PRIu64 " of %" PRIu64 ", line %d:\n\n", + static_cast<uint64_t>(i), static_cast<uint64_t>(expected_pos), + line); + Print(di, "mask", Load(di, mask_lanes.get()), 0, N); + Print(d, "in", Load(d, in.get()), 0, N); + Print(d, "expect", Load(d, expected.get()), 0, N); + Print(d, "actual", Load(d, actual_u), 0, N); + HWY_ASSERT(false); + } + } + } + + public: + template <class T, class D> + HWY_NOINLINE void operator()(T /*unused*/, D d) { + RandomState rng; + + using TI = MakeSigned<T>; // For mask > 0 comparison + const Rebind<TI, D> di; + const size_t N = Lanes(d); + + const T zero{0}; + + for (int frac : {0, 2, 3}) { + // For CompressStore + const size_t misalign = static_cast<size_t>(frac) * N / 4; + + auto in_lanes = AllocateAligned<T>(N); + auto mask_lanes = AllocateAligned<TI>(N); + auto expected = AllocateAligned<T>(N); + auto actual_a = AllocateAligned<T>(misalign + N); + T* actual_u = actual_a.get() + misalign; + auto bits = AllocateAligned<uint8_t>(HWY_MAX(8, (N + 7) / 8)); + + // Each lane should have a chance of having mask=true. + for (size_t rep = 0; rep < AdjustedReps(200); ++rep) { + size_t expected_pos = 0; + for (size_t i = 0; i < N; ++i) { + const uint64_t bits = Random32(&rng); + in_lanes[i] = T(); // cannot initialize float16_t directly. + CopyBytes<sizeof(T)>(&bits, &in_lanes[i]); + mask_lanes[i] = (Random32(&rng) & 1024) ? TI(1) : TI(0); + if (mask_lanes[i] > 0) { + expected[expected_pos++] = in_lanes[i]; + } + } + + const auto in = Load(d, in_lanes.get()); + const auto mask = + RebindMask(d, Gt(Load(di, mask_lanes.get()), Zero(di))); + StoreMaskBits(d, mask, bits.get()); + + // Compress + memset(actual_u, 0, N * sizeof(T)); + StoreU(Compress(in, mask), d, actual_u); + CheckStored(d, di, expected_pos, expected_pos, in_lanes, mask_lanes, + expected, actual_u, __LINE__); + + // CompressStore + memset(actual_u, 0, N * sizeof(T)); + const size_t size1 = CompressStore(in, mask, d, actual_u); + CheckStored(d, di, expected_pos, size1, in_lanes, mask_lanes, expected, + actual_u, __LINE__); + + // CompressBlendedStore + memset(actual_u, 0, N * sizeof(T)); + const size_t size2 = CompressBlendedStore(in, mask, d, actual_u); + CheckStored(d, di, expected_pos, size2, in_lanes, mask_lanes, expected, + actual_u, __LINE__); + // Subsequent lanes are untouched. + for (size_t i = size2; i < N; ++i) { + HWY_ASSERT_EQ(zero, actual_u[i]); + } + + // TODO(janwas): remove once implemented (cast or vse1) +#if HWY_TARGET != HWY_RVV + // CompressBits + memset(actual_u, 0, N * sizeof(T)); + StoreU(CompressBits(in, bits.get()), d, actual_u); + CheckStored(d, di, expected_pos, expected_pos, in_lanes, mask_lanes, + expected, actual_u, __LINE__); + + // CompressBitsStore + memset(actual_u, 0, N * sizeof(T)); + const size_t size3 = CompressBitsStore(in, bits.get(), d, actual_u); + CheckStored(d, di, expected_pos, size3, in_lanes, mask_lanes, expected, + actual_u, __LINE__); +#endif + } // rep + } // frac + } // operator() +}; + +#if HWY_PRINT_TABLES +namespace detail { // for code folding +void PrintCompress16x8Tables() { + printf("======================================= 16x8\n"); + constexpr size_t N = 8; // 128-bit SIMD + for (uint64_t code = 0; code < 1ull << N; ++code) { + std::array<uint8_t, N> indices{0}; + size_t pos = 0; + for (size_t i = 0; i < N; ++i) { + if (code & (1ull << i)) { + indices[pos++] = i; + } + } + + // Doubled (for converting lane to byte indices) + for (size_t i = 0; i < N; ++i) { + printf("%d,", 2 * indices[i]); + } + } + printf("\n"); +} + +// Similar to the above, but uses native 16-bit shuffle instead of bytes. +void PrintCompress16x16HalfTables() { + printf("======================================= 16x16Half\n"); + constexpr size_t N = 8; + for (uint64_t code = 0; code < 1ull << N; ++code) { + std::array<uint8_t, N> indices{0}; + size_t pos = 0; + for (size_t i = 0; i < N; ++i) { + if (code & (1ull << i)) { + indices[pos++] = i; + } + } + + for (size_t i = 0; i < N; ++i) { + printf("%d,", indices[i]); + } + printf("\n"); + } + printf("\n"); +} + +// Compressed to nibbles +void PrintCompress32x8Tables() { + printf("======================================= 32x8\n"); + constexpr size_t N = 8; // AVX2 + for (uint64_t code = 0; code < 1ull << N; ++code) { + std::array<uint32_t, N> indices{0}; + size_t pos = 0; + for (size_t i = 0; i < N; ++i) { + if (code & (1ull << i)) { + indices[pos++] = i; + } + } + + // Convert to nibbles + uint64_t packed = 0; + for (size_t i = 0; i < N; ++i) { + HWY_ASSERT(indices[i] < 16); + packed += indices[i] << (i * 4); + } + + HWY_ASSERT(packed < (1ull << 32)); + printf("0x%08x,", static_cast<uint32_t>(packed)); + } + printf("\n"); +} + +// Pairs of 32-bit lane indices +void PrintCompress64x4Tables() { + printf("======================================= 64x4\n"); + constexpr size_t N = 4; // AVX2 + for (uint64_t code = 0; code < 1ull << N; ++code) { + std::array<uint32_t, N> indices{0}; + size_t pos = 0; + for (size_t i = 0; i < N; ++i) { + if (code & (1ull << i)) { + indices[pos++] = i; + } + } + + for (size_t i = 0; i < N; ++i) { + printf("%d,%d,", 2 * indices[i], 2 * indices[i] + 1); + } + } + printf("\n"); +} + +// 4-tuple of byte indices +void PrintCompress32x4Tables() { + printf("======================================= 32x4\n"); + using T = uint32_t; + constexpr size_t N = 4; // SSE4 + for (uint64_t code = 0; code < 1ull << N; ++code) { + std::array<uint32_t, N> indices{0}; + size_t pos = 0; + for (size_t i = 0; i < N; ++i) { + if (code & (1ull << i)) { + indices[pos++] = i; + } + } + + for (size_t i = 0; i < N; ++i) { + for (size_t idx_byte = 0; idx_byte < sizeof(T); ++idx_byte) { + printf("%" PRIu64 ",", + static_cast<uint64_t>(sizeof(T) * indices[i] + idx_byte)); + } + } + } + printf("\n"); +} + +// 8-tuple of byte indices +void PrintCompress64x2Tables() { + printf("======================================= 64x2\n"); + using T = uint64_t; + constexpr size_t N = 2; // SSE4 + for (uint64_t code = 0; code < 1ull << N; ++code) { + std::array<uint32_t, N> indices{0}; + size_t pos = 0; + for (size_t i = 0; i < N; ++i) { + if (code & (1ull << i)) { + indices[pos++] = i; + } + } + + for (size_t i = 0; i < N; ++i) { + for (size_t idx_byte = 0; idx_byte < sizeof(T); ++idx_byte) { + printf("%" PRIu64 ",", + static_cast<uint64_t>(sizeof(T) * indices[i] + idx_byte)); + } + } + } + printf("\n"); +} +} // namespace detail +#endif // HWY_PRINT_TABLES + +HWY_NOINLINE void TestAllCompress() { +#if HWY_PRINT_TABLES + detail::PrintCompress32x8Tables(); + detail::PrintCompress64x4Tables(); + detail::PrintCompress32x4Tables(); + detail::PrintCompress64x2Tables(); + detail::PrintCompress16x8Tables(); + detail::PrintCompress16x16HalfTables(); +#endif + + const ForPartialVectors<TestCompress> test; + + test(uint16_t()); + test(int16_t()); +#if HWY_CAP_FLOAT16 + test(float16_t()); +#endif + + ForUIF3264(test); +} + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE + +namespace hwy { +HWY_BEFORE_TEST(HwySwizzleTest); +HWY_EXPORT_AND_TEST_P(HwySwizzleTest, TestAllGetLane); +HWY_EXPORT_AND_TEST_P(HwySwizzleTest, TestAllOddEven); +HWY_EXPORT_AND_TEST_P(HwySwizzleTest, TestAllOddEvenBlocks); +HWY_EXPORT_AND_TEST_P(HwySwizzleTest, TestAllSwapAdjacentBlocks); +HWY_EXPORT_AND_TEST_P(HwySwizzleTest, TestAllTableLookupLanes); +HWY_EXPORT_AND_TEST_P(HwySwizzleTest, TestAllReverse); +HWY_EXPORT_AND_TEST_P(HwySwizzleTest, TestAllCompress); +} // namespace hwy + +// Ought not to be necessary, but without this, no tests run on RVV. +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#endif diff --git a/third_party/highway/hwy/tests/test_util-inl.h b/third_party/highway/hwy/tests/test_util-inl.h new file mode 100644 index 0000000..8a82267 --- /dev/null +++ b/third_party/highway/hwy/tests/test_util-inl.h @@ -0,0 +1,440 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Target-specific helper functions for use by *_test.cc. + +#include <inttypes.h> +#include <stdint.h> + +#include "hwy/base.h" +#include "hwy/tests/hwy_gtest.h" +#include "hwy/tests/test_util.h" + +// Per-target include guard +#if defined(HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_) == \ + defined(HWY_TARGET_TOGGLE) +#ifdef HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_ +#undef HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_ +#else +#define HIGHWAY_HWY_TESTS_TEST_UTIL_INL_H_ +#endif + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +template <typename T, HWY_IF_LANE_SIZE(T, 1)> +HWY_NOINLINE void PrintValue(T value) { + uint8_t byte; + CopyBytes<1>(&value, &byte); // endian-safe: we ensured sizeof(T)=1. + fprintf(stderr, "0x%02X,", byte); +} + +#if HWY_CAP_FLOAT16 +HWY_NOINLINE void PrintValue(float16_t value) { + uint16_t bits; + CopyBytes<2>(&value, &bits); + fprintf(stderr, "0x%02X,", bits); +} +#endif + + + +template <typename T, HWY_IF_NOT_LANE_SIZE(T, 1)> +HWY_NOINLINE void PrintValue(T value) { + fprintf(stderr, "%g,", double(value)); +} + +// Prints lanes around `lane`, in memory order. +template <class D, class V = Vec<D>> +void Print(const D d, const char* caption, VecArg<V> v, size_t lane_u = 0, + size_t max_lanes = 7) { + using T = TFromD<D>; + const size_t N = Lanes(d); + auto lanes = AllocateAligned<T>(N); + Store(v, d, lanes.get()); + + const auto info = hwy::detail::MakeTypeInfo<T>(); + hwy::detail::PrintArray(info, caption, lanes.get(), N, lane_u, max_lanes); +} + +// Compare expected vector to vector. +template <class D, typename T = TFromD<D>, class V = Vec<D>> +void AssertVecEqual(D d, const T* expected, VecArg<V> actual, + const char* filename, const int line) { + const size_t N = Lanes(d); + auto actual_lanes = AllocateAligned<T>(N); + Store(actual, d, actual_lanes.get()); + + const auto info = hwy::detail::MakeTypeInfo<T>(); + const char* target_name = hwy::TargetName(HWY_TARGET); + hwy::detail::AssertArrayEqual(info, expected, actual_lanes.get(), N, + target_name, filename, line); +} + +// Compare expected lanes to vector. +template <class D, typename T = TFromD<D>, class V = Vec<D>> +HWY_NOINLINE void AssertVecEqual(D d, VecArg<V> expected, VecArg<V> actual, + const char* filename, int line) { + auto expected_lanes = AllocateAligned<T>(Lanes(d)); + Store(expected, d, expected_lanes.get()); + AssertVecEqual(d, expected_lanes.get(), actual, filename, line); +} + +// Only checks the valid mask elements (those whose index < Lanes(d)). +template <class D> +HWY_NOINLINE void AssertMaskEqual(D d, VecArg<Mask<D>> a, VecArg<Mask<D>> b, + const char* filename, int line) { + AssertVecEqual(d, VecFromMask(d, a), VecFromMask(d, b), filename, line); + + const char* target_name = hwy::TargetName(HWY_TARGET); + AssertEqual(CountTrue(d, a), CountTrue(d, b), target_name, filename, line); + AssertEqual(AllTrue(d, a), AllTrue(d, b), target_name, filename, line); + AssertEqual(AllFalse(d, a), AllFalse(d, b), target_name, filename, line); + + // TODO(janwas): remove RVV once implemented (cast or vse1) +#if HWY_TARGET != HWY_RVV && HWY_TARGET != HWY_SCALAR + const size_t N = Lanes(d); + const Repartition<uint8_t, D> d8; + const size_t N8 = Lanes(d8); + auto bits_a = AllocateAligned<uint8_t>(HWY_MAX(8, N8)); + auto bits_b = AllocateAligned<uint8_t>(HWY_MAX(8, N8)); + memset(bits_a.get(), 0, N8); + memset(bits_b.get(), 0, N8); + const size_t num_bytes_a = StoreMaskBits(d, a, bits_a.get()); + const size_t num_bytes_b = StoreMaskBits(d, b, bits_b.get()); + AssertEqual(num_bytes_a, num_bytes_b, target_name, filename, line); + size_t i = 0; + // First check whole bytes (if that many elements are still valid) + for (; i < N / 8; ++i) { + if (bits_a[i] != bits_b[i]) { + fprintf(stderr, "Mismatch in byte %" PRIu64 ": %d != %d\n", + static_cast<uint64_t>(i), bits_a[i], bits_b[i]); + Print(d8, "expect", Load(d8, bits_a.get()), 0, N8); + Print(d8, "actual", Load(d8, bits_b.get()), 0, N8); + hwy::Abort(filename, line, "Masks not equal"); + } + } + // Then the valid bit(s) in the last byte. + const size_t remainder = N % 8; + if (remainder != 0) { + const int mask = (1 << remainder) - 1; + const int valid_a = bits_a[i] & mask; + const int valid_b = bits_b[i] & mask; + if (valid_a != valid_b) { + fprintf(stderr, "Mismatch in last byte %" PRIu64 ": %d != %d\n", + static_cast<uint64_t>(i), valid_a, valid_b); + Print(d8, "expect", Load(d8, bits_a.get()), 0, N8); + Print(d8, "actual", Load(d8, bits_b.get()), 0, N8); + hwy::Abort(filename, line, "Masks not equal"); + } + } +#endif +} + +// Only sets valid elements (those whose index < Lanes(d)). This helps catch +// tests that are not masking off the (undefined) upper mask elements. +// +// TODO(janwas): with HWY_NOINLINE GCC zeros the upper half of AVX2 masks. +template <class D> +HWY_INLINE Mask<D> MaskTrue(const D d) { + return FirstN(d, Lanes(d)); +} + +template <class D> +HWY_INLINE Mask<D> MaskFalse(const D d) { + const auto zero = Zero(RebindToSigned<D>()); + return RebindMask(d, Lt(zero, zero)); +} + +#ifndef HWY_ASSERT_EQ + +#define HWY_ASSERT_EQ(expected, actual) \ + hwy::AssertEqual(expected, actual, hwy::TargetName(HWY_TARGET), __FILE__, \ + __LINE__) + +#define HWY_ASSERT_STRING_EQ(expected, actual) \ + hwy::AssertStringEqual(expected, actual, hwy::TargetName(HWY_TARGET), \ + __FILE__, __LINE__) + +#define HWY_ASSERT_VEC_EQ(d, expected, actual) \ + AssertVecEqual(d, expected, actual, __FILE__, __LINE__) + +#define HWY_ASSERT_MASK_EQ(d, expected, actual) \ + AssertMaskEqual(d, expected, actual, __FILE__, __LINE__) + +#endif // HWY_ASSERT_EQ + +// Helpers for instantiating tests with combinations of lane types / counts. + +// For ensuring we do not call tests with D such that widening D results in 0 +// lanes. Example: assume T=u32, VLEN=256, and fraction=1/8: there is no 1/8th +// of a u64 vector in this case. +template <class D, HWY_IF_NOT_LANE_SIZE_D(D, 8)> +HWY_INLINE size_t PromotedLanes(const D d) { + return Lanes(RepartitionToWide<decltype(d)>()); +} +// Already the widest possible T, cannot widen. +template <class D, HWY_IF_LANE_SIZE_D(D, 8)> +HWY_INLINE size_t PromotedLanes(const D d) { + return Lanes(d); +} + +// For all power of two N in [kMinLanes, kMul * kMinLanes] (so that recursion +// stops at kMul == 0). Note that N may be capped or a fraction. +template <typename T, size_t kMul, size_t kMinLanes, class Test, + bool kPromote = false> +struct ForeachSizeR { + static void Do() { + const Simd<T, kMul * kMinLanes> d; + + // Skip invalid fractions (e.g. 1/8th of u32x4). + const size_t lanes = kPromote ? PromotedLanes(d) : Lanes(d); + if (lanes < kMinLanes) return; + + Test()(T(), d); + + static_assert(kMul != 0, "Recursion should have ended already"); + ForeachSizeR<T, kMul / 2, kMinLanes, Test, kPromote>::Do(); + } +}; + +// Base case to stop the recursion. +template <typename T, size_t kMinLanes, class Test, bool kPromote> +struct ForeachSizeR<T, 0, kMinLanes, Test, kPromote> { + static void Do() {} +}; + +// These adapters may be called directly, or via For*Types: + +// Calls Test for all power of two N in [1, Lanes(d) / kFactor]. This is for +// ops that widen their input, e.g. Combine (not supported by HWY_SCALAR). +template <class Test, size_t kFactor = 2> +struct ForExtendableVectors { + template <typename T> + void operator()(T /*unused*/) const { +#if HWY_TARGET == HWY_SCALAR + // not supported +#else + constexpr bool kPromote = true; +#if HWY_TARGET == HWY_RVV + ForeachSizeR<T, 8 / kFactor, HWY_LANES(T), Test, kPromote>::Do(); + // TODO(janwas): also capped + // ForeachSizeR<T, (16 / sizeof(T)) / kFactor, 1, Test, kPromote>::Do(); +#elif HWY_TARGET == HWY_SVE || HWY_TARGET == HWY_SVE2 + // Capped + ForeachSizeR<T, (16 / sizeof(T)) / kFactor, 1, Test, kPromote>::Do(); + // Fractions + ForeachSizeR<T, 8 / kFactor, HWY_LANES(T) / 8, Test, kPromote>::Do(); +#else + ForeachSizeR<T, HWY_LANES(T) / kFactor, 1, Test, kPromote>::Do(); +#endif +#endif // HWY_SCALAR + } +}; + +// Calls Test for all power of two N in [kFactor, Lanes(d)]. This is for ops +// that narrow their input, e.g. UpperHalf. +template <class Test, size_t kFactor = 2> +struct ForShrinkableVectors { + template <typename T> + void operator()(T /*unused*/) const { +#if HWY_TARGET == HWY_SCALAR + // not supported +#elif HWY_TARGET == HWY_RVV + ForeachSizeR<T, 8 / kFactor, kFactor * HWY_LANES(T), Test>::Do(); + // TODO(janwas): also capped +#elif HWY_TARGET == HWY_SVE || HWY_TARGET == HWY_SVE2 + // Capped + ForeachSizeR<T, (16 / sizeof(T)) / kFactor, kFactor, Test>::Do(); + // Fractions + ForeachSizeR<T, 8 / kFactor, kFactor * HWY_LANES(T) / 8, Test>::Do(); +#elif HWY_TARGET == HWY_SCALAR + // not supported +#else + ForeachSizeR<T, HWY_LANES(T) / kFactor, kFactor, Test>::Do(); +#endif + } +}; + +// Calls Test for all power of two N in [16 / sizeof(T), Lanes(d)]. This is for +// ops that require at least 128 bits, e.g. AES or 64x64 = 128 mul. +template <class Test> +struct ForGE128Vectors { + template <typename T> + void operator()(T /*unused*/) const { +#if HWY_TARGET == HWY_SCALAR + // not supported +#elif HWY_TARGET == HWY_RVV + ForeachSizeR<T, 8, HWY_LANES(T), Test>::Do(); + // TODO(janwas): also capped + // ForeachSizeR<T, 1, (16 / sizeof(T)), Test>::Do(); +#elif HWY_TARGET == HWY_SVE || HWY_TARGET == HWY_SVE2 + // Capped + ForeachSizeR<T, 1, 16 / sizeof(T), Test>::Do(); + // Fractions + ForeachSizeR<T, 8, HWY_LANES(T) / 8, Test>::Do(); +#else + ForeachSizeR<T, HWY_LANES(T) / (16 / sizeof(T)), (16 / sizeof(T)), + Test>::Do(); +#endif + } +}; + +// Calls Test for all power of two N in [8 / sizeof(T), Lanes(d)]. This is for +// ops that require at least 64 bits, e.g. casts. +template <class Test> +struct ForGE64Vectors { + template <typename T> + void operator()(T /*unused*/) const { +#if HWY_TARGET == HWY_SCALAR + // not supported +#elif HWY_TARGET == HWY_RVV + ForeachSizeR<T, 8, HWY_LANES(T), Test>::Do(); + // TODO(janwas): also capped + // ForeachSizeR<T, 1, (8 / sizeof(T)), Test>::Do(); +#elif HWY_TARGET == HWY_SVE || HWY_TARGET == HWY_SVE2 + // Capped + ForeachSizeR<T, 1, 8 / sizeof(T), Test>::Do(); + // Fractions + ForeachSizeR<T, 8, HWY_LANES(T) / 8, Test>::Do(); +#else + ForeachSizeR<T, HWY_LANES(T) / (8 / sizeof(T)), (8 / sizeof(T)), + Test>::Do(); +#endif + } +}; + +// Calls Test for all N that can be promoted (not the same as Extendable because +// HWY_SCALAR has one lane). Also used for ZipLower, but not ZipUpper. +template <class Test, size_t kFactor = 2> +struct ForPromoteVectors { + template <typename T> + void operator()(T /*unused*/) const { +#if HWY_TARGET == HWY_SCALAR + ForeachSizeR<T, 1, 1, Test, /*kPromote=*/true>::Do(); +#else + return ForExtendableVectors<Test, kFactor>()(T()); +#endif + } +}; + +// Calls Test for all N than can be demoted (not the same as Shrinkable because +// HWY_SCALAR has one lane). Also used for LowerHalf, but not UpperHalf. +template <class Test, size_t kFactor = 2> +struct ForDemoteVectors { + template <typename T> + void operator()(T /*unused*/) const { +#if HWY_TARGET == HWY_SCALAR + ForeachSizeR<T, 1, 1, Test>::Do(); +#else + return ForShrinkableVectors<Test, kFactor>()(T()); +#endif + } +}; + +// Calls Test for all power of two N in [1, Lanes(d)]. This is the default +// for ops that do not narrow nor widen their input, nor require 128 bits. +template <class Test> +struct ForPartialVectors { + template <typename T> + void operator()(T t) const { + ForExtendableVectors<Test, 1>()(t); + } +}; + +// Type lists to shorten call sites: + +template <class Func> +void ForSignedTypes(const Func& func) { + func(int8_t()); + func(int16_t()); + func(int32_t()); +#if HWY_CAP_INTEGER64 + func(int64_t()); +#endif +} + +template <class Func> +void ForUnsignedTypes(const Func& func) { + func(uint8_t()); + func(uint16_t()); + func(uint32_t()); +#if HWY_CAP_INTEGER64 + func(uint64_t()); +#endif +} + +template <class Func> +void ForIntegerTypes(const Func& func) { + ForSignedTypes(func); + ForUnsignedTypes(func); +} + +template <class Func> +void ForFloatTypes(const Func& func) { + func(float()); +#if HWY_CAP_FLOAT64 + func(double()); +#endif +} + +template <class Func> +void ForAllTypes(const Func& func) { + ForIntegerTypes(func); + ForFloatTypes(func); +} + +template <class Func> +void ForUIF3264(const Func& func) { + func(uint32_t()); + func(int32_t()); +#if HWY_CAP_INTEGER64 + func(uint64_t()); + func(int64_t()); +#endif + + ForFloatTypes(func); +} + +template <class Func> +void ForUIF163264(const Func& func) { + ForUIF3264(func); + func(uint16_t()); + func(int16_t()); +#if HWY_CAP_FLOAT16 + func(float16_t()); +#endif +} + +// For tests that involve loops, adjust the trip count so that emulated tests +// finish quickly (but always at least 2 iterations to ensure some diversity). +constexpr size_t AdjustedReps(size_t max_reps) { +#if HWY_ARCH_RVV + return HWY_MAX(max_reps / 16, 2); +#elif HWY_ARCH_ARM + return HWY_MAX(max_reps / 4, 2); +#elif HWY_IS_DEBUG_BUILD + return HWY_MAX(max_reps / 8, 2); +#else + return HWY_MAX(max_reps, 2); +#endif +} + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); + +#endif // per-target include guard diff --git a/third_party/highway/hwy/tests/test_util.cc b/third_party/highway/hwy/tests/test_util.cc new file mode 100644 index 0000000..861f6a4 --- /dev/null +++ b/third_party/highway/hwy/tests/test_util.cc @@ -0,0 +1,198 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "hwy/tests/test_util.h" + +#include <inttypes.h> +#include <stddef.h> +#include <stdio.h> + +#include <cmath> + +#include "hwy/base.h" + +namespace hwy { + +bool BytesEqual(const void* p1, const void* p2, const size_t size, + size_t* pos) { + const uint8_t* bytes1 = reinterpret_cast<const uint8_t*>(p1); + const uint8_t* bytes2 = reinterpret_cast<const uint8_t*>(p2); + for (size_t i = 0; i < size; ++i) { + if (bytes1[i] != bytes2[i]) { + fprintf(stderr, "Mismatch at byte %" PRIu64 " of %" PRIu64 ": %d != %d\n", + static_cast<uint64_t>(i), static_cast<uint64_t>(size), bytes1[i], + bytes2[i]); + if (pos != nullptr) { + *pos = i; + } + return false; + } + } + return true; +} + +void AssertStringEqual(const char* expected, const char* actual, + const char* target_name, const char* filename, + int line) { + while (*expected == *actual++) { + if (*expected++ == '\0') return; + } + + Abort(filename, line, "%s string mismatch: expected '%s', got '%s'.\n", + target_name, expected, actual); +} + +namespace detail { + +bool IsEqual(const TypeInfo& info, const void* expected_ptr, + const void* actual_ptr) { + if (!info.is_float) { + return BytesEqual(expected_ptr, actual_ptr, info.sizeof_t); + } + + if (info.sizeof_t == 4) { + float expected, actual; + CopyBytes<4>(expected_ptr, &expected); + CopyBytes<4>(actual_ptr, &actual); + return ComputeUlpDelta(expected, actual) <= 1; + } else if (info.sizeof_t == 8) { + double expected, actual; + CopyBytes<8>(expected_ptr, &expected); + CopyBytes<8>(actual_ptr, &actual); + return ComputeUlpDelta(expected, actual) <= 1; + } else { + HWY_ABORT("Unexpected float size %" PRIu64 "\n", + static_cast<uint64_t>(info.sizeof_t)); + return false; + } +} + +void TypeName(const TypeInfo& info, size_t N, char* string100) { + const char prefix = info.is_float ? 'f' : (info.is_signed ? 'i' : 'u'); + // Omit the xN suffix for scalars. + if (N == 1) { + snprintf(string100, 64, "%c%" PRIu64, prefix, + static_cast<uint64_t>(info.sizeof_t * 8)); + } else { + snprintf(string100, 64, "%c%" PRIu64 "x%" PRIu64, prefix, + static_cast<uint64_t>(info.sizeof_t * 8), + static_cast<uint64_t>(N)); + } +} + +void ToString(const TypeInfo& info, const void* ptr, char* string100) { + if (info.sizeof_t == 1) { + uint8_t byte; + CopyBytes<1>(ptr, &byte); // endian-safe: we ensured sizeof(T)=1. + snprintf(string100, 100, "0x%02X", byte); + } else if (info.sizeof_t == 2) { + uint16_t bits; + CopyBytes<2>(ptr, &bits); + snprintf(string100, 100, "0x%04X", bits); + } else if (info.sizeof_t == 4) { + if (info.is_float) { + float value; + CopyBytes<4>(ptr, &value); + snprintf(string100, 100, "%g", double(value)); + } else if (info.is_signed) { + int32_t value; + CopyBytes<4>(ptr, &value); + snprintf(string100, 100, "%d", value); + } else { + uint32_t value; + CopyBytes<4>(ptr, &value); + snprintf(string100, 100, "%u", value); + } + } else { + HWY_ASSERT(info.sizeof_t == 8); + if (info.is_float) { + double value; + CopyBytes<8>(ptr, &value); + snprintf(string100, 100, "%g", value); + } else if (info.is_signed) { + int64_t value; + CopyBytes<8>(ptr, &value); + snprintf(string100, 100, "%" PRIi64 "", value); + } else { + uint64_t value; + CopyBytes<8>(ptr, &value); + snprintf(string100, 100, "%" PRIu64 "", value); + } + } +} + +void PrintArray(const TypeInfo& info, const char* caption, + const void* array_void, size_t N, size_t lane_u, + size_t max_lanes) { + const uint8_t* array_bytes = reinterpret_cast<const uint8_t*>(array_void); + + char type_name[100]; + TypeName(info, N, type_name); + + const intptr_t lane = intptr_t(lane_u); + const size_t begin = static_cast<size_t>(HWY_MAX(0, lane - 2)); + const size_t end = HWY_MIN(begin + max_lanes, N); + fprintf(stderr, "%s %s [%" PRIu64 "+ ->]:\n ", type_name, caption, + static_cast<uint64_t>(begin)); + for (size_t i = begin; i < end; ++i) { + const void* ptr = array_bytes + i * info.sizeof_t; + char str[100]; + ToString(info, ptr, str); + fprintf(stderr, "%s,", str); + } + if (begin >= end) fprintf(stderr, "(out of bounds)"); + fprintf(stderr, "\n"); +} + +HWY_NORETURN void PrintMismatchAndAbort(const TypeInfo& info, + const void* expected_ptr, + const void* actual_ptr, + const char* target_name, + const char* filename, int line, + size_t lane, size_t num_lanes) { + char type_name[100]; + TypeName(info, 1, type_name); + char expected_str[100]; + ToString(info, expected_ptr, expected_str); + char actual_str[100]; + ToString(info, actual_ptr, actual_str); + Abort(filename, line, + "%s, %sx%" PRIu64 " lane %" PRIu64 + " mismatch: expected '%s', got '%s'.\n", + target_name, type_name, static_cast<uint64_t>(num_lanes), + static_cast<uint64_t>(lane), expected_str, actual_str); +} + +void AssertArrayEqual(const TypeInfo& info, const void* expected_void, + const void* actual_void, size_t N, + const char* target_name, const char* filename, int line) { + const uint8_t* expected_array = + reinterpret_cast<const uint8_t*>(expected_void); + const uint8_t* actual_array = reinterpret_cast<const uint8_t*>(actual_void); + for (size_t i = 0; i < N; ++i) { + const void* expected_ptr = expected_array + i * info.sizeof_t; + const void* actual_ptr = actual_array + i * info.sizeof_t; + if (!IsEqual(info, expected_ptr, actual_ptr)) { + fprintf(stderr, "\n\n"); + PrintArray(info, "expect", expected_array, N, i); + PrintArray(info, "actual", actual_array, N, i); + + PrintMismatchAndAbort(info, expected_ptr, actual_ptr, target_name, + filename, line, i, N); + } + } +} + +} // namespace detail +} // namespace hwy diff --git a/third_party/highway/hwy/tests/test_util.h b/third_party/highway/hwy/tests/test_util.h new file mode 100644 index 0000000..076d82b --- /dev/null +++ b/third_party/highway/hwy/tests/test_util.h @@ -0,0 +1,185 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef HWY_TESTS_TEST_UTIL_H_ +#define HWY_TESTS_TEST_UTIL_H_ + +// Target-independent helper functions for use by *_test.cc. + +#include <stddef.h> +#include <stdint.h> +#include <string.h> + +#include <string> + +#include "hwy/aligned_allocator.h" +#include "hwy/base.h" +#include "hwy/highway.h" + +namespace hwy { + +// The maximum vector size used in tests when defining test data. DEPRECATED. +constexpr size_t kTestMaxVectorSize = 64; + +// 64-bit random generator (Xorshift128+). Much smaller state than std::mt19937, +// which triggers a compiler bug. +class RandomState { + public: + explicit RandomState(const uint64_t seed = 0x123456789ull) { + s0_ = SplitMix64(seed + 0x9E3779B97F4A7C15ull); + s1_ = SplitMix64(s0_); + } + + HWY_INLINE uint64_t operator()() { + uint64_t s1 = s0_; + const uint64_t s0 = s1_; + const uint64_t bits = s1 + s0; + s0_ = s0; + s1 ^= s1 << 23; + s1 ^= s0 ^ (s1 >> 18) ^ (s0 >> 5); + s1_ = s1; + return bits; + } + + private: + static uint64_t SplitMix64(uint64_t z) { + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBull; + return z ^ (z >> 31); + } + + uint64_t s0_; + uint64_t s1_; +}; + +static HWY_INLINE uint32_t Random32(RandomState* rng) { + return static_cast<uint32_t>((*rng)()); +} + +static HWY_INLINE uint64_t Random64(RandomState* rng) { + return (*rng)(); +} + +// Prevents the compiler from eliding the computations that led to "output". +// Works by indicating to the compiler that "output" is being read and modified. +// The +r constraint avoids unnecessary writes to memory, but only works for +// built-in types. +template <class T> +inline void PreventElision(T&& output) { +#if HWY_COMPILER_MSVC + (void)output; +#else // HWY_COMPILER_MSVC + asm volatile("" : "+r"(output) : : "memory"); +#endif // HWY_COMPILER_MSVC +} + +bool BytesEqual(const void* p1, const void* p2, const size_t size, + size_t* pos = nullptr); + +void AssertStringEqual(const char* expected, const char* actual, + const char* target_name, const char* filename, int line); + +namespace detail { + +template <typename T, typename TU = MakeUnsigned<T>> +TU ComputeUlpDelta(const T expected, const T actual) { + // Handle -0 == 0 and infinities. + if (expected == actual) return 0; + + // Consider "equal" if both are NaN, so we can verify an expected NaN. + // Needs a special case because there are many possible NaN representations. + if (std::isnan(expected) && std::isnan(actual)) return 0; + + // Compute the difference in units of last place. We do not need to check for + // differing signs; they will result in large differences, which is fine. + TU ux, uy; + CopyBytes<sizeof(T)>(&expected, &ux); + CopyBytes<sizeof(T)>(&actual, &uy); + + // Avoid unsigned->signed cast: 2's complement is only guaranteed by C++20. + const TU ulp = HWY_MAX(ux, uy) - HWY_MIN(ux, uy); + return ulp; +} + +// For implementing value comparisons etc. as type-erased functions to reduce +// template bloat. +struct TypeInfo { + size_t sizeof_t; + bool is_float; + bool is_signed; +}; + +template <typename T> +HWY_INLINE TypeInfo MakeTypeInfo() { + TypeInfo info; + info.sizeof_t = sizeof(T); + info.is_float = IsFloat<T>(); + info.is_signed = IsSigned<T>(); + return info; +} + +bool IsEqual(const TypeInfo& info, const void* expected_ptr, + const void* actual_ptr); + +void TypeName(const TypeInfo& info, size_t N, char* string100); + +void PrintArray(const TypeInfo& info, const char* caption, + const void* array_void, size_t N, size_t lane_u = 0, + size_t max_lanes = 7); + +HWY_NORETURN void PrintMismatchAndAbort(const TypeInfo& info, + const void* expected_ptr, + const void* actual_ptr, + const char* target_name, + const char* filename, int line, + size_t lane = 0, size_t num_lanes = 1); + +void AssertArrayEqual(const TypeInfo& info, const void* expected_void, + const void* actual_void, size_t N, + const char* target_name, const char* filename, int line); + +} // namespace detail + +// Returns a name for the vector/part/scalar. The type prefix is u/i/f for +// unsigned/signed/floating point, followed by the number of bits per lane; +// then 'x' followed by the number of lanes. Example: u8x16. This is useful for +// understanding which instantiation of a generic test failed. +template <typename T> +std::string TypeName(T /*unused*/, size_t N) { + char string100[100]; + detail::TypeName(detail::MakeTypeInfo<T>(), N, string100); + return string100; +} + +// Compare non-vector, non-string T. +template <typename T> +HWY_INLINE bool IsEqual(const T expected, const T actual) { + const auto info = detail::MakeTypeInfo<T>(); + return detail::IsEqual(info, &expected, &actual); +} + +template <typename T> +HWY_INLINE void AssertEqual(const T expected, const T actual, + const char* target_name, const char* filename, + int line, size_t lane = 0) { + const auto info = detail::MakeTypeInfo<T>(); + if (!detail::IsEqual(info, &expected, &actual)) { + detail::PrintMismatchAndAbort(info, &expected, &actual, target_name, + filename, line, lane); + } +} + +} // namespace hwy + +#endif // HWY_TESTS_TEST_UTIL_H_ diff --git a/third_party/highway/hwy/tests/test_util_test.cc b/third_party/highway/hwy/tests/test_util_test.cc new file mode 100644 index 0000000..af484ad --- /dev/null +++ b/third_party/highway/hwy/tests/test_util_test.cc @@ -0,0 +1,110 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <stddef.h> +#include <stdint.h> + +#undef HWY_TARGET_INCLUDE +#define HWY_TARGET_INCLUDE "tests/test_util_test.cc" +#include "hwy/foreach_target.h" +#include "hwy/highway.h" +#include "hwy/tests/test_util-inl.h" + +HWY_BEFORE_NAMESPACE(); +namespace hwy { +namespace HWY_NAMESPACE { + +struct TestName { + template <class T, class D> + HWY_NOINLINE void operator()(T t, D d) { + char num[10]; + std::string expected = IsFloat<T>() ? "f" : (IsSigned<T>() ? "i" : "u"); + snprintf(num, sizeof(num), "%u" , static_cast<unsigned>(sizeof(T) * 8)); + expected += num; + + const size_t N = Lanes(d); + if (N != 1) { + expected += 'x'; + snprintf(num, sizeof(num), "%u", static_cast<unsigned>(N)); + expected += num; + } + const std::string actual = TypeName(t, N); + if (expected != actual) { + HWY_ABORT("%s mismatch: expected '%s', got '%s'.\n", + hwy::TargetName(HWY_TARGET), expected.c_str(), actual.c_str()); + } + } +}; + +HWY_NOINLINE void TestAllName() { ForAllTypes(ForPartialVectors<TestName>()); } + +struct TestEqualInteger { + template <class T> + HWY_NOINLINE void operator()(T /*t*/) const { + HWY_ASSERT(IsEqual(T(0), T(0))); + HWY_ASSERT(IsEqual(T(1), T(1))); + HWY_ASSERT(IsEqual(T(-1), T(-1))); + HWY_ASSERT(IsEqual(LimitsMin<T>(), LimitsMin<T>())); + + HWY_ASSERT(!IsEqual(T(0), T(1))); + HWY_ASSERT(!IsEqual(T(1), T(0))); + HWY_ASSERT(!IsEqual(T(1), T(-1))); + HWY_ASSERT(!IsEqual(T(-1), T(1))); + HWY_ASSERT(!IsEqual(LimitsMin<T>(), LimitsMax<T>())); + HWY_ASSERT(!IsEqual(LimitsMax<T>(), LimitsMin<T>())); + } +}; + +struct TestEqualFloat { + template <class T> + HWY_NOINLINE void operator()(T /*t*/) const { + HWY_ASSERT(IsEqual(T(0), T(0))); + HWY_ASSERT(IsEqual(T(1), T(1))); + HWY_ASSERT(IsEqual(T(-1), T(-1))); + HWY_ASSERT(IsEqual(MantissaEnd<T>(), MantissaEnd<T>())); + + HWY_ASSERT(!IsEqual(T(0), T(1))); + HWY_ASSERT(!IsEqual(T(1), T(0))); + HWY_ASSERT(!IsEqual(T(1), T(-1))); + HWY_ASSERT(!IsEqual(T(-1), T(1))); + HWY_ASSERT(!IsEqual(LowestValue<T>(), HighestValue<T>())); + HWY_ASSERT(!IsEqual(HighestValue<T>(), LowestValue<T>())); + } +}; + +HWY_NOINLINE void TestAllEqual() { + ForIntegerTypes(TestEqualInteger()); + ForFloatTypes(TestEqualFloat()); +} + +// NOLINTNEXTLINE(google-readability-namespace-comments) +} // namespace HWY_NAMESPACE +} // namespace hwy +HWY_AFTER_NAMESPACE(); + +#if HWY_ONCE + +namespace hwy { +HWY_BEFORE_TEST(TestUtilTest); +HWY_EXPORT_AND_TEST_P(TestUtilTest, TestAllName); +HWY_EXPORT_AND_TEST_P(TestUtilTest, TestAllEqual); +} // namespace hwy + +// Ought not to be necessary, but without this, no tests run on RVV. +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +#endif diff --git a/third_party/highway/libhwy-contrib.pc.in b/third_party/highway/libhwy-contrib.pc.in new file mode 100644 index 0000000..260f15f --- /dev/null +++ b/third_party/highway/libhwy-contrib.pc.in @@ -0,0 +1,10 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=${prefix} +libdir=${exec_prefix}/@CMAKE_INSTALL_LIBDIR@ +includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ + +Name: libhwy-contrib +Description: Additions to Highway: image and math library +Version: @HWY_LIBRARY_VERSION@ +Libs: -L${libdir} -lhwy_contrib +Cflags: -I${includedir} diff --git a/third_party/highway/libhwy-test.pc.in b/third_party/highway/libhwy-test.pc.in new file mode 100644 index 0000000..ff91690 --- /dev/null +++ b/third_party/highway/libhwy-test.pc.in @@ -0,0 +1,10 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +libdir=${exec_prefix}/@CMAKE_INSTALL_LIBDIR@ +includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ + +Name: libhwy-test +Description: Efficient and performance-portable SIMD wrapper, test helpers. +Requires: gtest +Version: @HWY_LIBRARY_VERSION@ +Libs: -L${libdir} -lhwy_test +Cflags: -I${includedir} diff --git a/third_party/highway/libhwy.pc.in b/third_party/highway/libhwy.pc.in new file mode 100644 index 0000000..2ada0e8 --- /dev/null +++ b/third_party/highway/libhwy.pc.in @@ -0,0 +1,10 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=${prefix} +libdir=${exec_prefix}/@CMAKE_INSTALL_LIBDIR@ +includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ + +Name: libhwy +Description: Efficient and performance-portable SIMD wrapper +Version: @HWY_LIBRARY_VERSION@ +Libs: -L${libdir} -lhwy +Cflags: -I${includedir} diff --git a/third_party/highway/run_tests.bat b/third_party/highway/run_tests.bat new file mode 100644 index 0000000..26600a2 --- /dev/null +++ b/third_party/highway/run_tests.bat @@ -0,0 +1,20 @@ +@echo off +REM Switch directory of this batch file +cd %~dp0 + +if not exist build_win mkdir build_win + +cd build_win +cmake .. -DHWY_WARNINGS_ARE_ERRORS:BOOL=ON -G Ninja || goto error +ninja || goto error +ctest -j || goto error + +cd .. +echo Success +goto end + +:error +echo Failure +exit /b 1 + +:end diff --git a/third_party/highway/run_tests.sh b/third_party/highway/run_tests.sh new file mode 100644 index 0000000..4efae5c --- /dev/null +++ b/third_party/highway/run_tests.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +# Switch to directory of this script +MYDIR=$(dirname $(realpath "$0")) +cd "${MYDIR}" + +# Exit if anything fails +set -e + +####################################### +echo RELEASE +rm -rf build +mkdir build +cd build +cmake .. -DHWY_WARNINGS_ARE_ERRORS:BOOL=ON +make -j +ctest -j +cd .. +rm -rf build + +####################################### +echo DEBUG Clang 7 +rm -rf build_dbg +mkdir build_dbg +cd build_dbg +CXX=clang++-7 CC=clang-7 cmake .. -DHWY_WARNINGS_ARE_ERRORS:BOOL=ON -DCMAKE_BUILD_TYPE=Debug +make -j +ctest -j +cd .. +rm -rf build_dbg + +####################################### +echo 32-bit GCC +rm -rf build_32 +mkdir build_32 +cd build_32 +CFLAGS=-m32 CXXFLAGS=-m32 LDFLAGS=-m32 CXX=g++ CC=gcc cmake .. -DHWY_WARNINGS_ARE_ERRORS:BOOL=ON +make -j +ctest -j +cd .. +rm -rf build_32 + +####################################### +for VER in 8 9 10; do + echo GCC $VER + rm -rf build_g$VER + mkdir build_g$VER + cd build_g$VER + CC=gcc-$VER CXX=g++-$VER cmake .. -DHWY_WARNINGS_ARE_ERRORS:BOOL=ON + make -j + make test + cd .. + rm -rf build_g$VER +done + +####################################### +echo ARMv7 GCC +export QEMU_LD_PREFIX=/usr/arm-linux-gnueabihf +rm -rf build_arm7 +mkdir build_arm7 +cd build_arm7 +CC=arm-linux-gnueabihf-gcc CXX=arm-linux-gnueabihf-g++ cmake .. -DHWY_CMAKE_ARM7:BOOL=ON -DHWY_WARNINGS_ARE_ERRORS:BOOL=ON +make -j8 +ctest +cd .. +rm -rf build_arm7 + +####################################### +echo ARMv8 GCC +export QEMU_LD_PREFIX=/usr/aarch64-linux-gnu +rm -rf build_arm8 +mkdir build_arm8 +cd build_arm8 +CC=aarch64-linux-gnu-gcc CXX=aarch64-linux-gnu-g++ cmake .. -DHWY_WARNINGS_ARE_ERRORS:BOOL=ON +make -j8 +ctest +cd .. +rm -rf build_arm8 + +echo Success |