summaryrefslogtreecommitdiff
path: root/boost/atomic/detail/extra_ops_gcc_arm.hpp
diff options
context:
space:
mode:
Diffstat (limited to 'boost/atomic/detail/extra_ops_gcc_arm.hpp')
-rw-r--r--boost/atomic/detail/extra_ops_gcc_arm.hpp845
1 files changed, 810 insertions, 35 deletions
diff --git a/boost/atomic/detail/extra_ops_gcc_arm.hpp b/boost/atomic/detail/extra_ops_gcc_arm.hpp
index 9bd4829bd8..e84f1771da 100644
--- a/boost/atomic/detail/extra_ops_gcc_arm.hpp
+++ b/boost/atomic/detail/extra_ops_gcc_arm.hpp
@@ -3,7 +3,7 @@
* (See accompanying file LICENSE_1_0.txt or copy at
* http://www.boost.org/LICENSE_1_0.txt)
*
- * Copyright (c) 2017 Andrey Semashev
+ * Copyright (c) 2017 - 2018 Andrey Semashev
*/
/*!
* \file atomic/detail/extra_ops_gcc_arm.hpp
@@ -33,15 +33,71 @@ namespace boost {
namespace atomics {
namespace detail {
+template< typename Base >
+struct gcc_arm_extra_operations_common :
+ public Base
+{
+ typedef Base base_type;
+ typedef typename base_type::storage_type storage_type;
+
+ static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
+ {
+ base_type::fetch_negate(storage, order);
+ }
+
+ static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
+ {
+ base_type::fetch_complement(storage, order);
+ }
+
+ static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
+ {
+ return !!base_type::negate(storage, order);
+ }
+
+ static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
+ {
+ return !!base_type::add(storage, v, order);
+ }
+
+ static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
+ {
+ return !!base_type::sub(storage, v, order);
+ }
+
+ static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
+ {
+ return !!base_type::bitwise_and(storage, v, order);
+ }
+
+ static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
+ {
+ return !!base_type::bitwise_or(storage, v, order);
+ }
+
+ static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
+ {
+ return !!base_type::bitwise_xor(storage, v, order);
+ }
+
+ static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
+ {
+ return !!base_type::bitwise_complement(storage, order);
+ }
+};
+
+template< typename Base, std::size_t Size, bool Signed >
+struct gcc_arm_extra_operations;
+
#if defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXB_STREXB)
template< typename Base, bool Signed >
-struct extra_operations< Base, 1u, Signed > :
+struct gcc_arm_extra_operations< Base, 1u, Signed > :
public generic_extra_operations< Base, 1u, Signed >
{
typedef generic_extra_operations< Base, 1u, Signed > base_type;
typedef typename base_type::storage_type storage_type;
- typedef typename make_storage_type< 4u, Signed >::type extended_storage_type;
+ typedef typename make_storage_type< 4u >::type extended_storage_type;
static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
{
@@ -52,7 +108,7 @@ struct extra_operations< Base, 1u, Signed > :
(
BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
"1:\n"
- "ldrexb %[original], %[storage]\n" // original = *(&storage)
+ "ldrexb %[original], %[storage]\n" // original = zero_extend(*(&storage))
"rsb %[result], %[original], #0\n" // result = 0 - original
"strexb %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
"teq %[tmp], #0\n" // flags = tmp==0
@@ -69,6 +125,162 @@ struct extra_operations< Base, 1u, Signed > :
return static_cast< storage_type >(original);
}
+ static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
+ {
+ gcc_arm_operations_base::fence_before(order);
+ uint32_t tmp;
+ extended_storage_type original, result;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
+ "1:\n"
+ "ldrexb %[original], %[storage]\n" // original = zero_extend(*(&storage))
+ "rsb %[result], %[original], #0\n" // result = 0 - original
+ "strexb %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
+ "teq %[tmp], #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
+ : [original] "=&r" (original), // %0
+ [result] "=&r" (result), // %1
+ [tmp] "=&l" (tmp), // %2
+ [storage] "+Q" (storage) // %3
+ :
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return static_cast< storage_type >(result);
+ }
+
+ static BOOST_FORCEINLINE storage_type add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
+ {
+ gcc_arm_operations_base::fence_before(order);
+ uint32_t tmp;
+ extended_storage_type original, result;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
+ "1:\n"
+ "ldrexb %[original], %[storage]\n" // original = zero_extend(*(&storage))
+ "add %[result], %[original], %[value]\n" // result = original + value
+ "strexb %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
+ "teq %[tmp], #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
+ : [original] "=&r" (original), // %0
+ [result] "=&r" (result), // %1
+ [tmp] "=&l" (tmp), // %2
+ [storage] "+Q" (storage) // %3
+ : [value] "Ir" (v) // %4
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return static_cast< storage_type >(result);
+ }
+
+ static BOOST_FORCEINLINE storage_type sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
+ {
+ gcc_arm_operations_base::fence_before(order);
+ uint32_t tmp;
+ extended_storage_type original, result;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
+ "1:\n"
+ "ldrexb %[original], %[storage]\n" // original = zero_extend(*(&storage))
+ "sub %[result], %[original], %[value]\n" // result = original - value
+ "strexb %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
+ "teq %[tmp], #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
+ : [original] "=&r" (original), // %0
+ [result] "=&r" (result), // %1
+ [tmp] "=&l" (tmp), // %2
+ [storage] "+Q" (storage) // %3
+ : [value] "Ir" (v) // %4
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return static_cast< storage_type >(result);
+ }
+
+ static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
+ {
+ gcc_arm_operations_base::fence_before(order);
+ uint32_t tmp;
+ extended_storage_type original, result;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
+ "1:\n"
+ "ldrexb %[original], %[storage]\n" // original = zero_extend(*(&storage))
+ "and %[result], %[original], %[value]\n" // result = original & value
+ "strexb %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
+ "teq %[tmp], #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
+ : [original] "=&r" (original), // %0
+ [result] "=&r" (result), // %1
+ [tmp] "=&l" (tmp), // %2
+ [storage] "+Q" (storage) // %3
+ : [value] "Ir" (v) // %4
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return static_cast< storage_type >(result);
+ }
+
+ static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
+ {
+ gcc_arm_operations_base::fence_before(order);
+ uint32_t tmp;
+ extended_storage_type original, result;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
+ "1:\n"
+ "ldrexb %[original], %[storage]\n" // original = zero_extend(*(&storage))
+ "orr %[result], %[original], %[value]\n" // result = original | value
+ "strexb %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
+ "teq %[tmp], #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
+ : [original] "=&r" (original), // %0
+ [result] "=&r" (result), // %1
+ [tmp] "=&l" (tmp), // %2
+ [storage] "+Q" (storage) // %3
+ : [value] "Ir" (v) // %4
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return static_cast< storage_type >(result);
+ }
+
+ static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
+ {
+ gcc_arm_operations_base::fence_before(order);
+ uint32_t tmp;
+ extended_storage_type original, result;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
+ "1:\n"
+ "ldrexb %[original], %[storage]\n" // original = zero_extend(*(&storage))
+ "eor %[result], %[original], %[value]\n" // result = original ^ value
+ "strexb %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
+ "teq %[tmp], #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
+ : [original] "=&r" (original), // %0
+ [result] "=&r" (result), // %1
+ [tmp] "=&l" (tmp), // %2
+ [storage] "+Q" (storage) // %3
+ : [value] "Ir" (v) // %4
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return static_cast< storage_type >(result);
+ }
+
static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
{
gcc_arm_operations_base::fence_before(order);
@@ -78,7 +290,7 @@ struct extra_operations< Base, 1u, Signed > :
(
BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
"1:\n"
- "ldrexb %[original], %[storage]\n" // original = *(&storage)
+ "ldrexb %[original], %[storage]\n" // original = zero_extend(*(&storage))
"mvn %[result], %[original]\n" // result = NOT original
"strexb %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
"teq %[tmp], #0\n" // flags = tmp==0
@@ -95,15 +307,37 @@ struct extra_operations< Base, 1u, Signed > :
return static_cast< storage_type >(original);
}
- static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
+ static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
{
- fetch_negate(storage, order);
+ gcc_arm_operations_base::fence_before(order);
+ uint32_t tmp;
+ extended_storage_type original, result;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
+ "1:\n"
+ "ldrexb %[original], %[storage]\n" // original = zero_extend(*(&storage))
+ "mvn %[result], %[original]\n" // result = NOT original
+ "strexb %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
+ "teq %[tmp], #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
+ : [original] "=&r" (original), // %0
+ [result] "=&r" (result), // %1
+ [tmp] "=&l" (tmp), // %2
+ [storage] "+Q" (storage) // %3
+ :
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return static_cast< storage_type >(result);
}
+};
- static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
- {
- fetch_complement(storage, order);
- }
+template< typename Base, bool Signed >
+struct extra_operations< Base, 1u, Signed, true > :
+ public gcc_arm_extra_operations_common< gcc_arm_extra_operations< Base, 1u, Signed > >
+{
};
#endif // defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXB_STREXB)
@@ -111,12 +345,12 @@ struct extra_operations< Base, 1u, Signed > :
#if defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXH_STREXH)
template< typename Base, bool Signed >
-struct extra_operations< Base, 2u, Signed > :
+struct gcc_arm_extra_operations< Base, 2u, Signed > :
public generic_extra_operations< Base, 2u, Signed >
{
typedef generic_extra_operations< Base, 2u, Signed > base_type;
typedef typename base_type::storage_type storage_type;
- typedef typename make_storage_type< 4u, Signed >::type extended_storage_type;
+ typedef typename make_storage_type< 4u >::type extended_storage_type;
static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
{
@@ -127,7 +361,7 @@ struct extra_operations< Base, 2u, Signed > :
(
BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
"1:\n"
- "ldrexh %[original], %[storage]\n" // original = *(&storage)
+ "ldrexh %[original], %[storage]\n" // original = zero_extend(*(&storage))
"rsb %[result], %[original], #0\n" // result = 0 - original
"strexh %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
"teq %[tmp], #0\n" // flags = tmp==0
@@ -144,6 +378,162 @@ struct extra_operations< Base, 2u, Signed > :
return static_cast< storage_type >(original);
}
+ static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
+ {
+ gcc_arm_operations_base::fence_before(order);
+ uint32_t tmp;
+ extended_storage_type original, result;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
+ "1:\n"
+ "ldrexh %[original], %[storage]\n" // original = zero_extend(*(&storage))
+ "rsb %[result], %[original], #0\n" // result = 0 - original
+ "strexh %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
+ "teq %[tmp], #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
+ : [original] "=&r" (original), // %0
+ [result] "=&r" (result), // %1
+ [tmp] "=&l" (tmp), // %2
+ [storage] "+Q" (storage) // %3
+ :
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return static_cast< storage_type >(result);
+ }
+
+ static BOOST_FORCEINLINE storage_type add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
+ {
+ gcc_arm_operations_base::fence_before(order);
+ uint32_t tmp;
+ extended_storage_type original, result;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
+ "1:\n"
+ "ldrexh %[original], %[storage]\n" // original = zero_extend(*(&storage))
+ "add %[result], %[original], %[value]\n" // result = original + value
+ "strexh %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
+ "teq %[tmp], #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
+ : [original] "=&r" (original), // %0
+ [result] "=&r" (result), // %1
+ [tmp] "=&l" (tmp), // %2
+ [storage] "+Q" (storage) // %3
+ : [value] "Ir" (v) // %4
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return static_cast< storage_type >(result);
+ }
+
+ static BOOST_FORCEINLINE storage_type sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
+ {
+ gcc_arm_operations_base::fence_before(order);
+ uint32_t tmp;
+ extended_storage_type original, result;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
+ "1:\n"
+ "ldrexh %[original], %[storage]\n" // original = zero_extend(*(&storage))
+ "sub %[result], %[original], %[value]\n" // result = original - value
+ "strexh %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
+ "teq %[tmp], #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
+ : [original] "=&r" (original), // %0
+ [result] "=&r" (result), // %1
+ [tmp] "=&l" (tmp), // %2
+ [storage] "+Q" (storage) // %3
+ : [value] "Ir" (v) // %4
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return static_cast< storage_type >(result);
+ }
+
+ static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
+ {
+ gcc_arm_operations_base::fence_before(order);
+ uint32_t tmp;
+ extended_storage_type original, result;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
+ "1:\n"
+ "ldrexh %[original], %[storage]\n" // original = zero_extend(*(&storage))
+ "and %[result], %[original], %[value]\n" // result = original & value
+ "strexh %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
+ "teq %[tmp], #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
+ : [original] "=&r" (original), // %0
+ [result] "=&r" (result), // %1
+ [tmp] "=&l" (tmp), // %2
+ [storage] "+Q" (storage) // %3
+ : [value] "Ir" (v) // %4
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return static_cast< storage_type >(result);
+ }
+
+ static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
+ {
+ gcc_arm_operations_base::fence_before(order);
+ uint32_t tmp;
+ extended_storage_type original, result;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
+ "1:\n"
+ "ldrexh %[original], %[storage]\n" // original = zero_extend(*(&storage))
+ "orr %[result], %[original], %[value]\n" // result = original | value
+ "strexh %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
+ "teq %[tmp], #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
+ : [original] "=&r" (original), // %0
+ [result] "=&r" (result), // %1
+ [tmp] "=&l" (tmp), // %2
+ [storage] "+Q" (storage) // %3
+ : [value] "Ir" (v) // %4
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return static_cast< storage_type >(result);
+ }
+
+ static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
+ {
+ gcc_arm_operations_base::fence_before(order);
+ uint32_t tmp;
+ extended_storage_type original, result;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
+ "1:\n"
+ "ldrexh %[original], %[storage]\n" // original = zero_extend(*(&storage))
+ "eor %[result], %[original], %[value]\n" // result = original ^ value
+ "strexh %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
+ "teq %[tmp], #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
+ : [original] "=&r" (original), // %0
+ [result] "=&r" (result), // %1
+ [tmp] "=&l" (tmp), // %2
+ [storage] "+Q" (storage) // %3
+ : [value] "Ir" (v) // %4
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return static_cast< storage_type >(result);
+ }
+
static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
{
gcc_arm_operations_base::fence_before(order);
@@ -153,7 +543,7 @@ struct extra_operations< Base, 2u, Signed > :
(
BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
"1:\n"
- "ldrexh %[original], %[storage]\n" // original = *(&storage)
+ "ldrexh %[original], %[storage]\n" // original = zero_extend(*(&storage))
"mvn %[result], %[original]\n" // result = NOT original
"strexh %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
"teq %[tmp], #0\n" // flags = tmp==0
@@ -170,21 +560,43 @@ struct extra_operations< Base, 2u, Signed > :
return static_cast< storage_type >(original);
}
- static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
+ static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
{
- fetch_negate(storage, order);
+ gcc_arm_operations_base::fence_before(order);
+ uint32_t tmp;
+ extended_storage_type original, result;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
+ "1:\n"
+ "ldrexh %[original], %[storage]\n" // original = zero_extend(*(&storage))
+ "mvn %[result], %[original]\n" // result = NOT original
+ "strexh %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
+ "teq %[tmp], #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
+ : [original] "=&r" (original), // %0
+ [result] "=&r" (result), // %1
+ [tmp] "=&l" (tmp), // %2
+ [storage] "+Q" (storage) // %3
+ :
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return static_cast< storage_type >(result);
}
+};
- static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
- {
- fetch_complement(storage, order);
- }
+template< typename Base, bool Signed >
+struct extra_operations< Base, 2u, Signed, true > :
+ public gcc_arm_extra_operations_common< gcc_arm_extra_operations< Base, 2u, Signed > >
+{
};
#endif // defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXH_STREXH)
template< typename Base, bool Signed >
-struct extra_operations< Base, 4u, Signed > :
+struct gcc_arm_extra_operations< Base, 4u, Signed > :
public generic_extra_operations< Base, 4u, Signed >
{
typedef generic_extra_operations< Base, 4u, Signed > base_type;
@@ -216,6 +628,162 @@ struct extra_operations< Base, 4u, Signed > :
return original;
}
+ static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
+ {
+ gcc_arm_operations_base::fence_before(order);
+ uint32_t tmp;
+ storage_type original, result;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
+ "1:\n"
+ "ldrex %[original], %[storage]\n" // original = *(&storage)
+ "rsb %[result], %[original], #0\n" // result = 0 - original
+ "strex %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
+ "teq %[tmp], #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
+ : [original] "=&r" (original), // %0
+ [result] "=&r" (result), // %1
+ [tmp] "=&l" (tmp), // %2
+ [storage] "+Q" (storage) // %3
+ :
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return result;
+ }
+
+ static BOOST_FORCEINLINE storage_type add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
+ {
+ gcc_arm_operations_base::fence_before(order);
+ uint32_t tmp;
+ storage_type original, result;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
+ "1:\n"
+ "ldrex %[original], %[storage]\n" // original = *(&storage)
+ "add %[result], %[original], %[value]\n" // result = original + value
+ "strex %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
+ "teq %[tmp], #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
+ : [original] "=&r" (original), // %0
+ [result] "=&r" (result), // %1
+ [tmp] "=&l" (tmp), // %2
+ [storage] "+Q" (storage) // %3
+ : [value] "Ir" (v) // %4
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return result;
+ }
+
+ static BOOST_FORCEINLINE storage_type sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
+ {
+ gcc_arm_operations_base::fence_before(order);
+ uint32_t tmp;
+ storage_type original, result;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
+ "1:\n"
+ "ldrex %[original], %[storage]\n" // original = *(&storage)
+ "sub %[result], %[original], %[value]\n" // result = original - value
+ "strex %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
+ "teq %[tmp], #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
+ : [original] "=&r" (original), // %0
+ [result] "=&r" (result), // %1
+ [tmp] "=&l" (tmp), // %2
+ [storage] "+Q" (storage) // %3
+ : [value] "Ir" (v) // %4
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return result;
+ }
+
+ static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
+ {
+ gcc_arm_operations_base::fence_before(order);
+ uint32_t tmp;
+ storage_type original, result;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
+ "1:\n"
+ "ldrex %[original], %[storage]\n" // original = *(&storage)
+ "and %[result], %[original], %[value]\n" // result = original & value
+ "strex %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
+ "teq %[tmp], #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
+ : [original] "=&r" (original), // %0
+ [result] "=&r" (result), // %1
+ [tmp] "=&l" (tmp), // %2
+ [storage] "+Q" (storage) // %3
+ : [value] "Ir" (v) // %4
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return result;
+ }
+
+ static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
+ {
+ gcc_arm_operations_base::fence_before(order);
+ uint32_t tmp;
+ storage_type original, result;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
+ "1:\n"
+ "ldrex %[original], %[storage]\n" // original = *(&storage)
+ "orr %[result], %[original], %[value]\n" // result = original | value
+ "strex %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
+ "teq %[tmp], #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
+ : [original] "=&r" (original), // %0
+ [result] "=&r" (result), // %1
+ [tmp] "=&l" (tmp), // %2
+ [storage] "+Q" (storage) // %3
+ : [value] "Ir" (v) // %4
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return result;
+ }
+
+ static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
+ {
+ gcc_arm_operations_base::fence_before(order);
+ uint32_t tmp;
+ storage_type original, result;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
+ "1:\n"
+ "ldrex %[original], %[storage]\n" // original = *(&storage)
+ "eor %[result], %[original], %[value]\n" // result = original ^ value
+ "strex %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
+ "teq %[tmp], #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
+ : [original] "=&r" (original), // %0
+ [result] "=&r" (result), // %1
+ [tmp] "=&l" (tmp), // %2
+ [storage] "+Q" (storage) // %3
+ : [value] "Ir" (v) // %4
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return result;
+ }
+
static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
{
gcc_arm_operations_base::fence_before(order);
@@ -242,21 +810,43 @@ struct extra_operations< Base, 4u, Signed > :
return original;
}
- static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
+ static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
{
- fetch_negate(storage, order);
+ gcc_arm_operations_base::fence_before(order);
+ uint32_t tmp;
+ storage_type original, result;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%[tmp])
+ "1:\n"
+ "ldrex %[original], %[storage]\n" // original = *(&storage)
+ "mvn %[result], %[original]\n" // result = NOT original
+ "strex %[tmp], %[result], %[storage]\n" // *(&storage) = result, tmp = store failed
+ "teq %[tmp], #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%[tmp])
+ : [original] "=&r" (original), // %0
+ [result] "=&r" (result), // %1
+ [tmp] "=&l" (tmp), // %2
+ [storage] "+Q" (storage) // %3
+ :
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return result;
}
+};
- static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
- {
- fetch_complement(storage, order);
- }
+template< typename Base, bool Signed >
+struct extra_operations< Base, 4u, Signed, true > :
+ public gcc_arm_extra_operations_common< gcc_arm_extra_operations< Base, 4u, Signed > >
+{
};
#if defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXD_STREXD)
template< typename Base, bool Signed >
-struct extra_operations< Base, 8u, Signed > :
+struct gcc_arm_extra_operations< Base, 8u, Signed > :
public generic_extra_operations< Base, 8u, Signed >
{
typedef generic_extra_operations< Base, 8u, Signed > base_type;
@@ -290,7 +880,7 @@ struct extra_operations< Base, 8u, Signed > :
return original;
}
- static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
+ static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
{
gcc_arm_operations_base::fence_before(order);
storage_type original, result;
@@ -302,6 +892,8 @@ struct extra_operations< Base, 8u, Signed > :
"ldrexd %1, %H1, [%3]\n" // original = *(&storage)
"mvn %2, %1\n" // result = NOT original
"mvn %H2, %H1\n"
+ "adds %2, %2, #1\n" // result = result + 1
+ "adc %H2, %H2, #0\n"
"strexd %0, %2, %H2, [%3]\n" // *(&storage) = result, tmp = store failed
"teq %0, #0\n" // flags = tmp==0
"bne 1b\n" // if (!flags.equal) goto retry
@@ -313,18 +905,201 @@ struct extra_operations< Base, 8u, Signed > :
: BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
);
gcc_arm_operations_base::fence_after(order);
- return original;
+ return result;
}
- static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
+ static BOOST_FORCEINLINE storage_type add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
{
- fetch_negate(storage, order);
+ gcc_arm_operations_base::fence_before(order);
+ storage_type original, result;
+ uint32_t tmp;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
+ "1:\n"
+ "ldrexd %1, %H1, [%3]\n" // original = *(&storage)
+ "adds %2, %1, %4\n" // result = original + value
+ "adc %H2, %H1, %H4\n"
+ "strexd %0, %2, %H2, [%3]\n" // *(&storage) = result, tmp = store failed
+ "teq %0, #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
+ : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
+ "=&r" (original), // %1
+ "=&r" (result) // %2
+ : "r" (&storage), // %3
+ "r" (v) // %4
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return result;
}
- static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
+ static BOOST_FORCEINLINE storage_type sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
{
- fetch_complement(storage, order);
+ gcc_arm_operations_base::fence_before(order);
+ storage_type original, result;
+ uint32_t tmp;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
+ "1:\n"
+ "ldrexd %1, %H1, [%3]\n" // original = *(&storage)
+ "subs %2, %1, %4\n" // result = original - value
+ "sbc %H2, %H1, %H4\n"
+ "strexd %0, %2, %H2, [%3]\n" // *(&storage) = result, tmp = store failed
+ "teq %0, #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
+ : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
+ "=&r" (original), // %1
+ "=&r" (result) // %2
+ : "r" (&storage), // %3
+ "r" (v) // %4
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return result;
+ }
+
+ static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
+ {
+ gcc_arm_operations_base::fence_before(order);
+ storage_type original, result;
+ uint32_t tmp;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
+ "1:\n"
+ "ldrexd %1, %H1, [%3]\n" // original = *(&storage)
+ "and %2, %1, %4\n" // result = original & value
+ "and %H2, %H1, %H4\n"
+ "strexd %0, %2, %H2, [%3]\n" // *(&storage) = result, tmp = store failed
+ "teq %0, #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
+ : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
+ "=&r" (original), // %1
+ "=&r" (result) // %2
+ : "r" (&storage), // %3
+ "r" (v) // %4
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return result;
}
+
+ static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
+ {
+ gcc_arm_operations_base::fence_before(order);
+ storage_type original, result;
+ uint32_t tmp;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
+ "1:\n"
+ "ldrexd %1, %H1, [%3]\n" // original = *(&storage)
+ "orr %2, %1, %4\n" // result = original | value
+ "orr %H2, %H1, %H4\n"
+ "strexd %0, %2, %H2, [%3]\n" // *(&storage) = result, tmp = store failed
+ "teq %0, #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
+ : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
+ "=&r" (original), // %1
+ "=&r" (result) // %2
+ : "r" (&storage), // %3
+ "r" (v) // %4
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return result;
+ }
+
+ static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
+ {
+ gcc_arm_operations_base::fence_before(order);
+ storage_type original, result;
+ uint32_t tmp;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
+ "1:\n"
+ "ldrexd %1, %H1, [%3]\n" // original = *(&storage)
+ "eor %2, %1, %4\n" // result = original ^ value
+ "eor %H2, %H1, %H4\n"
+ "strexd %0, %2, %H2, [%3]\n" // *(&storage) = result, tmp = store failed
+ "teq %0, #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
+ : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
+ "=&r" (original), // %1
+ "=&r" (result) // %2
+ : "r" (&storage), // %3
+ "r" (v) // %4
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return result;
+ }
+
+ static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
+ {
+ gcc_arm_operations_base::fence_before(order);
+ storage_type original, result;
+ uint32_t tmp;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
+ "1:\n"
+ "ldrexd %1, %H1, [%3]\n" // original = *(&storage)
+ "mvn %2, %1\n" // result = NOT original
+ "mvn %H2, %H1\n"
+ "strexd %0, %2, %H2, [%3]\n" // *(&storage) = result, tmp = store failed
+ "teq %0, #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
+ : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
+ "=&r" (original), // %1
+ "=&r" (result) // %2
+ : "r" (&storage) // %3
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return original;
+ }
+
+ static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
+ {
+ gcc_arm_operations_base::fence_before(order);
+ storage_type original, result;
+ uint32_t tmp;
+ __asm__ __volatile__
+ (
+ BOOST_ATOMIC_DETAIL_ARM_ASM_START(%0)
+ "1:\n"
+ "ldrexd %1, %H1, [%3]\n" // original = *(&storage)
+ "mvn %2, %1\n" // result = NOT original
+ "mvn %H2, %H1\n"
+ "strexd %0, %2, %H2, [%3]\n" // *(&storage) = result, tmp = store failed
+ "teq %0, #0\n" // flags = tmp==0
+ "bne 1b\n" // if (!flags.equal) goto retry
+ BOOST_ATOMIC_DETAIL_ARM_ASM_END(%0)
+ : BOOST_ATOMIC_DETAIL_ARM_ASM_TMPREG_CONSTRAINT(tmp), // %0
+ "=&r" (original), // %1
+ "=&r" (result) // %2
+ : "r" (&storage) // %3
+ : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"
+ );
+ gcc_arm_operations_base::fence_after(order);
+ return result;
+ }
+};
+
+template< typename Base, bool Signed >
+struct extra_operations< Base, 8u, Signed, true > :
+ public gcc_arm_extra_operations_common< gcc_arm_extra_operations< Base, 8u, Signed > >
+{
};
#endif // defined(BOOST_ATOMIC_DETAIL_ARM_HAS_LDREXD_STREXD)