From 1cf7cadff1e0cab971be2279be8d6f4efa6c1399 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 10:55:58 +0300 Subject: [PATCH 001/132] atomic_ref (#2, p0019r8) --- stl/inc/atomic | 611 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 410 insertions(+), 201 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 8f78cb684dc..a223a9a246b 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -289,9 +289,24 @@ struct _Atomic_padded<_Ty, false> { }; #endif // TRANSITION, ABI +template +struct _Atomic_storage_types { + using _TVal = _Ty; + using _TStorage = _Atomic_padded<_Ty>; + using _TConstr = const _Ty; +}; + +template +struct _Atomic_storage_types<_Ty&> { + using _TVal = _Ty; + using _TStorage = _Ty&; + using _TConstr = _Ty&; +}; + + // STRUCT TEMPLATE _Atomic_storage #if 1 // TRANSITION, ABI -template +template ::_TVal)> #else // ^^^ don't break ABI / break ABI vvv template ::_Storage_size> #endif // TRANSITION, ABI @@ -299,13 +314,15 @@ struct _Atomic_storage { // Provides operations common to all specializations of std::atomic, load, store, exchange, and CAS. // Locking version used when hardware has no atomic operations for sizeof(_Ty). + using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; + _Atomic_storage() = default; - /* implicit */ constexpr _Atomic_storage(const _Ty _Value) noexcept : _Storage(_Value) { + /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept : _Storage(_Value) { // non-atomically initialize this atomic } - void store(const _Ty _Value, const memory_order _Order = memory_order_seq_cst) noexcept { + void store(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { // store with sequential consistency _Check_store_memory_order(_Order); _Lock(); @@ -313,37 +330,37 @@ struct _Atomic_storage { _Unlock(); } - _NODISCARD _Ty load(const memory_order _Order = memory_order_seq_cst) const noexcept { + _NODISCARD _TVal load(const memory_order _Order = memory_order_seq_cst) const noexcept { // load with sequential consistency _Check_load_memory_order(_Order); _Lock(); - _Ty _Local(_Storage); + _TVal _Local(_Storage); _Unlock(); return _Local; } - _Ty exchange(const _Ty _Value, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal exchange(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { // exchange _Value with _Storage with sequential consistency _Check_memory_order(_Order); _Lock(); - _Ty _Result(_Storage); + _TVal _Result(_Storage); _Storage = _Value; _Unlock(); return _Result; } - bool compare_exchange_strong(_Ty& _Expected, const _Ty _Desired, + bool compare_exchange_strong(_TVal& _Expected, const _TVal _Desired, const memory_order _Order = memory_order_seq_cst) noexcept { // CAS with sequential consistency, plain _Check_memory_order(_Order); const auto _Storage_ptr = _STD addressof(_Storage); const auto _Expected_ptr = _STD addressof(_Expected); bool _Result; _Lock(); - if (_CSTD memcmp(_Storage_ptr, _Expected_ptr, sizeof(_Ty)) == 0) { - _CSTD memcpy(_Storage_ptr, _STD addressof(_Desired), sizeof(_Ty)); + if (_CSTD memcmp(_Storage_ptr, _Expected_ptr, sizeof(_TVal)) == 0) { + _CSTD memcpy(_Storage_ptr, _STD addressof(_Desired), sizeof(_TVal)); _Result = true; } else { - _CSTD memcpy(_Expected_ptr, _Storage_ptr, sizeof(_Ty)); + _CSTD memcpy(_Expected_ptr, _Storage_ptr, sizeof(_TVal)); _Result = false; } @@ -398,13 +415,17 @@ public: template struct _Atomic_storage<_Ty, 1> { // lock-free using 1-byte intrinsics + + using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; + _Atomic_storage() = default; - /* implicit */ constexpr _Atomic_storage(const _Ty _Value) noexcept : _Storage{_Value} { + /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Storage{_Value} { // non-atomically initialize this atomic } - void store(const _Ty _Value) noexcept { // store with sequential consistency + void store(const _TVal _Value) noexcept { // store with sequential consistency const auto _Mem = _Atomic_address_as(_Storage); const char _As_bytes = _Atomic_reinterpret_as(_Value); #if defined(_M_ARM) || defined(_M_ARM64) @@ -416,7 +437,7 @@ struct _Atomic_storage<_Ty, 1> { // lock-free using 1-byte intrinsics #endif // hardware } - void store(const _Ty _Value, const memory_order _Order) noexcept { // store with given memory order + void store(const _TVal _Value, const memory_order _Order) noexcept { // store with given memory order const char _As_bytes = _Atomic_reinterpret_as(_Value); switch (_Order) { case memory_order_relaxed: @@ -438,19 +459,19 @@ struct _Atomic_storage<_Ty, 1> { // lock-free using 1-byte intrinsics } } - _NODISCARD _Ty load() const noexcept { // load with sequential consistency + _NODISCARD _TVal load() const noexcept { // load with sequential consistency char _As_bytes = _ISO_VOLATILE_LOAD8(_Storage); _Compiler_or_memory_barrier(); - return reinterpret_cast<_Ty&>(_As_bytes); + return reinterpret_cast<_TVal&>(_As_bytes); } - _NODISCARD _Ty load(const memory_order _Order) const noexcept { // load with given memory order + _NODISCARD _TVal load(const memory_order _Order) const noexcept { // load with given memory order char _As_bytes = _ISO_VOLATILE_LOAD8(_Storage); _Load_barrier(_Order); - return reinterpret_cast<_Ty&>(_As_bytes); + return reinterpret_cast<_TVal&>(_As_bytes); } - _Ty exchange(const _Ty _Value, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal exchange(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { // exchange with given memory order char _As_bytes; _ATOMIC_CHOOSE_INTRINSIC(_Order, _As_bytes, _InterlockedExchange8, _Atomic_address_as(_Storage), @@ -458,7 +479,7 @@ struct _Atomic_storage<_Ty, 1> { // lock-free using 1-byte intrinsics return reinterpret_cast<_Ty&>(_As_bytes); } - bool compare_exchange_strong(_Ty& _Expected, const _Ty _Desired, + bool compare_exchange_strong(_TVal& _Expected, const _TVal _Desired, const memory_order _Order = memory_order_seq_cst) noexcept { // CAS with given memory order const char _Expected_bytes = _Atomic_reinterpret_as(_Expected); // read before atomic operation char _Prev_bytes; @@ -472,18 +493,21 @@ struct _Atomic_storage<_Ty, 1> { // lock-free using 1-byte intrinsics return false; } - _Atomic_padded<_Ty> _Storage; + typename _Atomic_storage_types<_Ty>::_TStorage _Storage; }; template struct _Atomic_storage<_Ty, 2> { // lock-free using 2-byte intrinsics + + using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; + _Atomic_storage() = default; - /* implicit */ constexpr _Atomic_storage(const _Ty _Value) noexcept : _Storage{_Value} { + /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept : _Storage{_Value} { // non-atomically initialize this atomic } - void store(const _Ty _Value) noexcept { // store with sequential consistency + void store(const _TVal _Value) noexcept { // store with sequential consistency const auto _Mem = _Atomic_address_as(_Storage); const short _As_bytes = _Atomic_reinterpret_as(_Value); #if defined(_M_ARM) || defined(_M_ARM64) @@ -495,7 +519,7 @@ struct _Atomic_storage<_Ty, 2> { // lock-free using 2-byte intrinsics #endif // hardware } - void store(const _Ty _Value, const memory_order _Order) noexcept { // store with given memory order + void store(const _TVal _Value, const memory_order _Order) noexcept { // store with given memory order const short _As_bytes = _Atomic_reinterpret_as(_Value); switch (_Order) { case memory_order_relaxed: @@ -517,27 +541,27 @@ struct _Atomic_storage<_Ty, 2> { // lock-free using 2-byte intrinsics } } - _NODISCARD _Ty load() const noexcept { // load with sequential consistency + _NODISCARD _TVal load() const noexcept { // load with sequential consistency short _As_bytes = _ISO_VOLATILE_LOAD16(_Storage); _Compiler_or_memory_barrier(); - return reinterpret_cast<_Ty&>(_As_bytes); + return reinterpret_cast<_TVal&>(_As_bytes); } - _NODISCARD _Ty load(const memory_order _Order) const noexcept { // load with given memory order + _NODISCARD _TVal load(const memory_order _Order) const noexcept { // load with given memory order short _As_bytes = _ISO_VOLATILE_LOAD16(_Storage); _Load_barrier(_Order); - return reinterpret_cast<_Ty&>(_As_bytes); + return reinterpret_cast<_TVal&>(_As_bytes); } - _Ty exchange(const _Ty _Value, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal exchange(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { // exchange with given memory order short _As_bytes; _ATOMIC_CHOOSE_INTRINSIC(_Order, _As_bytes, _InterlockedExchange16, _Atomic_address_as(_Storage), _Atomic_reinterpret_as(_Value)); - return reinterpret_cast<_Ty&>(_As_bytes); + return reinterpret_cast<_TVal&>(_As_bytes); } - bool compare_exchange_strong(_Ty& _Expected, const _Ty _Desired, + bool compare_exchange_strong(_TVal& _Expected, const _TVal _Desired, const memory_order _Order = memory_order_seq_cst) noexcept { // CAS with given memory order const short _Expected_bytes = _Atomic_reinterpret_as(_Expected); // read before atomic operation short _Prev_bytes; @@ -551,18 +575,22 @@ struct _Atomic_storage<_Ty, 2> { // lock-free using 2-byte intrinsics return false; } - _Atomic_padded<_Ty> _Storage; + typename _Atomic_storage_types<_Ty>::_TStorage _Storage; }; template struct _Atomic_storage<_Ty, 4> { // lock-free using 4-byte intrinsics + + using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; + _Atomic_storage() = default; - /* implicit */ constexpr _Atomic_storage(const _Ty _Value) noexcept : _Storage{_Value} { + /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Storage{_Value} { // non-atomically initialize this atomic } - void store(const _Ty _Value) noexcept { // store with sequential consistency + void store(const _TVal _Value) noexcept { // store with sequential consistency #if defined(_M_ARM) || defined(_M_ARM64) _Memory_barrier(); _ISO_VOLATILE_STORE32(_Storage, _Atomic_reinterpret_as(_Value)); @@ -572,7 +600,7 @@ struct _Atomic_storage<_Ty, 4> { // lock-free using 4-byte intrinsics #endif // hardware } - void store(const _Ty _Value, const memory_order _Order) noexcept { // store with given memory order + void store(const _TVal _Value, const memory_order _Order) noexcept { // store with given memory order const int _As_bytes = _Atomic_reinterpret_as(_Value); switch (_Order) { case memory_order_relaxed: @@ -594,27 +622,27 @@ struct _Atomic_storage<_Ty, 4> { // lock-free using 4-byte intrinsics } } - _NODISCARD _Ty load() const noexcept { // load with sequential consistency + _NODISCARD _TVal load() const noexcept { // load with sequential consistency auto _As_bytes = _ISO_VOLATILE_LOAD32(_Storage); _Compiler_or_memory_barrier(); - return reinterpret_cast<_Ty&>(_As_bytes); + return reinterpret_cast<_TVal&>(_As_bytes); } - _NODISCARD _Ty load(const memory_order _Order) const noexcept { // load with given memory order + _NODISCARD _TVal load(const memory_order _Order) const noexcept { // load with given memory order auto _As_bytes = _ISO_VOLATILE_LOAD32(_Storage); _Load_barrier(_Order); - return reinterpret_cast<_Ty&>(_As_bytes); + return reinterpret_cast<_TVal&>(_As_bytes); } - _Ty exchange(const _Ty _Value, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal exchange(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { // exchange with given memory order long _As_bytes; _ATOMIC_CHOOSE_INTRINSIC(_Order, _As_bytes, _InterlockedExchange, _Atomic_address_as(_Storage), _Atomic_reinterpret_as(_Value)); - return reinterpret_cast<_Ty&>(_As_bytes); + return reinterpret_cast<_TVal&>(_As_bytes); } - bool compare_exchange_strong(_Ty& _Expected, const _Ty _Desired, + bool compare_exchange_strong(_TVal& _Expected, const _TVal _Desired, const memory_order _Order = memory_order_seq_cst) noexcept { // CAS with given memory order const long _Expected_bytes = _Atomic_reinterpret_as(_Expected); // read before atomic operation long _Prev_bytes; @@ -624,30 +652,33 @@ struct _Atomic_storage<_Ty, 4> { // lock-free using 4-byte intrinsics return true; } - _CSTD memcpy(_STD addressof(_Expected), &_Prev_bytes, sizeof(_Ty)); + _CSTD memcpy(_STD addressof(_Expected), &_Prev_bytes, sizeof(_TVal)); return false; } - _Atomic_padded<_Ty> _Storage; + typename _Atomic_storage_types<_Ty>::_TStorage _Storage; }; template struct _Atomic_storage<_Ty, 8> { // lock-free using 8-byte intrinsics + + using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; + _Atomic_storage() = default; - /* implicit */ constexpr _Atomic_storage(const _Ty _Value) noexcept : _Storage{_Value} { + /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept : _Storage{_Value} { // non-atomically initialize this atomic } #ifdef _M_IX86 - void store(const _Ty _Value, const memory_order _Order = memory_order_seq_cst) noexcept { + void store(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { // store with (effectively) sequential consistency _Check_store_memory_order(_Order); (void) exchange(_Value, _Order); } #else // ^^^ _M_IX86 / !_M_IX86 vvv - void store(const _Ty _Value) noexcept { // store with sequential consistency + void store(const _TVal _Value) noexcept { // store with sequential consistency const auto _Mem = _Atomic_address_as(_Storage); const long long _As_bytes = _Atomic_reinterpret_as(_Value); #ifdef _M_ARM64 @@ -659,7 +690,7 @@ struct _Atomic_storage<_Ty, 8> { // lock-free using 8-byte intrinsics #endif // _M_ARM64 } - void store(const _Ty _Value, const memory_order _Order) noexcept { // store with given memory order + void store(const _TVal _Value, const memory_order _Order) noexcept { // store with given memory order const long long _As_bytes = _Atomic_reinterpret_as(_Value); switch (_Order) { case memory_order_relaxed: @@ -682,7 +713,7 @@ struct _Atomic_storage<_Ty, 8> { // lock-free using 8-byte intrinsics } #endif // _M_IX86 - _NODISCARD _Ty load() const noexcept { // load with sequential consistency + _NODISCARD _TVal load() const noexcept { // load with sequential consistency const auto _Mem = _Atomic_address_as(_Storage); long long _As_bytes; #if defined(_M_ARM) @@ -695,10 +726,10 @@ struct _Atomic_storage<_Ty, 8> { // lock-free using 8-byte intrinsics _As_bytes = *_Mem; _Compiler_barrier(); #endif // hardware - return reinterpret_cast<_Ty&>(_As_bytes); + return reinterpret_cast<_TVal&>(_As_bytes); } - _NODISCARD _Ty load(const memory_order _Order) const noexcept { // load with given memory order + _NODISCARD _TVal load(const memory_order _Order) const noexcept { // load with given memory order const auto _Mem = _Atomic_address_as(_Storage); #if defined(_M_ARM) long long _As_bytes = __ldrexd(_Mem); @@ -708,29 +739,29 @@ struct _Atomic_storage<_Ty, 8> { // lock-free using 8-byte intrinsics long long _As_bytes = *_Mem; #endif // hardware _Load_barrier(_Order); - return reinterpret_cast<_Ty&>(_As_bytes); + return reinterpret_cast<_TVal&>(_As_bytes); } #ifdef _M_IX86 - _Ty exchange(const _Ty _Value, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal exchange(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { // exchange with (effectively) sequential consistency - _Ty _Temp{load()}; + _TVal _Temp{load()}; while (!compare_exchange_strong(_Temp, _Value, _Order)) { // keep trying } return _Temp; } #else // ^^^ _M_IX86 / !_M_IX86 vvv - _Ty exchange(const _Ty _Value, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal exchange(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { // exchange with given memory order long long _As_bytes; _ATOMIC_CHOOSE_INTRINSIC(_Order, _As_bytes, _InterlockedExchange64, _Atomic_address_as(_Storage), _Atomic_reinterpret_as(_Value)); - return reinterpret_cast<_Ty&>(_As_bytes); + return reinterpret_cast<_TVal&>(_As_bytes); } #endif // _M_IX86 - bool compare_exchange_strong(_Ty& _Expected, const _Ty _Desired, + bool compare_exchange_strong(_TVal& _Expected, const _TVal _Desired, const memory_order _Order = memory_order_seq_cst) noexcept { // CAS with given memory order const long long _Expected_bytes = _Atomic_reinterpret_as(_Expected); // read before atomic operation long long _Prev_bytes; @@ -740,39 +771,42 @@ struct _Atomic_storage<_Ty, 8> { // lock-free using 8-byte intrinsics return true; } - _CSTD memcpy(_STD addressof(_Expected), &_Prev_bytes, sizeof(_Ty)); + _CSTD memcpy(_STD addressof(_Expected), &_Prev_bytes, sizeof(_TVal)); return false; } - _Atomic_padded<_Ty> _Storage; + typename _Atomic_storage_types<_Ty>::_TStorage _Storage; }; #if 0 // TRANSITION, ABI #if defined(_M_X64) || defined(_M_ARM64) template struct _Atomic_storage<_Ty, 16> { // lock-free using 16-byte intrinsics + + using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; + _Atomic_storage() = default; - /* implicit */ constexpr _Atomic_storage(const _Ty _Value) noexcept + /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept : _Storage{_Value} {} // non-atomically initialize this atomic - void store(const _Ty _Value) noexcept { // store with sequential consistency + void store(const _TVal _Value) noexcept { // store with sequential consistency (void) exchange(_Value); } - void store(const _Ty _Value, const memory_order _Order) noexcept { // store with given memory order + void store(const _TVal _Value, const memory_order _Order) noexcept { // store with given memory order _Check_store_memory_order(_Order); (void) exchange(_Value, _Order); } - _NODISCARD _Ty load() const noexcept { // load with sequential consistency + _NODISCARD _TVal load() const noexcept { // load with sequential consistency long long* const _Storage_ptr = const_cast(_Atomic_address_as(_Storage)); _Int128 _Result{}; // atomic CAS 0 with 0 (void) _STD_COMPARE_EXCHANGE_128(_Storage_ptr, 0, 0, &_Result._Low); - return reinterpret_cast<_Ty&>(_Result); + return reinterpret_cast<_TVal&>(_Result); } - _NODISCARD _Ty load(const memory_order _Order) const noexcept { // load with given memory order + _NODISCARD _TVal load(const memory_order _Order) const noexcept { // load with given memory order #ifdef _M_ARM64 long long* const _Storage_ptr = const_cast(_Atomic_address_as(_Storage)); _Int128 _Result{}; // atomic CAS 0 with 0 @@ -794,14 +828,14 @@ struct _Atomic_storage<_Ty, 16> { // lock-free using 16-byte intrinsics break; } - return reinterpret_cast<_Ty&>(_Result); + return reinterpret_cast<_TVal&>(_Result); #else // ^^^ _M_ARM64 / _M_X64 vvv _Check_load_memory_order(_Order); return load(); #endif // _M_ARM64 } - _Ty exchange(const _Ty _Value) noexcept { // exchange with sequential consistency + _TVal exchange(const _TVal _Value) noexcept { // exchange with sequential consistency _Ty _Result{_Value}; while (!compare_exchange_strong(_Result, _Value)) { // keep trying } @@ -809,7 +843,7 @@ struct _Atomic_storage<_Ty, 16> { // lock-free using 16-byte intrinsics return _Result; } - _Ty exchange(const _Ty _Value, const memory_order _Order) noexcept { // exchange with given memory order + _TVal exchange(const _TVal _Value, const memory_order _Order) noexcept { // exchange with given memory order _Ty _Result{_Value}; while (!compare_exchange_strong(_Result, _Value, _Order)) { // keep trying } @@ -817,12 +851,12 @@ struct _Atomic_storage<_Ty, 16> { // lock-free using 16-byte intrinsics return _Result; } - bool compare_exchange_strong(_Ty& _Expected, const _Ty _Desired, + bool compare_exchange_strong(_TVal& _Expected, const _TVal _Desired, const memory_order _Order = memory_order_seq_cst) noexcept { // CAS with given memory order _Int128 _Desired_bytes{}; - _CSTD memcpy(&_Desired_bytes, _STD addressof(_Desired), sizeof(_Ty)); + _CSTD memcpy(&_Desired_bytes, _STD addressof(_Desired), sizeof(_TVal)); _Int128 _Expected_temp{}; - _CSTD memcpy(&_Expected_temp, _STD addressof(_Expected), sizeof(_Ty)); + _CSTD memcpy(&_Expected_temp, _STD addressof(_Expected), sizeof(_TVal)); unsigned char _Result; #ifdef _M_ARM64 _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedCompareExchange128, @@ -833,7 +867,7 @@ struct _Atomic_storage<_Ty, 16> { // lock-free using 16-byte intrinsics &reinterpret_cast(_Storage), _Desired_bytes._High, _Desired_bytes._Low, &_Expected_temp._Low); #endif // _M_ARM64 if (_Result == 0) { - _CSTD memcpy(_STD addressof(_Expected), &_Expected_temp, sizeof(_Ty)); + _CSTD memcpy(_STD addressof(_Expected), &_Expected_temp, sizeof(_TVal)); } return _Result != 0; @@ -844,7 +878,7 @@ struct _Atomic_storage<_Ty, 16> { // lock-free using 16-byte intrinsics long long _High; }; - _Atomic_padded<_Ty> _Storage; + typename _Atomic_storage_types<_Ty>::_TStorage _Storage; }; #endif // defined(_M_X64) || defined(_M_ARM64) #endif // TRANSITION, ABI @@ -859,59 +893,62 @@ struct _Atomic_integral<_Ty, 1> : _Atomic_storage<_Ty> { // atomic integral oper #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 _Atomic_integral() = default; - /* implicit */ constexpr _Atomic_integral(const _Ty _Value) noexcept : _Base(_Value) {} + /* implicit */ constexpr _Atomic_integral(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Base(_Value) {} #else // ^^^ workaround / no workaround vvv using _Base::_Base; #endif // ^^^ no workaround ^^^ - _Ty fetch_add(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + using _TVal = typename _Base::_TVal; + + _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { char _Result; _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedExchangeAdd8, _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_Ty>(_Result); + return static_cast<_TVal>(_Result); } - _Ty fetch_and(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal fetch_and(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { char _Result; _ATOMIC_CHOOSE_INTRINSIC( _Order, _Result, _InterlockedAnd8, _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_Ty>(_Result); + return static_cast<_TVal>(_Result); } - _Ty fetch_or(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal fetch_or(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { char _Result; _ATOMIC_CHOOSE_INTRINSIC( _Order, _Result, _InterlockedOr8, _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_Ty>(_Result); + return static_cast<_TVal>(_Result); } - _Ty fetch_xor(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal fetch_xor(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { char _Result; _ATOMIC_CHOOSE_INTRINSIC( _Order, _Result, _InterlockedXor8, _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_Ty>(_Result); + return static_cast<_TVal>(_Result); } - _Ty operator++(int) noexcept { - return static_cast<_Ty>(_InterlockedExchangeAdd8(_Atomic_address_as(this->_Storage), 1)); + _TVal operator++(int) noexcept { + return static_cast<_TVal>(_InterlockedExchangeAdd8(_Atomic_address_as(this->_Storage), 1)); } - _Ty operator++() noexcept { + _TVal operator++() noexcept { unsigned char _Before = static_cast(_InterlockedExchangeAdd8(_Atomic_address_as(this->_Storage), 1)); ++_Before; - return static_cast<_Ty>(_Before); + return static_cast<_TVal>(_Before); } - _Ty operator--(int) noexcept { + _TVal operator--(int) noexcept { return static_cast<_Ty>(_InterlockedExchangeAdd8(_Atomic_address_as(this->_Storage), -1)); } - _Ty operator--() noexcept { + _TVal operator--() noexcept { unsigned char _Before = static_cast(_InterlockedExchangeAdd8(_Atomic_address_as(this->_Storage), -1)); --_Before; - return static_cast<_Ty>(_Before); + return static_cast<_TVal>(_Before); } }; @@ -922,59 +959,62 @@ struct _Atomic_integral<_Ty, 2> : _Atomic_storage<_Ty> { // atomic integral oper #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 _Atomic_integral() = default; - /* implicit */ constexpr _Atomic_integral(const _Ty _Value) noexcept : _Base(_Value) {} + /* implicit */ constexpr _Atomic_integral(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Base(_Value) {} #else // ^^^ workaround / no workaround vvv using _Base::_Base; #endif // ^^^ no workaround ^^^ - _Ty fetch_add(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + using _TVal = typename _Base::_TVal; + + _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { short _Result; _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedExchangeAdd16, _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_Ty>(_Result); + return static_cast<_TVal>(_Result); } - _Ty fetch_and(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal fetch_and(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { short _Result; _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedAnd16, _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_Ty>(_Result); + return static_cast<_TVal>(_Result); } - _Ty fetch_or(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal fetch_or(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { short _Result; _ATOMIC_CHOOSE_INTRINSIC( _Order, _Result, _InterlockedOr16, _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_Ty>(_Result); + return static_cast<_TVal>(_Result); } - _Ty fetch_xor(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal fetch_xor(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { short _Result; _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedXor16, _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_Ty>(_Result); + return static_cast<_TVal>(_Result); } - _Ty operator++(int) noexcept { + _TVal operator++(int) noexcept { unsigned short _After = static_cast(_InterlockedIncrement16(_Atomic_address_as(this->_Storage))); --_After; - return static_cast<_Ty>(_After); + return static_cast<_TVal>(_After); } - _Ty operator++() noexcept { - return static_cast<_Ty>(_InterlockedIncrement16(_Atomic_address_as(this->_Storage))); + _TVal operator++() noexcept { + return static_cast<_TVal>(_InterlockedIncrement16(_Atomic_address_as(this->_Storage))); } - _Ty operator--(int) noexcept { + _TVal operator--(int) noexcept { unsigned short _After = static_cast(_InterlockedDecrement16(_Atomic_address_as(this->_Storage))); ++_After; - return static_cast<_Ty>(_After); + return static_cast<_TVal>(_After); } - _Ty operator--() noexcept { - return static_cast<_Ty>(_InterlockedDecrement16(_Atomic_address_as(this->_Storage))); + _TVal operator--() noexcept { + return static_cast<_TVal>(_InterlockedDecrement16(_Atomic_address_as(this->_Storage))); } }; @@ -984,59 +1024,62 @@ struct _Atomic_integral<_Ty, 4> : _Atomic_storage<_Ty> { // atomic integral oper #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 _Atomic_integral() = default; - /* implicit */ constexpr _Atomic_integral(const _Ty _Value) noexcept : _Base(_Value) {} + /* implicit */ constexpr _Atomic_integral(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Base(_Value) {} #else // ^^^ workaround / no workaround vvv using _Base::_Base; #endif // ^^^ no workaround ^^^ - _Ty fetch_add(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + using _TVal = typename _Base::_TVal; + + _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { long _Result; _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedExchangeAdd, _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_Ty>(_Result); + return static_cast<_TVal>(_Result); } - _Ty fetch_and(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal fetch_and(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { long _Result; _ATOMIC_CHOOSE_INTRINSIC( _Order, _Result, _InterlockedAnd, _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_Ty>(_Result); + return static_cast<_TVal>(_Result); } - _Ty fetch_or(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal fetch_or(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { long _Result; _ATOMIC_CHOOSE_INTRINSIC( _Order, _Result, _InterlockedOr, _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_Ty>(_Result); + return static_cast<_TVal>(_Result); } - _Ty fetch_xor(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal fetch_xor(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { long _Result; _ATOMIC_CHOOSE_INTRINSIC( _Order, _Result, _InterlockedXor, _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_Ty>(_Result); + return static_cast<_TVal>(_Result); } - _Ty operator++(int) noexcept { + _TVal operator++(int) noexcept { unsigned long _After = static_cast(_InterlockedIncrement(_Atomic_address_as(this->_Storage))); --_After; - return static_cast<_Ty>(_After); + return static_cast<_TVal>(_After); } - _Ty operator++() noexcept { - return static_cast<_Ty>(_InterlockedIncrement(_Atomic_address_as(this->_Storage))); + _TVal operator++() noexcept { + return static_cast<_TVal>(_InterlockedIncrement(_Atomic_address_as(this->_Storage))); } - _Ty operator--(int) noexcept { + _TVal operator--(int) noexcept { unsigned long _After = static_cast(_InterlockedDecrement(_Atomic_address_as(this->_Storage))); ++_After; - return static_cast<_Ty>(_After); + return static_cast<_TVal>(_After); } - _Ty operator--() noexcept { - return static_cast<_Ty>(_InterlockedDecrement(_Atomic_address_as(this->_Storage))); + _TVal operator--() noexcept { + return static_cast<_TVal>(_InterlockedDecrement(_Atomic_address_as(this->_Storage))); } }; @@ -1046,113 +1089,115 @@ struct _Atomic_integral<_Ty, 8> : _Atomic_storage<_Ty> { // atomic integral oper #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 _Atomic_integral() = default; - /* implicit */ constexpr _Atomic_integral(const _Ty _Value) noexcept : _Base(_Value) {} + /* implicit */ constexpr _Atomic_integral(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept : _Base(_Value) {} #else // ^^^ workaround / no workaround vvv using _Base::_Base; #endif // ^^^ no workaround ^^^ + using _TVal = typename _Base::_TVal; + #ifdef _M_IX86 - _Ty fetch_add(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { // effectively sequential consistency - _Ty _Temp{this->load()}; + _TVal _Temp{this->load()}; while (!this->compare_exchange_strong(_Temp, _Temp + _Operand, _Order)) { // keep trying } return _Temp; } - _Ty fetch_and(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal fetch_and(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { // effectively sequential consistency - _Ty _Temp{this->load()}; + _TVal _Temp{this->load()}; while (!this->compare_exchange_strong(_Temp, _Temp & _Operand, _Order)) { // keep trying } return _Temp; } - _Ty fetch_or(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal fetch_or(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { // effectively sequential consistency - _Ty _Temp{this->load()}; + _TVal _Temp{this->load()}; while (!this->compare_exchange_strong(_Temp, _Temp | _Operand, _Order)) { // keep trying } return _Temp; } - _Ty fetch_xor(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal fetch_xor(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { // effectively sequential consistency - _Ty _Temp{this->load()}; + _TVal _Temp{this->load()}; while (!this->compare_exchange_strong(_Temp, _Temp ^ _Operand, _Order)) { // keep trying } return _Temp; } - _Ty operator++(int) noexcept { - return fetch_add(static_cast<_Ty>(1)); + _TVal operator++(int) noexcept { + return fetch_add(static_cast<_TVal>(1)); } - _Ty operator++() noexcept { - return fetch_add(static_cast<_Ty>(1)) + static_cast<_Ty>(1); + _TVal operator++() noexcept { + return fetch_add(static_cast<_TVal>(1)) + static_cast<_TVal>(1); } - _Ty operator--(int) noexcept { - return fetch_add(static_cast<_Ty>(-1)); + _TVal operator--(int) noexcept { + return fetch_add(static_cast<_TVal>(-1)); } - _Ty operator--() noexcept { - return fetch_add(static_cast<_Ty>(-1)) - static_cast<_Ty>(1); + _TVal operator--() noexcept { + return fetch_add(static_cast<_TVal>(-1)) - static_cast<_TVal>(1); } #else // ^^^ _M_IX86 / !_M_IX86 vvv - _Ty fetch_add(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { long long _Result; _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedExchangeAdd64, _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_Ty>(_Result); + return static_cast<_TVal>(_Result); } - _Ty fetch_and(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal fetch_and(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { long long _Result; _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedAnd64, _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_Ty>(_Result); + return static_cast<_TVal>(_Result); } - _Ty fetch_or(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal fetch_or(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { long long _Result; _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedOr64, _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_Ty>(_Result); + return static_cast<_TVal>(_Result); } - _Ty fetch_xor(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal fetch_xor(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { long long _Result; _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedXor64, _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_Ty>(_Result); + return static_cast<_TVal>(_Result); } - _Ty operator++(int) noexcept { + _TVal operator++(int) noexcept { unsigned long long _After = static_cast(_InterlockedIncrement64(_Atomic_address_as(this->_Storage))); --_After; - return static_cast<_Ty>(_After); + return static_cast<_TVal>(_After); } - _Ty operator++() noexcept { - return static_cast<_Ty>(_InterlockedIncrement64(_Atomic_address_as(this->_Storage))); + _TVal operator++() noexcept { + return static_cast<_TVal>(_InterlockedIncrement64(_Atomic_address_as(this->_Storage))); } - _Ty operator--(int) noexcept { + _TVal operator--(int) noexcept { unsigned long long _After = static_cast(_InterlockedDecrement64(_Atomic_address_as(this->_Storage))); ++_After; - return static_cast<_Ty>(_After); + return static_cast<_TVal>(_After); } - _Ty operator--() noexcept { - return static_cast<_Ty>(_InterlockedDecrement64(_Atomic_address_as(this->_Storage))); + _TVal operator--() noexcept { + return static_cast<_TVal>(_InterlockedDecrement64(_Atomic_address_as(this->_Storage))); } #endif // _M_IX86 }; @@ -1185,128 +1230,131 @@ struct _Atomic_integral_facade : _Atomic_integral<_Ty> { #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 _Atomic_integral_facade() = default; - /* implicit */ constexpr _Atomic_integral_facade(const _Ty _Value) noexcept : _Base(_Value) {} + /* implicit */ constexpr _Atomic_integral_facade(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Base(_Value) {} #else // ^^^ workaround / no workaround vvv using _Base::_Base; #endif // ^^^ no workaround ^^^ + using _TVal = typename _Base::_TVal; + // _Deprecate_non_lock_free_volatile is unnecessary here. // note: const_cast-ing away volatile is safe because all our intrinsics add volatile back on. // We make the primary functions non-volatile for better debug codegen, as non-volatile atomics // are far more common than volatile ones. using _Base::fetch_add; - _Ty fetch_add(const _Ty _Operand) volatile noexcept { + _TVal fetch_add(const _TVal _Operand) volatile noexcept { return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_add(_Operand); } - _Ty fetch_add(const _Ty _Operand, const memory_order _Order) volatile noexcept { + _TVal fetch_add(const _TVal _Operand, const memory_order _Order) volatile noexcept { return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_add(_Operand, _Order); } - _NODISCARD static _Ty _Negate(const _Ty _Value) noexcept { // returns two's complement negated value of _Value - return static_cast<_Ty>(0U - static_cast>(_Value)); + _NODISCARD static _TVal _Negate(const _TVal _Value) noexcept { // returns two's complement negated value of _Value + return static_cast<_TVal>(0U - static_cast>(_Value)); } - _Ty fetch_sub(const _Ty _Operand) noexcept { + _TVal fetch_sub(const _TVal _Operand) noexcept { return fetch_add(_Negate(_Operand)); } - _Ty fetch_sub(const _Ty _Operand) volatile noexcept { + _TVal fetch_sub(const _TVal _Operand) volatile noexcept { return fetch_add(_Negate(_Operand)); } - _Ty fetch_sub(const _Ty _Operand, const memory_order _Order) noexcept { + _TVal fetch_sub(const _TVal _Operand, const memory_order _Order) noexcept { return fetch_add(_Negate(_Operand), _Order); } - _Ty fetch_sub(const _Ty _Operand, const memory_order _Order) volatile noexcept { + _TVal fetch_sub(const _TVal _Operand, const memory_order _Order) volatile noexcept { return fetch_add(_Negate(_Operand), _Order); } using _Base::fetch_and; - _Ty fetch_and(const _Ty _Operand) volatile noexcept { + _TVal fetch_and(const _TVal _Operand) volatile noexcept { return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_and(_Operand); } - _Ty fetch_and(const _Ty _Operand, const memory_order _Order) volatile noexcept { + _TVal fetch_and(const _TVal _Operand, const memory_order _Order) volatile noexcept { return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_and(_Operand, _Order); } using _Base::fetch_or; - _Ty fetch_or(const _Ty _Operand) volatile noexcept { + _TVal fetch_or(const _TVal _Operand) volatile noexcept { return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_or(_Operand); } - _Ty fetch_or(const _Ty _Operand, const memory_order _Order) volatile noexcept { + _TVal fetch_or(const _TVal _Operand, const memory_order _Order) volatile noexcept { return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_or(_Operand, _Order); } using _Base::fetch_xor; - _Ty fetch_xor(const _Ty _Operand) volatile noexcept { + _TVal fetch_xor(const _TVal _Operand) volatile noexcept { return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_xor(_Operand); } - _Ty fetch_xor(const _Ty _Operand, const memory_order _Order) volatile noexcept { + _TVal fetch_xor(const _TVal _Operand, const memory_order _Order) volatile noexcept { return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_xor(_Operand, _Order); } using _Base::operator++; - _Ty operator++(int) volatile noexcept { + _TVal operator++(int) volatile noexcept { return const_cast<_Atomic_integral_facade*>(this)->_Base::operator++(0); } - _Ty operator++() volatile noexcept { + _TVal operator++() volatile noexcept { return const_cast<_Atomic_integral_facade*>(this)->_Base::operator++(); } using _Base::operator--; - _Ty operator--(int) volatile noexcept { + _TVal operator--(int) volatile noexcept { return const_cast<_Atomic_integral_facade*>(this)->_Base::operator--(0); } - _Ty operator--() volatile noexcept { + _TVal operator--() volatile noexcept { return const_cast<_Atomic_integral_facade*>(this)->_Base::operator--(); } - _Ty operator+=(const _Ty _Operand) noexcept { - return static_cast<_Ty>(this->_Base::fetch_add(_Operand) + _Operand); + _TVal operator+=(const _TVal _Operand) noexcept { + return static_cast<_TVal>(this->_Base::fetch_add(_Operand) + _Operand); } - _Ty operator+=(const _Ty _Operand) volatile noexcept { - return static_cast<_Ty>(const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_add(_Operand) + _Operand); + _TVal operator+=(const _TVal _Operand) volatile noexcept { + return static_cast<_TVal>(const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_add(_Operand) + _Operand); } - _Ty operator-=(const _Ty _Operand) noexcept { - return static_cast<_Ty>(fetch_sub(_Operand) - _Operand); + _TVal operator-=(const _TVal _Operand) noexcept { + return static_cast<_TVal>(fetch_sub(_Operand) - _Operand); } - _Ty operator-=(const _Ty _Operand) volatile noexcept { - return static_cast<_Ty>(const_cast<_Atomic_integral_facade*>(this)->fetch_sub(_Operand) - _Operand); + _TVal operator-=(const _TVal _Operand) volatile noexcept { + return static_cast<_TVal>(const_cast<_Atomic_integral_facade*>(this)->fetch_sub(_Operand) - _Operand); } - _Ty operator&=(const _Ty _Operand) noexcept { - return static_cast<_Ty>(this->_Base::fetch_and(_Operand) & _Operand); + _TVal operator&=(const _TVal _Operand) noexcept { + return static_cast<_TVal>(this->_Base::fetch_and(_Operand) & _Operand); } - _Ty operator&=(const _Ty _Operand) volatile noexcept { - return static_cast<_Ty>(const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_and(_Operand) & _Operand); + _TVal operator&=(const _TVal _Operand) volatile noexcept { + return static_cast<_TVal>(const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_and(_Operand) & _Operand); } - _Ty operator|=(const _Ty _Operand) noexcept { - return static_cast<_Ty>(this->_Base::fetch_or(_Operand) | _Operand); + _TVal operator|=(const _TVal _Operand) noexcept { + return static_cast<_TVal>(this->_Base::fetch_or(_Operand) | _Operand); } - _Ty operator|=(const _Ty _Operand) volatile noexcept { - return static_cast<_Ty>(const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_or(_Operand) | _Operand); + _TVal operator|=(const _TVal _Operand) volatile noexcept { + return static_cast<_TVal>(const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_or(_Operand) | _Operand); } - _Ty operator^=(const _Ty _Operand) noexcept { - return static_cast<_Ty>(this->_Base::fetch_xor(_Operand) ^ _Operand); + _TVal operator^=(const _TVal _Operand) noexcept { + return static_cast<_TVal>(this->_Base::fetch_xor(_Operand) ^ _Operand); } - _Ty operator^=(const _Ty _Operand) volatile noexcept { - return static_cast<_Ty>(const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_xor(_Operand) ^ _Operand); + _TVal operator^=(const _TVal _Operand) volatile noexcept { + return static_cast<_TVal>(const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_xor(_Operand) ^ _Operand); } }; @@ -1319,13 +1367,16 @@ struct _Atomic_floating : _Atomic_storage<_Ty> { #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 _Atomic_floating() = default; - /* implicit */ constexpr _Atomic_floating(const _Ty _Value) noexcept : _Base(_Value) {} + /* implicit */ constexpr _Atomic_floating(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Base(_Value) {} #else // ^^^ workaround / no workaround vvv using _Base::_Base; #endif // ^^^ no workaround ^^^ - _Ty fetch_add(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - _Ty _Temp{this->load(memory_order_relaxed)}; + using _TVal = typename _Base::_TVal; + + _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal _Temp{this->load(memory_order_relaxed)}; while (!this->compare_exchange_strong(_Temp, _Temp + _Operand, _Order)) { // keep trying } @@ -1337,11 +1388,11 @@ struct _Atomic_floating : _Atomic_storage<_Ty> { // note: const_cast-ing away volatile is safe because all our intrinsics add volatile back on. // We make the primary functions non-volatile for better debug codegen, as non-volatile atomics // are far more common than volatile ones. - _Ty fetch_add(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) volatile noexcept { + _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) volatile noexcept { return const_cast<_Atomic_floating*>(this)->fetch_add(_Operand, _Order); } - _Ty fetch_sub(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal fetch_sub(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { _Ty _Temp{this->load(memory_order_relaxed)}; while (!this->compare_exchange_strong(_Temp, _Temp - _Operand, _Order)) { // keep trying } @@ -1379,7 +1430,8 @@ struct _Atomic_pointer : _Atomic_storage<_Ty> { #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 _Atomic_pointer() = default; - /* implicit */ constexpr _Atomic_pointer(const _Ty _Value) noexcept : _Base(_Value) {} + /* implicit */ constexpr _Atomic_pointer(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Base(_Value) {} #else // ^^^ workaround / no workaround vvv using _Base::_Base; #endif // ^^^ no workaround ^^^ @@ -1659,6 +1711,163 @@ template atomic(_Ty) -> atomic<_Ty>; #endif // _HAS_CXX17 +#if _HAS_CXX20 +template +struct atomic_ref : _Choose_atomic_base_t<_Ty&> { // atomic value +private: + using _Base = _Choose_atomic_base_t<_Ty&>; + +public: + // clang-format off + static_assert(is_trivially_copyable_v<_Ty> && is_copy_constructible_v<_Ty> && is_move_constructible_v<_Ty> + && is_copy_assignable_v<_Ty> && is_move_assignable_v<_Ty>, + "atomic requires T to be trivially copyable, copy constructible, move constructible, copy assignable, " + "and move assignable."); + // clang-format on + + using value_type = _Ty; + + explicit atomic_ref(_Ty& _Value) noexcept : _Base(_Value) {} + + atomic_ref(const atomic_ref&) noexcept = default; + + atomic_ref& operator=(const atomic_ref&) = delete; + + static constexpr bool is_always_lock_free = _Is_always_lock_free; + +#if 1 // TRANSITION, ABI + _NODISCARD bool is_lock_free() const volatile noexcept { + constexpr bool _Result = sizeof(_Ty) <= 8 && (sizeof(_Ty) & sizeof(_Ty) - 1) == 0; + return _Result; + } + +#else // ^^^ don't break ABI / break ABI vvv + + _NODISCARD bool is_lock_free() const volatile noexcept { +#if _ATOMIC_HAS_DCAS + return sizeof(_Ty) <= 2 * sizeof(void*); +#else // ^^^ _ATOMIC_HAS_DCAS / !_ATOMIC_HAS_DCAS vvv + return sizeof(_Ty) <= sizeof(void*) || (sizeof(_Ty) <= 2 * sizeof(void*) && __std_atomic_has_cmpxchg16b()); +#endif // _ATOMIC_HAS_DCAS + } +#endif // TRANSITION, ABI + + _NODISCARD bool is_lock_free() const noexcept { + return static_cast(this)->is_lock_free(); + } + + _Ty operator=(const _Ty _Value) volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + this->store(_Value); + return _Value; + } + + _Ty operator=(const _Ty _Value) noexcept { + this->store(_Value); + return _Value; + } + + // For the following, we do the real implementation in the non-volatile function, and const_cast + // to call the non-volatile function in the volatile one. This is safe because all of the + // non-volatile functions reapply volatile, as all our intrinsics accept only volatile T *. + // We expect most atomics to be non-volatile, so making the real implementations + // non-volatile should result in better debug codegen. + using _Base::store; + void store(const _Ty _Value) volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + const_cast(this)->_Base::store(_Value); + } + + void store(const _Ty _Value, const memory_order _Order) volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + const_cast(this)->_Base::store(_Value, _Order); + } + + using _Base::load; + _NODISCARD _Ty load() const volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return const_cast(this)->_Base::load(); + } + + _NODISCARD _Ty load(const memory_order _Order) const volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return const_cast(this)->_Base::load(_Order); + } + + using _Base::exchange; + _Ty exchange(const _Ty _Value) volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return const_cast(this)->_Base::exchange(_Value); + } + + _Ty exchange(const _Ty _Value, const memory_order _Order) volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return const_cast(this)->_Base::exchange(_Value, _Order); + } + + using _Base::compare_exchange_strong; + bool compare_exchange_strong(_Ty& _Expected, const _Ty _Desired) volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return const_cast(this)->_Base::compare_exchange_strong(_Expected, _Desired); + } + + bool compare_exchange_strong(_Ty& _Expected, const _Ty _Desired, const memory_order _Order) volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return const_cast(this)->_Base::compare_exchange_strong(_Expected, _Desired, _Order); + } + + bool compare_exchange_strong(_Ty& _Expected, const _Ty _Desired, const memory_order _Success, + const memory_order _Failure) volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return this->compare_exchange_strong(_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); + } + + bool compare_exchange_strong( + _Ty& _Expected, const _Ty _Desired, const memory_order _Success, const memory_order _Failure) noexcept { + return this->compare_exchange_strong(_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); + } + + bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired) volatile noexcept { + // we have no weak CAS intrinsics, even on ARM32/ARM64, so fall back to strong + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return this->compare_exchange_strong(_Expected, _Desired); + } + + bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired) noexcept { + return this->compare_exchange_strong(_Expected, _Desired); + } + + bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired, const memory_order _Order) volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return this->compare_exchange_strong(_Expected, _Desired, _Order); + } + + bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired, const memory_order _Order) noexcept { + return this->compare_exchange_strong(_Expected, _Desired, _Order); + } + + bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired, const memory_order _Success, + const memory_order _Failure) volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return this->compare_exchange_strong(_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); + } + + bool compare_exchange_weak( + _Ty& _Expected, const _Ty _Desired, const memory_order _Success, const memory_order _Failure) noexcept { + return this->compare_exchange_strong(_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); + } + + operator _Ty() const volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return this->load(); + } + + operator _Ty() const noexcept { + return this->load(); + } +}; +#endif // _HAS_CXX20 + // NONMEMBER OPERATIONS ON ATOMIC TYPES template _NODISCARD bool atomic_is_lock_free(const volatile atomic<_Ty>* _Mem) noexcept { From 001e340e2e5d76a89bb096150e23a2513cb1d097 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 11:12:23 +0300 Subject: [PATCH 002/132] clang format, volatile overload --- stl/inc/atomic | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index a223a9a246b..6524c8cebc3 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -289,21 +289,20 @@ struct _Atomic_padded<_Ty, false> { }; #endif // TRANSITION, ABI -template +template struct _Atomic_storage_types { - using _TVal = _Ty; + using _TVal = _Ty; using _TStorage = _Atomic_padded<_Ty>; - using _TConstr = const _Ty; + using _TConstr = const _Ty; }; template struct _Atomic_storage_types<_Ty&> { - using _TVal = _Ty; + using _TVal = _Ty; using _TStorage = _Ty&; - using _TConstr = _Ty&; + using _TConstr = _Ty&; }; - // STRUCT TEMPLATE _Atomic_storage #if 1 // TRANSITION, ABI template ::_TVal)> @@ -318,7 +317,8 @@ struct _Atomic_storage { _Atomic_storage() = default; - /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept : _Storage(_Value) { + /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Storage(_Value) { // non-atomically initialize this atomic } @@ -503,7 +503,8 @@ struct _Atomic_storage<_Ty, 2> { // lock-free using 2-byte intrinsics _Atomic_storage() = default; - /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept : _Storage{_Value} { + /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Storage{_Value} { // non-atomically initialize this atomic } @@ -666,7 +667,8 @@ struct _Atomic_storage<_Ty, 8> { // lock-free using 8-byte intrinsics _Atomic_storage() = default; - /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept : _Storage{_Value} { + /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Storage{_Value} { // non-atomically initialize this atomic } @@ -1089,7 +1091,8 @@ struct _Atomic_integral<_Ty, 8> : _Atomic_storage<_Ty> { // atomic integral oper #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 _Atomic_integral() = default; - /* implicit */ constexpr _Atomic_integral(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept : _Base(_Value) {} + /* implicit */ constexpr _Atomic_integral(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Base(_Value) {} #else // ^^^ workaround / no workaround vvv using _Base::_Base; #endif // ^^^ no workaround ^^^ @@ -1775,45 +1778,45 @@ public: using _Base::store; void store(const _Ty _Value) volatile noexcept { static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - const_cast(this)->_Base::store(_Value); + const_cast(this)->_Base::store(_Value); } void store(const _Ty _Value, const memory_order _Order) volatile noexcept { static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - const_cast(this)->_Base::store(_Value, _Order); + const_cast(this)->_Base::store(_Value, _Order); } using _Base::load; _NODISCARD _Ty load() const volatile noexcept { static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return const_cast(this)->_Base::load(); + return const_cast(this)->_Base::load(); } _NODISCARD _Ty load(const memory_order _Order) const volatile noexcept { static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return const_cast(this)->_Base::load(_Order); + return const_cast(this)->_Base::load(_Order); } using _Base::exchange; _Ty exchange(const _Ty _Value) volatile noexcept { static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return const_cast(this)->_Base::exchange(_Value); + return const_cast(this)->_Base::exchange(_Value); } _Ty exchange(const _Ty _Value, const memory_order _Order) volatile noexcept { static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return const_cast(this)->_Base::exchange(_Value, _Order); + return const_cast(this)->_Base::exchange(_Value, _Order); } using _Base::compare_exchange_strong; bool compare_exchange_strong(_Ty& _Expected, const _Ty _Desired) volatile noexcept { static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return const_cast(this)->_Base::compare_exchange_strong(_Expected, _Desired); + return const_cast(this)->_Base::compare_exchange_strong(_Expected, _Desired); } bool compare_exchange_strong(_Ty& _Expected, const _Ty _Desired, const memory_order _Order) volatile noexcept { static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return const_cast(this)->_Base::compare_exchange_strong(_Expected, _Desired, _Order); + return const_cast(this)->_Base::compare_exchange_strong(_Expected, _Desired, _Order); } bool compare_exchange_strong(_Ty& _Expected, const _Ty _Desired, const memory_order _Success, From 75a155d38aa7b8d92dfbea31c7bf3de4a232d8c0 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 11:17:47 +0300 Subject: [PATCH 003/132] no really `volatile` overload for atomic_ref --- stl/inc/atomic | 81 -------------------------------------------------- 1 file changed, 81 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 6524c8cebc3..a0e93663170 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -1770,101 +1770,20 @@ public: return _Value; } - // For the following, we do the real implementation in the non-volatile function, and const_cast - // to call the non-volatile function in the volatile one. This is safe because all of the - // non-volatile functions reapply volatile, as all our intrinsics accept only volatile T *. - // We expect most atomics to be non-volatile, so making the real implementations - // non-volatile should result in better debug codegen. - using _Base::store; - void store(const _Ty _Value) volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - const_cast(this)->_Base::store(_Value); - } - - void store(const _Ty _Value, const memory_order _Order) volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - const_cast(this)->_Base::store(_Value, _Order); - } - - using _Base::load; - _NODISCARD _Ty load() const volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return const_cast(this)->_Base::load(); - } - - _NODISCARD _Ty load(const memory_order _Order) const volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return const_cast(this)->_Base::load(_Order); - } - - using _Base::exchange; - _Ty exchange(const _Ty _Value) volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return const_cast(this)->_Base::exchange(_Value); - } - - _Ty exchange(const _Ty _Value, const memory_order _Order) volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return const_cast(this)->_Base::exchange(_Value, _Order); - } - - using _Base::compare_exchange_strong; - bool compare_exchange_strong(_Ty& _Expected, const _Ty _Desired) volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return const_cast(this)->_Base::compare_exchange_strong(_Expected, _Desired); - } - - bool compare_exchange_strong(_Ty& _Expected, const _Ty _Desired, const memory_order _Order) volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return const_cast(this)->_Base::compare_exchange_strong(_Expected, _Desired, _Order); - } - - bool compare_exchange_strong(_Ty& _Expected, const _Ty _Desired, const memory_order _Success, - const memory_order _Failure) volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return this->compare_exchange_strong(_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); - } - - bool compare_exchange_strong( - _Ty& _Expected, const _Ty _Desired, const memory_order _Success, const memory_order _Failure) noexcept { - return this->compare_exchange_strong(_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); - } - - bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired) volatile noexcept { - // we have no weak CAS intrinsics, even on ARM32/ARM64, so fall back to strong - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return this->compare_exchange_strong(_Expected, _Desired); - } - bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired) noexcept { return this->compare_exchange_strong(_Expected, _Desired); } - bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired, const memory_order _Order) volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return this->compare_exchange_strong(_Expected, _Desired, _Order); - } bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired, const memory_order _Order) noexcept { return this->compare_exchange_strong(_Expected, _Desired, _Order); } - bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired, const memory_order _Success, - const memory_order _Failure) volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return this->compare_exchange_strong(_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); - } - bool compare_exchange_weak( _Ty& _Expected, const _Ty _Desired, const memory_order _Success, const memory_order _Failure) noexcept { return this->compare_exchange_strong(_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); } - operator _Ty() const volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return this->load(); - } - operator _Ty() const noexcept { return this->load(); } From 85ba6a2ed2c2cdcea75aa6a7080c2a9cef51cbc0 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 11:18:54 +0300 Subject: [PATCH 004/132] no really `volatile` overload for atomic_ref --- stl/inc/atomic | 6 ------ 1 file changed, 6 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index a0e93663170..d08aa95f099 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -1759,12 +1759,6 @@ public: return static_cast(this)->is_lock_free(); } - _Ty operator=(const _Ty _Value) volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - this->store(_Value); - return _Value; - } - _Ty operator=(const _Ty _Value) noexcept { this->store(_Value); return _Value; From 9d610cc28b8810a3b1b476481eb2c4b7f86d3489 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 11:40:36 +0300 Subject: [PATCH 005/132] required_alignment --- stl/inc/atomic | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index d08aa95f099..09e3ba062af 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -386,6 +386,7 @@ struct _Atomic_storage { } private: + // TODO: Don't use it for atomic_ref mutable long _Spinlock = 0; public: @@ -1738,15 +1739,20 @@ public: static constexpr bool is_always_lock_free = _Is_always_lock_free; + static constexpr size_t required_alignment = + (sizeof(_Ty) == 1 || sizeof(_Ty) == 2 || sizeof(_Ty) == 4 || sizeof(_Ty) == 8 || sizeof(_Ty) == 16) + ? sizeof(_Ty) + : alignof(_Ty); + #if 1 // TRANSITION, ABI - _NODISCARD bool is_lock_free() const volatile noexcept { + _NODISCARD bool is_lock_free() const noexcept { constexpr bool _Result = sizeof(_Ty) <= 8 && (sizeof(_Ty) & sizeof(_Ty) - 1) == 0; return _Result; } #else // ^^^ don't break ABI / break ABI vvv - _NODISCARD bool is_lock_free() const volatile noexcept { + _NODISCARD bool is_lock_free() const noexcept { #if _ATOMIC_HAS_DCAS return sizeof(_Ty) <= 2 * sizeof(void*); #else // ^^^ _ATOMIC_HAS_DCAS / !_ATOMIC_HAS_DCAS vvv @@ -1755,10 +1761,6 @@ public: } #endif // TRANSITION, ABI - _NODISCARD bool is_lock_free() const noexcept { - return static_cast(this)->is_lock_free(); - } - _Ty operator=(const _Ty _Value) noexcept { this->store(_Value); return _Value; From 857c6209cf52428801c4ba107e161b3412215b8c Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 12:27:55 +0300 Subject: [PATCH 006/132] Don't use _Spinlock --- stl/inc/atomic | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 09e3ba062af..74d41605e29 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -294,6 +294,8 @@ struct _Atomic_storage_types { using _TVal = _Ty; using _TStorage = _Atomic_padded<_Ty>; using _TConstr = const _Ty; + + static constexpr bool _Is_ref = false; }; template @@ -301,8 +303,13 @@ struct _Atomic_storage_types<_Ty&> { using _TVal = _Ty; using _TStorage = _Ty&; using _TConstr = _Ty&; + + static constexpr bool _Is_ref = true; }; +void _Atomic_ref_lock(const void* key) noexcept; // TODO: Satellite +void _Atomic_ref_unlock(const void* key) noexcept; // TODO: Satellite + // STRUCT TEMPLATE _Atomic_storage #if 1 // TRANSITION, ABI template ::_TVal)> @@ -370,23 +377,31 @@ struct _Atomic_storage { #if 1 // TRANSITION, ABI void _Lock() const noexcept { // lock the spinlock - while (_InterlockedExchange(&_Spinlock, 1)) { - _YIELD_PROCESSOR(); + if _CONSTEXPR_IF (_Atomic_storage_types<_Ty>::_Is_ref) { + _Atomic_ref_lock(_STD addressof(_Storage)); + } else { + while (_InterlockedExchange(&_Spinlock, 1)) { + _YIELD_PROCESSOR(); + } } } void _Unlock() const noexcept { // unlock the spinlock + if _CONSTEXPR_IF (_Atomic_storage_types<_Ty>::_Is_ref) { + _Atomic_ref_unlock(_STD addressof(_Storage)); + } else { #if defined(_M_ARM) || defined(_M_ARM64) - _Memory_barrier(); - __iso_volatile_store32(reinterpret_cast(&_Spinlock), 0); - _Memory_barrier(); + _Memory_barrier(); + __iso_volatile_store32(reinterpret_cast(&_Spinlock), 0); + _Memory_barrier(); #else // ^^^ ARM32/ARM64 hardware / x86/x64 hardware vvv - _InterlockedExchange(&_Spinlock, 0); + _InterlockedExchange(&_Spinlock, 0); #endif // hardware + } } private: - // TODO: Don't use it for atomic_ref + // Used only for atomic_ref. Hard to avoid without ABI break, will eliminate with next ABI breaking release mutable long _Spinlock = 0; public: @@ -410,6 +425,7 @@ public: } _Ty _Storage; + // TODO: Emimitate for atomic_ref using [[no_unique_address]] or by re-defining hierarchy mutable char _Spinlock = 0; #endif // TRANSITION, ABI }; From 1ca420a11e121e644c4400b4acfa4ff57084eff1 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 13:31:00 +0300 Subject: [PATCH 007/132] Do use spinlock --- stl/inc/atomic | 57 +++++++++++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 22 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 74d41605e29..42a07b078ab 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -289,13 +289,34 @@ struct _Atomic_padded<_Ty, false> { }; #endif // TRANSITION, ABI + +long& _Atomic_ref_get_mutex(const void* const _Key) noexcept { + // TODO: Move to satellite +#pragma warning(push) +#pragma warning(disable : 4324) // '%s': structure was padded due to alignment specifier + struct alignas(64) _Table_entry { + long _Splinlock = 0; + }; + static _Table_entry _Table[256]; +#pragma warning(pop) + auto _Key_as_int = reinterpret_cast(_Key); + _Key_as_int ^= _Key_as_int << 16; + _Key_as_int ^= _Key_as_int << 8; + return _Table[_Key_as_int]._Splinlock; +} + template struct _Atomic_storage_types { using _TVal = _Ty; using _TStorage = _Atomic_padded<_Ty>; using _TConstr = const _Ty; - static constexpr bool _Is_ref = false; + using _Mx_val_or_ref = long; + + constexpr long _Mx_init(const void* const _Key) noexcept { + (void) _Key; + return 0; + } }; template @@ -304,11 +325,13 @@ struct _Atomic_storage_types<_Ty&> { using _TStorage = _Ty&; using _TConstr = _Ty&; - static constexpr bool _Is_ref = true; + using _Mx_val_or_ref = long&; + + long& _Mx_init(const void* const _Key) noexcept { + return _Atomic_ref_get_mutex(_Key); + } }; -void _Atomic_ref_lock(const void* key) noexcept; // TODO: Satellite -void _Atomic_ref_unlock(const void* key) noexcept; // TODO: Satellite // STRUCT TEMPLATE _Atomic_storage #if 1 // TRANSITION, ABI @@ -377,32 +400,23 @@ struct _Atomic_storage { #if 1 // TRANSITION, ABI void _Lock() const noexcept { // lock the spinlock - if _CONSTEXPR_IF (_Atomic_storage_types<_Ty>::_Is_ref) { - _Atomic_ref_lock(_STD addressof(_Storage)); - } else { - while (_InterlockedExchange(&_Spinlock, 1)) { - _YIELD_PROCESSOR(); - } + while (_InterlockedExchange(&_Spinlock, 1)) { + _YIELD_PROCESSOR(); } } void _Unlock() const noexcept { // unlock the spinlock - if _CONSTEXPR_IF (_Atomic_storage_types<_Ty>::_Is_ref) { - _Atomic_ref_unlock(_STD addressof(_Storage)); - } else { #if defined(_M_ARM) || defined(_M_ARM64) - _Memory_barrier(); - __iso_volatile_store32(reinterpret_cast(&_Spinlock), 0); - _Memory_barrier(); + _Memory_barrier(); + __iso_volatile_store32(reinterpret_cast(&_Spinlock), 0); + _Memory_barrier(); #else // ^^^ ARM32/ARM64 hardware / x86/x64 hardware vvv - _InterlockedExchange(&_Spinlock, 0); + _InterlockedExchange(&_Spinlock, 0); #endif // hardware - } } private: - // Used only for atomic_ref. Hard to avoid without ABI break, will eliminate with next ABI breaking release - mutable long _Spinlock = 0; + mutable typename _Atomic_storage_types<_Ty&>::_Mx_val_or_ref _Spinlock = _Atomic_storage_types<_Ty&>::_Mx_init(this); public: _Ty _Storage{}; @@ -425,8 +439,7 @@ public: } _Ty _Storage; - // TODO: Emimitate for atomic_ref using [[no_unique_address]] or by re-defining hierarchy - mutable char _Spinlock = 0; + mutable char _Spinlock = 0; // TODO: refisit for atomic_ref #endif // TRANSITION, ABI }; From a522d3b26c99d4ba83989cd8bf17eda17dc5a490 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 13:33:13 +0300 Subject: [PATCH 008/132] fix build --- stl/inc/atomic | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 42a07b078ab..dfd06287b3f 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -290,19 +290,19 @@ struct _Atomic_padded<_Ty, false> { #endif // TRANSITION, ABI -long& _Atomic_ref_get_mutex(const void* const _Key) noexcept { +inline long& _Atomic_ref_get_mutex(const void* const _Key) noexcept { // TODO: Move to satellite #pragma warning(push) #pragma warning(disable : 4324) // '%s': structure was padded due to alignment specifier struct alignas(64) _Table_entry { long _Splinlock = 0; }; - static _Table_entry _Table[256]; + static _Table_entry _Table[256]; #pragma warning(pop) auto _Key_as_int = reinterpret_cast(_Key); _Key_as_int ^= _Key_as_int << 16; _Key_as_int ^= _Key_as_int << 8; - return _Table[_Key_as_int]._Splinlock; + return _Table[_Key_as_int & 0xFF]._Splinlock; } template From 25f603b0b26eb2dcd3759714779582a2950c8669 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 13:34:26 +0300 Subject: [PATCH 009/132] direction --- stl/inc/atomic | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index dfd06287b3f..59b2c8f0ffe 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -300,8 +300,8 @@ inline long& _Atomic_ref_get_mutex(const void* const _Key) noexcept { static _Table_entry _Table[256]; #pragma warning(pop) auto _Key_as_int = reinterpret_cast(_Key); - _Key_as_int ^= _Key_as_int << 16; - _Key_as_int ^= _Key_as_int << 8; + _Key_as_int ^= _Key_as_int >> 16; + _Key_as_int ^= _Key_as_int >> 8; return _Table[_Key_as_int & 0xFF]._Splinlock; } From 85c044ec04f7d17a5a8d0f61873eeddf213ecfd8 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 13:40:26 +0300 Subject: [PATCH 010/132] Pointer to object for atomic ref mutex init --- stl/inc/atomic | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 59b2c8f0ffe..635b7899981 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -345,10 +345,10 @@ struct _Atomic_storage { using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; - _Atomic_storage() = default; + _Atomic_storage() : _Storage({}), _Atomic_storage_types<_Ty&>::_Mx_init(nullptr) {} /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept - : _Storage(_Value) { + : _Storage(_Value), _Atomic_storage_types<_Ty&>::_Mx_init(&_Storage) { // non-atomically initialize this atomic } @@ -416,7 +416,7 @@ struct _Atomic_storage { } private: - mutable typename _Atomic_storage_types<_Ty&>::_Mx_val_or_ref _Spinlock = _Atomic_storage_types<_Ty&>::_Mx_init(this); + mutable typename _Atomic_storage_types<_Ty&>::_Mx_val_or_ref _Spinlock; public: _Ty _Storage{}; From 2cb4d3ef5268ee6132d60ff85d548ef3b77a2a43 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 14:50:27 +0300 Subject: [PATCH 011/132] clang format --- stl/inc/atomic | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 635b7899981..21ed9032d34 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -307,11 +307,10 @@ inline long& _Atomic_ref_get_mutex(const void* const _Key) noexcept { template struct _Atomic_storage_types { - using _TVal = _Ty; - using _TStorage = _Atomic_padded<_Ty>; - using _TConstr = const _Ty; - - using _Mx_val_or_ref = long; + using _TVal = _Ty; + using _TStorage = _Atomic_padded<_Ty>; + using _TConstr = const _Ty; + using _Mx_val_or_ref = long; constexpr long _Mx_init(const void* const _Key) noexcept { (void) _Key; @@ -321,10 +320,9 @@ struct _Atomic_storage_types { template struct _Atomic_storage_types<_Ty&> { - using _TVal = _Ty; - using _TStorage = _Ty&; - using _TConstr = _Ty&; - + using _TVal = _Ty; + using _TStorage = _Ty&; + using _TConstr = _Ty&; using _Mx_val_or_ref = long&; long& _Mx_init(const void* const _Key) noexcept { @@ -332,7 +330,6 @@ struct _Atomic_storage_types<_Ty&> { } }; - // STRUCT TEMPLATE _Atomic_storage #if 1 // TRANSITION, ABI template ::_TVal)> From c938bf2a114b645893e5afb3692087a90c1b8600 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 15:35:40 +0300 Subject: [PATCH 012/132] Correct base for specializations --- stl/inc/atomic | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 21ed9032d34..7d20a3dd79f 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -1559,19 +1559,19 @@ struct _Atomic_pointer : _Atomic_storage<_Ty> { #define ATOMIC_VAR_INIT(_Value) \ { _Value } -template +template using _Choose_atomic_base2_t = - typename _Select && !is_same_v>::template _Apply<_Atomic_integral_facade<_Ty>, - typename _Select && is_object_v>>::template _Apply<_Atomic_pointer<_Ty>, - _Atomic_storage<_Ty>>>; + typename _Select && !is_same_v>::template _Apply<_Atomic_integral_facade<_Ty>, + typename _Select && is_object_v>>::template _Apply< + _Atomic_pointer<_Ty>, _Atomic_storage<_Ty>>>; #if _HAS_CXX20 -template -using _Choose_atomic_base_t = - typename _Select>::template _Apply<_Atomic_floating<_Ty>, _Choose_atomic_base2_t<_Ty>>; +template +using _Choose_atomic_base_t = typename _Select>::template _Apply<_Atomic_floating<_Ty>, + _Choose_atomic_base2_t<_TVal, _Ty>>; #else // ^^^ _HAS_CXX20 // !_HAS_CXX20 vvv -template -using _Choose_atomic_base_t = _Choose_atomic_base2_t<_Ty>; +template +using _Choose_atomic_base_t = _Choose_atomic_base2_t<_TVal, _Ty>; #endif //_HAS_CXX20 template @@ -1743,9 +1743,9 @@ atomic(_Ty) -> atomic<_Ty>; #if _HAS_CXX20 template -struct atomic_ref : _Choose_atomic_base_t<_Ty&> { // atomic value +struct atomic_ref : _Choose_atomic_base_t<_Ty, _Ty&> { // atomic value private: - using _Base = _Choose_atomic_base_t<_Ty&>; + using _Base = _Choose_atomic_base_t<_Ty, _Ty&>; public: // clang-format off From 012bf4662de50b9bbd66ad17b84c0adc7589f2ef Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 15:38:37 +0300 Subject: [PATCH 013/132] basic operations test --- tests/std/test.lst | 1 + tests/std/tests/P0019R8_atomic_ref/env.lst | 4 ++ tests/std/tests/P0019R8_atomic_ref/test.cpp | 54 +++++++++++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 tests/std/tests/P0019R8_atomic_ref/env.lst create mode 100644 tests/std/tests/P0019R8_atomic_ref/test.cpp diff --git a/tests/std/test.lst b/tests/std/test.lst index 88fafc09c47..a2674770662 100644 --- a/tests/std/test.lst +++ b/tests/std/test.lst @@ -158,6 +158,7 @@ tests\Dev11_1180290_filesystem_error_code tests\GH_000457_system_error_message tests\GH_000545_include_compare tests\GH_000690_overaligned_function +tests\P0019R8_atomic_ref tests\P0024R2_parallel_algorithms_adjacent_difference tests\P0024R2_parallel_algorithms_adjacent_find tests\P0024R2_parallel_algorithms_all_of diff --git a/tests/std/tests/P0019R8_atomic_ref/env.lst b/tests/std/tests/P0019R8_atomic_ref/env.lst new file mode 100644 index 00000000000..642f530ffad --- /dev/null +++ b/tests/std/tests/P0019R8_atomic_ref/env.lst @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +RUNALL_INCLUDE ..\usual_latest_matrix.lst diff --git a/tests/std/tests/P0019R8_atomic_ref/test.cpp b/tests/std/tests/P0019R8_atomic_ref/test.cpp new file mode 100644 index 00000000000..b2cd239d605 --- /dev/null +++ b/tests/std/tests/P0019R8_atomic_ref/test.cpp @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include +#include +#include +#include +#include + +// code reuse of ../P1135R6_atomic_flag_test/test.cpp + +template +void test_ops() { + constexpr std::size_t unique = 80; // small to avoid overflow even for char + constexpr std::size_t repetitions = 8000; + constexpr std::size_t total = unique * repetitions; + constexpr std::size_t range = 10; + + ValueType vals[unique] = {}; + std::vector> refs; + refs.reserve(total); + for (std::size_t i = 0; i != repetitions; ++i) { + for (auto& val : vals) { + refs.push_back(std::atomic_ref(val)); + } + } + + using std::execution::par; + + auto load = [](const std::atomic_ref& ref) { return static_cast(ref.load()); }; + auto add = [](std::atomic_ref& ref) { return static_cast(ref.fetch_add(1)); }; + + assert(std::transform_reduce(par, refs.begin(), refs.end(), 0, std::plus{}, load) == 0); + assert(std::transform_reduce(par, refs.begin(), refs.begin() + range, 0, std::plus{}, add) == 0); + assert(std::transform_reduce(par, refs.begin(), refs.end(), 0, std::plus{}, load) == range * repetitions); + assert(std::transform_reduce(par, refs.begin(), refs.begin() + range, 0, std::plus{}, add) == range); + assert(std::transform_reduce(par, refs.begin(), refs.end(), 0, std::plus{}, load) == range * repetitions * 2); +} + + +int main() { + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); +} From da247a600decf7419f5e06d7a2f3b56f658afc1b Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 16:10:53 +0300 Subject: [PATCH 014/132] Fix spinlock initialization --- stl/inc/atomic | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 7d20a3dd79f..a243efd4a87 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -312,7 +312,7 @@ struct _Atomic_storage_types { using _TConstr = const _Ty; using _Mx_val_or_ref = long; - constexpr long _Mx_init(const void* const _Key) noexcept { + static constexpr long _Mx_init(const void* const _Key) noexcept { (void) _Key; return 0; } @@ -325,7 +325,7 @@ struct _Atomic_storage_types<_Ty&> { using _TConstr = _Ty&; using _Mx_val_or_ref = long&; - long& _Mx_init(const void* const _Key) noexcept { + static long& _Mx_init(const void* const _Key) noexcept { return _Atomic_ref_get_mutex(_Key); } }; @@ -342,10 +342,10 @@ struct _Atomic_storage { using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; - _Atomic_storage() : _Storage({}), _Atomic_storage_types<_Ty&>::_Mx_init(nullptr) {} + _Atomic_storage() : _Storage({}), _Spinlock(_Atomic_storage_types<_Ty&>::_Mx_init(nullptr)) {} /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept - : _Storage(_Value), _Atomic_storage_types<_Ty&>::_Mx_init(&_Storage) { + : _Storage(_Value), _Spinlock(_Atomic_storage_types<_Ty&>::_Mx_init(&_Storage)) { // non-atomically initialize this atomic } @@ -397,7 +397,7 @@ struct _Atomic_storage { #if 1 // TRANSITION, ABI void _Lock() const noexcept { // lock the spinlock - while (_InterlockedExchange(&_Spinlock, 1)) { + while (_InterlockedExchange(const_cast(&_Spinlock), 1)) { _YIELD_PROCESSOR(); } } @@ -405,15 +405,15 @@ struct _Atomic_storage { void _Unlock() const noexcept { // unlock the spinlock #if defined(_M_ARM) || defined(_M_ARM64) _Memory_barrier(); - __iso_volatile_store32(reinterpret_cast(&_Spinlock), 0); + __iso_volatile_store32(reinterpret_cast(const_cast(&_Spinlock)), 0); _Memory_barrier(); #else // ^^^ ARM32/ARM64 hardware / x86/x64 hardware vvv - _InterlockedExchange(&_Spinlock, 0); + _InterlockedExchange(const_cast(&_Spinlock), 0); #endif // hardware } private: - mutable typename _Atomic_storage_types<_Ty&>::_Mx_val_or_ref _Spinlock; + typename _Atomic_storage_types<_Ty&>::_Mx_val_or_ref _Spinlock; public: _Ty _Storage{}; From 0c38cdc3960a8b90a065afa192b8b5b69b6b71b0 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 16:11:13 +0300 Subject: [PATCH 015/132] elaborate test --- tests/std/tests/P0019R8_atomic_ref/test.cpp | 78 +++++++++++++++++---- 1 file changed, 64 insertions(+), 14 deletions(-) diff --git a/tests/std/tests/P0019R8_atomic_ref/test.cpp b/tests/std/tests/P0019R8_atomic_ref/test.cpp index b2cd239d605..95bc8557545 100644 --- a/tests/std/tests/P0019R8_atomic_ref/test.cpp +++ b/tests/std/tests/P0019R8_atomic_ref/test.cpp @@ -8,9 +8,21 @@ #include #include + +struct bigint { + int value; + int more_value[10]; + + bigint(int value = 0) : value(value) {} + + operator int() const { + return value; + } +}; + // code reuse of ../P1135R6_atomic_flag_test/test.cpp -template +template void test_ops() { constexpr std::size_t unique = 80; // small to avoid overflow even for char constexpr std::size_t repetitions = 8000; @@ -28,27 +40,65 @@ void test_ops() { using std::execution::par; - auto load = [](const std::atomic_ref& ref) { return static_cast(ref.load()); }; - auto add = [](std::atomic_ref& ref) { return static_cast(ref.fetch_add(1)); }; + auto load = [](const std::atomic_ref& ref) { return static_cast(ref.load()); }; + auto xchg0 = [](std::atomic_ref& ref) { return static_cast(ref.exchange(0)); }; + + int (*add)(std::atomic_ref & ref); + if constexpr (AddViaCas) { + add = [](std::atomic_ref& ref) + { + for (;;) { + ValueType e = ref.load(); + ValueType d = e + 1; + if (ref.compare_exchange_weak(e, d)) + return static_cast(e); + } + }; + } else { + add = [](std::atomic_ref& ref) { return static_cast(ref.fetch_add(1)); }; + } assert(std::transform_reduce(par, refs.begin(), refs.end(), 0, std::plus{}, load) == 0); assert(std::transform_reduce(par, refs.begin(), refs.begin() + range, 0, std::plus{}, add) == 0); assert(std::transform_reduce(par, refs.begin(), refs.end(), 0, std::plus{}, load) == range * repetitions); assert(std::transform_reduce(par, refs.begin(), refs.begin() + range, 0, std::plus{}, add) == range); assert(std::transform_reduce(par, refs.begin(), refs.end(), 0, std::plus{}, load) == range * repetitions * 2); + assert(std::transform_reduce(par, refs.begin(), refs.end(), 0, std::plus{}, xchg0) == range * 2); + assert(std::transform_reduce(par, refs.begin(), refs.end(), 0, std::plus{}, load) == 0); } + + int main() { - test_ops(); - test_ops(); - test_ops(); - test_ops(); - test_ops(); - test_ops(); - test_ops(); - test_ops(); - test_ops(); - test_ops(); - test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); + test_ops(); } From 3e3cd47b0fbbb89bfebc29e6c97deb7d9dfcaa2b Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 16:13:03 +0300 Subject: [PATCH 016/132] better naming --- tests/std/tests/P0019R8_atomic_ref/test.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/std/tests/P0019R8_atomic_ref/test.cpp b/tests/std/tests/P0019R8_atomic_ref/test.cpp index 95bc8557545..a4011fc6d79 100644 --- a/tests/std/tests/P0019R8_atomic_ref/test.cpp +++ b/tests/std/tests/P0019R8_atomic_ref/test.cpp @@ -43,19 +43,20 @@ void test_ops() { auto load = [](const std::atomic_ref& ref) { return static_cast(ref.load()); }; auto xchg0 = [](std::atomic_ref& ref) { return static_cast(ref.exchange(0)); }; - int (*add)(std::atomic_ref & ref); + int (*inc)(std::atomic_ref & ref); if constexpr (AddViaCas) { - add = [](std::atomic_ref& ref) + inc = [](std::atomic_ref& ref) { for (;;) { ValueType e = ref.load(); ValueType d = e + 1; - if (ref.compare_exchange_weak(e, d)) + if (ref.compare_exchange_weak(e, d)) { return static_cast(e); + } } }; } else { - add = [](std::atomic_ref& ref) { return static_cast(ref.fetch_add(1)); }; + inc = [](std::atomic_ref& ref) { return static_cast(ref.fetch_add(1)); }; } assert(std::transform_reduce(par, refs.begin(), refs.end(), 0, std::plus{}, load) == 0); From 6fdcf4d8b8ec651a117e9fc3409c04d512569ec5 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 16:13:45 +0300 Subject: [PATCH 017/132] better naming --- tests/std/tests/P0019R8_atomic_ref/test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/std/tests/P0019R8_atomic_ref/test.cpp b/tests/std/tests/P0019R8_atomic_ref/test.cpp index a4011fc6d79..fa1b8c04af7 100644 --- a/tests/std/tests/P0019R8_atomic_ref/test.cpp +++ b/tests/std/tests/P0019R8_atomic_ref/test.cpp @@ -60,9 +60,9 @@ void test_ops() { } assert(std::transform_reduce(par, refs.begin(), refs.end(), 0, std::plus{}, load) == 0); - assert(std::transform_reduce(par, refs.begin(), refs.begin() + range, 0, std::plus{}, add) == 0); + assert(std::transform_reduce(par, refs.begin(), refs.begin() + range, 0, std::plus{}, inc) == 0); assert(std::transform_reduce(par, refs.begin(), refs.end(), 0, std::plus{}, load) == range * repetitions); - assert(std::transform_reduce(par, refs.begin(), refs.begin() + range, 0, std::plus{}, add) == range); + assert(std::transform_reduce(par, refs.begin(), refs.begin() + range, 0, std::plus{}, inc) == range); assert(std::transform_reduce(par, refs.begin(), refs.end(), 0, std::plus{}, load) == range * repetitions * 2); assert(std::transform_reduce(par, refs.begin(), refs.end(), 0, std::plus{}, xchg0) == range * 2); assert(std::transform_reduce(par, refs.begin(), refs.end(), 0, std::plus{}, load) == 0); From f385fa1d7e9ebe48a35e4ce58de592d24774cf81 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 16:16:33 +0300 Subject: [PATCH 018/132] warnings fix --- stl/inc/atomic | 4 ++-- tests/std/tests/P0019R8_atomic_ref/test.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index a243efd4a87..322956a9040 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -342,10 +342,10 @@ struct _Atomic_storage { using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; - _Atomic_storage() : _Storage({}), _Spinlock(_Atomic_storage_types<_Ty&>::_Mx_init(nullptr)) {} + _Atomic_storage() : _Spinlock(_Atomic_storage_types<_Ty&>::_Mx_init(nullptr)), _Storage({}) {} /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept - : _Storage(_Value), _Spinlock(_Atomic_storage_types<_Ty&>::_Mx_init(&_Storage)) { + : _Spinlock(_Atomic_storage_types<_Ty&>::_Mx_init(&_Storage)), _Storage(_Value) { // non-atomically initialize this atomic } diff --git a/tests/std/tests/P0019R8_atomic_ref/test.cpp b/tests/std/tests/P0019R8_atomic_ref/test.cpp index fa1b8c04af7..0a9e51ece1b 100644 --- a/tests/std/tests/P0019R8_atomic_ref/test.cpp +++ b/tests/std/tests/P0019R8_atomic_ref/test.cpp @@ -49,7 +49,7 @@ void test_ops() { { for (;;) { ValueType e = ref.load(); - ValueType d = e + 1; + ValueType d = static_cast(static_cast(e) + 1); if (ref.compare_exchange_weak(e, d)) { return static_cast(e); } From dac5c6a87c2e27f3a70155c274bbe5ad8da31fa0 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 16:25:18 +0300 Subject: [PATCH 019/132] get back mutable --- stl/inc/atomic | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 322956a9040..291f0a4468f 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -290,7 +290,7 @@ struct _Atomic_padded<_Ty, false> { #endif // TRANSITION, ABI -inline long& _Atomic_ref_get_mutex(const void* const _Key) noexcept { +inline long* _Atomic_ref_get_mutex(const void* const _Key) noexcept { // TODO: Move to satellite #pragma warning(push) #pragma warning(disable : 4324) // '%s': structure was padded due to alignment specifier @@ -302,7 +302,7 @@ inline long& _Atomic_ref_get_mutex(const void* const _Key) noexcept { auto _Key_as_int = reinterpret_cast(_Key); _Key_as_int ^= _Key_as_int >> 16; _Key_as_int ^= _Key_as_int >> 8; - return _Table[_Key_as_int & 0xFF]._Splinlock; + return &_Table[_Key_as_int & 0xFF]._Splinlock; } template @@ -310,12 +310,16 @@ struct _Atomic_storage_types { using _TVal = _Ty; using _TStorage = _Atomic_padded<_Ty>; using _TConstr = const _Ty; - using _Mx_val_or_ref = long; + using _Mx_val_or_ptr = long; static constexpr long _Mx_init(const void* const _Key) noexcept { (void) _Key; return 0; } + + static long* _Mx_ptr(_Mx_val_or_ptr& _Val) { + return &_Val; + } }; template @@ -323,11 +327,15 @@ struct _Atomic_storage_types<_Ty&> { using _TVal = _Ty; using _TStorage = _Ty&; using _TConstr = _Ty&; - using _Mx_val_or_ref = long&; + using _Mx_val_or_ptr = long*; - static long& _Mx_init(const void* const _Key) noexcept { + static long* _Mx_init(const void* const _Key) noexcept { return _Atomic_ref_get_mutex(_Key); } + + static long* _Mx_ptr(_Mx_val_or_ptr _Ptr) { + return _Ptr; + } }; // STRUCT TEMPLATE _Atomic_storage @@ -397,7 +405,7 @@ struct _Atomic_storage { #if 1 // TRANSITION, ABI void _Lock() const noexcept { // lock the spinlock - while (_InterlockedExchange(const_cast(&_Spinlock), 1)) { + while (_InterlockedExchange(_Atomic_storage_types<_Ty>::_Mx_ptr(_Spinlock), 1)) { _YIELD_PROCESSOR(); } } @@ -405,15 +413,15 @@ struct _Atomic_storage { void _Unlock() const noexcept { // unlock the spinlock #if defined(_M_ARM) || defined(_M_ARM64) _Memory_barrier(); - __iso_volatile_store32(reinterpret_cast(const_cast(&_Spinlock)), 0); + __iso_volatile_store32(reinterpret_cast(_Atomic_storage_types<_Ty>::_Mx_ptr(_Spinlock)), 0); _Memory_barrier(); #else // ^^^ ARM32/ARM64 hardware / x86/x64 hardware vvv - _InterlockedExchange(const_cast(&_Spinlock), 0); + _InterlockedExchange(_Atomic_storage_types<_Ty>::_Mx_ptr(_Spinlock), 0); #endif // hardware } private: - typename _Atomic_storage_types<_Ty&>::_Mx_val_or_ref _Spinlock; + mutable typename _Atomic_storage_types<_Ty&>::_Mx_val_or_ptr _Spinlock; public: _Ty _Storage{}; From c8d38df1aaad0deca113b51b3159a0ae984ad9fd Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 16:44:49 +0300 Subject: [PATCH 020/132] clang format --- tests/std/tests/P0019R8_atomic_ref/test.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/std/tests/P0019R8_atomic_ref/test.cpp b/tests/std/tests/P0019R8_atomic_ref/test.cpp index 0a9e51ece1b..b1328e20352 100644 --- a/tests/std/tests/P0019R8_atomic_ref/test.cpp +++ b/tests/std/tests/P0019R8_atomic_ref/test.cpp @@ -45,8 +45,7 @@ void test_ops() { int (*inc)(std::atomic_ref & ref); if constexpr (AddViaCas) { - inc = [](std::atomic_ref& ref) - { + inc = [](std::atomic_ref& ref) { for (;;) { ValueType e = ref.load(); ValueType d = static_cast(static_cast(e) + 1); @@ -68,9 +67,6 @@ void test_ops() { assert(std::transform_reduce(par, refs.begin(), refs.end(), 0, std::plus{}, load) == 0); } - - - int main() { test_ops(); test_ops(); From 3cf38d6e588f7cd3dca38c68460ec94ddc01fe3a Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 17:09:25 +0300 Subject: [PATCH 021/132] satellite --- stl/CMakeLists.txt | 21 +++++++++++++++++++-- stl/inc/atomic | 21 +++++---------------- stl/src/atomic_ref.cpp | 27 +++++++++++++++++++++++++++ stl/src/msvcp_atomic_ref.def | 7 +++++++ 4 files changed, 58 insertions(+), 18 deletions(-) create mode 100644 stl/src/atomic_ref.cpp create mode 100644 stl/src/msvcp_atomic_ref.def diff --git a/stl/CMakeLists.txt b/stl/CMakeLists.txt index dfb026bd299..ee0bcf39558 100644 --- a/stl/CMakeLists.txt +++ b/stl/CMakeLists.txt @@ -391,11 +391,16 @@ set(SOURCES_SATELLITE_CODECVT_IDS ${CMAKE_CURRENT_LIST_DIR}/src/ulocale.cpp ) +set(SOURCES_SATELLITE_ATOMIC_REF + ${CMAKE_CURRENT_LIST_DIR}/src/atomic_ref.cpp +) + # Objs that exist only in libcpmt[d][01].lib. set(STATIC_SOURCES ${SOURCES_SATELLITE_1} ${SOURCES_SATELLITE_2} ${SOURCES_SATELLITE_CODECVT_IDS} + ${SOURCES_SATELLITE_ATOMIC_REF} ) add_library(std_init_once_begin_initialize OBJECT IMPORTED) @@ -470,11 +475,23 @@ function(add_stl_dlls D_SUFFIX THIS_CONFIG_DEFINITIONS THIS_CONFIG_COMPILE_OPTIO set_target_properties(msvcp${D_SUFFIX}_codecvt_ids PROPERTIES OUTPUT_NAME "msvcp140${D_SUFFIX}_codecvt_ids${VCLIBS_SUFFIX}") target_link_options(msvcp${D_SUFFIX}_codecvt_ids PRIVATE "${THIS_CONFIG_LINK_OPTIONS}") + # msvcp140_atomic_ref.dll (the atomic ref satellite) + add_library(msvcp${D_SUFFIX}_atomic_ref_objects OBJECT ${SOURCES_SATELLITE_ATOMIC_REF}) + target_compile_definitions(msvcp${D_SUFFIX}_atomic_ref_objects PRIVATE "_BUILDING_SATELLITE_ATOMIC_REF;_DLL;${THIS_CONFIG_DEFINITIONS}") + target_compile_options(msvcp${D_SUFFIX}_atomic_ref_objects PRIVATE "${THIS_CONFIG_COMPILE_OPTIONS};${GL_FLAG};/EHsc") + + add_library(msvcp${D_SUFFIX}_atomic_ref SHARED) + target_link_libraries(msvcp${D_SUFFIX}_atomic_ref PRIVATE msvcp${D_SUFFIX}_atomic_ref_objects "msvcp${D_SUFFIX}" "${TOOLSET_LIB}/vcruntime${D_SUFFIX}.lib" "${TOOLSET_LIB}/msvcrt${D_SUFFIX}.lib" "ucrt${D_SUFFIX}.lib") + set_target_properties(msvcp${D_SUFFIX}_atomic_ref PROPERTIES ARCHIVE_OUTPUT_NAME "msvcp140_atomic_ref${D_SUFFIX}${VCLIBS_SUFFIX}") + set_target_properties(msvcp${D_SUFFIX}_atomic_ref PROPERTIES ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") + set_target_properties(msvcp${D_SUFFIX}_atomic_ref PROPERTIES OUTPUT_NAME "msvcp140${D_SUFFIX}_atomic_ref${VCLIBS_SUFFIX}") + target_link_options(msvcp${D_SUFFIX}_atomic_ref PRIVATE "${THIS_CONFIG_LINK_OPTIONS}" "/DEF:${CMAKE_CURRENT_LIST_DIR}/src/msvcp_atomic_ref.def") + # import library add_library(msvcp${D_SUFFIX}_implib STATIC ${HEADERS}) target_link_libraries(msvcp${D_SUFFIX}_implib msvcp${D_SUFFIX}_implib_objects std_init_once_begin_initialize std_init_once_complete) - add_dependencies(msvcp${D_SUFFIX}_implib msvcp${D_SUFFIX} msvcp_1${D_SUFFIX} msvcp_2${D_SUFFIX} msvcp${D_SUFFIX}_codecvt_ids) - set_target_properties(msvcp${D_SUFFIX}_implib PROPERTIES STATIC_LIBRARY_OPTIONS "/NOLOGO;/NODEFAULTLIB;/IGNORE:4006;$;$;$;$") + add_dependencies(msvcp${D_SUFFIX}_implib msvcp${D_SUFFIX} msvcp_1${D_SUFFIX} msvcp_2${D_SUFFIX} msvcp${D_SUFFIX}_codecvt_ids msvcp${D_SUFFIX}_atomic_ref) + set_target_properties(msvcp${D_SUFFIX}_implib PROPERTIES STATIC_LIBRARY_OPTIONS "/NOLOGO;/NODEFAULTLIB;/IGNORE:4006;$;$;$;$;$") set_target_properties(msvcp${D_SUFFIX}_implib PROPERTIES ARCHIVE_OUTPUT_NAME "msvcprt${D_SUFFIX}") endfunction() diff --git a/stl/inc/atomic b/stl/inc/atomic index 291f0a4468f..bb18546a5bd 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -131,6 +131,10 @@ _NODISCARD extern "C" bool __cdecl __std_atomic_has_cmpxchg16b() noexcept; #define ATOMIC_LLONG_LOCK_FREE 2 #define ATOMIC_POINTER_LOCK_FREE 2 +_EXTERN_C +long* __stdcall __std_atomic_ref_get_mutex(const void* _Key) noexcept; +_END_EXTERN_C + _STD_BEGIN // FUNCTION TEMPLATE kill_dependency @@ -290,21 +294,6 @@ struct _Atomic_padded<_Ty, false> { #endif // TRANSITION, ABI -inline long* _Atomic_ref_get_mutex(const void* const _Key) noexcept { - // TODO: Move to satellite -#pragma warning(push) -#pragma warning(disable : 4324) // '%s': structure was padded due to alignment specifier - struct alignas(64) _Table_entry { - long _Splinlock = 0; - }; - static _Table_entry _Table[256]; -#pragma warning(pop) - auto _Key_as_int = reinterpret_cast(_Key); - _Key_as_int ^= _Key_as_int >> 16; - _Key_as_int ^= _Key_as_int >> 8; - return &_Table[_Key_as_int & 0xFF]._Splinlock; -} - template struct _Atomic_storage_types { using _TVal = _Ty; @@ -330,7 +319,7 @@ struct _Atomic_storage_types<_Ty&> { using _Mx_val_or_ptr = long*; static long* _Mx_init(const void* const _Key) noexcept { - return _Atomic_ref_get_mutex(_Key); + return __std_atomic_ref_get_mutex(_Key); } static long* _Mx_ptr(_Mx_val_or_ptr _Ptr) { diff --git a/stl/src/atomic_ref.cpp b/stl/src/atomic_ref.cpp new file mode 100644 index 00000000000..8141d856ae5 --- /dev/null +++ b/stl/src/atomic_ref.cpp @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// implement atomic_ref spin lock + +#include +#include + +_EXTERN_C +long* __stdcall __std_atomic_ref_get_mutex(const void* const _Key) noexcept { + constexpr size_t _Table_size_power = 8; + constexpr size_t _Table_size = 1 << _Table_size_power; + constexpr size_t _Table_index_mask = _Table_size - 1; + +#pragma warning(push) +#pragma warning(disable : 4324) // '%s': structure was padded due to alignment specifier + struct alignas(std::hardware_destructive_interference_size) _Table_entry { + long _Splinlock = 0; + }; + static _Table_entry _Table[_Table_size]; +#pragma warning(pop) + auto _Index = reinterpret_cast(_Key); + _Index ^= _Index >> (_Table_size_power * 2); + _Index ^= _Index >> _Table_size_power; + return &_Table[_Index & _Table_index_mask]._Splinlock; +} +_END_EXTERN_C \ No newline at end of file diff --git a/stl/src/msvcp_atomic_ref.def b/stl/src/msvcp_atomic_ref.def new file mode 100644 index 00000000000..0cd8a922a46 --- /dev/null +++ b/stl/src/msvcp_atomic_ref.def @@ -0,0 +1,7 @@ +; Copyright (c) Microsoft Corporation. +; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +; atomic ref satellite DLL definition + +EXPORTS + __std_atomic_ref_get_mutex \ No newline at end of file From dd33205ff8a5cc8fc79381fb0d1c14743e0dcd34 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 17:14:29 +0300 Subject: [PATCH 022/132] feature --- stl/inc/yvals_core.h | 1 + .../tests/VSO_0157762_feature_test_macros/test.cpp | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/stl/inc/yvals_core.h b/stl/inc/yvals_core.h index c760ab29804..daf57ab2d2d 100644 --- a/stl/inc/yvals_core.h +++ b/stl/inc/yvals_core.h @@ -1134,6 +1134,7 @@ #define __cpp_lib_atomic_flag_test 201907L #define __cpp_lib_atomic_float 201711L #define __cpp_lib_atomic_lock_free_type_aliases 201907L +#define __cpp_lib_atomic_ref 201806L #define __cpp_lib_atomic_shared_ptr 201711L #define __cpp_lib_bind_front 201907L #define __cpp_lib_bit_cast 201806L diff --git a/tests/std/tests/VSO_0157762_feature_test_macros/test.cpp b/tests/std/tests/VSO_0157762_feature_test_macros/test.cpp index 378511931d1..42084e2b755 100644 --- a/tests/std/tests/VSO_0157762_feature_test_macros/test.cpp +++ b/tests/std/tests/VSO_0157762_feature_test_macros/test.cpp @@ -139,6 +139,20 @@ STATIC_ASSERT(__cpp_lib_atomic_lock_free_type_aliases == 201907L); #endif #endif +#if _HAS_CXX20 +#ifndef __cpp_lib_atomic_ref +#error __cpp_lib_atomic_ref is not defined +#elif __cpp_lib_atomic_ref != 201806L +#error __cpp_lib_atomic_ref is not 201806L +#else +STATIC_ASSERT(__cpp_lib_atomic_ref == 201806L); +#endif +#else +#ifdef __cpp_lib_atomic_ref +#error __cpp_lib_atomic_ref is defined +#endif +#endif + #if _HAS_CXX20 #ifndef __cpp_lib_atomic_shared_ptr #error __cpp_lib_atomic_shared_ptr is not defined From fc29559ec03571954077c6a723fcb0663eefecc1 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 18:38:32 +0300 Subject: [PATCH 023/132] Use SRWLOCK and abandon _Spinlock for atomic_ref --- stl/inc/atomic | 67 ++++++++++++++++-------------------- stl/src/atomic_ref.cpp | 18 +++++++--- stl/src/msvcp_atomic_ref.def | 3 +- 3 files changed, 45 insertions(+), 43 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index bb18546a5bd..b499308801d 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -132,7 +132,8 @@ _NODISCARD extern "C" bool __cdecl __std_atomic_has_cmpxchg16b() noexcept; #define ATOMIC_POINTER_LOCK_FREE 2 _EXTERN_C -long* __stdcall __std_atomic_ref_get_mutex(const void* _Key) noexcept; +void __stdcall __std_atomic_ref_lock(const void* _Key) noexcept; +void __stdcall __std_atomic_ref_unlock(const void* _Key) noexcept; _END_EXTERN_C _STD_BEGIN @@ -293,38 +294,20 @@ struct _Atomic_padded<_Ty, false> { }; #endif // TRANSITION, ABI - template struct _Atomic_storage_types { - using _TVal = _Ty; - using _TStorage = _Atomic_padded<_Ty>; - using _TConstr = const _Ty; - using _Mx_val_or_ptr = long; - - static constexpr long _Mx_init(const void* const _Key) noexcept { - (void) _Key; - return 0; - } - - static long* _Mx_ptr(_Mx_val_or_ptr& _Val) { - return &_Val; - } + using _TVal = _Ty; + using _TStorage = _Atomic_padded<_Ty>; + using _TConstr = const _Ty; + static constexpr bool _Is_ref = false; }; template struct _Atomic_storage_types<_Ty&> { - using _TVal = _Ty; - using _TStorage = _Ty&; - using _TConstr = _Ty&; - using _Mx_val_or_ptr = long*; - - static long* _Mx_init(const void* const _Key) noexcept { - return __std_atomic_ref_get_mutex(_Key); - } - - static long* _Mx_ptr(_Mx_val_or_ptr _Ptr) { - return _Ptr; - } + using _TVal = _Ty; + using _TStorage = _Ty&; + using _TConstr = _Ty&; + static constexpr bool _Is_ref = false; }; // STRUCT TEMPLATE _Atomic_storage @@ -339,10 +322,10 @@ struct _Atomic_storage { using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; - _Atomic_storage() : _Spinlock(_Atomic_storage_types<_Ty&>::_Mx_init(nullptr)), _Storage({}) {} + _Atomic_storage() = default; /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept - : _Spinlock(_Atomic_storage_types<_Ty&>::_Mx_init(&_Storage)), _Storage(_Value) { + : _Storage(_Value) { // non-atomically initialize this atomic } @@ -394,23 +377,32 @@ struct _Atomic_storage { #if 1 // TRANSITION, ABI void _Lock() const noexcept { // lock the spinlock - while (_InterlockedExchange(_Atomic_storage_types<_Ty>::_Mx_ptr(_Spinlock), 1)) { - _YIELD_PROCESSOR(); + if _CONSTEXPR_IF (_Atomic_storage_types<_Ty>::_Is_ref) { + __std_atomic_ref_lock(_STD addressof(_Storage)); + } else { + while (_InterlockedExchange(&_Spinlock, 1)) { + _YIELD_PROCESSOR(); + } } } void _Unlock() const noexcept { // unlock the spinlock + if _CONSTEXPR_IF (_Atomic_storage_types<_Ty>::_Is_ref) { + __std_atomic_ref_unlock(_STD addressof(_Storage)); + } else { #if defined(_M_ARM) || defined(_M_ARM64) - _Memory_barrier(); - __iso_volatile_store32(reinterpret_cast(_Atomic_storage_types<_Ty>::_Mx_ptr(_Spinlock)), 0); - _Memory_barrier(); + _Memory_barrier(); + __iso_volatile_store32(&_Spinlock, 0); + _Memory_barrier(); #else // ^^^ ARM32/ARM64 hardware / x86/x64 hardware vvv - _InterlockedExchange(_Atomic_storage_types<_Ty>::_Mx_ptr(_Spinlock), 0); + _InterlockedExchange(&_Spinlock, 0); #endif // hardware + } } private: - mutable typename _Atomic_storage_types<_Ty&>::_Mx_val_or_ptr _Spinlock; + // Spinlock for non-lock-free atomic. Not used by atomic_ref, but cannot be easily eliminated due to ABI + mutable long _Spinlock = 0; public: _Ty _Storage{}; @@ -1754,7 +1746,7 @@ public: using value_type = _Ty; - explicit atomic_ref(_Ty& _Value) noexcept : _Base(_Value) {} + explicit atomic_ref(_Ty& _Value) noexcept /* strengthened */ : _Base(_Value) {} atomic_ref(const atomic_ref&) noexcept = default; @@ -2213,7 +2205,6 @@ struct atomic_flag { // flag with test-and-set semantics #endif // TRANSITION, ABI }; - // atomic_flag NONMEMBERS #if _HAS_CXX20 _NODISCARD inline bool atomic_flag_test(const volatile atomic_flag* const _Flag) noexcept { diff --git a/stl/src/atomic_ref.cpp b/stl/src/atomic_ref.cpp index 8141d856ae5..f38237492f4 100644 --- a/stl/src/atomic_ref.cpp +++ b/stl/src/atomic_ref.cpp @@ -6,8 +6,9 @@ #include #include -_EXTERN_C -long* __stdcall __std_atomic_ref_get_mutex(const void* const _Key) noexcept { +#include + +SRWLOCK* _Atomic_atomic_ref_get_mutex(const void* const _Key) noexcept { constexpr size_t _Table_size_power = 8; constexpr size_t _Table_size = 1 << _Table_size_power; constexpr size_t _Table_index_mask = _Table_size - 1; @@ -15,13 +16,22 @@ long* __stdcall __std_atomic_ref_get_mutex(const void* const _Key) noexcept { #pragma warning(push) #pragma warning(disable : 4324) // '%s': structure was padded due to alignment specifier struct alignas(std::hardware_destructive_interference_size) _Table_entry { - long _Splinlock = 0; + SRWLOCK _Mutex = SRWLOCK_INIT; }; static _Table_entry _Table[_Table_size]; #pragma warning(pop) auto _Index = reinterpret_cast(_Key); _Index ^= _Index >> (_Table_size_power * 2); _Index ^= _Index >> _Table_size_power; - return &_Table[_Index & _Table_index_mask]._Splinlock; + return &_Table[_Index & _Table_index_mask]._Mutex; +} + +_EXTERN_C +void __stdcall __std_atomic_ref_lock(const void* const _Key) noexcept { + ::AcquireSRWLockExclusive(_Atomic_atomic_ref_get_mutex(_Key)); +} + +void __stdcall __std_atomic_ref_unlock(const void* const _Key) noexcept { + ::ReleaseSRWLockExclusive(_Atomic_atomic_ref_get_mutex(_Key)); } _END_EXTERN_C \ No newline at end of file diff --git a/stl/src/msvcp_atomic_ref.def b/stl/src/msvcp_atomic_ref.def index 0cd8a922a46..136679224ad 100644 --- a/stl/src/msvcp_atomic_ref.def +++ b/stl/src/msvcp_atomic_ref.def @@ -4,4 +4,5 @@ ; atomic ref satellite DLL definition EXPORTS - __std_atomic_ref_get_mutex \ No newline at end of file + __std_atomic_ref_lock + __std_atomic_ref_unlock \ No newline at end of file From e219b71cdc2c3f80038f97177ee4285a646a2cd0 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 18:41:47 +0300 Subject: [PATCH 024/132] ref --- stl/inc/atomic | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index b499308801d..ae7304e9211 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -307,7 +307,7 @@ struct _Atomic_storage_types<_Ty&> { using _TVal = _Ty; using _TStorage = _Ty&; using _TConstr = _Ty&; - static constexpr bool _Is_ref = false; + static constexpr bool _Is_ref = true; }; // STRUCT TEMPLATE _Atomic_storage From 9229c120a62af32f7d82061e41919eeb0a042621 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 18:43:58 +0300 Subject: [PATCH 025/132] ARM spinlock --- stl/inc/atomic | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index ae7304e9211..06e2bc7a69f 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -392,7 +392,7 @@ struct _Atomic_storage { } else { #if defined(_M_ARM) || defined(_M_ARM64) _Memory_barrier(); - __iso_volatile_store32(&_Spinlock, 0); + __iso_volatile_store32(reinterpret_cast(&_Spinlock), 0); _Memory_barrier(); #else // ^^^ ARM32/ARM64 hardware / x86/x64 hardware vvv _InterlockedExchange(&_Spinlock, 0); From a266baa3dcd4fded86908d7b57a9f81488ac9092 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 18:47:12 +0300 Subject: [PATCH 026/132] clang format --- stl/src/atomic_ref.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/stl/src/atomic_ref.cpp b/stl/src/atomic_ref.cpp index f38237492f4..9071c1488f6 100644 --- a/stl/src/atomic_ref.cpp +++ b/stl/src/atomic_ref.cpp @@ -5,7 +5,6 @@ #include #include - #include SRWLOCK* _Atomic_atomic_ref_get_mutex(const void* const _Key) noexcept { @@ -34,4 +33,4 @@ void __stdcall __std_atomic_ref_lock(const void* const _Key) noexcept { void __stdcall __std_atomic_ref_unlock(const void* const _Key) noexcept { ::ReleaseSRWLockExclusive(_Atomic_atomic_ref_get_mutex(_Key)); } -_END_EXTERN_C \ No newline at end of file +_END_EXTERN_C From 2f992db81f368ac1a5ca30cb837358c7eaaa1550 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 18:47:44 +0300 Subject: [PATCH 027/132] clang format --- stl/src/msvcp_atomic_ref.def | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/src/msvcp_atomic_ref.def b/stl/src/msvcp_atomic_ref.def index 136679224ad..8098d3c4da0 100644 --- a/stl/src/msvcp_atomic_ref.def +++ b/stl/src/msvcp_atomic_ref.def @@ -5,4 +5,4 @@ EXPORTS __std_atomic_ref_lock - __std_atomic_ref_unlock \ No newline at end of file + __std_atomic_ref_unlock From 24f1f01f5b2d1487197a88b288e40719ac035cee Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 20:12:10 +0300 Subject: [PATCH 028/132] found a way to use spinlock pointer for atomic_ref --- stl/inc/atomic | 87 ++++++++++++++++++++++-------------- stl/src/atomic_ref.cpp | 17 ++----- stl/src/msvcp_atomic_ref.def | 3 +- 3 files changed, 59 insertions(+), 48 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 06e2bc7a69f..4d2331b7d6b 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -132,8 +132,7 @@ _NODISCARD extern "C" bool __cdecl __std_atomic_has_cmpxchg16b() noexcept; #define ATOMIC_POINTER_LOCK_FREE 2 _EXTERN_C -void __stdcall __std_atomic_ref_lock(const void* _Key) noexcept; -void __stdcall __std_atomic_ref_unlock(const void* _Key) noexcept; +long* __stdcall __std_atomic_get_mutex(const void* _Key) noexcept; _END_EXTERN_C _STD_BEGIN @@ -281,6 +280,11 @@ struct _Atomic_storage_traits { // properties for how _Ty is stored in an atomic static constexpr bool _Uses_padding = _Padding_size != 0; }; +template +struct _Atomic_storage_traits<_Ty&> { // properties for how _Ty is stored in an atomic + static_assert("Should be defined for atomic ref"); +}; + // STRUCT TEMPLATE _Atomic_padded template ::_Uses_padding> struct _Atomic_padded { // aggregate to allow explicit constexpr zeroing of padding @@ -296,18 +300,26 @@ struct _Atomic_padded<_Ty, false> { template struct _Atomic_storage_types { - using _TVal = _Ty; - using _TStorage = _Atomic_padded<_Ty>; - using _TConstr = const _Ty; - static constexpr bool _Is_ref = false; + using _TVal = _Ty; + using _TStorage = _Atomic_padded<_Ty>; + using _TConstr = const _Ty; + using _Spinlock = long; + + static long* _Get_spinlock(long& _Spinlock) { + return &_Spinlock; + } }; template struct _Atomic_storage_types<_Ty&> { - using _TVal = _Ty; - using _TStorage = _Ty&; - using _TConstr = _Ty&; - static constexpr bool _Is_ref = true; + using _TVal = _Ty; + using _TStorage = _Ty&; + using _TConstr = _Ty&; + using _Spinlock = long*; + + static long* _Get_spinlock(long* _Spinlock) { + return _Spinlock; + } }; // STRUCT TEMPLATE _Atomic_storage @@ -377,32 +389,30 @@ struct _Atomic_storage { #if 1 // TRANSITION, ABI void _Lock() const noexcept { // lock the spinlock - if _CONSTEXPR_IF (_Atomic_storage_types<_Ty>::_Is_ref) { - __std_atomic_ref_lock(_STD addressof(_Storage)); - } else { - while (_InterlockedExchange(&_Spinlock, 1)) { - _YIELD_PROCESSOR(); - } + while (_InterlockedExchange(_Atomic_storage_types<_Ty>::_Get_spinlock(_Spinlock), 1)) { + _YIELD_PROCESSOR(); } } void _Unlock() const noexcept { // unlock the spinlock - if _CONSTEXPR_IF (_Atomic_storage_types<_Ty>::_Is_ref) { - __std_atomic_ref_unlock(_STD addressof(_Storage)); - } else { #if defined(_M_ARM) || defined(_M_ARM64) - _Memory_barrier(); - __iso_volatile_store32(reinterpret_cast(&_Spinlock), 0); - _Memory_barrier(); + _Memory_barrier(); + __iso_volatile_store32(reinterpret_cast(_Atomic_storage_types<_Ty>::_Get_spinlock(_Spinlock)), 0); + _Memory_barrier(); #else // ^^^ ARM32/ARM64 hardware / x86/x64 hardware vvv - _InterlockedExchange(&_Spinlock, 0); + _InterlockedExchange(_Atomic_storage_types<_Ty>::_Get_spinlock(_Spinlock), 0); #endif // hardware - } + + } + +protected: + void _Init_spinlock_for_ref() { + _Spinlock = __std_atomic_get_mutex(_STD addressof(_Storage)); } private: - // Spinlock for non-lock-free atomic. Not used by atomic_ref, but cannot be easily eliminated due to ABI - mutable long _Spinlock = 0; + // Spinlock for non-lock-free atomic. Spinlock pointer for non-lock-free atomic_ref + mutable typename _Atomic_storage_types<_Ty>::_Spinlock _Spinlock = 0; public: _Ty _Storage{}; @@ -425,7 +435,8 @@ public: } _Ty _Storage; - mutable char _Spinlock = 0; // TODO: refisit for atomic_ref + // Revisit for atomic_ref, should be removed completely + mutable char _Spinlock = 0; #endif // TRANSITION, ABI }; @@ -1732,7 +1743,7 @@ atomic(_Ty) -> atomic<_Ty>; #if _HAS_CXX20 template -struct atomic_ref : _Choose_atomic_base_t<_Ty, _Ty&> { // atomic value +struct atomic_ref : _Choose_atomic_base_t<_Ty, _Ty&> { // atomic reference private: using _Base = _Choose_atomic_base_t<_Ty, _Ty&>; @@ -1746,7 +1757,11 @@ public: using value_type = _Ty; - explicit atomic_ref(_Ty& _Value) noexcept /* strengthened */ : _Base(_Value) {} + explicit atomic_ref(_Ty& _Value) noexcept /* strengthened */ : _Base(_Value) { + if constexpr (!is_always_lock_free){ + this->_Init_spinlock_for_ref(); + } + } atomic_ref(const atomic_ref&) noexcept = default; @@ -1754,10 +1769,16 @@ public: static constexpr bool is_always_lock_free = _Is_always_lock_free; - static constexpr size_t required_alignment = - (sizeof(_Ty) == 1 || sizeof(_Ty) == 2 || sizeof(_Ty) == 4 || sizeof(_Ty) == 8 || sizeof(_Ty) == 16) - ? sizeof(_Ty) - : alignof(_Ty); +#if defined(_M_IX86) || defined(_M_ARM) + static constexpr bool _Is_potentially_lock_free = + sizeof(_Ty) == 1 || sizeof(_Ty) == 2 || sizeof(_Ty) == 4 || sizeof(_Ty) == 8; +#elif defined(_M_X64) || defined(_M_ARM64) + static constexpr bool _Is_potentially_lock_free = + sizeof(_Ty) == 1 || sizeof(_Ty) == 2 || sizeof(_Ty) == 4 || sizeof(_Ty) == 8 || sizeof(_Ty) == 16; +#else +#error Unsupported atchitecture +#endif + static constexpr size_t required_alignment = _Is_potentially_lock_free ? sizeof(_Ty) : alignof(_Ty); #if 1 // TRANSITION, ABI _NODISCARD bool is_lock_free() const noexcept { diff --git a/stl/src/atomic_ref.cpp b/stl/src/atomic_ref.cpp index 9071c1488f6..fe46f80b64e 100644 --- a/stl/src/atomic_ref.cpp +++ b/stl/src/atomic_ref.cpp @@ -5,17 +5,17 @@ #include #include -#include -SRWLOCK* _Atomic_atomic_ref_get_mutex(const void* const _Key) noexcept { + +_EXTERN_C +long* __stdcall __std_atomic_get_mutex(const void* const _Key) noexcept { constexpr size_t _Table_size_power = 8; constexpr size_t _Table_size = 1 << _Table_size_power; constexpr size_t _Table_index_mask = _Table_size - 1; - #pragma warning(push) #pragma warning(disable : 4324) // '%s': structure was padded due to alignment specifier struct alignas(std::hardware_destructive_interference_size) _Table_entry { - SRWLOCK _Mutex = SRWLOCK_INIT; + long _Mutex = 0; }; static _Table_entry _Table[_Table_size]; #pragma warning(pop) @@ -24,13 +24,4 @@ SRWLOCK* _Atomic_atomic_ref_get_mutex(const void* const _Key) noexcept { _Index ^= _Index >> _Table_size_power; return &_Table[_Index & _Table_index_mask]._Mutex; } - -_EXTERN_C -void __stdcall __std_atomic_ref_lock(const void* const _Key) noexcept { - ::AcquireSRWLockExclusive(_Atomic_atomic_ref_get_mutex(_Key)); -} - -void __stdcall __std_atomic_ref_unlock(const void* const _Key) noexcept { - ::ReleaseSRWLockExclusive(_Atomic_atomic_ref_get_mutex(_Key)); -} _END_EXTERN_C diff --git a/stl/src/msvcp_atomic_ref.def b/stl/src/msvcp_atomic_ref.def index 8098d3c4da0..f1d1c361b80 100644 --- a/stl/src/msvcp_atomic_ref.def +++ b/stl/src/msvcp_atomic_ref.def @@ -4,5 +4,4 @@ ; atomic ref satellite DLL definition EXPORTS - __std_atomic_ref_lock - __std_atomic_ref_unlock + __std_atomic_get_mutex From b166dd1e5715633b481cc7b3e43c1746ec096a65 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 20:22:21 +0300 Subject: [PATCH 029/132] clang format --- stl/inc/atomic | 8 +++----- stl/src/atomic_ref.cpp | 1 - 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 4d2331b7d6b..46a64285d0a 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -402,7 +402,6 @@ struct _Atomic_storage { #else // ^^^ ARM32/ARM64 hardware / x86/x64 hardware vvv _InterlockedExchange(_Atomic_storage_types<_Ty>::_Get_spinlock(_Spinlock), 0); #endif // hardware - } protected: @@ -435,8 +434,8 @@ public: } _Ty _Storage; - // Revisit for atomic_ref, should be removed completely - mutable char _Spinlock = 0; + // Revisit, could be removed completely for atomic_ref + mutable char _Spinlock = 0; #endif // TRANSITION, ABI }; @@ -1758,7 +1757,7 @@ public: using value_type = _Ty; explicit atomic_ref(_Ty& _Value) noexcept /* strengthened */ : _Base(_Value) { - if constexpr (!is_always_lock_free){ + if constexpr (!is_always_lock_free) { this->_Init_spinlock_for_ref(); } } @@ -1806,7 +1805,6 @@ public: return this->compare_exchange_strong(_Expected, _Desired); } - bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired, const memory_order _Order) noexcept { return this->compare_exchange_strong(_Expected, _Desired, _Order); } diff --git a/stl/src/atomic_ref.cpp b/stl/src/atomic_ref.cpp index fe46f80b64e..31a9bda8c7f 100644 --- a/stl/src/atomic_ref.cpp +++ b/stl/src/atomic_ref.cpp @@ -6,7 +6,6 @@ #include #include - _EXTERN_C long* __stdcall __std_atomic_get_mutex(const void* const _Key) noexcept { constexpr size_t _Table_size_power = 8; From e3c2331dcb2f07839f91d0e532ce6dba72f9c392 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 20:27:17 +0300 Subject: [PATCH 030/132] missing strong CAS --- stl/inc/atomic | 6 ++++++ tests/std/tests/P0019R8_atomic_ref/test.cpp | 1 - 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 46a64285d0a..87a4102cd32 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -1801,6 +1801,12 @@ public: return _Value; } + using _Base::compare_exchange_strong; + bool compare_exchange_strong( + _Ty& _Expected, const _Ty _Desired, const memory_order _Success, const memory_order _Failure) noexcept { + return this->compare_exchange_strong(_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); + } + bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired) noexcept { return this->compare_exchange_strong(_Expected, _Desired); } diff --git a/tests/std/tests/P0019R8_atomic_ref/test.cpp b/tests/std/tests/P0019R8_atomic_ref/test.cpp index b1328e20352..3fb1dd48634 100644 --- a/tests/std/tests/P0019R8_atomic_ref/test.cpp +++ b/tests/std/tests/P0019R8_atomic_ref/test.cpp @@ -8,7 +8,6 @@ #include #include - struct bigint { int value; int more_value[10]; From b4973b3c5afbcfdd1bb7f72d3561e18339d44a50 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 21:30:13 +0300 Subject: [PATCH 031/132] Consistently use value type --- stl/inc/atomic | 79 +++++++++++++++++++++++--------------------------- 1 file changed, 37 insertions(+), 42 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 87a4102cd32..58683ac23c2 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -864,7 +864,7 @@ struct _Atomic_storage<_Ty, 16> { // lock-free using 16-byte intrinsics } _TVal exchange(const _TVal _Value) noexcept { // exchange with sequential consistency - _Ty _Result{_Value}; + _TVal _Result{_Value}; while (!compare_exchange_strong(_Result, _Value)) { // keep trying } @@ -872,7 +872,7 @@ struct _Atomic_storage<_Ty, 16> { // lock-free using 16-byte intrinsics } _TVal exchange(const _TVal _Value, const memory_order _Order) noexcept { // exchange with given memory order - _Ty _Result{_Value}; + _TVal _Result{_Value}; while (!compare_exchange_strong(_Result, _Value, _Order)) { // keep trying } @@ -918,6 +918,7 @@ struct _Atomic_integral; // not defined template struct _Atomic_integral<_Ty, 1> : _Atomic_storage<_Ty> { // atomic integral operations using 1-byte intrinsics using _Base = _Atomic_storage<_Ty>; + using _TVal = typename _Base::_TVal; #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 _Atomic_integral() = default; @@ -927,8 +928,6 @@ struct _Atomic_integral<_Ty, 1> : _Atomic_storage<_Ty> { // atomic integral oper using _Base::_Base; #endif // ^^^ no workaround ^^^ - using _TVal = typename _Base::_TVal; - _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { char _Result; _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedExchangeAdd8, _Atomic_address_as(this->_Storage), @@ -984,6 +983,7 @@ struct _Atomic_integral<_Ty, 1> : _Atomic_storage<_Ty> { // atomic integral oper template struct _Atomic_integral<_Ty, 2> : _Atomic_storage<_Ty> { // atomic integral operations using 2-byte intrinsics using _Base = _Atomic_storage<_Ty>; + using _TVal = typename _Base::_TVal; #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 _Atomic_integral() = default; @@ -993,8 +993,6 @@ struct _Atomic_integral<_Ty, 2> : _Atomic_storage<_Ty> { // atomic integral oper using _Base::_Base; #endif // ^^^ no workaround ^^^ - using _TVal = typename _Base::_TVal; - _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { short _Result; _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedExchangeAdd16, _Atomic_address_as(this->_Storage), @@ -1049,6 +1047,7 @@ struct _Atomic_integral<_Ty, 2> : _Atomic_storage<_Ty> { // atomic integral oper template struct _Atomic_integral<_Ty, 4> : _Atomic_storage<_Ty> { // atomic integral operations using 4-byte intrinsics using _Base = _Atomic_storage<_Ty>; + using _TVal = typename _Base::_TVal; #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 _Atomic_integral() = default; @@ -1058,8 +1057,6 @@ struct _Atomic_integral<_Ty, 4> : _Atomic_storage<_Ty> { // atomic integral oper using _Base::_Base; #endif // ^^^ no workaround ^^^ - using _TVal = typename _Base::_TVal; - _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { long _Result; _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedExchangeAdd, _Atomic_address_as(this->_Storage), @@ -1114,6 +1111,7 @@ struct _Atomic_integral<_Ty, 4> : _Atomic_storage<_Ty> { // atomic integral oper template struct _Atomic_integral<_Ty, 8> : _Atomic_storage<_Ty> { // atomic integral operations using 8-byte intrinsics using _Base = _Atomic_storage<_Ty>; + using _TVal = typename _Base::_TVal; #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 _Atomic_integral() = default; @@ -1123,8 +1121,6 @@ struct _Atomic_integral<_Ty, 8> : _Atomic_storage<_Ty> { // atomic integral oper using _Base::_Base; #endif // ^^^ no workaround ^^^ - using _TVal = typename _Base::_TVal; - #ifdef _M_IX86 _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { // effectively sequential consistency @@ -1255,6 +1251,7 @@ template struct _Atomic_integral_facade : _Atomic_integral<_Ty> { // provides operator overloads and other support for atomic integral specializations using _Base = _Atomic_integral<_Ty>; + using _TVal = typename _Base::_TVal; using difference_type = _Ty; #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 @@ -1265,8 +1262,6 @@ struct _Atomic_integral_facade : _Atomic_integral<_Ty> { using _Base::_Base; #endif // ^^^ no workaround ^^^ - using _TVal = typename _Base::_TVal; - // _Deprecate_non_lock_free_volatile is unnecessary here. // note: const_cast-ing away volatile is safe because all our intrinsics add volatile back on. @@ -1392,7 +1387,8 @@ template struct _Atomic_floating : _Atomic_storage<_Ty> { // provides atomic floating-point operations using _Base = _Atomic_storage<_Ty>; - using difference_type = _Ty; + using _TVal = typename _Base::_TVal; + using difference_type = _TVal; #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 _Atomic_floating() = default; @@ -1402,8 +1398,6 @@ struct _Atomic_floating : _Atomic_storage<_Ty> { using _Base::_Base; #endif // ^^^ no workaround ^^^ - using _TVal = typename _Base::_TVal; - _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { _TVal _Temp{this->load(memory_order_relaxed)}; while (!this->compare_exchange_strong(_Temp, _Temp + _Operand, _Order)) { // keep trying @@ -1422,30 +1416,30 @@ struct _Atomic_floating : _Atomic_storage<_Ty> { } _TVal fetch_sub(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - _Ty _Temp{this->load(memory_order_relaxed)}; + _TVal _Temp{this->load(memory_order_relaxed)}; while (!this->compare_exchange_strong(_Temp, _Temp - _Operand, _Order)) { // keep trying } return _Temp; } - _Ty fetch_sub(const _Ty _Operand, const memory_order _Order = memory_order_seq_cst) volatile noexcept { + _TVal fetch_sub(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) volatile noexcept { return const_cast<_Atomic_floating*>(this)->fetch_sub(_Operand, _Order); } - _Ty operator+=(const _Ty _Operand) noexcept { + _TVal operator+=(const _TVal _Operand) noexcept { return fetch_add(_Operand) + _Operand; } - _Ty operator+=(const _Ty _Operand) volatile noexcept { + _TVal operator+=(const _TVal _Operand) volatile noexcept { return const_cast<_Atomic_floating*>(this)->fetch_add(_Operand) + _Operand; } - _Ty operator-=(const _Ty _Operand) noexcept { + _TVal operator-=(const _TVal _Operand) noexcept { return fetch_sub(_Operand) - _Operand; } - _Ty operator-=(const _Ty _Operand) volatile noexcept { + _TVal operator-=(const _TVal _Operand) volatile noexcept { return const_cast<_Atomic_floating*>(this)->fetch_sub(_Operand) - _Operand; } }; @@ -1455,6 +1449,7 @@ struct _Atomic_floating : _Atomic_storage<_Ty> { template struct _Atomic_pointer : _Atomic_storage<_Ty> { using _Base = _Atomic_storage<_Ty>; + using _TVal = typename _Base::_TVal; using difference_type = ptrdiff_t; #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 @@ -1465,9 +1460,9 @@ struct _Atomic_pointer : _Atomic_storage<_Ty> { using _Base::_Base; #endif // ^^^ no workaround ^^^ - _Ty fetch_add(const ptrdiff_t _Diff, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal fetch_add(const ptrdiff_t _Diff, const memory_order _Order = memory_order_seq_cst) noexcept { const ptrdiff_t _Shift_bytes = - static_cast(static_cast(_Diff) * sizeof(remove_pointer_t<_Ty>)); + static_cast(static_cast(_Diff) * sizeof(remove_pointer_t<_TVal>)); ptrdiff_t _Result; #if defined(_M_IX86) || defined(_M_ARM) _ATOMIC_CHOOSE_INTRINSIC( @@ -1476,80 +1471,80 @@ struct _Atomic_pointer : _Atomic_storage<_Ty> { _ATOMIC_CHOOSE_INTRINSIC( _Order, _Result, _InterlockedExchangeAdd64, _Atomic_address_as(this->_Storage), _Shift_bytes); #endif // hardware - return reinterpret_cast<_Ty>(_Result); + return reinterpret_cast<_TVal>(_Result); } // _Deprecate_non_lock_free_volatile is unnecessary here. - _Ty fetch_add(const ptrdiff_t _Diff) volatile noexcept { + _TVal fetch_add(const ptrdiff_t _Diff) volatile noexcept { return const_cast<_Atomic_pointer*>(this)->fetch_add(_Diff); } - _Ty fetch_add(const ptrdiff_t _Diff, const memory_order _Order) volatile noexcept { + _TVal fetch_add(const ptrdiff_t _Diff, const memory_order _Order) volatile noexcept { return const_cast<_Atomic_pointer*>(this)->fetch_add(_Diff, _Order); } - _Ty fetch_sub(const ptrdiff_t _Diff) volatile noexcept { + _TVal fetch_sub(const ptrdiff_t _Diff) volatile noexcept { return fetch_add(static_cast(0 - static_cast(_Diff))); } - _Ty fetch_sub(const ptrdiff_t _Diff) noexcept { + _TVal fetch_sub(const ptrdiff_t _Diff) noexcept { return fetch_add(static_cast(0 - static_cast(_Diff))); } - _Ty fetch_sub(const ptrdiff_t _Diff, const memory_order _Order) volatile noexcept { + _TVal fetch_sub(const ptrdiff_t _Diff, const memory_order _Order) volatile noexcept { return fetch_add(static_cast(0 - static_cast(_Diff)), _Order); } - _Ty fetch_sub(const ptrdiff_t _Diff, const memory_order _Order) noexcept { + _TVal fetch_sub(const ptrdiff_t _Diff, const memory_order _Order) noexcept { return fetch_add(static_cast(0 - static_cast(_Diff)), _Order); } - _Ty operator++(int) volatile noexcept { + _TVal operator++(int) volatile noexcept { return fetch_add(1); } - _Ty operator++(int) noexcept { + _TVal operator++(int) noexcept { return fetch_add(1); } - _Ty operator++() volatile noexcept { + _TVal operator++() volatile noexcept { return fetch_add(1) + 1; } - _Ty operator++() noexcept { + _TVal operator++() noexcept { return fetch_add(1) + 1; } - _Ty operator--(int) volatile noexcept { + _TVal operator--(int) volatile noexcept { return fetch_add(-1); } - _Ty operator--(int) noexcept { + _TVal operator--(int) noexcept { return fetch_add(-1); } - _Ty operator--() volatile noexcept { + _TVal operator--() volatile noexcept { return fetch_add(-1) - 1; } - _Ty operator--() noexcept { + _TVal operator--() noexcept { return fetch_add(-1) - 1; } - _Ty operator+=(const ptrdiff_t _Diff) volatile noexcept { + _TVal operator+=(const ptrdiff_t _Diff) volatile noexcept { return fetch_add(_Diff) + _Diff; } - _Ty operator+=(const ptrdiff_t _Diff) noexcept { + _TVal operator+=(const ptrdiff_t _Diff) noexcept { return fetch_add(_Diff) + _Diff; } - _Ty operator-=(const ptrdiff_t _Diff) volatile noexcept { + _TVal operator-=(const ptrdiff_t _Diff) volatile noexcept { return fetch_add(static_cast(0 - static_cast(_Diff))) - _Diff; } - _Ty operator-=(const ptrdiff_t _Diff) noexcept { + _TVal operator-=(const ptrdiff_t _Diff) noexcept { return fetch_add(static_cast(0 - static_cast(_Diff))) - _Diff; } }; From 15bbdbeca248d6462664498837c35db45259aa3b Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 17 May 2020 21:33:28 +0300 Subject: [PATCH 032/132] Consistently use value type --- stl/inc/atomic | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 58683ac23c2..191cf65b675 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -1252,7 +1252,7 @@ struct _Atomic_integral_facade : _Atomic_integral<_Ty> { // provides operator overloads and other support for atomic integral specializations using _Base = _Atomic_integral<_Ty>; using _TVal = typename _Base::_TVal; - using difference_type = _Ty; + using difference_type = _TVal; #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 _Atomic_integral_facade() = default; From 4d07c1cb5688722a60fc8768632c565a8c9dc43b Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 23 May 2020 14:03:09 +0300 Subject: [PATCH 033/132] Remove volatile overloads for atomic_ref --- stl/inc/atomic | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/stl/inc/atomic b/stl/inc/atomic index 191cf65b675..19322bd523e 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -305,6 +305,8 @@ struct _Atomic_storage_types { using _TConstr = const _Ty; using _Spinlock = long; + static constexpr bool _Has_volatile = true; + static long* _Get_spinlock(long& _Spinlock) { return &_Spinlock; } @@ -317,6 +319,8 @@ struct _Atomic_storage_types<_Ty&> { using _TConstr = _Ty&; using _Spinlock = long*; + static constexpr bool _Has_volatile = false; + static long* _Get_spinlock(long* _Spinlock) { return _Spinlock; } @@ -1269,10 +1273,12 @@ struct _Atomic_integral_facade : _Atomic_integral<_Ty> { // are far more common than volatile ones. using _Base::fetch_add; _TVal fetch_add(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_add(_Operand); } _TVal fetch_add(const _TVal _Operand, const memory_order _Order) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_add(_Operand, _Order); } @@ -1285,6 +1291,7 @@ struct _Atomic_integral_facade : _Atomic_integral<_Ty> { } _TVal fetch_sub(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return fetch_add(_Negate(_Operand)); } @@ -1293,51 +1300,62 @@ struct _Atomic_integral_facade : _Atomic_integral<_Ty> { } _TVal fetch_sub(const _TVal _Operand, const memory_order _Order) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return fetch_add(_Negate(_Operand), _Order); } using _Base::fetch_and; _TVal fetch_and(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_and(_Operand); } _TVal fetch_and(const _TVal _Operand, const memory_order _Order) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_and(_Operand, _Order); } using _Base::fetch_or; _TVal fetch_or(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_or(_Operand); } _TVal fetch_or(const _TVal _Operand, const memory_order _Order) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_or(_Operand, _Order); } using _Base::fetch_xor; _TVal fetch_xor(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_xor(_Operand); } _TVal fetch_xor(const _TVal _Operand, const memory_order _Order) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_xor(_Operand, _Order); } using _Base::operator++; _TVal operator++(int) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return const_cast<_Atomic_integral_facade*>(this)->_Base::operator++(0); } _TVal operator++() volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return const_cast<_Atomic_integral_facade*>(this)->_Base::operator++(); } using _Base::operator--; _TVal operator--(int) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return const_cast<_Atomic_integral_facade*>(this)->_Base::operator--(0); } _TVal operator--() volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return const_cast<_Atomic_integral_facade*>(this)->_Base::operator--(); } @@ -1346,6 +1364,7 @@ struct _Atomic_integral_facade : _Atomic_integral<_Ty> { } _TVal operator+=(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return static_cast<_TVal>(const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_add(_Operand) + _Operand); } @@ -1354,6 +1373,7 @@ struct _Atomic_integral_facade : _Atomic_integral<_Ty> { } _TVal operator-=(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return static_cast<_TVal>(const_cast<_Atomic_integral_facade*>(this)->fetch_sub(_Operand) - _Operand); } @@ -1362,6 +1382,7 @@ struct _Atomic_integral_facade : _Atomic_integral<_Ty> { } _TVal operator&=(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return static_cast<_TVal>(const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_and(_Operand) & _Operand); } @@ -1370,6 +1391,7 @@ struct _Atomic_integral_facade : _Atomic_integral<_Ty> { } _TVal operator|=(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return static_cast<_TVal>(const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_or(_Operand) | _Operand); } @@ -1378,6 +1400,7 @@ struct _Atomic_integral_facade : _Atomic_integral<_Ty> { } _TVal operator^=(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return static_cast<_TVal>(const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_xor(_Operand) ^ _Operand); } }; @@ -1412,6 +1435,7 @@ struct _Atomic_floating : _Atomic_storage<_Ty> { // We make the primary functions non-volatile for better debug codegen, as non-volatile atomics // are far more common than volatile ones. _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return const_cast<_Atomic_floating*>(this)->fetch_add(_Operand, _Order); } @@ -1424,6 +1448,7 @@ struct _Atomic_floating : _Atomic_storage<_Ty> { } _TVal fetch_sub(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return const_cast<_Atomic_floating*>(this)->fetch_sub(_Operand, _Order); } @@ -1432,6 +1457,7 @@ struct _Atomic_floating : _Atomic_storage<_Ty> { } _TVal operator+=(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return const_cast<_Atomic_floating*>(this)->fetch_add(_Operand) + _Operand; } @@ -1440,6 +1466,7 @@ struct _Atomic_floating : _Atomic_storage<_Ty> { } _TVal operator-=(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return const_cast<_Atomic_floating*>(this)->fetch_sub(_Operand) - _Operand; } }; @@ -1477,14 +1504,17 @@ struct _Atomic_pointer : _Atomic_storage<_Ty> { // _Deprecate_non_lock_free_volatile is unnecessary here. _TVal fetch_add(const ptrdiff_t _Diff) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return const_cast<_Atomic_pointer*>(this)->fetch_add(_Diff); } _TVal fetch_add(const ptrdiff_t _Diff, const memory_order _Order) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return const_cast<_Atomic_pointer*>(this)->fetch_add(_Diff, _Order); } _TVal fetch_sub(const ptrdiff_t _Diff) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return fetch_add(static_cast(0 - static_cast(_Diff))); } @@ -1493,6 +1523,7 @@ struct _Atomic_pointer : _Atomic_storage<_Ty> { } _TVal fetch_sub(const ptrdiff_t _Diff, const memory_order _Order) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return fetch_add(static_cast(0 - static_cast(_Diff)), _Order); } @@ -1501,6 +1532,7 @@ struct _Atomic_pointer : _Atomic_storage<_Ty> { } _TVal operator++(int) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return fetch_add(1); } @@ -1509,6 +1541,7 @@ struct _Atomic_pointer : _Atomic_storage<_Ty> { } _TVal operator++() volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return fetch_add(1) + 1; } @@ -1517,6 +1550,7 @@ struct _Atomic_pointer : _Atomic_storage<_Ty> { } _TVal operator--(int) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return fetch_add(-1); } @@ -1525,6 +1559,7 @@ struct _Atomic_pointer : _Atomic_storage<_Ty> { } _TVal operator--() volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return fetch_add(-1) - 1; } @@ -1533,6 +1568,7 @@ struct _Atomic_pointer : _Atomic_storage<_Ty> { } _TVal operator+=(const ptrdiff_t _Diff) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return fetch_add(_Diff) + _Diff; } @@ -1541,6 +1577,7 @@ struct _Atomic_pointer : _Atomic_storage<_Ty> { } _TVal operator-=(const ptrdiff_t _Diff) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); return fetch_add(static_cast(0 - static_cast(_Diff))) - _Diff; } From ef2adde6329f09de82fe01861995772feeba3b23 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 23 May 2020 15:31:10 +0300 Subject: [PATCH 034/132] try to define vNext as well --- stl/inc/atomic | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 19322bd523e..e2fe5f72b21 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -281,9 +281,10 @@ struct _Atomic_storage_traits { // properties for how _Ty is stored in an atomic }; template -struct _Atomic_storage_traits<_Ty&> { // properties for how _Ty is stored in an atomic - static_assert("Should be defined for atomic ref"); -}; +struct _Atomic_storage_traits<_Ty&> { // properties for how _Ty is stored in an atomic_ref + static constexpr size_t _Storage_size = sizeof(_Ty); + static constexpr bool _Uses_padding = false; +} // STRUCT TEMPLATE _Atomic_padded template ::_Uses_padding> @@ -296,6 +297,12 @@ template struct _Atomic_padded<_Ty, false> { alignas(sizeof(_Ty)) mutable _Ty _Value; // align to sizeof(T); x86 stack aligns 8-byte objects on 4-byte boundaries }; + +template +struct _Atomic_padded<_Ty&, false> { + _Ty& _Value; +}; + #endif // TRANSITION, ABI template From 49b7f95cd303dc7e19f95ae2ba14cf1dc812a89b Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 23 May 2020 17:25:25 +0300 Subject: [PATCH 035/132] Zero pad for potential future redefinition --- stl/src/atomic_ref.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stl/src/atomic_ref.cpp b/stl/src/atomic_ref.cpp index 31a9bda8c7f..4b7cf62d1b3 100644 --- a/stl/src/atomic_ref.cpp +++ b/stl/src/atomic_ref.cpp @@ -11,13 +11,13 @@ long* __stdcall __std_atomic_get_mutex(const void* const _Key) noexcept { constexpr size_t _Table_size_power = 8; constexpr size_t _Table_size = 1 << _Table_size_power; constexpr size_t _Table_index_mask = _Table_size - 1; -#pragma warning(push) -#pragma warning(disable : 4324) // '%s': structure was padded due to alignment specifier + struct alignas(std::hardware_destructive_interference_size) _Table_entry { - long _Mutex = 0; + long _Mutex = 0; + char _Pad[std::hardware_destructive_interference_size - sizeof(long)] = {}; }; static _Table_entry _Table[_Table_size]; -#pragma warning(pop) + auto _Index = reinterpret_cast(_Key); _Index ^= _Index >> (_Table_size_power * 2); _Index ^= _Index >> _Table_size_power; From 1be9a85301ec13d35ece73ebc6305b786e7591a9 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 25 May 2020 15:30:10 +0300 Subject: [PATCH 036/132] Warn about misuse "It is a quality-of-implementation issue to generate appropriate information when constraints are violated." - from P0019R8 --- stl/inc/atomic | 26 +++++++++++++++++--------- stl/inc/yvals.h | 15 +++++++++++++++ stl/inc/yvals_core.h | 14 +++++++++++++- 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index e2fe5f72b21..33d28957d9a 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -1796,7 +1796,12 @@ public: using value_type = _Ty; explicit atomic_ref(_Ty& _Value) noexcept /* strengthened */ : _Base(_Value) { - if constexpr (!is_always_lock_free) { + if constexpr (_Is_potentially_lock_free) { + if constexpr (alignof(_Ty) < sizeof(_Ty)) { + _Warn_potential_misaligment(); + } + _Check_aligment(_Value); + } else { this->_Init_spinlock_for_ref(); } } @@ -1807,15 +1812,9 @@ public: static constexpr bool is_always_lock_free = _Is_always_lock_free; -#if defined(_M_IX86) || defined(_M_ARM) static constexpr bool _Is_potentially_lock_free = - sizeof(_Ty) == 1 || sizeof(_Ty) == 2 || sizeof(_Ty) == 4 || sizeof(_Ty) == 8; -#elif defined(_M_X64) || defined(_M_ARM64) - static constexpr bool _Is_potentially_lock_free = - sizeof(_Ty) == 1 || sizeof(_Ty) == 2 || sizeof(_Ty) == 4 || sizeof(_Ty) == 8 || sizeof(_Ty) == 16; -#else -#error Unsupported atchitecture -#endif + sizeof(_Ty) <= 2 * sizeof(void*) && (sizeof(_Ty) & (sizeof(_Ty) - 1)) == 0; + static constexpr size_t required_alignment = _Is_potentially_lock_free ? sizeof(_Ty) : alignof(_Ty); #if 1 // TRANSITION, ABI @@ -1862,6 +1861,15 @@ public: operator _Ty() const noexcept { return this->load(); } + +private: + _CXX20_ATOMIC_REF_POTENTIALLY_MISALIGNED_TYPE void _Warn_potential_misaligment() {} + + void _Check_aligment(_Ty& _Value) { + _ATOMIC_REF_CHECK_ALIGNMENT( + (reinterpret_cast(_STD addressof(_Value)) & (required_alignment - 1)) == 0, + "atomic_ref underlying object is not aligned as required_alignment"); + } }; #endif // _HAS_CXX20 diff --git a/stl/inc/yvals.h b/stl/inc/yvals.h index 1a70aa17ea4..9e127afe153 100644 --- a/stl/inc/yvals.h +++ b/stl/inc/yvals.h @@ -208,6 +208,21 @@ _STL_DISABLE_CLANG_WARNINGS #define _STL_INTERNAL_STATIC_ASSERT(...) #endif // _ENABLE_STL_INTERNAL_CHECK +#ifndef _ENABLE_ATOMIC_REF_ALIGNMENT_CHECK +#ifdef _DEBUG +#define _ENABLE_ATOMIC_REF_ALIGNMENT_CHECK 1 +#else // ^^^ _DEBUG ^^^ // vvv !_DEBUG vvv +#define _ENABLE_ATOMIC_REF_ALIGNMENT_CHECK 0 +#endif // _DEBUG +#endif // _ENABLE_ATOMIC_REF_ALIGNMENT_CHECK + +#if _ENABLE_ATOMIC_REF_ALIGNMENT_CHECK +#define _ATOMIC_REF_CHECK_ALIGNMENT(cond, mesg) _STL_VERIFY(cond, mesg) +#else +#define _ATOMIC_REF_CHECK_ALIGNMENT(cond, mesg) _Analysis_assume_(cond) +#endif + + #include #define _WARNING_MESSAGE(NUMBER, MESSAGE) __FILE__ "(" _CRT_STRINGIZE(__LINE__) "): warning " NUMBER ": " MESSAGE diff --git a/stl/inc/yvals_core.h b/stl/inc/yvals_core.h index c1d12a5da2f..4cc78af71a2 100644 --- a/stl/inc/yvals_core.h +++ b/stl/inc/yvals_core.h @@ -983,7 +983,19 @@ #define _CXX20_DEPRECATE_MOVE_ITERATOR_ARROW #endif // ^^^ warning disabled ^^^ -// next warning number: STL4032 +#if _HAS_CXX20 && !defined(_SILENCE_CXX20_ATOMIC_REF_POTENTIALLY_MISALIGNED_TYPE) +#define _CXX20_ATOMIC_REF_POTENTIALLY_MISALIGNED_TYPE \ + [[deprecated("warning STL4032: " \ + "underlying type for atomic_ref has smaller aligment than required_alignment. " \ + "It is up to the caller to make sure that the actual aligment of the object " \ + "satisfies required_alignment requirement. " \ + "You can define _SILENCE_CXX20_ATOMIC_REF_POTENTIALLY_MISALIGNED_TYPE " \ + "to acknowledge that you have received this warning.")]] +#else // ^^^ warning enabled / warning disabled vvv +#define _CXX20_ATOMIC_REF_POTENTIALLY_MISALIGNED_TYPE +#endif // ^^^ warning disabled ^^^ + +// next warning number: STL4033 // P0619R4 Removing C++17-Deprecated Features #ifndef _HAS_FEATURES_REMOVED_IN_CXX20 From 91f3819b65352847f684b6644c692f9aa14dfaed Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 25 May 2020 15:53:23 +0300 Subject: [PATCH 037/132] clang-format --- stl/inc/yvals_core.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/inc/yvals_core.h b/stl/inc/yvals_core.h index 4cc78af71a2..1ee5a53900a 100644 --- a/stl/inc/yvals_core.h +++ b/stl/inc/yvals_core.h @@ -988,8 +988,8 @@ [[deprecated("warning STL4032: " \ "underlying type for atomic_ref has smaller aligment than required_alignment. " \ "It is up to the caller to make sure that the actual aligment of the object " \ - "satisfies required_alignment requirement. " \ - "You can define _SILENCE_CXX20_ATOMIC_REF_POTENTIALLY_MISALIGNED_TYPE " \ + "satisfies required_alignment requirement. " \ + "You can define _SILENCE_CXX20_ATOMIC_REF_POTENTIALLY_MISALIGNED_TYPE " \ "to acknowledge that you have received this warning.")]] #else // ^^^ warning enabled / warning disabled vvv #define _CXX20_ATOMIC_REF_POTENTIALLY_MISALIGNED_TYPE From f3e9c76a9e1a2b0fa15ff80e4e96efbea5bc81ae Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 25 May 2020 15:57:11 +0300 Subject: [PATCH 038/132] might --- stl/inc/yvals_core.h | 1 + 1 file changed, 1 insertion(+) diff --git a/stl/inc/yvals_core.h b/stl/inc/yvals_core.h index 1ee5a53900a..1430d86c473 100644 --- a/stl/inc/yvals_core.h +++ b/stl/inc/yvals_core.h @@ -987,6 +987,7 @@ #define _CXX20_ATOMIC_REF_POTENTIALLY_MISALIGNED_TYPE \ [[deprecated("warning STL4032: " \ "underlying type for atomic_ref has smaller aligment than required_alignment. " \ + "required_alignment requirement might be violated." \ "It is up to the caller to make sure that the actual aligment of the object " \ "satisfies required_alignment requirement. " \ "You can define _SILENCE_CXX20_ATOMIC_REF_POTENTIALLY_MISALIGNED_TYPE " \ From fe2e25f2a645f6aec761cbf4f546c938a541708b Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 25 May 2020 15:59:53 +0300 Subject: [PATCH 039/132] formatting --- stl/inc/yvals_core.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/inc/yvals_core.h b/stl/inc/yvals_core.h index 1430d86c473..d7e288bcc7e 100644 --- a/stl/inc/yvals_core.h +++ b/stl/inc/yvals_core.h @@ -987,7 +987,7 @@ #define _CXX20_ATOMIC_REF_POTENTIALLY_MISALIGNED_TYPE \ [[deprecated("warning STL4032: " \ "underlying type for atomic_ref has smaller aligment than required_alignment. " \ - "required_alignment requirement might be violated." \ + "required_alignment requirement might be violated. " \ "It is up to the caller to make sure that the actual aligment of the object " \ "satisfies required_alignment requirement. " \ "You can define _SILENCE_CXX20_ATOMIC_REF_POTENTIALLY_MISALIGNED_TYPE " \ From 17297a93260bc88ae1abd205f789fe1666fc9702 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 25 May 2020 17:41:48 +0300 Subject: [PATCH 040/132] drop static check, keep only runtime --- stl/inc/atomic | 3 --- stl/inc/yvals_core.h | 15 +-------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 33d28957d9a..d8668a915c8 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -1797,9 +1797,6 @@ public: explicit atomic_ref(_Ty& _Value) noexcept /* strengthened */ : _Base(_Value) { if constexpr (_Is_potentially_lock_free) { - if constexpr (alignof(_Ty) < sizeof(_Ty)) { - _Warn_potential_misaligment(); - } _Check_aligment(_Value); } else { this->_Init_spinlock_for_ref(); diff --git a/stl/inc/yvals_core.h b/stl/inc/yvals_core.h index d7e288bcc7e..c1d12a5da2f 100644 --- a/stl/inc/yvals_core.h +++ b/stl/inc/yvals_core.h @@ -983,20 +983,7 @@ #define _CXX20_DEPRECATE_MOVE_ITERATOR_ARROW #endif // ^^^ warning disabled ^^^ -#if _HAS_CXX20 && !defined(_SILENCE_CXX20_ATOMIC_REF_POTENTIALLY_MISALIGNED_TYPE) -#define _CXX20_ATOMIC_REF_POTENTIALLY_MISALIGNED_TYPE \ - [[deprecated("warning STL4032: " \ - "underlying type for atomic_ref has smaller aligment than required_alignment. " \ - "required_alignment requirement might be violated. " \ - "It is up to the caller to make sure that the actual aligment of the object " \ - "satisfies required_alignment requirement. " \ - "You can define _SILENCE_CXX20_ATOMIC_REF_POTENTIALLY_MISALIGNED_TYPE " \ - "to acknowledge that you have received this warning.")]] -#else // ^^^ warning enabled / warning disabled vvv -#define _CXX20_ATOMIC_REF_POTENTIALLY_MISALIGNED_TYPE -#endif // ^^^ warning disabled ^^^ - -// next warning number: STL4033 +// next warning number: STL4032 // P0619R4 Removing C++17-Deprecated Features #ifndef _HAS_FEATURES_REMOVED_IN_CXX20 From 45191a0288cde35dc0abf4cb5fd7b517fd962e1e Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 25 May 2020 17:46:11 +0300 Subject: [PATCH 041/132] drop static check, keep only runtime --- stl/inc/atomic | 2 -- 1 file changed, 2 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index d8668a915c8..c0c21009da4 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -1860,8 +1860,6 @@ public: } private: - _CXX20_ATOMIC_REF_POTENTIALLY_MISALIGNED_TYPE void _Warn_potential_misaligment() {} - void _Check_aligment(_Ty& _Value) { _ATOMIC_REF_CHECK_ALIGNMENT( (reinterpret_cast(_STD addressof(_Value)) & (required_alignment - 1)) == 0, From 8f18bbea29ac71d17ba987209409499c29307256 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 25 May 2020 19:35:44 +0300 Subject: [PATCH 042/132] fix tests pass --- stl/inc/atomic | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index c0c21009da4..0b0a60c4a2e 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -1860,7 +1860,7 @@ public: } private: - void _Check_aligment(_Ty& _Value) { + void _Check_aligment([[maybe_unused]] _Ty& _Value) { _ATOMIC_REF_CHECK_ALIGNMENT( (reinterpret_cast(_STD addressof(_Value)) & (required_alignment - 1)) == 0, "atomic_ref underlying object is not aligned as required_alignment"); From 35388d9398b20091f28d21aeb3ccacfc1904f305 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 25 May 2020 19:41:49 +0300 Subject: [PATCH 043/132] Correct _Init_spinlock_for_ref for current ABI --- stl/inc/atomic | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 0b0a60c4a2e..379702593f6 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -1798,7 +1798,13 @@ public: explicit atomic_ref(_Ty& _Value) noexcept /* strengthened */ : _Base(_Value) { if constexpr (_Is_potentially_lock_free) { _Check_aligment(_Value); - } else { + } + +#if 1 // TRANSITION, ABI + if (!is_always_lock_free) { +#else + if constexpr (!_Is_potentially_lock_free) { +#endif this->_Init_spinlock_for_ref(); } } From 0ff9148658fd6b166e99587116e813cf0c8c16ee Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 25 May 2020 19:42:15 +0300 Subject: [PATCH 044/132] constexpr --- stl/inc/atomic | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 379702593f6..74fe5d34ff7 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -1801,7 +1801,7 @@ public: } #if 1 // TRANSITION, ABI - if (!is_always_lock_free) { + if constexpr (!is_always_lock_free) { #else if constexpr (!_Is_potentially_lock_free) { #endif From 549a856e2751a514bdcc96b1c0d01e887f5c2fb4 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 25 May 2020 20:06:52 +0300 Subject: [PATCH 045/132] clang format, const, noexcept --- stl/inc/atomic | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 74fe5d34ff7..5b102234abc 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -416,7 +416,7 @@ struct _Atomic_storage { } protected: - void _Init_spinlock_for_ref() { + void _Init_spinlock_for_ref() noexcept { _Spinlock = __std_atomic_get_mutex(_STD addressof(_Storage)); } @@ -1795,11 +1795,11 @@ public: using value_type = _Ty; - explicit atomic_ref(_Ty& _Value) noexcept /* strengthened */ : _Base(_Value) { + explicit atomic_ref(_Ty& _Value) : _Base(_Value) { if constexpr (_Is_potentially_lock_free) { _Check_aligment(_Value); } - + #if 1 // TRANSITION, ABI if constexpr (!is_always_lock_free) { #else @@ -1866,7 +1866,7 @@ public: } private: - void _Check_aligment([[maybe_unused]] _Ty& _Value) { + void _Check_aligment([[maybe_unused]] const _Ty& _Value) { _ATOMIC_REF_CHECK_ALIGNMENT( (reinterpret_cast(_STD addressof(_Value)) & (required_alignment - 1)) == 0, "atomic_ref underlying object is not aligned as required_alignment"); From cec012a17be513d64b8dc0ebb1fc02ecd5eecf54 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 25 May 2020 22:01:03 +0300 Subject: [PATCH 046/132] whitespace --- stl/inc/atomic | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 5b102234abc..3a8c9d133c5 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -1798,7 +1798,7 @@ public: explicit atomic_ref(_Ty& _Value) : _Base(_Value) { if constexpr (_Is_potentially_lock_free) { _Check_aligment(_Value); - } + } #if 1 // TRANSITION, ABI if constexpr (!is_always_lock_free) { From f9f876ca56c6e131fd5e773a0929aa6a9b9ccbc4 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 26 May 2020 05:50:40 +0300 Subject: [PATCH 047/132] fix incorrect test, satisfy required alignment --- tests/std/tests/P0019R8_atomic_ref/test.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/std/tests/P0019R8_atomic_ref/test.cpp b/tests/std/tests/P0019R8_atomic_ref/test.cpp index 3fb1dd48634..31050ab4303 100644 --- a/tests/std/tests/P0019R8_atomic_ref/test.cpp +++ b/tests/std/tests/P0019R8_atomic_ref/test.cpp @@ -27,8 +27,13 @@ void test_ops() { constexpr std::size_t repetitions = 8000; constexpr std::size_t total = unique * repetitions; constexpr std::size_t range = 10; + + struct alignas(std::atomic_ref::required_alignment) Padded { + ValueType vals[unique] = {}; + } padded; + + auto& vals = padded.vals; - ValueType vals[unique] = {}; std::vector> refs; refs.reserve(total); for (std::size_t i = 0; i != repetitions; ++i) { From fed136c3e41defd5200fea03ce94648fdbf63460 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 26 May 2020 08:22:44 +0300 Subject: [PATCH 048/132] clang format --- tests/std/tests/P0019R8_atomic_ref/test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/std/tests/P0019R8_atomic_ref/test.cpp b/tests/std/tests/P0019R8_atomic_ref/test.cpp index 31050ab4303..14834dd15b2 100644 --- a/tests/std/tests/P0019R8_atomic_ref/test.cpp +++ b/tests/std/tests/P0019R8_atomic_ref/test.cpp @@ -27,11 +27,11 @@ void test_ops() { constexpr std::size_t repetitions = 8000; constexpr std::size_t total = unique * repetitions; constexpr std::size_t range = 10; - + struct alignas(std::atomic_ref::required_alignment) Padded { ValueType vals[unique] = {}; } padded; - + auto& vals = padded.vals; std::vector> refs; From f59ed291f99ee2cdc8cefa04fc47e7d5acae5197 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 9 Jun 2020 14:15:43 +0300 Subject: [PATCH 049/132] fail libcxx test for correct reason --- tests/libcxx/expected_results.txt | 4 +--- tests/libcxx/skipped_tests.txt | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/libcxx/expected_results.txt b/tests/libcxx/expected_results.txt index beea762b739..7609b65e508 100644 --- a/tests/libcxx/expected_results.txt +++ b/tests/libcxx/expected_results.txt @@ -49,6 +49,7 @@ std/containers/unord/unord.map/unord.map.modifiers/insert_and_emplace_allocator_ std/containers/unord/unord.set/insert_and_emplace_allocator_requirements.pass.cpp FAIL # libc++ doesn't yet implement P1423R3, so it expects an old value for `__cpp_lib_char8_t` +std/language.support/support.limits/support.limits.general/atomic.version.pass.cpp FAIL std/language.support/support.limits/support.limits.general/filesystem.version.pass.cpp FAIL std/language.support/support.limits/support.limits.general/istream.version.pass.cpp FAIL std/language.support/support.limits/support.limits.general/limits.version.pass.cpp FAIL @@ -250,9 +251,6 @@ std/utilities/memory/default.allocator/allocator.members/allocate.verify.cpp SKI # *** MISSING STL FEATURES *** -# C++20 P0019R8 "atomic_ref" -std/language.support/support.limits/support.limits.general/atomic.version.pass.cpp FAIL - # C++20 P0355R7 " Calendars And Time Zones" std/utilities/time/days.pass.cpp FAIL std/utilities/time/months.pass.cpp FAIL diff --git a/tests/libcxx/skipped_tests.txt b/tests/libcxx/skipped_tests.txt index 0912d0ae78f..6fd875bd4ca 100644 --- a/tests/libcxx/skipped_tests.txt +++ b/tests/libcxx/skipped_tests.txt @@ -49,6 +49,7 @@ containers\unord\unord.map\unord.map.modifiers\insert_and_emplace_allocator_requ containers\unord\unord.set\insert_and_emplace_allocator_requirements.pass.cpp # libc++ doesn't yet implement P1423R3, so it expects an old value for `__cpp_lib_char8_t` +language.support\support.limits\support.limits.general\atomic.version.pass.cpp language.support\support.limits\support.limits.general\filesystem.version.pass.cpp language.support\support.limits\support.limits.general\istream.version.pass.cpp language.support\support.limits\support.limits.general\limits.version.pass.cpp @@ -250,9 +251,6 @@ utilities\memory\default.allocator\allocator.members\allocate.verify.cpp # *** MISSING STL FEATURES *** -# C++20 P0019R8 "atomic_ref" -language.support\support.limits\support.limits.general\atomic.version.pass.cpp - # C++20 P0355R7 " Calendars And Time Zones" utilities\time\days.pass.cpp utilities\time\months.pass.cpp From bf1af3fd86fdccb427aa1a623e0f141c533fc1c0 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Wed, 8 Jul 2020 10:17:13 +0300 Subject: [PATCH 050/132] line ending fix after merge via GitHub web face --- stl/inc/atomic | 4668 ++++++++++++++++++++++++------------------------ 1 file changed, 2334 insertions(+), 2334 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 5b2093a6251..9b84e079f0e 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -1,2334 +1,2334 @@ -// atomic standard header - -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -#pragma once -#ifndef _ATOMIC_ -#define _ATOMIC_ -#include -#if _STL_COMPILER_PREPROCESSOR - -#ifdef _M_CEE_PURE -#error is not supported when compiling with /clr:pure. -#endif // _M_CEE_PURE - -#include // for size_t -#include -#include -#include - -#pragma pack(push, _CRT_PACKING) -#pragma warning(push, _STL_WARNING_LEVEL) -#pragma warning(disable : _STL_DISABLED_WARNINGS) -_STL_DISABLE_CLANG_WARNINGS -#pragma push_macro("new") -#undef new - -#define _Compiler_barrier() _STL_DISABLE_DEPRECATED_WARNING _ReadWriteBarrier() _STL_RESTORE_DEPRECATED_WARNING - -#if defined(_M_ARM) || defined(_M_ARM64) -#define _Memory_barrier() __dmb(0xB) // inner shared data memory barrier -#define _Compiler_or_memory_barrier() _Memory_barrier() -#elif defined(_M_IX86) || defined(_M_X64) -// x86/x64 hardware only emits memory barriers inside _Interlocked intrinsics -#define _Compiler_or_memory_barrier() _Compiler_barrier() -#else // ^^^ x86/x64 / unsupported hardware vvv -#error Unsupported hardware -#endif // hardware - -#ifndef _INVALID_MEMORY_ORDER -#ifdef _DEBUG -#define _INVALID_MEMORY_ORDER _STL_REPORT_ERROR("Invalid memory order") -#else // ^^^ _DEBUG / !_DEBUG vvv -#define _INVALID_MEMORY_ORDER -#endif // _DEBUG -#endif // _INVALID_MEMORY_ORDER - -#if 0 // TRANSITION, ABI -// MACRO _STD_COMPARE_EXCHANGE_128 -#if _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B || defined(_M_ARM64) -#define _STD_COMPARE_EXCHANGE_128 _InterlockedCompareExchange128 -#endif // _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B || defined(_M_ARM64) -#if defined(_M_X64) && !_STD_ATOMIC_ALWAYS_USE_CMPXCHG16B -// 16-byte atomics are separately compiled for x64, as not all x64 hardware has the cmpxchg16b -// instruction; in the event this instruction is not available, the fallback is a global -// CRITICAL_SECTION shared by all 16-byte atomics. -// (Note: machines without this instruction typically have 2 cores or fewer, so this isn't too bad) -// All pointer parameters must be 16-byte aligned. -_NODISCARD extern "C" unsigned char __cdecl __std_atomic_compare_exchange_128( - _Inout_bytecount_(16) long long* _Destination, _In_ long long _ExchangeHigh, _In_ long long _ExchangeLow, - _Inout_bytecount_(16) long long* _ComparandResult) noexcept; -_NODISCARD extern "C" bool __cdecl __std_atomic_has_cmpxchg16b() noexcept; -#define _STD_COMPARE_EXCHANGE_128 __std_atomic_compare_exchange_128 -#endif // defined(_M_X64) && !_STD_ATOMIC_ALWAYS_USE_CMPXCHG16B - -// MACRO _ATOMIC_HAS_DCAS -// Controls whether atomic::is_always_lock_free triggers for sizeof(void *) or 2 * sizeof(void *) -#if _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B || !defined(_M_X64) -#define _ATOMIC_HAS_DCAS 1 -#else // ^^ We always have DCAS / We only sometimes have DCAS vvv -#define _ATOMIC_HAS_DCAS 0 -#endif // _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B || !defined(_M_X64) -#endif // TRANSITION, ABI - -// MACRO _ATOMIC_CHOOSE_INTRINSIC -#if defined(_M_IX86) || defined(_M_X64) -#define _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _Intrinsic, ...) \ - _Check_memory_order(_Order); \ - _Result = _Intrinsic(__VA_ARGS__) -#elif defined(_M_ARM) || defined(_M_ARM64) -#define _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _Intrinsic, ...) \ - switch (_Order) { \ - case memory_order_relaxed: \ - _Result = _INTRIN_RELAXED(_Intrinsic)(__VA_ARGS__); \ - break; \ - case memory_order_consume: \ - case memory_order_acquire: \ - _Result = _INTRIN_ACQUIRE(_Intrinsic)(__VA_ARGS__); \ - break; \ - case memory_order_release: \ - _Result = _INTRIN_RELEASE(_Intrinsic)(__VA_ARGS__); \ - break; \ - default: \ - _INVALID_MEMORY_ORDER; \ - /* [[fallthrough]]; */ \ - case memory_order_acq_rel: \ - case memory_order_seq_cst: \ - _Result = _Intrinsic(__VA_ARGS__); \ - break; \ - } -#endif // hardware - -// LOCK-FREE PROPERTY -#define ATOMIC_BOOL_LOCK_FREE 2 -#define ATOMIC_CHAR_LOCK_FREE 2 -#ifdef __cpp_lib_char8_t -#define ATOMIC_CHAR8_T_LOCK_FREE 2 -#endif // __cpp_lib_char8_t -#define ATOMIC_CHAR16_T_LOCK_FREE 2 -#define ATOMIC_CHAR32_T_LOCK_FREE 2 -#define ATOMIC_WCHAR_T_LOCK_FREE 2 -#define ATOMIC_SHORT_LOCK_FREE 2 -#define ATOMIC_INT_LOCK_FREE 2 -#define ATOMIC_LONG_LOCK_FREE 2 -#define ATOMIC_LLONG_LOCK_FREE 2 -#define ATOMIC_POINTER_LOCK_FREE 2 - -_EXTERN_C -long* __stdcall __std_atomic_get_mutex(const void* _Key) noexcept; -_END_EXTERN_C - -_STD_BEGIN - -// FENCES -extern "C" inline void atomic_thread_fence(const memory_order _Order) noexcept { - if (_Order == memory_order_relaxed) { - return; - } - -#if defined(_M_IX86) || defined(_M_X64) - _Compiler_barrier(); - if (_Order == memory_order_seq_cst) { - volatile long _Guard; // Not initialized to avoid an unnecessary operation; the value does not matter - - // _mm_mfence could have been used, but it is not supported on older x86 CPUs and is slower on some recent CPUs. - // The memory fence provided by interlocked operations has some exceptions, but this is fine: - // std::atomic_thread_fence works with respect to other atomics only; it may not be a full fence for all ops. -#pragma warning(suppress : 6001) // "Using uninitialized memory '_Guard'" -#pragma warning(suppress : 28113) // "Accessing a local variable _Guard via an Interlocked function: This is an unusual - // usage which could be reconsidered." - (void) _InterlockedIncrement(&_Guard); - _Compiler_barrier(); - } -#elif defined(_M_ARM) || defined(_M_ARM64) - _Memory_barrier(); -#else // ^^^ ARM32/ARM64 / unsupported hardware vvv -#error Unsupported hardware -#endif // unsupported hardware -} - -extern "C" inline void atomic_signal_fence(const memory_order _Order) noexcept { - if (_Order != memory_order_relaxed) { - _Compiler_barrier(); - } -} - -// FUNCTION TEMPLATE kill_dependency -template -_Ty kill_dependency(_Ty _Arg) noexcept { // "magic" template that kills dependency ordering when called - return _Arg; -} - -// FUNCTION _Check_memory_order -inline void _Check_memory_order(const memory_order _Order) noexcept { - // check that _Order is a valid memory_order - if (static_cast(_Order) > static_cast(memory_order_seq_cst)) { - _INVALID_MEMORY_ORDER; - } -} - -// FUNCTION _Check_store_memory_order -inline void _Check_store_memory_order(const memory_order _Order) noexcept { - switch (_Order) { - case memory_order_relaxed: - case memory_order_release: - case memory_order_seq_cst: - // nothing to do - break; - case memory_order_consume: - case memory_order_acquire: - case memory_order_acq_rel: - default: - _INVALID_MEMORY_ORDER; - break; - } -} - -// FUNCTION _Check_load_memory_order -inline void _Check_load_memory_order(const memory_order _Order) noexcept { - switch (_Order) { - case memory_order_relaxed: - case memory_order_consume: - case memory_order_acquire: - case memory_order_seq_cst: - // nothing to do - break; - case memory_order_release: - case memory_order_acq_rel: - default: - _INVALID_MEMORY_ORDER; - break; - } -} - -// FUNCTION _Combine_cas_memory_orders -_NODISCARD inline memory_order _Combine_cas_memory_orders( - const memory_order _Success, const memory_order _Failure) noexcept { - // Finds upper bound of a compare/exchange memory order - // pair, according to the following partial order: - // seq_cst - // | - // acq_rel - // / \ - // acquire release - // | | - // consume | - // \ / - // relaxed - static constexpr memory_order _Combined_memory_orders[6][6] = {// combined upper bounds - {memory_order_relaxed, memory_order_consume, memory_order_acquire, memory_order_release, memory_order_acq_rel, - memory_order_seq_cst}, - {memory_order_consume, memory_order_consume, memory_order_acquire, memory_order_acq_rel, memory_order_acq_rel, - memory_order_seq_cst}, - {memory_order_acquire, memory_order_acquire, memory_order_acquire, memory_order_acq_rel, memory_order_acq_rel, - memory_order_seq_cst}, - {memory_order_release, memory_order_acq_rel, memory_order_acq_rel, memory_order_release, memory_order_acq_rel, - memory_order_seq_cst}, - {memory_order_acq_rel, memory_order_acq_rel, memory_order_acq_rel, memory_order_acq_rel, memory_order_acq_rel, - memory_order_seq_cst}, - {memory_order_seq_cst, memory_order_seq_cst, memory_order_seq_cst, memory_order_seq_cst, memory_order_seq_cst, - memory_order_seq_cst}}; - - _Check_memory_order(_Success); - _Check_load_memory_order(_Failure); - return _Combined_memory_orders[static_cast(_Success)][static_cast(_Failure)]; -} - -// FUNCTION TEMPLATE _Atomic_reinterpret_as -template -_NODISCARD _Integral _Atomic_reinterpret_as(const _Ty& _Source) noexcept { - // interprets _Source as the supplied integral type - static_assert(is_integral_v<_Integral>, "Tried to reinterpret memory as non-integral"); -#if _HAS_IF_CONSTEXPR - if constexpr (is_integral_v<_Ty> && sizeof(_Integral) == sizeof(_Ty)) { - return static_cast<_Integral>(_Source); - } else if constexpr (is_pointer_v<_Ty> && sizeof(_Integral) == sizeof(_Ty)) { - return reinterpret_cast<_Integral>(_Source); - } else -#endif // _HAS_IF_CONSTEXPR - { - _Integral _Result{}; // zero padding bits - _CSTD memcpy(&_Result, _STD addressof(_Source), sizeof(_Source)); - return _Result; - } -} - -// FUNCTION _Load_barrier -inline void _Load_barrier(const memory_order _Order) noexcept { // implement memory barrier for atomic load functions - switch (_Order) { - case memory_order_relaxed: - // no barrier - break; - default: - case memory_order_release: - case memory_order_acq_rel: - _INVALID_MEMORY_ORDER; - // [[fallthrough]]; - case memory_order_consume: - case memory_order_acquire: - case memory_order_seq_cst: - _Compiler_or_memory_barrier(); - break; - } -} - -#if 1 // TRANSITION, ABI -template -struct _Atomic_padded { - alignas(sizeof(_Ty)) mutable _Ty _Value; // align to sizeof(T); x86 stack aligns 8-byte objects on 4-byte boundaries -}; - -#else // ^^^ don't break ABI / break ABI vvv -// STRUCT TEMPLATE _Atomic_storage_traits -template -struct _Atomic_storage_traits { // properties for how _Ty is stored in an atomic - static constexpr size_t _Storage_size = - sizeof(_Ty) == 1 ? 1 - : sizeof(_Ty) == 2 ? 2 - : sizeof(_Ty) <= 4 ? 4 - : sizeof(_Ty) <= 8 ? 8 -#if defined(_M_X64) || defined(_M_ARM64) - : sizeof(_Ty) <= 16 ? 16 -#endif // 64 bits - : sizeof(_Ty); - - static constexpr size_t _Padding_size = _Storage_size - sizeof(_Ty); - static constexpr bool _Uses_padding = _Padding_size != 0; -}; - -template -struct _Atomic_storage_traits<_Ty&> { // properties for how _Ty is stored in an atomic_ref - static constexpr size_t _Storage_size = sizeof(_Ty); - static constexpr bool _Uses_padding = false; -} - -// STRUCT TEMPLATE _Atomic_padded -template ::_Uses_padding> -struct _Atomic_padded { // aggregate to allow explicit constexpr zeroing of padding - alignas(_Atomic_storage_traits<_Ty>::_Storage_size) mutable _Ty _Value; - mutable unsigned char _Padding[_Atomic_storage_traits<_Ty>::_Padding_size]; -}; - -template -struct _Atomic_padded<_Ty, false> { - alignas(sizeof(_Ty)) mutable _Ty _Value; // align to sizeof(T); x86 stack aligns 8-byte objects on 4-byte boundaries -}; - -template -struct _Atomic_padded<_Ty&, false> { - _Ty& _Value; -}; - -#endif // TRANSITION, ABI - -template -struct _Atomic_storage_types { - using _TVal = _Ty; - using _TStorage = _Atomic_padded<_Ty>; - using _TConstr = const _Ty; - using _Spinlock = long; - - static constexpr bool _Has_volatile = true; - - static long* _Get_spinlock(long& _Spinlock) { - return &_Spinlock; - } -}; - -template -struct _Atomic_storage_types<_Ty&> { - using _TVal = _Ty; - using _TStorage = _Ty&; - using _TConstr = _Ty&; - using _Spinlock = long*; - - static constexpr bool _Has_volatile = false; - - static long* _Get_spinlock(long* _Spinlock) { - return _Spinlock; - } -}; - -// STRUCT TEMPLATE _Atomic_storage -#if 1 // TRANSITION, ABI -template ::_TVal)> -#else // ^^^ don't break ABI / break ABI vvv -template ::_Storage_size> -#endif // TRANSITION, ABI -struct _Atomic_storage { - // Provides operations common to all specializations of std::atomic, load, store, exchange, and CAS. - // Locking version used when hardware has no atomic operations for sizeof(_Ty). - - using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; - - _Atomic_storage() = default; - - /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept - : _Storage(_Value) { - // non-atomically initialize this atomic - } - - void store(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { - // store with sequential consistency - _Check_store_memory_order(_Order); - _Lock(); - _Storage = _Value; - _Unlock(); - } - - _NODISCARD _TVal load(const memory_order _Order = memory_order_seq_cst) const noexcept { - // load with sequential consistency - _Check_load_memory_order(_Order); - _Lock(); - _TVal _Local(_Storage); - _Unlock(); - return _Local; - } - - _TVal exchange(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { - // exchange _Value with _Storage with sequential consistency - _Check_memory_order(_Order); - _Lock(); - _TVal _Result(_Storage); - _Storage = _Value; - _Unlock(); - return _Result; - } - - bool compare_exchange_strong(_TVal& _Expected, const _TVal _Desired, - const memory_order _Order = memory_order_seq_cst) noexcept { // CAS with sequential consistency, plain - _Check_memory_order(_Order); - const auto _Storage_ptr = _STD addressof(_Storage); - const auto _Expected_ptr = _STD addressof(_Expected); - bool _Result; - _Lock(); - if (_CSTD memcmp(_Storage_ptr, _Expected_ptr, sizeof(_TVal)) == 0) { - _CSTD memcpy(_Storage_ptr, _STD addressof(_Desired), sizeof(_TVal)); - _Result = true; - } else { - _CSTD memcpy(_Expected_ptr, _Storage_ptr, sizeof(_TVal)); - _Result = false; - } - - _Unlock(); - return _Result; - } - -#if 1 // TRANSITION, ABI - void _Lock() const noexcept { // lock the spinlock - while (_InterlockedExchange(_Atomic_storage_types<_Ty>::_Get_spinlock(_Spinlock), 1)) { - _YIELD_PROCESSOR(); - } - } - - void _Unlock() const noexcept { // unlock the spinlock -#if defined(_M_ARM) || defined(_M_ARM64) - _Memory_barrier(); - __iso_volatile_store32(reinterpret_cast(_Atomic_storage_types<_Ty>::_Get_spinlock(_Spinlock)), 0); - _Memory_barrier(); -#else // ^^^ ARM32/ARM64 hardware / x86/x64 hardware vvv - _InterlockedExchange(_Atomic_storage_types<_Ty>::_Get_spinlock(_Spinlock), 0); -#endif // hardware - } - -protected: - void _Init_spinlock_for_ref() noexcept { - _Spinlock = __std_atomic_get_mutex(_STD addressof(_Storage)); - } - -private: - // Spinlock for non-lock-free atomic. Spinlock pointer for non-lock-free atomic_ref - mutable typename _Atomic_storage_types<_Ty>::_Spinlock _Spinlock = 0; - -public: - _Ty _Storage{}; - -#else // ^^^ don't break ABI / break ABI vvv - void _Lock() const noexcept { // lock the spinlock - while (_InterlockedExchange8(&_Spinlock, 1)) { - _YIELD_PROCESSOR(); - } - } - - void _Unlock() const noexcept { // unlock the spinlock -#if defined(_M_ARM) || defined(_M_ARM64) - _Memory_barrier(); - __iso_volatile_store8(&_Spinlock, 0); - _Memory_barrier(); -#else // ^^^ ARM32/ARM64 hardware / x86/x64 hardware vvv - _InterlockedExchange8(&_Spinlock, 0); -#endif // hardware - } - - _Ty _Storage; - // Revisit, could be removed completely for atomic_ref - mutable char _Spinlock = 0; -#endif // TRANSITION, ABI -}; - -template -struct _Atomic_storage<_Ty, 1> { // lock-free using 1-byte intrinsics - - using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; - - _Atomic_storage() = default; - - /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept - : _Storage{_Value} { - // non-atomically initialize this atomic - } - - void store(const _TVal _Value) noexcept { // store with sequential consistency - const auto _Mem = _Atomic_address_as(_Storage); - const char _As_bytes = _Atomic_reinterpret_as(_Value); -#if defined(_M_ARM) || defined(_M_ARM64) - _Memory_barrier(); - __iso_volatile_store8(_Mem, _As_bytes); - _Memory_barrier(); -#else // ^^^ ARM32/ARM64 hardware / x86/x64 hardware vvv - (void) _InterlockedExchange8(_Mem, _As_bytes); -#endif // hardware - } - - void store(const _TVal _Value, const memory_order _Order) noexcept { // store with given memory order - const auto _Mem = _Atomic_address_as(_Storage); - const char _As_bytes = _Atomic_reinterpret_as(_Value); - switch (_Order) { - case memory_order_relaxed: - __iso_volatile_store8(_Mem, _As_bytes); - return; - case memory_order_release: - _Compiler_or_memory_barrier(); - __iso_volatile_store8(_Mem, _As_bytes); - return; - default: - case memory_order_consume: - case memory_order_acquire: - case memory_order_acq_rel: - _INVALID_MEMORY_ORDER; - // [[fallthrough]]; - case memory_order_seq_cst: - store(_Value); - return; - } - } - - _NODISCARD _TVal load() const noexcept { // load with sequential consistency - const auto _Mem = _Atomic_address_as(_Storage); - char _As_bytes = __iso_volatile_load8(_Mem); - _Compiler_or_memory_barrier(); - return reinterpret_cast<_TVal&>(_As_bytes); - } - - _NODISCARD _TVal load(const memory_order _Order) const noexcept { // load with given memory order - const auto _Mem = _Atomic_address_as(_Storage); - char _As_bytes = __iso_volatile_load8(_Mem); - _Load_barrier(_Order); - return reinterpret_cast<_TVal&>(_As_bytes); - } - - _TVal exchange(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { - // exchange with given memory order - char _As_bytes; - _ATOMIC_CHOOSE_INTRINSIC(_Order, _As_bytes, _InterlockedExchange8, _Atomic_address_as(_Storage), - _Atomic_reinterpret_as(_Value)); - return reinterpret_cast<_Ty&>(_As_bytes); - } - - bool compare_exchange_strong(_TVal& _Expected, const _TVal _Desired, - const memory_order _Order = memory_order_seq_cst) noexcept { // CAS with given memory order - const char _Expected_bytes = _Atomic_reinterpret_as(_Expected); // read before atomic operation - char _Prev_bytes; - _ATOMIC_CHOOSE_INTRINSIC(_Order, _Prev_bytes, _InterlockedCompareExchange8, _Atomic_address_as(_Storage), - _Atomic_reinterpret_as(_Desired), _Expected_bytes); - if (_Prev_bytes == _Expected_bytes) { - return true; - } - - reinterpret_cast(_Expected) = _Prev_bytes; - return false; - } - - typename _Atomic_storage_types<_Ty>::_TStorage _Storage; -}; - -template -struct _Atomic_storage<_Ty, 2> { // lock-free using 2-byte intrinsics - - using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; - - _Atomic_storage() = default; - - /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept - : _Storage{_Value} { - // non-atomically initialize this atomic - } - - void store(const _TVal _Value) noexcept { // store with sequential consistency - const auto _Mem = _Atomic_address_as(_Storage); - const short _As_bytes = _Atomic_reinterpret_as(_Value); -#if defined(_M_ARM) || defined(_M_ARM64) - _Memory_barrier(); - __iso_volatile_store16(_Mem, _As_bytes); - _Memory_barrier(); -#else // ^^^ ARM32/ARM64 hardware / x86/x64 hardware vvv - (void) _InterlockedExchange16(_Mem, _As_bytes); -#endif // hardware - } - - void store(const _TVal _Value, const memory_order _Order) noexcept { // store with given memory order - const auto _Mem = _Atomic_address_as(_Storage); - const short _As_bytes = _Atomic_reinterpret_as(_Value); - switch (_Order) { - case memory_order_relaxed: - __iso_volatile_store16(_Mem, _As_bytes); - return; - case memory_order_release: - _Compiler_or_memory_barrier(); - __iso_volatile_store16(_Mem, _As_bytes); - return; - default: - case memory_order_consume: - case memory_order_acquire: - case memory_order_acq_rel: - _INVALID_MEMORY_ORDER; - // [[fallthrough]]; - case memory_order_seq_cst: - store(_Value); - return; - } - } - - _NODISCARD _TVal load() const noexcept { // load with sequential consistency - const auto _Mem = _Atomic_address_as(_Storage); - short _As_bytes = __iso_volatile_load16(_Mem); - _Compiler_or_memory_barrier(); - return reinterpret_cast<_TVal&>(_As_bytes); - } - - _NODISCARD _TVal load(const memory_order _Order) const noexcept { // load with given memory order - const auto _Mem = _Atomic_address_as(_Storage); - short _As_bytes = __iso_volatile_load16(_Mem); - _Load_barrier(_Order); - return reinterpret_cast<_TVal&>(_As_bytes); - } - - _TVal exchange(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { - // exchange with given memory order - short _As_bytes; - _ATOMIC_CHOOSE_INTRINSIC(_Order, _As_bytes, _InterlockedExchange16, _Atomic_address_as(_Storage), - _Atomic_reinterpret_as(_Value)); - return reinterpret_cast<_TVal&>(_As_bytes); - } - - bool compare_exchange_strong(_TVal& _Expected, const _TVal _Desired, - const memory_order _Order = memory_order_seq_cst) noexcept { // CAS with given memory order - const short _Expected_bytes = _Atomic_reinterpret_as(_Expected); // read before atomic operation - short _Prev_bytes; - _ATOMIC_CHOOSE_INTRINSIC(_Order, _Prev_bytes, _InterlockedCompareExchange16, - _Atomic_address_as(_Storage), _Atomic_reinterpret_as(_Desired), _Expected_bytes); - if (_Prev_bytes == _Expected_bytes) { - return true; - } - - _CSTD memcpy(_STD addressof(_Expected), &_Prev_bytes, sizeof(_Ty)); - return false; - } - - typename _Atomic_storage_types<_Ty>::_TStorage _Storage; -}; - -template -struct _Atomic_storage<_Ty, 4> { // lock-free using 4-byte intrinsics - - using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; - - _Atomic_storage() = default; - - /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept - : _Storage{_Value} { - // non-atomically initialize this atomic - } - - void store(const _TVal _Value) noexcept { // store with sequential consistency -#if defined(_M_ARM) || defined(_M_ARM64) - _Memory_barrier(); - __iso_volatile_store32(_Atomic_address_as(_Storage), _Atomic_reinterpret_as(_Value)); - _Memory_barrier(); -#else // ^^^ ARM32/ARM64 hardware / x86/x64 hardware vvv - (void) _InterlockedExchange(_Atomic_address_as(_Storage), _Atomic_reinterpret_as(_Value)); -#endif // hardware - } - - void store(const _TVal _Value, const memory_order _Order) noexcept { // store with given memory order - const auto _Mem = _Atomic_address_as(_Storage); - const int _As_bytes = _Atomic_reinterpret_as(_Value); - switch (_Order) { - case memory_order_relaxed: - __iso_volatile_store32(_Mem, _As_bytes); - return; - case memory_order_release: - _Compiler_or_memory_barrier(); - __iso_volatile_store32(_Mem, _As_bytes); - return; - default: - case memory_order_consume: - case memory_order_acquire: - case memory_order_acq_rel: - _INVALID_MEMORY_ORDER; - // [[fallthrough]]; - case memory_order_seq_cst: - store(_Value); - return; - } - } - - _NODISCARD _TVal load() const noexcept { // load with sequential consistency - const auto _Mem = _Atomic_address_as(_Storage); - auto _As_bytes = __iso_volatile_load32(_Mem); - _Compiler_or_memory_barrier(); - return reinterpret_cast<_TVal&>(_As_bytes); - } - - _NODISCARD _TVal load(const memory_order _Order) const noexcept { // load with given memory order - const auto _Mem = _Atomic_address_as(_Storage); - auto _As_bytes = __iso_volatile_load32(_Mem); - _Load_barrier(_Order); - return reinterpret_cast<_TVal&>(_As_bytes); - } - - _TVal exchange(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { - // exchange with given memory order - long _As_bytes; - _ATOMIC_CHOOSE_INTRINSIC(_Order, _As_bytes, _InterlockedExchange, _Atomic_address_as(_Storage), - _Atomic_reinterpret_as(_Value)); - return reinterpret_cast<_TVal&>(_As_bytes); - } - - bool compare_exchange_strong(_TVal& _Expected, const _TVal _Desired, - const memory_order _Order = memory_order_seq_cst) noexcept { // CAS with given memory order - const long _Expected_bytes = _Atomic_reinterpret_as(_Expected); // read before atomic operation - long _Prev_bytes; - _ATOMIC_CHOOSE_INTRINSIC(_Order, _Prev_bytes, _InterlockedCompareExchange, _Atomic_address_as(_Storage), - _Atomic_reinterpret_as(_Desired), _Expected_bytes); - if (_Prev_bytes == _Expected_bytes) { - return true; - } - - _CSTD memcpy(_STD addressof(_Expected), &_Prev_bytes, sizeof(_TVal)); - return false; - } - - typename _Atomic_storage_types<_Ty>::_TStorage _Storage; -}; - -template -struct _Atomic_storage<_Ty, 8> { // lock-free using 8-byte intrinsics - - using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; - - _Atomic_storage() = default; - - /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept - : _Storage{_Value} { - // non-atomically initialize this atomic - } - - void store(const _TVal _Value) noexcept { // store with sequential consistency - const auto _Mem = _Atomic_address_as(_Storage); - const long long _As_bytes = _Atomic_reinterpret_as(_Value); -#if defined(_M_IX86) - _Compiler_barrier(); - __iso_volatile_store64(_Mem, _As_bytes); - _STD atomic_thread_fence(memory_order_seq_cst); -#elif defined(_M_ARM64) - _Memory_barrier(); - __iso_volatile_store64(_Mem, _As_bytes); - _Memory_barrier(); -#else // ^^^ _M_ARM64 / ARM32, x64 vvv - (void) _InterlockedExchange64(_Mem, _As_bytes); -#endif // _M_ARM64 - } - - void store(const _TVal _Value, const memory_order _Order) noexcept { // store with given memory order - const auto _Mem = _Atomic_address_as(_Storage); - const long long _As_bytes = _Atomic_reinterpret_as(_Value); - switch (_Order) { - case memory_order_relaxed: - __iso_volatile_store64(_Mem, _As_bytes); - return; - case memory_order_release: - _Compiler_or_memory_barrier(); - __iso_volatile_store64(_Mem, _As_bytes); - return; - default: - case memory_order_consume: - case memory_order_acquire: - case memory_order_acq_rel: - _INVALID_MEMORY_ORDER; - // [[fallthrough]]; - case memory_order_seq_cst: - store(_Value); - return; - } - } - - _NODISCARD _TVal load() const noexcept { // load with sequential consistency - const auto _Mem = _Atomic_address_as(_Storage); - long long _As_bytes; -#ifdef _M_ARM - _As_bytes = __ldrexd(_Mem); - _Memory_barrier(); -#else - _As_bytes = __iso_volatile_load64(_Mem); - _Compiler_or_memory_barrier(); -#endif - return reinterpret_cast<_TVal&>(_As_bytes); - } - - _NODISCARD _TVal load(const memory_order _Order) const noexcept { // load with given memory order - const auto _Mem = _Atomic_address_as(_Storage); -#ifdef _M_ARM - long long _As_bytes = __ldrexd(_Mem); -#else - long long _As_bytes = __iso_volatile_load64(_Mem); -#endif - _Load_barrier(_Order); - return reinterpret_cast<_TVal&>(_As_bytes); - } - -#if defined(_M_IX86) && defined(__clang__) // TRANSITION, LLVM-46595 - TVal exchange(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { - // exchange with (effectively) sequential consistency - _TVal _Temp{load()}; - while (!compare_exchange_strong(_Temp, _Value, _Order)) { // keep trying - } - - return _Temp; - } -#else // ^^^ defined(_M_IX86) && defined(__clang__), LLVM-46595 / !defined(_M_IX86) || !defined(__clang__) vvv - _TVal exchange(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { - // exchange with given memory order - long long _As_bytes; - _ATOMIC_CHOOSE_INTRINSIC(_Order, _As_bytes, _InterlockedExchange64, _Atomic_address_as(_Storage), - _Atomic_reinterpret_as(_Value)); - return reinterpret_cast<_TVal&>(_As_bytes); - } -#endif // ^^^ !defined(_M_IX86) || !defined(__clang__) ^^^ - - bool compare_exchange_strong(_TVal& _Expected, const _TVal _Desired, - const memory_order _Order = memory_order_seq_cst) noexcept { // CAS with given memory order - const long long _Expected_bytes = _Atomic_reinterpret_as(_Expected); // read before atomic operation - long long _Prev_bytes; - _ATOMIC_CHOOSE_INTRINSIC(_Order, _Prev_bytes, _InterlockedCompareExchange64, - _Atomic_address_as(_Storage), _Atomic_reinterpret_as(_Desired), _Expected_bytes); - if (_Prev_bytes == _Expected_bytes) { - return true; - } - - _CSTD memcpy(_STD addressof(_Expected), &_Prev_bytes, sizeof(_TVal)); - return false; - } - - typename _Atomic_storage_types<_Ty>::_TStorage _Storage; -}; - -#if 0 // TRANSITION, ABI -#if defined(_M_X64) || defined(_M_ARM64) -template -struct _Atomic_storage<_Ty, 16> { // lock-free using 16-byte intrinsics - - using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; - - _Atomic_storage() = default; - - /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept - : _Storage{_Value} {} // non-atomically initialize this atomic - - void store(const _TVal _Value) noexcept { // store with sequential consistency - (void) exchange(_Value); - } - - void store(const _TVal _Value, const memory_order _Order) noexcept { // store with given memory order - _Check_store_memory_order(_Order); - (void) exchange(_Value, _Order); - } - - _NODISCARD _TVal load() const noexcept { // load with sequential consistency - long long* const _Storage_ptr = const_cast(_Atomic_address_as(_Storage)); - _Int128 _Result{}; // atomic CAS 0 with 0 - (void) _STD_COMPARE_EXCHANGE_128(_Storage_ptr, 0, 0, &_Result._Low); - return reinterpret_cast<_TVal&>(_Result); - } - - _NODISCARD _TVal load(const memory_order _Order) const noexcept { // load with given memory order -#ifdef _M_ARM64 - long long* const _Storage_ptr = const_cast(_Atomic_address_as(_Storage)); - _Int128 _Result{}; // atomic CAS 0 with 0 - switch (_Order) { - case memory_order_relaxed: - (void) _INTRIN_RELAXED(_InterlockedCompareExchange128)(_Storage_ptr, 0, 0, &_Result._Low); - break; - case memory_order_consume: - case memory_order_acquire: - (void) _INTRIN_ACQUIRE(_InterlockedCompareExchange128)(_Storage_ptr, 0, 0, &_Result._Low); - break; - default: - case memory_order_release: - case memory_order_acq_rel: - _INVALID_MEMORY_ORDER; - // [[fallthrough]]; - case memory_order_seq_cst: - (void) _InterlockedCompareExchange128(_Storage_ptr, 0, 0, &_Result._Low); - break; - } - - return reinterpret_cast<_TVal&>(_Result); -#else // ^^^ _M_ARM64 / _M_X64 vvv - _Check_load_memory_order(_Order); - return load(); -#endif // _M_ARM64 - } - - _TVal exchange(const _TVal _Value) noexcept { // exchange with sequential consistency - _TVal _Result{_Value}; - while (!compare_exchange_strong(_Result, _Value)) { // keep trying - } - - return _Result; - } - - _TVal exchange(const _TVal _Value, const memory_order _Order) noexcept { // exchange with given memory order - _TVal _Result{_Value}; - while (!compare_exchange_strong(_Result, _Value, _Order)) { // keep trying - } - - return _Result; - } - - bool compare_exchange_strong(_TVal& _Expected, const _TVal _Desired, - const memory_order _Order = memory_order_seq_cst) noexcept { // CAS with given memory order - _Int128 _Desired_bytes{}; - _CSTD memcpy(&_Desired_bytes, _STD addressof(_Desired), sizeof(_TVal)); - _Int128 _Expected_temp{}; - _CSTD memcpy(&_Expected_temp, _STD addressof(_Expected), sizeof(_TVal)); - unsigned char _Result; -#ifdef _M_ARM64 - _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedCompareExchange128, - _Atomic_address_as(_Storage), _Desired_bytes._High, _Desired_bytes._Low, &_Expected_temp._Low); -#else // ^^^ _M_ARM64 / _M_X64 vvv - (void) _Order; - _Result = _STD_COMPARE_EXCHANGE_128( - &reinterpret_cast(_Storage), _Desired_bytes._High, _Desired_bytes._Low, &_Expected_temp._Low); -#endif // _M_ARM64 - if (_Result == 0) { - _CSTD memcpy(_STD addressof(_Expected), &_Expected_temp, sizeof(_TVal)); - } - - return _Result != 0; - } - - struct _Int128 { - alignas(16) long long _Low; - long long _High; - }; - - typename _Atomic_storage_types<_Ty>::_TStorage _Storage; -}; -#endif // defined(_M_X64) || defined(_M_ARM64) -#endif // TRANSITION, ABI - -// STRUCT TEMPLATE _Atomic_integral -template -struct _Atomic_integral; // not defined - -template -struct _Atomic_integral<_Ty, 1> : _Atomic_storage<_Ty> { // atomic integral operations using 1-byte intrinsics - using _Base = _Atomic_storage<_Ty>; - using _TVal = typename _Base::_TVal; - -#ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 - _Atomic_integral() = default; - /* implicit */ constexpr _Atomic_integral(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept - : _Base(_Value) {} -#else // ^^^ workaround / no workaround vvv - using _Base::_Base; -#endif // ^^^ no workaround ^^^ - - _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - char _Result; - _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedExchangeAdd8, _Atomic_address_as(this->_Storage), - static_cast(_Operand)); - return static_cast<_TVal>(_Result); - } - - _TVal fetch_and(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - char _Result; - _ATOMIC_CHOOSE_INTRINSIC( - _Order, _Result, _InterlockedAnd8, _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_TVal>(_Result); - } - - _TVal fetch_or(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - char _Result; - _ATOMIC_CHOOSE_INTRINSIC( - _Order, _Result, _InterlockedOr8, _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_TVal>(_Result); - } - - _TVal fetch_xor(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - char _Result; - _ATOMIC_CHOOSE_INTRINSIC( - _Order, _Result, _InterlockedXor8, _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_TVal>(_Result); - } - - _TVal operator++(int) noexcept { - return static_cast<_TVal>(_InterlockedExchangeAdd8(_Atomic_address_as(this->_Storage), 1)); - } - - _TVal operator++() noexcept { - unsigned char _Before = - static_cast(_InterlockedExchangeAdd8(_Atomic_address_as(this->_Storage), 1)); - ++_Before; - return static_cast<_TVal>(_Before); - } - - _TVal operator--(int) noexcept { - return static_cast<_Ty>(_InterlockedExchangeAdd8(_Atomic_address_as(this->_Storage), -1)); - } - - _TVal operator--() noexcept { - unsigned char _Before = - static_cast(_InterlockedExchangeAdd8(_Atomic_address_as(this->_Storage), -1)); - --_Before; - return static_cast<_TVal>(_Before); - } -}; - -template -struct _Atomic_integral<_Ty, 2> : _Atomic_storage<_Ty> { // atomic integral operations using 2-byte intrinsics - using _Base = _Atomic_storage<_Ty>; - using _TVal = typename _Base::_TVal; - -#ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 - _Atomic_integral() = default; - /* implicit */ constexpr _Atomic_integral(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept - : _Base(_Value) {} -#else // ^^^ workaround / no workaround vvv - using _Base::_Base; -#endif // ^^^ no workaround ^^^ - - _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - short _Result; - _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedExchangeAdd16, _Atomic_address_as(this->_Storage), - static_cast(_Operand)); - return static_cast<_TVal>(_Result); - } - - _TVal fetch_and(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - short _Result; - _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedAnd16, _Atomic_address_as(this->_Storage), - static_cast(_Operand)); - return static_cast<_TVal>(_Result); - } - - _TVal fetch_or(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - short _Result; - _ATOMIC_CHOOSE_INTRINSIC( - _Order, _Result, _InterlockedOr16, _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_TVal>(_Result); - } - - _TVal fetch_xor(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - short _Result; - _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedXor16, _Atomic_address_as(this->_Storage), - static_cast(_Operand)); - return static_cast<_TVal>(_Result); - } - - _TVal operator++(int) noexcept { - unsigned short _After = - static_cast(_InterlockedIncrement16(_Atomic_address_as(this->_Storage))); - --_After; - return static_cast<_TVal>(_After); - } - - _TVal operator++() noexcept { - return static_cast<_TVal>(_InterlockedIncrement16(_Atomic_address_as(this->_Storage))); - } - - _TVal operator--(int) noexcept { - unsigned short _After = - static_cast(_InterlockedDecrement16(_Atomic_address_as(this->_Storage))); - ++_After; - return static_cast<_TVal>(_After); - } - - _TVal operator--() noexcept { - return static_cast<_TVal>(_InterlockedDecrement16(_Atomic_address_as(this->_Storage))); - } -}; - -template -struct _Atomic_integral<_Ty, 4> : _Atomic_storage<_Ty> { // atomic integral operations using 4-byte intrinsics - using _Base = _Atomic_storage<_Ty>; - using _TVal = typename _Base::_TVal; - -#ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 - _Atomic_integral() = default; - /* implicit */ constexpr _Atomic_integral(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept - : _Base(_Value) {} -#else // ^^^ workaround / no workaround vvv - using _Base::_Base; -#endif // ^^^ no workaround ^^^ - - _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - long _Result; - _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedExchangeAdd, _Atomic_address_as(this->_Storage), - static_cast(_Operand)); - return static_cast<_TVal>(_Result); - } - - _TVal fetch_and(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - long _Result; - _ATOMIC_CHOOSE_INTRINSIC( - _Order, _Result, _InterlockedAnd, _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_TVal>(_Result); - } - - _TVal fetch_or(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - long _Result; - _ATOMIC_CHOOSE_INTRINSIC( - _Order, _Result, _InterlockedOr, _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_TVal>(_Result); - } - - _TVal fetch_xor(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - long _Result; - _ATOMIC_CHOOSE_INTRINSIC( - _Order, _Result, _InterlockedXor, _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_TVal>(_Result); - } - - _TVal operator++(int) noexcept { - unsigned long _After = - static_cast(_InterlockedIncrement(_Atomic_address_as(this->_Storage))); - --_After; - return static_cast<_TVal>(_After); - } - - _TVal operator++() noexcept { - return static_cast<_TVal>(_InterlockedIncrement(_Atomic_address_as(this->_Storage))); - } - - _TVal operator--(int) noexcept { - unsigned long _After = - static_cast(_InterlockedDecrement(_Atomic_address_as(this->_Storage))); - ++_After; - return static_cast<_TVal>(_After); - } - - _TVal operator--() noexcept { - return static_cast<_TVal>(_InterlockedDecrement(_Atomic_address_as(this->_Storage))); - } -}; - -template -struct _Atomic_integral<_Ty, 8> : _Atomic_storage<_Ty> { // atomic integral operations using 8-byte intrinsics - using _Base = _Atomic_storage<_Ty>; - using _TVal = typename _Base::_TVal; - -#ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 - _Atomic_integral() = default; - /* implicit */ constexpr _Atomic_integral(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept - : _Base(_Value) {} -#else // ^^^ workaround / no workaround vvv - using _Base::_Base; -#endif // ^^^ no workaround ^^^ - -#if defined(_M_IX86) && defined(__clang__) // TRANSITION, LLVM-46595 - _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - // effectively sequential consistency - _TVal _Temp{this->load()}; - while (!this->compare_exchange_strong(_Temp, _Temp + _Operand, _Order)) { // keep trying - } - - return _Temp; - } - - _TVal fetch_and(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - // effectively sequential consistency - _TVal _Temp{this->load()}; - while (!this->compare_exchange_strong(_Temp, _Temp & _Operand, _Order)) { // keep trying - } - - return _Temp; - } - - _TVal fetch_or(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - // effectively sequential consistency - _TVal _Temp{this->load()}; - while (!this->compare_exchange_strong(_Temp, _Temp | _Operand, _Order)) { // keep trying - } - - return _Temp; - } - - _TVal fetch_xor(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - // effectively sequential consistency - _TVal _Temp{this->load()}; - while (!this->compare_exchange_strong(_Temp, _Temp ^ _Operand, _Order)) { // keep trying - } - - return _Temp; - } - - _TVal operator++(int) noexcept { - return fetch_add(static_cast<_TVal>(1)); - } - - _TVal operator++() noexcept { - return fetch_add(static_cast<_TVal>(1)) + static_cast<_TVal>(1); - } - - _TVal operator--(int) noexcept { - return fetch_add(static_cast<_TVal>(-1)); - } - - _TVal operator--() noexcept { - return fetch_add(static_cast<_TVal>(-1)) - static_cast<_TVal>(1); - } - -#else // ^^^ defined(_M_IX86) && defined(__clang__), LLVM-46595 / !defined(_M_IX86) || !defined(__clang__) vvv - _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - long long _Result; - _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedExchangeAdd64, - _Atomic_address_as(this->_Storage), static_cast(_Operand)); - return static_cast<_TVal>(_Result); - } - - _TVal fetch_and(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - long long _Result; - _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedAnd64, _Atomic_address_as(this->_Storage), - static_cast(_Operand)); - return static_cast<_TVal>(_Result); - } - - _TVal fetch_or(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - long long _Result; - _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedOr64, _Atomic_address_as(this->_Storage), - static_cast(_Operand)); - return static_cast<_TVal>(_Result); - } - - _TVal fetch_xor(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - long long _Result; - _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedXor64, _Atomic_address_as(this->_Storage), - static_cast(_Operand)); - return static_cast<_TVal>(_Result); - } - - _TVal operator++(int) noexcept { - unsigned long long _After = - static_cast(_InterlockedIncrement64(_Atomic_address_as(this->_Storage))); - --_After; - return static_cast<_TVal>(_After); - } - - _TVal operator++() noexcept { - return static_cast<_TVal>(_InterlockedIncrement64(_Atomic_address_as(this->_Storage))); - } - - _TVal operator--(int) noexcept { - unsigned long long _After = - static_cast(_InterlockedDecrement64(_Atomic_address_as(this->_Storage))); - ++_After; - return static_cast<_TVal>(_After); - } - - _TVal operator--() noexcept { - return static_cast<_TVal>(_InterlockedDecrement64(_Atomic_address_as(this->_Storage))); - } -#endif // ^^^ !defined(_M_IX86) || !defined(__clang__) ^^^ -}; - -#if 1 // TRANSITION, ABI -template -_INLINE_VAR constexpr bool _Is_always_lock_free = _TypeSize <= 8 && (_TypeSize & (_TypeSize - 1)) == 0; -#else // ^^^ don't break ABI / break ABI vvv -#if _ATOMIC_HAS_DCAS -template -_INLINE_VAR constexpr bool _Is_always_lock_free = _TypeSize <= 2 * sizeof(void*); -#else // ^^^ _ATOMIC_HAS_DCAS / !_ATOMIC_HAS_DCAS vvv -template -_INLINE_VAR constexpr bool _Is_always_lock_free = _TypeSize <= sizeof(void*); -#endif // _ATOMIC_HAS_DCAS -#endif // break ABI - -template > -_INLINE_VAR constexpr bool _Deprecate_non_lock_free_volatile = true; - -template -_CXX20_DEPRECATE_VOLATILE _INLINE_VAR constexpr bool _Deprecate_non_lock_free_volatile<_Ty, false> = true; - -// STRUCT TEMPLATE _Atomic_integral_facade -template -struct _Atomic_integral_facade : _Atomic_integral<_Ty> { - // provides operator overloads and other support for atomic integral specializations - using _Base = _Atomic_integral<_Ty>; - using _TVal = typename _Base::_TVal; - using difference_type = _TVal; - -#ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 - _Atomic_integral_facade() = default; - /* implicit */ constexpr _Atomic_integral_facade(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept - : _Base(_Value) {} -#else // ^^^ workaround / no workaround vvv - using _Base::_Base; -#endif // ^^^ no workaround ^^^ - - // _Deprecate_non_lock_free_volatile is unnecessary here. - - // note: const_cast-ing away volatile is safe because all our intrinsics add volatile back on. - // We make the primary functions non-volatile for better debug codegen, as non-volatile atomics - // are far more common than volatile ones. - using _Base::fetch_add; - _TVal fetch_add(const _TVal _Operand) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_add(_Operand); - } - - _TVal fetch_add(const _TVal _Operand, const memory_order _Order) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_add(_Operand, _Order); - } - - _NODISCARD static _TVal _Negate(const _TVal _Value) noexcept { // returns two's complement negated value of _Value - return static_cast<_TVal>(0U - static_cast>(_Value)); - } - - _TVal fetch_sub(const _TVal _Operand) noexcept { - return fetch_add(_Negate(_Operand)); - } - - _TVal fetch_sub(const _TVal _Operand) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return fetch_add(_Negate(_Operand)); - } - - _TVal fetch_sub(const _TVal _Operand, const memory_order _Order) noexcept { - return fetch_add(_Negate(_Operand), _Order); - } - - _TVal fetch_sub(const _TVal _Operand, const memory_order _Order) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return fetch_add(_Negate(_Operand), _Order); - } - - using _Base::fetch_and; - _TVal fetch_and(const _TVal _Operand) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_and(_Operand); - } - - _TVal fetch_and(const _TVal _Operand, const memory_order _Order) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_and(_Operand, _Order); - } - - using _Base::fetch_or; - _TVal fetch_or(const _TVal _Operand) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_or(_Operand); - } - - _TVal fetch_or(const _TVal _Operand, const memory_order _Order) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_or(_Operand, _Order); - } - - using _Base::fetch_xor; - _TVal fetch_xor(const _TVal _Operand) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_xor(_Operand); - } - - _TVal fetch_xor(const _TVal _Operand, const memory_order _Order) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_xor(_Operand, _Order); - } - - using _Base::operator++; - _TVal operator++(int) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return const_cast<_Atomic_integral_facade*>(this)->_Base::operator++(0); - } - - _TVal operator++() volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return const_cast<_Atomic_integral_facade*>(this)->_Base::operator++(); - } - - using _Base::operator--; - _TVal operator--(int) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return const_cast<_Atomic_integral_facade*>(this)->_Base::operator--(0); - } - - _TVal operator--() volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return const_cast<_Atomic_integral_facade*>(this)->_Base::operator--(); - } - - _TVal operator+=(const _TVal _Operand) noexcept { - return static_cast<_TVal>(this->_Base::fetch_add(_Operand) + _Operand); - } - - _TVal operator+=(const _TVal _Operand) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return static_cast<_TVal>(const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_add(_Operand) + _Operand); - } - - _TVal operator-=(const _TVal _Operand) noexcept { - return static_cast<_TVal>(fetch_sub(_Operand) - _Operand); - } - - _TVal operator-=(const _TVal _Operand) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return static_cast<_TVal>(const_cast<_Atomic_integral_facade*>(this)->fetch_sub(_Operand) - _Operand); - } - - _TVal operator&=(const _TVal _Operand) noexcept { - return static_cast<_TVal>(this->_Base::fetch_and(_Operand) & _Operand); - } - - _TVal operator&=(const _TVal _Operand) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return static_cast<_TVal>(const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_and(_Operand) & _Operand); - } - - _TVal operator|=(const _TVal _Operand) noexcept { - return static_cast<_TVal>(this->_Base::fetch_or(_Operand) | _Operand); - } - - _TVal operator|=(const _TVal _Operand) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return static_cast<_TVal>(const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_or(_Operand) | _Operand); - } - - _TVal operator^=(const _TVal _Operand) noexcept { - return static_cast<_TVal>(this->_Base::fetch_xor(_Operand) ^ _Operand); - } - - _TVal operator^=(const _TVal _Operand) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return static_cast<_TVal>(const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_xor(_Operand) ^ _Operand); - } -}; - -#if _HAS_CXX20 -template -struct _Atomic_floating : _Atomic_storage<_Ty> { - // provides atomic floating-point operations - using _Base = _Atomic_storage<_Ty>; - using _TVal = typename _Base::_TVal; - using difference_type = _TVal; - -#ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 - _Atomic_floating() = default; - /* implicit */ constexpr _Atomic_floating(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept - : _Base(_Value) {} -#else // ^^^ workaround / no workaround vvv - using _Base::_Base; -#endif // ^^^ no workaround ^^^ - - _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - _TVal _Temp{this->load(memory_order_relaxed)}; - while (!this->compare_exchange_strong(_Temp, _Temp + _Operand, _Order)) { // keep trying - } - - return _Temp; - } - - // _Deprecate_non_lock_free_volatile is unnecessary here. - - // note: const_cast-ing away volatile is safe because all our intrinsics add volatile back on. - // We make the primary functions non-volatile for better debug codegen, as non-volatile atomics - // are far more common than volatile ones. - _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return const_cast<_Atomic_floating*>(this)->fetch_add(_Operand, _Order); - } - - _TVal fetch_sub(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { - _TVal _Temp{this->load(memory_order_relaxed)}; - while (!this->compare_exchange_strong(_Temp, _Temp - _Operand, _Order)) { // keep trying - } - - return _Temp; - } - - _TVal fetch_sub(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return const_cast<_Atomic_floating*>(this)->fetch_sub(_Operand, _Order); - } - - _TVal operator+=(const _TVal _Operand) noexcept { - return fetch_add(_Operand) + _Operand; - } - - _TVal operator+=(const _TVal _Operand) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return const_cast<_Atomic_floating*>(this)->fetch_add(_Operand) + _Operand; - } - - _TVal operator-=(const _TVal _Operand) noexcept { - return fetch_sub(_Operand) - _Operand; - } - - _TVal operator-=(const _TVal _Operand) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return const_cast<_Atomic_floating*>(this)->fetch_sub(_Operand) - _Operand; - } -}; -#endif // _HAS_CXX20 - -// STRUCT TEMPLATE _Atomic_pointer -template -struct _Atomic_pointer : _Atomic_storage<_Ty> { - using _Base = _Atomic_storage<_Ty>; - using _TVal = typename _Base::_TVal; - using difference_type = ptrdiff_t; - -#ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 - _Atomic_pointer() = default; - /* implicit */ constexpr _Atomic_pointer(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept - : _Base(_Value) {} -#else // ^^^ workaround / no workaround vvv - using _Base::_Base; -#endif // ^^^ no workaround ^^^ - - _TVal fetch_add(const ptrdiff_t _Diff, const memory_order _Order = memory_order_seq_cst) noexcept { - const ptrdiff_t _Shift_bytes = - static_cast(static_cast(_Diff) * sizeof(remove_pointer_t<_TVal>)); - ptrdiff_t _Result; -#if defined(_M_IX86) || defined(_M_ARM) - _ATOMIC_CHOOSE_INTRINSIC( - _Order, _Result, _InterlockedExchangeAdd, _Atomic_address_as(this->_Storage), _Shift_bytes); -#else // ^^^ 32 bits / 64 bits vvv - _ATOMIC_CHOOSE_INTRINSIC( - _Order, _Result, _InterlockedExchangeAdd64, _Atomic_address_as(this->_Storage), _Shift_bytes); -#endif // hardware - return reinterpret_cast<_TVal>(_Result); - } - - // _Deprecate_non_lock_free_volatile is unnecessary here. - - _TVal fetch_add(const ptrdiff_t _Diff) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return const_cast<_Atomic_pointer*>(this)->fetch_add(_Diff); - } - - _TVal fetch_add(const ptrdiff_t _Diff, const memory_order _Order) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return const_cast<_Atomic_pointer*>(this)->fetch_add(_Diff, _Order); - } - - _TVal fetch_sub(const ptrdiff_t _Diff) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return fetch_add(static_cast(0 - static_cast(_Diff))); - } - - _TVal fetch_sub(const ptrdiff_t _Diff) noexcept { - return fetch_add(static_cast(0 - static_cast(_Diff))); - } - - _TVal fetch_sub(const ptrdiff_t _Diff, const memory_order _Order) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return fetch_add(static_cast(0 - static_cast(_Diff)), _Order); - } - - _TVal fetch_sub(const ptrdiff_t _Diff, const memory_order _Order) noexcept { - return fetch_add(static_cast(0 - static_cast(_Diff)), _Order); - } - - _TVal operator++(int) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return fetch_add(1); - } - - _TVal operator++(int) noexcept { - return fetch_add(1); - } - - _TVal operator++() volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return fetch_add(1) + 1; - } - - _TVal operator++() noexcept { - return fetch_add(1) + 1; - } - - _TVal operator--(int) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return fetch_add(-1); - } - - _TVal operator--(int) noexcept { - return fetch_add(-1); - } - - _TVal operator--() volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return fetch_add(-1) - 1; - } - - _TVal operator--() noexcept { - return fetch_add(-1) - 1; - } - - _TVal operator+=(const ptrdiff_t _Diff) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return fetch_add(_Diff) + _Diff; - } - - _TVal operator+=(const ptrdiff_t _Diff) noexcept { - return fetch_add(_Diff) + _Diff; - } - - _TVal operator-=(const ptrdiff_t _Diff) volatile noexcept { - static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); - return fetch_add(static_cast(0 - static_cast(_Diff))) - _Diff; - } - - _TVal operator-=(const ptrdiff_t _Diff) noexcept { - return fetch_add(static_cast(0 - static_cast(_Diff))) - _Diff; - } -}; - -// STRUCT TEMPLATE atomic -#define ATOMIC_VAR_INIT(_Value) \ - { _Value } - -template -using _Choose_atomic_base2_t = - typename _Select && !is_same_v>::template _Apply<_Atomic_integral_facade<_Ty>, - typename _Select && is_object_v>>::template _Apply< - _Atomic_pointer<_Ty>, _Atomic_storage<_Ty>>>; - -#if _HAS_CXX20 -template -using _Choose_atomic_base_t = typename _Select>::template _Apply<_Atomic_floating<_Ty>, - _Choose_atomic_base2_t<_TVal, _Ty>>; -#else // ^^^ _HAS_CXX20 // !_HAS_CXX20 vvv -template -using _Choose_atomic_base_t = _Choose_atomic_base2_t<_TVal, _Ty>; -#endif //_HAS_CXX20 - -template -struct atomic : _Choose_atomic_base_t<_Ty> { // atomic value -private: - using _Base = _Choose_atomic_base_t<_Ty>; - -public: - // clang-format off - static_assert(is_trivially_copyable_v<_Ty> && is_copy_constructible_v<_Ty> && is_move_constructible_v<_Ty> - && is_copy_assignable_v<_Ty> && is_move_assignable_v<_Ty>, - "atomic requires T to be trivially copyable, copy constructible, move constructible, copy assignable, " - "and move assignable."); - // clang-format on - - using value_type = _Ty; - -#ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 - /* implicit */ constexpr atomic(const _Ty _Value) noexcept : _Base(_Value) {} -#else // ^^^ workaround / no workaround vvv - using _Base::_Base; -#endif // ^^^ no workaround ^^^ - - constexpr atomic() noexcept(is_nothrow_default_constructible_v<_Ty>) : _Base() {} - - atomic(const atomic&) = delete; - atomic& operator=(const atomic&) = delete; - -#if _HAS_CXX17 - static constexpr bool is_always_lock_free = _Is_always_lock_free; -#endif // _HAS_CXX17 - -#if 1 // TRANSITION, ABI - _NODISCARD bool is_lock_free() const volatile noexcept { - constexpr bool _Result = sizeof(_Ty) <= 8 && (sizeof(_Ty) & sizeof(_Ty) - 1) == 0; - return _Result; - } - -#else // ^^^ don't break ABI / break ABI vvv - - _NODISCARD bool is_lock_free() const volatile noexcept { -#if _ATOMIC_HAS_DCAS - return sizeof(_Ty) <= 2 * sizeof(void*); -#else // ^^^ _ATOMIC_HAS_DCAS / !_ATOMIC_HAS_DCAS vvv - return sizeof(_Ty) <= sizeof(void*) || (sizeof(_Ty) <= 2 * sizeof(void*) && __std_atomic_has_cmpxchg16b()); -#endif // _ATOMIC_HAS_DCAS - } -#endif // TRANSITION, ABI - - _NODISCARD bool is_lock_free() const noexcept { - return static_cast(this)->is_lock_free(); - } - - _Ty operator=(const _Ty _Value) volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - this->store(_Value); - return _Value; - } - - _Ty operator=(const _Ty _Value) noexcept { - this->store(_Value); - return _Value; - } - - // For the following, we do the real implementation in the non-volatile function, and const_cast - // to call the non-volatile function in the volatile one. This is safe because all of the - // non-volatile functions reapply volatile, as all our intrinsics accept only volatile T *. - // We expect most atomics to be non-volatile, so making the real implementations - // non-volatile should result in better debug codegen. - using _Base::store; - void store(const _Ty _Value) volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - const_cast(this)->_Base::store(_Value); - } - - void store(const _Ty _Value, const memory_order _Order) volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - const_cast(this)->_Base::store(_Value, _Order); - } - - using _Base::load; - _NODISCARD _Ty load() const volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return const_cast(this)->_Base::load(); - } - - _NODISCARD _Ty load(const memory_order _Order) const volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return const_cast(this)->_Base::load(_Order); - } - - using _Base::exchange; - _Ty exchange(const _Ty _Value) volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return const_cast(this)->_Base::exchange(_Value); - } - - _Ty exchange(const _Ty _Value, const memory_order _Order) volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return const_cast(this)->_Base::exchange(_Value, _Order); - } - - using _Base::compare_exchange_strong; - bool compare_exchange_strong(_Ty& _Expected, const _Ty _Desired) volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return const_cast(this)->_Base::compare_exchange_strong(_Expected, _Desired); - } - - bool compare_exchange_strong(_Ty& _Expected, const _Ty _Desired, const memory_order _Order) volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return const_cast(this)->_Base::compare_exchange_strong(_Expected, _Desired, _Order); - } - - bool compare_exchange_strong(_Ty& _Expected, const _Ty _Desired, const memory_order _Success, - const memory_order _Failure) volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return this->compare_exchange_strong(_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); - } - - bool compare_exchange_strong( - _Ty& _Expected, const _Ty _Desired, const memory_order _Success, const memory_order _Failure) noexcept { - return this->compare_exchange_strong(_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); - } - - bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired) volatile noexcept { - // we have no weak CAS intrinsics, even on ARM32/ARM64, so fall back to strong - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return this->compare_exchange_strong(_Expected, _Desired); - } - - bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired) noexcept { - return this->compare_exchange_strong(_Expected, _Desired); - } - - bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired, const memory_order _Order) volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return this->compare_exchange_strong(_Expected, _Desired, _Order); - } - - bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired, const memory_order _Order) noexcept { - return this->compare_exchange_strong(_Expected, _Desired, _Order); - } - - bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired, const memory_order _Success, - const memory_order _Failure) volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return this->compare_exchange_strong(_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); - } - - bool compare_exchange_weak( - _Ty& _Expected, const _Ty _Desired, const memory_order _Success, const memory_order _Failure) noexcept { - return this->compare_exchange_strong(_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); - } - - operator _Ty() const volatile noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return this->load(); - } - - operator _Ty() const noexcept { - return this->load(); - } -}; - -#if _HAS_CXX17 -template -atomic(_Ty) -> atomic<_Ty>; -#endif // _HAS_CXX17 - -#if _HAS_CXX20 -template -struct atomic_ref : _Choose_atomic_base_t<_Ty, _Ty&> { // atomic reference -private: - using _Base = _Choose_atomic_base_t<_Ty, _Ty&>; - -public: - // clang-format off - static_assert(is_trivially_copyable_v<_Ty> && is_copy_constructible_v<_Ty> && is_move_constructible_v<_Ty> - && is_copy_assignable_v<_Ty> && is_move_assignable_v<_Ty>, - "atomic requires T to be trivially copyable, copy constructible, move constructible, copy assignable, " - "and move assignable."); - // clang-format on - - using value_type = _Ty; - - explicit atomic_ref(_Ty& _Value) : _Base(_Value) { - if constexpr (_Is_potentially_lock_free) { - _Check_aligment(_Value); - } - -#if 1 // TRANSITION, ABI - if constexpr (!is_always_lock_free) { -#else - if constexpr (!_Is_potentially_lock_free) { -#endif - this->_Init_spinlock_for_ref(); - } - } - - atomic_ref(const atomic_ref&) noexcept = default; - - atomic_ref& operator=(const atomic_ref&) = delete; - - static constexpr bool is_always_lock_free = _Is_always_lock_free; - - static constexpr bool _Is_potentially_lock_free = - sizeof(_Ty) <= 2 * sizeof(void*) && (sizeof(_Ty) & (sizeof(_Ty) - 1)) == 0; - - static constexpr size_t required_alignment = _Is_potentially_lock_free ? sizeof(_Ty) : alignof(_Ty); - -#if 1 // TRANSITION, ABI - _NODISCARD bool is_lock_free() const noexcept { - constexpr bool _Result = sizeof(_Ty) <= 8 && (sizeof(_Ty) & sizeof(_Ty) - 1) == 0; - return _Result; - } - -#else // ^^^ don't break ABI / break ABI vvv - - _NODISCARD bool is_lock_free() const noexcept { -#if _ATOMIC_HAS_DCAS - return sizeof(_Ty) <= 2 * sizeof(void*); -#else // ^^^ _ATOMIC_HAS_DCAS / !_ATOMIC_HAS_DCAS vvv - return sizeof(_Ty) <= sizeof(void*) || (sizeof(_Ty) <= 2 * sizeof(void*) && __std_atomic_has_cmpxchg16b()); -#endif // _ATOMIC_HAS_DCAS - } -#endif // TRANSITION, ABI - - _Ty operator=(const _Ty _Value) noexcept { - this->store(_Value); - return _Value; - } - - using _Base::compare_exchange_strong; - bool compare_exchange_strong( - _Ty& _Expected, const _Ty _Desired, const memory_order _Success, const memory_order _Failure) noexcept { - return this->compare_exchange_strong(_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); - } - - bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired) noexcept { - return this->compare_exchange_strong(_Expected, _Desired); - } - - bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired, const memory_order _Order) noexcept { - return this->compare_exchange_strong(_Expected, _Desired, _Order); - } - - bool compare_exchange_weak( - _Ty& _Expected, const _Ty _Desired, const memory_order _Success, const memory_order _Failure) noexcept { - return this->compare_exchange_strong(_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); - } - - operator _Ty() const noexcept { - return this->load(); - } - -private: - void _Check_aligment([[maybe_unused]] const _Ty& _Value) { - _ATOMIC_REF_CHECK_ALIGNMENT( - (reinterpret_cast(_STD addressof(_Value)) & (required_alignment - 1)) == 0, - "atomic_ref underlying object is not aligned as required_alignment"); - } -}; -#endif // _HAS_CXX20 - -// NONMEMBER OPERATIONS ON ATOMIC TYPES -template -_NODISCARD bool atomic_is_lock_free(const volatile atomic<_Ty>* _Mem) noexcept { - return _Mem->is_lock_free(); -} - -template -_NODISCARD bool atomic_is_lock_free(const atomic<_Ty>* _Mem) noexcept { - return _Mem->is_lock_free(); -} - -template -_CXX20_DEPRECATE_ATOMIC_INIT void atomic_init( - atomic<_Ty>* const _Mem, const typename atomic<_Ty>::value_type _Value) noexcept { -#if 1 // TRANSITION, ABI - _CSTD memcpy(_STD addressof(_Mem->_Storage), _STD addressof(_Value), sizeof(_Ty)); -#else // ^^^ don't break ABI / break ABI vvv - _CSTD memcpy(_Mem, _STD addressof(_Value), sizeof(_Ty)); -#endif // TRANSITION, ABI - _CSTD memset(reinterpret_cast(_Mem) + sizeof(_Ty), 0, sizeof(atomic<_Ty>) - sizeof(_Ty)); -} - -template -_CXX20_DEPRECATE_ATOMIC_INIT void atomic_init( - volatile atomic<_Ty>* const _Mem, const typename atomic<_Ty>::value_type _Value) noexcept { - // NB: respecting volatility here appears unimplementable - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - _STD atomic_init(const_cast*>(_Mem), _Value); -} - -template -void atomic_store(volatile atomic<_Ty>* const _Mem, const _Identity_t<_Ty> _Value) noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - _Mem->store(_Value); -} - -template -void atomic_store(atomic<_Ty>* const _Mem, const _Identity_t<_Ty> _Value) noexcept { - _Mem->store(_Value); -} - -template -void atomic_store_explicit( - volatile atomic<_Ty>* const _Mem, const _Identity_t<_Ty> _Value, const memory_order _Order) noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - _Mem->store(_Value, _Order); -} - -template -void atomic_store_explicit(atomic<_Ty>* const _Mem, const _Identity_t<_Ty> _Value, const memory_order _Order) noexcept { - _Mem->store(_Value, _Order); -} - -template -_NODISCARD _Ty atomic_load(const volatile atomic<_Ty>* const _Mem) noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return _Mem->load(); -} - -template -_NODISCARD _Ty atomic_load(const atomic<_Ty>* const _Mem) noexcept { - return _Mem->load(); -} - -template -_NODISCARD _Ty atomic_load_explicit(const volatile atomic<_Ty>* const _Mem, const memory_order _Order) noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return _Mem->load(_Order); -} - -template -_NODISCARD _Ty atomic_load_explicit(const atomic<_Ty>* const _Mem, const memory_order _Order) noexcept { - return _Mem->load(_Order); -} - -template -_Ty atomic_exchange(volatile atomic<_Ty>* const _Mem, const _Identity_t<_Ty> _Value) noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return _Mem->exchange(_Value); -} - -template -_Ty atomic_exchange(atomic<_Ty>* const _Mem, const _Identity_t<_Ty> _Value) noexcept { - return _Mem->exchange(_Value); -} - -template -_Ty atomic_exchange_explicit( - volatile atomic<_Ty>* const _Mem, const _Identity_t<_Ty> _Value, const memory_order _Order) noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return _Mem->exchange(_Value, _Order); -} - -template -_Ty atomic_exchange_explicit( - atomic<_Ty>* const _Mem, const _Identity_t<_Ty> _Value, const memory_order _Order) noexcept { - return _Mem->exchange(_Value, _Order); -} - -template -bool atomic_compare_exchange_strong( - volatile atomic<_Ty>* const _Mem, _Identity_t<_Ty>* const _Expected, const _Identity_t<_Ty> _Desired) noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return _Mem->compare_exchange_strong(*_Expected, _Desired); -} - -template -bool atomic_compare_exchange_strong( - atomic<_Ty>* const _Mem, _Identity_t<_Ty>* const _Expected, const _Identity_t<_Ty> _Desired) noexcept { - return _Mem->compare_exchange_strong(*_Expected, _Desired); -} - -template -bool atomic_compare_exchange_strong_explicit(volatile atomic<_Ty>* const _Mem, _Identity_t<_Ty>* const _Expected, - const _Identity_t<_Ty> _Desired, const memory_order _Success, const memory_order _Failure) noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return _Mem->compare_exchange_strong(*_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); -} - -template -bool atomic_compare_exchange_strong_explicit(atomic<_Ty>* const _Mem, _Identity_t<_Ty>* const _Expected, - const _Identity_t<_Ty> _Desired, const memory_order _Success, const memory_order _Failure) noexcept { - return _Mem->compare_exchange_strong(*_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); -} - -template -bool atomic_compare_exchange_weak( - volatile atomic<_Ty>* const _Mem, _Identity_t<_Ty>* const _Expected, const _Identity_t<_Ty> _Desired) noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return _Mem->compare_exchange_strong(*_Expected, _Desired); -} - -template -bool atomic_compare_exchange_weak( - atomic<_Ty>* const _Mem, _Identity_t<_Ty>* const _Expected, const _Identity_t<_Ty> _Desired) noexcept { - return _Mem->compare_exchange_strong(*_Expected, _Desired); -} - -template -bool atomic_compare_exchange_weak_explicit(volatile atomic<_Ty>* const _Mem, _Identity_t<_Ty>* const _Expected, - const _Identity_t<_Ty> _Desired, const memory_order _Success, const memory_order _Failure) noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return _Mem->compare_exchange_strong(*_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); -} - -template -bool atomic_compare_exchange_weak_explicit(atomic<_Ty>* const _Mem, _Identity_t<_Ty>* const _Expected, - const _Identity_t<_Ty> _Desired, const memory_order _Success, const memory_order _Failure) noexcept { - return _Mem->compare_exchange_strong(*_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); -} - -template -_Ty atomic_fetch_add(volatile atomic<_Ty>* _Mem, const typename atomic<_Ty>::difference_type _Value) noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return _Mem->fetch_add(_Value); -} - -template -_Ty atomic_fetch_add(atomic<_Ty>* _Mem, const typename atomic<_Ty>::difference_type _Value) noexcept { - return _Mem->fetch_add(_Value); -} - -template -_Ty atomic_fetch_add_explicit(volatile atomic<_Ty>* _Mem, const typename atomic<_Ty>::difference_type _Value, - const memory_order _Order) noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return _Mem->fetch_add(_Value, _Order); -} - -template -_Ty atomic_fetch_add_explicit( - atomic<_Ty>* _Mem, const typename atomic<_Ty>::difference_type _Value, const memory_order _Order) noexcept { - return _Mem->fetch_add(_Value, _Order); -} - -template -_Ty atomic_fetch_sub(volatile atomic<_Ty>* _Mem, const typename atomic<_Ty>::difference_type _Value) noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return _Mem->fetch_sub(_Value); -} - -template -_Ty atomic_fetch_sub(atomic<_Ty>* _Mem, const typename atomic<_Ty>::difference_type _Value) noexcept { - return _Mem->fetch_sub(_Value); -} - -template -_Ty atomic_fetch_sub_explicit(volatile atomic<_Ty>* _Mem, const typename atomic<_Ty>::difference_type _Value, - const memory_order _Order) noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return _Mem->fetch_sub(_Value, _Order); -} - -template -_Ty atomic_fetch_sub_explicit( - atomic<_Ty>* _Mem, const typename atomic<_Ty>::difference_type _Value, const memory_order _Order) noexcept { - return _Mem->fetch_sub(_Value, _Order); -} - -template -_Ty atomic_fetch_and(volatile atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value) noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return _Mem->fetch_and(_Value); -} - -template -_Ty atomic_fetch_and(atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value) noexcept { - return _Mem->fetch_and(_Value); -} - -template -_Ty atomic_fetch_and_explicit( - volatile atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value, const memory_order _Order) noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return _Mem->fetch_and(_Value, _Order); -} - -template -_Ty atomic_fetch_and_explicit( - atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value, const memory_order _Order) noexcept { - return _Mem->fetch_and(_Value, _Order); -} - -template -_Ty atomic_fetch_or(volatile atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value) noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return _Mem->fetch_or(_Value); -} - -template -_Ty atomic_fetch_or(atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value) noexcept { - return _Mem->fetch_or(_Value); -} - -template -_Ty atomic_fetch_or_explicit( - volatile atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value, const memory_order _Order) noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return _Mem->fetch_or(_Value, _Order); -} - -template -_Ty atomic_fetch_or_explicit( - atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value, const memory_order _Order) noexcept { - return _Mem->fetch_or(_Value, _Order); -} - -template -_Ty atomic_fetch_xor(volatile atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value) noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return _Mem->fetch_xor(_Value); -} - -template -_Ty atomic_fetch_xor(atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value) noexcept { - return _Mem->fetch_xor(_Value); -} - -template -_Ty atomic_fetch_xor_explicit( - volatile atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value, const memory_order _Order) noexcept { - static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); - return _Mem->fetch_xor(_Value, _Order); -} - -template -_Ty atomic_fetch_xor_explicit( - atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value, const memory_order _Order) noexcept { - return _Mem->fetch_xor(_Value, _Order); -} - -// ATOMIC TYPEDEFS -using atomic_bool = atomic; - -using atomic_char = atomic; -using atomic_schar = atomic; -using atomic_uchar = atomic; -using atomic_short = atomic; -using atomic_ushort = atomic; -using atomic_int = atomic; -using atomic_uint = atomic; -using atomic_long = atomic; -using atomic_ulong = atomic; -using atomic_llong = atomic; -using atomic_ullong = atomic; - -#ifdef __cpp_lib_char8_t -using atomic_char8_t = atomic; -#endif // __cpp_lib_char8_t -using atomic_char16_t = atomic; -using atomic_char32_t = atomic; -using atomic_wchar_t = atomic; - -using atomic_int8_t = atomic; -using atomic_uint8_t = atomic; -using atomic_int16_t = atomic; -using atomic_uint16_t = atomic; -using atomic_int32_t = atomic; -using atomic_uint32_t = atomic; -using atomic_int64_t = atomic; -using atomic_uint64_t = atomic; - -using atomic_int_least8_t = atomic; -using atomic_uint_least8_t = atomic; -using atomic_int_least16_t = atomic; -using atomic_uint_least16_t = atomic; -using atomic_int_least32_t = atomic; -using atomic_uint_least32_t = atomic; -using atomic_int_least64_t = atomic; -using atomic_uint_least64_t = atomic; - -using atomic_int_fast8_t = atomic; -using atomic_uint_fast8_t = atomic; -using atomic_int_fast16_t = atomic; -using atomic_uint_fast16_t = atomic; -using atomic_int_fast32_t = atomic; -using atomic_uint_fast32_t = atomic; -using atomic_int_fast64_t = atomic; -using atomic_uint_fast64_t = atomic; - -using atomic_intptr_t = atomic; -using atomic_uintptr_t = atomic; -using atomic_size_t = atomic; -using atomic_ptrdiff_t = atomic; -using atomic_intmax_t = atomic; -using atomic_uintmax_t = atomic; - -#if _HAS_CXX20 -// Though there are CMPXCHG8B and CMPXCHG16B, -// the largest atomics with a full set of efficient operations are pointer-sized. -using atomic_signed_lock_free = atomic_intptr_t; -using atomic_unsigned_lock_free = atomic_uintptr_t; -#endif // _HAS_CXX20 - -// STRUCT atomic_flag -#define ATOMIC_FLAG_INIT \ - {} -struct atomic_flag { // flag with test-and-set semantics -#if _HAS_CXX20 - _NODISCARD bool test(const memory_order _Order = memory_order_seq_cst) const noexcept { - return _Storage.load(_Order) != 0; - } - - _NODISCARD bool test(const memory_order _Order = memory_order_seq_cst) const volatile noexcept { - return _Storage.load(_Order) != 0; - } -#endif // _HAS_CXX20 - - bool test_and_set(const memory_order _Order = memory_order_seq_cst) noexcept { - return _Storage.exchange(true, _Order) != 0; - } - - bool test_and_set(const memory_order _Order = memory_order_seq_cst) volatile noexcept { - return _Storage.exchange(true, _Order) != 0; - } - - void clear(const memory_order _Order = memory_order_seq_cst) noexcept { - _Storage.store(false, _Order); - } - - void clear(const memory_order _Order = memory_order_seq_cst) volatile noexcept { - _Storage.store(false, _Order); - } - - constexpr atomic_flag() noexcept = default; - -#if 1 // TRANSITION, ABI - atomic _Storage; -#else // ^^^ don't break ABI / break ABI vvv - atomic _Storage; -#endif // TRANSITION, ABI -}; - -// atomic_flag NONMEMBERS -#if _HAS_CXX20 -_NODISCARD inline bool atomic_flag_test(const volatile atomic_flag* const _Flag) noexcept { - return _Flag->test(); -} - -_NODISCARD inline bool atomic_flag_test(const atomic_flag* const _Flag) noexcept { - return _Flag->test(); -} - -_NODISCARD inline bool atomic_flag_test_explicit( - const volatile atomic_flag* const _Flag, const memory_order _Order) noexcept { - return _Flag->test(_Order); -} - -_NODISCARD inline bool atomic_flag_test_explicit(const atomic_flag* const _Flag, const memory_order _Order) noexcept { - return _Flag->test(_Order); -} -#endif // _HAS_CXX20 - -inline bool atomic_flag_test_and_set(atomic_flag* _Flag) noexcept { - return _Flag->test_and_set(); -} - -inline bool atomic_flag_test_and_set(volatile atomic_flag* _Flag) noexcept { - return _Flag->test_and_set(); -} - -inline bool atomic_flag_test_and_set_explicit(atomic_flag* _Flag, memory_order _Order) noexcept { - return _Flag->test_and_set(_Order); -} - -inline bool atomic_flag_test_and_set_explicit(volatile atomic_flag* _Flag, memory_order _Order) noexcept { - return _Flag->test_and_set(_Order); -} - -inline void atomic_flag_clear(atomic_flag* _Flag) noexcept { - _Flag->clear(); -} - -inline void atomic_flag_clear(volatile atomic_flag* _Flag) noexcept { - _Flag->clear(); -} - -inline void atomic_flag_clear_explicit(atomic_flag* _Flag, memory_order _Order) noexcept { - _Flag->clear(_Order); -} - -inline void atomic_flag_clear_explicit(volatile atomic_flag* _Flag, memory_order _Order) noexcept { - _Flag->clear(_Order); -} - -_STD_END - -#undef _ATOMIC_CHOOSE_INTRINSIC -#undef _ATOMIC_HAS_DCAS - -#undef _STD_COMPARE_EXCHANGE_128 -#undef _INVALID_MEMORY_ORDER -#undef _Compiler_or_memory_barrier -#undef _Memory_barrier -#undef _Compiler_barrier - -#pragma pop_macro("new") -_STL_RESTORE_CLANG_WARNINGS -#pragma warning(pop) -#pragma pack(pop) -#endif // _STL_COMPILER_PREPROCESSOR -#endif // _ATOMIC_ +// atomic standard header + +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#pragma once +#ifndef _ATOMIC_ +#define _ATOMIC_ +#include +#if _STL_COMPILER_PREPROCESSOR + +#ifdef _M_CEE_PURE +#error is not supported when compiling with /clr:pure. +#endif // _M_CEE_PURE + +#include // for size_t +#include +#include +#include + +#pragma pack(push, _CRT_PACKING) +#pragma warning(push, _STL_WARNING_LEVEL) +#pragma warning(disable : _STL_DISABLED_WARNINGS) +_STL_DISABLE_CLANG_WARNINGS +#pragma push_macro("new") +#undef new + +#define _Compiler_barrier() _STL_DISABLE_DEPRECATED_WARNING _ReadWriteBarrier() _STL_RESTORE_DEPRECATED_WARNING + +#if defined(_M_ARM) || defined(_M_ARM64) +#define _Memory_barrier() __dmb(0xB) // inner shared data memory barrier +#define _Compiler_or_memory_barrier() _Memory_barrier() +#elif defined(_M_IX86) || defined(_M_X64) +// x86/x64 hardware only emits memory barriers inside _Interlocked intrinsics +#define _Compiler_or_memory_barrier() _Compiler_barrier() +#else // ^^^ x86/x64 / unsupported hardware vvv +#error Unsupported hardware +#endif // hardware + +#ifndef _INVALID_MEMORY_ORDER +#ifdef _DEBUG +#define _INVALID_MEMORY_ORDER _STL_REPORT_ERROR("Invalid memory order") +#else // ^^^ _DEBUG / !_DEBUG vvv +#define _INVALID_MEMORY_ORDER +#endif // _DEBUG +#endif // _INVALID_MEMORY_ORDER + +#if 0 // TRANSITION, ABI +// MACRO _STD_COMPARE_EXCHANGE_128 +#if _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B || defined(_M_ARM64) +#define _STD_COMPARE_EXCHANGE_128 _InterlockedCompareExchange128 +#endif // _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B || defined(_M_ARM64) +#if defined(_M_X64) && !_STD_ATOMIC_ALWAYS_USE_CMPXCHG16B +// 16-byte atomics are separately compiled for x64, as not all x64 hardware has the cmpxchg16b +// instruction; in the event this instruction is not available, the fallback is a global +// CRITICAL_SECTION shared by all 16-byte atomics. +// (Note: machines without this instruction typically have 2 cores or fewer, so this isn't too bad) +// All pointer parameters must be 16-byte aligned. +_NODISCARD extern "C" unsigned char __cdecl __std_atomic_compare_exchange_128( + _Inout_bytecount_(16) long long* _Destination, _In_ long long _ExchangeHigh, _In_ long long _ExchangeLow, + _Inout_bytecount_(16) long long* _ComparandResult) noexcept; +_NODISCARD extern "C" bool __cdecl __std_atomic_has_cmpxchg16b() noexcept; +#define _STD_COMPARE_EXCHANGE_128 __std_atomic_compare_exchange_128 +#endif // defined(_M_X64) && !_STD_ATOMIC_ALWAYS_USE_CMPXCHG16B + +// MACRO _ATOMIC_HAS_DCAS +// Controls whether atomic::is_always_lock_free triggers for sizeof(void *) or 2 * sizeof(void *) +#if _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B || !defined(_M_X64) +#define _ATOMIC_HAS_DCAS 1 +#else // ^^ We always have DCAS / We only sometimes have DCAS vvv +#define _ATOMIC_HAS_DCAS 0 +#endif // _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B || !defined(_M_X64) +#endif // TRANSITION, ABI + +// MACRO _ATOMIC_CHOOSE_INTRINSIC +#if defined(_M_IX86) || defined(_M_X64) +#define _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _Intrinsic, ...) \ + _Check_memory_order(_Order); \ + _Result = _Intrinsic(__VA_ARGS__) +#elif defined(_M_ARM) || defined(_M_ARM64) +#define _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _Intrinsic, ...) \ + switch (_Order) { \ + case memory_order_relaxed: \ + _Result = _INTRIN_RELAXED(_Intrinsic)(__VA_ARGS__); \ + break; \ + case memory_order_consume: \ + case memory_order_acquire: \ + _Result = _INTRIN_ACQUIRE(_Intrinsic)(__VA_ARGS__); \ + break; \ + case memory_order_release: \ + _Result = _INTRIN_RELEASE(_Intrinsic)(__VA_ARGS__); \ + break; \ + default: \ + _INVALID_MEMORY_ORDER; \ + /* [[fallthrough]]; */ \ + case memory_order_acq_rel: \ + case memory_order_seq_cst: \ + _Result = _Intrinsic(__VA_ARGS__); \ + break; \ + } +#endif // hardware + +// LOCK-FREE PROPERTY +#define ATOMIC_BOOL_LOCK_FREE 2 +#define ATOMIC_CHAR_LOCK_FREE 2 +#ifdef __cpp_lib_char8_t +#define ATOMIC_CHAR8_T_LOCK_FREE 2 +#endif // __cpp_lib_char8_t +#define ATOMIC_CHAR16_T_LOCK_FREE 2 +#define ATOMIC_CHAR32_T_LOCK_FREE 2 +#define ATOMIC_WCHAR_T_LOCK_FREE 2 +#define ATOMIC_SHORT_LOCK_FREE 2 +#define ATOMIC_INT_LOCK_FREE 2 +#define ATOMIC_LONG_LOCK_FREE 2 +#define ATOMIC_LLONG_LOCK_FREE 2 +#define ATOMIC_POINTER_LOCK_FREE 2 + +_EXTERN_C +long* __stdcall __std_atomic_get_mutex(const void* _Key) noexcept; +_END_EXTERN_C + +_STD_BEGIN + +// FENCES +extern "C" inline void atomic_thread_fence(const memory_order _Order) noexcept { + if (_Order == memory_order_relaxed) { + return; + } + +#if defined(_M_IX86) || defined(_M_X64) + _Compiler_barrier(); + if (_Order == memory_order_seq_cst) { + volatile long _Guard; // Not initialized to avoid an unnecessary operation; the value does not matter + + // _mm_mfence could have been used, but it is not supported on older x86 CPUs and is slower on some recent CPUs. + // The memory fence provided by interlocked operations has some exceptions, but this is fine: + // std::atomic_thread_fence works with respect to other atomics only; it may not be a full fence for all ops. +#pragma warning(suppress : 6001) // "Using uninitialized memory '_Guard'" +#pragma warning(suppress : 28113) // "Accessing a local variable _Guard via an Interlocked function: This is an unusual + // usage which could be reconsidered." + (void) _InterlockedIncrement(&_Guard); + _Compiler_barrier(); + } +#elif defined(_M_ARM) || defined(_M_ARM64) + _Memory_barrier(); +#else // ^^^ ARM32/ARM64 / unsupported hardware vvv +#error Unsupported hardware +#endif // unsupported hardware +} + +extern "C" inline void atomic_signal_fence(const memory_order _Order) noexcept { + if (_Order != memory_order_relaxed) { + _Compiler_barrier(); + } +} + +// FUNCTION TEMPLATE kill_dependency +template +_Ty kill_dependency(_Ty _Arg) noexcept { // "magic" template that kills dependency ordering when called + return _Arg; +} + +// FUNCTION _Check_memory_order +inline void _Check_memory_order(const memory_order _Order) noexcept { + // check that _Order is a valid memory_order + if (static_cast(_Order) > static_cast(memory_order_seq_cst)) { + _INVALID_MEMORY_ORDER; + } +} + +// FUNCTION _Check_store_memory_order +inline void _Check_store_memory_order(const memory_order _Order) noexcept { + switch (_Order) { + case memory_order_relaxed: + case memory_order_release: + case memory_order_seq_cst: + // nothing to do + break; + case memory_order_consume: + case memory_order_acquire: + case memory_order_acq_rel: + default: + _INVALID_MEMORY_ORDER; + break; + } +} + +// FUNCTION _Check_load_memory_order +inline void _Check_load_memory_order(const memory_order _Order) noexcept { + switch (_Order) { + case memory_order_relaxed: + case memory_order_consume: + case memory_order_acquire: + case memory_order_seq_cst: + // nothing to do + break; + case memory_order_release: + case memory_order_acq_rel: + default: + _INVALID_MEMORY_ORDER; + break; + } +} + +// FUNCTION _Combine_cas_memory_orders +_NODISCARD inline memory_order _Combine_cas_memory_orders( + const memory_order _Success, const memory_order _Failure) noexcept { + // Finds upper bound of a compare/exchange memory order + // pair, according to the following partial order: + // seq_cst + // | + // acq_rel + // / \ + // acquire release + // | | + // consume | + // \ / + // relaxed + static constexpr memory_order _Combined_memory_orders[6][6] = {// combined upper bounds + {memory_order_relaxed, memory_order_consume, memory_order_acquire, memory_order_release, memory_order_acq_rel, + memory_order_seq_cst}, + {memory_order_consume, memory_order_consume, memory_order_acquire, memory_order_acq_rel, memory_order_acq_rel, + memory_order_seq_cst}, + {memory_order_acquire, memory_order_acquire, memory_order_acquire, memory_order_acq_rel, memory_order_acq_rel, + memory_order_seq_cst}, + {memory_order_release, memory_order_acq_rel, memory_order_acq_rel, memory_order_release, memory_order_acq_rel, + memory_order_seq_cst}, + {memory_order_acq_rel, memory_order_acq_rel, memory_order_acq_rel, memory_order_acq_rel, memory_order_acq_rel, + memory_order_seq_cst}, + {memory_order_seq_cst, memory_order_seq_cst, memory_order_seq_cst, memory_order_seq_cst, memory_order_seq_cst, + memory_order_seq_cst}}; + + _Check_memory_order(_Success); + _Check_load_memory_order(_Failure); + return _Combined_memory_orders[static_cast(_Success)][static_cast(_Failure)]; +} + +// FUNCTION TEMPLATE _Atomic_reinterpret_as +template +_NODISCARD _Integral _Atomic_reinterpret_as(const _Ty& _Source) noexcept { + // interprets _Source as the supplied integral type + static_assert(is_integral_v<_Integral>, "Tried to reinterpret memory as non-integral"); +#if _HAS_IF_CONSTEXPR + if constexpr (is_integral_v<_Ty> && sizeof(_Integral) == sizeof(_Ty)) { + return static_cast<_Integral>(_Source); + } else if constexpr (is_pointer_v<_Ty> && sizeof(_Integral) == sizeof(_Ty)) { + return reinterpret_cast<_Integral>(_Source); + } else +#endif // _HAS_IF_CONSTEXPR + { + _Integral _Result{}; // zero padding bits + _CSTD memcpy(&_Result, _STD addressof(_Source), sizeof(_Source)); + return _Result; + } +} + +// FUNCTION _Load_barrier +inline void _Load_barrier(const memory_order _Order) noexcept { // implement memory barrier for atomic load functions + switch (_Order) { + case memory_order_relaxed: + // no barrier + break; + default: + case memory_order_release: + case memory_order_acq_rel: + _INVALID_MEMORY_ORDER; + // [[fallthrough]]; + case memory_order_consume: + case memory_order_acquire: + case memory_order_seq_cst: + _Compiler_or_memory_barrier(); + break; + } +} + +#if 1 // TRANSITION, ABI +template +struct _Atomic_padded { + alignas(sizeof(_Ty)) mutable _Ty _Value; // align to sizeof(T); x86 stack aligns 8-byte objects on 4-byte boundaries +}; + +#else // ^^^ don't break ABI / break ABI vvv +// STRUCT TEMPLATE _Atomic_storage_traits +template +struct _Atomic_storage_traits { // properties for how _Ty is stored in an atomic + static constexpr size_t _Storage_size = + sizeof(_Ty) == 1 ? 1 + : sizeof(_Ty) == 2 ? 2 + : sizeof(_Ty) <= 4 ? 4 + : sizeof(_Ty) <= 8 ? 8 +#if defined(_M_X64) || defined(_M_ARM64) + : sizeof(_Ty) <= 16 ? 16 +#endif // 64 bits + : sizeof(_Ty); + + static constexpr size_t _Padding_size = _Storage_size - sizeof(_Ty); + static constexpr bool _Uses_padding = _Padding_size != 0; +}; + +template +struct _Atomic_storage_traits<_Ty&> { // properties for how _Ty is stored in an atomic_ref + static constexpr size_t _Storage_size = sizeof(_Ty); + static constexpr bool _Uses_padding = false; +} + +// STRUCT TEMPLATE _Atomic_padded +template ::_Uses_padding> +struct _Atomic_padded { // aggregate to allow explicit constexpr zeroing of padding + alignas(_Atomic_storage_traits<_Ty>::_Storage_size) mutable _Ty _Value; + mutable unsigned char _Padding[_Atomic_storage_traits<_Ty>::_Padding_size]; +}; + +template +struct _Atomic_padded<_Ty, false> { + alignas(sizeof(_Ty)) mutable _Ty _Value; // align to sizeof(T); x86 stack aligns 8-byte objects on 4-byte boundaries +}; + +template +struct _Atomic_padded<_Ty&, false> { + _Ty& _Value; +}; + +#endif // TRANSITION, ABI + +template +struct _Atomic_storage_types { + using _TVal = _Ty; + using _TStorage = _Atomic_padded<_Ty>; + using _TConstr = const _Ty; + using _Spinlock = long; + + static constexpr bool _Has_volatile = true; + + static long* _Get_spinlock(long& _Spinlock) { + return &_Spinlock; + } +}; + +template +struct _Atomic_storage_types<_Ty&> { + using _TVal = _Ty; + using _TStorage = _Ty&; + using _TConstr = _Ty&; + using _Spinlock = long*; + + static constexpr bool _Has_volatile = false; + + static long* _Get_spinlock(long* _Spinlock) { + return _Spinlock; + } +}; + +// STRUCT TEMPLATE _Atomic_storage +#if 1 // TRANSITION, ABI +template ::_TVal)> +#else // ^^^ don't break ABI / break ABI vvv +template ::_Storage_size> +#endif // TRANSITION, ABI +struct _Atomic_storage { + // Provides operations common to all specializations of std::atomic, load, store, exchange, and CAS. + // Locking version used when hardware has no atomic operations for sizeof(_Ty). + + using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; + + _Atomic_storage() = default; + + /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Storage(_Value) { + // non-atomically initialize this atomic + } + + void store(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { + // store with sequential consistency + _Check_store_memory_order(_Order); + _Lock(); + _Storage = _Value; + _Unlock(); + } + + _NODISCARD _TVal load(const memory_order _Order = memory_order_seq_cst) const noexcept { + // load with sequential consistency + _Check_load_memory_order(_Order); + _Lock(); + _TVal _Local(_Storage); + _Unlock(); + return _Local; + } + + _TVal exchange(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { + // exchange _Value with _Storage with sequential consistency + _Check_memory_order(_Order); + _Lock(); + _TVal _Result(_Storage); + _Storage = _Value; + _Unlock(); + return _Result; + } + + bool compare_exchange_strong(_TVal& _Expected, const _TVal _Desired, + const memory_order _Order = memory_order_seq_cst) noexcept { // CAS with sequential consistency, plain + _Check_memory_order(_Order); + const auto _Storage_ptr = _STD addressof(_Storage); + const auto _Expected_ptr = _STD addressof(_Expected); + bool _Result; + _Lock(); + if (_CSTD memcmp(_Storage_ptr, _Expected_ptr, sizeof(_TVal)) == 0) { + _CSTD memcpy(_Storage_ptr, _STD addressof(_Desired), sizeof(_TVal)); + _Result = true; + } else { + _CSTD memcpy(_Expected_ptr, _Storage_ptr, sizeof(_TVal)); + _Result = false; + } + + _Unlock(); + return _Result; + } + +#if 1 // TRANSITION, ABI + void _Lock() const noexcept { // lock the spinlock + while (_InterlockedExchange(_Atomic_storage_types<_Ty>::_Get_spinlock(_Spinlock), 1)) { + _YIELD_PROCESSOR(); + } + } + + void _Unlock() const noexcept { // unlock the spinlock +#if defined(_M_ARM) || defined(_M_ARM64) + _Memory_barrier(); + __iso_volatile_store32(reinterpret_cast(_Atomic_storage_types<_Ty>::_Get_spinlock(_Spinlock)), 0); + _Memory_barrier(); +#else // ^^^ ARM32/ARM64 hardware / x86/x64 hardware vvv + _InterlockedExchange(_Atomic_storage_types<_Ty>::_Get_spinlock(_Spinlock), 0); +#endif // hardware + } + +protected: + void _Init_spinlock_for_ref() noexcept { + _Spinlock = __std_atomic_get_mutex(_STD addressof(_Storage)); + } + +private: + // Spinlock for non-lock-free atomic. Spinlock pointer for non-lock-free atomic_ref + mutable typename _Atomic_storage_types<_Ty>::_Spinlock _Spinlock = 0; + +public: + _Ty _Storage{}; + +#else // ^^^ don't break ABI / break ABI vvv + void _Lock() const noexcept { // lock the spinlock + while (_InterlockedExchange8(&_Spinlock, 1)) { + _YIELD_PROCESSOR(); + } + } + + void _Unlock() const noexcept { // unlock the spinlock +#if defined(_M_ARM) || defined(_M_ARM64) + _Memory_barrier(); + __iso_volatile_store8(&_Spinlock, 0); + _Memory_barrier(); +#else // ^^^ ARM32/ARM64 hardware / x86/x64 hardware vvv + _InterlockedExchange8(&_Spinlock, 0); +#endif // hardware + } + + _Ty _Storage; + // Revisit, could be removed completely for atomic_ref + mutable char _Spinlock = 0; +#endif // TRANSITION, ABI +}; + +template +struct _Atomic_storage<_Ty, 1> { // lock-free using 1-byte intrinsics + + using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; + + _Atomic_storage() = default; + + /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Storage{_Value} { + // non-atomically initialize this atomic + } + + void store(const _TVal _Value) noexcept { // store with sequential consistency + const auto _Mem = _Atomic_address_as(_Storage); + const char _As_bytes = _Atomic_reinterpret_as(_Value); +#if defined(_M_ARM) || defined(_M_ARM64) + _Memory_barrier(); + __iso_volatile_store8(_Mem, _As_bytes); + _Memory_barrier(); +#else // ^^^ ARM32/ARM64 hardware / x86/x64 hardware vvv + (void) _InterlockedExchange8(_Mem, _As_bytes); +#endif // hardware + } + + void store(const _TVal _Value, const memory_order _Order) noexcept { // store with given memory order + const auto _Mem = _Atomic_address_as(_Storage); + const char _As_bytes = _Atomic_reinterpret_as(_Value); + switch (_Order) { + case memory_order_relaxed: + __iso_volatile_store8(_Mem, _As_bytes); + return; + case memory_order_release: + _Compiler_or_memory_barrier(); + __iso_volatile_store8(_Mem, _As_bytes); + return; + default: + case memory_order_consume: + case memory_order_acquire: + case memory_order_acq_rel: + _INVALID_MEMORY_ORDER; + // [[fallthrough]]; + case memory_order_seq_cst: + store(_Value); + return; + } + } + + _NODISCARD _TVal load() const noexcept { // load with sequential consistency + const auto _Mem = _Atomic_address_as(_Storage); + char _As_bytes = __iso_volatile_load8(_Mem); + _Compiler_or_memory_barrier(); + return reinterpret_cast<_TVal&>(_As_bytes); + } + + _NODISCARD _TVal load(const memory_order _Order) const noexcept { // load with given memory order + const auto _Mem = _Atomic_address_as(_Storage); + char _As_bytes = __iso_volatile_load8(_Mem); + _Load_barrier(_Order); + return reinterpret_cast<_TVal&>(_As_bytes); + } + + _TVal exchange(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { + // exchange with given memory order + char _As_bytes; + _ATOMIC_CHOOSE_INTRINSIC(_Order, _As_bytes, _InterlockedExchange8, _Atomic_address_as(_Storage), + _Atomic_reinterpret_as(_Value)); + return reinterpret_cast<_Ty&>(_As_bytes); + } + + bool compare_exchange_strong(_TVal& _Expected, const _TVal _Desired, + const memory_order _Order = memory_order_seq_cst) noexcept { // CAS with given memory order + const char _Expected_bytes = _Atomic_reinterpret_as(_Expected); // read before atomic operation + char _Prev_bytes; + _ATOMIC_CHOOSE_INTRINSIC(_Order, _Prev_bytes, _InterlockedCompareExchange8, _Atomic_address_as(_Storage), + _Atomic_reinterpret_as(_Desired), _Expected_bytes); + if (_Prev_bytes == _Expected_bytes) { + return true; + } + + reinterpret_cast(_Expected) = _Prev_bytes; + return false; + } + + typename _Atomic_storage_types<_Ty>::_TStorage _Storage; +}; + +template +struct _Atomic_storage<_Ty, 2> { // lock-free using 2-byte intrinsics + + using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; + + _Atomic_storage() = default; + + /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Storage{_Value} { + // non-atomically initialize this atomic + } + + void store(const _TVal _Value) noexcept { // store with sequential consistency + const auto _Mem = _Atomic_address_as(_Storage); + const short _As_bytes = _Atomic_reinterpret_as(_Value); +#if defined(_M_ARM) || defined(_M_ARM64) + _Memory_barrier(); + __iso_volatile_store16(_Mem, _As_bytes); + _Memory_barrier(); +#else // ^^^ ARM32/ARM64 hardware / x86/x64 hardware vvv + (void) _InterlockedExchange16(_Mem, _As_bytes); +#endif // hardware + } + + void store(const _TVal _Value, const memory_order _Order) noexcept { // store with given memory order + const auto _Mem = _Atomic_address_as(_Storage); + const short _As_bytes = _Atomic_reinterpret_as(_Value); + switch (_Order) { + case memory_order_relaxed: + __iso_volatile_store16(_Mem, _As_bytes); + return; + case memory_order_release: + _Compiler_or_memory_barrier(); + __iso_volatile_store16(_Mem, _As_bytes); + return; + default: + case memory_order_consume: + case memory_order_acquire: + case memory_order_acq_rel: + _INVALID_MEMORY_ORDER; + // [[fallthrough]]; + case memory_order_seq_cst: + store(_Value); + return; + } + } + + _NODISCARD _TVal load() const noexcept { // load with sequential consistency + const auto _Mem = _Atomic_address_as(_Storage); + short _As_bytes = __iso_volatile_load16(_Mem); + _Compiler_or_memory_barrier(); + return reinterpret_cast<_TVal&>(_As_bytes); + } + + _NODISCARD _TVal load(const memory_order _Order) const noexcept { // load with given memory order + const auto _Mem = _Atomic_address_as(_Storage); + short _As_bytes = __iso_volatile_load16(_Mem); + _Load_barrier(_Order); + return reinterpret_cast<_TVal&>(_As_bytes); + } + + _TVal exchange(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { + // exchange with given memory order + short _As_bytes; + _ATOMIC_CHOOSE_INTRINSIC(_Order, _As_bytes, _InterlockedExchange16, _Atomic_address_as(_Storage), + _Atomic_reinterpret_as(_Value)); + return reinterpret_cast<_TVal&>(_As_bytes); + } + + bool compare_exchange_strong(_TVal& _Expected, const _TVal _Desired, + const memory_order _Order = memory_order_seq_cst) noexcept { // CAS with given memory order + const short _Expected_bytes = _Atomic_reinterpret_as(_Expected); // read before atomic operation + short _Prev_bytes; + _ATOMIC_CHOOSE_INTRINSIC(_Order, _Prev_bytes, _InterlockedCompareExchange16, + _Atomic_address_as(_Storage), _Atomic_reinterpret_as(_Desired), _Expected_bytes); + if (_Prev_bytes == _Expected_bytes) { + return true; + } + + _CSTD memcpy(_STD addressof(_Expected), &_Prev_bytes, sizeof(_Ty)); + return false; + } + + typename _Atomic_storage_types<_Ty>::_TStorage _Storage; +}; + +template +struct _Atomic_storage<_Ty, 4> { // lock-free using 4-byte intrinsics + + using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; + + _Atomic_storage() = default; + + /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Storage{_Value} { + // non-atomically initialize this atomic + } + + void store(const _TVal _Value) noexcept { // store with sequential consistency +#if defined(_M_ARM) || defined(_M_ARM64) + _Memory_barrier(); + __iso_volatile_store32(_Atomic_address_as(_Storage), _Atomic_reinterpret_as(_Value)); + _Memory_barrier(); +#else // ^^^ ARM32/ARM64 hardware / x86/x64 hardware vvv + (void) _InterlockedExchange(_Atomic_address_as(_Storage), _Atomic_reinterpret_as(_Value)); +#endif // hardware + } + + void store(const _TVal _Value, const memory_order _Order) noexcept { // store with given memory order + const auto _Mem = _Atomic_address_as(_Storage); + const int _As_bytes = _Atomic_reinterpret_as(_Value); + switch (_Order) { + case memory_order_relaxed: + __iso_volatile_store32(_Mem, _As_bytes); + return; + case memory_order_release: + _Compiler_or_memory_barrier(); + __iso_volatile_store32(_Mem, _As_bytes); + return; + default: + case memory_order_consume: + case memory_order_acquire: + case memory_order_acq_rel: + _INVALID_MEMORY_ORDER; + // [[fallthrough]]; + case memory_order_seq_cst: + store(_Value); + return; + } + } + + _NODISCARD _TVal load() const noexcept { // load with sequential consistency + const auto _Mem = _Atomic_address_as(_Storage); + auto _As_bytes = __iso_volatile_load32(_Mem); + _Compiler_or_memory_barrier(); + return reinterpret_cast<_TVal&>(_As_bytes); + } + + _NODISCARD _TVal load(const memory_order _Order) const noexcept { // load with given memory order + const auto _Mem = _Atomic_address_as(_Storage); + auto _As_bytes = __iso_volatile_load32(_Mem); + _Load_barrier(_Order); + return reinterpret_cast<_TVal&>(_As_bytes); + } + + _TVal exchange(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { + // exchange with given memory order + long _As_bytes; + _ATOMIC_CHOOSE_INTRINSIC(_Order, _As_bytes, _InterlockedExchange, _Atomic_address_as(_Storage), + _Atomic_reinterpret_as(_Value)); + return reinterpret_cast<_TVal&>(_As_bytes); + } + + bool compare_exchange_strong(_TVal& _Expected, const _TVal _Desired, + const memory_order _Order = memory_order_seq_cst) noexcept { // CAS with given memory order + const long _Expected_bytes = _Atomic_reinterpret_as(_Expected); // read before atomic operation + long _Prev_bytes; + _ATOMIC_CHOOSE_INTRINSIC(_Order, _Prev_bytes, _InterlockedCompareExchange, _Atomic_address_as(_Storage), + _Atomic_reinterpret_as(_Desired), _Expected_bytes); + if (_Prev_bytes == _Expected_bytes) { + return true; + } + + _CSTD memcpy(_STD addressof(_Expected), &_Prev_bytes, sizeof(_TVal)); + return false; + } + + typename _Atomic_storage_types<_Ty>::_TStorage _Storage; +}; + +template +struct _Atomic_storage<_Ty, 8> { // lock-free using 8-byte intrinsics + + using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; + + _Atomic_storage() = default; + + /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Storage{_Value} { + // non-atomically initialize this atomic + } + + void store(const _TVal _Value) noexcept { // store with sequential consistency + const auto _Mem = _Atomic_address_as(_Storage); + const long long _As_bytes = _Atomic_reinterpret_as(_Value); +#if defined(_M_IX86) + _Compiler_barrier(); + __iso_volatile_store64(_Mem, _As_bytes); + _STD atomic_thread_fence(memory_order_seq_cst); +#elif defined(_M_ARM64) + _Memory_barrier(); + __iso_volatile_store64(_Mem, _As_bytes); + _Memory_barrier(); +#else // ^^^ _M_ARM64 / ARM32, x64 vvv + (void) _InterlockedExchange64(_Mem, _As_bytes); +#endif // _M_ARM64 + } + + void store(const _TVal _Value, const memory_order _Order) noexcept { // store with given memory order + const auto _Mem = _Atomic_address_as(_Storage); + const long long _As_bytes = _Atomic_reinterpret_as(_Value); + switch (_Order) { + case memory_order_relaxed: + __iso_volatile_store64(_Mem, _As_bytes); + return; + case memory_order_release: + _Compiler_or_memory_barrier(); + __iso_volatile_store64(_Mem, _As_bytes); + return; + default: + case memory_order_consume: + case memory_order_acquire: + case memory_order_acq_rel: + _INVALID_MEMORY_ORDER; + // [[fallthrough]]; + case memory_order_seq_cst: + store(_Value); + return; + } + } + + _NODISCARD _TVal load() const noexcept { // load with sequential consistency + const auto _Mem = _Atomic_address_as(_Storage); + long long _As_bytes; +#ifdef _M_ARM + _As_bytes = __ldrexd(_Mem); + _Memory_barrier(); +#else + _As_bytes = __iso_volatile_load64(_Mem); + _Compiler_or_memory_barrier(); +#endif + return reinterpret_cast<_TVal&>(_As_bytes); + } + + _NODISCARD _TVal load(const memory_order _Order) const noexcept { // load with given memory order + const auto _Mem = _Atomic_address_as(_Storage); +#ifdef _M_ARM + long long _As_bytes = __ldrexd(_Mem); +#else + long long _As_bytes = __iso_volatile_load64(_Mem); +#endif + _Load_barrier(_Order); + return reinterpret_cast<_TVal&>(_As_bytes); + } + +#if defined(_M_IX86) && defined(__clang__) // TRANSITION, LLVM-46595 + TVal exchange(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { + // exchange with (effectively) sequential consistency + _TVal _Temp{load()}; + while (!compare_exchange_strong(_Temp, _Value, _Order)) { // keep trying + } + + return _Temp; + } +#else // ^^^ defined(_M_IX86) && defined(__clang__), LLVM-46595 / !defined(_M_IX86) || !defined(__clang__) vvv + _TVal exchange(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { + // exchange with given memory order + long long _As_bytes; + _ATOMIC_CHOOSE_INTRINSIC(_Order, _As_bytes, _InterlockedExchange64, _Atomic_address_as(_Storage), + _Atomic_reinterpret_as(_Value)); + return reinterpret_cast<_TVal&>(_As_bytes); + } +#endif // ^^^ !defined(_M_IX86) || !defined(__clang__) ^^^ + + bool compare_exchange_strong(_TVal& _Expected, const _TVal _Desired, + const memory_order _Order = memory_order_seq_cst) noexcept { // CAS with given memory order + const long long _Expected_bytes = _Atomic_reinterpret_as(_Expected); // read before atomic operation + long long _Prev_bytes; + _ATOMIC_CHOOSE_INTRINSIC(_Order, _Prev_bytes, _InterlockedCompareExchange64, + _Atomic_address_as(_Storage), _Atomic_reinterpret_as(_Desired), _Expected_bytes); + if (_Prev_bytes == _Expected_bytes) { + return true; + } + + _CSTD memcpy(_STD addressof(_Expected), &_Prev_bytes, sizeof(_TVal)); + return false; + } + + typename _Atomic_storage_types<_Ty>::_TStorage _Storage; +}; + +#if 0 // TRANSITION, ABI +#if defined(_M_X64) || defined(_M_ARM64) +template +struct _Atomic_storage<_Ty, 16> { // lock-free using 16-byte intrinsics + + using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; + + _Atomic_storage() = default; + + /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Storage{_Value} {} // non-atomically initialize this atomic + + void store(const _TVal _Value) noexcept { // store with sequential consistency + (void) exchange(_Value); + } + + void store(const _TVal _Value, const memory_order _Order) noexcept { // store with given memory order + _Check_store_memory_order(_Order); + (void) exchange(_Value, _Order); + } + + _NODISCARD _TVal load() const noexcept { // load with sequential consistency + long long* const _Storage_ptr = const_cast(_Atomic_address_as(_Storage)); + _Int128 _Result{}; // atomic CAS 0 with 0 + (void) _STD_COMPARE_EXCHANGE_128(_Storage_ptr, 0, 0, &_Result._Low); + return reinterpret_cast<_TVal&>(_Result); + } + + _NODISCARD _TVal load(const memory_order _Order) const noexcept { // load with given memory order +#ifdef _M_ARM64 + long long* const _Storage_ptr = const_cast(_Atomic_address_as(_Storage)); + _Int128 _Result{}; // atomic CAS 0 with 0 + switch (_Order) { + case memory_order_relaxed: + (void) _INTRIN_RELAXED(_InterlockedCompareExchange128)(_Storage_ptr, 0, 0, &_Result._Low); + break; + case memory_order_consume: + case memory_order_acquire: + (void) _INTRIN_ACQUIRE(_InterlockedCompareExchange128)(_Storage_ptr, 0, 0, &_Result._Low); + break; + default: + case memory_order_release: + case memory_order_acq_rel: + _INVALID_MEMORY_ORDER; + // [[fallthrough]]; + case memory_order_seq_cst: + (void) _InterlockedCompareExchange128(_Storage_ptr, 0, 0, &_Result._Low); + break; + } + + return reinterpret_cast<_TVal&>(_Result); +#else // ^^^ _M_ARM64 / _M_X64 vvv + _Check_load_memory_order(_Order); + return load(); +#endif // _M_ARM64 + } + + _TVal exchange(const _TVal _Value) noexcept { // exchange with sequential consistency + _TVal _Result{_Value}; + while (!compare_exchange_strong(_Result, _Value)) { // keep trying + } + + return _Result; + } + + _TVal exchange(const _TVal _Value, const memory_order _Order) noexcept { // exchange with given memory order + _TVal _Result{_Value}; + while (!compare_exchange_strong(_Result, _Value, _Order)) { // keep trying + } + + return _Result; + } + + bool compare_exchange_strong(_TVal& _Expected, const _TVal _Desired, + const memory_order _Order = memory_order_seq_cst) noexcept { // CAS with given memory order + _Int128 _Desired_bytes{}; + _CSTD memcpy(&_Desired_bytes, _STD addressof(_Desired), sizeof(_TVal)); + _Int128 _Expected_temp{}; + _CSTD memcpy(&_Expected_temp, _STD addressof(_Expected), sizeof(_TVal)); + unsigned char _Result; +#ifdef _M_ARM64 + _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedCompareExchange128, + _Atomic_address_as(_Storage), _Desired_bytes._High, _Desired_bytes._Low, &_Expected_temp._Low); +#else // ^^^ _M_ARM64 / _M_X64 vvv + (void) _Order; + _Result = _STD_COMPARE_EXCHANGE_128( + &reinterpret_cast(_Storage), _Desired_bytes._High, _Desired_bytes._Low, &_Expected_temp._Low); +#endif // _M_ARM64 + if (_Result == 0) { + _CSTD memcpy(_STD addressof(_Expected), &_Expected_temp, sizeof(_TVal)); + } + + return _Result != 0; + } + + struct _Int128 { + alignas(16) long long _Low; + long long _High; + }; + + typename _Atomic_storage_types<_Ty>::_TStorage _Storage; +}; +#endif // defined(_M_X64) || defined(_M_ARM64) +#endif // TRANSITION, ABI + +// STRUCT TEMPLATE _Atomic_integral +template +struct _Atomic_integral; // not defined + +template +struct _Atomic_integral<_Ty, 1> : _Atomic_storage<_Ty> { // atomic integral operations using 1-byte intrinsics + using _Base = _Atomic_storage<_Ty>; + using _TVal = typename _Base::_TVal; + +#ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 + _Atomic_integral() = default; + /* implicit */ constexpr _Atomic_integral(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Base(_Value) {} +#else // ^^^ workaround / no workaround vvv + using _Base::_Base; +#endif // ^^^ no workaround ^^^ + + _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + char _Result; + _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedExchangeAdd8, _Atomic_address_as(this->_Storage), + static_cast(_Operand)); + return static_cast<_TVal>(_Result); + } + + _TVal fetch_and(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + char _Result; + _ATOMIC_CHOOSE_INTRINSIC( + _Order, _Result, _InterlockedAnd8, _Atomic_address_as(this->_Storage), static_cast(_Operand)); + return static_cast<_TVal>(_Result); + } + + _TVal fetch_or(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + char _Result; + _ATOMIC_CHOOSE_INTRINSIC( + _Order, _Result, _InterlockedOr8, _Atomic_address_as(this->_Storage), static_cast(_Operand)); + return static_cast<_TVal>(_Result); + } + + _TVal fetch_xor(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + char _Result; + _ATOMIC_CHOOSE_INTRINSIC( + _Order, _Result, _InterlockedXor8, _Atomic_address_as(this->_Storage), static_cast(_Operand)); + return static_cast<_TVal>(_Result); + } + + _TVal operator++(int) noexcept { + return static_cast<_TVal>(_InterlockedExchangeAdd8(_Atomic_address_as(this->_Storage), 1)); + } + + _TVal operator++() noexcept { + unsigned char _Before = + static_cast(_InterlockedExchangeAdd8(_Atomic_address_as(this->_Storage), 1)); + ++_Before; + return static_cast<_TVal>(_Before); + } + + _TVal operator--(int) noexcept { + return static_cast<_Ty>(_InterlockedExchangeAdd8(_Atomic_address_as(this->_Storage), -1)); + } + + _TVal operator--() noexcept { + unsigned char _Before = + static_cast(_InterlockedExchangeAdd8(_Atomic_address_as(this->_Storage), -1)); + --_Before; + return static_cast<_TVal>(_Before); + } +}; + +template +struct _Atomic_integral<_Ty, 2> : _Atomic_storage<_Ty> { // atomic integral operations using 2-byte intrinsics + using _Base = _Atomic_storage<_Ty>; + using _TVal = typename _Base::_TVal; + +#ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 + _Atomic_integral() = default; + /* implicit */ constexpr _Atomic_integral(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Base(_Value) {} +#else // ^^^ workaround / no workaround vvv + using _Base::_Base; +#endif // ^^^ no workaround ^^^ + + _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + short _Result; + _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedExchangeAdd16, _Atomic_address_as(this->_Storage), + static_cast(_Operand)); + return static_cast<_TVal>(_Result); + } + + _TVal fetch_and(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + short _Result; + _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedAnd16, _Atomic_address_as(this->_Storage), + static_cast(_Operand)); + return static_cast<_TVal>(_Result); + } + + _TVal fetch_or(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + short _Result; + _ATOMIC_CHOOSE_INTRINSIC( + _Order, _Result, _InterlockedOr16, _Atomic_address_as(this->_Storage), static_cast(_Operand)); + return static_cast<_TVal>(_Result); + } + + _TVal fetch_xor(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + short _Result; + _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedXor16, _Atomic_address_as(this->_Storage), + static_cast(_Operand)); + return static_cast<_TVal>(_Result); + } + + _TVal operator++(int) noexcept { + unsigned short _After = + static_cast(_InterlockedIncrement16(_Atomic_address_as(this->_Storage))); + --_After; + return static_cast<_TVal>(_After); + } + + _TVal operator++() noexcept { + return static_cast<_TVal>(_InterlockedIncrement16(_Atomic_address_as(this->_Storage))); + } + + _TVal operator--(int) noexcept { + unsigned short _After = + static_cast(_InterlockedDecrement16(_Atomic_address_as(this->_Storage))); + ++_After; + return static_cast<_TVal>(_After); + } + + _TVal operator--() noexcept { + return static_cast<_TVal>(_InterlockedDecrement16(_Atomic_address_as(this->_Storage))); + } +}; + +template +struct _Atomic_integral<_Ty, 4> : _Atomic_storage<_Ty> { // atomic integral operations using 4-byte intrinsics + using _Base = _Atomic_storage<_Ty>; + using _TVal = typename _Base::_TVal; + +#ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 + _Atomic_integral() = default; + /* implicit */ constexpr _Atomic_integral(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Base(_Value) {} +#else // ^^^ workaround / no workaround vvv + using _Base::_Base; +#endif // ^^^ no workaround ^^^ + + _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + long _Result; + _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedExchangeAdd, _Atomic_address_as(this->_Storage), + static_cast(_Operand)); + return static_cast<_TVal>(_Result); + } + + _TVal fetch_and(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + long _Result; + _ATOMIC_CHOOSE_INTRINSIC( + _Order, _Result, _InterlockedAnd, _Atomic_address_as(this->_Storage), static_cast(_Operand)); + return static_cast<_TVal>(_Result); + } + + _TVal fetch_or(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + long _Result; + _ATOMIC_CHOOSE_INTRINSIC( + _Order, _Result, _InterlockedOr, _Atomic_address_as(this->_Storage), static_cast(_Operand)); + return static_cast<_TVal>(_Result); + } + + _TVal fetch_xor(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + long _Result; + _ATOMIC_CHOOSE_INTRINSIC( + _Order, _Result, _InterlockedXor, _Atomic_address_as(this->_Storage), static_cast(_Operand)); + return static_cast<_TVal>(_Result); + } + + _TVal operator++(int) noexcept { + unsigned long _After = + static_cast(_InterlockedIncrement(_Atomic_address_as(this->_Storage))); + --_After; + return static_cast<_TVal>(_After); + } + + _TVal operator++() noexcept { + return static_cast<_TVal>(_InterlockedIncrement(_Atomic_address_as(this->_Storage))); + } + + _TVal operator--(int) noexcept { + unsigned long _After = + static_cast(_InterlockedDecrement(_Atomic_address_as(this->_Storage))); + ++_After; + return static_cast<_TVal>(_After); + } + + _TVal operator--() noexcept { + return static_cast<_TVal>(_InterlockedDecrement(_Atomic_address_as(this->_Storage))); + } +}; + +template +struct _Atomic_integral<_Ty, 8> : _Atomic_storage<_Ty> { // atomic integral operations using 8-byte intrinsics + using _Base = _Atomic_storage<_Ty>; + using _TVal = typename _Base::_TVal; + +#ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 + _Atomic_integral() = default; + /* implicit */ constexpr _Atomic_integral(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Base(_Value) {} +#else // ^^^ workaround / no workaround vvv + using _Base::_Base; +#endif // ^^^ no workaround ^^^ + +#if defined(_M_IX86) && defined(__clang__) // TRANSITION, LLVM-46595 + _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + // effectively sequential consistency + _TVal _Temp{this->load()}; + while (!this->compare_exchange_strong(_Temp, _Temp + _Operand, _Order)) { // keep trying + } + + return _Temp; + } + + _TVal fetch_and(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + // effectively sequential consistency + _TVal _Temp{this->load()}; + while (!this->compare_exchange_strong(_Temp, _Temp & _Operand, _Order)) { // keep trying + } + + return _Temp; + } + + _TVal fetch_or(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + // effectively sequential consistency + _TVal _Temp{this->load()}; + while (!this->compare_exchange_strong(_Temp, _Temp | _Operand, _Order)) { // keep trying + } + + return _Temp; + } + + _TVal fetch_xor(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + // effectively sequential consistency + _TVal _Temp{this->load()}; + while (!this->compare_exchange_strong(_Temp, _Temp ^ _Operand, _Order)) { // keep trying + } + + return _Temp; + } + + _TVal operator++(int) noexcept { + return fetch_add(static_cast<_TVal>(1)); + } + + _TVal operator++() noexcept { + return fetch_add(static_cast<_TVal>(1)) + static_cast<_TVal>(1); + } + + _TVal operator--(int) noexcept { + return fetch_add(static_cast<_TVal>(-1)); + } + + _TVal operator--() noexcept { + return fetch_add(static_cast<_TVal>(-1)) - static_cast<_TVal>(1); + } + +#else // ^^^ defined(_M_IX86) && defined(__clang__), LLVM-46595 / !defined(_M_IX86) || !defined(__clang__) vvv + _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + long long _Result; + _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedExchangeAdd64, + _Atomic_address_as(this->_Storage), static_cast(_Operand)); + return static_cast<_TVal>(_Result); + } + + _TVal fetch_and(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + long long _Result; + _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedAnd64, _Atomic_address_as(this->_Storage), + static_cast(_Operand)); + return static_cast<_TVal>(_Result); + } + + _TVal fetch_or(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + long long _Result; + _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedOr64, _Atomic_address_as(this->_Storage), + static_cast(_Operand)); + return static_cast<_TVal>(_Result); + } + + _TVal fetch_xor(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + long long _Result; + _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedXor64, _Atomic_address_as(this->_Storage), + static_cast(_Operand)); + return static_cast<_TVal>(_Result); + } + + _TVal operator++(int) noexcept { + unsigned long long _After = + static_cast(_InterlockedIncrement64(_Atomic_address_as(this->_Storage))); + --_After; + return static_cast<_TVal>(_After); + } + + _TVal operator++() noexcept { + return static_cast<_TVal>(_InterlockedIncrement64(_Atomic_address_as(this->_Storage))); + } + + _TVal operator--(int) noexcept { + unsigned long long _After = + static_cast(_InterlockedDecrement64(_Atomic_address_as(this->_Storage))); + ++_After; + return static_cast<_TVal>(_After); + } + + _TVal operator--() noexcept { + return static_cast<_TVal>(_InterlockedDecrement64(_Atomic_address_as(this->_Storage))); + } +#endif // ^^^ !defined(_M_IX86) || !defined(__clang__) ^^^ +}; + +#if 1 // TRANSITION, ABI +template +_INLINE_VAR constexpr bool _Is_always_lock_free = _TypeSize <= 8 && (_TypeSize & (_TypeSize - 1)) == 0; +#else // ^^^ don't break ABI / break ABI vvv +#if _ATOMIC_HAS_DCAS +template +_INLINE_VAR constexpr bool _Is_always_lock_free = _TypeSize <= 2 * sizeof(void*); +#else // ^^^ _ATOMIC_HAS_DCAS / !_ATOMIC_HAS_DCAS vvv +template +_INLINE_VAR constexpr bool _Is_always_lock_free = _TypeSize <= sizeof(void*); +#endif // _ATOMIC_HAS_DCAS +#endif // break ABI + +template > +_INLINE_VAR constexpr bool _Deprecate_non_lock_free_volatile = true; + +template +_CXX20_DEPRECATE_VOLATILE _INLINE_VAR constexpr bool _Deprecate_non_lock_free_volatile<_Ty, false> = true; + +// STRUCT TEMPLATE _Atomic_integral_facade +template +struct _Atomic_integral_facade : _Atomic_integral<_Ty> { + // provides operator overloads and other support for atomic integral specializations + using _Base = _Atomic_integral<_Ty>; + using _TVal = typename _Base::_TVal; + using difference_type = _TVal; + +#ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 + _Atomic_integral_facade() = default; + /* implicit */ constexpr _Atomic_integral_facade(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Base(_Value) {} +#else // ^^^ workaround / no workaround vvv + using _Base::_Base; +#endif // ^^^ no workaround ^^^ + + // _Deprecate_non_lock_free_volatile is unnecessary here. + + // note: const_cast-ing away volatile is safe because all our intrinsics add volatile back on. + // We make the primary functions non-volatile for better debug codegen, as non-volatile atomics + // are far more common than volatile ones. + using _Base::fetch_add; + _TVal fetch_add(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_add(_Operand); + } + + _TVal fetch_add(const _TVal _Operand, const memory_order _Order) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_add(_Operand, _Order); + } + + _NODISCARD static _TVal _Negate(const _TVal _Value) noexcept { // returns two's complement negated value of _Value + return static_cast<_TVal>(0U - static_cast>(_Value)); + } + + _TVal fetch_sub(const _TVal _Operand) noexcept { + return fetch_add(_Negate(_Operand)); + } + + _TVal fetch_sub(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return fetch_add(_Negate(_Operand)); + } + + _TVal fetch_sub(const _TVal _Operand, const memory_order _Order) noexcept { + return fetch_add(_Negate(_Operand), _Order); + } + + _TVal fetch_sub(const _TVal _Operand, const memory_order _Order) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return fetch_add(_Negate(_Operand), _Order); + } + + using _Base::fetch_and; + _TVal fetch_and(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_and(_Operand); + } + + _TVal fetch_and(const _TVal _Operand, const memory_order _Order) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_and(_Operand, _Order); + } + + using _Base::fetch_or; + _TVal fetch_or(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_or(_Operand); + } + + _TVal fetch_or(const _TVal _Operand, const memory_order _Order) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_or(_Operand, _Order); + } + + using _Base::fetch_xor; + _TVal fetch_xor(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_xor(_Operand); + } + + _TVal fetch_xor(const _TVal _Operand, const memory_order _Order) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_xor(_Operand, _Order); + } + + using _Base::operator++; + _TVal operator++(int) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return const_cast<_Atomic_integral_facade*>(this)->_Base::operator++(0); + } + + _TVal operator++() volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return const_cast<_Atomic_integral_facade*>(this)->_Base::operator++(); + } + + using _Base::operator--; + _TVal operator--(int) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return const_cast<_Atomic_integral_facade*>(this)->_Base::operator--(0); + } + + _TVal operator--() volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return const_cast<_Atomic_integral_facade*>(this)->_Base::operator--(); + } + + _TVal operator+=(const _TVal _Operand) noexcept { + return static_cast<_TVal>(this->_Base::fetch_add(_Operand) + _Operand); + } + + _TVal operator+=(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return static_cast<_TVal>(const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_add(_Operand) + _Operand); + } + + _TVal operator-=(const _TVal _Operand) noexcept { + return static_cast<_TVal>(fetch_sub(_Operand) - _Operand); + } + + _TVal operator-=(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return static_cast<_TVal>(const_cast<_Atomic_integral_facade*>(this)->fetch_sub(_Operand) - _Operand); + } + + _TVal operator&=(const _TVal _Operand) noexcept { + return static_cast<_TVal>(this->_Base::fetch_and(_Operand) & _Operand); + } + + _TVal operator&=(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return static_cast<_TVal>(const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_and(_Operand) & _Operand); + } + + _TVal operator|=(const _TVal _Operand) noexcept { + return static_cast<_TVal>(this->_Base::fetch_or(_Operand) | _Operand); + } + + _TVal operator|=(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return static_cast<_TVal>(const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_or(_Operand) | _Operand); + } + + _TVal operator^=(const _TVal _Operand) noexcept { + return static_cast<_TVal>(this->_Base::fetch_xor(_Operand) ^ _Operand); + } + + _TVal operator^=(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return static_cast<_TVal>(const_cast<_Atomic_integral_facade*>(this)->_Base::fetch_xor(_Operand) ^ _Operand); + } +}; + +#if _HAS_CXX20 +template +struct _Atomic_floating : _Atomic_storage<_Ty> { + // provides atomic floating-point operations + using _Base = _Atomic_storage<_Ty>; + using _TVal = typename _Base::_TVal; + using difference_type = _TVal; + +#ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 + _Atomic_floating() = default; + /* implicit */ constexpr _Atomic_floating(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Base(_Value) {} +#else // ^^^ workaround / no workaround vvv + using _Base::_Base; +#endif // ^^^ no workaround ^^^ + + _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal _Temp{this->load(memory_order_relaxed)}; + while (!this->compare_exchange_strong(_Temp, _Temp + _Operand, _Order)) { // keep trying + } + + return _Temp; + } + + // _Deprecate_non_lock_free_volatile is unnecessary here. + + // note: const_cast-ing away volatile is safe because all our intrinsics add volatile back on. + // We make the primary functions non-volatile for better debug codegen, as non-volatile atomics + // are far more common than volatile ones. + _TVal fetch_add(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return const_cast<_Atomic_floating*>(this)->fetch_add(_Operand, _Order); + } + + _TVal fetch_sub(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal _Temp{this->load(memory_order_relaxed)}; + while (!this->compare_exchange_strong(_Temp, _Temp - _Operand, _Order)) { // keep trying + } + + return _Temp; + } + + _TVal fetch_sub(const _TVal _Operand, const memory_order _Order = memory_order_seq_cst) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return const_cast<_Atomic_floating*>(this)->fetch_sub(_Operand, _Order); + } + + _TVal operator+=(const _TVal _Operand) noexcept { + return fetch_add(_Operand) + _Operand; + } + + _TVal operator+=(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return const_cast<_Atomic_floating*>(this)->fetch_add(_Operand) + _Operand; + } + + _TVal operator-=(const _TVal _Operand) noexcept { + return fetch_sub(_Operand) - _Operand; + } + + _TVal operator-=(const _TVal _Operand) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return const_cast<_Atomic_floating*>(this)->fetch_sub(_Operand) - _Operand; + } +}; +#endif // _HAS_CXX20 + +// STRUCT TEMPLATE _Atomic_pointer +template +struct _Atomic_pointer : _Atomic_storage<_Ty> { + using _Base = _Atomic_storage<_Ty>; + using _TVal = typename _Base::_TVal; + using difference_type = ptrdiff_t; + +#ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 + _Atomic_pointer() = default; + /* implicit */ constexpr _Atomic_pointer(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + : _Base(_Value) {} +#else // ^^^ workaround / no workaround vvv + using _Base::_Base; +#endif // ^^^ no workaround ^^^ + + _TVal fetch_add(const ptrdiff_t _Diff, const memory_order _Order = memory_order_seq_cst) noexcept { + const ptrdiff_t _Shift_bytes = + static_cast(static_cast(_Diff) * sizeof(remove_pointer_t<_TVal>)); + ptrdiff_t _Result; +#if defined(_M_IX86) || defined(_M_ARM) + _ATOMIC_CHOOSE_INTRINSIC( + _Order, _Result, _InterlockedExchangeAdd, _Atomic_address_as(this->_Storage), _Shift_bytes); +#else // ^^^ 32 bits / 64 bits vvv + _ATOMIC_CHOOSE_INTRINSIC( + _Order, _Result, _InterlockedExchangeAdd64, _Atomic_address_as(this->_Storage), _Shift_bytes); +#endif // hardware + return reinterpret_cast<_TVal>(_Result); + } + + // _Deprecate_non_lock_free_volatile is unnecessary here. + + _TVal fetch_add(const ptrdiff_t _Diff) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return const_cast<_Atomic_pointer*>(this)->fetch_add(_Diff); + } + + _TVal fetch_add(const ptrdiff_t _Diff, const memory_order _Order) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return const_cast<_Atomic_pointer*>(this)->fetch_add(_Diff, _Order); + } + + _TVal fetch_sub(const ptrdiff_t _Diff) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return fetch_add(static_cast(0 - static_cast(_Diff))); + } + + _TVal fetch_sub(const ptrdiff_t _Diff) noexcept { + return fetch_add(static_cast(0 - static_cast(_Diff))); + } + + _TVal fetch_sub(const ptrdiff_t _Diff, const memory_order _Order) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return fetch_add(static_cast(0 - static_cast(_Diff)), _Order); + } + + _TVal fetch_sub(const ptrdiff_t _Diff, const memory_order _Order) noexcept { + return fetch_add(static_cast(0 - static_cast(_Diff)), _Order); + } + + _TVal operator++(int) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return fetch_add(1); + } + + _TVal operator++(int) noexcept { + return fetch_add(1); + } + + _TVal operator++() volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return fetch_add(1) + 1; + } + + _TVal operator++() noexcept { + return fetch_add(1) + 1; + } + + _TVal operator--(int) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return fetch_add(-1); + } + + _TVal operator--(int) noexcept { + return fetch_add(-1); + } + + _TVal operator--() volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return fetch_add(-1) - 1; + } + + _TVal operator--() noexcept { + return fetch_add(-1) - 1; + } + + _TVal operator+=(const ptrdiff_t _Diff) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return fetch_add(_Diff) + _Diff; + } + + _TVal operator+=(const ptrdiff_t _Diff) noexcept { + return fetch_add(_Diff) + _Diff; + } + + _TVal operator-=(const ptrdiff_t _Diff) volatile noexcept { + static_assert(_Atomic_storage_types<_Ty>::_Has_volatile, "no volatile for atomic_ref"); + return fetch_add(static_cast(0 - static_cast(_Diff))) - _Diff; + } + + _TVal operator-=(const ptrdiff_t _Diff) noexcept { + return fetch_add(static_cast(0 - static_cast(_Diff))) - _Diff; + } +}; + +// STRUCT TEMPLATE atomic +#define ATOMIC_VAR_INIT(_Value) \ + { _Value } + +template +using _Choose_atomic_base2_t = + typename _Select && !is_same_v>::template _Apply<_Atomic_integral_facade<_Ty>, + typename _Select && is_object_v>>::template _Apply< + _Atomic_pointer<_Ty>, _Atomic_storage<_Ty>>>; + +#if _HAS_CXX20 +template +using _Choose_atomic_base_t = typename _Select>::template _Apply<_Atomic_floating<_Ty>, + _Choose_atomic_base2_t<_TVal, _Ty>>; +#else // ^^^ _HAS_CXX20 // !_HAS_CXX20 vvv +template +using _Choose_atomic_base_t = _Choose_atomic_base2_t<_TVal, _Ty>; +#endif //_HAS_CXX20 + +template +struct atomic : _Choose_atomic_base_t<_Ty> { // atomic value +private: + using _Base = _Choose_atomic_base_t<_Ty>; + +public: + // clang-format off + static_assert(is_trivially_copyable_v<_Ty> && is_copy_constructible_v<_Ty> && is_move_constructible_v<_Ty> + && is_copy_assignable_v<_Ty> && is_move_assignable_v<_Ty>, + "atomic requires T to be trivially copyable, copy constructible, move constructible, copy assignable, " + "and move assignable."); + // clang-format on + + using value_type = _Ty; + +#ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 + /* implicit */ constexpr atomic(const _Ty _Value) noexcept : _Base(_Value) {} +#else // ^^^ workaround / no workaround vvv + using _Base::_Base; +#endif // ^^^ no workaround ^^^ + + constexpr atomic() noexcept(is_nothrow_default_constructible_v<_Ty>) : _Base() {} + + atomic(const atomic&) = delete; + atomic& operator=(const atomic&) = delete; + +#if _HAS_CXX17 + static constexpr bool is_always_lock_free = _Is_always_lock_free; +#endif // _HAS_CXX17 + +#if 1 // TRANSITION, ABI + _NODISCARD bool is_lock_free() const volatile noexcept { + constexpr bool _Result = sizeof(_Ty) <= 8 && (sizeof(_Ty) & sizeof(_Ty) - 1) == 0; + return _Result; + } + +#else // ^^^ don't break ABI / break ABI vvv + + _NODISCARD bool is_lock_free() const volatile noexcept { +#if _ATOMIC_HAS_DCAS + return sizeof(_Ty) <= 2 * sizeof(void*); +#else // ^^^ _ATOMIC_HAS_DCAS / !_ATOMIC_HAS_DCAS vvv + return sizeof(_Ty) <= sizeof(void*) || (sizeof(_Ty) <= 2 * sizeof(void*) && __std_atomic_has_cmpxchg16b()); +#endif // _ATOMIC_HAS_DCAS + } +#endif // TRANSITION, ABI + + _NODISCARD bool is_lock_free() const noexcept { + return static_cast(this)->is_lock_free(); + } + + _Ty operator=(const _Ty _Value) volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + this->store(_Value); + return _Value; + } + + _Ty operator=(const _Ty _Value) noexcept { + this->store(_Value); + return _Value; + } + + // For the following, we do the real implementation in the non-volatile function, and const_cast + // to call the non-volatile function in the volatile one. This is safe because all of the + // non-volatile functions reapply volatile, as all our intrinsics accept only volatile T *. + // We expect most atomics to be non-volatile, so making the real implementations + // non-volatile should result in better debug codegen. + using _Base::store; + void store(const _Ty _Value) volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + const_cast(this)->_Base::store(_Value); + } + + void store(const _Ty _Value, const memory_order _Order) volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + const_cast(this)->_Base::store(_Value, _Order); + } + + using _Base::load; + _NODISCARD _Ty load() const volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return const_cast(this)->_Base::load(); + } + + _NODISCARD _Ty load(const memory_order _Order) const volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return const_cast(this)->_Base::load(_Order); + } + + using _Base::exchange; + _Ty exchange(const _Ty _Value) volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return const_cast(this)->_Base::exchange(_Value); + } + + _Ty exchange(const _Ty _Value, const memory_order _Order) volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return const_cast(this)->_Base::exchange(_Value, _Order); + } + + using _Base::compare_exchange_strong; + bool compare_exchange_strong(_Ty& _Expected, const _Ty _Desired) volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return const_cast(this)->_Base::compare_exchange_strong(_Expected, _Desired); + } + + bool compare_exchange_strong(_Ty& _Expected, const _Ty _Desired, const memory_order _Order) volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return const_cast(this)->_Base::compare_exchange_strong(_Expected, _Desired, _Order); + } + + bool compare_exchange_strong(_Ty& _Expected, const _Ty _Desired, const memory_order _Success, + const memory_order _Failure) volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return this->compare_exchange_strong(_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); + } + + bool compare_exchange_strong( + _Ty& _Expected, const _Ty _Desired, const memory_order _Success, const memory_order _Failure) noexcept { + return this->compare_exchange_strong(_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); + } + + bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired) volatile noexcept { + // we have no weak CAS intrinsics, even on ARM32/ARM64, so fall back to strong + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return this->compare_exchange_strong(_Expected, _Desired); + } + + bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired) noexcept { + return this->compare_exchange_strong(_Expected, _Desired); + } + + bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired, const memory_order _Order) volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return this->compare_exchange_strong(_Expected, _Desired, _Order); + } + + bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired, const memory_order _Order) noexcept { + return this->compare_exchange_strong(_Expected, _Desired, _Order); + } + + bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired, const memory_order _Success, + const memory_order _Failure) volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return this->compare_exchange_strong(_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); + } + + bool compare_exchange_weak( + _Ty& _Expected, const _Ty _Desired, const memory_order _Success, const memory_order _Failure) noexcept { + return this->compare_exchange_strong(_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); + } + + operator _Ty() const volatile noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return this->load(); + } + + operator _Ty() const noexcept { + return this->load(); + } +}; + +#if _HAS_CXX17 +template +atomic(_Ty) -> atomic<_Ty>; +#endif // _HAS_CXX17 + +#if _HAS_CXX20 +template +struct atomic_ref : _Choose_atomic_base_t<_Ty, _Ty&> { // atomic reference +private: + using _Base = _Choose_atomic_base_t<_Ty, _Ty&>; + +public: + // clang-format off + static_assert(is_trivially_copyable_v<_Ty> && is_copy_constructible_v<_Ty> && is_move_constructible_v<_Ty> + && is_copy_assignable_v<_Ty> && is_move_assignable_v<_Ty>, + "atomic requires T to be trivially copyable, copy constructible, move constructible, copy assignable, " + "and move assignable."); + // clang-format on + + using value_type = _Ty; + + explicit atomic_ref(_Ty& _Value) : _Base(_Value) { + if constexpr (_Is_potentially_lock_free) { + _Check_aligment(_Value); + } + +#if 1 // TRANSITION, ABI + if constexpr (!is_always_lock_free) { +#else + if constexpr (!_Is_potentially_lock_free) { +#endif + this->_Init_spinlock_for_ref(); + } + } + + atomic_ref(const atomic_ref&) noexcept = default; + + atomic_ref& operator=(const atomic_ref&) = delete; + + static constexpr bool is_always_lock_free = _Is_always_lock_free; + + static constexpr bool _Is_potentially_lock_free = + sizeof(_Ty) <= 2 * sizeof(void*) && (sizeof(_Ty) & (sizeof(_Ty) - 1)) == 0; + + static constexpr size_t required_alignment = _Is_potentially_lock_free ? sizeof(_Ty) : alignof(_Ty); + +#if 1 // TRANSITION, ABI + _NODISCARD bool is_lock_free() const noexcept { + constexpr bool _Result = sizeof(_Ty) <= 8 && (sizeof(_Ty) & sizeof(_Ty) - 1) == 0; + return _Result; + } + +#else // ^^^ don't break ABI / break ABI vvv + + _NODISCARD bool is_lock_free() const noexcept { +#if _ATOMIC_HAS_DCAS + return sizeof(_Ty) <= 2 * sizeof(void*); +#else // ^^^ _ATOMIC_HAS_DCAS / !_ATOMIC_HAS_DCAS vvv + return sizeof(_Ty) <= sizeof(void*) || (sizeof(_Ty) <= 2 * sizeof(void*) && __std_atomic_has_cmpxchg16b()); +#endif // _ATOMIC_HAS_DCAS + } +#endif // TRANSITION, ABI + + _Ty operator=(const _Ty _Value) noexcept { + this->store(_Value); + return _Value; + } + + using _Base::compare_exchange_strong; + bool compare_exchange_strong( + _Ty& _Expected, const _Ty _Desired, const memory_order _Success, const memory_order _Failure) noexcept { + return this->compare_exchange_strong(_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); + } + + bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired) noexcept { + return this->compare_exchange_strong(_Expected, _Desired); + } + + bool compare_exchange_weak(_Ty& _Expected, const _Ty _Desired, const memory_order _Order) noexcept { + return this->compare_exchange_strong(_Expected, _Desired, _Order); + } + + bool compare_exchange_weak( + _Ty& _Expected, const _Ty _Desired, const memory_order _Success, const memory_order _Failure) noexcept { + return this->compare_exchange_strong(_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); + } + + operator _Ty() const noexcept { + return this->load(); + } + +private: + void _Check_aligment([[maybe_unused]] const _Ty& _Value) { + _ATOMIC_REF_CHECK_ALIGNMENT( + (reinterpret_cast(_STD addressof(_Value)) & (required_alignment - 1)) == 0, + "atomic_ref underlying object is not aligned as required_alignment"); + } +}; +#endif // _HAS_CXX20 + +// NONMEMBER OPERATIONS ON ATOMIC TYPES +template +_NODISCARD bool atomic_is_lock_free(const volatile atomic<_Ty>* _Mem) noexcept { + return _Mem->is_lock_free(); +} + +template +_NODISCARD bool atomic_is_lock_free(const atomic<_Ty>* _Mem) noexcept { + return _Mem->is_lock_free(); +} + +template +_CXX20_DEPRECATE_ATOMIC_INIT void atomic_init( + atomic<_Ty>* const _Mem, const typename atomic<_Ty>::value_type _Value) noexcept { +#if 1 // TRANSITION, ABI + _CSTD memcpy(_STD addressof(_Mem->_Storage), _STD addressof(_Value), sizeof(_Ty)); +#else // ^^^ don't break ABI / break ABI vvv + _CSTD memcpy(_Mem, _STD addressof(_Value), sizeof(_Ty)); +#endif // TRANSITION, ABI + _CSTD memset(reinterpret_cast(_Mem) + sizeof(_Ty), 0, sizeof(atomic<_Ty>) - sizeof(_Ty)); +} + +template +_CXX20_DEPRECATE_ATOMIC_INIT void atomic_init( + volatile atomic<_Ty>* const _Mem, const typename atomic<_Ty>::value_type _Value) noexcept { + // NB: respecting volatility here appears unimplementable + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + _STD atomic_init(const_cast*>(_Mem), _Value); +} + +template +void atomic_store(volatile atomic<_Ty>* const _Mem, const _Identity_t<_Ty> _Value) noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + _Mem->store(_Value); +} + +template +void atomic_store(atomic<_Ty>* const _Mem, const _Identity_t<_Ty> _Value) noexcept { + _Mem->store(_Value); +} + +template +void atomic_store_explicit( + volatile atomic<_Ty>* const _Mem, const _Identity_t<_Ty> _Value, const memory_order _Order) noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + _Mem->store(_Value, _Order); +} + +template +void atomic_store_explicit(atomic<_Ty>* const _Mem, const _Identity_t<_Ty> _Value, const memory_order _Order) noexcept { + _Mem->store(_Value, _Order); +} + +template +_NODISCARD _Ty atomic_load(const volatile atomic<_Ty>* const _Mem) noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return _Mem->load(); +} + +template +_NODISCARD _Ty atomic_load(const atomic<_Ty>* const _Mem) noexcept { + return _Mem->load(); +} + +template +_NODISCARD _Ty atomic_load_explicit(const volatile atomic<_Ty>* const _Mem, const memory_order _Order) noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return _Mem->load(_Order); +} + +template +_NODISCARD _Ty atomic_load_explicit(const atomic<_Ty>* const _Mem, const memory_order _Order) noexcept { + return _Mem->load(_Order); +} + +template +_Ty atomic_exchange(volatile atomic<_Ty>* const _Mem, const _Identity_t<_Ty> _Value) noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return _Mem->exchange(_Value); +} + +template +_Ty atomic_exchange(atomic<_Ty>* const _Mem, const _Identity_t<_Ty> _Value) noexcept { + return _Mem->exchange(_Value); +} + +template +_Ty atomic_exchange_explicit( + volatile atomic<_Ty>* const _Mem, const _Identity_t<_Ty> _Value, const memory_order _Order) noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return _Mem->exchange(_Value, _Order); +} + +template +_Ty atomic_exchange_explicit( + atomic<_Ty>* const _Mem, const _Identity_t<_Ty> _Value, const memory_order _Order) noexcept { + return _Mem->exchange(_Value, _Order); +} + +template +bool atomic_compare_exchange_strong( + volatile atomic<_Ty>* const _Mem, _Identity_t<_Ty>* const _Expected, const _Identity_t<_Ty> _Desired) noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return _Mem->compare_exchange_strong(*_Expected, _Desired); +} + +template +bool atomic_compare_exchange_strong( + atomic<_Ty>* const _Mem, _Identity_t<_Ty>* const _Expected, const _Identity_t<_Ty> _Desired) noexcept { + return _Mem->compare_exchange_strong(*_Expected, _Desired); +} + +template +bool atomic_compare_exchange_strong_explicit(volatile atomic<_Ty>* const _Mem, _Identity_t<_Ty>* const _Expected, + const _Identity_t<_Ty> _Desired, const memory_order _Success, const memory_order _Failure) noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return _Mem->compare_exchange_strong(*_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); +} + +template +bool atomic_compare_exchange_strong_explicit(atomic<_Ty>* const _Mem, _Identity_t<_Ty>* const _Expected, + const _Identity_t<_Ty> _Desired, const memory_order _Success, const memory_order _Failure) noexcept { + return _Mem->compare_exchange_strong(*_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); +} + +template +bool atomic_compare_exchange_weak( + volatile atomic<_Ty>* const _Mem, _Identity_t<_Ty>* const _Expected, const _Identity_t<_Ty> _Desired) noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return _Mem->compare_exchange_strong(*_Expected, _Desired); +} + +template +bool atomic_compare_exchange_weak( + atomic<_Ty>* const _Mem, _Identity_t<_Ty>* const _Expected, const _Identity_t<_Ty> _Desired) noexcept { + return _Mem->compare_exchange_strong(*_Expected, _Desired); +} + +template +bool atomic_compare_exchange_weak_explicit(volatile atomic<_Ty>* const _Mem, _Identity_t<_Ty>* const _Expected, + const _Identity_t<_Ty> _Desired, const memory_order _Success, const memory_order _Failure) noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return _Mem->compare_exchange_strong(*_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); +} + +template +bool atomic_compare_exchange_weak_explicit(atomic<_Ty>* const _Mem, _Identity_t<_Ty>* const _Expected, + const _Identity_t<_Ty> _Desired, const memory_order _Success, const memory_order _Failure) noexcept { + return _Mem->compare_exchange_strong(*_Expected, _Desired, _Combine_cas_memory_orders(_Success, _Failure)); +} + +template +_Ty atomic_fetch_add(volatile atomic<_Ty>* _Mem, const typename atomic<_Ty>::difference_type _Value) noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return _Mem->fetch_add(_Value); +} + +template +_Ty atomic_fetch_add(atomic<_Ty>* _Mem, const typename atomic<_Ty>::difference_type _Value) noexcept { + return _Mem->fetch_add(_Value); +} + +template +_Ty atomic_fetch_add_explicit(volatile atomic<_Ty>* _Mem, const typename atomic<_Ty>::difference_type _Value, + const memory_order _Order) noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return _Mem->fetch_add(_Value, _Order); +} + +template +_Ty atomic_fetch_add_explicit( + atomic<_Ty>* _Mem, const typename atomic<_Ty>::difference_type _Value, const memory_order _Order) noexcept { + return _Mem->fetch_add(_Value, _Order); +} + +template +_Ty atomic_fetch_sub(volatile atomic<_Ty>* _Mem, const typename atomic<_Ty>::difference_type _Value) noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return _Mem->fetch_sub(_Value); +} + +template +_Ty atomic_fetch_sub(atomic<_Ty>* _Mem, const typename atomic<_Ty>::difference_type _Value) noexcept { + return _Mem->fetch_sub(_Value); +} + +template +_Ty atomic_fetch_sub_explicit(volatile atomic<_Ty>* _Mem, const typename atomic<_Ty>::difference_type _Value, + const memory_order _Order) noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return _Mem->fetch_sub(_Value, _Order); +} + +template +_Ty atomic_fetch_sub_explicit( + atomic<_Ty>* _Mem, const typename atomic<_Ty>::difference_type _Value, const memory_order _Order) noexcept { + return _Mem->fetch_sub(_Value, _Order); +} + +template +_Ty atomic_fetch_and(volatile atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value) noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return _Mem->fetch_and(_Value); +} + +template +_Ty atomic_fetch_and(atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value) noexcept { + return _Mem->fetch_and(_Value); +} + +template +_Ty atomic_fetch_and_explicit( + volatile atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value, const memory_order _Order) noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return _Mem->fetch_and(_Value, _Order); +} + +template +_Ty atomic_fetch_and_explicit( + atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value, const memory_order _Order) noexcept { + return _Mem->fetch_and(_Value, _Order); +} + +template +_Ty atomic_fetch_or(volatile atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value) noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return _Mem->fetch_or(_Value); +} + +template +_Ty atomic_fetch_or(atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value) noexcept { + return _Mem->fetch_or(_Value); +} + +template +_Ty atomic_fetch_or_explicit( + volatile atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value, const memory_order _Order) noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return _Mem->fetch_or(_Value, _Order); +} + +template +_Ty atomic_fetch_or_explicit( + atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value, const memory_order _Order) noexcept { + return _Mem->fetch_or(_Value, _Order); +} + +template +_Ty atomic_fetch_xor(volatile atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value) noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return _Mem->fetch_xor(_Value); +} + +template +_Ty atomic_fetch_xor(atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value) noexcept { + return _Mem->fetch_xor(_Value); +} + +template +_Ty atomic_fetch_xor_explicit( + volatile atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value, const memory_order _Order) noexcept { + static_assert(_Deprecate_non_lock_free_volatile<_Ty>, "Never fails"); + return _Mem->fetch_xor(_Value, _Order); +} + +template +_Ty atomic_fetch_xor_explicit( + atomic<_Ty>* _Mem, const typename atomic<_Ty>::value_type _Value, const memory_order _Order) noexcept { + return _Mem->fetch_xor(_Value, _Order); +} + +// ATOMIC TYPEDEFS +using atomic_bool = atomic; + +using atomic_char = atomic; +using atomic_schar = atomic; +using atomic_uchar = atomic; +using atomic_short = atomic; +using atomic_ushort = atomic; +using atomic_int = atomic; +using atomic_uint = atomic; +using atomic_long = atomic; +using atomic_ulong = atomic; +using atomic_llong = atomic; +using atomic_ullong = atomic; + +#ifdef __cpp_lib_char8_t +using atomic_char8_t = atomic; +#endif // __cpp_lib_char8_t +using atomic_char16_t = atomic; +using atomic_char32_t = atomic; +using atomic_wchar_t = atomic; + +using atomic_int8_t = atomic; +using atomic_uint8_t = atomic; +using atomic_int16_t = atomic; +using atomic_uint16_t = atomic; +using atomic_int32_t = atomic; +using atomic_uint32_t = atomic; +using atomic_int64_t = atomic; +using atomic_uint64_t = atomic; + +using atomic_int_least8_t = atomic; +using atomic_uint_least8_t = atomic; +using atomic_int_least16_t = atomic; +using atomic_uint_least16_t = atomic; +using atomic_int_least32_t = atomic; +using atomic_uint_least32_t = atomic; +using atomic_int_least64_t = atomic; +using atomic_uint_least64_t = atomic; + +using atomic_int_fast8_t = atomic; +using atomic_uint_fast8_t = atomic; +using atomic_int_fast16_t = atomic; +using atomic_uint_fast16_t = atomic; +using atomic_int_fast32_t = atomic; +using atomic_uint_fast32_t = atomic; +using atomic_int_fast64_t = atomic; +using atomic_uint_fast64_t = atomic; + +using atomic_intptr_t = atomic; +using atomic_uintptr_t = atomic; +using atomic_size_t = atomic; +using atomic_ptrdiff_t = atomic; +using atomic_intmax_t = atomic; +using atomic_uintmax_t = atomic; + +#if _HAS_CXX20 +// Though there are CMPXCHG8B and CMPXCHG16B, +// the largest atomics with a full set of efficient operations are pointer-sized. +using atomic_signed_lock_free = atomic_intptr_t; +using atomic_unsigned_lock_free = atomic_uintptr_t; +#endif // _HAS_CXX20 + +// STRUCT atomic_flag +#define ATOMIC_FLAG_INIT \ + {} +struct atomic_flag { // flag with test-and-set semantics +#if _HAS_CXX20 + _NODISCARD bool test(const memory_order _Order = memory_order_seq_cst) const noexcept { + return _Storage.load(_Order) != 0; + } + + _NODISCARD bool test(const memory_order _Order = memory_order_seq_cst) const volatile noexcept { + return _Storage.load(_Order) != 0; + } +#endif // _HAS_CXX20 + + bool test_and_set(const memory_order _Order = memory_order_seq_cst) noexcept { + return _Storage.exchange(true, _Order) != 0; + } + + bool test_and_set(const memory_order _Order = memory_order_seq_cst) volatile noexcept { + return _Storage.exchange(true, _Order) != 0; + } + + void clear(const memory_order _Order = memory_order_seq_cst) noexcept { + _Storage.store(false, _Order); + } + + void clear(const memory_order _Order = memory_order_seq_cst) volatile noexcept { + _Storage.store(false, _Order); + } + + constexpr atomic_flag() noexcept = default; + +#if 1 // TRANSITION, ABI + atomic _Storage; +#else // ^^^ don't break ABI / break ABI vvv + atomic _Storage; +#endif // TRANSITION, ABI +}; + +// atomic_flag NONMEMBERS +#if _HAS_CXX20 +_NODISCARD inline bool atomic_flag_test(const volatile atomic_flag* const _Flag) noexcept { + return _Flag->test(); +} + +_NODISCARD inline bool atomic_flag_test(const atomic_flag* const _Flag) noexcept { + return _Flag->test(); +} + +_NODISCARD inline bool atomic_flag_test_explicit( + const volatile atomic_flag* const _Flag, const memory_order _Order) noexcept { + return _Flag->test(_Order); +} + +_NODISCARD inline bool atomic_flag_test_explicit(const atomic_flag* const _Flag, const memory_order _Order) noexcept { + return _Flag->test(_Order); +} +#endif // _HAS_CXX20 + +inline bool atomic_flag_test_and_set(atomic_flag* _Flag) noexcept { + return _Flag->test_and_set(); +} + +inline bool atomic_flag_test_and_set(volatile atomic_flag* _Flag) noexcept { + return _Flag->test_and_set(); +} + +inline bool atomic_flag_test_and_set_explicit(atomic_flag* _Flag, memory_order _Order) noexcept { + return _Flag->test_and_set(_Order); +} + +inline bool atomic_flag_test_and_set_explicit(volatile atomic_flag* _Flag, memory_order _Order) noexcept { + return _Flag->test_and_set(_Order); +} + +inline void atomic_flag_clear(atomic_flag* _Flag) noexcept { + _Flag->clear(); +} + +inline void atomic_flag_clear(volatile atomic_flag* _Flag) noexcept { + _Flag->clear(); +} + +inline void atomic_flag_clear_explicit(atomic_flag* _Flag, memory_order _Order) noexcept { + _Flag->clear(_Order); +} + +inline void atomic_flag_clear_explicit(volatile atomic_flag* _Flag, memory_order _Order) noexcept { + _Flag->clear(_Order); +} + +_STD_END + +#undef _ATOMIC_CHOOSE_INTRINSIC +#undef _ATOMIC_HAS_DCAS + +#undef _STD_COMPARE_EXCHANGE_128 +#undef _INVALID_MEMORY_ORDER +#undef _Compiler_or_memory_barrier +#undef _Memory_barrier +#undef _Compiler_barrier + +#pragma pop_macro("new") +_STL_RESTORE_CLANG_WARNINGS +#pragma warning(pop) +#pragma pack(pop) +#endif // _STL_COMPILER_PREPROCESSOR +#endif // _ATOMIC_ From 5421ee909489029561c5ceedc376c769bb8f3fd7 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Fri, 10 Jul 2020 11:50:15 +0300 Subject: [PATCH 051/132] Fix merge error --- stl/inc/atomic | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 9b84e079f0e..6dc60ea6849 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -799,7 +799,7 @@ struct _Atomic_storage<_Ty, 8> { // lock-free using 8-byte intrinsics } #if defined(_M_IX86) && defined(__clang__) // TRANSITION, LLVM-46595 - TVal exchange(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { + _TVal exchange(const _TVal _Value, const memory_order _Order = memory_order_seq_cst) noexcept { // exchange with (effectively) sequential consistency _TVal _Temp{load()}; while (!compare_exchange_strong(_Temp, _Value, _Order)) { // keep trying From 5992ce57e23f5c8438688b0dcbfa6ddf9db68947 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Wed, 15 Jul 2020 07:50:45 +0300 Subject: [PATCH 052/132] Update stl/inc/atomic Co-authored-by: Billy O'Neal --- stl/inc/atomic | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 6dc60ea6849..730b6c868a1 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -1805,7 +1805,8 @@ public: // clang-format off static_assert(is_trivially_copyable_v<_Ty> && is_copy_constructible_v<_Ty> && is_move_constructible_v<_Ty> && is_copy_assignable_v<_Ty> && is_move_assignable_v<_Ty>, - "atomic requires T to be trivially copyable, copy constructible, move constructible, copy assignable, " + "atomic_ref requires T to be trivially copyable, copy constructible, move constructible, copy assignable, " + "and move assignable."); // clang-format on From a08ebbfeb8dc700e2e0279a5588ce4ef1dfcb468 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Fri, 17 Jul 2020 20:31:28 +0300 Subject: [PATCH 053/132] revive atomic_ref<16 bytes> --- stl/inc/atomic | 31 ++++++--------------- stl/src/atomic_ref.cpp | 17 +++++++++++ stl/src/msvcp_atomic_ref.def | 2 ++ tests/std/tests/P0019R8_atomic_ref/test.cpp | 13 +++++++++ 4 files changed, 40 insertions(+), 23 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 6dc60ea6849..0b34505ffad 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -45,7 +45,6 @@ _STL_DISABLE_CLANG_WARNINGS #endif // _DEBUG #endif // _INVALID_MEMORY_ORDER -#if 0 // TRANSITION, ABI // MACRO _STD_COMPARE_EXCHANGE_128 #if _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B || defined(_M_ARM64) #define _STD_COMPARE_EXCHANGE_128 _InterlockedCompareExchange128 @@ -56,10 +55,10 @@ _STL_DISABLE_CLANG_WARNINGS // CRITICAL_SECTION shared by all 16-byte atomics. // (Note: machines without this instruction typically have 2 cores or fewer, so this isn't too bad) // All pointer parameters must be 16-byte aligned. -_NODISCARD extern "C" unsigned char __cdecl __std_atomic_compare_exchange_128( +extern "C" _NODISCARD unsigned char __stdcall __std_atomic_compare_exchange_128( _Inout_bytecount_(16) long long* _Destination, _In_ long long _ExchangeHigh, _In_ long long _ExchangeLow, _Inout_bytecount_(16) long long* _ComparandResult) noexcept; -_NODISCARD extern "C" bool __cdecl __std_atomic_has_cmpxchg16b() noexcept; +extern "C" _NODISCARD bool __stdcall __std_atomic_has_cmpxchg16b() noexcept; #define _STD_COMPARE_EXCHANGE_128 __std_atomic_compare_exchange_128 #endif // defined(_M_X64) && !_STD_ATOMIC_ALWAYS_USE_CMPXCHG16B @@ -70,7 +69,6 @@ _NODISCARD extern "C" bool __cdecl __std_atomic_has_cmpxchg16b() noexcept; #else // ^^ We always have DCAS / We only sometimes have DCAS vvv #define _ATOMIC_HAS_DCAS 0 #endif // _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B || !defined(_M_X64) -#endif // TRANSITION, ABI // MACRO _ATOMIC_CHOOSE_INTRINSIC #if defined(_M_IX86) || defined(_M_X64) @@ -834,16 +832,17 @@ struct _Atomic_storage<_Ty, 8> { // lock-free using 8-byte intrinsics typename _Atomic_storage_types<_Ty>::_TStorage _Storage; }; -#if 0 // TRANSITION, ABI + #if defined(_M_X64) || defined(_M_ARM64) template -struct _Atomic_storage<_Ty, 16> { // lock-free using 16-byte intrinsics +struct _Atomic_storage<_Ty&, 16> { // lock-free using 16-byte intrinsics + // TRANSITION, ABI: replace '_Ty&' with '_Ty' in parameter of everywhere in this specialization - using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; + using _TVal = typename _Atomic_storage_types<_Ty&>::_TVal; _Atomic_storage() = default; - /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty>::_TConstr _Value) noexcept + /* implicit */ constexpr _Atomic_storage(typename _Atomic_storage_types<_Ty&>::_TConstr _Value) noexcept : _Storage{_Value} {} // non-atomically initialize this atomic void store(const _TVal _Value) noexcept { // store with sequential consistency @@ -934,10 +933,9 @@ struct _Atomic_storage<_Ty, 16> { // lock-free using 16-byte intrinsics long long _High; }; - typename _Atomic_storage_types<_Ty>::_TStorage _Storage; + typename _Atomic_storage_types<_Ty&>::_TStorage _Storage; }; #endif // defined(_M_X64) || defined(_M_ARM64) -#endif // TRANSITION, ABI // STRUCT TEMPLATE _Atomic_integral template @@ -1816,11 +1814,7 @@ public: _Check_aligment(_Value); } -#if 1 // TRANSITION, ABI - if constexpr (!is_always_lock_free) { -#else if constexpr (!_Is_potentially_lock_free) { -#endif this->_Init_spinlock_for_ref(); } } @@ -1836,14 +1830,6 @@ public: static constexpr size_t required_alignment = _Is_potentially_lock_free ? sizeof(_Ty) : alignof(_Ty); -#if 1 // TRANSITION, ABI - _NODISCARD bool is_lock_free() const noexcept { - constexpr bool _Result = sizeof(_Ty) <= 8 && (sizeof(_Ty) & sizeof(_Ty) - 1) == 0; - return _Result; - } - -#else // ^^^ don't break ABI / break ABI vvv - _NODISCARD bool is_lock_free() const noexcept { #if _ATOMIC_HAS_DCAS return sizeof(_Ty) <= 2 * sizeof(void*); @@ -1851,7 +1837,6 @@ public: return sizeof(_Ty) <= sizeof(void*) || (sizeof(_Ty) <= 2 * sizeof(void*) && __std_atomic_has_cmpxchg16b()); #endif // _ATOMIC_HAS_DCAS } -#endif // TRANSITION, ABI _Ty operator=(const _Ty _Value) noexcept { this->store(_Value); diff --git a/stl/src/atomic_ref.cpp b/stl/src/atomic_ref.cpp index 4b7cf62d1b3..f3987d59239 100644 --- a/stl/src/atomic_ref.cpp +++ b/stl/src/atomic_ref.cpp @@ -6,6 +6,8 @@ #include #include +#include + _EXTERN_C long* __stdcall __std_atomic_get_mutex(const void* const _Key) noexcept { constexpr size_t _Table_size_power = 8; @@ -23,4 +25,19 @@ long* __stdcall __std_atomic_get_mutex(const void* const _Key) noexcept { _Index ^= _Index >> _Table_size_power; return &_Table[_Index & _Table_index_mask]._Mutex; } + +#if defined(_M_X64) + +_NODISCARD unsigned char __stdcall __std_atomic_compare_exchange_128( + _Inout_bytecount_(16) long long* _Destination, _In_ long long _ExchangeHigh, _In_ long long _ExchangeLow, + _Inout_bytecount_(16) long long* _ComparandResult) noexcept { + return _InterlockedCompareExchange128(_Destination, _ExchangeHigh, _ExchangeLow, _ComparandResult); +} + +_NODISCARD bool __stdcall __std_atomic_has_cmpxchg16b() noexcept { + return true; +} + +#endif + _END_EXTERN_C diff --git a/stl/src/msvcp_atomic_ref.def b/stl/src/msvcp_atomic_ref.def index f1d1c361b80..f423d63798c 100644 --- a/stl/src/msvcp_atomic_ref.def +++ b/stl/src/msvcp_atomic_ref.def @@ -5,3 +5,5 @@ EXPORTS __std_atomic_get_mutex + __std_atomic_compare_exchange_128 + __std_atomic_has_cmpxchg16b diff --git a/tests/std/tests/P0019R8_atomic_ref/test.cpp b/tests/std/tests/P0019R8_atomic_ref/test.cpp index 14834dd15b2..cae306b0f07 100644 --- a/tests/std/tests/P0019R8_atomic_ref/test.cpp +++ b/tests/std/tests/P0019R8_atomic_ref/test.cpp @@ -19,6 +19,18 @@ struct bigint { } }; +struct int128 { + long long value; + long long more_value; + + int128(int value = 0) : value(value), more_value(0) {} + + operator int() const { + return static_cast(value); + } +}; + + // code reuse of ../P1135R6_atomic_flag_test/test.cpp template @@ -102,4 +114,5 @@ int main() { test_ops(); test_ops(); test_ops(); + test_ops(); } From 5bf9c939d4d940eb031804b4d19b2fef64234d50 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Fri, 17 Jul 2020 20:46:27 +0300 Subject: [PATCH 054/132] good mutex --- stl/inc/atomic | 35 ++++++++++++++++++----------------- stl/src/atomic_ref.cpp | 10 +++++----- 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 0bfe9f5b008..3ee18f033e6 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -17,6 +17,7 @@ #include #include #include +#include #pragma pack(push, _CRT_PACKING) #pragma warning(push, _STL_WARNING_LEVEL) @@ -114,7 +115,7 @@ extern "C" _NODISCARD bool __stdcall __std_atomic_has_cmpxchg16b() noexcept; #define ATOMIC_POINTER_LOCK_FREE 2 _EXTERN_C -long* __stdcall __std_atomic_get_mutex(const void* _Key) noexcept; +_Smtx_t* __stdcall __std_atomic_get_mutex(const void* _Key) noexcept; _END_EXTERN_C _STD_BEGIN @@ -328,10 +329,6 @@ struct _Atomic_storage_types { using _Spinlock = long; static constexpr bool _Has_volatile = true; - - static long* _Get_spinlock(long& _Spinlock) { - return &_Spinlock; - } }; template @@ -339,13 +336,9 @@ struct _Atomic_storage_types<_Ty&> { using _TVal = _Ty; using _TStorage = _Ty&; using _TConstr = _Ty&; - using _Spinlock = long*; + using _Spinlock = _Smtx_t*; // POINTER TO mutex static constexpr bool _Has_volatile = false; - - static long* _Get_spinlock(long* _Spinlock) { - return _Spinlock; - } }; // STRUCT TEMPLATE _Atomic_storage @@ -415,19 +408,27 @@ struct _Atomic_storage { #if 1 // TRANSITION, ABI void _Lock() const noexcept { // lock the spinlock - while (_InterlockedExchange(_Atomic_storage_types<_Ty>::_Get_spinlock(_Spinlock), 1)) { - _YIELD_PROCESSOR(); + if constexpr (!is_reference_v<_Ty>) { + while (_InterlockedExchange(&_Spinlock, 1)) { + _YIELD_PROCESSOR(); + } + } else { + _Smtx_lock_exclusive(_Spinlock); } } void _Unlock() const noexcept { // unlock the spinlock + if constexpr (!is_reference_v<_Ty>) { #if defined(_M_ARM) || defined(_M_ARM64) - _Memory_barrier(); - __iso_volatile_store32(reinterpret_cast(_Atomic_storage_types<_Ty>::_Get_spinlock(_Spinlock)), 0); - _Memory_barrier(); + _Memory_barrier(); + __iso_volatile_store32(reinterpret_cast(&_Spinlock), 0); + _Memory_barrier(); #else // ^^^ ARM32/ARM64 hardware / x86/x64 hardware vvv - _InterlockedExchange(_Atomic_storage_types<_Ty>::_Get_spinlock(_Spinlock), 0); + _InterlockedExchange(&_Spinlock, 0); #endif // hardware + } else { + _Smtx_unlock_exclusive(_Spinlock); + } } protected: @@ -436,7 +437,7 @@ protected: } private: - // Spinlock for non-lock-free atomic. Spinlock pointer for non-lock-free atomic_ref + // Spinlock integer for non-lock-free atomic. mutex pointer for non-lock-free atomic_ref mutable typename _Atomic_storage_types<_Ty>::_Spinlock _Spinlock = 0; public: diff --git a/stl/src/atomic_ref.cpp b/stl/src/atomic_ref.cpp index f3987d59239..5cf4e794e64 100644 --- a/stl/src/atomic_ref.cpp +++ b/stl/src/atomic_ref.cpp @@ -4,19 +4,19 @@ // implement atomic_ref spin lock #include +#include #include - -#include +#include _EXTERN_C -long* __stdcall __std_atomic_get_mutex(const void* const _Key) noexcept { +_Smtx_t* __stdcall __std_atomic_get_mutex(const void* const _Key) noexcept { constexpr size_t _Table_size_power = 8; constexpr size_t _Table_size = 1 << _Table_size_power; constexpr size_t _Table_index_mask = _Table_size - 1; struct alignas(std::hardware_destructive_interference_size) _Table_entry { - long _Mutex = 0; - char _Pad[std::hardware_destructive_interference_size - sizeof(long)] = {}; + _Smtx_t _Mutex = 0; + char _Pad[std::hardware_destructive_interference_size - sizeof(_Smtx_t)] = {}; }; static _Table_entry _Table[_Table_size]; From 886824643af693f4ad0eafd391562e4aca62191c Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Fri, 17 Jul 2020 20:48:10 +0300 Subject: [PATCH 055/132] clang format --- stl/src/atomic_ref.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stl/src/atomic_ref.cpp b/stl/src/atomic_ref.cpp index 5cf4e794e64..132bd0dba56 100644 --- a/stl/src/atomic_ref.cpp +++ b/stl/src/atomic_ref.cpp @@ -15,7 +15,7 @@ _Smtx_t* __stdcall __std_atomic_get_mutex(const void* const _Key) noexcept { constexpr size_t _Table_index_mask = _Table_size - 1; struct alignas(std::hardware_destructive_interference_size) _Table_entry { - _Smtx_t _Mutex = 0; + _Smtx_t _Mutex = 0; char _Pad[std::hardware_destructive_interference_size - sizeof(_Smtx_t)] = {}; }; static _Table_entry _Table[_Table_size]; @@ -28,12 +28,12 @@ _Smtx_t* __stdcall __std_atomic_get_mutex(const void* const _Key) noexcept { #if defined(_M_X64) -_NODISCARD unsigned char __stdcall __std_atomic_compare_exchange_128( - _Inout_bytecount_(16) long long* _Destination, _In_ long long _ExchangeHigh, _In_ long long _ExchangeLow, +_NODISCARD unsigned char __stdcall __std_atomic_compare_exchange_128(_Inout_bytecount_(16) long long* _Destination, + _In_ long long _ExchangeHigh, _In_ long long _ExchangeLow, _Inout_bytecount_(16) long long* _ComparandResult) noexcept { return _InterlockedCompareExchange128(_Destination, _ExchangeHigh, _ExchangeLow, _ComparandResult); } - + _NODISCARD bool __stdcall __std_atomic_has_cmpxchg16b() noexcept { return true; } From f35df6b0d0bb9f77f9a835238453cb2d127c6ecc Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Fri, 17 Jul 2020 21:21:21 +0300 Subject: [PATCH 056/132] fix build --- stl/src/atomic_ref.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/stl/src/atomic_ref.cpp b/stl/src/atomic_ref.cpp index 132bd0dba56..1198cf2279e 100644 --- a/stl/src/atomic_ref.cpp +++ b/stl/src/atomic_ref.cpp @@ -26,18 +26,21 @@ _Smtx_t* __stdcall __std_atomic_get_mutex(const void* const _Key) noexcept { return &_Table[_Index & _Table_index_mask]._Mutex; } -#if defined(_M_X64) + _NODISCARD unsigned char __stdcall __std_atomic_compare_exchange_128(_Inout_bytecount_(16) long long* _Destination, _In_ long long _ExchangeHigh, _In_ long long _ExchangeLow, _Inout_bytecount_(16) long long* _ComparandResult) noexcept { +#if defined(_M_X64) || defined(_M_ARM64) return _InterlockedCompareExchange128(_Destination, _ExchangeHigh, _ExchangeLow, _ComparandResult); +#else + __debugbreak(); + return 0; +#endif } _NODISCARD bool __stdcall __std_atomic_has_cmpxchg16b() noexcept { return true; } -#endif - _END_EXTERN_C From 64d364e1ff9f1526b5577cce545b8169029d9e04 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Fri, 17 Jul 2020 21:27:10 +0300 Subject: [PATCH 057/132] clang format --- stl/src/atomic_ref.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/stl/src/atomic_ref.cpp b/stl/src/atomic_ref.cpp index 1198cf2279e..efd98552338 100644 --- a/stl/src/atomic_ref.cpp +++ b/stl/src/atomic_ref.cpp @@ -26,8 +26,6 @@ _Smtx_t* __stdcall __std_atomic_get_mutex(const void* const _Key) noexcept { return &_Table[_Index & _Table_index_mask]._Mutex; } - - _NODISCARD unsigned char __stdcall __std_atomic_compare_exchange_128(_Inout_bytecount_(16) long long* _Destination, _In_ long long _ExchangeHigh, _In_ long long _ExchangeLow, _Inout_bytecount_(16) long long* _ComparandResult) noexcept { From 2b7bc842268ff14e8d780360060437c50ca4362c Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Fri, 17 Jul 2020 21:33:27 +0300 Subject: [PATCH 058/132] fix build --- stl/src/atomic_ref.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/stl/src/atomic_ref.cpp b/stl/src/atomic_ref.cpp index efd98552338..3ddc79908ad 100644 --- a/stl/src/atomic_ref.cpp +++ b/stl/src/atomic_ref.cpp @@ -30,15 +30,21 @@ _NODISCARD unsigned char __stdcall __std_atomic_compare_exchange_128(_Inout_byte _In_ long long _ExchangeHigh, _In_ long long _ExchangeLow, _Inout_bytecount_(16) long long* _ComparandResult) noexcept { #if defined(_M_X64) || defined(_M_ARM64) + // TODO: Fallback return _InterlockedCompareExchange128(_Destination, _ExchangeHigh, _ExchangeLow, _ComparandResult); #else + (void) _Destination, _ExchangeHigh, _ExchangeLow, _ComparandResult; __debugbreak(); return 0; #endif } _NODISCARD bool __stdcall __std_atomic_has_cmpxchg16b() noexcept { - return true; +#if defined(_M_X64) || defined(_M_ARM64) + return true; // TODO: Detect +#else + return false; +#endif } _END_EXTERN_C From 58295410f332fd83fff0da2f4804c576f1054541 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Fri, 17 Jul 2020 22:45:55 +0300 Subject: [PATCH 059/132] implement fallback --- stl/src/atomic_ref.cpp | 64 ++++++++++++++++++--- tests/std/tests/P0019R8_atomic_ref/test.cpp | 2 + 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/stl/src/atomic_ref.cpp b/stl/src/atomic_ref.cpp index 3ddc79908ad..e02b7e05339 100644 --- a/stl/src/atomic_ref.cpp +++ b/stl/src/atomic_ref.cpp @@ -3,9 +3,11 @@ // implement atomic_ref spin lock +#include #include #include #include +#include #include _EXTERN_C @@ -26,22 +28,70 @@ _Smtx_t* __stdcall __std_atomic_get_mutex(const void* const _Key) noexcept { return &_Table[_Index & _Table_index_mask]._Mutex; } +namespace { + + _NODISCARD unsigned char __std_atomic_compare_exchange_128_fallback(_Inout_bytecount_(16) long long* _Destination, + _In_ long long _ExchangeHigh, _In_ long long _ExchangeLow, + _Inout_bytecount_(16) long long* _ComparandResult) noexcept { + static SRWLOCK _Mtx = SRWLOCK_INIT; + AcquireSRWLockExclusive(&_Mtx); + if (_Destination[0] == _ComparandResult[0] && _Destination[1] == _ComparandResult[1]) { + _ComparandResult[0] = _Destination[0]; + _ComparandResult[1] = _Destination[1]; + _Destination[0] = _ExchangeLow; + _Destination[1] = _ExchangeHigh; + ReleaseSRWLockExclusive(&_Mtx); + return true; + } else { + _ComparandResult[0] = _Destination[0]; + _ComparandResult[1] = _Destination[1]; + ReleaseSRWLockExclusive(&_Mtx); + return false; + } + } + +} // unnamed namespace + _NODISCARD unsigned char __stdcall __std_atomic_compare_exchange_128(_Inout_bytecount_(16) long long* _Destination, _In_ long long _ExchangeHigh, _In_ long long _ExchangeLow, _Inout_bytecount_(16) long long* _ComparandResult) noexcept { -#if defined(_M_X64) || defined(_M_ARM64) - // TODO: Fallback +#if defined(_M_X64) + if (__std_atomic_has_cmpxchg16b()) { + return _InterlockedCompareExchange128(_Destination, _ExchangeHigh, _ExchangeLow, _ComparandResult); + } else { + return __std_atomic_compare_exchange_128_fallback(_Destination, _ExchangeHigh, _ExchangeLow, _ComparandResult); + } +#elif defined(_M_ARM64) return _InterlockedCompareExchange128(_Destination, _ExchangeHigh, _ExchangeLow, _ComparandResult); #else - (void) _Destination, _ExchangeHigh, _ExchangeLow, _ComparandResult; - __debugbreak(); - return 0; + return __std_atomic_compare_exchange_128_fallback(_Destination, _ExchangeHigh, _ExchangeLow, _ComparandResult); #endif } _NODISCARD bool __stdcall __std_atomic_has_cmpxchg16b() noexcept { -#if defined(_M_X64) || defined(_M_ARM64) - return true; // TODO: Detect +#if defined(_M_X64) + enum class _Cmpxchg16_support : char { + _Absent = false, + _Present = true, + _Unknown, + }; + static std::atomic<_Cmpxchg16_support> _Cached_value{_Cmpxchg16_support::_Unknown}; + + _Cmpxchg16_support _Value = _Cached_value.load(std::memory_order_relaxed); + + if (_Value == _Cmpxchg16_support::_Unknown) { + int regs[4]; + __cpuid(regs, 1); // assume leaf 1 exists + if (regs[2] & (1 << 13)) { + _Value = _Cmpxchg16_support::_Present; + } else { + _Value = _Cmpxchg16_support::_Absent; + } + _Cached_value.store(_Value, std::memory_order_relaxed); + } + return reinterpret_cast(_Value); +#elif defined(_M_ARM64) + return true; #else return false; #endif diff --git a/tests/std/tests/P0019R8_atomic_ref/test.cpp b/tests/std/tests/P0019R8_atomic_ref/test.cpp index cae306b0f07..96461e9f940 100644 --- a/tests/std/tests/P0019R8_atomic_ref/test.cpp +++ b/tests/std/tests/P0019R8_atomic_ref/test.cpp @@ -83,6 +83,8 @@ void test_ops() { assert(std::transform_reduce(par, refs.begin(), refs.end(), 0, std::plus{}, load) == 0); } +#include + int main() { test_ops(); test_ops(); From 68b134f87d26bbb29f2e9bc3b8bb1add3f20c6f9 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Fri, 17 Jul 2020 22:47:18 +0300 Subject: [PATCH 060/132] try to fix tests --- stl/inc/atomic | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 3ee18f033e6..c69da6f94cf 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -47,10 +47,10 @@ _STL_DISABLE_CLANG_WARNINGS #endif // _INVALID_MEMORY_ORDER // MACRO _STD_COMPARE_EXCHANGE_128 -#if _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B || defined(_M_ARM64) +#if defined(_STD_ATOMIC_ALWAYS_USE_CMPXCHG16B) || defined(_M_ARM64) #define _STD_COMPARE_EXCHANGE_128 _InterlockedCompareExchange128 -#endif // _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B || defined(_M_ARM64) -#if defined(_M_X64) && !_STD_ATOMIC_ALWAYS_USE_CMPXCHG16B +#endif // defined(_STD_ATOMIC_ALWAYS_USE_CMPXCHG16B) || defined(_M_ARM64) +#if defined(_M_X64) && !defined(_STD_ATOMIC_ALWAYS_USE_CMPXCHG16B) // 16-byte atomics are separately compiled for x64, as not all x64 hardware has the cmpxchg16b // instruction; in the event this instruction is not available, the fallback is a global // CRITICAL_SECTION shared by all 16-byte atomics. @@ -61,15 +61,15 @@ extern "C" _NODISCARD unsigned char __stdcall __std_atomic_compare_exchange_128( _Inout_bytecount_(16) long long* _ComparandResult) noexcept; extern "C" _NODISCARD bool __stdcall __std_atomic_has_cmpxchg16b() noexcept; #define _STD_COMPARE_EXCHANGE_128 __std_atomic_compare_exchange_128 -#endif // defined(_M_X64) && !_STD_ATOMIC_ALWAYS_USE_CMPXCHG16B +#endif // defined(_M_X64) && !defined(_STD_ATOMIC_ALWAYS_USE_CMPXCHG16B) // MACRO _ATOMIC_HAS_DCAS // Controls whether atomic::is_always_lock_free triggers for sizeof(void *) or 2 * sizeof(void *) -#if _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B || !defined(_M_X64) +#if defined(_STD_ATOMIC_ALWAYS_USE_CMPXCHG16B) || !defined(_M_X64) #define _ATOMIC_HAS_DCAS 1 #else // ^^ We always have DCAS / We only sometimes have DCAS vvv #define _ATOMIC_HAS_DCAS 0 -#endif // _STD_ATOMIC_ALWAYS_USE_CMPXCHG16B || !defined(_M_X64) +#endif // defined(_STD_ATOMIC_ALWAYS_USE_CMPXCHG16B) || !defined(_M_X64) // MACRO _ATOMIC_CHOOSE_INTRINSIC #if defined(_M_IX86) || defined(_M_X64) @@ -408,17 +408,23 @@ struct _Atomic_storage { #if 1 // TRANSITION, ABI void _Lock() const noexcept { // lock the spinlock +#if _HAS_CXX20 if constexpr (!is_reference_v<_Ty>) { +#endif while (_InterlockedExchange(&_Spinlock, 1)) { _YIELD_PROCESSOR(); } +#if _HAS_CXX20 } else { _Smtx_lock_exclusive(_Spinlock); } +#endif } void _Unlock() const noexcept { // unlock the spinlock +#if _HAS_CXX20 if constexpr (!is_reference_v<_Ty>) { +#endif #if defined(_M_ARM) || defined(_M_ARM64) _Memory_barrier(); __iso_volatile_store32(reinterpret_cast(&_Spinlock), 0); @@ -426,9 +432,11 @@ struct _Atomic_storage { #else // ^^^ ARM32/ARM64 hardware / x86/x64 hardware vvv _InterlockedExchange(&_Spinlock, 0); #endif // hardware +#if _HAS_CXX20 } else { _Smtx_unlock_exclusive(_Spinlock); } +#endif } protected: From c9a3baac83604e966aa28c5d6ab0d6c57555e95e Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Fri, 17 Jul 2020 23:05:13 +0300 Subject: [PATCH 061/132] go home clang format you're drunk --- stl/inc/atomic | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stl/inc/atomic b/stl/inc/atomic index c69da6f94cf..58fbaaa3161 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -421,6 +421,7 @@ struct _Atomic_storage { #endif } + // clang-format off void _Unlock() const noexcept { // unlock the spinlock #if _HAS_CXX20 if constexpr (!is_reference_v<_Ty>) { @@ -438,6 +439,7 @@ struct _Atomic_storage { } #endif } + // clang-format on protected: void _Init_spinlock_for_ref() noexcept { From 081890dd4f49d8402e3b812bd49478d845bb28a2 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 25 Jul 2020 13:03:12 +0300 Subject: [PATCH 062/132] merge typo fix --- stl/inc/atomic | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index d1dc807a360..9689b6631ef 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -443,7 +443,7 @@ struct _Atomic_storage { _Lock(); #if _CMPXCHG_MASK_OUT_PADDING_BITS if constexpr (_Might_have_non_value_bits<_TVal>) { - _Storage_for _Local; + _Storage_for<_TVal> _Local; const auto _Local_ptr = _Local._Ptr(); _CSTD memcpy(_Local_ptr, _Storage_ptr, sizeof(_TVal)); __builtin_zero_non_value_bits(_Local_ptr); From a9130fec802362e43fb887c941361e2f38bcd800 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 25 Jul 2020 13:16:36 +0300 Subject: [PATCH 063/132] merge: fix _Ty with _TVal in padding bits part --- stl/inc/atomic | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 9689b6631ef..9985599cfb9 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -609,8 +609,8 @@ struct _Atomic_storage<_Ty, 1> { // lock-free using 1-byte intrinsics char _Prev_bytes; #if _CMPXCHG_MASK_OUT_PADDING_BITS - if constexpr (_Might_have_non_value_bits<_Ty>) { - _Storage_for<_Ty> _Mask{_Form_mask}; + if constexpr (_Might_have_non_value_bits<_TVal>) { + _Storage_for<_TVal> _Mask{_Form_mask}; const char _Mask_val = _Atomic_reinterpret_as(_Mask._Ref()); for (;;) { @@ -716,7 +716,7 @@ struct _Atomic_storage<_Ty, 2> { // lock-free using 2-byte intrinsics short _Prev_bytes; #if _CMPXCHG_MASK_OUT_PADDING_BITS if constexpr (_Might_have_non_value_bits<_Ty>) { - _Storage_for<_Ty> _Mask{_Form_mask}; + _Storage_for<_TVal> _Mask{_Form_mask}; const short _Mask_val = _Atomic_reinterpret_as(_Mask._Ref()); for (;;) { @@ -727,7 +727,7 @@ struct _Atomic_storage<_Ty, 2> { // lock-free using 2-byte intrinsics } if ((_Prev_bytes ^ _Expected_bytes) & _Mask_val) { - _CSTD memcpy(_STD addressof(_Expected), &_Prev_bytes, sizeof(_Ty)); + _CSTD memcpy(_STD addressof(_Expected), &_Prev_bytes, sizeof(_TVal)); return false; } _Expected_bytes = (_Expected_bytes & _Mask_val) | (_Prev_bytes & ~_Mask_val); @@ -819,8 +819,8 @@ struct _Atomic_storage<_Ty, 4> { // lock-free using 4-byte intrinsics long _Expected_bytes = _Atomic_reinterpret_as(_Expected); // read before atomic operation long _Prev_bytes; #if _CMPXCHG_MASK_OUT_PADDING_BITS - if constexpr (_Might_have_non_value_bits<_Ty>) { - _Storage_for<_Ty> _Mask{_Form_mask}; + if constexpr (_Might_have_non_value_bits<_TVal>) { + _Storage_for<_TVal> _Mask{_Form_mask}; const long _Mask_val = _Atomic_reinterpret_as(_Mask); for (;;) { @@ -831,7 +831,7 @@ struct _Atomic_storage<_Ty, 4> { // lock-free using 4-byte intrinsics } if ((_Prev_bytes ^ _Expected_bytes) & _Mask_val) { - _CSTD memcpy(_STD addressof(_Expected), &_Prev_bytes, sizeof(_Ty)); + _CSTD memcpy(_STD addressof(_Expected), &_Prev_bytes, sizeof(_TVal)); return false; } _Expected_bytes = (_Expected_bytes & _Mask_val) | (_Prev_bytes & ~_Mask_val); @@ -951,8 +951,8 @@ struct _Atomic_storage<_Ty, 8> { // lock-free using 8-byte intrinsics long long _Prev_bytes; #if _CMPXCHG_MASK_OUT_PADDING_BITS - if constexpr (_Might_have_non_value_bits<_Ty>) { - _Storage_for<_Ty> _Mask{_Form_mask}; + if constexpr (_Might_have_non_value_bits<_TVal>) { + _Storage_for<_TVal> _Mask{_Form_mask}; const long long _Mask_val = _Atomic_reinterpret_as(_Mask); for (;;) { @@ -964,7 +964,7 @@ struct _Atomic_storage<_Ty, 8> { // lock-free using 8-byte intrinsics } if ((_Prev_bytes ^ _Expected_bytes) & _Mask_val) { - _CSTD memcpy(_STD addressof(_Expected), &_Prev_bytes, sizeof(_Ty)); + _CSTD memcpy(_STD addressof(_Expected), &_Prev_bytes, sizeof(_TVal)); return false; } _Expected_bytes = (_Expected_bytes & _Mask_val) | (_Prev_bytes & ~_Mask_val); From 48262adec88aece6879a03392aec03ba1878be53 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 25 Jul 2020 14:20:53 +0300 Subject: [PATCH 064/132] Implement padding bits for 16 byte type --- stl/inc/atomic | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/stl/inc/atomic b/stl/inc/atomic index 9985599cfb9..5a213de3f23 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -1065,6 +1065,41 @@ struct _Atomic_storage<_Ty&, 16> { // lock-free using 16-byte intrinsics _Int128 _Expected_temp{}; _CSTD memcpy(&_Expected_temp, _STD addressof(_Expected), sizeof(_TVal)); unsigned char _Result; +#if _CMPXCHG_MASK_OUT_PADDING_BITS + if constexpr (_Might_have_non_value_bits<_TVal>) { + _Int128 _Expected_originally{}; + _CSTD memcpy(&_Expected_originally, _STD addressof(_Expected), sizeof(_TVal)); + + _Storage_for<_TVal> _Mask{_Form_mask}; + _Int128 _Mask_val{}; + _CSTD memcpy(&_Mask_val, _Mask._Ptr(), sizeof(_TVal)); + for (;;) { +#ifdef _M_ARM64 + _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedCompareExchange128, + _Atomic_address_as(_Storage), _Desired_bytes._High, _Desired_bytes._Low, + &_Expected_temp._Low); +#else // ^^^ _M_ARM64 / _M_X64 vvv + (void) _Order; + _Result = _STD_COMPARE_EXCHANGE_128(&reinterpret_cast(_Storage), _Desired_bytes._High, + _Desired_bytes._Low, &_Expected_temp._Low); +#endif // _M_ARM64 + if (_Result) { + return true; + } + + if (((_Expected_temp._Low ^ _Expected_originally._Low) & _Mask_val._Low) != 0 + || ((_Expected_temp._High ^ _Expected_originally._High) & _Mask_val._High) != 0) { + _CSTD memcpy(_STD addressof(_Expected), &_Expected_temp, sizeof(_TVal)); + return false; + } + + _Expected_temp._Low = + (_Expected_originally._Low & _Mask_val._Low) | (_Expected_temp._Low & ~_Mask_val._Low); + _Expected_temp._High = + (_Expected_originally._High & _Mask_val._High) | (_Expected_temp._High & ~_Mask_val._High); + } + } +#endif // _CMPXCHG_MASK_OUT_PADDING_BITS #ifdef _M_ARM64 _ATOMIC_CHOOSE_INTRINSIC(_Order, _Result, _InterlockedCompareExchange128, _Atomic_address_as(_Storage), _Desired_bytes._High, _Desired_bytes._Low, &_Expected_temp._Low); From 547e7755e74bbfa031a972518625a754c2ba95cc Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 25 Jul 2020 14:21:44 +0300 Subject: [PATCH 065/132] test coverage for 16 byte type --- tests/std/tests/P0528R3_cmpxchg_pad/test.cpp | 88 ++++++++++++++++++-- 1 file changed, 80 insertions(+), 8 deletions(-) diff --git a/tests/std/tests/P0528R3_cmpxchg_pad/test.cpp b/tests/std/tests/P0528R3_cmpxchg_pad/test.cpp index 44419f57410..5c17d82d1eb 100644 --- a/tests/std/tests/P0528R3_cmpxchg_pad/test.cpp +++ b/tests/std/tests/P0528R3_cmpxchg_pad/test.cpp @@ -170,11 +170,8 @@ struct X20 { }; -template -void test() { - static_assert(sizeof(X) == S, "Unexpected size"); - static_assert( - !std::has_unique_object_representations_v, "Type without padding is not useful for testing P0528."); +template +void test_atomic() { X x1; X x2; X x3; @@ -219,9 +216,8 @@ void test() { assert(v.load().check(6)); } - template -void test0() { +void test_atomic_0() { X x1; X x2; X x3; @@ -233,8 +229,84 @@ void test0() { v.store(x1); X x; std::memcpy(std::addressof(x), std::addressof(x3), sizeof(x)); + assert(v.compare_exchange_strong(x1, x2)); +} + +template +void test_atomic_ref() { + X x1; + X x2; + X x3; + X x4; + std::memset(std::addressof(x1), 0xaa, sizeof(x1)); + std::memset(std::addressof(x2), 0x55, sizeof(x2)); + std::memset(std::addressof(x3), 0x55, sizeof(x3)); + std::memset(std::addressof(x4), 0x55, sizeof(x4)); + x1.set(5); + x2.set(5); + x3.set(6); + x4.set(7); + + alignas(std::atomic_ref::required_alignment) X v = x1; + X x; + std::memcpy(std::addressof(x), std::addressof(x3), sizeof(x)); + assert(!std::atomic_ref(v).compare_exchange_strong(x, x4)); + assert(std::atomic_ref(v).load().check(5)); + + std::atomic_ref(v).store(x1); + for (int retry = 0; retry != 10; ++retry) { + X xw; + std::memcpy(std::addressof(xw), std::addressof(x3), sizeof(x)); + assert(!std::atomic_ref(v).compare_exchange_weak(xw, x4)); + assert(std::atomic_ref(v).load().check(5)); + } + + std::atomic_ref(v).store(x1); + std::memcpy(std::addressof(x), std::addressof(x2), sizeof(x)); + assert(std::atomic_ref(v).compare_exchange_strong(x, x3)); + assert(std::atomic_ref(v).load().check(6)); + + std::atomic_ref(v).store(x1); + for (;;) { + X xw; + std::memcpy(std::addressof(xw), std::addressof(x2), sizeof(x)); + if (std::atomic_ref(v).compare_exchange_weak(xw, x3)) { + break; + } + } + assert(std::atomic_ref(v).load().check(6)); +} + +template +void test_atomic_ref_0() { + X x1; + X x2; + X x3; + std::memset(std::addressof(x1), 0xaa, sizeof(x1)); + std::memset(std::addressof(x2), 0x55, sizeof(x2)); + std::memset(std::addressof(x3), 0x55, sizeof(x3)); + + alignas(std::atomic_ref::required_alignment) X v = x1; + X x; + std::memcpy(std::addressof(x), std::addressof(x3), sizeof(x)); + assert(std::atomic_ref(v).compare_exchange_strong(x1, x2)); +} + +template +void test() { + static_assert(sizeof(X) == S, "Unexpected size"); + static_assert( + !std::has_unique_object_representations_v, "Type without padding is not useful for testing P0528."); + test_atomic(); + test_atomic_ref(); +} - assert(v.compare_exchange_strong(x, x2)); +template +void test0() { + static_assert( + !std::has_unique_object_representations_v, "Type without padding is not useful for testing P0528."); + test_atomic_0(); + test_atomic_ref_0(); } int main() { From 516a09c162d1640ef4b40fa27d74daaebf435848 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 2 Aug 2020 16:23:41 +0300 Subject: [PATCH 066/132] post merge fixes --- stl/CMakeLists.txt | 1 + stl/inc/atomic | 21 +++++++++++++++++---- stl/src/msvcp_atomic_ref.def | 9 --------- stl/src/msvcp_atomic_wait.src | 3 +++ 4 files changed, 21 insertions(+), 13 deletions(-) delete mode 100644 stl/src/msvcp_atomic_ref.def diff --git a/stl/CMakeLists.txt b/stl/CMakeLists.txt index 23dc688099e..c1344e18352 100644 --- a/stl/CMakeLists.txt +++ b/stl/CMakeLists.txt @@ -389,6 +389,7 @@ set(SOURCES_SATELLITE_2 ) set(SOURCES_SATELLITE_ATOMIC_WAIT + ${CMAKE_CURRENT_LIST_DIR}/src/atomic_ref.cpp ${CMAKE_CURRENT_LIST_DIR}/src/atomic_wait.cpp ${CMAKE_CURRENT_LIST_DIR}/src/parallel_algorithms.cpp ) diff --git a/stl/inc/atomic b/stl/inc/atomic index ab02b37e574..2eed5f75d5b 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -441,9 +441,19 @@ inline void _Atomic_unlock_spinlock(long& _Spinlock) noexcept { #endif // hardware } + +inline void _Atomic_lock_spinlock(_Smtx_t* _Spinlock) noexcept { + _Smtx_lock_exclusive(_Spinlock); +} + +inline void _Atomic_unlock_spinlock(_Smtx_t* _Spinlock) noexcept { + _Smtx_unlock_exclusive(_Spinlock); +} + +template class _Spinlock_guard { public: - explicit _Spinlock_guard(long& _Spinlock_) noexcept : _Spinlock(_Spinlock_) { + explicit _Spinlock_guard(_Spinlock_t& _Spinlock_) noexcept : _Spinlock(_Spinlock_) { _Atomic_lock_spinlock(_Spinlock); } @@ -455,7 +465,7 @@ public: _Spinlock_guard& operator=(const _Spinlock_guard&) = delete; private: - long& _Spinlock; + _Spinlock_t& _Spinlock; }; #if _HAS_CXX20 @@ -599,6 +609,11 @@ struct _Atomic_storage { _Atomic_unlock_spinlock(_Spinlock); } +protected: + void _Init_spinlock_for_ref() noexcept { + _Spinlock = __std_atomic_get_mutex(_STD addressof(_Storage)); + } + private: // Spinlock integer for non-lock-free atomic. mutex pointer for non-lock-free atomic_ref mutable typename _Atomic_storage_types<_Ty>::_Spinlock _Spinlock = 0; @@ -724,8 +739,6 @@ struct _Atomic_storage<_Ty, 1> { // lock-free using 1-byte intrinsics return false; } - _Atomic_padded<_Ty> _Storage; - typename _Atomic_storage_types<_Ty>::_TStorage _Storage; #if _HAS_CXX20 void wait(const _Ty _Expected, const memory_order _Order = memory_order_seq_cst) const noexcept { _Atomic_wait_direct(this, _Atomic_reinterpret_as(_Expected), _Order); diff --git a/stl/src/msvcp_atomic_ref.def b/stl/src/msvcp_atomic_ref.def deleted file mode 100644 index f423d63798c..00000000000 --- a/stl/src/msvcp_atomic_ref.def +++ /dev/null @@ -1,9 +0,0 @@ -; Copyright (c) Microsoft Corporation. -; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -; atomic ref satellite DLL definition - -EXPORTS - __std_atomic_get_mutex - __std_atomic_compare_exchange_128 - __std_atomic_has_cmpxchg16b diff --git a/stl/src/msvcp_atomic_wait.src b/stl/src/msvcp_atomic_wait.src index ec335cc161c..5af6d3ffa77 100644 --- a/stl/src/msvcp_atomic_wait.src +++ b/stl/src/msvcp_atomic_wait.src @@ -6,6 +6,9 @@ LIBRARY LIBRARYNAME EXPORTS + __std_atomic_compare_exchange_128 + __std_atomic_has_cmpxchg16b + __std_atomic_get_mutex __std_atomic_wait_get_deadline __std_atomic_wait_get_remaining_timeout __std_atomic_notify_all_direct From 456ff028865bdf45cdc48f2d703a238f68ee35a5 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 2 Aug 2020 19:27:00 +0300 Subject: [PATCH 067/132] more post merge fixes --- stl/inc/atomic | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 2eed5f75d5b..f6cd052e9b6 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -19,8 +19,8 @@ #include #if _HAS_CXX20 #include -#include #endif // _HAS_CXX20 +#include #pragma pack(push, _CRT_PACKING) #pragma warning(push, _STL_WARNING_LEVEL) @@ -405,8 +405,9 @@ void _Atomic_wait_direct( const _Value_type _Observed_bytes = _Atomic_reinterpret_as<_Value_type>(_This->load(_Order)); if (_Expected_bytes != _Observed_bytes) { #if _CMPXCHG_MASK_OUT_PADDING_BITS - if constexpr (_Might_have_non_value_bits<_Ty>) { - _Storage_for<_Ty> _Mask{_Form_mask}; + using _TVal = _Atomic_storage_types<_Ty>::_TVal; + if constexpr (_Might_have_non_value_bits<_TVal>) { + _Storage_for<_TVal> _Mask{_Form_mask}; const _Value_type _Mask_val = _Atomic_reinterpret_as<_Value_type>(_Mask._Ref()); if (((_Expected_bytes ^ _Observed_bytes) & _Mask_val) == 0) { @@ -555,25 +556,25 @@ struct _Atomic_storage { } #if _HAS_CXX20 - void wait(_Ty _Expected, memory_order = memory_order_seq_cst) const noexcept { + void wait(_TVal _Expected, memory_order = memory_order_seq_cst) const noexcept { const auto _Storage_ptr = _STD addressof(_Storage); const auto _Expected_ptr = _STD addressof(_Expected); for (;;) { { _Spinlock_guard _Lock{_Spinlock}; - if (_CSTD memcmp(_Storage_ptr, _Expected_ptr, sizeof(_Ty)) != 0) { + if (_CSTD memcmp(_Storage_ptr, _Expected_ptr, sizeof(_TVal)) != 0) { // contents differed, we might be done, check for padding #if _CMPXCHG_MASK_OUT_PADDING_BITS - if constexpr (_Might_have_non_value_bits<_Ty>) { - _Storage_for<_Ty> _Local; + if constexpr (_Might_have_non_value_bits<_TVal>) { + _Storage_for<_TVal> _Local; const auto _Local_ptr = _Local._Ptr(); - _CSTD memcpy(_Local_ptr, _Storage_ptr, sizeof(_Ty)); + _CSTD memcpy(_Local_ptr, _Storage_ptr, sizeof(_TVal)); __builtin_zero_non_value_bits(_Local_ptr); __builtin_zero_non_value_bits(_Expected_ptr); - if (_CSTD memcmp(_Local_ptr, _Expected_ptr, sizeof(_Ty)) == 0) { + if (_CSTD memcmp(_Local_ptr, _Expected_ptr, sizeof(_TVal)) == 0) { // _Storage differs from _Expected only by padding; copy the padding from _Storage into // _Expected - _CSTD memcpy(_Expected_ptr, _Storage_ptr, sizeof(_Ty)); + _CSTD memcpy(_Expected_ptr, _Storage_ptr, sizeof(_TVal)); } else { // truly different, we're done return; @@ -586,7 +587,7 @@ struct _Atomic_storage { } } // unlock - __std_atomic_wait_indirect(_Storage_ptr, _Expected_ptr, sizeof(_Ty), &_Spinlock, + __std_atomic_wait_indirect(_Storage_ptr, _Expected_ptr, sizeof(_TVal), &_Spinlock, &_Atomic_wait_compare_non_lock_free, _Atomic_wait_no_timeout); } } @@ -740,7 +741,7 @@ struct _Atomic_storage<_Ty, 1> { // lock-free using 1-byte intrinsics } #if _HAS_CXX20 - void wait(const _Ty _Expected, const memory_order _Order = memory_order_seq_cst) const noexcept { + void wait(const _TVal _Expected, const memory_order _Order = memory_order_seq_cst) const noexcept { _Atomic_wait_direct(this, _Atomic_reinterpret_as(_Expected), _Order); } @@ -860,7 +861,7 @@ struct _Atomic_storage<_Ty, 2> { // lock-free using 2-byte intrinsics } #if _HAS_CXX20 - void wait(const _Ty _Expected, const memory_order _Order = memory_order_seq_cst) const noexcept { + void wait(const _TVal _Expected, const memory_order _Order = memory_order_seq_cst) const noexcept { _Atomic_wait_direct(this, _Atomic_reinterpret_as(_Expected), _Order); } @@ -978,7 +979,7 @@ struct _Atomic_storage<_Ty, 4> { // lock-free using 4-byte intrinsics } #if _HAS_CXX20 - void wait(const _Ty _Expected, const memory_order _Order = memory_order_seq_cst) const noexcept { + void wait(const _TVal _Expected, const memory_order _Order = memory_order_seq_cst) const noexcept { _Atomic_wait_direct(this, _Atomic_reinterpret_as(_Expected), _Order); } @@ -1125,7 +1126,7 @@ struct _Atomic_storage<_Ty, 8> { // lock-free using 8-byte intrinsics } #if _HAS_CXX20 - void wait(const _Ty _Expected, const memory_order _Order = memory_order_seq_cst) const noexcept { + void wait(const _TVal _Expected, const memory_order _Order = memory_order_seq_cst) const noexcept { _Atomic_wait_direct(this, _Atomic_reinterpret_as(_Expected), _Order); } From 0a3b79e4973c21a1c6c5d88f6e0727370c232090 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 2 Aug 2020 20:17:36 +0300 Subject: [PATCH 068/132] Apply suggestions from code review Co-authored-by: Adam Bucior <35536269+AdamBucior@users.noreply.github.com> --- stl/inc/atomic | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index f6cd052e9b6..a813dfcf28e 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -1299,7 +1299,7 @@ struct _Atomic_integral; // not defined template struct _Atomic_integral<_Ty, 1> : _Atomic_storage<_Ty> { // atomic integral operations using 1-byte intrinsics using _Base = _Atomic_storage<_Ty>; - using _TVal = typename _Base::_TVal; + using _Base::_TVal; #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 _Atomic_integral() = default; @@ -1363,7 +1363,7 @@ struct _Atomic_integral<_Ty, 1> : _Atomic_storage<_Ty> { // atomic integral oper template struct _Atomic_integral<_Ty, 2> : _Atomic_storage<_Ty> { // atomic integral operations using 2-byte intrinsics using _Base = _Atomic_storage<_Ty>; - using _TVal = typename _Base::_TVal; + using _Base::_TVal; #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 _Atomic_integral() = default; @@ -1491,7 +1491,7 @@ struct _Atomic_integral<_Ty, 4> : _Atomic_storage<_Ty> { // atomic integral oper template struct _Atomic_integral<_Ty, 8> : _Atomic_storage<_Ty> { // atomic integral operations using 8-byte intrinsics using _Base = _Atomic_storage<_Ty>; - using _TVal = typename _Base::_TVal; + using _Base::_TVal; #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 _Atomic_integral() = default; @@ -1631,7 +1631,7 @@ template struct _Atomic_integral_facade : _Atomic_integral<_Ty> { // provides operator overloads and other support for atomic integral specializations using _Base = _Atomic_integral<_Ty>; - using _TVal = typename _Base::_TVal; + using _Base::_TVal; using difference_type = _TVal; #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 @@ -1786,7 +1786,7 @@ template struct _Atomic_floating : _Atomic_storage<_Ty> { // provides atomic floating-point operations using _Base = _Atomic_storage<_Ty>; - using _TVal = typename _Base::_TVal; + using _Base::_TVal; using difference_type = _TVal; #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 @@ -1852,7 +1852,7 @@ struct _Atomic_floating : _Atomic_storage<_Ty> { template struct _Atomic_pointer : _Atomic_storage<_Ty> { using _Base = _Atomic_storage<_Ty>; - using _TVal = typename _Base::_TVal; + using _Base::_TVal; using difference_type = ptrdiff_t; #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 @@ -2184,7 +2184,7 @@ public: explicit atomic_ref(_Ty& _Value) : _Base(_Value) { if constexpr (_Is_potentially_lock_free) { - _Check_aligment(_Value); + _Check_alignment(_Value); } if constexpr (!_Is_potentially_lock_free) { @@ -2240,7 +2240,7 @@ public: } private: - void _Check_aligment([[maybe_unused]] const _Ty& _Value) { + void _Check_alignment([[maybe_unused]] const _Ty& _Value) { _ATOMIC_REF_CHECK_ALIGNMENT( (reinterpret_cast(_STD addressof(_Value)) & (required_alignment - 1)) == 0, "atomic_ref underlying object is not aligned as required_alignment"); From 74d925b566d346eaa781c8491c15bf37deaeaeaa Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 2 Aug 2020 20:23:20 +0300 Subject: [PATCH 069/132] clang format --- stl/inc/atomic | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index a813dfcf28e..39e78e31fe6 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -1630,7 +1630,7 @@ _CXX20_DEPRECATE_VOLATILE _INLINE_VAR constexpr bool _Deprecate_non_lock_free_vo template struct _Atomic_integral_facade : _Atomic_integral<_Ty> { // provides operator overloads and other support for atomic integral specializations - using _Base = _Atomic_integral<_Ty>; + using _Base = _Atomic_integral<_Ty>; using _Base::_TVal; using difference_type = _TVal; @@ -1785,7 +1785,7 @@ struct _Atomic_integral_facade : _Atomic_integral<_Ty> { template struct _Atomic_floating : _Atomic_storage<_Ty> { // provides atomic floating-point operations - using _Base = _Atomic_storage<_Ty>; + using _Base = _Atomic_storage<_Ty>; using _Base::_TVal; using difference_type = _TVal; @@ -1851,7 +1851,7 @@ struct _Atomic_floating : _Atomic_storage<_Ty> { // STRUCT TEMPLATE _Atomic_pointer template struct _Atomic_pointer : _Atomic_storage<_Ty> { - using _Base = _Atomic_storage<_Ty>; + using _Base = _Atomic_storage<_Ty>; using _Base::_TVal; using difference_type = ptrdiff_t; From 6ec6912cc16600b20459126f2125180cf6cac94f Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 2 Aug 2020 21:08:49 +0300 Subject: [PATCH 070/132] typename --- stl/inc/atomic | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 39e78e31fe6..35aa253e132 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -405,7 +405,7 @@ void _Atomic_wait_direct( const _Value_type _Observed_bytes = _Atomic_reinterpret_as<_Value_type>(_This->load(_Order)); if (_Expected_bytes != _Observed_bytes) { #if _CMPXCHG_MASK_OUT_PADDING_BITS - using _TVal = _Atomic_storage_types<_Ty>::_TVal; + using _TVal = typename _Atomic_storage_types<_Ty>::_TVal; if constexpr (_Might_have_non_value_bits<_TVal>) { _Storage_for<_TVal> _Mask{_Form_mask}; const _Value_type _Mask_val = _Atomic_reinterpret_as<_Value_type>(_Mask._Ref()); @@ -1299,7 +1299,7 @@ struct _Atomic_integral; // not defined template struct _Atomic_integral<_Ty, 1> : _Atomic_storage<_Ty> { // atomic integral operations using 1-byte intrinsics using _Base = _Atomic_storage<_Ty>; - using _Base::_TVal; + using typename _Base::_TVal; #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 _Atomic_integral() = default; @@ -1363,7 +1363,7 @@ struct _Atomic_integral<_Ty, 1> : _Atomic_storage<_Ty> { // atomic integral oper template struct _Atomic_integral<_Ty, 2> : _Atomic_storage<_Ty> { // atomic integral operations using 2-byte intrinsics using _Base = _Atomic_storage<_Ty>; - using _Base::_TVal; + using typename _Base::_TVal; #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 _Atomic_integral() = default; @@ -1427,7 +1427,7 @@ struct _Atomic_integral<_Ty, 2> : _Atomic_storage<_Ty> { // atomic integral oper template struct _Atomic_integral<_Ty, 4> : _Atomic_storage<_Ty> { // atomic integral operations using 4-byte intrinsics using _Base = _Atomic_storage<_Ty>; - using _TVal = typename _Base::_TVal; + using typename _Base::_TVal; #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 _Atomic_integral() = default; @@ -1491,7 +1491,7 @@ struct _Atomic_integral<_Ty, 4> : _Atomic_storage<_Ty> { // atomic integral oper template struct _Atomic_integral<_Ty, 8> : _Atomic_storage<_Ty> { // atomic integral operations using 8-byte intrinsics using _Base = _Atomic_storage<_Ty>; - using _Base::_TVal; + using typename _Base::_TVal; #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 _Atomic_integral() = default; @@ -1631,7 +1631,7 @@ template struct _Atomic_integral_facade : _Atomic_integral<_Ty> { // provides operator overloads and other support for atomic integral specializations using _Base = _Atomic_integral<_Ty>; - using _Base::_TVal; + using typename _Base::_TVal; using difference_type = _TVal; #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 @@ -1786,7 +1786,7 @@ template struct _Atomic_floating : _Atomic_storage<_Ty> { // provides atomic floating-point operations using _Base = _Atomic_storage<_Ty>; - using _Base::_TVal; + using typename _Base::_TVal; using difference_type = _TVal; #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 @@ -1852,7 +1852,7 @@ struct _Atomic_floating : _Atomic_storage<_Ty> { template struct _Atomic_pointer : _Atomic_storage<_Ty> { using _Base = _Atomic_storage<_Ty>; - using _Base::_TVal; + using typename _Base::_TVal; using difference_type = ptrdiff_t; #ifdef __cplusplus_winrt // TRANSITION, VSO-1083296 From a3b1d45ab89a339babd891050d3a93b40d5b65a7 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 3 Aug 2020 17:51:08 +0300 Subject: [PATCH 071/132] atomic ref wait --- stl/inc/atomic | 47 ++++++++++++++++++++++++-- tests/std/include/test_atomic_wait.hpp | 35 +++++++++++++++---- 2 files changed, 73 insertions(+), 9 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 35aa253e132..2387d3671cb 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -470,14 +470,28 @@ private: }; #if _HAS_CXX20 +template inline bool __stdcall _Atomic_wait_compare_non_lock_free( const void* _Storage, void* _Comparand, size_t _Size, void* _Spinlock_raw) noexcept { - long& _Spinlock = *static_cast(_Spinlock_raw); + Spinlock& _Spinlock = *static_cast(_Spinlock_raw); _Atomic_lock_spinlock(_Spinlock); const auto _Cmp_result = _CSTD memcmp(_Storage, _Comparand, _Size); _Atomic_unlock_spinlock(_Spinlock); return _Cmp_result == 0; } + +#ifdef _STD_COMPARE_EXCHANGE_128 +inline bool __stdcall _Atomic_wait_compare_16_bytes(const void* _Storage, void* _Comparand, size_t, void*) noexcept { + const auto _Dest = reinterpret_cast(const_cast(_Storage)); + const auto _Cmp = reinterpret_cast(_Comparand); + long long _Tmp[2] = {_Cmp[0], _Cmp[1]}; +#ifdef _M_X64 + return _STD_COMPARE_EXCHANGE_128(_Dest, _Dest[1], _Dest[0], _Tmp) != 0; +#else // ^^^ _M_X64 / ARM64 vvv + return _InterlockedCompareExchange128_nf(_Dest, _Dest[1], _Dest[0], _Tmp) != 0; +#endif // ^^^ ARM64 ^^^ +} +#endif // _STD_COMPARE_EXCHANGE_128 #endif // _HAS_CXX20 #endif // TRANSITION, ABI @@ -588,7 +602,7 @@ struct _Atomic_storage { } // unlock __std_atomic_wait_indirect(_Storage_ptr, _Expected_ptr, sizeof(_TVal), &_Spinlock, - &_Atomic_wait_compare_non_lock_free, _Atomic_wait_no_timeout); + &_Atomic_wait_compare_non_lock_free, _Atomic_wait_no_timeout); } } @@ -1274,6 +1288,35 @@ struct _Atomic_storage<_Ty&, 16> { // lock-free using 16-byte intrinsics } #if _HAS_CXX20 + void wait(_TVal _Expected, memory_order _Order = memory_order_seq_cst) const noexcept { + const auto _Storage_ptr = _STD addressof(_Storage); + const auto _Expected_ptr = _STD addressof(_Expected); + _Int128 _Expected_bytes = reinterpret_cast(_Expected); + + for (;;) { + const _TVal _Observed = load(_Order); + _Int128 _Observed_bytes = reinterpret_cast(_Observed); + if (_Observed_bytes._Low != _Expected_bytes._Low || _Observed_bytes._High != _Expected_bytes._High) { +#if _CMPXCHG_MASK_OUT_PADDING_BITS + if constexpr (_Might_have_non_value_bits<_TVal>) { + _Storage_for<_TVal> _Mask{_Form_mask}; + const _Int128 _Mask_val = reinterpret_cast<_Int128&>(_Mask._Ref()); + + if (((_Expected_bytes._Low ^ _Observed_bytes._Low) & _Mask_val._Low) == 0 + && ((_Expected_bytes._High ^ _Observed_bytes._High) & _Mask_val._High) == 0) { + _Expected_bytes = _Observed_bytes; + continue; + } + } +#endif // _CMPXCHG_MASK_OUT_PADDING_BITS + return; + } + + __std_atomic_wait_indirect(_Storage_ptr, _Expected_ptr, sizeof(_TVal), nullptr, + &_Atomic_wait_compare_16_bytes, _Atomic_wait_no_timeout); + } + } + void notify_one() noexcept { __std_atomic_notify_one_indirect(_STD addressof(_Storage)); } diff --git a/tests/std/include/test_atomic_wait.hpp b/tests/std/include/test_atomic_wait.hpp index 248615cdf42..3cfe1de4c87 100644 --- a/tests/std/include/test_atomic_wait.hpp +++ b/tests/std/include/test_atomic_wait.hpp @@ -9,8 +9,8 @@ #include #include -template -void test_atomic_wait_func(const UnderlyingType old_value, const UnderlyingType new_value, +template