[libc++] Implement P0528R3 std::atomic CAS for types with padding (#76180)
Fixes #99984 Implement P0528R3 The Curious Case of Padding Bits, Featuring Atomic Compare-and-Exchange The unit test padding.pass.cpp is assisted by Cursor. However, all tests have been modified to test what I wanted to test --------- Co-authored-by: Louis Dionne <ldionne.2@gmail.com>
This commit is contained in:
Hui
GitHub
parent
0a30b0c890
commit
de962dda12
11 changed files with 739 additions and 119 deletions
|
|
@ -43,6 +43,7 @@ Implemented Papers
|
|||
- P3948R1: ``constant_wrapper`` is the only tool needed for passing constant expressions via function arguments (`Github <https://llvm.org/PR189604>`__)
|
||||
- P3961R1: Less double indirection in ``function_ref`` (RU-220) (`Github <https://llvm.org/PR189606>`__)
|
||||
- P1901R2: Enabling the Use of ``weak_ptr`` as Keys in Unordered Associative Containers (`Github <https://llvm.org/PR105372>`__)
|
||||
- P0528R3: The Curious Case of Padding Bits, Featuring Atomic Compare-and-Exchange (`Github <https://llvm.org/PR76180>`__)
|
||||
|
||||
Improvements and New Features
|
||||
-----------------------------
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@
|
|||
"`P0458R2 <https://wg21.link/P0458R2>`__","Checking for Existence of an Element in Associative Containers","2018-06 (Rapperswil)","|Complete|","13","`#104029 <https://github.com/llvm/llvm-project/issues/104029>`__",""
|
||||
"`P0475R1 <https://wg21.link/P0475R1>`__","LWG 2511: guaranteed copy elision for piecewise construction","2018-06 (Rapperswil)","|Complete|","","`#104031 <https://github.com/llvm/llvm-project/issues/104031>`__",""
|
||||
"`P0476R2 <https://wg21.link/P0476R2>`__","Bit-casting object representations","2018-06 (Rapperswil)","|Complete|","14","`#104087 <https://github.com/llvm/llvm-project/issues/104087>`__",""
|
||||
"`P0528R3 <https://wg21.link/P0528R3>`__","The Curious Case of Padding Bits, Featuring Atomic Compare-and-Exchange","2018-06 (Rapperswil)","","","`#99984 <https://github.com/llvm/llvm-project/issues/99984>`__",""
|
||||
"`P0528R3 <https://wg21.link/P0528R3>`__","The Curious Case of Padding Bits, Featuring Atomic Compare-and-Exchange","2018-06 (Rapperswil)","|Complete|","24","`#99984 <https://github.com/llvm/llvm-project/issues/99984>`__",""
|
||||
"`P0542R5 <https://wg21.link/P0542R5>`__","Support for contract based programming in C++","2018-06 (Rapperswil)","|Nothing To Do|","n/a","`#104089 <https://github.com/llvm/llvm-project/issues/104089>`__","Pulled at the 2019-07 meeting in Cologne"
|
||||
"`P0556R3 <https://wg21.link/P0556R3>`__","Integral power-of-2 operations","2018-06 (Rapperswil)","|Complete|","9","`#104090 <https://github.com/llvm/llvm-project/issues/104090>`__",""
|
||||
"`P0619R4 <https://wg21.link/P0619R4>`__","Reviewing Deprecated Facilities of C++17 for C++20","2018-06 (Rapperswil)","|Complete|","20","`#99985 <https://github.com/llvm/llvm-project/issues/99985>`__","Removed headers are still provided as an extension, but with deprecation warnings."
|
||||
|
|
|
|||
|
|
|
@ -219,6 +219,7 @@ set(files
|
|||
__atomic/atomic_sync_timed.h
|
||||
__atomic/atomic_waitable_traits.h
|
||||
__atomic/check_memory_order.h
|
||||
__atomic/clear_padding.h
|
||||
__atomic/contention_t.h
|
||||
__atomic/fence.h
|
||||
__atomic/floating_point_helper.h
|
||||
|
|
|
|||
92
libcxx/include/__atomic/clear_padding.h
Normal file
92
libcxx/include/__atomic/clear_padding.h
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
#ifndef _LIBCPP___ATOMIC_CLEAR_PADDING_H
|
||||
#define _LIBCPP___ATOMIC_CLEAR_PADDING_H
|
||||
|
||||
#include <__config>
|
||||
#include <__memory/addressof.h>
|
||||
#include <__type_traits/conjunction.h>
|
||||
#include <__type_traits/enable_if.h>
|
||||
#include <__type_traits/has_unique_object_representation.h>
|
||||
#include <__type_traits/integral_constant.h>
|
||||
#include <__type_traits/is_same.h>
|
||||
#include <__type_traits/negation.h>
|
||||
#include <__type_traits/remove_cv.h>
|
||||
#include <__type_traits/remove_cvref.h>
|
||||
#include <__utility/forward.h>
|
||||
#include <cstring>
|
||||
|
||||
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
|
||||
# pragma GCC system_header
|
||||
#endif
|
||||
|
||||
_LIBCPP_BEGIN_NAMESPACE_STD
|
||||
|
||||
#if __has_builtin(__builtin_clear_padding)
|
||||
|
||||
template <class _Tp>
|
||||
struct __needs_clear_padding
|
||||
: _And<_Not<integral_constant<bool, __has_unique_object_representations(_Tp)> >,
|
||||
_Not<is_same<_Tp, float> >,
|
||||
_Not<is_same<_Tp, double> > > {};
|
||||
|
||||
template <class _Tp, __enable_if_t<!__needs_clear_padding<__remove_cvref_t<_Tp> >::value, int> = 0>
|
||||
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Tp& __clear_padding_if_needed(_Tp& __obj) _NOEXCEPT {
|
||||
return __obj;
|
||||
}
|
||||
|
||||
template <class _Tp, __enable_if_t<__needs_clear_padding<__remove_cvref_t<_Tp> >::value, int> = 0>
|
||||
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Tp& __clear_padding_if_needed(_Tp& __obj) _NOEXCEPT {
|
||||
return __builtin_is_constant_evaluated() ? __obj : (__builtin_clear_padding(std::addressof(__obj)), __obj);
|
||||
}
|
||||
|
||||
// clang fails to inline the function when the memory order is a constant
|
||||
template <class _Tp,
|
||||
class _Up,
|
||||
class _CasFunc,
|
||||
__enable_if_t<!__needs_clear_padding<__remove_cvref_t<_Tp> >::value, int> = 0>
|
||||
_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE bool
|
||||
__atomic_cas_with_clear_padding(_Tp* __expected, _Up __value, _CasFunc&& __cas_func) {
|
||||
return __cas_func(__expected, __value);
|
||||
}
|
||||
|
||||
template <class _Tp,
|
||||
class _Up,
|
||||
class _CasFunc,
|
||||
__enable_if_t<__needs_clear_padding<__remove_cvref_t<_Tp> >::value, int> = 0>
|
||||
_LIBCPP_HIDE_FROM_ABI bool __atomic_cas_with_clear_padding(_Tp* __expected, _Up __value, _CasFunc&& __cas_func) {
|
||||
std::__clear_padding_if_needed(__value);
|
||||
__remove_cvref_t<_Tp> __expected_copy = *__expected;
|
||||
std::__clear_padding_if_needed(__expected_copy);
|
||||
if (__cas_func(std::addressof(__expected_copy), __value)) {
|
||||
return true;
|
||||
} else {
|
||||
std::memcpy(__expected, std::addressof(__expected_copy), sizeof(__remove_cvref_t<_Tp>));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
#else // __has_builtin(__builtin_clear_padding)
|
||||
|
||||
template <class _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR _Tp& __clear_padding_if_needed(_Tp& __obj) _NOEXCEPT {
|
||||
return __obj;
|
||||
}
|
||||
|
||||
template <class _Tp, class _Up, class _CasFunc>
|
||||
_LIBCPP_HIDE_FROM_ABI _LIBCPP_ALWAYS_INLINE bool
|
||||
__atomic_cas_with_clear_padding(_Tp* __expected, _Up __value, _CasFunc&& __cas_func) {
|
||||
return __cas_func(__expected, __value);
|
||||
}
|
||||
|
||||
#endif // __has_builtin(__builtin_clear_padding)
|
||||
|
||||
_LIBCPP_END_NAMESPACE_STD
|
||||
|
||||
#endif // _LIBCPP___ATOMIC_CLEAR_PADDING_H
|
||||
|
|
@ -9,6 +9,7 @@
|
|||
#ifndef _LIBCPP___ATOMIC_SUPPORT_C11_H
|
||||
#define _LIBCPP___ATOMIC_SUPPORT_C11_H
|
||||
|
||||
#include <__atomic/clear_padding.h>
|
||||
#include <__atomic/memory_order.h>
|
||||
#include <__config>
|
||||
#include <__cstddef/ptrdiff_t.h>
|
||||
|
|
@ -36,7 +37,10 @@ struct __cxx_atomic_base_impl {
|
|||
__cxx_atomic_base_impl() _NOEXCEPT : __a_value() {
|
||||
}
|
||||
#endif // _LIBCPP_CXX03_LANG
|
||||
_LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT : __a_value(__value) {}
|
||||
|
||||
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT
|
||||
: __a_value(std::__clear_padding_if_needed(__value)) {}
|
||||
|
||||
_Atomic(_Tp) __a_value;
|
||||
};
|
||||
|
||||
|
|
@ -52,21 +56,25 @@ _LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order
|
|||
|
||||
template <class _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val) _NOEXCEPT {
|
||||
std::__clear_padding_if_needed(__val);
|
||||
__c11_atomic_init(std::addressof(__a->__a_value), __val);
|
||||
}
|
||||
template <class _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) _NOEXCEPT {
|
||||
std::__clear_padding_if_needed(__val);
|
||||
__c11_atomic_init(std::addressof(__a->__a_value), __val);
|
||||
}
|
||||
|
||||
template <class _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI void
|
||||
__cxx_atomic_store(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __val, memory_order __order) _NOEXCEPT {
|
||||
std::__clear_padding_if_needed(__val);
|
||||
__c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
|
||||
}
|
||||
template <class _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI void
|
||||
__cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) _NOEXCEPT {
|
||||
std::__clear_padding_if_needed(__val);
|
||||
__c11_atomic_store(std::addressof(__a->__a_value), __val, static_cast<__memory_order_underlying_t>(__order));
|
||||
}
|
||||
|
||||
|
|
@ -102,12 +110,14 @@ __cxx_atomic_load_inplace(__cxx_atomic_base_impl<_Tp> const* __a, _Tp* __dst, me
|
|||
template <class _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI _Tp
|
||||
__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp> volatile* __a, _Tp __value, memory_order __order) _NOEXCEPT {
|
||||
std::__clear_padding_if_needed(__value);
|
||||
return __c11_atomic_exchange(
|
||||
std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
|
||||
}
|
||||
template <class _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI _Tp
|
||||
__cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) _NOEXCEPT {
|
||||
std::__clear_padding_if_needed(__value);
|
||||
return __c11_atomic_exchange(
|
||||
std::addressof(__a->__a_value), __value, static_cast<__memory_order_underlying_t>(__order));
|
||||
}
|
||||
|
|
@ -126,23 +136,34 @@ _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
|
|||
_Tp __value,
|
||||
memory_order __success,
|
||||
memory_order __failure) _NOEXCEPT {
|
||||
return __c11_atomic_compare_exchange_strong(
|
||||
std::addressof(__a->__a_value),
|
||||
return std::__atomic_cas_with_clear_padding(
|
||||
__expected,
|
||||
__value,
|
||||
static_cast<__memory_order_underlying_t>(__success),
|
||||
static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
|
||||
// Use always_inline since Clang fails to inline the lambda, even when the memory order is a constant
|
||||
[__a, __success, __failure](_Tp* __expected_or_copy, _Tp __value_maybe_padding_cleared) _LIBCPP_ALWAYS_INLINE {
|
||||
return __c11_atomic_compare_exchange_strong(
|
||||
std::addressof(__a->__a_value),
|
||||
__expected_or_copy,
|
||||
__value_maybe_padding_cleared,
|
||||
static_cast<__memory_order_underlying_t>(__success),
|
||||
static_cast<__memory_order_underlying_t>(std::__to_failure_order(__failure)));
|
||||
});
|
||||
}
|
||||
template <class _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
|
||||
__cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
|
||||
_NOEXCEPT {
|
||||
return __c11_atomic_compare_exchange_strong(
|
||||
std::addressof(__a->__a_value),
|
||||
return std::__atomic_cas_with_clear_padding(
|
||||
__expected,
|
||||
__value,
|
||||
static_cast<__memory_order_underlying_t>(__success),
|
||||
static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
|
||||
[__a, __success, __failure](_Tp* __expected_or_copy, _Tp __value_maybe_padding_cleared) _LIBCPP_ALWAYS_INLINE {
|
||||
return __c11_atomic_compare_exchange_strong(
|
||||
std::addressof(__a->__a_value),
|
||||
__expected_or_copy,
|
||||
__value_maybe_padding_cleared,
|
||||
static_cast<__memory_order_underlying_t>(__success),
|
||||
static_cast<__memory_order_underlying_t>(std::__to_failure_order(__failure)));
|
||||
});
|
||||
}
|
||||
|
||||
template <class _Tp>
|
||||
|
|
@ -152,23 +173,34 @@ _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
|
|||
_Tp __value,
|
||||
memory_order __success,
|
||||
memory_order __failure) _NOEXCEPT {
|
||||
return __c11_atomic_compare_exchange_weak(
|
||||
std::addressof(__a->__a_value),
|
||||
return std::__atomic_cas_with_clear_padding(
|
||||
__expected,
|
||||
__value,
|
||||
static_cast<__memory_order_underlying_t>(__success),
|
||||
static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
|
||||
[__a, __success, __failure](_Tp* __expected_or_copy, _Tp __value_maybe_padding_cleared) _LIBCPP_ALWAYS_INLINE {
|
||||
return __c11_atomic_compare_exchange_weak(
|
||||
std::addressof(__a->__a_value),
|
||||
__expected_or_copy,
|
||||
__value_maybe_padding_cleared,
|
||||
static_cast<__memory_order_underlying_t>(__success),
|
||||
static_cast<__memory_order_underlying_t>(std::__to_failure_order(__failure)));
|
||||
});
|
||||
}
|
||||
|
||||
template <class _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
|
||||
__cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure)
|
||||
_NOEXCEPT {
|
||||
return __c11_atomic_compare_exchange_weak(
|
||||
std::addressof(__a->__a_value),
|
||||
return std::__atomic_cas_with_clear_padding(
|
||||
__expected,
|
||||
__value,
|
||||
static_cast<__memory_order_underlying_t>(__success),
|
||||
static_cast<__memory_order_underlying_t>(__to_failure_order(__failure)));
|
||||
[__a, __success, __failure](_Tp* __expected_or_copy, _Tp __value_maybe_padding_cleared) _LIBCPP_ALWAYS_INLINE {
|
||||
return __c11_atomic_compare_exchange_weak(
|
||||
std::addressof(__a->__a_value),
|
||||
__expected_or_copy,
|
||||
__value_maybe_padding_cleared,
|
||||
static_cast<__memory_order_underlying_t>(__success),
|
||||
static_cast<__memory_order_underlying_t>(std::__to_failure_order(__failure)));
|
||||
});
|
||||
}
|
||||
|
||||
template <class _Tp>
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
#ifndef _LIBCPP___ATOMIC_SUPPORT_GCC_H
|
||||
#define _LIBCPP___ATOMIC_SUPPORT_GCC_H
|
||||
|
||||
#include <__atomic/clear_padding.h>
|
||||
#include <__atomic/memory_order.h>
|
||||
#include <__atomic/to_gcc_order.h>
|
||||
#include <__config>
|
||||
|
|
@ -52,62 +53,69 @@ struct __cxx_atomic_base_impl {
|
|||
__cxx_atomic_base_impl() _NOEXCEPT : __a_value() {
|
||||
}
|
||||
#endif // _LIBCPP_CXX03_LANG
|
||||
_LIBCPP_CONSTEXPR explicit __cxx_atomic_base_impl(_Tp value) _NOEXCEPT : __a_value(value) {}
|
||||
|
||||
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR __cxx_atomic_base_impl(_Tp __value) _NOEXCEPT
|
||||
: __a_value(std::__clear_padding_if_needed(__value)) {}
|
||||
|
||||
_Tp __a_value;
|
||||
};
|
||||
|
||||
template <typename _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val) {
|
||||
std::__clear_padding_if_needed(__val);
|
||||
__cxx_atomic_assign_volatile(__a->__a_value, __val);
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_init(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val) {
|
||||
std::__clear_padding_if_needed(__val);
|
||||
__a->__a_value = __val;
|
||||
}
|
||||
|
||||
_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_thread_fence(memory_order __order) {
|
||||
__atomic_thread_fence(__to_gcc_order(__order));
|
||||
__atomic_thread_fence(std::__to_gcc_order(__order));
|
||||
}
|
||||
|
||||
_LIBCPP_HIDE_FROM_ABI inline void __cxx_atomic_signal_fence(memory_order __order) {
|
||||
__atomic_signal_fence(__to_gcc_order(__order));
|
||||
__atomic_signal_fence(std::__to_gcc_order(__order));
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI void
|
||||
__cxx_atomic_store(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) {
|
||||
__atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order));
|
||||
std::__clear_padding_if_needed(__val);
|
||||
__atomic_store(std::addressof(__a->__a_value), std::addressof(__val), std::__to_gcc_order(__order));
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI void __cxx_atomic_store(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) {
|
||||
__atomic_store(std::addressof(__a->__a_value), std::addressof(__val), __to_gcc_order(__order));
|
||||
std::__clear_padding_if_needed(__val);
|
||||
__atomic_store(std::addressof(__a->__a_value), std::addressof(__val), std::__to_gcc_order(__order));
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const volatile __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) {
|
||||
_Tp __ret;
|
||||
__atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order));
|
||||
__atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), std::__to_gcc_order(__order));
|
||||
return __ret;
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI void
|
||||
__cxx_atomic_load_inplace(const volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) {
|
||||
__atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order));
|
||||
__atomic_load(std::addressof(__a->__a_value), __dst, std::__to_gcc_order(__order));
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI void
|
||||
__cxx_atomic_load_inplace(const __cxx_atomic_base_impl<_Tp>* __a, _Tp* __dst, memory_order __order) {
|
||||
__atomic_load(std::addressof(__a->__a_value), __dst, __to_gcc_order(__order));
|
||||
__atomic_load(std::addressof(__a->__a_value), __dst, std::__to_gcc_order(__order));
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_load(const __cxx_atomic_base_impl<_Tp>* __a, memory_order __order) {
|
||||
_Tp __ret;
|
||||
__atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), __to_gcc_order(__order));
|
||||
__atomic_load(std::addressof(__a->__a_value), std::addressof(__ret), std::__to_gcc_order(__order));
|
||||
return __ret;
|
||||
}
|
||||
|
||||
|
|
@ -115,16 +123,18 @@ template <typename _Tp>
|
|||
_LIBCPP_HIDE_FROM_ABI _Tp
|
||||
__cxx_atomic_exchange(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) {
|
||||
_Tp __ret;
|
||||
std::__clear_padding_if_needed(__value);
|
||||
__atomic_exchange(
|
||||
std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order));
|
||||
std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), std::__to_gcc_order(__order));
|
||||
return __ret;
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_exchange(__cxx_atomic_base_impl<_Tp>* __a, _Tp __value, memory_order __order) {
|
||||
_Tp __ret;
|
||||
std::__clear_padding_if_needed(__value);
|
||||
__atomic_exchange(
|
||||
std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), __to_gcc_order(__order));
|
||||
std::addressof(__a->__a_value), std::addressof(__value), std::addressof(__ret), std::__to_gcc_order(__order));
|
||||
return __ret;
|
||||
}
|
||||
|
||||
|
|
@ -135,25 +145,31 @@ _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
|
|||
_Tp __value,
|
||||
memory_order __success,
|
||||
memory_order __failure) {
|
||||
return __atomic_compare_exchange(
|
||||
std::addressof(__a->__a_value),
|
||||
__expected,
|
||||
std::addressof(__value),
|
||||
false,
|
||||
__to_gcc_order(__success),
|
||||
__to_gcc_failure_order(__failure));
|
||||
return std::__atomic_cas_with_clear_padding(
|
||||
__expected, __value, [__a, __success, __failure](_Tp* __expected_or_copy, _Tp __value_maybe_padding_cleared) {
|
||||
return __atomic_compare_exchange(
|
||||
std::addressof(__a->__a_value),
|
||||
__expected_or_copy,
|
||||
std::addressof(__value_maybe_padding_cleared),
|
||||
false,
|
||||
std::__to_gcc_order(__success),
|
||||
std::__to_gcc_failure_order(__failure));
|
||||
});
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_strong(
|
||||
__cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) {
|
||||
return __atomic_compare_exchange(
|
||||
std::addressof(__a->__a_value),
|
||||
__expected,
|
||||
std::addressof(__value),
|
||||
false,
|
||||
__to_gcc_order(__success),
|
||||
__to_gcc_failure_order(__failure));
|
||||
return std::__atomic_cas_with_clear_padding(
|
||||
__expected, __value, [__a, __success, __failure](_Tp* __expected_or_copy, _Tp __value_maybe_padding_cleared) {
|
||||
return __atomic_compare_exchange(
|
||||
std::addressof(__a->__a_value),
|
||||
__expected_or_copy,
|
||||
std::addressof(__value_maybe_padding_cleared),
|
||||
false,
|
||||
std::__to_gcc_order(__success),
|
||||
std::__to_gcc_failure_order(__failure));
|
||||
});
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
|
|
@ -163,25 +179,31 @@ _LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
|
|||
_Tp __value,
|
||||
memory_order __success,
|
||||
memory_order __failure) {
|
||||
return __atomic_compare_exchange(
|
||||
std::addressof(__a->__a_value),
|
||||
__expected,
|
||||
std::addressof(__value),
|
||||
true,
|
||||
__to_gcc_order(__success),
|
||||
__to_gcc_failure_order(__failure));
|
||||
return std::__atomic_cas_with_clear_padding(
|
||||
__expected, __value, [__a, __success, __failure](_Tp* __expected_or_copy, _Tp __value_maybe_padding_cleared) {
|
||||
return __atomic_compare_exchange(
|
||||
std::addressof(__a->__a_value),
|
||||
__expected_or_copy,
|
||||
std::addressof(__value_maybe_padding_cleared),
|
||||
true,
|
||||
std::__to_gcc_order(__success),
|
||||
std::__to_gcc_failure_order(__failure));
|
||||
});
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI bool __cxx_atomic_compare_exchange_weak(
|
||||
__cxx_atomic_base_impl<_Tp>* __a, _Tp* __expected, _Tp __value, memory_order __success, memory_order __failure) {
|
||||
return __atomic_compare_exchange(
|
||||
std::addressof(__a->__a_value),
|
||||
__expected,
|
||||
std::addressof(__value),
|
||||
true,
|
||||
__to_gcc_order(__success),
|
||||
__to_gcc_failure_order(__failure));
|
||||
return std::__atomic_cas_with_clear_padding(
|
||||
__expected, __value, [__a, __success, __failure](_Tp* __expected_or_copy, _Tp __value_maybe_padding_cleared) {
|
||||
return __atomic_compare_exchange(
|
||||
std::addressof(__a->__a_value),
|
||||
__expected_or_copy,
|
||||
std::addressof(__value_maybe_padding_cleared),
|
||||
true,
|
||||
std::__to_gcc_order(__success),
|
||||
std::__to_gcc_failure_order(__failure));
|
||||
});
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
|
|
@ -204,65 +226,69 @@ struct __skip_amt<_Tp[n]> {};
|
|||
template <typename _Tp, typename _Td>
|
||||
_LIBCPP_HIDE_FROM_ABI _Tp
|
||||
__cxx_atomic_fetch_add(volatile __cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
|
||||
return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
|
||||
return __atomic_fetch_add(
|
||||
std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, std::__to_gcc_order(__order));
|
||||
}
|
||||
|
||||
template <typename _Tp, typename _Td>
|
||||
_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_add(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
|
||||
return __atomic_fetch_add(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
|
||||
return __atomic_fetch_add(
|
||||
std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, std::__to_gcc_order(__order));
|
||||
}
|
||||
|
||||
template <typename _Tp, typename _Td>
|
||||
_LIBCPP_HIDE_FROM_ABI _Tp
|
||||
__cxx_atomic_fetch_sub(volatile __cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
|
||||
return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
|
||||
return __atomic_fetch_sub(
|
||||
std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, std::__to_gcc_order(__order));
|
||||
}
|
||||
|
||||
template <typename _Tp, typename _Td>
|
||||
_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_sub(__cxx_atomic_base_impl<_Tp>* __a, _Td __delta, memory_order __order) {
|
||||
return __atomic_fetch_sub(std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, __to_gcc_order(__order));
|
||||
return __atomic_fetch_sub(
|
||||
std::addressof(__a->__a_value), __delta * __skip_amt<_Tp>::value, std::__to_gcc_order(__order));
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI _Tp
|
||||
__cxx_atomic_fetch_and(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
|
||||
return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
|
||||
return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, std::__to_gcc_order(__order));
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI _Tp
|
||||
__cxx_atomic_fetch_and(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
|
||||
return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
|
||||
return __atomic_fetch_and(std::addressof(__a->__a_value), __pattern, std::__to_gcc_order(__order));
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI _Tp
|
||||
__cxx_atomic_fetch_or(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
|
||||
return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
|
||||
return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, std::__to_gcc_order(__order));
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_or(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
|
||||
return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
|
||||
return __atomic_fetch_or(std::addressof(__a->__a_value), __pattern, std::__to_gcc_order(__order));
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI _Tp
|
||||
__cxx_atomic_fetch_xor(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
|
||||
return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
|
||||
return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, std::__to_gcc_order(__order));
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI _Tp
|
||||
__cxx_atomic_fetch_xor(__cxx_atomic_base_impl<_Tp>* __a, _Tp __pattern, memory_order __order) {
|
||||
return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, __to_gcc_order(__order));
|
||||
return __atomic_fetch_xor(std::addressof(__a->__a_value), __pattern, std::__to_gcc_order(__order));
|
||||
}
|
||||
|
||||
template <typename _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI _Tp
|
||||
__cxx_atomic_fetch_max(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) {
|
||||
#if __has_builtin(__atomic_fetch_max)
|
||||
return __atomic_fetch_max(std::addressof(__a->__a_value), __val, __to_gcc_order(__order));
|
||||
return __atomic_fetch_max(std::addressof(__a->__a_value), __val, std::__to_gcc_order(__order));
|
||||
#else
|
||||
_Tp __ret = __cxx_atomic_load(__a, memory_order_relaxed);
|
||||
_Tp __value;
|
||||
|
|
@ -276,7 +302,7 @@ __cxx_atomic_fetch_max(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, mem
|
|||
template <typename _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_max(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) {
|
||||
#if __has_builtin(__atomic_fetch_max)
|
||||
return __atomic_fetch_max(std::addressof(__a->__a_value), __val, __to_gcc_order(__order));
|
||||
return __atomic_fetch_max(std::addressof(__a->__a_value), __val, std::__to_gcc_order(__order));
|
||||
#else
|
||||
_Tp __ret = __cxx_atomic_load(__a, memory_order_relaxed);
|
||||
_Tp __value;
|
||||
|
|
@ -291,7 +317,7 @@ template <typename _Tp>
|
|||
_LIBCPP_HIDE_FROM_ABI _Tp
|
||||
__cxx_atomic_fetch_min(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) {
|
||||
#if __has_builtin(__atomic_fetch_min)
|
||||
return __atomic_fetch_min(std::addressof(__a->__a_value), __val, __to_gcc_order(__order));
|
||||
return __atomic_fetch_min(std::addressof(__a->__a_value), __val, std::__to_gcc_order(__order));
|
||||
#else
|
||||
_Tp __ret = __cxx_atomic_load(__a, memory_order_relaxed);
|
||||
_Tp __value;
|
||||
|
|
@ -305,7 +331,7 @@ __cxx_atomic_fetch_min(volatile __cxx_atomic_base_impl<_Tp>* __a, _Tp __val, mem
|
|||
template <typename _Tp>
|
||||
_LIBCPP_HIDE_FROM_ABI _Tp __cxx_atomic_fetch_min(__cxx_atomic_base_impl<_Tp>* __a, _Tp __val, memory_order __order) {
|
||||
#if __has_builtin(__atomic_fetch_min)
|
||||
return __atomic_fetch_min(std::addressof(__a->__a_value), __val, __to_gcc_order(__order));
|
||||
return __atomic_fetch_min(std::addressof(__a->__a_value), __val, std::__to_gcc_order(__order));
|
||||
#else
|
||||
_Tp __ret = __cxx_atomic_load(__a, memory_order_relaxed);
|
||||
_Tp __value;
|
||||
|
|
|
|||
|
|
@ -885,6 +885,7 @@ module std {
|
|||
export std.atomic.atomic_base // most of std::atomic methods are defined there
|
||||
}
|
||||
module check_memory_order { header "__atomic/check_memory_order.h" }
|
||||
module clear_padding { header "__atomic/clear_padding.h" }
|
||||
module contention_t { header "__atomic/contention_t.h" }
|
||||
module fence { header "__atomic/fence.h" }
|
||||
module floating_point_helper { header "__atomic/floating_point_helper.h" }
|
||||
|
|
|
|||
|
|
@ -0,0 +1,187 @@
|
|||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
// UNSUPPORTED: c++03
|
||||
// XFAIL: !has-128-bit-atomics
|
||||
|
||||
// Older Clang doesn't implement __builtin_clear_padding
|
||||
// XFAIL: clang-21, apple-clang-21, clang-22
|
||||
|
||||
// atomic_init is deprecated
|
||||
// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
|
||||
|
||||
// atomic<T>::atomic()
|
||||
// atomic<T>::atomic(T)
|
||||
// atomic<T>::store(T)
|
||||
// atomic<T>::exchange(T)
|
||||
// atomic_init(T)
|
||||
// libc++ maintains the invariant of the atomic to have zero for all padding bits
|
||||
|
||||
#include <atomic>
|
||||
#include <cassert>
|
||||
#include <cstring>
|
||||
#include <type_traits>
|
||||
|
||||
#include "test_macros.h"
|
||||
|
||||
struct WithTailPadding {
|
||||
int i;
|
||||
char c;
|
||||
};
|
||||
|
||||
static_assert(sizeof(WithTailPadding) > sizeof(int) + sizeof(char), "");
|
||||
|
||||
struct WithInternalPadding {
|
||||
char c;
|
||||
int i;
|
||||
};
|
||||
|
||||
static_assert(sizeof(WithInternalPadding) > sizeof(int) + sizeof(char), "");
|
||||
|
||||
struct WithInternalAndTailPadding {
|
||||
char c;
|
||||
int i;
|
||||
char c2;
|
||||
};
|
||||
|
||||
static_assert(sizeof(WithInternalAndTailPadding) > sizeof(int) + 2 * sizeof(char), "");
|
||||
|
||||
template <class T>
|
||||
void set(T& obj, int i, char c) {
|
||||
obj.i = i;
|
||||
obj.c = c;
|
||||
}
|
||||
|
||||
void set(WithInternalAndTailPadding& obj, int i, char c) {
|
||||
obj.i = i;
|
||||
obj.c = c;
|
||||
obj.c2 = c;
|
||||
}
|
||||
|
||||
template <class T>
|
||||
void initialize(T& obj, int i, char c, unsigned char pad_byte) {
|
||||
std::memset(&obj, pad_byte, sizeof(T));
|
||||
set(obj, i, c);
|
||||
}
|
||||
|
||||
template <class T>
|
||||
void assert_padding(const T& obj, unsigned char pad_byte) {
|
||||
alignas(T) unsigned char buf[sizeof(T)];
|
||||
std::memset(buf, pad_byte, sizeof(T));
|
||||
T& reference = *reinterpret_cast<T*>(buf);
|
||||
set(reference, obj.i, obj.c);
|
||||
assert(std::memcmp(&obj, &reference, sizeof(T)) == 0);
|
||||
}
|
||||
|
||||
template <class T>
|
||||
void assert_padding(const std::atomic<T>& obj, unsigned char pad_byte) {
|
||||
alignas(T) unsigned char buf[sizeof(T)];
|
||||
std::memset(buf, pad_byte, sizeof(T));
|
||||
T& reference = *reinterpret_cast<T*>(buf);
|
||||
T loaded = obj.load();
|
||||
set(reference, loaded.i, loaded.c);
|
||||
assert(std::memcmp(&obj, &reference, sizeof(T)) == 0);
|
||||
}
|
||||
|
||||
template <class T>
|
||||
void assert_padding(volatile const std::atomic<T>& obj, unsigned char pad_byte) {
|
||||
alignas(T) unsigned char buf[sizeof(T)];
|
||||
std::memset(buf, pad_byte, sizeof(T));
|
||||
T& reference = *reinterpret_cast<T*>(buf);
|
||||
T loaded = obj.load();
|
||||
set(reference, loaded.i, loaded.c);
|
||||
assert(std::memcmp(const_cast<const void*>(static_cast<const volatile void*>(&obj)), &reference, sizeof(T)) == 0);
|
||||
}
|
||||
|
||||
template <class T, template <class> class MaybeVolatile>
|
||||
void test() {
|
||||
{
|
||||
// atomic();
|
||||
// Prio to C++20, the default constructor leaves std::atomic unitialized,
|
||||
// so padding bytes are not guaranteed zero
|
||||
#if TEST_STD_VER >= 20
|
||||
MaybeVolatile<std::atomic<T>> a;
|
||||
assert_padding(a, 0);
|
||||
T loaded = a.load();
|
||||
assert(loaded.i == 0);
|
||||
assert(loaded.c == '\0');
|
||||
#endif
|
||||
}
|
||||
|
||||
{
|
||||
// atomic(T);
|
||||
T init;
|
||||
initialize(init, 10, 'a', 0xBB);
|
||||
assert_padding(init, 0xBB);
|
||||
MaybeVolatile<std::atomic<T>> a(init);
|
||||
T loaded = a.load();
|
||||
assert(loaded.i == 10);
|
||||
assert(loaded.c == 'a');
|
||||
assert_padding(a, 0);
|
||||
}
|
||||
{
|
||||
// atomic::store
|
||||
MaybeVolatile<std::atomic<T>> a;
|
||||
T value;
|
||||
initialize(value, 5, 'x', 0xAB);
|
||||
assert_padding(value, 0xAB);
|
||||
a.store(value);
|
||||
T loaded = a.load();
|
||||
assert(loaded.i == 5);
|
||||
assert(loaded.c == 'x');
|
||||
assert_padding(a, 0);
|
||||
}
|
||||
{
|
||||
// atomic::exchange
|
||||
T initial;
|
||||
initialize(initial, 1, 'a', 0x00);
|
||||
assert_padding(initial, 0x00);
|
||||
MaybeVolatile<std::atomic<T>> a(initial);
|
||||
T new_val;
|
||||
initialize(new_val, 2, 'b', 0xCD);
|
||||
assert_padding(new_val, 0xCD);
|
||||
T old = a.exchange(new_val);
|
||||
assert(old.i == 1);
|
||||
assert(old.c == 'a');
|
||||
T loaded = a.load();
|
||||
assert(loaded.i == 2);
|
||||
assert(loaded.c == 'b');
|
||||
assert_padding(a, 0);
|
||||
}
|
||||
{
|
||||
// atomic_init
|
||||
MaybeVolatile<std::atomic<T>> a;
|
||||
T init;
|
||||
initialize(init, 7, 'z', 0xEF);
|
||||
assert_padding(init, 0xEF);
|
||||
std::atomic_init(&a, init);
|
||||
T loaded = a.load();
|
||||
assert(loaded.i == 7);
|
||||
assert(loaded.c == 'z');
|
||||
assert_padding(a, 0);
|
||||
}
|
||||
}
|
||||
|
||||
template <class T>
|
||||
using TypeIdentity = T;
|
||||
|
||||
template <class T>
|
||||
using AddVolatile = typename std::add_volatile<T>::type;
|
||||
|
||||
template <class T>
|
||||
void test() {
|
||||
test<T, TypeIdentity>();
|
||||
test<T, AddVolatile>();
|
||||
}
|
||||
|
||||
int main(int, char**) {
|
||||
test<WithTailPadding>();
|
||||
test<WithInternalPadding>();
|
||||
test<WithInternalAndTailPadding>();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -24,8 +24,8 @@
|
|||
|
||||
// ADDITIONAL_COMPILE_FLAGS: -Wno-deprecated-volatile -Wno-dynamic-class-memaccess
|
||||
|
||||
#include <atomic>
|
||||
#include <cassert>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <new>
|
||||
|
||||
|
|
@ -96,7 +96,7 @@ void testAllStructsForType(T a, T b, T c, T d) {
|
|||
basic2.x = a;
|
||||
basic2.y = b;
|
||||
assert(memcmp(&basic1, &basic2, sizeof(B)) != 0);
|
||||
__builtin_clear_padding(&basic2);
|
||||
std::__clear_padding_if_needed(basic2);
|
||||
assert(memcmp(&basic1, &basic2, sizeof(B)) == 0);
|
||||
}
|
||||
|
||||
|
|
@ -117,7 +117,7 @@ void testAllStructsForType(T a, T b, T c, T d) {
|
|||
arr2.y[1] = d;
|
||||
arr2.c = 0;
|
||||
assert(memcmp(&arr1, &arr2, sizeof(A)) != 0);
|
||||
__builtin_clear_padding(&arr2);
|
||||
std::__clear_padding_if_needed(arr2);
|
||||
assert(memcmp(&arr1, &arr2, sizeof(A)) == 0);
|
||||
}
|
||||
|
||||
|
|
@ -133,7 +133,7 @@ void testAllStructsForType(T a, T b, T c, T d) {
|
|||
ptr2.x = &a;
|
||||
ptr2.y = &b;
|
||||
assert(memcmp(&ptr1, &ptr2, sizeof(P)) != 0);
|
||||
__builtin_clear_padding(&ptr2);
|
||||
std::__clear_padding_if_needed(ptr2);
|
||||
assert(memcmp(&ptr1, &ptr2, sizeof(P)) == 0);
|
||||
}
|
||||
|
||||
|
|
@ -150,7 +150,7 @@ void testAllStructsForType(T a, T b, T c, T d) {
|
|||
three2.x = a;
|
||||
three2.y = b;
|
||||
three2.z = c;
|
||||
__builtin_clear_padding(&three2);
|
||||
std::__clear_padding_if_needed(three2);
|
||||
assert(memcmp(&three1, &three2, sizeof(Three)) == 0);
|
||||
}
|
||||
|
||||
|
|
@ -165,7 +165,7 @@ void testAllStructsForType(T a, T b, T c, T d) {
|
|||
memset(&normal2, 42, sizeof(N));
|
||||
normal2.a = a;
|
||||
normal2.b = b;
|
||||
__builtin_clear_padding(&normal2);
|
||||
std::__clear_padding_if_needed(normal2);
|
||||
assert(memcmp(&normal1, &normal2, sizeof(N)) == 0);
|
||||
}
|
||||
|
||||
|
|
@ -187,7 +187,7 @@ void testAllStructsForType(T a, T b, T c, T d) {
|
|||
base2.y = d;
|
||||
base2.z = a;
|
||||
assert(memcmp(&base1, &base2, sizeof(H)) != 0);
|
||||
__builtin_clear_padding(&base2);
|
||||
std::__clear_padding_if_needed(base2);
|
||||
assert(memcmp(&base1, &base2, sizeof(H)) == 0);
|
||||
}
|
||||
}
|
||||
|
|
@ -205,7 +205,7 @@ void otherStructTests() {
|
|||
basic2->x = 1;
|
||||
basic2->y = 2;
|
||||
assert(memcmp(basic1, basic2, sizeof(B)) != 0);
|
||||
__builtin_clear_padding(basic2);
|
||||
std::__clear_padding_if_needed(*basic2);
|
||||
assert(memcmp(basic1, basic2, sizeof(B)) == 0);
|
||||
delete basic2;
|
||||
delete basic1;
|
||||
|
|
@ -223,8 +223,9 @@ void otherStructTests() {
|
|||
basic4->x = 1;
|
||||
basic4->y = 2;
|
||||
assert(memcmp(basic3, basic4, sizeof(B)) != 0);
|
||||
// libc++'s __clear_padding_if_needed does not handle volatile as std::atomic only clears the
|
||||
// incoming object which is always a non-volatile copy
|
||||
__builtin_clear_padding(const_cast<volatile B*>(basic4));
|
||||
__builtin_clear_padding(basic4);
|
||||
assert(memcmp(basic3, basic4, sizeof(B)) == 0);
|
||||
delete basic4;
|
||||
delete basic3;
|
||||
|
|
@ -245,7 +246,7 @@ void primitiveTests() {
|
|||
// no padding
|
||||
{
|
||||
int i1 = 42, i2 = 42;
|
||||
__builtin_clear_padding(&i1); // does nothing
|
||||
std::__clear_padding_if_needed(i1); // does nothing
|
||||
assert(i1 == 42);
|
||||
assert(memcmp(&i1, &i2, sizeof(int)) == 0);
|
||||
}
|
||||
|
|
@ -259,7 +260,7 @@ void primitiveTests() {
|
|||
d1 = 3.0L;
|
||||
d2 = 3.0L;
|
||||
|
||||
__builtin_clear_padding(&d1);
|
||||
std::__clear_padding_if_needed(d1);
|
||||
assert(d1 == 3.0L);
|
||||
assert(memcmp(&d1, &d2, sizeof(long double)) == 0);
|
||||
}
|
||||
|
|
@ -273,7 +274,7 @@ void primitiveTests() {
|
|||
|
||||
i1 = 37;
|
||||
i2 = 37;
|
||||
__builtin_clear_padding(&i1);
|
||||
std::__clear_padding_if_needed(i1);
|
||||
assert(i1 == 37);
|
||||
assert(memcmp(&i1, &i2, sizeof(T)) == 0);
|
||||
}
|
||||
|
|
@ -286,7 +287,7 @@ void primitiveTests() {
|
|||
memset(&c2, 0, sizeof(_Complex long double));
|
||||
c1 = 3.0L;
|
||||
c1 = 3.0L;
|
||||
__builtin_clear_padding(&c1);
|
||||
std::__clear_padding_if_needed(c1);
|
||||
//TODO
|
||||
}
|
||||
}
|
||||
|
|
@ -316,7 +317,7 @@ void structTests() {
|
|||
s2.b = true;
|
||||
|
||||
assert(memcmp(&s1, &s2, sizeof(S2)) != 0);
|
||||
__builtin_clear_padding(&s1);
|
||||
std::__clear_padding_if_needed(s1);
|
||||
assert(s1.s.x == 4);
|
||||
assert(s1.s.c == 'a');
|
||||
assert(s1.b == true);
|
||||
|
|
@ -341,7 +342,7 @@ void structTests() {
|
|||
s2.b = true;
|
||||
|
||||
assert(memcmp(&s1, &s2, sizeof(S)) != 0);
|
||||
__builtin_clear_padding(&s1);
|
||||
std::__clear_padding_if_needed(s1);
|
||||
assert(s1.l == 3.0L);
|
||||
assert(s1.b == true);
|
||||
assert(memcmp(&s1, &s2, sizeof(S)) == 0);
|
||||
|
|
@ -367,7 +368,7 @@ void structTests() {
|
|||
s2.b = true;
|
||||
|
||||
assert(memcmp(&s1, &s2, sizeof(S)) != 0);
|
||||
__builtin_clear_padding(&s1);
|
||||
std::__clear_padding_if_needed(s1);
|
||||
assert(s1.i == 4);
|
||||
assert(s1.b == true);
|
||||
assert(memcmp(&s1, &s2, sizeof(S)) == 0);
|
||||
|
|
@ -394,7 +395,7 @@ void structTests() {
|
|||
s2.c2 = 'b';
|
||||
|
||||
assert(memcmp(&s1, &s2, sizeof(S)) != 0);
|
||||
__builtin_clear_padding(&s1);
|
||||
std::__clear_padding_if_needed(s1);
|
||||
assert(s1.c1 == 'a');
|
||||
assert(s1.c2 == 'b');
|
||||
assert(memcmp(&s1, &s2, sizeof(S)) == 0);
|
||||
|
|
@ -425,7 +426,7 @@ void structTests() {
|
|||
s2.c3 = 'c';
|
||||
|
||||
assert(memcmp(&s1, &s2, sizeof(S)) != 0);
|
||||
__builtin_clear_padding(&s1);
|
||||
std::__clear_padding_if_needed(s1);
|
||||
assert(s1.c1 == 'a');
|
||||
assert(s1.c2 == 'b');
|
||||
assert(s1.c3 == 'c');
|
||||
|
|
@ -469,6 +470,10 @@ void structTests() {
|
|||
s2.y = 'a';
|
||||
s1.z = true;
|
||||
s2.z = true;
|
||||
// for non trivially copyable types, __builtin_clear_padding matches gcc's behaviour
|
||||
// where the pointer to the object must be known to the compiler
|
||||
// using libc++'s __clear_padding_if_needed would make the object "unknown" to the compiler
|
||||
// note that in atomic<T>, T must be trivially copyable
|
||||
__builtin_clear_padding(&s2);
|
||||
assert(s2.x == 0xFFFFFFFF);
|
||||
assert(s2.y == 'a');
|
||||
|
|
@ -530,6 +535,10 @@ void structTests() {
|
|||
s2.y = 'a';
|
||||
s1.z = true;
|
||||
s2.z = true;
|
||||
// for non trivially copyable types, __builtin_clear_padding matches gcc's behaviour
|
||||
// where the pointer to the object must be known to the compiler
|
||||
// using libc++'s __clear_padding_if_needed would make the object "unknown" to the compiler
|
||||
// note that in atomic<T>, T must be trivially copyable
|
||||
__builtin_clear_padding(&s2);
|
||||
assert(s2.x1 == 0xFFFFFFFF);
|
||||
assert(s2.x2 == 0xFAFAFAFA);
|
||||
|
|
@ -592,6 +601,10 @@ void structTests() {
|
|||
s2.y = 'a';
|
||||
s1.z = true;
|
||||
s2.z = true;
|
||||
// for non trivially copyable types, __builtin_clear_padding matches gcc's behaviour
|
||||
// where the pointer to the object must be known to the compiler
|
||||
// using libc++'s __clear_padding_if_needed would make the object "unknown" to the compiler
|
||||
// note that in atomic<T>, T must be trivially copyable
|
||||
__builtin_clear_padding(&s2);
|
||||
assert(memcmp(&s1, &s2, sizeof(S)) == 0);
|
||||
}
|
||||
|
|
@ -639,6 +652,10 @@ void structTests() {
|
|||
s2.b2 = true;
|
||||
s1.s = true;
|
||||
s2.s = true;
|
||||
// for non trivially copyable types, __builtin_clear_padding matches gcc's behaviour
|
||||
// where the pointer to the object must be known to the compiler
|
||||
// using libc++'s __clear_padding_if_needed would make the object "unknown" to the compiler
|
||||
// note that in atomic<T>, T must be trivially copyable
|
||||
__builtin_clear_padding(&s2);
|
||||
assert(memcmp(&s1, &s2, sizeof(S)) == 0);
|
||||
}
|
||||
|
|
@ -665,7 +682,7 @@ void structTests() {
|
|||
s2.b3 = 27;
|
||||
s1.b4 = 3;
|
||||
s2.b4 = 3;
|
||||
__builtin_clear_padding(&s2);
|
||||
std::__clear_padding_if_needed(s2);
|
||||
assert(memcmp(&s1, &s2, sizeof(S)) == 0);
|
||||
}
|
||||
|
||||
|
|
@ -685,7 +702,7 @@ void structTests() {
|
|||
s1.b2 = 1;
|
||||
s2.b1 = 1;
|
||||
s2.b2 = 1;
|
||||
__builtin_clear_padding(&s2);
|
||||
std::__clear_padding_if_needed(s2);
|
||||
assert(memcmp(&s1, &s2, sizeof(S)) == 0);
|
||||
}
|
||||
|
||||
|
|
@ -727,7 +744,7 @@ void unionTests() {
|
|||
u1.c = '4';
|
||||
u2.c = '4';
|
||||
|
||||
__builtin_clear_padding(&u1); // should have no effect
|
||||
std::__clear_padding_if_needed(u1); // should have no effect
|
||||
assert(u1.c == '4');
|
||||
|
||||
assert(memcmp(&u1, &u2, sizeof(u)) == 0);
|
||||
|
|
@ -752,12 +769,14 @@ void unionTests() {
|
|||
u2.s1.c1 = '4';
|
||||
|
||||
assert(memcmp(&u1, &u2, sizeof(u)) != 0);
|
||||
__builtin_clear_padding(&u1);
|
||||
std::__clear_padding_if_needed(u1);
|
||||
assert(u1.s1.c1 == '4');
|
||||
assert(memcmp(&u1, &u2, sizeof(u)) == 0);
|
||||
}
|
||||
}
|
||||
|
||||
// libc++'s __clear_padding_if_needed does not handle arrays
|
||||
// as std::atomic does not support arrays
|
||||
void arrayTests() {
|
||||
// no padding
|
||||
{
|
||||
|
|
@ -859,7 +878,7 @@ void vectorTests() {
|
|||
v2[7] = true;
|
||||
v2[8] = false;
|
||||
|
||||
__builtin_clear_padding(&v1);
|
||||
std::__clear_padding_if_needed(v1);
|
||||
assert(v1[0] == true);
|
||||
assert(v1[1] == false);
|
||||
assert(v1[7] == true);
|
||||
|
|
@ -880,7 +899,7 @@ void vectorTests() {
|
|||
v2[1] = 2.0L;
|
||||
v2[2] = 3.0L;
|
||||
|
||||
__builtin_clear_padding(&v1);
|
||||
std::__clear_padding_if_needed(v1);
|
||||
assert(v1[0] == 1.0L);
|
||||
assert(v1[1] == 2.0L);
|
||||
assert(v1[2] == 3.0L);
|
||||
|
|
@ -900,7 +919,7 @@ void vectorTests() {
|
|||
v2[1] = 2.0f;
|
||||
v2[2] = 3.0f;
|
||||
|
||||
__builtin_clear_padding(&v1);
|
||||
std::__clear_padding_if_needed(v1);
|
||||
assert(v1[0] == 1.0f);
|
||||
assert(v1[1] == 2.0f);
|
||||
assert(v1[2] == 3.0f);
|
||||
270
libcxx/test/std/atomics/atomics.types.generic/padding.pass.cpp
Normal file
270
libcxx/test/std/atomics/atomics.types.generic/padding.pass.cpp
Normal file
|
|
@ -0,0 +1,270 @@
|
|||
//===----------------------------------------------------------------------===//
|
||||
//
|
||||
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
||||
// See https://llvm.org/LICENSE.txt for license information.
|
||||
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
||||
//
|
||||
//===----------------------------------------------------------------------===//
|
||||
// UNSUPPORTED: c++03
|
||||
// XFAIL: !has-128-bit-atomics
|
||||
|
||||
// Older Clang doesn't implement __builtin_clear_padding
|
||||
// XFAIL: clang-21, apple-clang-21, clang-22
|
||||
|
||||
// atomic<T>::compare_exchange_weak
|
||||
// atomic<T>::compare_exchange_strong
|
||||
// CAS should work on types with padding bits
|
||||
|
||||
#include <atomic>
|
||||
#include <cassert>
|
||||
#include <cstring>
|
||||
#include <type_traits>
|
||||
|
||||
struct WithTailPadding {
|
||||
int i;
|
||||
char c;
|
||||
};
|
||||
|
||||
static_assert(sizeof(WithTailPadding) > sizeof(int) + sizeof(char), "");
|
||||
|
||||
struct WithInternalPadding {
|
||||
char c;
|
||||
int i;
|
||||
};
|
||||
|
||||
static_assert(sizeof(WithInternalPadding) > sizeof(int) + sizeof(char), "");
|
||||
|
||||
struct WithInternalAndTailPadding {
|
||||
char c;
|
||||
int i;
|
||||
char c2;
|
||||
};
|
||||
|
||||
static_assert(sizeof(WithInternalAndTailPadding) > sizeof(int) + 2 * sizeof(char), "");
|
||||
|
||||
template <class T>
|
||||
void set(T& obj, int i, char c) {
|
||||
obj.i = i;
|
||||
obj.c = c;
|
||||
}
|
||||
|
||||
void set(WithInternalAndTailPadding& obj, int i, char c) {
|
||||
obj.i = i;
|
||||
obj.c = c;
|
||||
obj.c2 = c;
|
||||
}
|
||||
|
||||
template <class T>
|
||||
void initialize(T& obj, int i, char c, unsigned char pad_byte) {
|
||||
std::memset(&obj, pad_byte, sizeof(T));
|
||||
set(obj, i, c);
|
||||
}
|
||||
|
||||
template <class T>
|
||||
void assert_padding(const T& obj, unsigned char pad_byte) {
|
||||
alignas(T) unsigned char buf[sizeof(T)];
|
||||
std::memset(buf, pad_byte, sizeof(T));
|
||||
T& reference = *reinterpret_cast<T*>(buf);
|
||||
set(reference, obj.i, obj.c);
|
||||
assert(std::memcmp(&obj, &reference, sizeof(T)) == 0);
|
||||
}
|
||||
|
||||
template <class T, template <class> class MaybeVolatile>
|
||||
void test() {
|
||||
{
|
||||
// compare_exchange_strong
|
||||
// CAS should succeed when only padding differs in expected; expected is unchanged.
|
||||
MaybeVolatile<std::atomic<T>> a;
|
||||
|
||||
T init;
|
||||
initialize(init, 10, 'a', 0xBB);
|
||||
assert_padding(init, 0xBB);
|
||||
a.store(init);
|
||||
|
||||
T expected;
|
||||
initialize(expected, 10, 'a', 0xAA);
|
||||
assert_padding(expected, 0xAA);
|
||||
|
||||
T original_expected; // make a copy including padding bits
|
||||
std::memcpy(&original_expected, &expected, sizeof(T));
|
||||
|
||||
T new_value;
|
||||
initialize(new_value, 42, 'b', 0xCC);
|
||||
assert_padding(new_value, 0xCC);
|
||||
|
||||
bool r = a.compare_exchange_strong(expected, new_value);
|
||||
|
||||
assert(r);
|
||||
assert(std::memcmp(&expected, &original_expected, sizeof(T)) == 0);
|
||||
T loaded = a.load();
|
||||
assert(loaded.i == 42);
|
||||
assert(loaded.c == 'b');
|
||||
}
|
||||
|
||||
{
|
||||
// compare_exchange_strong
|
||||
// atomic and expected values are different; failure
|
||||
MaybeVolatile<std::atomic<T>> a;
|
||||
T stored;
|
||||
initialize(stored, 10, 'a', 0xBB);
|
||||
assert_padding(stored, 0xBB);
|
||||
a.store(stored);
|
||||
|
||||
T expected;
|
||||
initialize(expected, 99, 'a', 0xAA);
|
||||
assert_padding(expected, 0xAA);
|
||||
T new_value;
|
||||
initialize(new_value, 42, 'b', 0xCC);
|
||||
assert_padding(new_value, 0xCC);
|
||||
|
||||
bool r = a.compare_exchange_strong(expected, new_value);
|
||||
|
||||
assert(!r);
|
||||
assert(expected.i == 10);
|
||||
assert(expected.c == 'a');
|
||||
// expected is updated to contain atomic's value and in libc++, the paddings bits are always zero
|
||||
#ifdef _LIBCPP_VERSION
|
||||
assert_padding(expected, 0);
|
||||
#endif
|
||||
T loaded = a.load();
|
||||
assert(loaded.i == 10);
|
||||
assert(loaded.c == 'a');
|
||||
}
|
||||
{
|
||||
// compare_exchange_strong
|
||||
// atomic and expected are the same, including padding
|
||||
MaybeVolatile<std::atomic<T>> a;
|
||||
T init;
|
||||
initialize(init, 10, 'a', 0x00);
|
||||
assert_padding(init, 0x00);
|
||||
a.store(init);
|
||||
|
||||
T expected;
|
||||
initialize(expected, 10, 'a', 0x00);
|
||||
assert_padding(expected, 0x00);
|
||||
|
||||
T original_expected; // make a copy including padding bits
|
||||
std::memcpy(&original_expected, &expected, sizeof(T));
|
||||
|
||||
T new_value;
|
||||
initialize(new_value, 42, 'b', 0x42);
|
||||
assert_padding(new_value, 0x42);
|
||||
|
||||
bool r = a.compare_exchange_strong(expected, new_value);
|
||||
|
||||
assert(r);
|
||||
assert(std::memcmp(&expected, &original_expected, sizeof(T)) == 0);
|
||||
T loaded = a.load();
|
||||
assert(loaded.i == 42);
|
||||
assert(loaded.c == 'b');
|
||||
}
|
||||
|
||||
{
|
||||
// compare_exchange_weak
|
||||
// atomic and expected only differs in padding bits. It should either succeed or spuriously fail
|
||||
MaybeVolatile<std::atomic<T>> a;
|
||||
T stored;
|
||||
initialize(stored, 10, 'a', 0xBB);
|
||||
assert_padding(stored, 0xBB);
|
||||
a.store(stored);
|
||||
|
||||
T new_value;
|
||||
initialize(new_value, 42, 'b', 0xCC);
|
||||
assert_padding(new_value, 0xCC);
|
||||
|
||||
T original_expected;
|
||||
initialize(original_expected, 10, 'a', 0xAA);
|
||||
assert_padding(original_expected, 0xAA);
|
||||
|
||||
bool r = false;
|
||||
const auto max_attempts = 100;
|
||||
auto current_attempt = 0;
|
||||
while (!r) {
|
||||
++current_attempt;
|
||||
assert(current_attempt < max_attempts && "compare_exchange_weak did not succeed within 3 seconds");
|
||||
T expected;
|
||||
initialize(expected, 10, 'a', 0xAA);
|
||||
assert_padding(expected, 0xAA);
|
||||
r = a.compare_exchange_weak(expected, new_value);
|
||||
if (r) {
|
||||
assert(std::memcmp(&expected, &original_expected, sizeof(T)) == 0);
|
||||
} else {
|
||||
// Spurious failure: expected is updated to the current atomic value.
|
||||
assert(expected.i == 10);
|
||||
assert(expected.c == 'a');
|
||||
// expected is updated to contain atomic's value and in libc++, the paddings bits are always zero
|
||||
#ifdef _LIBCPP_VERSION
|
||||
assert_padding(expected, 0);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
T loaded = a.load();
|
||||
assert(loaded.i == 42);
|
||||
assert(loaded.c == 'b');
|
||||
}
|
||||
|
||||
{
|
||||
// compare_exchange_strong
|
||||
// atomic and expected values are different; failure
|
||||
MaybeVolatile<std::atomic<T>> a;
|
||||
T stored;
|
||||
initialize(stored, 10, 'a', 0xBB);
|
||||
assert_padding(stored, 0xBB);
|
||||
a.store(stored);
|
||||
|
||||
T expected;
|
||||
initialize(expected, 99, 'a', 0xAA);
|
||||
assert_padding(expected, 0xAA);
|
||||
T new_value;
|
||||
initialize(new_value, 42, 'b', 0xCC);
|
||||
assert_padding(new_value, 0xCC);
|
||||
|
||||
bool r = a.compare_exchange_weak(expected, new_value);
|
||||
|
||||
assert(!r);
|
||||
assert(expected.i == 10);
|
||||
assert(expected.c == 'a');
|
||||
// expected is updated to contain atomic's value and in libc++, the paddings bits are always zero
|
||||
#ifdef _LIBCPP_VERSION
|
||||
assert_padding(expected, 0);
|
||||
#endif
|
||||
T loaded = a.load();
|
||||
assert(loaded.i == 10);
|
||||
assert(loaded.c == 'a');
|
||||
}
|
||||
|
||||
{
|
||||
// Types with unique object representations skip the padding-clearing path.
|
||||
MaybeVolatile<std::atomic<int>> a(1);
|
||||
int expected = 1;
|
||||
assert(a.compare_exchange_strong(expected, 2));
|
||||
assert(expected == 1);
|
||||
assert(a.load() == 2);
|
||||
|
||||
expected = 3;
|
||||
assert(!a.compare_exchange_strong(expected, 4));
|
||||
assert(expected == 2);
|
||||
assert(a.load() == 2);
|
||||
}
|
||||
}
|
||||
|
||||
template <class T>
|
||||
using TypeIdentity = T;
|
||||
|
||||
template <class T>
|
||||
using AddVolatile = typename std::add_volatile<T>::type;
|
||||
|
||||
template <class T>
|
||||
void test() {
|
||||
test<T, TypeIdentity>();
|
||||
test<T, AddVolatile>();
|
||||
}
|
||||
|
||||
int main(int, char**) {
|
||||
test<WithTailPadding>();
|
||||
test<WithInternalPadding>();
|
||||
test<WithInternalAndTailPadding>();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -91,29 +91,20 @@ features = [
|
|||
),
|
||||
actions=[AddLinkFlag("-latomic")],
|
||||
),
|
||||
Feature(
|
||||
name="has-64-bit-atomics",
|
||||
when=lambda cfg: sourceBuilds(
|
||||
cfg,
|
||||
"""
|
||||
#include <atomic>
|
||||
struct Large { char storage[64/8]; };
|
||||
std::atomic<Large> x;
|
||||
int main(int, char**) { (void)x.load(); (void)x.is_lock_free(); return 0; }
|
||||
""",
|
||||
),
|
||||
),
|
||||
Feature(
|
||||
name="has-1024-bit-atomics",
|
||||
when=lambda cfg: sourceBuilds(
|
||||
cfg,
|
||||
"""
|
||||
#include <atomic>
|
||||
struct Large { char storage[1024/8]; };
|
||||
std::atomic<Large> x;
|
||||
int main(int, char**) { (void)x.load(); (void)x.is_lock_free(); return 0; }
|
||||
""",
|
||||
),
|
||||
*(
|
||||
Feature(
|
||||
name=f"has-{n}-bit-atomics",
|
||||
when=lambda cfg, n=n: sourceBuilds(
|
||||
cfg,
|
||||
f"""
|
||||
#include <atomic>
|
||||
struct Large {{ char storage[{n}/8]; }};
|
||||
std::atomic<Large> x;
|
||||
int main(int, char**) {{ (void)x.load(); (void)x.is_lock_free(); return 0; }}
|
||||
""",
|
||||
),
|
||||
)
|
||||
for n in [64, 128, 1024]
|
||||
),
|
||||
# Tests that require 64-bit architecture
|
||||
Feature(
|
||||
|
|
|
|||
Loading…
Reference in a new issue