3 // Copyright (C) 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
5 // This file is part of the GNU ISO C++ Library. This library is free
6 // software; you can redistribute it and/or modify it under the
7 // terms of the GNU General Public License as published by the
8 // Free Software Foundation; either version 3, or (at your option)
11 // This library is distributed in the hope that it will be useful,
12 // but WITHOUT ANY WARRANTY; without even the implied warranty of
13 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 // GNU General Public License for more details.
16 // Under Section 7 of GPL version 3, you are granted additional
17 // permissions described in the GCC Runtime Library Exception, version
18 // 3.1, as published by the Free Software Foundation.
20 // You should have received a copy of the GNU General Public License and
21 // a copy of the GCC Runtime Library Exception along with this program;
22 // see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
23 // <http://www.gnu.org/licenses/>.
25 /** @file bits/atomic_base.h
26 * This is an internal header file, included by other library headers.
27 * Do not attempt to use it directly. @headername{atomic}
30 #ifndef _GLIBCXX_ATOMIC_BASE_H
31 #define _GLIBCXX_ATOMIC_BASE_H 1
33 #pragma GCC system_header
35 #include <bits/c++config.h>
38 #include <bits/atomic_lockfree_defines.h>
40 namespace std _GLIBCXX_VISIBILITY(default)
42 _GLIBCXX_BEGIN_NAMESPACE_VERSION
45 * @defgroup atomics Atomics
47 * Components for performing atomic operations.
51 /// Enumeration for memory_order
52 typedef enum memory_order
62 // Drop release ordering as per [atomics.types.operations.req]/21
63 constexpr memory_order
64 __cmpexch_failure_order(memory_order __m) noexcept
66 return __m == memory_order_acq_rel ? memory_order_acquire
67 : __m == memory_order_release ? memory_order_relaxed : __m;
71 atomic_thread_fence(memory_order __m) noexcept
72 { __atomic_thread_fence(__m); }
75 atomic_signal_fence(memory_order __m) noexcept
76 { __atomic_signal_fence(__m); }
79 template<typename _Tp>
81 kill_dependency(_Tp __y) noexcept
88 // Base types for atomics.
89 template<typename _IntTp>
93 typedef __atomic_base<char> atomic_char;
96 typedef __atomic_base<signed char> atomic_schar;
99 typedef __atomic_base<unsigned char> atomic_uchar;
102 typedef __atomic_base<short> atomic_short;
105 typedef __atomic_base<unsigned short> atomic_ushort;
108 typedef __atomic_base<int> atomic_int;
111 typedef __atomic_base<unsigned int> atomic_uint;
114 typedef __atomic_base<long> atomic_long;
117 typedef __atomic_base<unsigned long> atomic_ulong;
120 typedef __atomic_base<long long> atomic_llong;
123 typedef __atomic_base<unsigned long long> atomic_ullong;
126 typedef __atomic_base<wchar_t> atomic_wchar_t;
129 typedef __atomic_base<char16_t> atomic_char16_t;
132 typedef __atomic_base<char32_t> atomic_char32_t;
135 typedef __atomic_base<char32_t> atomic_char32_t;
138 /// atomic_int_least8_t
139 typedef __atomic_base<int_least8_t> atomic_int_least8_t;
141 /// atomic_uint_least8_t
142 typedef __atomic_base<uint_least8_t> atomic_uint_least8_t;
144 /// atomic_int_least16_t
145 typedef __atomic_base<int_least16_t> atomic_int_least16_t;
147 /// atomic_uint_least16_t
148 typedef __atomic_base<uint_least16_t> atomic_uint_least16_t;
150 /// atomic_int_least32_t
151 typedef __atomic_base<int_least32_t> atomic_int_least32_t;
153 /// atomic_uint_least32_t
154 typedef __atomic_base<uint_least32_t> atomic_uint_least32_t;
156 /// atomic_int_least64_t
157 typedef __atomic_base<int_least64_t> atomic_int_least64_t;
159 /// atomic_uint_least64_t
160 typedef __atomic_base<uint_least64_t> atomic_uint_least64_t;
163 /// atomic_int_fast8_t
164 typedef __atomic_base<int_fast8_t> atomic_int_fast8_t;
166 /// atomic_uint_fast8_t
167 typedef __atomic_base<uint_fast8_t> atomic_uint_fast8_t;
169 /// atomic_int_fast16_t
170 typedef __atomic_base<int_fast16_t> atomic_int_fast16_t;
172 /// atomic_uint_fast16_t
173 typedef __atomic_base<uint_fast16_t> atomic_uint_fast16_t;
175 /// atomic_int_fast32_t
176 typedef __atomic_base<int_fast32_t> atomic_int_fast32_t;
178 /// atomic_uint_fast32_t
179 typedef __atomic_base<uint_fast32_t> atomic_uint_fast32_t;
181 /// atomic_int_fast64_t
182 typedef __atomic_base<int_fast64_t> atomic_int_fast64_t;
184 /// atomic_uint_fast64_t
185 typedef __atomic_base<uint_fast64_t> atomic_uint_fast64_t;
189 typedef __atomic_base<intptr_t> atomic_intptr_t;
192 typedef __atomic_base<uintptr_t> atomic_uintptr_t;
195 typedef __atomic_base<size_t> atomic_size_t;
198 typedef __atomic_base<intmax_t> atomic_intmax_t;
201 typedef __atomic_base<uintmax_t> atomic_uintmax_t;
204 typedef __atomic_base<ptrdiff_t> atomic_ptrdiff_t;
207 #define ATOMIC_VAR_INIT(_VI) { _VI }
209 template<typename _Tp>
212 template<typename _Tp>
215 /* The target's "set" value for test-and-set may not be exactly 1. */
216 #if __GCC_ATOMIC_TEST_AND_SET_TRUEVAL == 1
217 typedef bool __atomic_flag_data_type;
219 typedef unsigned char __atomic_flag_data_type;
223 * @brief Base type for atomic_flag.
225 * Base type is POD with data, allowing atomic_flag to derive from
226 * it and meet the standard layout type requirement. In addition to
227 * compatibilty with a C interface, this allows different
228 * implementations of atomic_flag to use the same atomic operation
229 * functions, via a standard conversion to the __atomic_flag_base
232 _GLIBCXX_BEGIN_EXTERN_C
234 struct __atomic_flag_base
236 __atomic_flag_data_type _M_i;
239 _GLIBCXX_END_EXTERN_C
241 #define ATOMIC_FLAG_INIT { 0 }
244 struct atomic_flag : public __atomic_flag_base
246 atomic_flag() noexcept = default;
247 ~atomic_flag() noexcept = default;
248 atomic_flag(const atomic_flag&) = delete;
249 atomic_flag& operator=(const atomic_flag&) = delete;
250 atomic_flag& operator=(const atomic_flag&) volatile = delete;
252 // Conversion to ATOMIC_FLAG_INIT.
253 constexpr atomic_flag(bool __i) noexcept
254 : __atomic_flag_base{ _S_init(__i) }
258 test_and_set(memory_order __m = memory_order_seq_cst) noexcept
260 return __atomic_test_and_set (&_M_i, __m);
264 test_and_set(memory_order __m = memory_order_seq_cst) volatile noexcept
266 return __atomic_test_and_set (&_M_i, __m);
270 clear(memory_order __m = memory_order_seq_cst) noexcept
272 __glibcxx_assert(__m != memory_order_consume);
273 __glibcxx_assert(__m != memory_order_acquire);
274 __glibcxx_assert(__m != memory_order_acq_rel);
276 __atomic_clear (&_M_i, __m);
280 clear(memory_order __m = memory_order_seq_cst) volatile noexcept
282 __glibcxx_assert(__m != memory_order_consume);
283 __glibcxx_assert(__m != memory_order_acquire);
284 __glibcxx_assert(__m != memory_order_acq_rel);
286 __atomic_clear (&_M_i, __m);
290 static constexpr __atomic_flag_data_type
292 { return __i ? __GCC_ATOMIC_TEST_AND_SET_TRUEVAL : 0; }
296 /// Base class for atomic integrals.
298 // For each of the integral types, define atomic_[integral type] struct
302 // atomic_schar signed char
303 // atomic_uchar unsigned char
304 // atomic_short short
305 // atomic_ushort unsigned short
307 // atomic_uint unsigned int
309 // atomic_ulong unsigned long
310 // atomic_llong long long
311 // atomic_ullong unsigned long long
312 // atomic_char16_t char16_t
313 // atomic_char32_t char32_t
314 // atomic_wchar_t wchar_t
316 // NB: Assuming _ITp is an integral scalar type that is 1, 2, 4, or
317 // 8 bytes, since that is what GCC built-in functions for atomic
318 // memory access expect.
319 template<typename _ITp>
323 typedef _ITp __int_type;
328 __atomic_base() noexcept = default;
329 ~__atomic_base() noexcept = default;
330 __atomic_base(const __atomic_base&) = delete;
331 __atomic_base& operator=(const __atomic_base&) = delete;
332 __atomic_base& operator=(const __atomic_base&) volatile = delete;
334 // Requires __int_type convertible to _M_i.
335 constexpr __atomic_base(__int_type __i) noexcept : _M_i (__i) { }
337 operator __int_type() const noexcept
340 operator __int_type() const volatile noexcept
344 operator=(__int_type __i) noexcept
351 operator=(__int_type __i) volatile noexcept
358 operator++(int) noexcept
359 { return fetch_add(1); }
362 operator++(int) volatile noexcept
363 { return fetch_add(1); }
366 operator--(int) noexcept
367 { return fetch_sub(1); }
370 operator--(int) volatile noexcept
371 { return fetch_sub(1); }
374 operator++() noexcept
375 { return __atomic_add_fetch(&_M_i, 1, memory_order_seq_cst); }
378 operator++() volatile noexcept
379 { return __atomic_add_fetch(&_M_i, 1, memory_order_seq_cst); }
382 operator--() noexcept
383 { return __atomic_sub_fetch(&_M_i, 1, memory_order_seq_cst); }
386 operator--() volatile noexcept
387 { return __atomic_sub_fetch(&_M_i, 1, memory_order_seq_cst); }
390 operator+=(__int_type __i) noexcept
391 { return __atomic_add_fetch(&_M_i, __i, memory_order_seq_cst); }
394 operator+=(__int_type __i) volatile noexcept
395 { return __atomic_add_fetch(&_M_i, __i, memory_order_seq_cst); }
398 operator-=(__int_type __i) noexcept
399 { return __atomic_sub_fetch(&_M_i, __i, memory_order_seq_cst); }
402 operator-=(__int_type __i) volatile noexcept
403 { return __atomic_sub_fetch(&_M_i, __i, memory_order_seq_cst); }
406 operator&=(__int_type __i) noexcept
407 { return __atomic_and_fetch(&_M_i, __i, memory_order_seq_cst); }
410 operator&=(__int_type __i) volatile noexcept
411 { return __atomic_and_fetch(&_M_i, __i, memory_order_seq_cst); }
414 operator|=(__int_type __i) noexcept
415 { return __atomic_or_fetch(&_M_i, __i, memory_order_seq_cst); }
418 operator|=(__int_type __i) volatile noexcept
419 { return __atomic_or_fetch(&_M_i, __i, memory_order_seq_cst); }
422 operator^=(__int_type __i) noexcept
423 { return __atomic_xor_fetch(&_M_i, __i, memory_order_seq_cst); }
426 operator^=(__int_type __i) volatile noexcept
427 { return __atomic_xor_fetch(&_M_i, __i, memory_order_seq_cst); }
430 is_lock_free() const noexcept
431 { return __atomic_is_lock_free (sizeof (_M_i), &_M_i); }
434 is_lock_free() const volatile noexcept
435 { return __atomic_is_lock_free (sizeof (_M_i), &_M_i); }
438 store(__int_type __i, memory_order __m = memory_order_seq_cst) noexcept
440 __glibcxx_assert(__m != memory_order_acquire);
441 __glibcxx_assert(__m != memory_order_acq_rel);
442 __glibcxx_assert(__m != memory_order_consume);
444 __atomic_store_n(&_M_i, __i, __m);
448 store(__int_type __i,
449 memory_order __m = memory_order_seq_cst) volatile noexcept
451 __glibcxx_assert(__m != memory_order_acquire);
452 __glibcxx_assert(__m != memory_order_acq_rel);
453 __glibcxx_assert(__m != memory_order_consume);
455 __atomic_store_n(&_M_i, __i, __m);
459 load(memory_order __m = memory_order_seq_cst) const noexcept
461 __glibcxx_assert(__m != memory_order_release);
462 __glibcxx_assert(__m != memory_order_acq_rel);
464 return __atomic_load_n(&_M_i, __m);
468 load(memory_order __m = memory_order_seq_cst) const volatile noexcept
470 __glibcxx_assert(__m != memory_order_release);
471 __glibcxx_assert(__m != memory_order_acq_rel);
473 return __atomic_load_n(&_M_i, __m);
477 exchange(__int_type __i,
478 memory_order __m = memory_order_seq_cst) noexcept
480 return __atomic_exchange_n(&_M_i, __i, __m);
485 exchange(__int_type __i,
486 memory_order __m = memory_order_seq_cst) volatile noexcept
488 return __atomic_exchange_n(&_M_i, __i, __m);
492 compare_exchange_weak(__int_type& __i1, __int_type __i2,
493 memory_order __m1, memory_order __m2) noexcept
495 __glibcxx_assert(__m2 != memory_order_release);
496 __glibcxx_assert(__m2 != memory_order_acq_rel);
497 __glibcxx_assert(__m2 <= __m1);
499 return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 1, __m1, __m2);
503 compare_exchange_weak(__int_type& __i1, __int_type __i2,
505 memory_order __m2) volatile noexcept
507 __glibcxx_assert(__m2 != memory_order_release);
508 __glibcxx_assert(__m2 != memory_order_acq_rel);
509 __glibcxx_assert(__m2 <= __m1);
511 return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 1, __m1, __m2);
515 compare_exchange_weak(__int_type& __i1, __int_type __i2,
516 memory_order __m = memory_order_seq_cst) noexcept
518 return compare_exchange_weak(__i1, __i2, __m,
519 __cmpexch_failure_order(__m));
523 compare_exchange_weak(__int_type& __i1, __int_type __i2,
524 memory_order __m = memory_order_seq_cst) volatile noexcept
526 return compare_exchange_weak(__i1, __i2, __m,
527 __cmpexch_failure_order(__m));
531 compare_exchange_strong(__int_type& __i1, __int_type __i2,
532 memory_order __m1, memory_order __m2) noexcept
534 __glibcxx_assert(__m2 != memory_order_release);
535 __glibcxx_assert(__m2 != memory_order_acq_rel);
536 __glibcxx_assert(__m2 <= __m1);
538 return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 0, __m1, __m2);
542 compare_exchange_strong(__int_type& __i1, __int_type __i2,
544 memory_order __m2) volatile noexcept
546 __glibcxx_assert(__m2 != memory_order_release);
547 __glibcxx_assert(__m2 != memory_order_acq_rel);
548 __glibcxx_assert(__m2 <= __m1);
550 return __atomic_compare_exchange_n(&_M_i, &__i1, __i2, 0, __m1, __m2);
554 compare_exchange_strong(__int_type& __i1, __int_type __i2,
555 memory_order __m = memory_order_seq_cst) noexcept
557 return compare_exchange_strong(__i1, __i2, __m,
558 __cmpexch_failure_order(__m));
562 compare_exchange_strong(__int_type& __i1, __int_type __i2,
563 memory_order __m = memory_order_seq_cst) volatile noexcept
565 return compare_exchange_strong(__i1, __i2, __m,
566 __cmpexch_failure_order(__m));
570 fetch_add(__int_type __i,
571 memory_order __m = memory_order_seq_cst) noexcept
572 { return __atomic_fetch_add(&_M_i, __i, __m); }
575 fetch_add(__int_type __i,
576 memory_order __m = memory_order_seq_cst) volatile noexcept
577 { return __atomic_fetch_add(&_M_i, __i, __m); }
580 fetch_sub(__int_type __i,
581 memory_order __m = memory_order_seq_cst) noexcept
582 { return __atomic_fetch_sub(&_M_i, __i, __m); }
585 fetch_sub(__int_type __i,
586 memory_order __m = memory_order_seq_cst) volatile noexcept
587 { return __atomic_fetch_sub(&_M_i, __i, __m); }
590 fetch_and(__int_type __i,
591 memory_order __m = memory_order_seq_cst) noexcept
592 { return __atomic_fetch_and(&_M_i, __i, __m); }
595 fetch_and(__int_type __i,
596 memory_order __m = memory_order_seq_cst) volatile noexcept
597 { return __atomic_fetch_and(&_M_i, __i, __m); }
600 fetch_or(__int_type __i,
601 memory_order __m = memory_order_seq_cst) noexcept
602 { return __atomic_fetch_or(&_M_i, __i, __m); }
605 fetch_or(__int_type __i,
606 memory_order __m = memory_order_seq_cst) volatile noexcept
607 { return __atomic_fetch_or(&_M_i, __i, __m); }
610 fetch_xor(__int_type __i,
611 memory_order __m = memory_order_seq_cst) noexcept
612 { return __atomic_fetch_xor(&_M_i, __i, __m); }
615 fetch_xor(__int_type __i,
616 memory_order __m = memory_order_seq_cst) volatile noexcept
617 { return __atomic_fetch_xor(&_M_i, __i, __m); }
621 /// Partial specialization for pointer types.
622 template<typename _PTp>
623 struct __atomic_base<_PTp*>
626 typedef _PTp* __pointer_type;
630 // Factored out to facilitate explicit specialization.
632 _M_type_size(ptrdiff_t __d) { return __d * sizeof(_PTp); }
635 _M_type_size(ptrdiff_t __d) volatile { return __d * sizeof(_PTp); }
638 __atomic_base() noexcept = default;
639 ~__atomic_base() noexcept = default;
640 __atomic_base(const __atomic_base&) = delete;
641 __atomic_base& operator=(const __atomic_base&) = delete;
642 __atomic_base& operator=(const __atomic_base&) volatile = delete;
644 // Requires __pointer_type convertible to _M_p.
645 constexpr __atomic_base(__pointer_type __p) noexcept : _M_p (__p) { }
647 operator __pointer_type() const noexcept
650 operator __pointer_type() const volatile noexcept
654 operator=(__pointer_type __p) noexcept
661 operator=(__pointer_type __p) volatile noexcept
668 operator++(int) noexcept
669 { return fetch_add(1); }
672 operator++(int) volatile noexcept
673 { return fetch_add(1); }
676 operator--(int) noexcept
677 { return fetch_sub(1); }
680 operator--(int) volatile noexcept
681 { return fetch_sub(1); }
684 operator++() noexcept
685 { return __atomic_add_fetch(&_M_p, _M_type_size(1),
686 memory_order_seq_cst); }
689 operator++() volatile noexcept
690 { return __atomic_add_fetch(&_M_p, _M_type_size(1),
691 memory_order_seq_cst); }
694 operator--() noexcept
695 { return __atomic_sub_fetch(&_M_p, _M_type_size(1),
696 memory_order_seq_cst); }
699 operator--() volatile noexcept
700 { return __atomic_sub_fetch(&_M_p, _M_type_size(1),
701 memory_order_seq_cst); }
704 operator+=(ptrdiff_t __d) noexcept
705 { return __atomic_add_fetch(&_M_p, _M_type_size(__d),
706 memory_order_seq_cst); }
709 operator+=(ptrdiff_t __d) volatile noexcept
710 { return __atomic_add_fetch(&_M_p, _M_type_size(__d),
711 memory_order_seq_cst); }
714 operator-=(ptrdiff_t __d) noexcept
715 { return __atomic_sub_fetch(&_M_p, _M_type_size(__d),
716 memory_order_seq_cst); }
719 operator-=(ptrdiff_t __d) volatile noexcept
720 { return __atomic_sub_fetch(&_M_p, _M_type_size(__d),
721 memory_order_seq_cst); }
724 is_lock_free() const noexcept
725 { return __atomic_is_lock_free(_M_type_size(1), &_M_p); }
728 is_lock_free() const volatile noexcept
729 { return __atomic_is_lock_free(_M_type_size(1), &_M_p); }
732 store(__pointer_type __p,
733 memory_order __m = memory_order_seq_cst) noexcept
735 __glibcxx_assert(__m != memory_order_acquire);
736 __glibcxx_assert(__m != memory_order_acq_rel);
737 __glibcxx_assert(__m != memory_order_consume);
739 __atomic_store_n(&_M_p, __p, __m);
743 store(__pointer_type __p,
744 memory_order __m = memory_order_seq_cst) volatile noexcept
746 __glibcxx_assert(__m != memory_order_acquire);
747 __glibcxx_assert(__m != memory_order_acq_rel);
748 __glibcxx_assert(__m != memory_order_consume);
750 __atomic_store_n(&_M_p, __p, __m);
754 load(memory_order __m = memory_order_seq_cst) const noexcept
756 __glibcxx_assert(__m != memory_order_release);
757 __glibcxx_assert(__m != memory_order_acq_rel);
759 return __atomic_load_n(&_M_p, __m);
763 load(memory_order __m = memory_order_seq_cst) const volatile noexcept
765 __glibcxx_assert(__m != memory_order_release);
766 __glibcxx_assert(__m != memory_order_acq_rel);
768 return __atomic_load_n(&_M_p, __m);
772 exchange(__pointer_type __p,
773 memory_order __m = memory_order_seq_cst) noexcept
775 return __atomic_exchange_n(&_M_p, __p, __m);
780 exchange(__pointer_type __p,
781 memory_order __m = memory_order_seq_cst) volatile noexcept
783 return __atomic_exchange_n(&_M_p, __p, __m);
787 compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
789 memory_order __m2) noexcept
791 __glibcxx_assert(__m2 != memory_order_release);
792 __glibcxx_assert(__m2 != memory_order_acq_rel);
793 __glibcxx_assert(__m2 <= __m1);
795 return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 0, __m1, __m2);
799 compare_exchange_strong(__pointer_type& __p1, __pointer_type __p2,
801 memory_order __m2) volatile noexcept
803 __glibcxx_assert(__m2 != memory_order_release);
804 __glibcxx_assert(__m2 != memory_order_acq_rel);
805 __glibcxx_assert(__m2 <= __m1);
807 return __atomic_compare_exchange_n(&_M_p, &__p1, __p2, 0, __m1, __m2);
811 fetch_add(ptrdiff_t __d,
812 memory_order __m = memory_order_seq_cst) noexcept
813 { return __atomic_fetch_add(&_M_p, _M_type_size(__d), __m); }
816 fetch_add(ptrdiff_t __d,
817 memory_order __m = memory_order_seq_cst) volatile noexcept
818 { return __atomic_fetch_add(&_M_p, _M_type_size(__d), __m); }
821 fetch_sub(ptrdiff_t __d,
822 memory_order __m = memory_order_seq_cst) noexcept
823 { return __atomic_fetch_sub(&_M_p, _M_type_size(__d), __m); }
826 fetch_sub(ptrdiff_t __d,
827 memory_order __m = memory_order_seq_cst) volatile noexcept
828 { return __atomic_fetch_sub(&_M_p, _M_type_size(__d), __m); }
833 _GLIBCXX_END_NAMESPACE_VERSION