Atomics.h 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800
  1. /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
  2. /* vim: set ts=8 sts=2 et sw=2 tw=80: */
  3. /* This Source Code Form is subject to the terms of the Mozilla Public
  4. * License, v. 2.0. If a copy of the MPL was not distributed with this
  5. * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
  6. /*
  7. * Implements (almost always) lock-free atomic operations. The operations here
  8. * are a subset of that which can be found in C++11's <atomic> header, with a
  9. * different API to enforce consistent memory ordering constraints.
  10. *
  11. * Anyone caught using |volatile| for inter-thread memory safety needs to be
  12. * sent a copy of this header and the C++11 standard.
  13. */
  14. #ifndef mozilla_Atomics_h
  15. #define mozilla_Atomics_h
  16. #include "mozilla/Assertions.h"
  17. #include "mozilla/Attributes.h"
  18. #include "mozilla/Compiler.h"
  19. #include "mozilla/TypeTraits.h"
  20. #include <stdint.h>
  21. /*
  22. * Our minimum deployment target on clang/OS X is OS X 10.6, whose SDK
  23. * does not have <atomic>. So be sure to check for <atomic> support
  24. * along with C++0x support.
  25. */
  26. #if defined(_MSC_VER)
  27. # define MOZ_HAVE_CXX11_ATOMICS
  28. #elif defined(__clang__) || defined(__GNUC__)
  29. /*
  30. * Clang doesn't like <atomic> from libstdc++ before 4.7 due to the
  31. * loose typing of the atomic builtins. GCC 4.5 and 4.6 lacks inline
  32. * definitions for unspecialized std::atomic and causes linking errors.
  33. * Therefore, we require at least 4.7.0 for using libstdc++.
  34. *
  35. * libc++ <atomic> is only functional with clang.
  36. */
  37. # if MOZ_USING_LIBSTDCXX && MOZ_LIBSTDCXX_VERSION_AT_LEAST(4, 7, 0)
  38. # define MOZ_HAVE_CXX11_ATOMICS
  39. # elif MOZ_USING_LIBCXX && defined(__clang__)
  40. # define MOZ_HAVE_CXX11_ATOMICS
  41. # endif
  42. #endif
  43. namespace mozilla {
  44. /**
  45. * An enum of memory ordering possibilities for atomics.
  46. *
  47. * Memory ordering is the observable state of distinct values in memory.
  48. * (It's a separate concept from atomicity, which concerns whether an
  49. * operation can ever be observed in an intermediate state. Don't
  50. * conflate the two!) Given a sequence of operations in source code on
  51. * memory, it is *not* always the case that, at all times and on all
  52. * cores, those operations will appear to have occurred in that exact
  53. * sequence. First, the compiler might reorder that sequence, if it
  54. * thinks another ordering will be more efficient. Second, the CPU may
  55. * not expose so consistent a view of memory. CPUs will often perform
  56. * their own instruction reordering, above and beyond that performed by
  57. * the compiler. And each core has its own memory caches, and accesses
  58. * (reads and writes both) to "memory" may only resolve to out-of-date
  59. * cache entries -- not to the "most recently" performed operation in
  60. * some global sense. Any access to a value that may be used by
  61. * multiple threads, potentially across multiple cores, must therefore
  62. * have a memory ordering imposed on it, for all code on all
  63. * threads/cores to have a sufficiently coherent worldview.
  64. *
  65. * http://gcc.gnu.org/wiki/Atomic/GCCMM/AtomicSync and
  66. * http://en.cppreference.com/w/cpp/atomic/memory_order go into more
  67. * detail on all this, including examples of how each mode works.
  68. *
  69. * Note that for simplicity and practicality, not all of the modes in
  70. * C++11 are supported. The missing C++11 modes are either subsumed by
  71. * the modes we provide below, or not relevant for the CPUs we support
  72. * in Gecko. These three modes are confusing enough as it is!
  73. */
  74. enum MemoryOrdering {
  75. /*
  76. * Relaxed ordering is the simplest memory ordering: none at all.
  77. * When the result of a write is observed, nothing may be inferred
  78. * about other memory. Writes ostensibly performed "before" on the
  79. * writing thread may not yet be visible. Writes performed "after" on
  80. * the writing thread may already be visible, if the compiler or CPU
  81. * reordered them. (The latter can happen if reads and/or writes get
  82. * held up in per-processor caches.) Relaxed ordering means
  83. * operations can always use cached values (as long as the actual
  84. * updates to atomic values actually occur, correctly, eventually), so
  85. * it's usually the fastest sort of atomic access. For this reason,
  86. * *it's also the most dangerous kind of access*.
  87. *
  88. * Relaxed ordering is good for things like process-wide statistics
  89. * counters that don't need to be consistent with anything else, so
  90. * long as updates themselves are atomic. (And so long as any
  91. * observations of that value can tolerate being out-of-date -- if you
  92. * need some sort of up-to-date value, you need some sort of other
  93. * synchronizing operation.) It's *not* good for locks, mutexes,
  94. * reference counts, etc. that mediate access to other memory, or must
  95. * be observably consistent with other memory.
  96. *
  97. * x86 architectures don't take advantage of the optimization
  98. * opportunities that relaxed ordering permits. Thus it's possible
  99. * that using relaxed ordering will "work" on x86 but fail elsewhere
  100. * (ARM, say, which *does* implement non-sequentially-consistent
  101. * relaxed ordering semantics). Be extra-careful using relaxed
  102. * ordering if you can't easily test non-x86 architectures!
  103. */
  104. Relaxed,
  105. /*
  106. * When an atomic value is updated with ReleaseAcquire ordering, and
  107. * that new value is observed with ReleaseAcquire ordering, prior
  108. * writes (atomic or not) are also observable. What ReleaseAcquire
  109. * *doesn't* give you is any observable ordering guarantees for
  110. * ReleaseAcquire-ordered operations on different objects. For
  111. * example, if there are two cores that each perform ReleaseAcquire
  112. * operations on separate objects, each core may or may not observe
  113. * the operations made by the other core. The only way the cores can
  114. * be synchronized with ReleaseAcquire is if they both
  115. * ReleaseAcquire-access the same object. This implies that you can't
  116. * necessarily describe some global total ordering of ReleaseAcquire
  117. * operations.
  118. *
  119. * ReleaseAcquire ordering is good for (as the name implies) atomic
  120. * operations on values controlling ownership of things: reference
  121. * counts, mutexes, and the like. However, if you are thinking about
  122. * using these to implement your own locks or mutexes, you should take
  123. * a good, hard look at actual lock or mutex primitives first.
  124. */
  125. ReleaseAcquire,
  126. /*
  127. * When an atomic value is updated with SequentiallyConsistent
  128. * ordering, all writes observable when the update is observed, just
  129. * as with ReleaseAcquire ordering. But, furthermore, a global total
  130. * ordering of SequentiallyConsistent operations *can* be described.
  131. * For example, if two cores perform SequentiallyConsistent operations
  132. * on separate objects, one core will observably perform its update
  133. * (and all previous operations will have completed), then the other
  134. * core will observably perform its update (and all previous
  135. * operations will have completed). (Although those previous
  136. * operations aren't themselves ordered -- they could be intermixed,
  137. * or ordered if they occur on atomic values with ordering
  138. * requirements.) SequentiallyConsistent is the *simplest and safest*
  139. * ordering of atomic operations -- it's always as if one operation
  140. * happens, then another, then another, in some order -- and every
  141. * core observes updates to happen in that single order. Because it
  142. * has the most synchronization requirements, operations ordered this
  143. * way also tend to be slowest.
  144. *
  145. * SequentiallyConsistent ordering can be desirable when multiple
  146. * threads observe objects, and they all have to agree on the
  147. * observable order of changes to them. People expect
  148. * SequentiallyConsistent ordering, even if they shouldn't, when
  149. * writing code, atomic or otherwise. SequentiallyConsistent is also
  150. * the ordering of choice when designing lockless data structures. If
  151. * you don't know what order to use, use this one.
  152. */
  153. SequentiallyConsistent,
  154. };
  155. } // namespace mozilla
  156. // Build up the underlying intrinsics.
  157. #ifdef MOZ_HAVE_CXX11_ATOMICS
  158. # include <atomic>
  159. namespace mozilla {
  160. namespace detail {
  161. /*
  162. * We provide CompareExchangeFailureOrder to work around a bug in some
  163. * versions of GCC's <atomic> header. See bug 898491.
  164. */
  165. template<MemoryOrdering Order> struct AtomicOrderConstraints;
  166. template<>
  167. struct AtomicOrderConstraints<Relaxed>
  168. {
  169. static const std::memory_order AtomicRMWOrder = std::memory_order_relaxed;
  170. static const std::memory_order LoadOrder = std::memory_order_relaxed;
  171. static const std::memory_order StoreOrder = std::memory_order_relaxed;
  172. static const std::memory_order CompareExchangeFailureOrder =
  173. std::memory_order_relaxed;
  174. };
  175. template<>
  176. struct AtomicOrderConstraints<ReleaseAcquire>
  177. {
  178. static const std::memory_order AtomicRMWOrder = std::memory_order_acq_rel;
  179. static const std::memory_order LoadOrder = std::memory_order_acquire;
  180. static const std::memory_order StoreOrder = std::memory_order_release;
  181. static const std::memory_order CompareExchangeFailureOrder =
  182. std::memory_order_acquire;
  183. };
  184. template<>
  185. struct AtomicOrderConstraints<SequentiallyConsistent>
  186. {
  187. static const std::memory_order AtomicRMWOrder = std::memory_order_seq_cst;
  188. static const std::memory_order LoadOrder = std::memory_order_seq_cst;
  189. static const std::memory_order StoreOrder = std::memory_order_seq_cst;
  190. static const std::memory_order CompareExchangeFailureOrder =
  191. std::memory_order_seq_cst;
  192. };
  193. template<typename T, MemoryOrdering Order>
  194. struct IntrinsicBase
  195. {
  196. typedef std::atomic<T> ValueType;
  197. typedef AtomicOrderConstraints<Order> OrderedOp;
  198. };
  199. template<typename T, MemoryOrdering Order>
  200. struct IntrinsicMemoryOps : public IntrinsicBase<T, Order>
  201. {
  202. typedef IntrinsicBase<T, Order> Base;
  203. static T load(const typename Base::ValueType& aPtr)
  204. {
  205. return aPtr.load(Base::OrderedOp::LoadOrder);
  206. }
  207. static void store(typename Base::ValueType& aPtr, T aVal)
  208. {
  209. aPtr.store(aVal, Base::OrderedOp::StoreOrder);
  210. }
  211. static T exchange(typename Base::ValueType& aPtr, T aVal)
  212. {
  213. return aPtr.exchange(aVal, Base::OrderedOp::AtomicRMWOrder);
  214. }
  215. static bool compareExchange(typename Base::ValueType& aPtr,
  216. T aOldVal, T aNewVal)
  217. {
  218. return aPtr.compare_exchange_strong(aOldVal, aNewVal,
  219. Base::OrderedOp::AtomicRMWOrder,
  220. Base::OrderedOp::CompareExchangeFailureOrder);
  221. }
  222. };
  223. template<typename T, MemoryOrdering Order>
  224. struct IntrinsicAddSub : public IntrinsicBase<T, Order>
  225. {
  226. typedef IntrinsicBase<T, Order> Base;
  227. static T add(typename Base::ValueType& aPtr, T aVal)
  228. {
  229. return aPtr.fetch_add(aVal, Base::OrderedOp::AtomicRMWOrder);
  230. }
  231. static T sub(typename Base::ValueType& aPtr, T aVal)
  232. {
  233. return aPtr.fetch_sub(aVal, Base::OrderedOp::AtomicRMWOrder);
  234. }
  235. };
  236. template<typename T, MemoryOrdering Order>
  237. struct IntrinsicAddSub<T*, Order> : public IntrinsicBase<T*, Order>
  238. {
  239. typedef IntrinsicBase<T*, Order> Base;
  240. static T* add(typename Base::ValueType& aPtr, ptrdiff_t aVal)
  241. {
  242. return aPtr.fetch_add(aVal, Base::OrderedOp::AtomicRMWOrder);
  243. }
  244. static T* sub(typename Base::ValueType& aPtr, ptrdiff_t aVal)
  245. {
  246. return aPtr.fetch_sub(aVal, Base::OrderedOp::AtomicRMWOrder);
  247. }
  248. };
  249. template<typename T, MemoryOrdering Order>
  250. struct IntrinsicIncDec : public IntrinsicAddSub<T, Order>
  251. {
  252. typedef IntrinsicBase<T, Order> Base;
  253. static T inc(typename Base::ValueType& aPtr)
  254. {
  255. return IntrinsicAddSub<T, Order>::add(aPtr, 1);
  256. }
  257. static T dec(typename Base::ValueType& aPtr)
  258. {
  259. return IntrinsicAddSub<T, Order>::sub(aPtr, 1);
  260. }
  261. };
  262. template<typename T, MemoryOrdering Order>
  263. struct AtomicIntrinsics : public IntrinsicMemoryOps<T, Order>,
  264. public IntrinsicIncDec<T, Order>
  265. {
  266. typedef IntrinsicBase<T, Order> Base;
  267. static T or_(typename Base::ValueType& aPtr, T aVal)
  268. {
  269. return aPtr.fetch_or(aVal, Base::OrderedOp::AtomicRMWOrder);
  270. }
  271. static T xor_(typename Base::ValueType& aPtr, T aVal)
  272. {
  273. return aPtr.fetch_xor(aVal, Base::OrderedOp::AtomicRMWOrder);
  274. }
  275. static T and_(typename Base::ValueType& aPtr, T aVal)
  276. {
  277. return aPtr.fetch_and(aVal, Base::OrderedOp::AtomicRMWOrder);
  278. }
  279. };
  280. template<typename T, MemoryOrdering Order>
  281. struct AtomicIntrinsics<T*, Order>
  282. : public IntrinsicMemoryOps<T*, Order>, public IntrinsicIncDec<T*, Order>
  283. {
  284. };
  285. template<typename T>
  286. struct ToStorageTypeArgument
  287. {
  288. static constexpr T convert (T aT) { return aT; }
  289. };
  290. } // namespace detail
  291. } // namespace mozilla
  292. #elif defined(__GNUC__)
  293. namespace mozilla {
  294. namespace detail {
  295. /*
  296. * The __sync_* family of intrinsics is documented here:
  297. *
  298. * http://gcc.gnu.org/onlinedocs/gcc-4.6.4/gcc/Atomic-Builtins.html
  299. *
  300. * While these intrinsics are deprecated in favor of the newer __atomic_*
  301. * family of intrincs:
  302. *
  303. * http://gcc.gnu.org/onlinedocs/gcc-4.7.3/gcc/_005f_005fatomic-Builtins.html
  304. *
  305. * any GCC version that supports the __atomic_* intrinsics will also support
  306. * the <atomic> header and so will be handled above. We provide a version of
  307. * atomics using the __sync_* intrinsics to support older versions of GCC.
  308. *
  309. * All __sync_* intrinsics that we use below act as full memory barriers, for
  310. * both compiler and hardware reordering, except for __sync_lock_test_and_set,
  311. * which is a only an acquire barrier. When we call __sync_lock_test_and_set,
  312. * we add a barrier above it as appropriate.
  313. */
  314. template<MemoryOrdering Order> struct Barrier;
  315. /*
  316. * Some processors (in particular, x86) don't require quite so many calls to
  317. * __sync_sychronize as our specializations of Barrier produce. If
  318. * performance turns out to be an issue, defining these specializations
  319. * on a per-processor basis would be a good first tuning step.
  320. */
  321. template<>
  322. struct Barrier<Relaxed>
  323. {
  324. static void beforeLoad() {}
  325. static void afterLoad() {}
  326. static void beforeStore() {}
  327. static void afterStore() {}
  328. };
  329. template<>
  330. struct Barrier<ReleaseAcquire>
  331. {
  332. static void beforeLoad() {}
  333. static void afterLoad() { __sync_synchronize(); }
  334. static void beforeStore() { __sync_synchronize(); }
  335. static void afterStore() {}
  336. };
  337. template<>
  338. struct Barrier<SequentiallyConsistent>
  339. {
  340. static void beforeLoad() { __sync_synchronize(); }
  341. static void afterLoad() { __sync_synchronize(); }
  342. static void beforeStore() { __sync_synchronize(); }
  343. static void afterStore() { __sync_synchronize(); }
  344. };
  345. template<typename T, bool TIsEnum = IsEnum<T>::value>
  346. struct AtomicStorageType
  347. {
  348. // For non-enums, just use the type directly.
  349. typedef T Type;
  350. };
  351. template<typename T>
  352. struct AtomicStorageType<T, true>
  353. : Conditional<sizeof(T) == 4, uint32_t, uint64_t>
  354. {
  355. static_assert(sizeof(T) == 4 || sizeof(T) == 8,
  356. "wrong type computed in conditional above");
  357. };
  358. template<typename T, MemoryOrdering Order>
  359. struct IntrinsicMemoryOps
  360. {
  361. typedef typename AtomicStorageType<T>::Type ValueType;
  362. static T load(const ValueType& aPtr)
  363. {
  364. Barrier<Order>::beforeLoad();
  365. T val = T(aPtr);
  366. Barrier<Order>::afterLoad();
  367. return val;
  368. }
  369. static void store(ValueType& aPtr, T aVal)
  370. {
  371. Barrier<Order>::beforeStore();
  372. aPtr = ValueType(aVal);
  373. Barrier<Order>::afterStore();
  374. }
  375. static T exchange(ValueType& aPtr, T aVal)
  376. {
  377. // __sync_lock_test_and_set is only an acquire barrier; loads and stores
  378. // can't be moved up from after to before it, but they can be moved down
  379. // from before to after it. We may want a stricter ordering, so we need
  380. // an explicit barrier.
  381. Barrier<Order>::beforeStore();
  382. return T(__sync_lock_test_and_set(&aPtr, ValueType(aVal)));
  383. }
  384. static bool compareExchange(ValueType& aPtr, T aOldVal, T aNewVal)
  385. {
  386. return __sync_bool_compare_and_swap(&aPtr, ValueType(aOldVal), ValueType(aNewVal));
  387. }
  388. };
  389. template<typename T, MemoryOrdering Order>
  390. struct IntrinsicAddSub
  391. : public IntrinsicMemoryOps<T, Order>
  392. {
  393. typedef IntrinsicMemoryOps<T, Order> Base;
  394. typedef typename Base::ValueType ValueType;
  395. static T add(ValueType& aPtr, T aVal)
  396. {
  397. return T(__sync_fetch_and_add(&aPtr, ValueType(aVal)));
  398. }
  399. static T sub(ValueType& aPtr, T aVal)
  400. {
  401. return T(__sync_fetch_and_sub(&aPtr, ValueType(aVal)));
  402. }
  403. };
  404. template<typename T, MemoryOrdering Order>
  405. struct IntrinsicAddSub<T*, Order>
  406. : public IntrinsicMemoryOps<T*, Order>
  407. {
  408. typedef IntrinsicMemoryOps<T*, Order> Base;
  409. typedef typename Base::ValueType ValueType;
  410. /*
  411. * The reinterpret_casts are needed so that
  412. * __sync_fetch_and_{add,sub} will properly type-check.
  413. *
  414. * Also, these functions do not provide standard semantics for
  415. * pointer types, so we need to adjust the addend.
  416. */
  417. static ValueType add(ValueType& aPtr, ptrdiff_t aVal)
  418. {
  419. ValueType amount = reinterpret_cast<ValueType>(aVal * sizeof(T));
  420. return __sync_fetch_and_add(&aPtr, amount);
  421. }
  422. static ValueType sub(ValueType& aPtr, ptrdiff_t aVal)
  423. {
  424. ValueType amount = reinterpret_cast<ValueType>(aVal * sizeof(T));
  425. return __sync_fetch_and_sub(&aPtr, amount);
  426. }
  427. };
  428. template<typename T, MemoryOrdering Order>
  429. struct IntrinsicIncDec : public IntrinsicAddSub<T, Order>
  430. {
  431. typedef IntrinsicAddSub<T, Order> Base;
  432. typedef typename Base::ValueType ValueType;
  433. static T inc(ValueType& aPtr) { return Base::add(aPtr, 1); }
  434. static T dec(ValueType& aPtr) { return Base::sub(aPtr, 1); }
  435. };
  436. template<typename T, MemoryOrdering Order>
  437. struct AtomicIntrinsics : public IntrinsicIncDec<T, Order>
  438. {
  439. static T or_( T& aPtr, T aVal) { return __sync_fetch_and_or(&aPtr, aVal); }
  440. static T xor_(T& aPtr, T aVal) { return __sync_fetch_and_xor(&aPtr, aVal); }
  441. static T and_(T& aPtr, T aVal) { return __sync_fetch_and_and(&aPtr, aVal); }
  442. };
  443. template<typename T, MemoryOrdering Order>
  444. struct AtomicIntrinsics<T*, Order> : public IntrinsicIncDec<T*, Order>
  445. {
  446. };
  447. template<typename T, bool TIsEnum = IsEnum<T>::value>
  448. struct ToStorageTypeArgument
  449. {
  450. typedef typename AtomicStorageType<T>::Type ResultType;
  451. static constexpr ResultType convert (T aT) { return ResultType(aT); }
  452. };
  453. template<typename T>
  454. struct ToStorageTypeArgument<T, false>
  455. {
  456. static constexpr T convert (T aT) { return aT; }
  457. };
  458. } // namespace detail
  459. } // namespace mozilla
  460. #else
  461. # error "Atomic compiler intrinsics are not supported on your platform"
  462. #endif
  463. namespace mozilla {
  464. namespace detail {
  465. template<typename T, MemoryOrdering Order>
  466. class AtomicBase
  467. {
  468. static_assert(sizeof(T) == 4 || sizeof(T) == 8,
  469. "mozilla/Atomics.h only supports 32-bit and 64-bit types");
  470. protected:
  471. typedef typename detail::AtomicIntrinsics<T, Order> Intrinsics;
  472. typedef typename Intrinsics::ValueType ValueType;
  473. ValueType mValue;
  474. public:
  475. constexpr AtomicBase() : mValue() {}
  476. explicit constexpr AtomicBase(T aInit)
  477. : mValue(ToStorageTypeArgument<T>::convert(aInit))
  478. {}
  479. // Note: we can't provide operator T() here because Atomic<bool> inherits
  480. // from AtomcBase with T=uint32_t and not T=bool. If we implemented
  481. // operator T() here, it would cause errors when comparing Atomic<bool> with
  482. // a regular bool.
  483. T operator=(T aVal)
  484. {
  485. Intrinsics::store(mValue, aVal);
  486. return aVal;
  487. }
  488. /**
  489. * Performs an atomic swap operation. aVal is stored and the previous
  490. * value of this variable is returned.
  491. */
  492. T exchange(T aVal)
  493. {
  494. return Intrinsics::exchange(mValue, aVal);
  495. }
  496. /**
  497. * Performs an atomic compare-and-swap operation and returns true if it
  498. * succeeded. This is equivalent to atomically doing
  499. *
  500. * if (mValue == aOldValue) {
  501. * mValue = aNewValue;
  502. * return true;
  503. * } else {
  504. * return false;
  505. * }
  506. */
  507. bool compareExchange(T aOldValue, T aNewValue)
  508. {
  509. return Intrinsics::compareExchange(mValue, aOldValue, aNewValue);
  510. }
  511. private:
  512. template<MemoryOrdering AnyOrder>
  513. AtomicBase(const AtomicBase<T, AnyOrder>& aCopy) = delete;
  514. };
  515. template<typename T, MemoryOrdering Order>
  516. class AtomicBaseIncDec : public AtomicBase<T, Order>
  517. {
  518. typedef typename detail::AtomicBase<T, Order> Base;
  519. public:
  520. constexpr AtomicBaseIncDec() : Base() {}
  521. explicit constexpr AtomicBaseIncDec(T aInit) : Base(aInit) {}
  522. using Base::operator=;
  523. operator T() const { return Base::Intrinsics::load(Base::mValue); }
  524. T operator++(int) { return Base::Intrinsics::inc(Base::mValue); }
  525. T operator--(int) { return Base::Intrinsics::dec(Base::mValue); }
  526. T operator++() { return Base::Intrinsics::inc(Base::mValue) + 1; }
  527. T operator--() { return Base::Intrinsics::dec(Base::mValue) - 1; }
  528. private:
  529. template<MemoryOrdering AnyOrder>
  530. AtomicBaseIncDec(const AtomicBaseIncDec<T, AnyOrder>& aCopy) = delete;
  531. };
  532. } // namespace detail
  533. /**
  534. * A wrapper for a type that enforces that all memory accesses are atomic.
  535. *
  536. * In general, where a variable |T foo| exists, |Atomic<T> foo| can be used in
  537. * its place. Implementations for integral and pointer types are provided
  538. * below.
  539. *
  540. * Atomic accesses are sequentially consistent by default. You should
  541. * use the default unless you are tall enough to ride the
  542. * memory-ordering roller coaster (if you're not sure, you aren't) and
  543. * you have a compelling reason to do otherwise.
  544. *
  545. * There is one exception to the case of atomic memory accesses: providing an
  546. * initial value of the atomic value is not guaranteed to be atomic. This is a
  547. * deliberate design choice that enables static atomic variables to be declared
  548. * without introducing extra static constructors.
  549. */
  550. template<typename T,
  551. MemoryOrdering Order = SequentiallyConsistent,
  552. typename Enable = void>
  553. class Atomic;
  554. /**
  555. * Atomic<T> implementation for integral types.
  556. *
  557. * In addition to atomic store and load operations, compound assignment and
  558. * increment/decrement operators are implemented which perform the
  559. * corresponding read-modify-write operation atomically. Finally, an atomic
  560. * swap method is provided.
  561. */
  562. template<typename T, MemoryOrdering Order>
  563. class Atomic<T, Order, typename EnableIf<IsIntegral<T>::value &&
  564. !IsSame<T, bool>::value>::Type>
  565. : public detail::AtomicBaseIncDec<T, Order>
  566. {
  567. typedef typename detail::AtomicBaseIncDec<T, Order> Base;
  568. public:
  569. constexpr Atomic() : Base() {}
  570. explicit constexpr Atomic(T aInit) : Base(aInit) {}
  571. using Base::operator=;
  572. T operator+=(T aDelta)
  573. {
  574. return Base::Intrinsics::add(Base::mValue, aDelta) + aDelta;
  575. }
  576. T operator-=(T aDelta)
  577. {
  578. return Base::Intrinsics::sub(Base::mValue, aDelta) - aDelta;
  579. }
  580. T operator|=(T aVal)
  581. {
  582. return Base::Intrinsics::or_(Base::mValue, aVal) | aVal;
  583. }
  584. T operator^=(T aVal)
  585. {
  586. return Base::Intrinsics::xor_(Base::mValue, aVal) ^ aVal;
  587. }
  588. T operator&=(T aVal)
  589. {
  590. return Base::Intrinsics::and_(Base::mValue, aVal) & aVal;
  591. }
  592. private:
  593. Atomic(Atomic<T, Order>& aOther) = delete;
  594. };
  595. /**
  596. * Atomic<T> implementation for pointer types.
  597. *
  598. * An atomic compare-and-swap primitive for pointer variables is provided, as
  599. * are atomic increment and decement operators. Also provided are the compound
  600. * assignment operators for addition and subtraction. Atomic swap (via
  601. * exchange()) is included as well.
  602. */
  603. template<typename T, MemoryOrdering Order>
  604. class Atomic<T*, Order> : public detail::AtomicBaseIncDec<T*, Order>
  605. {
  606. typedef typename detail::AtomicBaseIncDec<T*, Order> Base;
  607. public:
  608. constexpr Atomic() : Base() {}
  609. explicit constexpr Atomic(T* aInit) : Base(aInit) {}
  610. using Base::operator=;
  611. T* operator+=(ptrdiff_t aDelta)
  612. {
  613. return Base::Intrinsics::add(Base::mValue, aDelta) + aDelta;
  614. }
  615. T* operator-=(ptrdiff_t aDelta)
  616. {
  617. return Base::Intrinsics::sub(Base::mValue, aDelta) - aDelta;
  618. }
  619. private:
  620. Atomic(Atomic<T*, Order>& aOther) = delete;
  621. };
  622. /**
  623. * Atomic<T> implementation for enum types.
  624. *
  625. * The atomic store and load operations and the atomic swap method is provided.
  626. */
  627. template<typename T, MemoryOrdering Order>
  628. class Atomic<T, Order, typename EnableIf<IsEnum<T>::value>::Type>
  629. : public detail::AtomicBase<T, Order>
  630. {
  631. typedef typename detail::AtomicBase<T, Order> Base;
  632. public:
  633. constexpr Atomic() : Base() {}
  634. explicit constexpr Atomic(T aInit) : Base(aInit) {}
  635. operator T() const { return T(Base::Intrinsics::load(Base::mValue)); }
  636. using Base::operator=;
  637. private:
  638. Atomic(Atomic<T, Order>& aOther) = delete;
  639. };
  640. /**
  641. * Atomic<T> implementation for boolean types.
  642. *
  643. * The atomic store and load operations and the atomic swap method is provided.
  644. *
  645. * Note:
  646. *
  647. * - sizeof(Atomic<bool>) != sizeof(bool) for some implementations of
  648. * bool and/or some implementations of std::atomic. This is allowed in
  649. * [atomic.types.generic]p9.
  650. *
  651. * - It's not obvious whether the 8-bit atomic functions on Windows are always
  652. * inlined or not. If they are not inlined, the corresponding functions in the
  653. * runtime library are not available on Windows XP. This is why we implement
  654. * Atomic<bool> with an underlying type of uint32_t.
  655. */
  656. template<MemoryOrdering Order>
  657. class Atomic<bool, Order>
  658. : protected detail::AtomicBase<uint32_t, Order>
  659. {
  660. typedef typename detail::AtomicBase<uint32_t, Order> Base;
  661. public:
  662. constexpr Atomic() : Base() {}
  663. explicit constexpr Atomic(bool aInit) : Base(aInit) {}
  664. // We provide boolean wrappers for the underlying AtomicBase methods.
  665. MOZ_IMPLICIT operator bool() const
  666. {
  667. return Base::Intrinsics::load(Base::mValue);
  668. }
  669. bool operator=(bool aVal)
  670. {
  671. return Base::operator=(aVal);
  672. }
  673. bool exchange(bool aVal)
  674. {
  675. return Base::exchange(aVal);
  676. }
  677. bool compareExchange(bool aOldValue, bool aNewValue)
  678. {
  679. return Base::compareExchange(aOldValue, aNewValue);
  680. }
  681. private:
  682. Atomic(Atomic<bool, Order>& aOther) = delete;
  683. };
  684. } // namespace mozilla
  685. #endif /* mozilla_Atomics_h */