/* * Copyright (c) 2018 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in * compliance with the License. The rights granted to you under the License * may not be used to create, or enable the creation or redistribution of, * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * This header provides some gory details to implement the * interfaces. Nothing in this header should be called directly, no promise is * made to keep this interface stable. */ #ifndef __OS_ATOMIC_PRIVATE_H__ #error "Do not include directly, use " #endif #ifndef __OS_ATOMIC_PRIVATE_IMPL_H__ #define __OS_ATOMIC_PRIVATE_IMPL_H__ #pragma mark - implementation details static inline int _os_atomic_mo_has_acquire(OS_ATOMIC_STD memory_order ord) { switch (ord) { case os_atomic_std(memory_order_consume): case os_atomic_std(memory_order_acquire): case os_atomic_std(memory_order_acq_rel): case os_atomic_std(memory_order_seq_cst): return 1; default: return 0; } } static inline int _os_atomic_mo_has_release(OS_ATOMIC_STD memory_order ord) { switch (ord) { case os_atomic_std(memory_order_release): case os_atomic_std(memory_order_acq_rel): case os_atomic_std(memory_order_seq_cst): return 1; default: return 0; } } #define _os_atomic_mo_relaxed os_atomic_std(memory_order_relaxed) #define _os_atomic_mo_compiler_acquire os_atomic_std(memory_order_relaxed) #define _os_atomic_mo_compiler_release os_atomic_std(memory_order_relaxed) #define _os_atomic_mo_compiler_acq_rel os_atomic_std(memory_order_relaxed) #define _os_atomic_mo_consume os_atomic_std(memory_order_consume) #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY #define _os_atomic_mo_dependency os_atomic_std(memory_order_acquire) #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY #define _os_atomic_mo_acquire os_atomic_std(memory_order_acquire) #define _os_atomic_mo_release os_atomic_std(memory_order_release) #define _os_atomic_mo_acq_rel os_atomic_std(memory_order_acq_rel) #define _os_atomic_mo_seq_cst os_atomic_std(memory_order_seq_cst) /* * Mapping between symbolic memory orderings and actual ones * to take SMP into account. */ #if OS_ATOMIC_CONFIG_SMP #define _os_atomic_mo_relaxed_smp _os_atomic_mo_relaxed #define _os_atomic_mo_compiler_acquire_smp _os_atomic_mo_relaxed #define _os_atomic_mo_compiler_release_smp _os_atomic_mo_relaxed #define _os_atomic_mo_compiler_acq_rel_smp _os_atomic_mo_relaxed #define _os_atomic_mo_consume_smp _os_atomic_mo_consume #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY #define _os_atomic_mo_dependency_smp _os_atomic_mo_dependency #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY #define _os_atomic_mo_acquire_smp _os_atomic_mo_acquire #define _os_atomic_mo_release_smp _os_atomic_mo_release #define _os_atomic_mo_acq_rel_smp _os_atomic_mo_acq_rel #define _os_atomic_mo_seq_cst_smp _os_atomic_mo_seq_cst #else #define _os_atomic_mo_relaxed_smp _os_atomic_mo_relaxed #define _os_atomic_mo_compiler_acquire_smp _os_atomic_mo_relaxed #define _os_atomic_mo_compiler_release_smp _os_atomic_mo_relaxed #define _os_atomic_mo_compiler_acq_rel_smp _os_atomic_mo_relaxed #define _os_atomic_mo_consume_smp _os_atomic_mo_relaxed #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY #define _os_atomic_mo_dependency_smp _os_atomic_mo_relaxed #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY #define _os_atomic_mo_acquire_smp _os_atomic_mo_relaxed #define _os_atomic_mo_release_smp _os_atomic_mo_relaxed #define _os_atomic_mo_acq_rel_smp _os_atomic_mo_relaxed #define _os_atomic_mo_seq_cst_smp _os_atomic_mo_relaxed #endif #if KERNEL_PRIVATE #define memory_order_relaxed_smp _os_atomic_mo_relaxed_smp #define memory_order_compiler_acquire_smp _os_atomic_mo_compiler_acquire_smp #define memory_order_compiler_release_smp _os_atomic_mo_compiler_release_smp #define memory_order_compiler_acq_rel_smp _os_atomic_mo_compiler_acq_rel_smp #define memory_order_consume_smp _os_atomic_mo_consume_smp #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY #define memory_order_dependency _os_atomic_mo_dependency #define memory_order_dependency_smp _os_atomic_mo_dependency_smp #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY #define memory_order_acquire_smp _os_atomic_mo_acquire_smp #define memory_order_release_smp _os_atomic_mo_release_smp #define memory_order_acq_rel_smp _os_atomic_mo_acq_rel_smp #define memory_order_seq_cst_smp _os_atomic_mo_seq_cst_smp #endif /* * Hack needed for os_compiler_barrier() to work (including with empty argument) */ #define _os_compiler_barrier_relaxed _os_atomic_mo_relaxed #define _os_compiler_barrier_acquire _os_atomic_mo_acquire #define _os_compiler_barrier_release _os_atomic_mo_release #define _os_compiler_barrier_acq_rel _os_atomic_mo_acq_rel #define _os_compiler_barrier_ _os_atomic_mo_acq_rel /* * Mapping between compiler barrier/memory orders and: * - compiler barriers before atomics ("rel_barrier") * - compiler barriers after atomics ("acq_barrier") */ #define _os_rel_barrier_relaxed _os_atomic_mo_relaxed #define _os_rel_barrier_compiler_acquire _os_atomic_mo_relaxed #define _os_rel_barrier_compiler_release _os_atomic_mo_release #define _os_rel_barrier_compiler_acq_rel _os_atomic_mo_release #define _os_rel_barrier_consume _os_atomic_mo_relaxed #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY #define _os_rel_barrier_dependency _os_atomic_mo_relaxed #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY #define _os_rel_barrier_acquire _os_atomic_mo_relaxed #define _os_rel_barrier_release _os_atomic_mo_release #define _os_rel_barrier_acq_rel _os_atomic_mo_release #define _os_rel_barrier_seq_cst _os_atomic_mo_release #define _os_acq_barrier_relaxed _os_atomic_mo_relaxed #define _os_acq_barrier_compiler_acquire _os_atomic_mo_acquire #define _os_acq_barrier_compiler_release _os_atomic_mo_relaxed #define _os_acq_barrier_compiler_acq_rel _os_atomic_mo_acquire #define _os_acq_barrier_consume _os_atomic_mo_acquire #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY #define _os_acq_barrier_dependency _os_atomic_mo_acquire #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY #define _os_acq_barrier_acquire _os_atomic_mo_acquire #define _os_acq_barrier_release _os_atomic_mo_relaxed #define _os_acq_barrier_acq_rel _os_atomic_mo_acquire #define _os_acq_barrier_seq_cst _os_atomic_mo_acquire #define _os_compiler_barrier_before_atomic(m) \ os_atomic_std(atomic_signal_fence)(_os_rel_barrier_##m) #define _os_compiler_barrier_after_atomic(m) \ os_atomic_std(atomic_signal_fence)(_os_acq_barrier_##m) /* * Mapping between compiler barrier/memmory orders and: * - memory fences before atomics ("rel_fence") * - memory fences after atomics ("acq_fence") */ #define _os_rel_fence_relaxed _os_atomic_mo_relaxed #define _os_rel_fence_compiler_acquire _os_atomic_mo_relaxed #define _os_rel_fence_compiler_release _os_atomic_mo_release #define _os_rel_fence_compiler_acq_rel _os_atomic_mo_release #define _os_rel_fence_consume _os_atomic_mo_relaxed_smp #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY #define _os_rel_fence_dependency _os_atomic_mo_relaxed_smp #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY #define _os_rel_fence_acquire _os_atomic_mo_relaxed_smp #define _os_rel_fence_release _os_atomic_mo_release_smp #define _os_rel_fence_acq_rel _os_atomic_mo_release_smp #define _os_rel_fence_seq_cst _os_atomic_mo_release_smp #define _os_acq_fence_relaxed _os_atomic_mo_relaxed #define _os_acq_fence_compiler_acquire _os_atomic_mo_relaxed #define _os_acq_fence_compiler_release _os_atomic_mo_relaxed #define _os_acq_fence_compiler_acq_rel _os_atomic_mo_relaxed #define _os_acq_fence_consume _os_atomic_mo_acquire_smp #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY #define _os_acq_fence_dependency _os_atomic_mo_dependency_smp #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY #define _os_acq_fence_acquire _os_atomic_mo_acquire_smp #define _os_acq_fence_release _os_atomic_mo_relaxed_smp #define _os_acq_fence_acq_rel _os_atomic_mo_acquire_smp #define _os_acq_fence_seq_cst _os_atomic_mo_acquire_smp #define _os_memory_fence_before_atomic(m) \ os_atomic_std(atomic_thread_fence)(_os_rel_fence_##m) #define _os_memory_fence_after_atomic(m) \ os_atomic_std(atomic_thread_fence)(_os_acq_fence_##m) /* * Misc. helpers */ #define _os_atomic_value_cast(p, v) \ ({ __typeof__(*os_cast_to_nonatomic_pointer(p)) ___v = (v); ___v; }) #define _os_atomic_c11_op_orig(p, v, m, o) ({ \ _os_compiler_barrier_before_atomic(m); \ __auto_type _r = os_atomic_std(atomic_##o##_explicit)(\ os_cast_to_atomic_pointer(p), \ _os_atomic_value_cast(p, v), \ _os_atomic_mo_##m##_smp); \ _os_compiler_barrier_after_atomic(m); \ _r; \ }) #define _os_atomic_c11_op(p, v, m, o, op) ({ \ __auto_type _v = _os_atomic_value_cast(p, v); \ _os_atomic_c11_op_orig(p, _v, m, o) op _v; \ }) #define _os_atomic_clang_op_orig(p, v, m, o) ({ \ _os_compiler_barrier_before_atomic(m); \ __auto_type _r = __atomic_##o(os_cast_to_nonatomic_pointer(p), \ _os_atomic_value_cast(p, v), \ _os_atomic_mo_##m##_smp); \ _os_compiler_barrier_after_atomic(m); \ _r; \ }) #define _os_atomic_clang_op(p, v, m, o, op) ({ \ __auto_type _v = _os_atomic_value_cast(p, v); \ __auto_type _s = _os_atomic_clang_op_orig(p, _v, m, o); \ op(_s, _v); \ }) #if OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY #define _os_atomic_auto_dependency(e) \ _Generic(e, \ os_atomic_dependency_t: (e), \ default: os_atomic_make_dependency(e)) #endif // OS_ATOMIC_CONFIG_MEMORY_ORDER_DEPENDENCY #endif /* __OS_ATOMIC_PRIVATE_IMPL_H__ */