19 #include "kmp_debug.h"
32 #define KMP_PAD(type, sz) \
33 (sizeof(type) + (sz - ((sizeof(type) - 1) % (sz)) - 1))
34 #define KMP_GTID_DNE (-2)
53 #if (KMP_OS_LINUX || KMP_OS_AIX) && defined(KMP_GOMP_COMPAT)
54 #define OMP_LOCK_T_SIZE sizeof(int)
55 #define OMP_NEST_LOCK_T_SIZE sizeof(void *)
57 #define OMP_LOCK_T_SIZE sizeof(void *)
58 #define OMP_NEST_LOCK_T_SIZE sizeof(void *)
64 #define OMP_CRITICAL_SIZE sizeof(void *)
65 #define INTEL_CRITICAL_SIZE 32
68 typedef kmp_uint32 kmp_lock_flags_t;
70 #define kmp_lf_critical_section 1
73 typedef kmp_uint32 kmp_lock_index_t;
77 struct kmp_lock_pool {
78 union kmp_user_lock *next;
79 kmp_lock_index_t index;
82 typedef struct kmp_lock_pool kmp_lock_pool_t;
84 extern void __kmp_validate_locks(
void);
121 struct kmp_base_tas_lock {
123 #if defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) && \
127 kmp_int32 depth_locked;
128 std::atomic<kmp_int32> poll;
130 std::atomic<kmp_int32> poll;
131 kmp_int32 depth_locked;
135 typedef struct kmp_base_tas_lock kmp_base_tas_lock_t;
138 kmp_base_tas_lock_t lk;
139 kmp_lock_pool_t pool;
143 typedef union kmp_tas_lock kmp_tas_lock_t;
147 #define KMP_TAS_LOCK_INITIALIZER(lock) \
149 { KMP_LOCK_FREE(tas), 0 } \
152 extern int __kmp_acquire_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
153 extern int __kmp_test_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
154 extern int __kmp_release_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
155 extern void __kmp_init_tas_lock(kmp_tas_lock_t *lck);
156 extern void __kmp_destroy_tas_lock(kmp_tas_lock_t *lck);
158 extern int __kmp_acquire_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
159 extern int __kmp_test_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
160 extern int __kmp_release_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
161 extern void __kmp_init_nested_tas_lock(kmp_tas_lock_t *lck);
162 extern void __kmp_destroy_nested_tas_lock(kmp_tas_lock_t *lck);
164 #define KMP_LOCK_RELEASED 1
165 #define KMP_LOCK_STILL_HELD 0
166 #define KMP_LOCK_ACQUIRED_FIRST 1
167 #define KMP_LOCK_ACQUIRED_NEXT 0
168 #ifndef KMP_USE_FUTEX
169 #define KMP_USE_FUTEX \
171 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64))
186 struct kmp_base_futex_lock {
187 volatile kmp_int32 poll;
190 kmp_int32 depth_locked;
193 typedef struct kmp_base_futex_lock kmp_base_futex_lock_t;
195 union kmp_futex_lock {
196 kmp_base_futex_lock_t lk;
197 kmp_lock_pool_t pool;
202 typedef union kmp_futex_lock kmp_futex_lock_t;
206 #define KMP_FUTEX_LOCK_INITIALIZER(lock) \
208 { KMP_LOCK_FREE(futex), 0 } \
211 extern int __kmp_acquire_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid);
212 extern int __kmp_test_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid);
213 extern int __kmp_release_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid);
214 extern void __kmp_init_futex_lock(kmp_futex_lock_t *lck);
215 extern void __kmp_destroy_futex_lock(kmp_futex_lock_t *lck);
217 extern int __kmp_acquire_nested_futex_lock(kmp_futex_lock_t *lck,
219 extern int __kmp_test_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid);
220 extern int __kmp_release_nested_futex_lock(kmp_futex_lock_t *lck,
222 extern void __kmp_init_nested_futex_lock(kmp_futex_lock_t *lck);
223 extern void __kmp_destroy_nested_futex_lock(kmp_futex_lock_t *lck);
236 struct kmp_base_ticket_lock {
238 std::atomic_bool initialized;
239 volatile union kmp_ticket_lock *
self;
243 std::atomic_uint now_serving;
244 std::atomic_int owner_id;
245 std::atomic_int depth_locked;
246 kmp_lock_flags_t flags;
249 struct kmp_base_ticket_lock {
251 std::atomic<bool> initialized;
252 volatile union kmp_ticket_lock *
self;
254 std::atomic<unsigned>
256 std::atomic<unsigned>
258 std::atomic<int> owner_id;
259 std::atomic<int> depth_locked;
260 kmp_lock_flags_t flags;
266 struct kmp_base_ticket_lock;
270 typedef struct kmp_base_ticket_lock kmp_base_ticket_lock_t;
272 union KMP_ALIGN_CACHE kmp_ticket_lock {
273 kmp_base_ticket_lock_t
275 kmp_lock_pool_t pool;
277 char lk_pad[KMP_PAD(kmp_base_ticket_lock_t, CACHE_LINE)];
280 typedef union kmp_ticket_lock kmp_ticket_lock_t;
285 #define KMP_TICKET_LOCK_INITIALIZER(lock) \
287 { true, &(lock), NULL, 0U, 0U, 0, -1 } \
290 extern int __kmp_acquire_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid);
291 extern int __kmp_test_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid);
292 extern int __kmp_test_ticket_lock_with_cheks(kmp_ticket_lock_t *lck,
294 extern int __kmp_release_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid);
295 extern void __kmp_init_ticket_lock(kmp_ticket_lock_t *lck);
296 extern void __kmp_destroy_ticket_lock(kmp_ticket_lock_t *lck);
298 extern int __kmp_acquire_nested_ticket_lock(kmp_ticket_lock_t *lck,
300 extern int __kmp_test_nested_ticket_lock(kmp_ticket_lock_t *lck,
302 extern int __kmp_release_nested_ticket_lock(kmp_ticket_lock_t *lck,
304 extern void __kmp_init_nested_ticket_lock(kmp_ticket_lock_t *lck);
305 extern void __kmp_destroy_nested_ticket_lock(kmp_ticket_lock_t *lck);
310 #if KMP_USE_ADAPTIVE_LOCKS
312 struct kmp_adaptive_lock_info;
314 typedef struct kmp_adaptive_lock_info kmp_adaptive_lock_info_t;
316 #if KMP_DEBUG_ADAPTIVE_LOCKS
318 struct kmp_adaptive_lock_statistics {
320 kmp_adaptive_lock_info_t *next;
321 kmp_adaptive_lock_info_t *prev;
324 kmp_uint32 successfulSpeculations;
325 kmp_uint32 hardFailedSpeculations;
326 kmp_uint32 softFailedSpeculations;
327 kmp_uint32 nonSpeculativeAcquires;
328 kmp_uint32 nonSpeculativeAcquireAttempts;
329 kmp_uint32 lemmingYields;
332 typedef struct kmp_adaptive_lock_statistics kmp_adaptive_lock_statistics_t;
334 extern void __kmp_print_speculative_stats();
335 extern void __kmp_init_speculative_stats();
339 struct kmp_adaptive_lock_info {
344 kmp_uint32
volatile badness;
345 kmp_uint32
volatile acquire_attempts;
347 kmp_uint32 max_badness;
348 kmp_uint32 max_soft_retries;
350 #if KMP_DEBUG_ADAPTIVE_LOCKS
351 kmp_adaptive_lock_statistics_t
volatile stats;
357 struct kmp_base_queuing_lock {
360 volatile union kmp_queuing_lock
378 volatile kmp_int32 owner_id;
379 kmp_int32 depth_locked;
381 kmp_lock_flags_t flags;
384 typedef struct kmp_base_queuing_lock kmp_base_queuing_lock_t;
386 KMP_BUILD_ASSERT(offsetof(kmp_base_queuing_lock_t, tail_id) % 8 == 0);
388 union KMP_ALIGN_CACHE kmp_queuing_lock {
389 kmp_base_queuing_lock_t
391 kmp_lock_pool_t pool;
393 char lk_pad[KMP_PAD(kmp_base_queuing_lock_t, CACHE_LINE)];
396 typedef union kmp_queuing_lock kmp_queuing_lock_t;
398 extern int __kmp_acquire_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid);
399 extern int __kmp_test_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid);
400 extern int __kmp_release_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid);
401 extern void __kmp_init_queuing_lock(kmp_queuing_lock_t *lck);
402 extern void __kmp_destroy_queuing_lock(kmp_queuing_lock_t *lck);
404 extern int __kmp_acquire_nested_queuing_lock(kmp_queuing_lock_t *lck,
406 extern int __kmp_test_nested_queuing_lock(kmp_queuing_lock_t *lck,
408 extern int __kmp_release_nested_queuing_lock(kmp_queuing_lock_t *lck,
410 extern void __kmp_init_nested_queuing_lock(kmp_queuing_lock_t *lck);
411 extern void __kmp_destroy_nested_queuing_lock(kmp_queuing_lock_t *lck);
413 #if KMP_USE_ADAPTIVE_LOCKS
417 struct kmp_base_adaptive_lock {
418 kmp_base_queuing_lock qlk;
419 KMP_ALIGN(CACHE_LINE)
420 kmp_adaptive_lock_info_t
424 typedef struct kmp_base_adaptive_lock kmp_base_adaptive_lock_t;
426 union KMP_ALIGN_CACHE kmp_adaptive_lock {
427 kmp_base_adaptive_lock_t lk;
428 kmp_lock_pool_t pool;
430 char lk_pad[KMP_PAD(kmp_base_adaptive_lock_t, CACHE_LINE)];
432 typedef union kmp_adaptive_lock kmp_adaptive_lock_t;
434 #define GET_QLK_PTR(l) ((kmp_queuing_lock_t *)&(l)->lk.qlk)
440 struct kmp_base_drdpa_lock {
449 volatile union kmp_drdpa_lock
452 std::atomic<std::atomic<kmp_uint64> *> polls;
453 std::atomic<kmp_uint64> mask;
454 kmp_uint64 cleanup_ticket;
455 std::atomic<kmp_uint64> *old_polls;
456 kmp_uint32 num_polls;
462 std::atomic<kmp_uint64> next_ticket;
477 kmp_uint64 now_serving;
478 volatile kmp_uint32 owner_id;
479 kmp_int32 depth_locked;
480 kmp_lock_flags_t flags;
483 typedef struct kmp_base_drdpa_lock kmp_base_drdpa_lock_t;
485 union KMP_ALIGN_CACHE kmp_drdpa_lock {
486 kmp_base_drdpa_lock_t
488 kmp_lock_pool_t pool;
490 char lk_pad[KMP_PAD(kmp_base_drdpa_lock_t, CACHE_LINE)];
493 typedef union kmp_drdpa_lock kmp_drdpa_lock_t;
495 extern int __kmp_acquire_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid);
496 extern int __kmp_test_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid);
497 extern int __kmp_release_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid);
498 extern void __kmp_init_drdpa_lock(kmp_drdpa_lock_t *lck);
499 extern void __kmp_destroy_drdpa_lock(kmp_drdpa_lock_t *lck);
501 extern int __kmp_acquire_nested_drdpa_lock(kmp_drdpa_lock_t *lck,
503 extern int __kmp_test_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid);
504 extern int __kmp_release_nested_drdpa_lock(kmp_drdpa_lock_t *lck,
506 extern void __kmp_init_nested_drdpa_lock(kmp_drdpa_lock_t *lck);
507 extern void __kmp_destroy_nested_drdpa_lock(kmp_drdpa_lock_t *lck);
521 typedef kmp_ticket_lock_t kmp_bootstrap_lock_t;
523 #define KMP_BOOTSTRAP_LOCK_INITIALIZER(lock) KMP_TICKET_LOCK_INITIALIZER((lock))
524 #define KMP_BOOTSTRAP_LOCK_INIT(lock) \
525 kmp_bootstrap_lock_t lock = KMP_TICKET_LOCK_INITIALIZER(lock)
527 static inline int __kmp_acquire_bootstrap_lock(kmp_bootstrap_lock_t *lck) {
528 return __kmp_acquire_ticket_lock(lck, KMP_GTID_DNE);
531 static inline int __kmp_test_bootstrap_lock(kmp_bootstrap_lock_t *lck) {
532 return __kmp_test_ticket_lock(lck, KMP_GTID_DNE);
535 static inline void __kmp_release_bootstrap_lock(kmp_bootstrap_lock_t *lck) {
536 __kmp_release_ticket_lock(lck, KMP_GTID_DNE);
539 static inline void __kmp_init_bootstrap_lock(kmp_bootstrap_lock_t *lck) {
540 __kmp_init_ticket_lock(lck);
543 static inline void __kmp_destroy_bootstrap_lock(kmp_bootstrap_lock_t *lck) {
544 __kmp_destroy_ticket_lock(lck);
555 typedef kmp_ticket_lock_t kmp_lock_t;
557 #define KMP_LOCK_INIT(lock) kmp_lock_t lock = KMP_TICKET_LOCK_INITIALIZER(lock)
559 static inline int __kmp_acquire_lock(kmp_lock_t *lck, kmp_int32 gtid) {
560 return __kmp_acquire_ticket_lock(lck, gtid);
563 static inline int __kmp_test_lock(kmp_lock_t *lck, kmp_int32 gtid) {
564 return __kmp_test_ticket_lock(lck, gtid);
567 static inline void __kmp_release_lock(kmp_lock_t *lck, kmp_int32 gtid) {
568 __kmp_release_ticket_lock(lck, gtid);
571 static inline void __kmp_init_lock(kmp_lock_t *lck) {
572 __kmp_init_ticket_lock(lck);
575 static inline void __kmp_destroy_lock(kmp_lock_t *lck) {
576 __kmp_destroy_ticket_lock(lck);
592 #if KMP_USE_DYNAMIC_LOCK && KMP_USE_TSX
600 #if KMP_USE_ADAPTIVE_LOCKS
605 typedef enum kmp_lock_kind kmp_lock_kind_t;
607 extern kmp_lock_kind_t __kmp_user_lock_kind;
609 union kmp_user_lock {
612 kmp_futex_lock_t futex;
614 kmp_ticket_lock_t ticket;
615 kmp_queuing_lock_t queuing;
616 kmp_drdpa_lock_t drdpa;
617 #if KMP_USE_ADAPTIVE_LOCKS
618 kmp_adaptive_lock_t adaptive;
620 kmp_lock_pool_t pool;
623 typedef union kmp_user_lock *kmp_user_lock_p;
625 #if !KMP_USE_DYNAMIC_LOCK
627 extern size_t __kmp_base_user_lock_size;
628 extern size_t __kmp_user_lock_size;
630 extern kmp_int32 (*__kmp_get_user_lock_owner_)(kmp_user_lock_p lck);
632 static inline kmp_int32 __kmp_get_user_lock_owner(kmp_user_lock_p lck) {
633 KMP_DEBUG_ASSERT(__kmp_get_user_lock_owner_ != NULL);
634 return (*__kmp_get_user_lock_owner_)(lck);
637 extern int (*__kmp_acquire_user_lock_with_checks_)(kmp_user_lock_p lck,
640 #if KMP_OS_LINUX && \
641 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
643 #define __kmp_acquire_user_lock_with_checks(lck, gtid) \
644 if (__kmp_user_lock_kind == lk_tas) { \
645 if (__kmp_env_consistency_check) { \
646 char const *const func = "omp_set_lock"; \
647 if ((sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) && \
648 lck->tas.lk.depth_locked != -1) { \
649 KMP_FATAL(LockNestableUsedAsSimple, func); \
651 if ((gtid >= 0) && (lck->tas.lk.poll - 1 == gtid)) { \
652 KMP_FATAL(LockIsAlreadyOwned, func); \
655 if (lck->tas.lk.poll != 0 || \
656 !__kmp_atomic_compare_store_acq(&lck->tas.lk.poll, 0, gtid + 1)) { \
659 KMP_FSYNC_PREPARE(lck); \
660 KMP_INIT_YIELD(spins); \
661 KMP_INIT_BACKOFF(time); \
663 KMP_YIELD_OVERSUB_ELSE_SPIN(spins, time); \
665 lck->tas.lk.poll != 0 || \
666 !__kmp_atomic_compare_store_acq(&lck->tas.lk.poll, 0, gtid + 1)); \
668 KMP_FSYNC_ACQUIRED(lck); \
670 KMP_DEBUG_ASSERT(__kmp_acquire_user_lock_with_checks_ != NULL); \
671 (*__kmp_acquire_user_lock_with_checks_)(lck, gtid); \
675 static inline int __kmp_acquire_user_lock_with_checks(kmp_user_lock_p lck,
677 KMP_DEBUG_ASSERT(__kmp_acquire_user_lock_with_checks_ != NULL);
678 return (*__kmp_acquire_user_lock_with_checks_)(lck, gtid);
682 extern int (*__kmp_test_user_lock_with_checks_)(kmp_user_lock_p lck,
685 #if KMP_OS_LINUX && \
686 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
688 #include "kmp_i18n.h"
689 extern int __kmp_env_consistency_check;
690 static inline int __kmp_test_user_lock_with_checks(kmp_user_lock_p lck,
692 if (__kmp_user_lock_kind == lk_tas) {
693 if (__kmp_env_consistency_check) {
694 char const *
const func =
"omp_test_lock";
695 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
696 lck->tas.lk.depth_locked != -1) {
697 KMP_FATAL(LockNestableUsedAsSimple, func);
700 return ((lck->tas.lk.poll == 0) &&
701 __kmp_atomic_compare_store_acq(&lck->tas.lk.poll, 0, gtid + 1));
703 KMP_DEBUG_ASSERT(__kmp_test_user_lock_with_checks_ != NULL);
704 return (*__kmp_test_user_lock_with_checks_)(lck, gtid);
708 static inline int __kmp_test_user_lock_with_checks(kmp_user_lock_p lck,
710 KMP_DEBUG_ASSERT(__kmp_test_user_lock_with_checks_ != NULL);
711 return (*__kmp_test_user_lock_with_checks_)(lck, gtid);
715 extern int (*__kmp_release_user_lock_with_checks_)(kmp_user_lock_p lck,
718 static inline void __kmp_release_user_lock_with_checks(kmp_user_lock_p lck,
720 KMP_DEBUG_ASSERT(__kmp_release_user_lock_with_checks_ != NULL);
721 (*__kmp_release_user_lock_with_checks_)(lck, gtid);
724 extern void (*__kmp_init_user_lock_with_checks_)(kmp_user_lock_p lck);
726 static inline void __kmp_init_user_lock_with_checks(kmp_user_lock_p lck) {
727 KMP_DEBUG_ASSERT(__kmp_init_user_lock_with_checks_ != NULL);
728 (*__kmp_init_user_lock_with_checks_)(lck);
733 extern void (*__kmp_destroy_user_lock_)(kmp_user_lock_p lck);
735 static inline void __kmp_destroy_user_lock(kmp_user_lock_p lck) {
736 KMP_DEBUG_ASSERT(__kmp_destroy_user_lock_ != NULL);
737 (*__kmp_destroy_user_lock_)(lck);
740 extern void (*__kmp_destroy_user_lock_with_checks_)(kmp_user_lock_p lck);
742 static inline void __kmp_destroy_user_lock_with_checks(kmp_user_lock_p lck) {
743 KMP_DEBUG_ASSERT(__kmp_destroy_user_lock_with_checks_ != NULL);
744 (*__kmp_destroy_user_lock_with_checks_)(lck);
747 extern int (*__kmp_acquire_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
750 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
752 #define __kmp_acquire_nested_user_lock_with_checks(lck, gtid, depth) \
753 if (__kmp_user_lock_kind == lk_tas) { \
754 if (__kmp_env_consistency_check) { \
755 char const *const func = "omp_set_nest_lock"; \
756 if ((sizeof(kmp_tas_lock_t) <= OMP_NEST_LOCK_T_SIZE) && \
757 lck->tas.lk.depth_locked == -1) { \
758 KMP_FATAL(LockSimpleUsedAsNestable, func); \
761 if (lck->tas.lk.poll - 1 == gtid) { \
762 lck->tas.lk.depth_locked += 1; \
763 *depth = KMP_LOCK_ACQUIRED_NEXT; \
765 if ((lck->tas.lk.poll != 0) || \
766 !__kmp_atomic_compare_store_acq(&lck->tas.lk.poll, 0, gtid + 1)) { \
769 KMP_FSYNC_PREPARE(lck); \
770 KMP_INIT_YIELD(spins); \
771 KMP_INIT_BACKOFF(time); \
773 KMP_YIELD_OVERSUB_ELSE_SPIN(spins, time); \
775 (lck->tas.lk.poll != 0) || \
776 !__kmp_atomic_compare_store_acq(&lck->tas.lk.poll, 0, gtid + 1)); \
778 lck->tas.lk.depth_locked = 1; \
779 *depth = KMP_LOCK_ACQUIRED_FIRST; \
781 KMP_FSYNC_ACQUIRED(lck); \
783 KMP_DEBUG_ASSERT(__kmp_acquire_nested_user_lock_with_checks_ != NULL); \
784 *depth = (*__kmp_acquire_nested_user_lock_with_checks_)(lck, gtid); \
789 __kmp_acquire_nested_user_lock_with_checks(kmp_user_lock_p lck, kmp_int32 gtid,
791 KMP_DEBUG_ASSERT(__kmp_acquire_nested_user_lock_with_checks_ != NULL);
792 *depth = (*__kmp_acquire_nested_user_lock_with_checks_)(lck, gtid);
796 extern int (*__kmp_test_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
799 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64)
800 static inline int __kmp_test_nested_user_lock_with_checks(kmp_user_lock_p lck,
802 if (__kmp_user_lock_kind == lk_tas) {
804 if (__kmp_env_consistency_check) {
805 char const *
const func =
"omp_test_nest_lock";
806 if ((
sizeof(kmp_tas_lock_t) <= OMP_NEST_LOCK_T_SIZE) &&
807 lck->tas.lk.depth_locked == -1) {
808 KMP_FATAL(LockSimpleUsedAsNestable, func);
811 KMP_DEBUG_ASSERT(gtid >= 0);
812 if (lck->tas.lk.poll - 1 ==
814 return ++lck->tas.lk.depth_locked;
816 retval = ((lck->tas.lk.poll == 0) &&
817 __kmp_atomic_compare_store_acq(&lck->tas.lk.poll, 0, gtid + 1));
820 lck->tas.lk.depth_locked = 1;
824 KMP_DEBUG_ASSERT(__kmp_test_nested_user_lock_with_checks_ != NULL);
825 return (*__kmp_test_nested_user_lock_with_checks_)(lck, gtid);
829 static inline int __kmp_test_nested_user_lock_with_checks(kmp_user_lock_p lck,
831 KMP_DEBUG_ASSERT(__kmp_test_nested_user_lock_with_checks_ != NULL);
832 return (*__kmp_test_nested_user_lock_with_checks_)(lck, gtid);
836 extern int (*__kmp_release_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
840 __kmp_release_nested_user_lock_with_checks(kmp_user_lock_p lck,
842 KMP_DEBUG_ASSERT(__kmp_release_nested_user_lock_with_checks_ != NULL);
843 return (*__kmp_release_nested_user_lock_with_checks_)(lck, gtid);
846 extern void (*__kmp_init_nested_user_lock_with_checks_)(kmp_user_lock_p lck);
849 __kmp_init_nested_user_lock_with_checks(kmp_user_lock_p lck) {
850 KMP_DEBUG_ASSERT(__kmp_init_nested_user_lock_with_checks_ != NULL);
851 (*__kmp_init_nested_user_lock_with_checks_)(lck);
854 extern void (*__kmp_destroy_nested_user_lock_with_checks_)(kmp_user_lock_p lck);
857 __kmp_destroy_nested_user_lock_with_checks(kmp_user_lock_p lck) {
858 KMP_DEBUG_ASSERT(__kmp_destroy_nested_user_lock_with_checks_ != NULL);
859 (*__kmp_destroy_nested_user_lock_with_checks_)(lck);
875 extern int (*__kmp_is_user_lock_initialized_)(kmp_user_lock_p lck);
879 extern const ident_t *(*__kmp_get_user_lock_location_)(kmp_user_lock_p lck);
881 static inline const ident_t *__kmp_get_user_lock_location(kmp_user_lock_p lck) {
882 if (__kmp_get_user_lock_location_ != NULL) {
883 return (*__kmp_get_user_lock_location_)(lck);
889 extern void (*__kmp_set_user_lock_location_)(kmp_user_lock_p lck,
892 static inline void __kmp_set_user_lock_location(kmp_user_lock_p lck,
894 if (__kmp_set_user_lock_location_ != NULL) {
895 (*__kmp_set_user_lock_location_)(lck, loc);
899 extern kmp_lock_flags_t (*__kmp_get_user_lock_flags_)(kmp_user_lock_p lck);
901 extern void (*__kmp_set_user_lock_flags_)(kmp_user_lock_p lck,
902 kmp_lock_flags_t flags);
904 static inline void __kmp_set_user_lock_flags(kmp_user_lock_p lck,
905 kmp_lock_flags_t flags) {
906 if (__kmp_set_user_lock_flags_ != NULL) {
907 (*__kmp_set_user_lock_flags_)(lck, flags);
912 extern void __kmp_set_user_lock_vptrs(kmp_lock_kind_t user_lock_kind);
915 #define KMP_BIND_USER_LOCK_TEMPLATE(nest, kind, suffix) \
917 __kmp_acquire##nest##user_lock_with_checks_ = (int (*)( \
918 kmp_user_lock_p, kmp_int32))__kmp_acquire##nest##kind##_##suffix; \
919 __kmp_release##nest##user_lock_with_checks_ = (int (*)( \
920 kmp_user_lock_p, kmp_int32))__kmp_release##nest##kind##_##suffix; \
921 __kmp_test##nest##user_lock_with_checks_ = (int (*)( \
922 kmp_user_lock_p, kmp_int32))__kmp_test##nest##kind##_##suffix; \
923 __kmp_init##nest##user_lock_with_checks_ = \
924 (void (*)(kmp_user_lock_p))__kmp_init##nest##kind##_##suffix; \
925 __kmp_destroy##nest##user_lock_with_checks_ = \
926 (void (*)(kmp_user_lock_p))__kmp_destroy##nest##kind##_##suffix; \
929 #define KMP_BIND_USER_LOCK(kind) KMP_BIND_USER_LOCK_TEMPLATE(_, kind, lock)
930 #define KMP_BIND_USER_LOCK_WITH_CHECKS(kind) \
931 KMP_BIND_USER_LOCK_TEMPLATE(_, kind, lock_with_checks)
932 #define KMP_BIND_NESTED_USER_LOCK(kind) \
933 KMP_BIND_USER_LOCK_TEMPLATE(_nested_, kind, lock)
934 #define KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(kind) \
935 KMP_BIND_USER_LOCK_TEMPLATE(_nested_, kind, lock_with_checks)
960 struct kmp_lock_table {
961 kmp_lock_index_t used;
962 kmp_lock_index_t allocated;
963 kmp_user_lock_p *table;
966 typedef struct kmp_lock_table kmp_lock_table_t;
968 extern kmp_lock_table_t __kmp_user_lock_table;
969 extern kmp_user_lock_p __kmp_lock_pool;
971 struct kmp_block_of_locks {
972 struct kmp_block_of_locks *next_block;
976 typedef struct kmp_block_of_locks kmp_block_of_locks_t;
978 extern kmp_block_of_locks_t *__kmp_lock_blocks;
979 extern int __kmp_num_locks_in_block;
981 extern kmp_user_lock_p __kmp_user_lock_allocate(
void **user_lock,
983 kmp_lock_flags_t flags);
984 extern void __kmp_user_lock_free(
void **user_lock, kmp_int32 gtid,
985 kmp_user_lock_p lck);
986 extern kmp_user_lock_p __kmp_lookup_user_lock(
void **user_lock,
988 extern void __kmp_cleanup_user_locks();
990 #define KMP_CHECK_USER_LOCK_INIT() \
992 if (!TCR_4(__kmp_init_user_locks)) { \
993 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); \
994 if (!TCR_4(__kmp_init_user_locks)) { \
995 TCW_4(__kmp_init_user_locks, TRUE); \
997 __kmp_release_bootstrap_lock(&__kmp_initz_lock); \
1006 #if KMP_USE_DYNAMIC_LOCK
1044 #define KMP_USE_INLINED_TAS \
1045 (KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)) && 1
1046 #define KMP_USE_INLINED_FUTEX KMP_USE_FUTEX && 0
1053 #define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a) m(hle, a) m(rtm_spin, a)
1054 #define KMP_FOREACH_I_LOCK(m, a) \
1055 m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) m(rtm_queuing, a) \
1056 m(nested_tas, a) m(nested_futex, a) m(nested_ticket, a) \
1057 m(nested_queuing, a) m(nested_drdpa, a)
1059 #define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(hle, a) m(rtm_spin, a)
1060 #define KMP_FOREACH_I_LOCK(m, a) \
1061 m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) m(rtm_queuing, a) \
1062 m(nested_tas, a) m(nested_ticket, a) m(nested_queuing, a) \
1065 #define KMP_LAST_D_LOCK lockseq_rtm_spin
1068 #define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a)
1069 #define KMP_FOREACH_I_LOCK(m, a) \
1070 m(ticket, a) m(queuing, a) m(drdpa, a) m(nested_tas, a) m(nested_futex, a) \
1071 m(nested_ticket, a) m(nested_queuing, a) m(nested_drdpa, a)
1072 #define KMP_LAST_D_LOCK lockseq_futex
1074 #define KMP_FOREACH_D_LOCK(m, a) m(tas, a)
1075 #define KMP_FOREACH_I_LOCK(m, a) \
1076 m(ticket, a) m(queuing, a) m(drdpa, a) m(nested_tas, a) m(nested_ticket, a) \
1077 m(nested_queuing, a) m(nested_drdpa, a)
1078 #define KMP_LAST_D_LOCK lockseq_tas
1083 #define KMP_LOCK_SHIFT \
1085 #define KMP_FIRST_D_LOCK lockseq_tas
1086 #define KMP_FIRST_I_LOCK lockseq_ticket
1087 #define KMP_LAST_I_LOCK lockseq_nested_drdpa
1088 #define KMP_NUM_I_LOCKS \
1089 (locktag_nested_drdpa + 1)
1092 typedef kmp_uint32 kmp_dyna_lock_t;
1097 lockseq_indirect = 0,
1098 #define expand_seq(l, a) lockseq_##l,
1099 KMP_FOREACH_D_LOCK(expand_seq, 0) KMP_FOREACH_I_LOCK(expand_seq, 0)
1101 } kmp_dyna_lockseq_t;
1105 #define expand_tag(l, a) locktag_##l,
1106 KMP_FOREACH_I_LOCK(expand_tag, 0)
1108 } kmp_indirect_locktag_t;
1111 #define KMP_IS_D_LOCK(seq) \
1112 ((seq) >= KMP_FIRST_D_LOCK && (seq) <= KMP_LAST_D_LOCK)
1113 #define KMP_IS_I_LOCK(seq) \
1114 ((seq) >= KMP_FIRST_I_LOCK && (seq) <= KMP_LAST_I_LOCK)
1115 #define KMP_GET_I_TAG(seq) (kmp_indirect_locktag_t)((seq)-KMP_FIRST_I_LOCK)
1116 #define KMP_GET_D_TAG(seq) ((seq) << 1 | 1)
1120 #define expand_tag(l, a) locktag_##l = KMP_GET_D_TAG(lockseq_##l),
1121 KMP_FOREACH_D_LOCK(expand_tag, 0)
1123 } kmp_direct_locktag_t;
1127 kmp_user_lock_p lock;
1128 kmp_indirect_locktag_t type;
1129 } kmp_indirect_lock_t;
1133 extern void (*__kmp_direct_init[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t);
1134 extern void (**__kmp_direct_destroy)(kmp_dyna_lock_t *);
1135 extern int (**__kmp_direct_set)(kmp_dyna_lock_t *, kmp_int32);
1136 extern int (**__kmp_direct_unset)(kmp_dyna_lock_t *, kmp_int32);
1137 extern int (**__kmp_direct_test)(kmp_dyna_lock_t *, kmp_int32);
1141 extern void (*__kmp_indirect_init[])(kmp_user_lock_p);
1142 extern void (**__kmp_indirect_destroy)(kmp_user_lock_p);
1143 extern int (**__kmp_indirect_set)(kmp_user_lock_p, kmp_int32);
1144 extern int (**__kmp_indirect_unset)(kmp_user_lock_p, kmp_int32);
1145 extern int (**__kmp_indirect_test)(kmp_user_lock_p, kmp_int32);
1148 #define KMP_EXTRACT_D_TAG(l) \
1149 ((kmp_dyna_lock_t)((kmp_base_tas_lock_t *)(l))->poll & \
1150 ((1 << KMP_LOCK_SHIFT) - 1) & \
1151 -((kmp_dyna_lock_t)((kmp_tas_lock_t *)(l))->lk.poll & 1))
1154 #define KMP_EXTRACT_I_INDEX(l) \
1155 ((kmp_lock_index_t)((kmp_base_tas_lock_t *)(l))->poll >> 1)
1159 #define KMP_D_LOCK_FUNC(l, op) __kmp_direct_##op[KMP_EXTRACT_D_TAG(l)]
1163 #define KMP_I_LOCK_FUNC(l, op) \
1164 __kmp_indirect_##op[((kmp_indirect_lock_t *)(l))->type]
1167 #define KMP_INIT_D_LOCK(l, seq) \
1168 __kmp_direct_init[KMP_GET_D_TAG(seq)]((kmp_dyna_lock_t *)l, seq)
1171 #define KMP_INIT_I_LOCK(l, seq) \
1172 __kmp_direct_init[0]((kmp_dyna_lock_t *)(l), seq)
1175 #define KMP_LOCK_FREE(type) (locktag_##type)
1178 #define KMP_LOCK_BUSY(v, type) ((v) << KMP_LOCK_SHIFT | locktag_##type)
1181 #define KMP_LOCK_STRIP(v) ((v) >> KMP_LOCK_SHIFT)
1185 extern void __kmp_init_dynamic_user_locks();
1188 extern kmp_indirect_lock_t *
1189 __kmp_allocate_indirect_lock(
void **, kmp_int32, kmp_indirect_locktag_t);
1192 extern void __kmp_cleanup_indirect_user_locks();
1195 extern kmp_dyna_lockseq_t __kmp_user_lock_seq;
1198 extern void (*__kmp_indirect_set_location[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
1200 #define KMP_SET_I_LOCK_LOCATION(lck, loc) \
1202 if (__kmp_indirect_set_location[(lck)->type] != NULL) \
1203 __kmp_indirect_set_location[(lck)->type]((lck)->lock, loc); \
1207 extern void (*__kmp_indirect_set_flags[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
1209 #define KMP_SET_I_LOCK_FLAGS(lck, flag) \
1211 if (__kmp_indirect_set_flags[(lck)->type] != NULL) \
1212 __kmp_indirect_set_flags[(lck)->type]((lck)->lock, flag); \
1216 extern const ident_t *(*__kmp_indirect_get_location[KMP_NUM_I_LOCKS])(
1218 #define KMP_GET_I_LOCK_LOCATION(lck) \
1219 (__kmp_indirect_get_location[(lck)->type] != NULL \
1220 ? __kmp_indirect_get_location[(lck)->type]((lck)->lock) \
1224 extern kmp_lock_flags_t (*__kmp_indirect_get_flags[KMP_NUM_I_LOCKS])(
1226 #define KMP_GET_I_LOCK_FLAGS(lck) \
1227 (__kmp_indirect_get_flags[(lck)->type] != NULL \
1228 ? __kmp_indirect_get_flags[(lck)->type]((lck)->lock) \
1232 #define KMP_I_LOCK_CHUNK 1024
1234 KMP_BUILD_ASSERT(KMP_I_LOCK_CHUNK % 2 == 0);
1236 #define KMP_I_LOCK_TABLE_INIT_NROW_PTRS 8
1239 typedef struct kmp_indirect_lock_table {
1240 kmp_indirect_lock_t **table;
1241 kmp_uint32 nrow_ptrs;
1242 kmp_lock_index_t next;
1243 struct kmp_indirect_lock_table *next_table;
1244 } kmp_indirect_lock_table_t;
1246 extern kmp_indirect_lock_table_t __kmp_i_lock_table;
1250 static inline kmp_indirect_lock_t *__kmp_get_i_lock(kmp_lock_index_t idx) {
1251 kmp_indirect_lock_table_t *lock_table = &__kmp_i_lock_table;
1252 while (lock_table) {
1253 kmp_lock_index_t max_locks = lock_table->nrow_ptrs * KMP_I_LOCK_CHUNK;
1254 if (idx < max_locks) {
1255 kmp_lock_index_t row = idx / KMP_I_LOCK_CHUNK;
1256 kmp_lock_index_t col = idx % KMP_I_LOCK_CHUNK;
1257 if (!lock_table->table[row] || idx >= lock_table->next)
1259 return &lock_table->table[row][col];
1262 lock_table = lock_table->next_table;
1270 extern int __kmp_num_locks_in_block;
1273 #define KMP_LOOKUP_I_LOCK(l) \
1274 ((OMP_LOCK_T_SIZE < sizeof(void *)) \
1275 ? __kmp_get_i_lock(KMP_EXTRACT_I_INDEX(l)) \
1276 : *((kmp_indirect_lock_t **)(l)))
1279 extern kmp_int32 __kmp_get_user_lock_owner(kmp_user_lock_p, kmp_uint32);
1283 #define KMP_LOCK_BUSY(v, type) (v)
1284 #define KMP_LOCK_FREE(type) 0
1285 #define KMP_LOCK_STRIP(v) (v)
1292 kmp_uint32 max_backoff;
1293 kmp_uint32 min_tick;
1297 extern kmp_backoff_t __kmp_spin_backoff_params;
1300 extern void __kmp_spin_backoff(kmp_backoff_t *);