19 #include "kmp_debug.h" 32 #define KMP_PAD(type, sz) \ 33 (sizeof(type) + (sz - ((sizeof(type) - 1) % (sz)) - 1)) 34 #define KMP_GTID_DNE (-2) 53 #if KMP_OS_LINUX && defined(KMP_GOMP_COMPAT) 54 #define OMP_LOCK_T_SIZE sizeof(int) 55 #define OMP_NEST_LOCK_T_SIZE sizeof(void *) 57 #define OMP_LOCK_T_SIZE sizeof(void *) 58 #define OMP_NEST_LOCK_T_SIZE sizeof(void *) 64 #define OMP_CRITICAL_SIZE sizeof(void *) 65 #define INTEL_CRITICAL_SIZE 32 68 typedef kmp_uint32 kmp_lock_flags_t;
70 #define kmp_lf_critical_section 1 73 typedef kmp_uint32 kmp_lock_index_t;
77 struct kmp_lock_pool {
78 union kmp_user_lock *next;
79 kmp_lock_index_t index;
82 typedef struct kmp_lock_pool kmp_lock_pool_t;
84 extern void __kmp_validate_locks(
void);
121 struct kmp_base_tas_lock {
123 std::atomic<kmp_int32> poll;
124 kmp_int32 depth_locked;
127 typedef struct kmp_base_tas_lock kmp_base_tas_lock_t;
130 kmp_base_tas_lock_t lk;
131 kmp_lock_pool_t pool;
135 typedef union kmp_tas_lock kmp_tas_lock_t;
139 #define KMP_TAS_LOCK_INITIALIZER(lock) \ 141 { KMP_LOCK_FREE(tas), 0 } \ 144 extern int __kmp_acquire_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
145 extern int __kmp_test_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
146 extern int __kmp_release_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
147 extern void __kmp_init_tas_lock(kmp_tas_lock_t *lck);
148 extern void __kmp_destroy_tas_lock(kmp_tas_lock_t *lck);
150 extern int __kmp_acquire_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
151 extern int __kmp_test_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
152 extern int __kmp_release_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid);
153 extern void __kmp_init_nested_tas_lock(kmp_tas_lock_t *lck);
154 extern void __kmp_destroy_nested_tas_lock(kmp_tas_lock_t *lck);
156 #define KMP_LOCK_RELEASED 1 157 #define KMP_LOCK_STILL_HELD 0 158 #define KMP_LOCK_ACQUIRED_FIRST 1 159 #define KMP_LOCK_ACQUIRED_NEXT 0 160 #ifndef KMP_USE_FUTEX 161 #define KMP_USE_FUTEX \ 163 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)) 178 struct kmp_base_futex_lock {
179 volatile kmp_int32 poll;
182 kmp_int32 depth_locked;
185 typedef struct kmp_base_futex_lock kmp_base_futex_lock_t;
187 union kmp_futex_lock {
188 kmp_base_futex_lock_t lk;
189 kmp_lock_pool_t pool;
194 typedef union kmp_futex_lock kmp_futex_lock_t;
198 #define KMP_FUTEX_LOCK_INITIALIZER(lock) \ 200 { KMP_LOCK_FREE(futex), 0 } \ 203 extern int __kmp_acquire_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid);
204 extern int __kmp_test_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid);
205 extern int __kmp_release_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid);
206 extern void __kmp_init_futex_lock(kmp_futex_lock_t *lck);
207 extern void __kmp_destroy_futex_lock(kmp_futex_lock_t *lck);
209 extern int __kmp_acquire_nested_futex_lock(kmp_futex_lock_t *lck,
211 extern int __kmp_test_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid);
212 extern int __kmp_release_nested_futex_lock(kmp_futex_lock_t *lck,
214 extern void __kmp_init_nested_futex_lock(kmp_futex_lock_t *lck);
215 extern void __kmp_destroy_nested_futex_lock(kmp_futex_lock_t *lck);
217 #endif // KMP_USE_FUTEX 228 struct kmp_base_ticket_lock {
230 std::atomic_bool initialized;
231 volatile union kmp_ticket_lock *
self;
235 std::atomic_uint now_serving;
236 std::atomic_int owner_id;
237 std::atomic_int depth_locked;
238 kmp_lock_flags_t flags;
241 struct kmp_base_ticket_lock {
243 std::atomic<bool> initialized;
244 volatile union kmp_ticket_lock *
self;
246 std::atomic<unsigned>
248 std::atomic<unsigned>
250 std::atomic<int> owner_id;
251 std::atomic<int> depth_locked;
252 kmp_lock_flags_t flags;
258 struct kmp_base_ticket_lock;
260 #endif // !__cplusplus 262 typedef struct kmp_base_ticket_lock kmp_base_ticket_lock_t;
264 union KMP_ALIGN_CACHE kmp_ticket_lock {
265 kmp_base_ticket_lock_t
267 kmp_lock_pool_t pool;
269 char lk_pad[KMP_PAD(kmp_base_ticket_lock_t, CACHE_LINE)];
272 typedef union kmp_ticket_lock kmp_ticket_lock_t;
277 #define KMP_TICKET_LOCK_INITIALIZER(lock) \ 279 { true, &(lock), NULL, 0U, 0U, 0, -1 } \ 282 extern int __kmp_acquire_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid);
283 extern int __kmp_test_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid);
284 extern int __kmp_test_ticket_lock_with_cheks(kmp_ticket_lock_t *lck,
286 extern int __kmp_release_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid);
287 extern void __kmp_init_ticket_lock(kmp_ticket_lock_t *lck);
288 extern void __kmp_destroy_ticket_lock(kmp_ticket_lock_t *lck);
290 extern int __kmp_acquire_nested_ticket_lock(kmp_ticket_lock_t *lck,
292 extern int __kmp_test_nested_ticket_lock(kmp_ticket_lock_t *lck,
294 extern int __kmp_release_nested_ticket_lock(kmp_ticket_lock_t *lck,
296 extern void __kmp_init_nested_ticket_lock(kmp_ticket_lock_t *lck);
297 extern void __kmp_destroy_nested_ticket_lock(kmp_ticket_lock_t *lck);
302 #if KMP_USE_ADAPTIVE_LOCKS 304 struct kmp_adaptive_lock_info;
306 typedef struct kmp_adaptive_lock_info kmp_adaptive_lock_info_t;
308 #if KMP_DEBUG_ADAPTIVE_LOCKS 310 struct kmp_adaptive_lock_statistics {
312 kmp_adaptive_lock_info_t *next;
313 kmp_adaptive_lock_info_t *prev;
316 kmp_uint32 successfulSpeculations;
317 kmp_uint32 hardFailedSpeculations;
318 kmp_uint32 softFailedSpeculations;
319 kmp_uint32 nonSpeculativeAcquires;
320 kmp_uint32 nonSpeculativeAcquireAttempts;
321 kmp_uint32 lemmingYields;
324 typedef struct kmp_adaptive_lock_statistics kmp_adaptive_lock_statistics_t;
326 extern void __kmp_print_speculative_stats();
327 extern void __kmp_init_speculative_stats();
329 #endif // KMP_DEBUG_ADAPTIVE_LOCKS 331 struct kmp_adaptive_lock_info {
336 kmp_uint32
volatile badness;
337 kmp_uint32
volatile acquire_attempts;
339 kmp_uint32 max_badness;
340 kmp_uint32 max_soft_retries;
342 #if KMP_DEBUG_ADAPTIVE_LOCKS 343 kmp_adaptive_lock_statistics_t
volatile stats;
347 #endif // KMP_USE_ADAPTIVE_LOCKS 349 struct kmp_base_queuing_lock {
352 volatile union kmp_queuing_lock
370 volatile kmp_int32 owner_id;
371 kmp_int32 depth_locked;
373 kmp_lock_flags_t flags;
376 typedef struct kmp_base_queuing_lock kmp_base_queuing_lock_t;
378 KMP_BUILD_ASSERT(offsetof(kmp_base_queuing_lock_t, tail_id) % 8 == 0);
380 union KMP_ALIGN_CACHE kmp_queuing_lock {
381 kmp_base_queuing_lock_t
383 kmp_lock_pool_t pool;
385 char lk_pad[KMP_PAD(kmp_base_queuing_lock_t, CACHE_LINE)];
388 typedef union kmp_queuing_lock kmp_queuing_lock_t;
390 extern int __kmp_acquire_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid);
391 extern int __kmp_test_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid);
392 extern int __kmp_release_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid);
393 extern void __kmp_init_queuing_lock(kmp_queuing_lock_t *lck);
394 extern void __kmp_destroy_queuing_lock(kmp_queuing_lock_t *lck);
396 extern int __kmp_acquire_nested_queuing_lock(kmp_queuing_lock_t *lck,
398 extern int __kmp_test_nested_queuing_lock(kmp_queuing_lock_t *lck,
400 extern int __kmp_release_nested_queuing_lock(kmp_queuing_lock_t *lck,
402 extern void __kmp_init_nested_queuing_lock(kmp_queuing_lock_t *lck);
403 extern void __kmp_destroy_nested_queuing_lock(kmp_queuing_lock_t *lck);
405 #if KMP_USE_ADAPTIVE_LOCKS 409 struct kmp_base_adaptive_lock {
410 kmp_base_queuing_lock qlk;
411 KMP_ALIGN(CACHE_LINE)
412 kmp_adaptive_lock_info_t
416 typedef struct kmp_base_adaptive_lock kmp_base_adaptive_lock_t;
418 union KMP_ALIGN_CACHE kmp_adaptive_lock {
419 kmp_base_adaptive_lock_t lk;
420 kmp_lock_pool_t pool;
422 char lk_pad[KMP_PAD(kmp_base_adaptive_lock_t, CACHE_LINE)];
424 typedef union kmp_adaptive_lock kmp_adaptive_lock_t;
426 #define GET_QLK_PTR(l) ((kmp_queuing_lock_t *)&(l)->lk.qlk) 428 #endif // KMP_USE_ADAPTIVE_LOCKS 432 struct kmp_base_drdpa_lock {
441 volatile union kmp_drdpa_lock
444 std::atomic<std::atomic<kmp_uint64> *> polls;
445 std::atomic<kmp_uint64> mask;
446 kmp_uint64 cleanup_ticket;
447 std::atomic<kmp_uint64> *old_polls;
448 kmp_uint32 num_polls;
454 std::atomic<kmp_uint64> next_ticket;
469 kmp_uint64 now_serving;
470 volatile kmp_uint32 owner_id;
471 kmp_int32 depth_locked;
472 kmp_lock_flags_t flags;
475 typedef struct kmp_base_drdpa_lock kmp_base_drdpa_lock_t;
477 union KMP_ALIGN_CACHE kmp_drdpa_lock {
478 kmp_base_drdpa_lock_t
480 kmp_lock_pool_t pool;
482 char lk_pad[KMP_PAD(kmp_base_drdpa_lock_t, CACHE_LINE)];
485 typedef union kmp_drdpa_lock kmp_drdpa_lock_t;
487 extern int __kmp_acquire_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid);
488 extern int __kmp_test_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid);
489 extern int __kmp_release_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid);
490 extern void __kmp_init_drdpa_lock(kmp_drdpa_lock_t *lck);
491 extern void __kmp_destroy_drdpa_lock(kmp_drdpa_lock_t *lck);
493 extern int __kmp_acquire_nested_drdpa_lock(kmp_drdpa_lock_t *lck,
495 extern int __kmp_test_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid);
496 extern int __kmp_release_nested_drdpa_lock(kmp_drdpa_lock_t *lck,
498 extern void __kmp_init_nested_drdpa_lock(kmp_drdpa_lock_t *lck);
499 extern void __kmp_destroy_nested_drdpa_lock(kmp_drdpa_lock_t *lck);
513 typedef kmp_ticket_lock_t kmp_bootstrap_lock_t;
515 #define KMP_BOOTSTRAP_LOCK_INITIALIZER(lock) KMP_TICKET_LOCK_INITIALIZER((lock)) 516 #define KMP_BOOTSTRAP_LOCK_INIT(lock) \ 517 kmp_bootstrap_lock_t lock = KMP_TICKET_LOCK_INITIALIZER(lock) 519 static inline int __kmp_acquire_bootstrap_lock(kmp_bootstrap_lock_t *lck) {
520 return __kmp_acquire_ticket_lock(lck, KMP_GTID_DNE);
523 static inline int __kmp_test_bootstrap_lock(kmp_bootstrap_lock_t *lck) {
524 return __kmp_test_ticket_lock(lck, KMP_GTID_DNE);
527 static inline void __kmp_release_bootstrap_lock(kmp_bootstrap_lock_t *lck) {
528 __kmp_release_ticket_lock(lck, KMP_GTID_DNE);
531 static inline void __kmp_init_bootstrap_lock(kmp_bootstrap_lock_t *lck) {
532 __kmp_init_ticket_lock(lck);
535 static inline void __kmp_destroy_bootstrap_lock(kmp_bootstrap_lock_t *lck) {
536 __kmp_destroy_ticket_lock(lck);
547 typedef kmp_ticket_lock_t kmp_lock_t;
549 #define KMP_LOCK_INIT(lock) kmp_lock_t lock = KMP_TICKET_LOCK_INITIALIZER(lock) 551 static inline int __kmp_acquire_lock(kmp_lock_t *lck, kmp_int32 gtid) {
552 return __kmp_acquire_ticket_lock(lck, gtid);
555 static inline int __kmp_test_lock(kmp_lock_t *lck, kmp_int32 gtid) {
556 return __kmp_test_ticket_lock(lck, gtid);
559 static inline void __kmp_release_lock(kmp_lock_t *lck, kmp_int32 gtid) {
560 __kmp_release_ticket_lock(lck, gtid);
563 static inline void __kmp_init_lock(kmp_lock_t *lck) {
564 __kmp_init_ticket_lock(lck);
567 static inline void __kmp_destroy_lock(kmp_lock_t *lck) {
568 __kmp_destroy_ticket_lock(lck);
584 #if KMP_USE_DYNAMIC_LOCK && KMP_USE_TSX 592 #if KMP_USE_ADAPTIVE_LOCKS 594 #endif // KMP_USE_ADAPTIVE_LOCKS 597 typedef enum kmp_lock_kind kmp_lock_kind_t;
599 extern kmp_lock_kind_t __kmp_user_lock_kind;
601 union kmp_user_lock {
604 kmp_futex_lock_t futex;
606 kmp_ticket_lock_t ticket;
607 kmp_queuing_lock_t queuing;
608 kmp_drdpa_lock_t drdpa;
609 #if KMP_USE_ADAPTIVE_LOCKS 610 kmp_adaptive_lock_t adaptive;
611 #endif // KMP_USE_ADAPTIVE_LOCKS 612 kmp_lock_pool_t pool;
615 typedef union kmp_user_lock *kmp_user_lock_p;
617 #if !KMP_USE_DYNAMIC_LOCK 619 extern size_t __kmp_base_user_lock_size;
620 extern size_t __kmp_user_lock_size;
622 extern kmp_int32 (*__kmp_get_user_lock_owner_)(kmp_user_lock_p lck);
624 static inline kmp_int32 __kmp_get_user_lock_owner(kmp_user_lock_p lck) {
625 KMP_DEBUG_ASSERT(__kmp_get_user_lock_owner_ != NULL);
626 return (*__kmp_get_user_lock_owner_)(lck);
629 extern int (*__kmp_acquire_user_lock_with_checks_)(kmp_user_lock_p lck,
632 #if KMP_OS_LINUX && \ 633 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 635 #define __kmp_acquire_user_lock_with_checks(lck, gtid) \ 636 if (__kmp_user_lock_kind == lk_tas) { \ 637 if (__kmp_env_consistency_check) { \ 638 char const *const func = "omp_set_lock"; \ 639 if ((sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) && \ 640 lck->tas.lk.depth_locked != -1) { \ 641 KMP_FATAL(LockNestableUsedAsSimple, func); \ 643 if ((gtid >= 0) && (lck->tas.lk.poll - 1 == gtid)) { \ 644 KMP_FATAL(LockIsAlreadyOwned, func); \ 647 if (lck->tas.lk.poll != 0 || \ 648 !__kmp_atomic_compare_store_acq(&lck->tas.lk.poll, 0, gtid + 1)) { \ 651 KMP_FSYNC_PREPARE(lck); \ 652 KMP_INIT_YIELD(spins); \ 653 KMP_INIT_BACKOFF(time); \ 655 KMP_YIELD_OVERSUB_ELSE_SPIN(spins, time); \ 657 lck->tas.lk.poll != 0 || \ 658 !__kmp_atomic_compare_store_acq(&lck->tas.lk.poll, 0, gtid + 1)); \ 660 KMP_FSYNC_ACQUIRED(lck); \ 662 KMP_DEBUG_ASSERT(__kmp_acquire_user_lock_with_checks_ != NULL); \ 663 (*__kmp_acquire_user_lock_with_checks_)(lck, gtid); \ 667 static inline int __kmp_acquire_user_lock_with_checks(kmp_user_lock_p lck,
669 KMP_DEBUG_ASSERT(__kmp_acquire_user_lock_with_checks_ != NULL);
670 return (*__kmp_acquire_user_lock_with_checks_)(lck, gtid);
674 extern int (*__kmp_test_user_lock_with_checks_)(kmp_user_lock_p lck,
677 #if KMP_OS_LINUX && \ 678 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64) 680 #include "kmp_i18n.h" 681 extern int __kmp_env_consistency_check;
682 static inline int __kmp_test_user_lock_with_checks(kmp_user_lock_p lck,
684 if (__kmp_user_lock_kind == lk_tas) {
685 if (__kmp_env_consistency_check) {
686 char const *
const func =
"omp_test_lock";
687 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
688 lck->tas.lk.depth_locked != -1) {
689 KMP_FATAL(LockNestableUsedAsSimple, func);
692 return ((lck->tas.lk.poll == 0) &&
693 __kmp_atomic_compare_store_acq(&lck->tas.lk.poll, 0, gtid + 1));
695 KMP_DEBUG_ASSERT(__kmp_test_user_lock_with_checks_ != NULL);
696 return (*__kmp_test_user_lock_with_checks_)(lck, gtid);
700 static inline int __kmp_test_user_lock_with_checks(kmp_user_lock_p lck,
702 KMP_DEBUG_ASSERT(__kmp_test_user_lock_with_checks_ != NULL);
703 return (*__kmp_test_user_lock_with_checks_)(lck, gtid);
707 extern int (*__kmp_release_user_lock_with_checks_)(kmp_user_lock_p lck,
710 static inline void __kmp_release_user_lock_with_checks(kmp_user_lock_p lck,
712 KMP_DEBUG_ASSERT(__kmp_release_user_lock_with_checks_ != NULL);
713 (*__kmp_release_user_lock_with_checks_)(lck, gtid);
716 extern void (*__kmp_init_user_lock_with_checks_)(kmp_user_lock_p lck);
718 static inline void __kmp_init_user_lock_with_checks(kmp_user_lock_p lck) {
719 KMP_DEBUG_ASSERT(__kmp_init_user_lock_with_checks_ != NULL);
720 (*__kmp_init_user_lock_with_checks_)(lck);
725 extern void (*__kmp_destroy_user_lock_)(kmp_user_lock_p lck);
727 static inline void __kmp_destroy_user_lock(kmp_user_lock_p lck) {
728 KMP_DEBUG_ASSERT(__kmp_destroy_user_lock_ != NULL);
729 (*__kmp_destroy_user_lock_)(lck);
732 extern void (*__kmp_destroy_user_lock_with_checks_)(kmp_user_lock_p lck);
734 static inline void __kmp_destroy_user_lock_with_checks(kmp_user_lock_p lck) {
735 KMP_DEBUG_ASSERT(__kmp_destroy_user_lock_with_checks_ != NULL);
736 (*__kmp_destroy_user_lock_with_checks_)(lck);
739 extern int (*__kmp_acquire_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
742 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64) 744 #define __kmp_acquire_nested_user_lock_with_checks(lck, gtid, depth) \ 745 if (__kmp_user_lock_kind == lk_tas) { \ 746 if (__kmp_env_consistency_check) { \ 747 char const *const func = "omp_set_nest_lock"; \ 748 if ((sizeof(kmp_tas_lock_t) <= OMP_NEST_LOCK_T_SIZE) && \ 749 lck->tas.lk.depth_locked == -1) { \ 750 KMP_FATAL(LockSimpleUsedAsNestable, func); \ 753 if (lck->tas.lk.poll - 1 == gtid) { \ 754 lck->tas.lk.depth_locked += 1; \ 755 *depth = KMP_LOCK_ACQUIRED_NEXT; \ 757 if ((lck->tas.lk.poll != 0) || \ 758 !__kmp_atomic_compare_store_acq(&lck->tas.lk.poll, 0, gtid + 1)) { \ 761 KMP_FSYNC_PREPARE(lck); \ 762 KMP_INIT_YIELD(spins); \ 763 KMP_INIT_BACKOFF(time); \ 765 KMP_YIELD_OVERSUB_ELSE_SPIN(spins, time); \ 767 (lck->tas.lk.poll != 0) || \ 768 !__kmp_atomic_compare_store_acq(&lck->tas.lk.poll, 0, gtid + 1)); \ 770 lck->tas.lk.depth_locked = 1; \ 771 *depth = KMP_LOCK_ACQUIRED_FIRST; \ 773 KMP_FSYNC_ACQUIRED(lck); \ 775 KMP_DEBUG_ASSERT(__kmp_acquire_nested_user_lock_with_checks_ != NULL); \ 776 *depth = (*__kmp_acquire_nested_user_lock_with_checks_)(lck, gtid); \ 781 __kmp_acquire_nested_user_lock_with_checks(kmp_user_lock_p lck, kmp_int32 gtid,
783 KMP_DEBUG_ASSERT(__kmp_acquire_nested_user_lock_with_checks_ != NULL);
784 *depth = (*__kmp_acquire_nested_user_lock_with_checks_)(lck, gtid);
788 extern int (*__kmp_test_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
791 #if KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64) 792 static inline int __kmp_test_nested_user_lock_with_checks(kmp_user_lock_p lck,
794 if (__kmp_user_lock_kind == lk_tas) {
796 if (__kmp_env_consistency_check) {
797 char const *
const func =
"omp_test_nest_lock";
798 if ((
sizeof(kmp_tas_lock_t) <= OMP_NEST_LOCK_T_SIZE) &&
799 lck->tas.lk.depth_locked == -1) {
800 KMP_FATAL(LockSimpleUsedAsNestable, func);
803 KMP_DEBUG_ASSERT(gtid >= 0);
804 if (lck->tas.lk.poll - 1 ==
806 return ++lck->tas.lk.depth_locked;
808 retval = ((lck->tas.lk.poll == 0) &&
809 __kmp_atomic_compare_store_acq(&lck->tas.lk.poll, 0, gtid + 1));
812 lck->tas.lk.depth_locked = 1;
816 KMP_DEBUG_ASSERT(__kmp_test_nested_user_lock_with_checks_ != NULL);
817 return (*__kmp_test_nested_user_lock_with_checks_)(lck, gtid);
821 static inline int __kmp_test_nested_user_lock_with_checks(kmp_user_lock_p lck,
823 KMP_DEBUG_ASSERT(__kmp_test_nested_user_lock_with_checks_ != NULL);
824 return (*__kmp_test_nested_user_lock_with_checks_)(lck, gtid);
828 extern int (*__kmp_release_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
832 __kmp_release_nested_user_lock_with_checks(kmp_user_lock_p lck,
834 KMP_DEBUG_ASSERT(__kmp_release_nested_user_lock_with_checks_ != NULL);
835 return (*__kmp_release_nested_user_lock_with_checks_)(lck, gtid);
838 extern void (*__kmp_init_nested_user_lock_with_checks_)(kmp_user_lock_p lck);
841 __kmp_init_nested_user_lock_with_checks(kmp_user_lock_p lck) {
842 KMP_DEBUG_ASSERT(__kmp_init_nested_user_lock_with_checks_ != NULL);
843 (*__kmp_init_nested_user_lock_with_checks_)(lck);
846 extern void (*__kmp_destroy_nested_user_lock_with_checks_)(kmp_user_lock_p lck);
849 __kmp_destroy_nested_user_lock_with_checks(kmp_user_lock_p lck) {
850 KMP_DEBUG_ASSERT(__kmp_destroy_nested_user_lock_with_checks_ != NULL);
851 (*__kmp_destroy_nested_user_lock_with_checks_)(lck);
867 extern int (*__kmp_is_user_lock_initialized_)(kmp_user_lock_p lck);
871 extern const ident_t *(*__kmp_get_user_lock_location_)(kmp_user_lock_p lck);
873 static inline const ident_t *__kmp_get_user_lock_location(kmp_user_lock_p lck) {
874 if (__kmp_get_user_lock_location_ != NULL) {
875 return (*__kmp_get_user_lock_location_)(lck);
881 extern void (*__kmp_set_user_lock_location_)(kmp_user_lock_p lck,
884 static inline void __kmp_set_user_lock_location(kmp_user_lock_p lck,
886 if (__kmp_set_user_lock_location_ != NULL) {
887 (*__kmp_set_user_lock_location_)(lck, loc);
891 extern kmp_lock_flags_t (*__kmp_get_user_lock_flags_)(kmp_user_lock_p lck);
893 extern void (*__kmp_set_user_lock_flags_)(kmp_user_lock_p lck,
894 kmp_lock_flags_t flags);
896 static inline void __kmp_set_user_lock_flags(kmp_user_lock_p lck,
897 kmp_lock_flags_t flags) {
898 if (__kmp_set_user_lock_flags_ != NULL) {
899 (*__kmp_set_user_lock_flags_)(lck, flags);
904 extern void __kmp_set_user_lock_vptrs(kmp_lock_kind_t user_lock_kind);
907 #define KMP_BIND_USER_LOCK_TEMPLATE(nest, kind, suffix) \ 909 __kmp_acquire##nest##user_lock_with_checks_ = (int (*)( \ 910 kmp_user_lock_p, kmp_int32))__kmp_acquire##nest##kind##_##suffix; \ 911 __kmp_release##nest##user_lock_with_checks_ = (int (*)( \ 912 kmp_user_lock_p, kmp_int32))__kmp_release##nest##kind##_##suffix; \ 913 __kmp_test##nest##user_lock_with_checks_ = (int (*)( \ 914 kmp_user_lock_p, kmp_int32))__kmp_test##nest##kind##_##suffix; \ 915 __kmp_init##nest##user_lock_with_checks_ = \ 916 (void (*)(kmp_user_lock_p))__kmp_init##nest##kind##_##suffix; \ 917 __kmp_destroy##nest##user_lock_with_checks_ = \ 918 (void (*)(kmp_user_lock_p))__kmp_destroy##nest##kind##_##suffix; \ 921 #define KMP_BIND_USER_LOCK(kind) KMP_BIND_USER_LOCK_TEMPLATE(_, kind, lock) 922 #define KMP_BIND_USER_LOCK_WITH_CHECKS(kind) \ 923 KMP_BIND_USER_LOCK_TEMPLATE(_, kind, lock_with_checks) 924 #define KMP_BIND_NESTED_USER_LOCK(kind) \ 925 KMP_BIND_USER_LOCK_TEMPLATE(_nested_, kind, lock) 926 #define KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(kind) \ 927 KMP_BIND_USER_LOCK_TEMPLATE(_nested_, kind, lock_with_checks) 952 struct kmp_lock_table {
953 kmp_lock_index_t used;
954 kmp_lock_index_t allocated;
955 kmp_user_lock_p *table;
958 typedef struct kmp_lock_table kmp_lock_table_t;
960 extern kmp_lock_table_t __kmp_user_lock_table;
961 extern kmp_user_lock_p __kmp_lock_pool;
963 struct kmp_block_of_locks {
964 struct kmp_block_of_locks *next_block;
968 typedef struct kmp_block_of_locks kmp_block_of_locks_t;
970 extern kmp_block_of_locks_t *__kmp_lock_blocks;
971 extern int __kmp_num_locks_in_block;
973 extern kmp_user_lock_p __kmp_user_lock_allocate(
void **user_lock,
975 kmp_lock_flags_t flags);
976 extern void __kmp_user_lock_free(
void **user_lock, kmp_int32 gtid,
977 kmp_user_lock_p lck);
978 extern kmp_user_lock_p __kmp_lookup_user_lock(
void **user_lock,
980 extern void __kmp_cleanup_user_locks();
982 #define KMP_CHECK_USER_LOCK_INIT() \ 984 if (!TCR_4(__kmp_init_user_locks)) { \ 985 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock); \ 986 if (!TCR_4(__kmp_init_user_locks)) { \ 987 TCW_4(__kmp_init_user_locks, TRUE); \ 989 __kmp_release_bootstrap_lock(&__kmp_initz_lock); \ 993 #endif // KMP_USE_DYNAMIC_LOCK 998 #if KMP_USE_DYNAMIC_LOCK 1036 #define KMP_USE_INLINED_TAS \ 1037 (KMP_OS_LINUX && (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM)) && 1 1038 #define KMP_USE_INLINED_FUTEX KMP_USE_FUTEX && 0 1045 #define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a) m(hle, a) m(rtm_spin, a) 1046 #define KMP_FOREACH_I_LOCK(m, a) \ 1047 m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) m(rtm_queuing, a) \ 1048 m(nested_tas, a) m(nested_futex, a) m(nested_ticket, a) \ 1049 m(nested_queuing, a) m(nested_drdpa, a) 1051 #define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(hle, a) m(rtm_spin, a) 1052 #define KMP_FOREACH_I_LOCK(m, a) \ 1053 m(ticket, a) m(queuing, a) m(adaptive, a) m(drdpa, a) m(rtm_queuing, a) \ 1054 m(nested_tas, a) m(nested_ticket, a) m(nested_queuing, a) \ 1056 #endif // KMP_USE_FUTEX 1057 #define KMP_LAST_D_LOCK lockseq_rtm_spin 1060 #define KMP_FOREACH_D_LOCK(m, a) m(tas, a) m(futex, a) 1061 #define KMP_FOREACH_I_LOCK(m, a) \ 1062 m(ticket, a) m(queuing, a) m(drdpa, a) m(nested_tas, a) m(nested_futex, a) \ 1063 m(nested_ticket, a) m(nested_queuing, a) m(nested_drdpa, a) 1064 #define KMP_LAST_D_LOCK lockseq_futex 1066 #define KMP_FOREACH_D_LOCK(m, a) m(tas, a) 1067 #define KMP_FOREACH_I_LOCK(m, a) \ 1068 m(ticket, a) m(queuing, a) m(drdpa, a) m(nested_tas, a) m(nested_ticket, a) \ 1069 m(nested_queuing, a) m(nested_drdpa, a) 1070 #define KMP_LAST_D_LOCK lockseq_tas 1071 #endif // KMP_USE_FUTEX 1072 #endif // KMP_USE_TSX 1075 #define KMP_LOCK_SHIFT \ 1076 8 // number of low bits to be used as tag for direct locks 1077 #define KMP_FIRST_D_LOCK lockseq_tas 1078 #define KMP_FIRST_I_LOCK lockseq_ticket 1079 #define KMP_LAST_I_LOCK lockseq_nested_drdpa 1080 #define KMP_NUM_I_LOCKS \ 1081 (locktag_nested_drdpa + 1) // number of indirect lock types 1084 typedef kmp_uint32 kmp_dyna_lock_t;
1089 lockseq_indirect = 0,
1090 #define expand_seq(l, a) lockseq_##l, 1091 KMP_FOREACH_D_LOCK(expand_seq, 0) KMP_FOREACH_I_LOCK(expand_seq, 0)
1093 } kmp_dyna_lockseq_t;
1097 #define expand_tag(l, a) locktag_##l, 1098 KMP_FOREACH_I_LOCK(expand_tag, 0)
1100 } kmp_indirect_locktag_t;
1103 #define KMP_IS_D_LOCK(seq) \ 1104 ((seq) >= KMP_FIRST_D_LOCK && (seq) <= KMP_LAST_D_LOCK) 1105 #define KMP_IS_I_LOCK(seq) \ 1106 ((seq) >= KMP_FIRST_I_LOCK && (seq) <= KMP_LAST_I_LOCK) 1107 #define KMP_GET_I_TAG(seq) (kmp_indirect_locktag_t)((seq)-KMP_FIRST_I_LOCK) 1108 #define KMP_GET_D_TAG(seq) ((seq) << 1 | 1) 1112 #define expand_tag(l, a) locktag_##l = KMP_GET_D_TAG(lockseq_##l), 1113 KMP_FOREACH_D_LOCK(expand_tag, 0)
1115 } kmp_direct_locktag_t;
1119 kmp_user_lock_p lock;
1120 kmp_indirect_locktag_t type;
1121 } kmp_indirect_lock_t;
1125 extern void (*__kmp_direct_init[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t);
1126 extern void (**__kmp_direct_destroy)(kmp_dyna_lock_t *);
1127 extern int (**__kmp_direct_set)(kmp_dyna_lock_t *, kmp_int32);
1128 extern int (**__kmp_direct_unset)(kmp_dyna_lock_t *, kmp_int32);
1129 extern int (**__kmp_direct_test)(kmp_dyna_lock_t *, kmp_int32);
1133 extern void (*__kmp_indirect_init[])(kmp_user_lock_p);
1134 extern void (**__kmp_indirect_destroy)(kmp_user_lock_p);
1135 extern int (**__kmp_indirect_set)(kmp_user_lock_p, kmp_int32);
1136 extern int (**__kmp_indirect_unset)(kmp_user_lock_p, kmp_int32);
1137 extern int (**__kmp_indirect_test)(kmp_user_lock_p, kmp_int32);
1140 #define KMP_EXTRACT_D_TAG(l) \ 1141 (*((kmp_dyna_lock_t *)(l)) & ((1 << KMP_LOCK_SHIFT) - 1) & \ 1142 -(*((kmp_dyna_lock_t *)(l)) & 1)) 1145 #define KMP_EXTRACT_I_INDEX(l) (*(kmp_lock_index_t *)(l) >> 1) 1149 #define KMP_D_LOCK_FUNC(l, op) __kmp_direct_##op[KMP_EXTRACT_D_TAG(l)] 1153 #define KMP_I_LOCK_FUNC(l, op) \ 1154 __kmp_indirect_##op[((kmp_indirect_lock_t *)(l))->type] 1157 #define KMP_INIT_D_LOCK(l, seq) \ 1158 __kmp_direct_init[KMP_GET_D_TAG(seq)]((kmp_dyna_lock_t *)l, seq) 1161 #define KMP_INIT_I_LOCK(l, seq) \ 1162 __kmp_direct_init[0]((kmp_dyna_lock_t *)(l), seq) 1165 #define KMP_LOCK_FREE(type) (locktag_##type) 1168 #define KMP_LOCK_BUSY(v, type) ((v) << KMP_LOCK_SHIFT | locktag_##type) 1171 #define KMP_LOCK_STRIP(v) ((v) >> KMP_LOCK_SHIFT) 1175 extern void __kmp_init_dynamic_user_locks();
1178 extern kmp_indirect_lock_t *
1179 __kmp_allocate_indirect_lock(
void **, kmp_int32, kmp_indirect_locktag_t);
1182 extern void __kmp_cleanup_indirect_user_locks();
1185 extern kmp_dyna_lockseq_t __kmp_user_lock_seq;
1188 extern void (*__kmp_indirect_set_location[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
1190 #define KMP_SET_I_LOCK_LOCATION(lck, loc) \ 1192 if (__kmp_indirect_set_location[(lck)->type] != NULL) \ 1193 __kmp_indirect_set_location[(lck)->type]((lck)->lock, loc); \ 1197 extern void (*__kmp_indirect_set_flags[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
1199 #define KMP_SET_I_LOCK_FLAGS(lck, flag) \ 1201 if (__kmp_indirect_set_flags[(lck)->type] != NULL) \ 1202 __kmp_indirect_set_flags[(lck)->type]((lck)->lock, flag); \ 1206 extern const ident_t *(*__kmp_indirect_get_location[KMP_NUM_I_LOCKS])(
1208 #define KMP_GET_I_LOCK_LOCATION(lck) \ 1209 (__kmp_indirect_get_location[(lck)->type] != NULL \ 1210 ? __kmp_indirect_get_location[(lck)->type]((lck)->lock) \ 1214 extern kmp_lock_flags_t (*__kmp_indirect_get_flags[KMP_NUM_I_LOCKS])(
1216 #define KMP_GET_I_LOCK_FLAGS(lck) \ 1217 (__kmp_indirect_get_flags[(lck)->type] != NULL \ 1218 ? __kmp_indirect_get_flags[(lck)->type]((lck)->lock) \ 1222 #define KMP_I_LOCK_CHUNK 1024 1224 KMP_BUILD_ASSERT(KMP_I_LOCK_CHUNK % 2 == 0);
1226 #define KMP_I_LOCK_TABLE_INIT_NROW_PTRS 8 1229 typedef struct kmp_indirect_lock_table {
1230 kmp_indirect_lock_t **table;
1231 kmp_uint32 nrow_ptrs;
1232 kmp_lock_index_t next;
1233 struct kmp_indirect_lock_table *next_table;
1234 } kmp_indirect_lock_table_t;
1236 extern kmp_indirect_lock_table_t __kmp_i_lock_table;
1240 static inline kmp_indirect_lock_t *__kmp_get_i_lock(kmp_lock_index_t idx) {
1241 kmp_indirect_lock_table_t *lock_table = &__kmp_i_lock_table;
1242 while (lock_table) {
1243 kmp_lock_index_t max_locks = lock_table->nrow_ptrs * KMP_I_LOCK_CHUNK;
1244 if (idx < max_locks) {
1245 kmp_lock_index_t row = idx / KMP_I_LOCK_CHUNK;
1246 kmp_lock_index_t col = idx % KMP_I_LOCK_CHUNK;
1247 if (!lock_table->table[row] || idx >= lock_table->next)
1249 return &lock_table->table[row][col];
1252 lock_table = lock_table->next_table;
1260 extern int __kmp_num_locks_in_block;
1263 #define KMP_LOOKUP_I_LOCK(l) \ 1264 ((OMP_LOCK_T_SIZE < sizeof(void *)) \ 1265 ? __kmp_get_i_lock(KMP_EXTRACT_I_INDEX(l)) \ 1266 : *((kmp_indirect_lock_t **)(l))) 1269 extern kmp_int32 __kmp_get_user_lock_owner(kmp_user_lock_p, kmp_uint32);
1271 #else // KMP_USE_DYNAMIC_LOCK 1273 #define KMP_LOCK_BUSY(v, type) (v) 1274 #define KMP_LOCK_FREE(type) 0 1275 #define KMP_LOCK_STRIP(v) (v) 1277 #endif // KMP_USE_DYNAMIC_LOCK 1282 kmp_uint32 max_backoff;
1283 kmp_uint32 min_tick;
1287 extern kmp_backoff_t __kmp_spin_backoff_params;
1290 extern void __kmp_spin_backoff(kmp_backoff_t *);
1294 #endif // __cplusplus