21 #include "kmp_wait_release.h"
22 #include "kmp_wrapper_getpid.h"
24 #include "tsan_annotations.h"
27 #include <sys/syscall.h>
45 void __kmp_validate_locks(
void) {
50 x = ~((kmp_uint32)0) - 2;
53 for (i = 0; i < 8; ++i, ++x, ++y) {
54 kmp_uint32 z = (x - y);
58 KMP_ASSERT(offsetof(kmp_base_queuing_lock, tail_id) % 8 == 0);
72 static kmp_int32 __kmp_get_tas_lock_owner(kmp_tas_lock_t *lck) {
73 return KMP_LOCK_STRIP(KMP_ATOMIC_LD_RLX(&lck->lk.poll)) - 1;
76 static inline bool __kmp_is_tas_lock_nestable(kmp_tas_lock_t *lck) {
77 return lck->lk.depth_locked != -1;
80 __forceinline
static int
81 __kmp_acquire_tas_lock_timed_template(kmp_tas_lock_t *lck, kmp_int32 gtid) {
84 #ifdef USE_LOCK_PROFILE
85 kmp_uint32 curr = KMP_LOCK_STRIP(lck->lk.poll);
86 if ((curr != 0) && (curr != gtid + 1))
87 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
91 kmp_int32 tas_free = KMP_LOCK_FREE(tas);
92 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas);
94 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == tas_free &&
95 __kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy)) {
96 KMP_FSYNC_ACQUIRED(lck);
97 return KMP_LOCK_ACQUIRED_FIRST;
101 KMP_FSYNC_PREPARE(lck);
102 KMP_INIT_YIELD(spins);
103 if (TCR_4(__kmp_nth) > (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {
106 KMP_YIELD_SPIN(spins);
109 kmp_backoff_t backoff = __kmp_spin_backoff_params;
110 while (KMP_ATOMIC_LD_RLX(&lck->lk.poll) != tas_free ||
111 !__kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy)) {
112 __kmp_spin_backoff(&backoff);
113 if (TCR_4(__kmp_nth) >
114 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {
117 KMP_YIELD_SPIN(spins);
120 KMP_FSYNC_ACQUIRED(lck);
121 return KMP_LOCK_ACQUIRED_FIRST;
124 int __kmp_acquire_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
125 int retval = __kmp_acquire_tas_lock_timed_template(lck, gtid);
126 ANNOTATE_TAS_ACQUIRED(lck);
130 static int __kmp_acquire_tas_lock_with_checks(kmp_tas_lock_t *lck,
132 char const *
const func =
"omp_set_lock";
133 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
134 __kmp_is_tas_lock_nestable(lck)) {
135 KMP_FATAL(LockNestableUsedAsSimple, func);
137 if ((gtid >= 0) && (__kmp_get_tas_lock_owner(lck) == gtid)) {
138 KMP_FATAL(LockIsAlreadyOwned, func);
140 return __kmp_acquire_tas_lock(lck, gtid);
143 int __kmp_test_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
144 kmp_int32 tas_free = KMP_LOCK_FREE(tas);
145 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas);
146 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == tas_free &&
147 __kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy)) {
148 KMP_FSYNC_ACQUIRED(lck);
154 static int __kmp_test_tas_lock_with_checks(kmp_tas_lock_t *lck,
156 char const *
const func =
"omp_test_lock";
157 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
158 __kmp_is_tas_lock_nestable(lck)) {
159 KMP_FATAL(LockNestableUsedAsSimple, func);
161 return __kmp_test_tas_lock(lck, gtid);
164 int __kmp_release_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
167 KMP_FSYNC_RELEASING(lck);
168 ANNOTATE_TAS_RELEASED(lck);
169 KMP_ATOMIC_ST_REL(&lck->lk.poll, KMP_LOCK_FREE(tas));
172 KMP_YIELD(TCR_4(__kmp_nth) >
173 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
174 return KMP_LOCK_RELEASED;
177 static int __kmp_release_tas_lock_with_checks(kmp_tas_lock_t *lck,
179 char const *
const func =
"omp_unset_lock";
181 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
182 __kmp_is_tas_lock_nestable(lck)) {
183 KMP_FATAL(LockNestableUsedAsSimple, func);
185 if (__kmp_get_tas_lock_owner(lck) == -1) {
186 KMP_FATAL(LockUnsettingFree, func);
188 if ((gtid >= 0) && (__kmp_get_tas_lock_owner(lck) >= 0) &&
189 (__kmp_get_tas_lock_owner(lck) != gtid)) {
190 KMP_FATAL(LockUnsettingSetByAnother, func);
192 return __kmp_release_tas_lock(lck, gtid);
195 void __kmp_init_tas_lock(kmp_tas_lock_t *lck) {
196 lck->lk.poll = KMP_LOCK_FREE(tas);
199 void __kmp_destroy_tas_lock(kmp_tas_lock_t *lck) { lck->lk.poll = 0; }
201 static void __kmp_destroy_tas_lock_with_checks(kmp_tas_lock_t *lck) {
202 char const *
const func =
"omp_destroy_lock";
203 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
204 __kmp_is_tas_lock_nestable(lck)) {
205 KMP_FATAL(LockNestableUsedAsSimple, func);
207 if (__kmp_get_tas_lock_owner(lck) != -1) {
208 KMP_FATAL(LockStillOwned, func);
210 __kmp_destroy_tas_lock(lck);
215 int __kmp_acquire_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
216 KMP_DEBUG_ASSERT(gtid >= 0);
218 if (__kmp_get_tas_lock_owner(lck) == gtid) {
219 lck->lk.depth_locked += 1;
220 return KMP_LOCK_ACQUIRED_NEXT;
222 __kmp_acquire_tas_lock_timed_template(lck, gtid);
223 ANNOTATE_TAS_ACQUIRED(lck);
224 lck->lk.depth_locked = 1;
225 return KMP_LOCK_ACQUIRED_FIRST;
229 static int __kmp_acquire_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
231 char const *
const func =
"omp_set_nest_lock";
232 if (!__kmp_is_tas_lock_nestable(lck)) {
233 KMP_FATAL(LockSimpleUsedAsNestable, func);
235 return __kmp_acquire_nested_tas_lock(lck, gtid);
238 int __kmp_test_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
241 KMP_DEBUG_ASSERT(gtid >= 0);
243 if (__kmp_get_tas_lock_owner(lck) == gtid) {
244 retval = ++lck->lk.depth_locked;
245 }
else if (!__kmp_test_tas_lock(lck, gtid)) {
249 retval = lck->lk.depth_locked = 1;
254 static int __kmp_test_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
256 char const *
const func =
"omp_test_nest_lock";
257 if (!__kmp_is_tas_lock_nestable(lck)) {
258 KMP_FATAL(LockSimpleUsedAsNestable, func);
260 return __kmp_test_nested_tas_lock(lck, gtid);
263 int __kmp_release_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
264 KMP_DEBUG_ASSERT(gtid >= 0);
267 if (--(lck->lk.depth_locked) == 0) {
268 __kmp_release_tas_lock(lck, gtid);
269 return KMP_LOCK_RELEASED;
271 return KMP_LOCK_STILL_HELD;
274 static int __kmp_release_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
276 char const *
const func =
"omp_unset_nest_lock";
278 if (!__kmp_is_tas_lock_nestable(lck)) {
279 KMP_FATAL(LockSimpleUsedAsNestable, func);
281 if (__kmp_get_tas_lock_owner(lck) == -1) {
282 KMP_FATAL(LockUnsettingFree, func);
284 if (__kmp_get_tas_lock_owner(lck) != gtid) {
285 KMP_FATAL(LockUnsettingSetByAnother, func);
287 return __kmp_release_nested_tas_lock(lck, gtid);
290 void __kmp_init_nested_tas_lock(kmp_tas_lock_t *lck) {
291 __kmp_init_tas_lock(lck);
292 lck->lk.depth_locked = 0;
295 void __kmp_destroy_nested_tas_lock(kmp_tas_lock_t *lck) {
296 __kmp_destroy_tas_lock(lck);
297 lck->lk.depth_locked = 0;
300 static void __kmp_destroy_nested_tas_lock_with_checks(kmp_tas_lock_t *lck) {
301 char const *
const func =
"omp_destroy_nest_lock";
302 if (!__kmp_is_tas_lock_nestable(lck)) {
303 KMP_FATAL(LockSimpleUsedAsNestable, func);
305 if (__kmp_get_tas_lock_owner(lck) != -1) {
306 KMP_FATAL(LockStillOwned, func);
308 __kmp_destroy_nested_tas_lock(lck);
321 static kmp_int32 __kmp_get_futex_lock_owner(kmp_futex_lock_t *lck) {
322 return KMP_LOCK_STRIP((TCR_4(lck->lk.poll) >> 1)) - 1;
325 static inline bool __kmp_is_futex_lock_nestable(kmp_futex_lock_t *lck) {
326 return lck->lk.depth_locked != -1;
329 __forceinline
static int
330 __kmp_acquire_futex_lock_timed_template(kmp_futex_lock_t *lck, kmp_int32 gtid) {
331 kmp_int32 gtid_code = (gtid + 1) << 1;
335 #ifdef USE_LOCK_PROFILE
336 kmp_uint32 curr = KMP_LOCK_STRIP(TCR_4(lck->lk.poll));
337 if ((curr != 0) && (curr != gtid_code))
338 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
342 KMP_FSYNC_PREPARE(lck);
343 KA_TRACE(1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d entering\n",
344 lck, lck->lk.poll, gtid));
348 while ((poll_val = KMP_COMPARE_AND_STORE_RET32(
349 &(lck->lk.poll), KMP_LOCK_FREE(futex),
350 KMP_LOCK_BUSY(gtid_code, futex))) != KMP_LOCK_FREE(futex)) {
352 kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1;
355 (
"__kmp_acquire_futex_lock: lck:%p, T#%d poll_val = 0x%x cond = 0x%x\n",
356 lck, gtid, poll_val, cond));
367 if (!KMP_COMPARE_AND_STORE_REL32(&(lck->lk.poll), poll_val,
368 poll_val | KMP_LOCK_BUSY(1, futex))) {
371 (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d can't set bit 0\n",
372 lck, lck->lk.poll, gtid));
375 poll_val |= KMP_LOCK_BUSY(1, futex);
378 (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d bit 0 set\n", lck,
379 lck->lk.poll, gtid));
384 (
"__kmp_acquire_futex_lock: lck:%p, T#%d before futex_wait(0x%x)\n",
385 lck, gtid, poll_val));
388 if ((rc = syscall(__NR_futex, &(lck->lk.poll), FUTEX_WAIT, poll_val, NULL,
390 KA_TRACE(1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d futex_wait(0x%x) "
391 "failed (rc=%d errno=%d)\n",
392 lck, gtid, poll_val, rc, errno));
397 (
"__kmp_acquire_futex_lock: lck:%p, T#%d after futex_wait(0x%x)\n",
398 lck, gtid, poll_val));
405 KMP_FSYNC_ACQUIRED(lck);
406 KA_TRACE(1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d exiting\n", lck,
407 lck->lk.poll, gtid));
408 return KMP_LOCK_ACQUIRED_FIRST;
411 int __kmp_acquire_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
412 int retval = __kmp_acquire_futex_lock_timed_template(lck, gtid);
413 ANNOTATE_FUTEX_ACQUIRED(lck);
417 static int __kmp_acquire_futex_lock_with_checks(kmp_futex_lock_t *lck,
419 char const *
const func =
"omp_set_lock";
420 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
421 __kmp_is_futex_lock_nestable(lck)) {
422 KMP_FATAL(LockNestableUsedAsSimple, func);
424 if ((gtid >= 0) && (__kmp_get_futex_lock_owner(lck) == gtid)) {
425 KMP_FATAL(LockIsAlreadyOwned, func);
427 return __kmp_acquire_futex_lock(lck, gtid);
430 int __kmp_test_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
431 if (KMP_COMPARE_AND_STORE_ACQ32(&(lck->lk.poll), KMP_LOCK_FREE(futex),
432 KMP_LOCK_BUSY((gtid + 1) << 1, futex))) {
433 KMP_FSYNC_ACQUIRED(lck);
439 static int __kmp_test_futex_lock_with_checks(kmp_futex_lock_t *lck,
441 char const *
const func =
"omp_test_lock";
442 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
443 __kmp_is_futex_lock_nestable(lck)) {
444 KMP_FATAL(LockNestableUsedAsSimple, func);
446 return __kmp_test_futex_lock(lck, gtid);
449 int __kmp_release_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
452 KA_TRACE(1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d entering\n",
453 lck, lck->lk.poll, gtid));
455 KMP_FSYNC_RELEASING(lck);
456 ANNOTATE_FUTEX_RELEASED(lck);
458 kmp_int32 poll_val = KMP_XCHG_FIXED32(&(lck->lk.poll), KMP_LOCK_FREE(futex));
461 (
"__kmp_release_futex_lock: lck:%p, T#%d released poll_val = 0x%x\n",
462 lck, gtid, poll_val));
464 if (KMP_LOCK_STRIP(poll_val) & 1) {
466 (
"__kmp_release_futex_lock: lck:%p, T#%d futex_wake 1 thread\n",
468 syscall(__NR_futex, &(lck->lk.poll), FUTEX_WAKE, KMP_LOCK_BUSY(1, futex),
474 KA_TRACE(1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d exiting\n", lck,
475 lck->lk.poll, gtid));
477 KMP_YIELD(TCR_4(__kmp_nth) >
478 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
479 return KMP_LOCK_RELEASED;
482 static int __kmp_release_futex_lock_with_checks(kmp_futex_lock_t *lck,
484 char const *
const func =
"omp_unset_lock";
486 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
487 __kmp_is_futex_lock_nestable(lck)) {
488 KMP_FATAL(LockNestableUsedAsSimple, func);
490 if (__kmp_get_futex_lock_owner(lck) == -1) {
491 KMP_FATAL(LockUnsettingFree, func);
493 if ((gtid >= 0) && (__kmp_get_futex_lock_owner(lck) >= 0) &&
494 (__kmp_get_futex_lock_owner(lck) != gtid)) {
495 KMP_FATAL(LockUnsettingSetByAnother, func);
497 return __kmp_release_futex_lock(lck, gtid);
500 void __kmp_init_futex_lock(kmp_futex_lock_t *lck) {
501 TCW_4(lck->lk.poll, KMP_LOCK_FREE(futex));
504 void __kmp_destroy_futex_lock(kmp_futex_lock_t *lck) { lck->lk.poll = 0; }
506 static void __kmp_destroy_futex_lock_with_checks(kmp_futex_lock_t *lck) {
507 char const *
const func =
"omp_destroy_lock";
508 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
509 __kmp_is_futex_lock_nestable(lck)) {
510 KMP_FATAL(LockNestableUsedAsSimple, func);
512 if (__kmp_get_futex_lock_owner(lck) != -1) {
513 KMP_FATAL(LockStillOwned, func);
515 __kmp_destroy_futex_lock(lck);
520 int __kmp_acquire_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
521 KMP_DEBUG_ASSERT(gtid >= 0);
523 if (__kmp_get_futex_lock_owner(lck) == gtid) {
524 lck->lk.depth_locked += 1;
525 return KMP_LOCK_ACQUIRED_NEXT;
527 __kmp_acquire_futex_lock_timed_template(lck, gtid);
528 ANNOTATE_FUTEX_ACQUIRED(lck);
529 lck->lk.depth_locked = 1;
530 return KMP_LOCK_ACQUIRED_FIRST;
534 static int __kmp_acquire_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
536 char const *
const func =
"omp_set_nest_lock";
537 if (!__kmp_is_futex_lock_nestable(lck)) {
538 KMP_FATAL(LockSimpleUsedAsNestable, func);
540 return __kmp_acquire_nested_futex_lock(lck, gtid);
543 int __kmp_test_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
546 KMP_DEBUG_ASSERT(gtid >= 0);
548 if (__kmp_get_futex_lock_owner(lck) == gtid) {
549 retval = ++lck->lk.depth_locked;
550 }
else if (!__kmp_test_futex_lock(lck, gtid)) {
554 retval = lck->lk.depth_locked = 1;
559 static int __kmp_test_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
561 char const *
const func =
"omp_test_nest_lock";
562 if (!__kmp_is_futex_lock_nestable(lck)) {
563 KMP_FATAL(LockSimpleUsedAsNestable, func);
565 return __kmp_test_nested_futex_lock(lck, gtid);
568 int __kmp_release_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
569 KMP_DEBUG_ASSERT(gtid >= 0);
572 if (--(lck->lk.depth_locked) == 0) {
573 __kmp_release_futex_lock(lck, gtid);
574 return KMP_LOCK_RELEASED;
576 return KMP_LOCK_STILL_HELD;
579 static int __kmp_release_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
581 char const *
const func =
"omp_unset_nest_lock";
583 if (!__kmp_is_futex_lock_nestable(lck)) {
584 KMP_FATAL(LockSimpleUsedAsNestable, func);
586 if (__kmp_get_futex_lock_owner(lck) == -1) {
587 KMP_FATAL(LockUnsettingFree, func);
589 if (__kmp_get_futex_lock_owner(lck) != gtid) {
590 KMP_FATAL(LockUnsettingSetByAnother, func);
592 return __kmp_release_nested_futex_lock(lck, gtid);
595 void __kmp_init_nested_futex_lock(kmp_futex_lock_t *lck) {
596 __kmp_init_futex_lock(lck);
597 lck->lk.depth_locked = 0;
600 void __kmp_destroy_nested_futex_lock(kmp_futex_lock_t *lck) {
601 __kmp_destroy_futex_lock(lck);
602 lck->lk.depth_locked = 0;
605 static void __kmp_destroy_nested_futex_lock_with_checks(kmp_futex_lock_t *lck) {
606 char const *
const func =
"omp_destroy_nest_lock";
607 if (!__kmp_is_futex_lock_nestable(lck)) {
608 KMP_FATAL(LockSimpleUsedAsNestable, func);
610 if (__kmp_get_futex_lock_owner(lck) != -1) {
611 KMP_FATAL(LockStillOwned, func);
613 __kmp_destroy_nested_futex_lock(lck);
616 #endif // KMP_USE_FUTEX
621 static kmp_int32 __kmp_get_ticket_lock_owner(kmp_ticket_lock_t *lck) {
622 return std::atomic_load_explicit(&lck->lk.owner_id,
623 std::memory_order_relaxed) -
627 static inline bool __kmp_is_ticket_lock_nestable(kmp_ticket_lock_t *lck) {
628 return std::atomic_load_explicit(&lck->lk.depth_locked,
629 std::memory_order_relaxed) != -1;
632 static kmp_uint32 __kmp_bakery_check(
void *now_serving, kmp_uint32 my_ticket) {
633 return std::atomic_load_explicit((std::atomic<unsigned> *)now_serving,
634 std::memory_order_acquire) == my_ticket;
637 __forceinline
static int
638 __kmp_acquire_ticket_lock_timed_template(kmp_ticket_lock_t *lck,
640 kmp_uint32 my_ticket = std::atomic_fetch_add_explicit(
641 &lck->lk.next_ticket, 1U, std::memory_order_relaxed);
643 #ifdef USE_LOCK_PROFILE
644 if (std::atomic_load_explicit(&lck->lk.now_serving,
645 std::memory_order_relaxed) != my_ticket)
646 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
650 if (std::atomic_load_explicit(&lck->lk.now_serving,
651 std::memory_order_acquire) == my_ticket) {
652 return KMP_LOCK_ACQUIRED_FIRST;
654 KMP_WAIT_YIELD_PTR(&lck->lk.now_serving, my_ticket, __kmp_bakery_check, lck);
655 return KMP_LOCK_ACQUIRED_FIRST;
658 int __kmp_acquire_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
659 int retval = __kmp_acquire_ticket_lock_timed_template(lck, gtid);
660 ANNOTATE_TICKET_ACQUIRED(lck);
664 static int __kmp_acquire_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
666 char const *
const func =
"omp_set_lock";
668 if (!std::atomic_load_explicit(&lck->lk.initialized,
669 std::memory_order_relaxed)) {
670 KMP_FATAL(LockIsUninitialized, func);
672 if (lck->lk.self != lck) {
673 KMP_FATAL(LockIsUninitialized, func);
675 if (__kmp_is_ticket_lock_nestable(lck)) {
676 KMP_FATAL(LockNestableUsedAsSimple, func);
678 if ((gtid >= 0) && (__kmp_get_ticket_lock_owner(lck) == gtid)) {
679 KMP_FATAL(LockIsAlreadyOwned, func);
682 __kmp_acquire_ticket_lock(lck, gtid);
684 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
685 std::memory_order_relaxed);
686 return KMP_LOCK_ACQUIRED_FIRST;
689 int __kmp_test_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
690 kmp_uint32 my_ticket = std::atomic_load_explicit(&lck->lk.next_ticket,
691 std::memory_order_relaxed);
693 if (std::atomic_load_explicit(&lck->lk.now_serving,
694 std::memory_order_relaxed) == my_ticket) {
695 kmp_uint32 next_ticket = my_ticket + 1;
696 if (std::atomic_compare_exchange_strong_explicit(
697 &lck->lk.next_ticket, &my_ticket, next_ticket,
698 std::memory_order_acquire, std::memory_order_acquire)) {
705 static int __kmp_test_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
707 char const *
const func =
"omp_test_lock";
709 if (!std::atomic_load_explicit(&lck->lk.initialized,
710 std::memory_order_relaxed)) {
711 KMP_FATAL(LockIsUninitialized, func);
713 if (lck->lk.self != lck) {
714 KMP_FATAL(LockIsUninitialized, func);
716 if (__kmp_is_ticket_lock_nestable(lck)) {
717 KMP_FATAL(LockNestableUsedAsSimple, func);
720 int retval = __kmp_test_ticket_lock(lck, gtid);
723 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
724 std::memory_order_relaxed);
729 int __kmp_release_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
730 kmp_uint32 distance = std::atomic_load_explicit(&lck->lk.next_ticket,
731 std::memory_order_relaxed) -
732 std::atomic_load_explicit(&lck->lk.now_serving,
733 std::memory_order_relaxed);
735 ANNOTATE_TICKET_RELEASED(lck);
736 std::atomic_fetch_add_explicit(&lck->lk.now_serving, 1U,
737 std::memory_order_release);
740 (kmp_uint32)(__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
741 return KMP_LOCK_RELEASED;
744 static int __kmp_release_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
746 char const *
const func =
"omp_unset_lock";
748 if (!std::atomic_load_explicit(&lck->lk.initialized,
749 std::memory_order_relaxed)) {
750 KMP_FATAL(LockIsUninitialized, func);
752 if (lck->lk.self != lck) {
753 KMP_FATAL(LockIsUninitialized, func);
755 if (__kmp_is_ticket_lock_nestable(lck)) {
756 KMP_FATAL(LockNestableUsedAsSimple, func);
758 if (__kmp_get_ticket_lock_owner(lck) == -1) {
759 KMP_FATAL(LockUnsettingFree, func);
761 if ((gtid >= 0) && (__kmp_get_ticket_lock_owner(lck) >= 0) &&
762 (__kmp_get_ticket_lock_owner(lck) != gtid)) {
763 KMP_FATAL(LockUnsettingSetByAnother, func);
765 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
766 return __kmp_release_ticket_lock(lck, gtid);
769 void __kmp_init_ticket_lock(kmp_ticket_lock_t *lck) {
770 lck->lk.location = NULL;
772 std::atomic_store_explicit(&lck->lk.next_ticket, 0U,
773 std::memory_order_relaxed);
774 std::atomic_store_explicit(&lck->lk.now_serving, 0U,
775 std::memory_order_relaxed);
776 std::atomic_store_explicit(
777 &lck->lk.owner_id, 0,
778 std::memory_order_relaxed);
779 std::atomic_store_explicit(
780 &lck->lk.depth_locked, -1,
781 std::memory_order_relaxed);
782 std::atomic_store_explicit(&lck->lk.initialized,
true,
783 std::memory_order_release);
786 void __kmp_destroy_ticket_lock(kmp_ticket_lock_t *lck) {
787 std::atomic_store_explicit(&lck->lk.initialized,
false,
788 std::memory_order_release);
790 lck->lk.location = NULL;
791 std::atomic_store_explicit(&lck->lk.next_ticket, 0U,
792 std::memory_order_relaxed);
793 std::atomic_store_explicit(&lck->lk.now_serving, 0U,
794 std::memory_order_relaxed);
795 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
796 std::atomic_store_explicit(&lck->lk.depth_locked, -1,
797 std::memory_order_relaxed);
800 static void __kmp_destroy_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
801 char const *
const func =
"omp_destroy_lock";
803 if (!std::atomic_load_explicit(&lck->lk.initialized,
804 std::memory_order_relaxed)) {
805 KMP_FATAL(LockIsUninitialized, func);
807 if (lck->lk.self != lck) {
808 KMP_FATAL(LockIsUninitialized, func);
810 if (__kmp_is_ticket_lock_nestable(lck)) {
811 KMP_FATAL(LockNestableUsedAsSimple, func);
813 if (__kmp_get_ticket_lock_owner(lck) != -1) {
814 KMP_FATAL(LockStillOwned, func);
816 __kmp_destroy_ticket_lock(lck);
821 int __kmp_acquire_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
822 KMP_DEBUG_ASSERT(gtid >= 0);
824 if (__kmp_get_ticket_lock_owner(lck) == gtid) {
825 std::atomic_fetch_add_explicit(&lck->lk.depth_locked, 1,
826 std::memory_order_relaxed);
827 return KMP_LOCK_ACQUIRED_NEXT;
829 __kmp_acquire_ticket_lock_timed_template(lck, gtid);
830 ANNOTATE_TICKET_ACQUIRED(lck);
831 std::atomic_store_explicit(&lck->lk.depth_locked, 1,
832 std::memory_order_relaxed);
833 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
834 std::memory_order_relaxed);
835 return KMP_LOCK_ACQUIRED_FIRST;
839 static int __kmp_acquire_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
841 char const *
const func =
"omp_set_nest_lock";
843 if (!std::atomic_load_explicit(&lck->lk.initialized,
844 std::memory_order_relaxed)) {
845 KMP_FATAL(LockIsUninitialized, func);
847 if (lck->lk.self != lck) {
848 KMP_FATAL(LockIsUninitialized, func);
850 if (!__kmp_is_ticket_lock_nestable(lck)) {
851 KMP_FATAL(LockSimpleUsedAsNestable, func);
853 return __kmp_acquire_nested_ticket_lock(lck, gtid);
856 int __kmp_test_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
859 KMP_DEBUG_ASSERT(gtid >= 0);
861 if (__kmp_get_ticket_lock_owner(lck) == gtid) {
862 retval = std::atomic_fetch_add_explicit(&lck->lk.depth_locked, 1,
863 std::memory_order_relaxed) +
865 }
else if (!__kmp_test_ticket_lock(lck, gtid)) {
868 std::atomic_store_explicit(&lck->lk.depth_locked, 1,
869 std::memory_order_relaxed);
870 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
871 std::memory_order_relaxed);
877 static int __kmp_test_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
879 char const *
const func =
"omp_test_nest_lock";
881 if (!std::atomic_load_explicit(&lck->lk.initialized,
882 std::memory_order_relaxed)) {
883 KMP_FATAL(LockIsUninitialized, func);
885 if (lck->lk.self != lck) {
886 KMP_FATAL(LockIsUninitialized, func);
888 if (!__kmp_is_ticket_lock_nestable(lck)) {
889 KMP_FATAL(LockSimpleUsedAsNestable, func);
891 return __kmp_test_nested_ticket_lock(lck, gtid);
894 int __kmp_release_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
895 KMP_DEBUG_ASSERT(gtid >= 0);
897 if ((std::atomic_fetch_add_explicit(&lck->lk.depth_locked, -1,
898 std::memory_order_relaxed) -
900 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
901 __kmp_release_ticket_lock(lck, gtid);
902 return KMP_LOCK_RELEASED;
904 return KMP_LOCK_STILL_HELD;
907 static int __kmp_release_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
909 char const *
const func =
"omp_unset_nest_lock";
911 if (!std::atomic_load_explicit(&lck->lk.initialized,
912 std::memory_order_relaxed)) {
913 KMP_FATAL(LockIsUninitialized, func);
915 if (lck->lk.self != lck) {
916 KMP_FATAL(LockIsUninitialized, func);
918 if (!__kmp_is_ticket_lock_nestable(lck)) {
919 KMP_FATAL(LockSimpleUsedAsNestable, func);
921 if (__kmp_get_ticket_lock_owner(lck) == -1) {
922 KMP_FATAL(LockUnsettingFree, func);
924 if (__kmp_get_ticket_lock_owner(lck) != gtid) {
925 KMP_FATAL(LockUnsettingSetByAnother, func);
927 return __kmp_release_nested_ticket_lock(lck, gtid);
930 void __kmp_init_nested_ticket_lock(kmp_ticket_lock_t *lck) {
931 __kmp_init_ticket_lock(lck);
932 std::atomic_store_explicit(&lck->lk.depth_locked, 0,
933 std::memory_order_relaxed);
937 void __kmp_destroy_nested_ticket_lock(kmp_ticket_lock_t *lck) {
938 __kmp_destroy_ticket_lock(lck);
939 std::atomic_store_explicit(&lck->lk.depth_locked, 0,
940 std::memory_order_relaxed);
944 __kmp_destroy_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
945 char const *
const func =
"omp_destroy_nest_lock";
947 if (!std::atomic_load_explicit(&lck->lk.initialized,
948 std::memory_order_relaxed)) {
949 KMP_FATAL(LockIsUninitialized, func);
951 if (lck->lk.self != lck) {
952 KMP_FATAL(LockIsUninitialized, func);
954 if (!__kmp_is_ticket_lock_nestable(lck)) {
955 KMP_FATAL(LockSimpleUsedAsNestable, func);
957 if (__kmp_get_ticket_lock_owner(lck) != -1) {
958 KMP_FATAL(LockStillOwned, func);
960 __kmp_destroy_nested_ticket_lock(lck);
965 static const ident_t *__kmp_get_ticket_lock_location(kmp_ticket_lock_t *lck) {
966 return lck->lk.location;
969 static void __kmp_set_ticket_lock_location(kmp_ticket_lock_t *lck,
971 lck->lk.location = loc;
974 static kmp_lock_flags_t __kmp_get_ticket_lock_flags(kmp_ticket_lock_t *lck) {
975 return lck->lk.flags;
978 static void __kmp_set_ticket_lock_flags(kmp_ticket_lock_t *lck,
979 kmp_lock_flags_t flags) {
980 lck->lk.flags = flags;
1038 #ifdef DEBUG_QUEUING_LOCKS
1041 #define TRACE_BUF_ELE 1024
1042 static char traces[TRACE_BUF_ELE][128] = {0};
1044 #define TRACE_LOCK(X, Y) \
1045 KMP_SNPRINTF(traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s\n", X, Y);
1046 #define TRACE_LOCK_T(X, Y, Z) \
1047 KMP_SNPRINTF(traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s%d\n", X, Y, Z);
1048 #define TRACE_LOCK_HT(X, Y, Z, Q) \
1049 KMP_SNPRINTF(traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s %d,%d\n", X, Y, \
1052 static void __kmp_dump_queuing_lock(kmp_info_t *this_thr, kmp_int32 gtid,
1053 kmp_queuing_lock_t *lck, kmp_int32 head_id,
1054 kmp_int32 tail_id) {
1057 __kmp_printf_no_lock(
"\n__kmp_dump_queuing_lock: TRACE BEGINS HERE! \n");
1059 i = tc % TRACE_BUF_ELE;
1060 __kmp_printf_no_lock(
"%s\n", traces[i]);
1061 i = (i + 1) % TRACE_BUF_ELE;
1062 while (i != (tc % TRACE_BUF_ELE)) {
1063 __kmp_printf_no_lock(
"%s", traces[i]);
1064 i = (i + 1) % TRACE_BUF_ELE;
1066 __kmp_printf_no_lock(
"\n");
1068 __kmp_printf_no_lock(
"\n__kmp_dump_queuing_lock: gtid+1:%d, spin_here:%d, "
1069 "next_wait:%d, head_id:%d, tail_id:%d\n",
1070 gtid + 1, this_thr->th.th_spin_here,
1071 this_thr->th.th_next_waiting, head_id, tail_id);
1073 __kmp_printf_no_lock(
"\t\thead: %d ", lck->lk.head_id);
1075 if (lck->lk.head_id >= 1) {
1076 t = __kmp_threads[lck->lk.head_id - 1]->th.th_next_waiting;
1078 __kmp_printf_no_lock(
"-> %d ", t);
1079 t = __kmp_threads[t - 1]->th.th_next_waiting;
1082 __kmp_printf_no_lock(
"; tail: %d ", lck->lk.tail_id);
1083 __kmp_printf_no_lock(
"\n\n");
1088 static kmp_int32 __kmp_get_queuing_lock_owner(kmp_queuing_lock_t *lck) {
1089 return TCR_4(lck->lk.owner_id) - 1;
1092 static inline bool __kmp_is_queuing_lock_nestable(kmp_queuing_lock_t *lck) {
1093 return lck->lk.depth_locked != -1;
1097 template <
bool takeTime>
1100 __forceinline
static int
1101 __kmp_acquire_queuing_lock_timed_template(kmp_queuing_lock_t *lck,
1103 kmp_info_t *this_thr = __kmp_thread_from_gtid(gtid);
1104 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1105 volatile kmp_int32 *tail_id_p = &lck->lk.tail_id;
1106 volatile kmp_uint32 *spin_here_p;
1107 kmp_int32 need_mf = 1;
1110 ompt_state_t prev_state = ompt_state_undefined;
1114 (
"__kmp_acquire_queuing_lock: lck:%p, T#%d entering\n", lck, gtid));
1116 KMP_FSYNC_PREPARE(lck);
1117 KMP_DEBUG_ASSERT(this_thr != NULL);
1118 spin_here_p = &this_thr->th.th_spin_here;
1120 #ifdef DEBUG_QUEUING_LOCKS
1121 TRACE_LOCK(gtid + 1,
"acq ent");
1123 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1124 if (this_thr->th.th_next_waiting != 0)
1125 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1127 KMP_DEBUG_ASSERT(!*spin_here_p);
1128 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
1137 *spin_here_p = TRUE;
1149 #ifdef DEBUG_QUEUING_LOCKS
1151 TRACE_LOCK_HT(gtid + 1,
"acq read: ", head, tail);
1160 enqueued = KMP_COMPARE_AND_STORE_ACQ64((
volatile kmp_int64 *)tail_id_p,
1162 KMP_PACK_64(gtid + 1, gtid + 1));
1163 #ifdef DEBUG_QUEUING_LOCKS
1165 TRACE_LOCK(gtid + 1,
"acq enq: (-1,0)->(tid,tid)");
1171 KMP_DEBUG_ASSERT(tail != gtid + 1);
1173 #ifdef DEBUG_QUEUING_LOCKS
1174 TRACE_LOCK_HT(gtid + 1,
"acq read: ", head, tail);
1182 enqueued = KMP_COMPARE_AND_STORE_ACQ32(tail_id_p, tail, gtid + 1);
1184 #ifdef DEBUG_QUEUING_LOCKS
1186 TRACE_LOCK(gtid + 1,
"acq enq: (h,t)->(h,tid)");
1193 kmp_int32 grabbed_lock;
1195 #ifdef DEBUG_QUEUING_LOCKS
1197 TRACE_LOCK_HT(gtid + 1,
"acq read: ", head, tail);
1203 grabbed_lock = KMP_COMPARE_AND_STORE_ACQ32(head_id_p, 0, -1);
1207 *spin_here_p = FALSE;
1211 (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: no queuing\n",
1213 #ifdef DEBUG_QUEUING_LOCKS
1214 TRACE_LOCK_HT(gtid + 1,
"acq exit: ", head, 0);
1218 if (ompt_enabled.enabled && prev_state != ompt_state_undefined) {
1220 this_thr->th.ompt_thread_info.state = prev_state;
1221 this_thr->th.ompt_thread_info.wait_id = 0;
1225 KMP_FSYNC_ACQUIRED(lck);
1226 return KMP_LOCK_ACQUIRED_FIRST;
1233 if (ompt_enabled.enabled && prev_state == ompt_state_undefined) {
1235 prev_state = this_thr->th.ompt_thread_info.state;
1236 this_thr->th.ompt_thread_info.wait_id = (uint64_t)lck;
1237 this_thr->th.ompt_thread_info.state = ompt_state_wait_lock;
1243 kmp_info_t *tail_thr = __kmp_thread_from_gtid(tail - 1);
1244 KMP_ASSERT(tail_thr != NULL);
1245 tail_thr->th.th_next_waiting = gtid + 1;
1249 (
"__kmp_acquire_queuing_lock: lck:%p, T#%d waiting for lock\n",
1255 KMP_WAIT_YIELD(spin_here_p, FALSE, KMP_EQ, lck);
1257 #ifdef DEBUG_QUEUING_LOCKS
1258 TRACE_LOCK(gtid + 1,
"acq spin");
1260 if (this_thr->th.th_next_waiting != 0)
1261 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1263 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
1264 KA_TRACE(1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: after "
1265 "waiting on queue\n",
1268 #ifdef DEBUG_QUEUING_LOCKS
1269 TRACE_LOCK(gtid + 1,
"acq exit 2");
1274 this_thr->th.ompt_thread_info.state = prev_state;
1275 this_thr->th.ompt_thread_info.wait_id = 0;
1279 return KMP_LOCK_ACQUIRED_FIRST;
1285 KMP_YIELD(TCR_4(__kmp_nth) >
1286 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
1287 #ifdef DEBUG_QUEUING_LOCKS
1288 TRACE_LOCK(gtid + 1,
"acq retry");
1291 KMP_ASSERT2(0,
"should not get here");
1292 return KMP_LOCK_ACQUIRED_FIRST;
1295 int __kmp_acquire_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1296 KMP_DEBUG_ASSERT(gtid >= 0);
1298 int retval = __kmp_acquire_queuing_lock_timed_template<false>(lck, gtid);
1299 ANNOTATE_QUEUING_ACQUIRED(lck);
1303 static int __kmp_acquire_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1305 char const *
const func =
"omp_set_lock";
1306 if (lck->lk.initialized != lck) {
1307 KMP_FATAL(LockIsUninitialized, func);
1309 if (__kmp_is_queuing_lock_nestable(lck)) {
1310 KMP_FATAL(LockNestableUsedAsSimple, func);
1312 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1313 KMP_FATAL(LockIsAlreadyOwned, func);
1316 __kmp_acquire_queuing_lock(lck, gtid);
1318 lck->lk.owner_id = gtid + 1;
1319 return KMP_LOCK_ACQUIRED_FIRST;
1322 int __kmp_test_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1323 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1326 kmp_info_t *this_thr;
1329 KA_TRACE(1000, (
"__kmp_test_queuing_lock: T#%d entering\n", gtid));
1330 KMP_DEBUG_ASSERT(gtid >= 0);
1332 this_thr = __kmp_thread_from_gtid(gtid);
1333 KMP_DEBUG_ASSERT(this_thr != NULL);
1334 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
1341 if (KMP_COMPARE_AND_STORE_ACQ32(head_id_p, 0, -1)) {
1343 (
"__kmp_test_queuing_lock: T#%d exiting: holding lock\n", gtid));
1344 KMP_FSYNC_ACQUIRED(lck);
1345 ANNOTATE_QUEUING_ACQUIRED(lck);
1351 (
"__kmp_test_queuing_lock: T#%d exiting: without lock\n", gtid));
1355 static int __kmp_test_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1357 char const *
const func =
"omp_test_lock";
1358 if (lck->lk.initialized != lck) {
1359 KMP_FATAL(LockIsUninitialized, func);
1361 if (__kmp_is_queuing_lock_nestable(lck)) {
1362 KMP_FATAL(LockNestableUsedAsSimple, func);
1365 int retval = __kmp_test_queuing_lock(lck, gtid);
1368 lck->lk.owner_id = gtid + 1;
1373 int __kmp_release_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1374 kmp_info_t *this_thr;
1375 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1376 volatile kmp_int32 *tail_id_p = &lck->lk.tail_id;
1379 (
"__kmp_release_queuing_lock: lck:%p, T#%d entering\n", lck, gtid));
1380 KMP_DEBUG_ASSERT(gtid >= 0);
1381 this_thr = __kmp_thread_from_gtid(gtid);
1382 KMP_DEBUG_ASSERT(this_thr != NULL);
1383 #ifdef DEBUG_QUEUING_LOCKS
1384 TRACE_LOCK(gtid + 1,
"rel ent");
1386 if (this_thr->th.th_spin_here)
1387 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1388 if (this_thr->th.th_next_waiting != 0)
1389 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1391 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
1392 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
1394 KMP_FSYNC_RELEASING(lck);
1395 ANNOTATE_QUEUING_RELEASED(lck);
1404 #ifdef DEBUG_QUEUING_LOCKS
1406 TRACE_LOCK_HT(gtid + 1,
"rel read: ", head, tail);
1408 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1410 KMP_DEBUG_ASSERT(head !=
1415 if (KMP_COMPARE_AND_STORE_REL32(head_id_p, -1, 0)) {
1418 (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: queue empty\n",
1420 #ifdef DEBUG_QUEUING_LOCKS
1421 TRACE_LOCK_HT(gtid + 1,
"rel exit: ", 0, 0);
1427 return KMP_LOCK_RELEASED;
1434 #ifdef DEBUG_QUEUING_LOCKS
1436 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1438 KMP_DEBUG_ASSERT(head > 0);
1441 dequeued = KMP_COMPARE_AND_STORE_REL64(
1442 RCAST(
volatile kmp_int64 *, tail_id_p), KMP_PACK_64(head, head),
1443 KMP_PACK_64(-1, 0));
1444 #ifdef DEBUG_QUEUING_LOCKS
1445 TRACE_LOCK(gtid + 1,
"rel deq: (h,h)->(-1,0)");
1449 volatile kmp_int32 *waiting_id_p;
1450 kmp_info_t *head_thr = __kmp_thread_from_gtid(head - 1);
1451 KMP_DEBUG_ASSERT(head_thr != NULL);
1452 waiting_id_p = &head_thr->th.th_next_waiting;
1455 #ifdef DEBUG_QUEUING_LOCKS
1456 if (head <= 0 || tail <= 0)
1457 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1459 KMP_DEBUG_ASSERT(head > 0 && tail > 0);
1465 *head_id_p = KMP_WAIT_YIELD((
volatile kmp_uint32 *)waiting_id_p, 0,
1467 #ifdef DEBUG_QUEUING_LOCKS
1468 TRACE_LOCK(gtid + 1,
"rel deq: (h,t)->(h',t)");
1475 kmp_info_t *head_thr = __kmp_thread_from_gtid(head - 1);
1476 KMP_DEBUG_ASSERT(head_thr != NULL);
1479 #ifdef DEBUG_QUEUING_LOCKS
1480 if (head <= 0 || tail <= 0)
1481 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1483 KMP_DEBUG_ASSERT(head > 0 && tail > 0);
1487 head_thr->th.th_next_waiting = 0;
1488 #ifdef DEBUG_QUEUING_LOCKS
1489 TRACE_LOCK_T(gtid + 1,
"rel nw=0 for t=", head);
1494 head_thr->th.th_spin_here = FALSE;
1496 KA_TRACE(1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: after "
1499 #ifdef DEBUG_QUEUING_LOCKS
1500 TRACE_LOCK(gtid + 1,
"rel exit 2");
1502 return KMP_LOCK_RELEASED;
1507 #ifdef DEBUG_QUEUING_LOCKS
1508 TRACE_LOCK(gtid + 1,
"rel retry");
1512 KMP_ASSERT2(0,
"should not get here");
1513 return KMP_LOCK_RELEASED;
1516 static int __kmp_release_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1518 char const *
const func =
"omp_unset_lock";
1520 if (lck->lk.initialized != lck) {
1521 KMP_FATAL(LockIsUninitialized, func);
1523 if (__kmp_is_queuing_lock_nestable(lck)) {
1524 KMP_FATAL(LockNestableUsedAsSimple, func);
1526 if (__kmp_get_queuing_lock_owner(lck) == -1) {
1527 KMP_FATAL(LockUnsettingFree, func);
1529 if (__kmp_get_queuing_lock_owner(lck) != gtid) {
1530 KMP_FATAL(LockUnsettingSetByAnother, func);
1532 lck->lk.owner_id = 0;
1533 return __kmp_release_queuing_lock(lck, gtid);
1536 void __kmp_init_queuing_lock(kmp_queuing_lock_t *lck) {
1537 lck->lk.location = NULL;
1538 lck->lk.head_id = 0;
1539 lck->lk.tail_id = 0;
1540 lck->lk.next_ticket = 0;
1541 lck->lk.now_serving = 0;
1542 lck->lk.owner_id = 0;
1543 lck->lk.depth_locked = -1;
1544 lck->lk.initialized = lck;
1546 KA_TRACE(1000, (
"__kmp_init_queuing_lock: lock %p initialized\n", lck));
1549 void __kmp_destroy_queuing_lock(kmp_queuing_lock_t *lck) {
1550 lck->lk.initialized = NULL;
1551 lck->lk.location = NULL;
1552 lck->lk.head_id = 0;
1553 lck->lk.tail_id = 0;
1554 lck->lk.next_ticket = 0;
1555 lck->lk.now_serving = 0;
1556 lck->lk.owner_id = 0;
1557 lck->lk.depth_locked = -1;
1560 static void __kmp_destroy_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
1561 char const *
const func =
"omp_destroy_lock";
1562 if (lck->lk.initialized != lck) {
1563 KMP_FATAL(LockIsUninitialized, func);
1565 if (__kmp_is_queuing_lock_nestable(lck)) {
1566 KMP_FATAL(LockNestableUsedAsSimple, func);
1568 if (__kmp_get_queuing_lock_owner(lck) != -1) {
1569 KMP_FATAL(LockStillOwned, func);
1571 __kmp_destroy_queuing_lock(lck);
1576 int __kmp_acquire_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1577 KMP_DEBUG_ASSERT(gtid >= 0);
1579 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1580 lck->lk.depth_locked += 1;
1581 return KMP_LOCK_ACQUIRED_NEXT;
1583 __kmp_acquire_queuing_lock_timed_template<false>(lck, gtid);
1584 ANNOTATE_QUEUING_ACQUIRED(lck);
1586 lck->lk.depth_locked = 1;
1588 lck->lk.owner_id = gtid + 1;
1589 return KMP_LOCK_ACQUIRED_FIRST;
1594 __kmp_acquire_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1596 char const *
const func =
"omp_set_nest_lock";
1597 if (lck->lk.initialized != lck) {
1598 KMP_FATAL(LockIsUninitialized, func);
1600 if (!__kmp_is_queuing_lock_nestable(lck)) {
1601 KMP_FATAL(LockSimpleUsedAsNestable, func);
1603 return __kmp_acquire_nested_queuing_lock(lck, gtid);
1606 int __kmp_test_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1609 KMP_DEBUG_ASSERT(gtid >= 0);
1611 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1612 retval = ++lck->lk.depth_locked;
1613 }
else if (!__kmp_test_queuing_lock(lck, gtid)) {
1617 retval = lck->lk.depth_locked = 1;
1619 lck->lk.owner_id = gtid + 1;
1624 static int __kmp_test_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1626 char const *
const func =
"omp_test_nest_lock";
1627 if (lck->lk.initialized != lck) {
1628 KMP_FATAL(LockIsUninitialized, func);
1630 if (!__kmp_is_queuing_lock_nestable(lck)) {
1631 KMP_FATAL(LockSimpleUsedAsNestable, func);
1633 return __kmp_test_nested_queuing_lock(lck, gtid);
1636 int __kmp_release_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1637 KMP_DEBUG_ASSERT(gtid >= 0);
1640 if (--(lck->lk.depth_locked) == 0) {
1642 lck->lk.owner_id = 0;
1643 __kmp_release_queuing_lock(lck, gtid);
1644 return KMP_LOCK_RELEASED;
1646 return KMP_LOCK_STILL_HELD;
1650 __kmp_release_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1652 char const *
const func =
"omp_unset_nest_lock";
1654 if (lck->lk.initialized != lck) {
1655 KMP_FATAL(LockIsUninitialized, func);
1657 if (!__kmp_is_queuing_lock_nestable(lck)) {
1658 KMP_FATAL(LockSimpleUsedAsNestable, func);
1660 if (__kmp_get_queuing_lock_owner(lck) == -1) {
1661 KMP_FATAL(LockUnsettingFree, func);
1663 if (__kmp_get_queuing_lock_owner(lck) != gtid) {
1664 KMP_FATAL(LockUnsettingSetByAnother, func);
1666 return __kmp_release_nested_queuing_lock(lck, gtid);
1669 void __kmp_init_nested_queuing_lock(kmp_queuing_lock_t *lck) {
1670 __kmp_init_queuing_lock(lck);
1671 lck->lk.depth_locked = 0;
1674 void __kmp_destroy_nested_queuing_lock(kmp_queuing_lock_t *lck) {
1675 __kmp_destroy_queuing_lock(lck);
1676 lck->lk.depth_locked = 0;
1680 __kmp_destroy_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
1681 char const *
const func =
"omp_destroy_nest_lock";
1682 if (lck->lk.initialized != lck) {
1683 KMP_FATAL(LockIsUninitialized, func);
1685 if (!__kmp_is_queuing_lock_nestable(lck)) {
1686 KMP_FATAL(LockSimpleUsedAsNestable, func);
1688 if (__kmp_get_queuing_lock_owner(lck) != -1) {
1689 KMP_FATAL(LockStillOwned, func);
1691 __kmp_destroy_nested_queuing_lock(lck);
1696 static const ident_t *__kmp_get_queuing_lock_location(kmp_queuing_lock_t *lck) {
1697 return lck->lk.location;
1700 static void __kmp_set_queuing_lock_location(kmp_queuing_lock_t *lck,
1702 lck->lk.location = loc;
1705 static kmp_lock_flags_t __kmp_get_queuing_lock_flags(kmp_queuing_lock_t *lck) {
1706 return lck->lk.flags;
1709 static void __kmp_set_queuing_lock_flags(kmp_queuing_lock_t *lck,
1710 kmp_lock_flags_t flags) {
1711 lck->lk.flags = flags;
1714 #if KMP_USE_ADAPTIVE_LOCKS
1718 #if (KMP_COMPILER_ICC && __INTEL_COMPILER >= 1300) || \
1719 (KMP_COMPILER_MSVC && _MSC_VER >= 1700) || \
1720 (KMP_COMPILER_CLANG && KMP_MSVC_COMPAT)
1722 #include <immintrin.h>
1723 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT)
1728 #define _XBEGIN_STARTED (~0u)
1729 #define _XABORT_EXPLICIT (1 << 0)
1730 #define _XABORT_RETRY (1 << 1)
1731 #define _XABORT_CONFLICT (1 << 2)
1732 #define _XABORT_CAPACITY (1 << 3)
1733 #define _XABORT_DEBUG (1 << 4)
1734 #define _XABORT_NESTED (1 << 5)
1735 #define _XABORT_CODE(x) ((unsigned char)(((x) >> 24) & 0xFF))
1738 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT)
1740 #define STRINGIZE_INTERNAL(arg) #arg
1741 #define STRINGIZE(arg) STRINGIZE_INTERNAL(arg)
1747 static __inline
int _xbegin() {
1775 #endif // KMP_ARCH_X86_64
1784 __asm__
volatile(
"1: .byte 0xC7; .byte 0xF8;\n"
1787 "1: movl %%eax,%0\n"
1789 :
"+r"(res)::
"memory",
"%eax");
1790 #endif // KMP_OS_WINDOWS
1795 static __inline
void _xend() {
1803 __asm__
volatile(
".byte 0x0f; .byte 0x01; .byte 0xd5" :::
"memory");
1812 #define _xabort(ARG) _asm _emit 0xc6 _asm _emit 0xf8 _asm _emit ARG
1814 #define _xabort(ARG) \
1815 __asm__ volatile(".byte 0xC6; .byte 0xF8; .byte " STRINGIZE(ARG):::"memory");
1818 #endif // KMP_COMPILER_ICC && __INTEL_COMPILER >= 1300
1821 #if KMP_DEBUG_ADAPTIVE_LOCKS
1826 static kmp_adaptive_lock_statistics_t destroyedStats;
1829 static kmp_adaptive_lock_info_t liveLocks;
1832 static kmp_bootstrap_lock_t chain_lock =
1833 KMP_BOOTSTRAP_LOCK_INITIALIZER(chain_lock);
1836 void __kmp_init_speculative_stats() {
1837 kmp_adaptive_lock_info_t *lck = &liveLocks;
1839 memset(CCAST(kmp_adaptive_lock_statistics_t *, &(lck->stats)), 0,
1840 sizeof(lck->stats));
1841 lck->stats.next = lck;
1842 lck->stats.prev = lck;
1844 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1845 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1847 __kmp_init_bootstrap_lock(&chain_lock);
1851 static void __kmp_remember_lock(kmp_adaptive_lock_info_t *lck) {
1852 __kmp_acquire_bootstrap_lock(&chain_lock);
1854 lck->stats.next = liveLocks.stats.next;
1855 lck->stats.prev = &liveLocks;
1857 liveLocks.stats.next = lck;
1858 lck->stats.next->stats.prev = lck;
1860 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1861 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1863 __kmp_release_bootstrap_lock(&chain_lock);
1866 static void __kmp_forget_lock(kmp_adaptive_lock_info_t *lck) {
1867 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1868 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1870 kmp_adaptive_lock_info_t *n = lck->stats.next;
1871 kmp_adaptive_lock_info_t *p = lck->stats.prev;
1877 static void __kmp_zero_speculative_stats(kmp_adaptive_lock_info_t *lck) {
1878 memset(CCAST(kmp_adaptive_lock_statistics_t *, &lck->stats), 0,
1879 sizeof(lck->stats));
1880 __kmp_remember_lock(lck);
1883 static void __kmp_add_stats(kmp_adaptive_lock_statistics_t *t,
1884 kmp_adaptive_lock_info_t *lck) {
1885 kmp_adaptive_lock_statistics_t
volatile *s = &lck->stats;
1887 t->nonSpeculativeAcquireAttempts += lck->acquire_attempts;
1888 t->successfulSpeculations += s->successfulSpeculations;
1889 t->hardFailedSpeculations += s->hardFailedSpeculations;
1890 t->softFailedSpeculations += s->softFailedSpeculations;
1891 t->nonSpeculativeAcquires += s->nonSpeculativeAcquires;
1892 t->lemmingYields += s->lemmingYields;
1895 static void __kmp_accumulate_speculative_stats(kmp_adaptive_lock_info_t *lck) {
1896 __kmp_acquire_bootstrap_lock(&chain_lock);
1898 __kmp_add_stats(&destroyedStats, lck);
1899 __kmp_forget_lock(lck);
1901 __kmp_release_bootstrap_lock(&chain_lock);
1904 static float percent(kmp_uint32 count, kmp_uint32 total) {
1905 return (total == 0) ? 0.0 : (100.0 * count) / total;
1908 static FILE *__kmp_open_stats_file() {
1909 if (strcmp(__kmp_speculative_statsfile,
"-") == 0)
1912 size_t buffLen = KMP_STRLEN(__kmp_speculative_statsfile) + 20;
1913 char buffer[buffLen];
1914 KMP_SNPRINTF(&buffer[0], buffLen, __kmp_speculative_statsfile,
1915 (kmp_int32)getpid());
1916 FILE *result = fopen(&buffer[0],
"w");
1919 return result ? result : stdout;
1922 void __kmp_print_speculative_stats() {
1923 kmp_adaptive_lock_statistics_t total = destroyedStats;
1924 kmp_adaptive_lock_info_t *lck;
1926 for (lck = liveLocks.stats.next; lck != &liveLocks; lck = lck->stats.next) {
1927 __kmp_add_stats(&total, lck);
1929 kmp_adaptive_lock_statistics_t *t = &total;
1930 kmp_uint32 totalSections =
1931 t->nonSpeculativeAcquires + t->successfulSpeculations;
1932 kmp_uint32 totalSpeculations = t->successfulSpeculations +
1933 t->hardFailedSpeculations +
1934 t->softFailedSpeculations;
1935 if (totalSections <= 0)
1938 FILE *statsFile = __kmp_open_stats_file();
1940 fprintf(statsFile,
"Speculative lock statistics (all approximate!)\n");
1941 fprintf(statsFile,
" Lock parameters: \n"
1942 " max_soft_retries : %10d\n"
1943 " max_badness : %10d\n",
1944 __kmp_adaptive_backoff_params.max_soft_retries,
1945 __kmp_adaptive_backoff_params.max_badness);
1946 fprintf(statsFile,
" Non-speculative acquire attempts : %10d\n",
1947 t->nonSpeculativeAcquireAttempts);
1948 fprintf(statsFile,
" Total critical sections : %10d\n",
1950 fprintf(statsFile,
" Successful speculations : %10d (%5.1f%%)\n",
1951 t->successfulSpeculations,
1952 percent(t->successfulSpeculations, totalSections));
1953 fprintf(statsFile,
" Non-speculative acquires : %10d (%5.1f%%)\n",
1954 t->nonSpeculativeAcquires,
1955 percent(t->nonSpeculativeAcquires, totalSections));
1956 fprintf(statsFile,
" Lemming yields : %10d\n\n",
1959 fprintf(statsFile,
" Speculative acquire attempts : %10d\n",
1961 fprintf(statsFile,
" Successes : %10d (%5.1f%%)\n",
1962 t->successfulSpeculations,
1963 percent(t->successfulSpeculations, totalSpeculations));
1964 fprintf(statsFile,
" Soft failures : %10d (%5.1f%%)\n",
1965 t->softFailedSpeculations,
1966 percent(t->softFailedSpeculations, totalSpeculations));
1967 fprintf(statsFile,
" Hard failures : %10d (%5.1f%%)\n",
1968 t->hardFailedSpeculations,
1969 percent(t->hardFailedSpeculations, totalSpeculations));
1971 if (statsFile != stdout)
1975 #define KMP_INC_STAT(lck, stat) (lck->lk.adaptive.stats.stat++)
1977 #define KMP_INC_STAT(lck, stat)
1979 #endif // KMP_DEBUG_ADAPTIVE_LOCKS
1981 static inline bool __kmp_is_unlocked_queuing_lock(kmp_queuing_lock_t *lck) {
1984 bool res = lck->lk.head_id == 0;
1988 #if KMP_COMPILER_ICC
1991 __sync_synchronize();
1998 static __inline
void
1999 __kmp_update_badness_after_success(kmp_adaptive_lock_t *lck) {
2001 lck->lk.adaptive.badness = 0;
2002 KMP_INC_STAT(lck, successfulSpeculations);
2006 static __inline
void __kmp_step_badness(kmp_adaptive_lock_t *lck) {
2007 kmp_uint32 newBadness = (lck->lk.adaptive.badness << 1) | 1;
2008 if (newBadness > lck->lk.adaptive.max_badness) {
2011 lck->lk.adaptive.badness = newBadness;
2016 static __inline
int __kmp_should_speculate(kmp_adaptive_lock_t *lck,
2018 kmp_uint32 badness = lck->lk.adaptive.badness;
2019 kmp_uint32 attempts = lck->lk.adaptive.acquire_attempts;
2020 int res = (attempts & badness) == 0;
2026 static int __kmp_test_adaptive_lock_only(kmp_adaptive_lock_t *lck,
2028 int retries = lck->lk.adaptive.max_soft_retries;
2035 kmp_uint32 status = _xbegin();
2040 if (status == _XBEGIN_STARTED) {
2045 if (!__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2049 KMP_ASSERT2(0,
"should not get here");
2054 if (status & SOFT_ABORT_MASK) {
2055 KMP_INC_STAT(lck, softFailedSpeculations);
2058 KMP_INC_STAT(lck, hardFailedSpeculations);
2063 }
while (retries--);
2067 __kmp_step_badness(lck);
2074 static int __kmp_test_adaptive_lock(kmp_adaptive_lock_t *lck, kmp_int32 gtid) {
2076 if (__kmp_should_speculate(lck, gtid) &&
2077 __kmp_test_adaptive_lock_only(lck, gtid))
2082 lck->lk.adaptive.acquire_attempts++;
2085 if (__kmp_test_queuing_lock(GET_QLK_PTR(lck), gtid)) {
2086 KMP_INC_STAT(lck, nonSpeculativeAcquires);
2093 static int __kmp_test_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2095 char const *
const func =
"omp_test_lock";
2096 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2097 KMP_FATAL(LockIsUninitialized, func);
2100 int retval = __kmp_test_adaptive_lock(lck, gtid);
2103 lck->lk.qlk.owner_id = gtid + 1;
2119 static void __kmp_acquire_adaptive_lock(kmp_adaptive_lock_t *lck,
2121 if (__kmp_should_speculate(lck, gtid)) {
2122 if (__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2123 if (__kmp_test_adaptive_lock_only(lck, gtid))
2132 while (!__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2133 KMP_INC_STAT(lck, lemmingYields);
2137 if (__kmp_test_adaptive_lock_only(lck, gtid))
2144 lck->lk.adaptive.acquire_attempts++;
2146 __kmp_acquire_queuing_lock_timed_template<FALSE>(GET_QLK_PTR(lck), gtid);
2148 KMP_INC_STAT(lck, nonSpeculativeAcquires);
2149 ANNOTATE_QUEUING_ACQUIRED(lck);
2152 static void __kmp_acquire_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2154 char const *
const func =
"omp_set_lock";
2155 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2156 KMP_FATAL(LockIsUninitialized, func);
2158 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) == gtid) {
2159 KMP_FATAL(LockIsAlreadyOwned, func);
2162 __kmp_acquire_adaptive_lock(lck, gtid);
2164 lck->lk.qlk.owner_id = gtid + 1;
2167 static int __kmp_release_adaptive_lock(kmp_adaptive_lock_t *lck,
2169 if (__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(
2174 __kmp_update_badness_after_success(lck);
2177 __kmp_release_queuing_lock(GET_QLK_PTR(lck), gtid);
2179 return KMP_LOCK_RELEASED;
2182 static int __kmp_release_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2184 char const *
const func =
"omp_unset_lock";
2186 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2187 KMP_FATAL(LockIsUninitialized, func);
2189 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) == -1) {
2190 KMP_FATAL(LockUnsettingFree, func);
2192 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) != gtid) {
2193 KMP_FATAL(LockUnsettingSetByAnother, func);
2195 lck->lk.qlk.owner_id = 0;
2196 __kmp_release_adaptive_lock(lck, gtid);
2197 return KMP_LOCK_RELEASED;
2200 static void __kmp_init_adaptive_lock(kmp_adaptive_lock_t *lck) {
2201 __kmp_init_queuing_lock(GET_QLK_PTR(lck));
2202 lck->lk.adaptive.badness = 0;
2203 lck->lk.adaptive.acquire_attempts = 0;
2204 lck->lk.adaptive.max_soft_retries =
2205 __kmp_adaptive_backoff_params.max_soft_retries;
2206 lck->lk.adaptive.max_badness = __kmp_adaptive_backoff_params.max_badness;
2207 #if KMP_DEBUG_ADAPTIVE_LOCKS
2208 __kmp_zero_speculative_stats(&lck->lk.adaptive);
2210 KA_TRACE(1000, (
"__kmp_init_adaptive_lock: lock %p initialized\n", lck));
2213 static void __kmp_destroy_adaptive_lock(kmp_adaptive_lock_t *lck) {
2214 #if KMP_DEBUG_ADAPTIVE_LOCKS
2215 __kmp_accumulate_speculative_stats(&lck->lk.adaptive);
2217 __kmp_destroy_queuing_lock(GET_QLK_PTR(lck));
2221 static void __kmp_destroy_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck) {
2222 char const *
const func =
"omp_destroy_lock";
2223 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2224 KMP_FATAL(LockIsUninitialized, func);
2226 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) != -1) {
2227 KMP_FATAL(LockStillOwned, func);
2229 __kmp_destroy_adaptive_lock(lck);
2232 #endif // KMP_USE_ADAPTIVE_LOCKS
2238 static kmp_int32 __kmp_get_drdpa_lock_owner(kmp_drdpa_lock_t *lck) {
2239 return lck->lk.owner_id - 1;
2242 static inline bool __kmp_is_drdpa_lock_nestable(kmp_drdpa_lock_t *lck) {
2243 return lck->lk.depth_locked != -1;
2246 __forceinline
static int
2247 __kmp_acquire_drdpa_lock_timed_template(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2248 kmp_uint64 ticket = KMP_ATOMIC_INC(&lck->lk.next_ticket);
2249 kmp_uint64 mask = lck->lk.mask;
2250 std::atomic<kmp_uint64> *polls = lck->lk.polls;
2252 #ifdef USE_LOCK_PROFILE
2253 if (polls[ticket & mask] != ticket)
2254 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
2268 KMP_FSYNC_PREPARE(lck);
2269 KMP_INIT_YIELD(spins);
2270 while (polls[ticket & mask] < ticket) {
2275 KMP_YIELD(TCR_4(__kmp_nth) >
2276 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
2277 KMP_YIELD_SPIN(spins);
2286 mask = lck->lk.mask;
2287 polls = lck->lk.polls;
2291 KMP_FSYNC_ACQUIRED(lck);
2292 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld acquired lock %p\n",
2294 lck->lk.now_serving = ticket;
2301 if ((lck->lk.old_polls != NULL) && (ticket >= lck->lk.cleanup_ticket)) {
2302 __kmp_free(lck->lk.old_polls);
2303 lck->lk.old_polls = NULL;
2304 lck->lk.cleanup_ticket = 0;
2310 if (lck->lk.old_polls == NULL) {
2311 bool reconfigure =
false;
2312 std::atomic<kmp_uint64> *old_polls = polls;
2313 kmp_uint32 num_polls = TCR_4(lck->lk.num_polls);
2315 if (TCR_4(__kmp_nth) >
2316 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {
2319 if (num_polls > 1) {
2321 num_polls = TCR_4(lck->lk.num_polls);
2324 polls = (std::atomic<kmp_uint64> *)__kmp_allocate(num_polls *
2332 kmp_uint64 num_waiting = TCR_8(lck->lk.next_ticket) - ticket - 1;
2333 if (num_waiting > num_polls) {
2334 kmp_uint32 old_num_polls = num_polls;
2337 mask = (mask << 1) | 1;
2339 }
while (num_polls <= num_waiting);
2345 polls = (std::atomic<kmp_uint64> *)__kmp_allocate(num_polls *
2348 for (i = 0; i < old_num_polls; i++) {
2349 polls[i].store(old_polls[i]);
2364 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld reconfiguring "
2365 "lock %p to %d polls\n",
2366 ticket, lck, num_polls));
2368 lck->lk.old_polls = old_polls;
2369 lck->lk.polls = polls;
2373 lck->lk.num_polls = num_polls;
2374 lck->lk.mask = mask;
2382 lck->lk.cleanup_ticket = lck->lk.next_ticket;
2385 return KMP_LOCK_ACQUIRED_FIRST;
2388 int __kmp_acquire_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2389 int retval = __kmp_acquire_drdpa_lock_timed_template(lck, gtid);
2390 ANNOTATE_DRDPA_ACQUIRED(lck);
2394 static int __kmp_acquire_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2396 char const *
const func =
"omp_set_lock";
2397 if (lck->lk.initialized != lck) {
2398 KMP_FATAL(LockIsUninitialized, func);
2400 if (__kmp_is_drdpa_lock_nestable(lck)) {
2401 KMP_FATAL(LockNestableUsedAsSimple, func);
2403 if ((gtid >= 0) && (__kmp_get_drdpa_lock_owner(lck) == gtid)) {
2404 KMP_FATAL(LockIsAlreadyOwned, func);
2407 __kmp_acquire_drdpa_lock(lck, gtid);
2409 lck->lk.owner_id = gtid + 1;
2410 return KMP_LOCK_ACQUIRED_FIRST;
2413 int __kmp_test_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2416 kmp_uint64 ticket = lck->lk.next_ticket;
2417 std::atomic<kmp_uint64> *polls = lck->lk.polls;
2418 kmp_uint64 mask = lck->lk.mask;
2419 if (polls[ticket & mask] == ticket) {
2420 kmp_uint64 next_ticket = ticket + 1;
2421 if (__kmp_atomic_compare_store_acq(&lck->lk.next_ticket, ticket,
2423 KMP_FSYNC_ACQUIRED(lck);
2424 KA_TRACE(1000, (
"__kmp_test_drdpa_lock: ticket #%lld acquired lock %p\n",
2426 lck->lk.now_serving = ticket;
2440 static int __kmp_test_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2442 char const *
const func =
"omp_test_lock";
2443 if (lck->lk.initialized != lck) {
2444 KMP_FATAL(LockIsUninitialized, func);
2446 if (__kmp_is_drdpa_lock_nestable(lck)) {
2447 KMP_FATAL(LockNestableUsedAsSimple, func);
2450 int retval = __kmp_test_drdpa_lock(lck, gtid);
2453 lck->lk.owner_id = gtid + 1;
2458 int __kmp_release_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2461 kmp_uint64 ticket = lck->lk.now_serving + 1;
2462 std::atomic<kmp_uint64> *polls = lck->lk.polls;
2463 kmp_uint64 mask = lck->lk.mask;
2464 KA_TRACE(1000, (
"__kmp_release_drdpa_lock: ticket #%lld released lock %p\n",
2466 KMP_FSYNC_RELEASING(lck);
2467 ANNOTATE_DRDPA_RELEASED(lck);
2468 polls[ticket & mask] = ticket;
2469 return KMP_LOCK_RELEASED;
2472 static int __kmp_release_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2474 char const *
const func =
"omp_unset_lock";
2476 if (lck->lk.initialized != lck) {
2477 KMP_FATAL(LockIsUninitialized, func);
2479 if (__kmp_is_drdpa_lock_nestable(lck)) {
2480 KMP_FATAL(LockNestableUsedAsSimple, func);
2482 if (__kmp_get_drdpa_lock_owner(lck) == -1) {
2483 KMP_FATAL(LockUnsettingFree, func);
2485 if ((gtid >= 0) && (__kmp_get_drdpa_lock_owner(lck) >= 0) &&
2486 (__kmp_get_drdpa_lock_owner(lck) != gtid)) {
2487 KMP_FATAL(LockUnsettingSetByAnother, func);
2489 lck->lk.owner_id = 0;
2490 return __kmp_release_drdpa_lock(lck, gtid);
2493 void __kmp_init_drdpa_lock(kmp_drdpa_lock_t *lck) {
2494 lck->lk.location = NULL;
2496 lck->lk.num_polls = 1;
2497 lck->lk.polls = (std::atomic<kmp_uint64> *)__kmp_allocate(
2498 lck->lk.num_polls *
sizeof(*(lck->lk.polls)));
2499 lck->lk.cleanup_ticket = 0;
2500 lck->lk.old_polls = NULL;
2501 lck->lk.next_ticket = 0;
2502 lck->lk.now_serving = 0;
2503 lck->lk.owner_id = 0;
2504 lck->lk.depth_locked = -1;
2505 lck->lk.initialized = lck;
2507 KA_TRACE(1000, (
"__kmp_init_drdpa_lock: lock %p initialized\n", lck));
2510 void __kmp_destroy_drdpa_lock(kmp_drdpa_lock_t *lck) {
2511 lck->lk.initialized = NULL;
2512 lck->lk.location = NULL;
2513 if (lck->lk.polls.load() != NULL) {
2514 __kmp_free(lck->lk.polls.load());
2515 lck->lk.polls = NULL;
2517 if (lck->lk.old_polls != NULL) {
2518 __kmp_free(lck->lk.old_polls);
2519 lck->lk.old_polls = NULL;
2522 lck->lk.num_polls = 0;
2523 lck->lk.cleanup_ticket = 0;
2524 lck->lk.next_ticket = 0;
2525 lck->lk.now_serving = 0;
2526 lck->lk.owner_id = 0;
2527 lck->lk.depth_locked = -1;
2530 static void __kmp_destroy_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
2531 char const *
const func =
"omp_destroy_lock";
2532 if (lck->lk.initialized != lck) {
2533 KMP_FATAL(LockIsUninitialized, func);
2535 if (__kmp_is_drdpa_lock_nestable(lck)) {
2536 KMP_FATAL(LockNestableUsedAsSimple, func);
2538 if (__kmp_get_drdpa_lock_owner(lck) != -1) {
2539 KMP_FATAL(LockStillOwned, func);
2541 __kmp_destroy_drdpa_lock(lck);
2546 int __kmp_acquire_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2547 KMP_DEBUG_ASSERT(gtid >= 0);
2549 if (__kmp_get_drdpa_lock_owner(lck) == gtid) {
2550 lck->lk.depth_locked += 1;
2551 return KMP_LOCK_ACQUIRED_NEXT;
2553 __kmp_acquire_drdpa_lock_timed_template(lck, gtid);
2554 ANNOTATE_DRDPA_ACQUIRED(lck);
2556 lck->lk.depth_locked = 1;
2558 lck->lk.owner_id = gtid + 1;
2559 return KMP_LOCK_ACQUIRED_FIRST;
2563 static void __kmp_acquire_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2565 char const *
const func =
"omp_set_nest_lock";
2566 if (lck->lk.initialized != lck) {
2567 KMP_FATAL(LockIsUninitialized, func);
2569 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2570 KMP_FATAL(LockSimpleUsedAsNestable, func);
2572 __kmp_acquire_nested_drdpa_lock(lck, gtid);
2575 int __kmp_test_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2578 KMP_DEBUG_ASSERT(gtid >= 0);
2580 if (__kmp_get_drdpa_lock_owner(lck) == gtid) {
2581 retval = ++lck->lk.depth_locked;
2582 }
else if (!__kmp_test_drdpa_lock(lck, gtid)) {
2586 retval = lck->lk.depth_locked = 1;
2588 lck->lk.owner_id = gtid + 1;
2593 static int __kmp_test_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2595 char const *
const func =
"omp_test_nest_lock";
2596 if (lck->lk.initialized != lck) {
2597 KMP_FATAL(LockIsUninitialized, func);
2599 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2600 KMP_FATAL(LockSimpleUsedAsNestable, func);
2602 return __kmp_test_nested_drdpa_lock(lck, gtid);
2605 int __kmp_release_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2606 KMP_DEBUG_ASSERT(gtid >= 0);
2609 if (--(lck->lk.depth_locked) == 0) {
2611 lck->lk.owner_id = 0;
2612 __kmp_release_drdpa_lock(lck, gtid);
2613 return KMP_LOCK_RELEASED;
2615 return KMP_LOCK_STILL_HELD;
2618 static int __kmp_release_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2620 char const *
const func =
"omp_unset_nest_lock";
2622 if (lck->lk.initialized != lck) {
2623 KMP_FATAL(LockIsUninitialized, func);
2625 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2626 KMP_FATAL(LockSimpleUsedAsNestable, func);
2628 if (__kmp_get_drdpa_lock_owner(lck) == -1) {
2629 KMP_FATAL(LockUnsettingFree, func);
2631 if (__kmp_get_drdpa_lock_owner(lck) != gtid) {
2632 KMP_FATAL(LockUnsettingSetByAnother, func);
2634 return __kmp_release_nested_drdpa_lock(lck, gtid);
2637 void __kmp_init_nested_drdpa_lock(kmp_drdpa_lock_t *lck) {
2638 __kmp_init_drdpa_lock(lck);
2639 lck->lk.depth_locked = 0;
2642 void __kmp_destroy_nested_drdpa_lock(kmp_drdpa_lock_t *lck) {
2643 __kmp_destroy_drdpa_lock(lck);
2644 lck->lk.depth_locked = 0;
2647 static void __kmp_destroy_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
2648 char const *
const func =
"omp_destroy_nest_lock";
2649 if (lck->lk.initialized != lck) {
2650 KMP_FATAL(LockIsUninitialized, func);
2652 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2653 KMP_FATAL(LockSimpleUsedAsNestable, func);
2655 if (__kmp_get_drdpa_lock_owner(lck) != -1) {
2656 KMP_FATAL(LockStillOwned, func);
2658 __kmp_destroy_nested_drdpa_lock(lck);
2663 static const ident_t *__kmp_get_drdpa_lock_location(kmp_drdpa_lock_t *lck) {
2664 return lck->lk.location;
2667 static void __kmp_set_drdpa_lock_location(kmp_drdpa_lock_t *lck,
2669 lck->lk.location = loc;
2672 static kmp_lock_flags_t __kmp_get_drdpa_lock_flags(kmp_drdpa_lock_t *lck) {
2673 return lck->lk.flags;
2676 static void __kmp_set_drdpa_lock_flags(kmp_drdpa_lock_t *lck,
2677 kmp_lock_flags_t flags) {
2678 lck->lk.flags = flags;
2682 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
2683 #define __kmp_tsc() __kmp_hardware_timestamp()
2685 kmp_backoff_t __kmp_spin_backoff_params = {1, 4096, 100};
2688 extern kmp_uint64 __kmp_now_nsec();
2689 kmp_backoff_t __kmp_spin_backoff_params = {1, 256, 100};
2690 #define __kmp_tsc() __kmp_now_nsec()
2700 static inline bool before(kmp_uint64 a, kmp_uint64 b) {
2701 return ((kmp_int64)b - (kmp_int64)a) > 0;
2705 void __kmp_spin_backoff(kmp_backoff_t *boff) {
2708 for (i = boff->step; i > 0; i--) {
2709 kmp_uint64 goal = __kmp_tsc() + boff->min_tick;
2712 }
while (before(__kmp_tsc(), goal));
2714 boff->step = (boff->step << 1 | 1) & (boff->max_backoff - 1);
2717 #if KMP_USE_DYNAMIC_LOCK
2721 static void __kmp_init_direct_lock(kmp_dyna_lock_t *lck,
2722 kmp_dyna_lockseq_t seq) {
2723 TCW_4(*lck, KMP_GET_D_TAG(seq));
2726 (
"__kmp_init_direct_lock: initialized direct lock with type#%d\n", seq));
2732 #define HLE_ACQUIRE ".byte 0xf2;"
2733 #define HLE_RELEASE ".byte 0xf3;"
2735 static inline kmp_uint32 swap4(kmp_uint32
volatile *p, kmp_uint32 v) {
2736 __asm__
volatile(HLE_ACQUIRE
"xchg %1,%0" :
"+r"(v),
"+m"(*p) : :
"memory");
2740 static void __kmp_destroy_hle_lock(kmp_dyna_lock_t *lck) { TCW_4(*lck, 0); }
2742 static void __kmp_destroy_hle_lock_with_checks(kmp_dyna_lock_t *lck) {
2746 static void __kmp_acquire_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2748 if (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle)) {
2751 while (*(kmp_uint32
volatile *)lck != KMP_LOCK_FREE(hle)) {
2752 for (
int i = delay; i != 0; --i)
2754 delay = ((delay << 1) | 1) & 7;
2756 }
while (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle));
2760 static void __kmp_acquire_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2762 __kmp_acquire_hle_lock(lck, gtid);
2765 static int __kmp_release_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2766 __asm__
volatile(HLE_RELEASE
"movl %1,%0"
2768 :
"r"(KMP_LOCK_FREE(hle))
2770 return KMP_LOCK_RELEASED;
2773 static int __kmp_release_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2775 return __kmp_release_hle_lock(lck, gtid);
2778 static int __kmp_test_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2779 return swap4(lck, KMP_LOCK_BUSY(1, hle)) == KMP_LOCK_FREE(hle);
2782 static int __kmp_test_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2784 return __kmp_test_hle_lock(lck, gtid);
2787 static void __kmp_init_rtm_lock(kmp_queuing_lock_t *lck) {
2788 __kmp_init_queuing_lock(lck);
2791 static void __kmp_destroy_rtm_lock(kmp_queuing_lock_t *lck) {
2792 __kmp_destroy_queuing_lock(lck);
2795 static void __kmp_destroy_rtm_lock_with_checks(kmp_queuing_lock_t *lck) {
2796 __kmp_destroy_queuing_lock_with_checks(lck);
2799 static void __kmp_acquire_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
2800 unsigned retries = 3, status;
2803 if (status == _XBEGIN_STARTED) {
2804 if (__kmp_is_unlocked_queuing_lock(lck))
2808 if ((status & _XABORT_EXPLICIT) && _XABORT_CODE(status) == 0xff) {
2810 while (!__kmp_is_unlocked_queuing_lock(lck))
2812 }
else if (!(status & _XABORT_RETRY))
2814 }
while (retries--);
2817 __kmp_acquire_queuing_lock(lck, gtid);
2820 static void __kmp_acquire_rtm_lock_with_checks(kmp_queuing_lock_t *lck,
2822 __kmp_acquire_rtm_lock(lck, gtid);
2825 static int __kmp_release_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
2826 if (__kmp_is_unlocked_queuing_lock(lck)) {
2831 __kmp_release_queuing_lock(lck, gtid);
2833 return KMP_LOCK_RELEASED;
2836 static int __kmp_release_rtm_lock_with_checks(kmp_queuing_lock_t *lck,
2838 return __kmp_release_rtm_lock(lck, gtid);
2841 static int __kmp_test_rtm_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
2842 unsigned retries = 3, status;
2845 if (status == _XBEGIN_STARTED && __kmp_is_unlocked_queuing_lock(lck)) {
2848 if (!(status & _XABORT_RETRY))
2850 }
while (retries--);
2852 return (__kmp_is_unlocked_queuing_lock(lck)) ? 1 : 0;
2855 static int __kmp_test_rtm_lock_with_checks(kmp_queuing_lock_t *lck,
2857 return __kmp_test_rtm_lock(lck, gtid);
2860 #endif // KMP_USE_TSX
2863 static void __kmp_init_indirect_lock(kmp_dyna_lock_t *l,
2864 kmp_dyna_lockseq_t tag);
2865 static void __kmp_destroy_indirect_lock(kmp_dyna_lock_t *lock);
2866 static int __kmp_set_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32);
2867 static int __kmp_unset_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32);
2868 static int __kmp_test_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32);
2869 static int __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
2871 static int __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
2873 static int __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
2877 #define KMP_FOREACH_LOCK_KIND(m, a) m(ticket, a) m(queuing, a) m(drdpa, a)
2879 #define expand1(lk, op) \
2880 static void __kmp_##op##_##lk##_##lock(kmp_user_lock_p lock) { \
2881 __kmp_##op##_##lk##_##lock(&lock->lk); \
2883 #define expand2(lk, op) \
2884 static int __kmp_##op##_##lk##_##lock(kmp_user_lock_p lock, \
2886 return __kmp_##op##_##lk##_##lock(&lock->lk, gtid); \
2888 #define expand3(lk, op) \
2889 static void __kmp_set_##lk##_##lock_flags(kmp_user_lock_p lock, \
2890 kmp_lock_flags_t flags) { \
2891 __kmp_set_##lk##_lock_flags(&lock->lk, flags); \
2893 #define expand4(lk, op) \
2894 static void __kmp_set_##lk##_##lock_location(kmp_user_lock_p lock, \
2895 const ident_t *loc) { \
2896 __kmp_set_##lk##_lock_location(&lock->lk, loc); \
2899 KMP_FOREACH_LOCK_KIND(expand1, init)
2900 KMP_FOREACH_LOCK_KIND(expand1, init_nested)
2901 KMP_FOREACH_LOCK_KIND(expand1, destroy)
2902 KMP_FOREACH_LOCK_KIND(expand1, destroy_nested)
2903 KMP_FOREACH_LOCK_KIND(expand2, acquire)
2904 KMP_FOREACH_LOCK_KIND(expand2, acquire_nested)
2905 KMP_FOREACH_LOCK_KIND(expand2, release)
2906 KMP_FOREACH_LOCK_KIND(expand2, release_nested)
2907 KMP_FOREACH_LOCK_KIND(expand2, test)
2908 KMP_FOREACH_LOCK_KIND(expand2, test_nested)
2909 KMP_FOREACH_LOCK_KIND(expand3, )
2910 KMP_FOREACH_LOCK_KIND(expand4, )
2921 #define expand(l, op) 0, __kmp_init_direct_lock,
2922 void (*__kmp_direct_init[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t) = {
2923 __kmp_init_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, init)};
2927 #define expand(l, op) 0, (void (*)(kmp_dyna_lock_t *))__kmp_##op##_##l##_lock,
2928 static void (*direct_destroy[])(kmp_dyna_lock_t *) = {
2929 __kmp_destroy_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, destroy)};
2931 #define expand(l, op) \
2932 0, (void (*)(kmp_dyna_lock_t *))__kmp_destroy_##l##_lock_with_checks,
2933 static void (*direct_destroy_check[])(kmp_dyna_lock_t *) = {
2934 __kmp_destroy_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, destroy)};
2938 #define expand(l, op) \
2939 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock,
2940 static int (*direct_set[])(kmp_dyna_lock_t *, kmp_int32) = {
2941 __kmp_set_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, acquire)};
2943 #define expand(l, op) \
2944 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock_with_checks,
2945 static int (*direct_set_check[])(kmp_dyna_lock_t *, kmp_int32) = {
2946 __kmp_set_indirect_lock_with_checks, 0,
2947 KMP_FOREACH_D_LOCK(expand, acquire)};
2951 #define expand(l, op) \
2952 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock,
2953 static int (*direct_unset[])(kmp_dyna_lock_t *, kmp_int32) = {
2954 __kmp_unset_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, release)};
2955 static int (*direct_test[])(kmp_dyna_lock_t *, kmp_int32) = {
2956 __kmp_test_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, test)};
2958 #define expand(l, op) \
2959 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock_with_checks,
2960 static int (*direct_unset_check[])(kmp_dyna_lock_t *, kmp_int32) = {
2961 __kmp_unset_indirect_lock_with_checks, 0,
2962 KMP_FOREACH_D_LOCK(expand, release)};
2963 static int (*direct_test_check[])(kmp_dyna_lock_t *, kmp_int32) = {
2964 __kmp_test_indirect_lock_with_checks, 0, KMP_FOREACH_D_LOCK(expand, test)};
2968 void (*(*__kmp_direct_destroy))(kmp_dyna_lock_t *) = 0;
2969 int (*(*__kmp_direct_set))(kmp_dyna_lock_t *, kmp_int32) = 0;
2970 int (*(*__kmp_direct_unset))(kmp_dyna_lock_t *, kmp_int32) = 0;
2971 int (*(*__kmp_direct_test))(kmp_dyna_lock_t *, kmp_int32) = 0;
2974 #define expand(l, op) (void (*)(kmp_user_lock_p)) __kmp_##op##_##l##_##lock,
2975 void (*__kmp_indirect_init[])(kmp_user_lock_p) = {
2976 KMP_FOREACH_I_LOCK(expand, init)};
2979 #define expand(l, op) (void (*)(kmp_user_lock_p)) __kmp_##op##_##l##_##lock,
2980 static void (*indirect_destroy[])(kmp_user_lock_p) = {
2981 KMP_FOREACH_I_LOCK(expand, destroy)};
2983 #define expand(l, op) \
2984 (void (*)(kmp_user_lock_p)) __kmp_##op##_##l##_##lock_with_checks,
2985 static void (*indirect_destroy_check[])(kmp_user_lock_p) = {
2986 KMP_FOREACH_I_LOCK(expand, destroy)};
2990 #define expand(l, op) \
2991 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock,
2992 static int (*indirect_set[])(kmp_user_lock_p,
2993 kmp_int32) = {KMP_FOREACH_I_LOCK(expand, acquire)};
2995 #define expand(l, op) \
2996 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock_with_checks,
2997 static int (*indirect_set_check[])(kmp_user_lock_p, kmp_int32) = {
2998 KMP_FOREACH_I_LOCK(expand, acquire)};
3002 #define expand(l, op) \
3003 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock,
3004 static int (*indirect_unset[])(kmp_user_lock_p, kmp_int32) = {
3005 KMP_FOREACH_I_LOCK(expand, release)};
3006 static int (*indirect_test[])(kmp_user_lock_p,
3007 kmp_int32) = {KMP_FOREACH_I_LOCK(expand, test)};
3009 #define expand(l, op) \
3010 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock_with_checks,
3011 static int (*indirect_unset_check[])(kmp_user_lock_p, kmp_int32) = {
3012 KMP_FOREACH_I_LOCK(expand, release)};
3013 static int (*indirect_test_check[])(kmp_user_lock_p, kmp_int32) = {
3014 KMP_FOREACH_I_LOCK(expand, test)};
3018 void (*(*__kmp_indirect_destroy))(kmp_user_lock_p) = 0;
3019 int (*(*__kmp_indirect_set))(kmp_user_lock_p, kmp_int32) = 0;
3020 int (*(*__kmp_indirect_unset))(kmp_user_lock_p, kmp_int32) = 0;
3021 int (*(*__kmp_indirect_test))(kmp_user_lock_p, kmp_int32) = 0;
3024 kmp_indirect_lock_table_t __kmp_i_lock_table;
3027 static kmp_uint32 __kmp_indirect_lock_size[KMP_NUM_I_LOCKS] = {0};
3030 void (*__kmp_indirect_set_location[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
3032 void (*__kmp_indirect_set_flags[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
3033 kmp_lock_flags_t) = {0};
3034 const ident_t *(*__kmp_indirect_get_location[KMP_NUM_I_LOCKS])(
3035 kmp_user_lock_p) = {0};
3036 kmp_lock_flags_t (*__kmp_indirect_get_flags[KMP_NUM_I_LOCKS])(
3037 kmp_user_lock_p) = {0};
3040 static kmp_indirect_lock_t *__kmp_indirect_lock_pool[KMP_NUM_I_LOCKS] = {0};
3047 kmp_indirect_lock_t *__kmp_allocate_indirect_lock(
void **user_lock,
3049 kmp_indirect_locktag_t tag) {
3050 kmp_indirect_lock_t *lck;
3051 kmp_lock_index_t idx;
3053 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3055 if (__kmp_indirect_lock_pool[tag] != NULL) {
3057 lck = __kmp_indirect_lock_pool[tag];
3058 if (OMP_LOCK_T_SIZE <
sizeof(
void *))
3059 idx = lck->lock->pool.index;
3060 __kmp_indirect_lock_pool[tag] = (kmp_indirect_lock_t *)lck->lock->pool.next;
3061 KA_TRACE(20, (
"__kmp_allocate_indirect_lock: reusing an existing lock %p\n",
3064 idx = __kmp_i_lock_table.next;
3066 if (idx == __kmp_i_lock_table.size) {
3068 int row = __kmp_i_lock_table.size / KMP_I_LOCK_CHUNK;
3069 kmp_indirect_lock_t **new_table = (kmp_indirect_lock_t **)__kmp_allocate(
3070 2 * row *
sizeof(kmp_indirect_lock_t *));
3071 KMP_MEMCPY(new_table, __kmp_i_lock_table.table,
3072 row *
sizeof(kmp_indirect_lock_t *));
3073 kmp_indirect_lock_t **old_table = __kmp_i_lock_table.table;
3074 __kmp_i_lock_table.table = new_table;
3075 __kmp_free(old_table);
3077 for (
int i = row; i < 2 * row; ++i)
3078 *(__kmp_i_lock_table.table + i) = (kmp_indirect_lock_t *)__kmp_allocate(
3079 KMP_I_LOCK_CHUNK *
sizeof(kmp_indirect_lock_t));
3080 __kmp_i_lock_table.size = 2 * idx;
3082 __kmp_i_lock_table.next++;
3083 lck = KMP_GET_I_LOCK(idx);
3085 lck->lock = (kmp_user_lock_p)__kmp_allocate(__kmp_indirect_lock_size[tag]);
3087 (
"__kmp_allocate_indirect_lock: allocated a new lock %p\n", lck));
3090 __kmp_release_lock(&__kmp_global_lock, gtid);
3094 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3095 *((kmp_lock_index_t *)user_lock) = idx
3098 *((kmp_indirect_lock_t **)user_lock) = lck;
3105 static __forceinline kmp_indirect_lock_t *
3106 __kmp_lookup_indirect_lock(
void **user_lock,
const char *func) {
3107 if (__kmp_env_consistency_check) {
3108 kmp_indirect_lock_t *lck = NULL;
3109 if (user_lock == NULL) {
3110 KMP_FATAL(LockIsUninitialized, func);
3112 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3113 kmp_lock_index_t idx = KMP_EXTRACT_I_INDEX(user_lock);
3114 if (idx >= __kmp_i_lock_table.size) {
3115 KMP_FATAL(LockIsUninitialized, func);
3117 lck = KMP_GET_I_LOCK(idx);
3119 lck = *((kmp_indirect_lock_t **)user_lock);
3122 KMP_FATAL(LockIsUninitialized, func);
3126 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3127 return KMP_GET_I_LOCK(KMP_EXTRACT_I_INDEX(user_lock));
3129 return *((kmp_indirect_lock_t **)user_lock);
3134 static void __kmp_init_indirect_lock(kmp_dyna_lock_t *lock,
3135 kmp_dyna_lockseq_t seq) {
3136 #if KMP_USE_ADAPTIVE_LOCKS
3137 if (seq == lockseq_adaptive && !__kmp_cpuinfo.rtm) {
3138 KMP_WARNING(AdaptiveNotSupported,
"kmp_lockseq_t",
"adaptive");
3139 seq = lockseq_queuing;
3143 if (seq == lockseq_rtm && !__kmp_cpuinfo.rtm) {
3144 seq = lockseq_queuing;
3147 kmp_indirect_locktag_t tag = KMP_GET_I_TAG(seq);
3148 kmp_indirect_lock_t *l =
3149 __kmp_allocate_indirect_lock((
void **)lock, __kmp_entry_gtid(), tag);
3150 KMP_I_LOCK_FUNC(l, init)(l->lock);
3152 20, (
"__kmp_init_indirect_lock: initialized indirect lock with type#%d\n",
3156 static void __kmp_destroy_indirect_lock(kmp_dyna_lock_t *lock) {
3157 kmp_uint32 gtid = __kmp_entry_gtid();
3158 kmp_indirect_lock_t *l =
3159 __kmp_lookup_indirect_lock((
void **)lock,
"omp_destroy_lock");
3160 KMP_I_LOCK_FUNC(l, destroy)(l->lock);
3161 kmp_indirect_locktag_t tag = l->type;
3163 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3166 l->lock->pool.next = (kmp_user_lock_p)__kmp_indirect_lock_pool[tag];
3167 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3168 l->lock->pool.index = KMP_EXTRACT_I_INDEX(lock);
3170 __kmp_indirect_lock_pool[tag] = l;
3172 __kmp_release_lock(&__kmp_global_lock, gtid);
3175 static int __kmp_set_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) {
3176 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3177 return KMP_I_LOCK_FUNC(l, set)(l->lock, gtid);
3180 static int __kmp_unset_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) {
3181 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3182 return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3185 static int __kmp_test_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) {
3186 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3187 return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid);
3190 static int __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
3192 kmp_indirect_lock_t *l =
3193 __kmp_lookup_indirect_lock((
void **)lock,
"omp_set_lock");
3194 return KMP_I_LOCK_FUNC(l, set)(l->lock, gtid);
3197 static int __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
3199 kmp_indirect_lock_t *l =
3200 __kmp_lookup_indirect_lock((
void **)lock,
"omp_unset_lock");
3201 return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3204 static int __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
3206 kmp_indirect_lock_t *l =
3207 __kmp_lookup_indirect_lock((
void **)lock,
"omp_test_lock");
3208 return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid);
3211 kmp_dyna_lockseq_t __kmp_user_lock_seq = lockseq_queuing;
3214 kmp_int32 __kmp_get_user_lock_owner(kmp_user_lock_p lck, kmp_uint32 seq) {
3217 case lockseq_nested_tas:
3218 return __kmp_get_tas_lock_owner((kmp_tas_lock_t *)lck);
3221 case lockseq_nested_futex:
3222 return __kmp_get_futex_lock_owner((kmp_futex_lock_t *)lck);
3224 case lockseq_ticket:
3225 case lockseq_nested_ticket:
3226 return __kmp_get_ticket_lock_owner((kmp_ticket_lock_t *)lck);
3227 case lockseq_queuing:
3228 case lockseq_nested_queuing:
3229 #if KMP_USE_ADAPTIVE_LOCKS
3230 case lockseq_adaptive:
3232 return __kmp_get_queuing_lock_owner((kmp_queuing_lock_t *)lck);
3234 case lockseq_nested_drdpa:
3235 return __kmp_get_drdpa_lock_owner((kmp_drdpa_lock_t *)lck);
3242 void __kmp_init_dynamic_user_locks() {
3244 if (__kmp_env_consistency_check) {
3245 __kmp_direct_set = direct_set_check;
3246 __kmp_direct_unset = direct_unset_check;
3247 __kmp_direct_test = direct_test_check;
3248 __kmp_direct_destroy = direct_destroy_check;
3249 __kmp_indirect_set = indirect_set_check;
3250 __kmp_indirect_unset = indirect_unset_check;
3251 __kmp_indirect_test = indirect_test_check;
3252 __kmp_indirect_destroy = indirect_destroy_check;
3254 __kmp_direct_set = direct_set;
3255 __kmp_direct_unset = direct_unset;
3256 __kmp_direct_test = direct_test;
3257 __kmp_direct_destroy = direct_destroy;
3258 __kmp_indirect_set = indirect_set;
3259 __kmp_indirect_unset = indirect_unset;
3260 __kmp_indirect_test = indirect_test;
3261 __kmp_indirect_destroy = indirect_destroy;
3266 if (__kmp_init_user_locks)
3270 __kmp_i_lock_table.size = KMP_I_LOCK_CHUNK;
3271 __kmp_i_lock_table.table =
3272 (kmp_indirect_lock_t **)__kmp_allocate(
sizeof(kmp_indirect_lock_t *));
3273 *(__kmp_i_lock_table.table) = (kmp_indirect_lock_t *)__kmp_allocate(
3274 KMP_I_LOCK_CHUNK *
sizeof(kmp_indirect_lock_t));
3275 __kmp_i_lock_table.next = 0;
3278 __kmp_indirect_lock_size[locktag_ticket] =
sizeof(kmp_ticket_lock_t);
3279 __kmp_indirect_lock_size[locktag_queuing] =
sizeof(kmp_queuing_lock_t);
3280 #if KMP_USE_ADAPTIVE_LOCKS
3281 __kmp_indirect_lock_size[locktag_adaptive] =
sizeof(kmp_adaptive_lock_t);
3283 __kmp_indirect_lock_size[locktag_drdpa] =
sizeof(kmp_drdpa_lock_t);
3285 __kmp_indirect_lock_size[locktag_rtm] =
sizeof(kmp_queuing_lock_t);
3287 __kmp_indirect_lock_size[locktag_nested_tas] =
sizeof(kmp_tas_lock_t);
3289 __kmp_indirect_lock_size[locktag_nested_futex] =
sizeof(kmp_futex_lock_t);
3291 __kmp_indirect_lock_size[locktag_nested_ticket] =
sizeof(kmp_ticket_lock_t);
3292 __kmp_indirect_lock_size[locktag_nested_queuing] =
sizeof(kmp_queuing_lock_t);
3293 __kmp_indirect_lock_size[locktag_nested_drdpa] =
sizeof(kmp_drdpa_lock_t);
3296 #define fill_jumps(table, expand, sep) \
3298 table[locktag##sep##ticket] = expand(ticket); \
3299 table[locktag##sep##queuing] = expand(queuing); \
3300 table[locktag##sep##drdpa] = expand(drdpa); \
3303 #if KMP_USE_ADAPTIVE_LOCKS
3304 #define fill_table(table, expand) \
3306 fill_jumps(table, expand, _); \
3307 table[locktag_adaptive] = expand(queuing); \
3308 fill_jumps(table, expand, _nested_); \
3311 #define fill_table(table, expand) \
3313 fill_jumps(table, expand, _); \
3314 fill_jumps(table, expand, _nested_); \
3316 #endif // KMP_USE_ADAPTIVE_LOCKS
3319 (void (*)(kmp_user_lock_p, const ident_t *)) __kmp_set_##l##_lock_location
3320 fill_table(__kmp_indirect_set_location, expand);
3323 (void (*)(kmp_user_lock_p, kmp_lock_flags_t)) __kmp_set_##l##_lock_flags
3324 fill_table(__kmp_indirect_set_flags, expand);
3327 (const ident_t *(*)(kmp_user_lock_p)) __kmp_get_##l##_lock_location
3328 fill_table(__kmp_indirect_get_location, expand);
3331 (kmp_lock_flags_t(*)(kmp_user_lock_p)) __kmp_get_##l##_lock_flags
3332 fill_table(__kmp_indirect_get_flags, expand);
3335 __kmp_init_user_locks = TRUE;
3339 void __kmp_cleanup_indirect_user_locks() {
3345 for (k = 0; k < KMP_NUM_I_LOCKS; ++k) {
3346 kmp_indirect_lock_t *l = __kmp_indirect_lock_pool[k];
3348 kmp_indirect_lock_t *ll = l;
3349 l = (kmp_indirect_lock_t *)l->lock->pool.next;
3350 KA_TRACE(20, (
"__kmp_cleanup_indirect_user_locks: freeing %p from pool\n",
3352 __kmp_free(ll->lock);
3355 __kmp_indirect_lock_pool[k] = NULL;
3358 for (i = 0; i < __kmp_i_lock_table.next; i++) {
3359 kmp_indirect_lock_t *l = KMP_GET_I_LOCK(i);
3360 if (l->lock != NULL) {
3362 KMP_I_LOCK_FUNC(l, destroy)(l->lock);
3365 (
"__kmp_cleanup_indirect_user_locks: destroy/freeing %p from table\n",
3367 __kmp_free(l->lock);
3371 for (i = 0; i < __kmp_i_lock_table.size / KMP_I_LOCK_CHUNK; i++)
3372 __kmp_free(__kmp_i_lock_table.table[i]);
3373 __kmp_free(__kmp_i_lock_table.table);
3375 __kmp_init_user_locks = FALSE;
3378 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3379 int __kmp_num_locks_in_block = 1;
3381 #else // KMP_USE_DYNAMIC_LOCK
3383 static void __kmp_init_tas_lock_with_checks(kmp_tas_lock_t *lck) {
3384 __kmp_init_tas_lock(lck);
3387 static void __kmp_init_nested_tas_lock_with_checks(kmp_tas_lock_t *lck) {
3388 __kmp_init_nested_tas_lock(lck);
3392 static void __kmp_init_futex_lock_with_checks(kmp_futex_lock_t *lck) {
3393 __kmp_init_futex_lock(lck);
3396 static void __kmp_init_nested_futex_lock_with_checks(kmp_futex_lock_t *lck) {
3397 __kmp_init_nested_futex_lock(lck);
3401 static int __kmp_is_ticket_lock_initialized(kmp_ticket_lock_t *lck) {
3402 return lck == lck->lk.self;
3405 static void __kmp_init_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
3406 __kmp_init_ticket_lock(lck);
3409 static void __kmp_init_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
3410 __kmp_init_nested_ticket_lock(lck);
3413 static int __kmp_is_queuing_lock_initialized(kmp_queuing_lock_t *lck) {
3414 return lck == lck->lk.initialized;
3417 static void __kmp_init_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
3418 __kmp_init_queuing_lock(lck);
3422 __kmp_init_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
3423 __kmp_init_nested_queuing_lock(lck);
3426 #if KMP_USE_ADAPTIVE_LOCKS
3427 static void __kmp_init_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck) {
3428 __kmp_init_adaptive_lock(lck);
3432 static int __kmp_is_drdpa_lock_initialized(kmp_drdpa_lock_t *lck) {
3433 return lck == lck->lk.initialized;
3436 static void __kmp_init_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
3437 __kmp_init_drdpa_lock(lck);
3440 static void __kmp_init_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
3441 __kmp_init_nested_drdpa_lock(lck);
3448 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3450 size_t __kmp_base_user_lock_size = 0;
3451 size_t __kmp_user_lock_size = 0;
3453 kmp_int32 (*__kmp_get_user_lock_owner_)(kmp_user_lock_p lck) = NULL;
3454 int (*__kmp_acquire_user_lock_with_checks_)(kmp_user_lock_p lck,
3455 kmp_int32 gtid) = NULL;
3457 int (*__kmp_test_user_lock_with_checks_)(kmp_user_lock_p lck,
3458 kmp_int32 gtid) = NULL;
3459 int (*__kmp_release_user_lock_with_checks_)(kmp_user_lock_p lck,
3460 kmp_int32 gtid) = NULL;
3461 void (*__kmp_init_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3462 void (*__kmp_destroy_user_lock_)(kmp_user_lock_p lck) = NULL;
3463 void (*__kmp_destroy_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3464 int (*__kmp_acquire_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3465 kmp_int32 gtid) = NULL;
3467 int (*__kmp_test_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3468 kmp_int32 gtid) = NULL;
3469 int (*__kmp_release_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3470 kmp_int32 gtid) = NULL;
3471 void (*__kmp_init_nested_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3472 void (*__kmp_destroy_nested_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3474 int (*__kmp_is_user_lock_initialized_)(kmp_user_lock_p lck) = NULL;
3475 const ident_t *(*__kmp_get_user_lock_location_)(kmp_user_lock_p lck) = NULL;
3476 void (*__kmp_set_user_lock_location_)(kmp_user_lock_p lck,
3478 kmp_lock_flags_t (*__kmp_get_user_lock_flags_)(kmp_user_lock_p lck) = NULL;
3479 void (*__kmp_set_user_lock_flags_)(kmp_user_lock_p lck,
3480 kmp_lock_flags_t flags) = NULL;
3482 void __kmp_set_user_lock_vptrs(kmp_lock_kind_t user_lock_kind) {
3483 switch (user_lock_kind) {
3489 __kmp_base_user_lock_size =
sizeof(kmp_base_tas_lock_t);
3490 __kmp_user_lock_size =
sizeof(kmp_tas_lock_t);
3492 __kmp_get_user_lock_owner_ =
3493 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_tas_lock_owner);
3495 if (__kmp_env_consistency_check) {
3496 KMP_BIND_USER_LOCK_WITH_CHECKS(tas);
3497 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(tas);
3499 KMP_BIND_USER_LOCK(tas);
3500 KMP_BIND_NESTED_USER_LOCK(tas);
3503 __kmp_destroy_user_lock_ =
3504 (void (*)(kmp_user_lock_p))(&__kmp_destroy_tas_lock);
3506 __kmp_is_user_lock_initialized_ = (int (*)(kmp_user_lock_p))NULL;
3508 __kmp_get_user_lock_location_ = (
const ident_t *(*)(kmp_user_lock_p))NULL;
3510 __kmp_set_user_lock_location_ =
3511 (void (*)(kmp_user_lock_p,
const ident_t *))NULL;
3513 __kmp_get_user_lock_flags_ = (kmp_lock_flags_t(*)(kmp_user_lock_p))NULL;
3515 __kmp_set_user_lock_flags_ =
3516 (void (*)(kmp_user_lock_p, kmp_lock_flags_t))NULL;
3522 __kmp_base_user_lock_size =
sizeof(kmp_base_futex_lock_t);
3523 __kmp_user_lock_size =
sizeof(kmp_futex_lock_t);
3525 __kmp_get_user_lock_owner_ =
3526 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_futex_lock_owner);
3528 if (__kmp_env_consistency_check) {
3529 KMP_BIND_USER_LOCK_WITH_CHECKS(futex);
3530 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(futex);
3532 KMP_BIND_USER_LOCK(futex);
3533 KMP_BIND_NESTED_USER_LOCK(futex);
3536 __kmp_destroy_user_lock_ =
3537 (void (*)(kmp_user_lock_p))(&__kmp_destroy_futex_lock);
3539 __kmp_is_user_lock_initialized_ = (int (*)(kmp_user_lock_p))NULL;
3541 __kmp_get_user_lock_location_ = (
const ident_t *(*)(kmp_user_lock_p))NULL;
3543 __kmp_set_user_lock_location_ =
3544 (void (*)(kmp_user_lock_p,
const ident_t *))NULL;
3546 __kmp_get_user_lock_flags_ = (kmp_lock_flags_t(*)(kmp_user_lock_p))NULL;
3548 __kmp_set_user_lock_flags_ =
3549 (void (*)(kmp_user_lock_p, kmp_lock_flags_t))NULL;
3552 #endif // KMP_USE_FUTEX
3555 __kmp_base_user_lock_size =
sizeof(kmp_base_ticket_lock_t);
3556 __kmp_user_lock_size =
sizeof(kmp_ticket_lock_t);
3558 __kmp_get_user_lock_owner_ =
3559 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_ticket_lock_owner);
3561 if (__kmp_env_consistency_check) {
3562 KMP_BIND_USER_LOCK_WITH_CHECKS(ticket);
3563 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(ticket);
3565 KMP_BIND_USER_LOCK(ticket);
3566 KMP_BIND_NESTED_USER_LOCK(ticket);
3569 __kmp_destroy_user_lock_ =
3570 (void (*)(kmp_user_lock_p))(&__kmp_destroy_ticket_lock);
3572 __kmp_is_user_lock_initialized_ =
3573 (int (*)(kmp_user_lock_p))(&__kmp_is_ticket_lock_initialized);
3575 __kmp_get_user_lock_location_ =
3576 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_ticket_lock_location);
3578 __kmp_set_user_lock_location_ = (void (*)(
3579 kmp_user_lock_p,
const ident_t *))(&__kmp_set_ticket_lock_location);
3581 __kmp_get_user_lock_flags_ =
3582 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_ticket_lock_flags);
3584 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3585 &__kmp_set_ticket_lock_flags);
3589 __kmp_base_user_lock_size =
sizeof(kmp_base_queuing_lock_t);
3590 __kmp_user_lock_size =
sizeof(kmp_queuing_lock_t);
3592 __kmp_get_user_lock_owner_ =
3593 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_owner);
3595 if (__kmp_env_consistency_check) {
3596 KMP_BIND_USER_LOCK_WITH_CHECKS(queuing);
3597 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(queuing);
3599 KMP_BIND_USER_LOCK(queuing);
3600 KMP_BIND_NESTED_USER_LOCK(queuing);
3603 __kmp_destroy_user_lock_ =
3604 (void (*)(kmp_user_lock_p))(&__kmp_destroy_queuing_lock);
3606 __kmp_is_user_lock_initialized_ =
3607 (int (*)(kmp_user_lock_p))(&__kmp_is_queuing_lock_initialized);
3609 __kmp_get_user_lock_location_ =
3610 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_location);
3612 __kmp_set_user_lock_location_ = (void (*)(
3613 kmp_user_lock_p,
const ident_t *))(&__kmp_set_queuing_lock_location);
3615 __kmp_get_user_lock_flags_ =
3616 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_flags);
3618 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3619 &__kmp_set_queuing_lock_flags);
3622 #if KMP_USE_ADAPTIVE_LOCKS
3624 __kmp_base_user_lock_size =
sizeof(kmp_base_adaptive_lock_t);
3625 __kmp_user_lock_size =
sizeof(kmp_adaptive_lock_t);
3627 __kmp_get_user_lock_owner_ =
3628 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_owner);
3630 if (__kmp_env_consistency_check) {
3631 KMP_BIND_USER_LOCK_WITH_CHECKS(adaptive);
3633 KMP_BIND_USER_LOCK(adaptive);
3636 __kmp_destroy_user_lock_ =
3637 (void (*)(kmp_user_lock_p))(&__kmp_destroy_adaptive_lock);
3639 __kmp_is_user_lock_initialized_ =
3640 (int (*)(kmp_user_lock_p))(&__kmp_is_queuing_lock_initialized);
3642 __kmp_get_user_lock_location_ =
3643 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_location);
3645 __kmp_set_user_lock_location_ = (void (*)(
3646 kmp_user_lock_p,
const ident_t *))(&__kmp_set_queuing_lock_location);
3648 __kmp_get_user_lock_flags_ =
3649 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_flags);
3651 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3652 &__kmp_set_queuing_lock_flags);
3655 #endif // KMP_USE_ADAPTIVE_LOCKS
3658 __kmp_base_user_lock_size =
sizeof(kmp_base_drdpa_lock_t);
3659 __kmp_user_lock_size =
sizeof(kmp_drdpa_lock_t);
3661 __kmp_get_user_lock_owner_ =
3662 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_drdpa_lock_owner);
3664 if (__kmp_env_consistency_check) {
3665 KMP_BIND_USER_LOCK_WITH_CHECKS(drdpa);
3666 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(drdpa);
3668 KMP_BIND_USER_LOCK(drdpa);
3669 KMP_BIND_NESTED_USER_LOCK(drdpa);
3672 __kmp_destroy_user_lock_ =
3673 (void (*)(kmp_user_lock_p))(&__kmp_destroy_drdpa_lock);
3675 __kmp_is_user_lock_initialized_ =
3676 (int (*)(kmp_user_lock_p))(&__kmp_is_drdpa_lock_initialized);
3678 __kmp_get_user_lock_location_ =
3679 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_drdpa_lock_location);
3681 __kmp_set_user_lock_location_ = (void (*)(
3682 kmp_user_lock_p,
const ident_t *))(&__kmp_set_drdpa_lock_location);
3684 __kmp_get_user_lock_flags_ =
3685 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_drdpa_lock_flags);
3687 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3688 &__kmp_set_drdpa_lock_flags);
3696 kmp_lock_table_t __kmp_user_lock_table = {1, 0, NULL};
3697 kmp_user_lock_p __kmp_lock_pool = NULL;
3700 kmp_block_of_locks *__kmp_lock_blocks = NULL;
3701 int __kmp_num_locks_in_block = 1;
3703 static kmp_lock_index_t __kmp_lock_table_insert(kmp_user_lock_p lck) {
3705 kmp_lock_index_t index;
3706 if (__kmp_user_lock_table.used >= __kmp_user_lock_table.allocated) {
3707 kmp_lock_index_t size;
3708 kmp_user_lock_p *table;
3710 if (__kmp_user_lock_table.allocated == 0) {
3713 size = __kmp_user_lock_table.allocated * 2;
3715 table = (kmp_user_lock_p *)__kmp_allocate(
sizeof(kmp_user_lock_p) * size);
3716 KMP_MEMCPY(table + 1, __kmp_user_lock_table.table + 1,
3717 sizeof(kmp_user_lock_p) * (__kmp_user_lock_table.used - 1));
3718 table[0] = (kmp_user_lock_p)__kmp_user_lock_table.table;
3723 __kmp_user_lock_table.table = table;
3724 __kmp_user_lock_table.allocated = size;
3726 KMP_DEBUG_ASSERT(__kmp_user_lock_table.used <
3727 __kmp_user_lock_table.allocated);
3728 index = __kmp_user_lock_table.used;
3729 __kmp_user_lock_table.table[index] = lck;
3730 ++__kmp_user_lock_table.used;
3734 static kmp_user_lock_p __kmp_lock_block_allocate() {
3736 static int last_index = 0;
3737 if ((last_index >= __kmp_num_locks_in_block) || (__kmp_lock_blocks == NULL)) {
3741 KMP_DEBUG_ASSERT(__kmp_user_lock_size > 0);
3742 size_t space_for_locks = __kmp_user_lock_size * __kmp_num_locks_in_block;
3744 (
char *)__kmp_allocate(space_for_locks +
sizeof(kmp_block_of_locks));
3746 kmp_block_of_locks *new_block =
3747 (kmp_block_of_locks *)(&buffer[space_for_locks]);
3748 new_block->next_block = __kmp_lock_blocks;
3749 new_block->locks = (
void *)buffer;
3752 __kmp_lock_blocks = new_block;
3754 kmp_user_lock_p ret = (kmp_user_lock_p)(&(
3755 ((
char *)(__kmp_lock_blocks->locks))[last_index * __kmp_user_lock_size]));
3762 kmp_user_lock_p __kmp_user_lock_allocate(
void **user_lock, kmp_int32 gtid,
3763 kmp_lock_flags_t flags) {
3764 kmp_user_lock_p lck;
3765 kmp_lock_index_t index;
3766 KMP_DEBUG_ASSERT(user_lock);
3768 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3770 if (__kmp_lock_pool == NULL) {
3775 ANNOTATE_IGNORE_WRITES_BEGIN();
3776 if (__kmp_num_locks_in_block <= 1) {
3777 lck = (kmp_user_lock_p)__kmp_allocate(__kmp_user_lock_size);
3779 lck = __kmp_lock_block_allocate();
3781 ANNOTATE_IGNORE_WRITES_END();
3785 index = __kmp_lock_table_insert(lck);
3788 lck = __kmp_lock_pool;
3789 index = __kmp_lock_pool->pool.index;
3790 __kmp_lock_pool = __kmp_lock_pool->pool.next;
3795 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3796 *((kmp_lock_index_t *)user_lock) = index;
3798 *((kmp_user_lock_p *)user_lock) = lck;
3802 __kmp_set_user_lock_flags(lck, flags);
3804 __kmp_release_lock(&__kmp_global_lock, gtid);
3810 void __kmp_user_lock_free(
void **user_lock, kmp_int32 gtid,
3811 kmp_user_lock_p lck) {
3812 KMP_DEBUG_ASSERT(user_lock != NULL);
3813 KMP_DEBUG_ASSERT(lck != NULL);
3815 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3817 lck->pool.next = __kmp_lock_pool;
3818 __kmp_lock_pool = lck;
3819 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3820 kmp_lock_index_t index = *((kmp_lock_index_t *)user_lock);
3821 KMP_DEBUG_ASSERT(0 < index && index <= __kmp_user_lock_table.used);
3822 lck->pool.index = index;
3825 __kmp_release_lock(&__kmp_global_lock, gtid);
3828 kmp_user_lock_p __kmp_lookup_user_lock(
void **user_lock,
char const *func) {
3829 kmp_user_lock_p lck = NULL;
3831 if (__kmp_env_consistency_check) {
3832 if (user_lock == NULL) {
3833 KMP_FATAL(LockIsUninitialized, func);
3837 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3838 kmp_lock_index_t index = *((kmp_lock_index_t *)user_lock);
3839 if (__kmp_env_consistency_check) {
3840 if (!(0 < index && index < __kmp_user_lock_table.used)) {
3841 KMP_FATAL(LockIsUninitialized, func);
3844 KMP_DEBUG_ASSERT(0 < index && index < __kmp_user_lock_table.used);
3845 KMP_DEBUG_ASSERT(__kmp_user_lock_size > 0);
3846 lck = __kmp_user_lock_table.table[index];
3848 lck = *((kmp_user_lock_p *)user_lock);
3851 if (__kmp_env_consistency_check) {
3853 KMP_FATAL(LockIsUninitialized, func);
3860 void __kmp_cleanup_user_locks(
void) {
3863 __kmp_lock_pool = NULL;
3865 #define IS_CRITICAL(lck) \
3866 ((__kmp_get_user_lock_flags_ != NULL) && \
3867 ((*__kmp_get_user_lock_flags_)(lck)&kmp_lf_critical_section))
3892 while (__kmp_user_lock_table.used > 1) {
3897 kmp_user_lock_p lck =
3898 __kmp_user_lock_table.table[--__kmp_user_lock_table.used];
3900 if ((__kmp_is_user_lock_initialized_ != NULL) &&
3901 (*__kmp_is_user_lock_initialized_)(lck)) {
3905 if (__kmp_env_consistency_check && (!IS_CRITICAL(lck)) &&
3906 ((loc = __kmp_get_user_lock_location(lck)) != NULL) &&
3908 kmp_str_loc_t str_loc = __kmp_str_loc_init(loc->
psource, 0);
3909 KMP_WARNING(CnsLockNotDestroyed, str_loc.file, str_loc.line);
3910 __kmp_str_loc_free(&str_loc);
3914 if (IS_CRITICAL(lck)) {
3917 (
"__kmp_cleanup_user_locks: free critical section lock %p (%p)\n",
3918 lck, *(
void **)lck));
3920 KA_TRACE(20, (
"__kmp_cleanup_user_locks: free lock %p (%p)\n", lck,
3926 __kmp_destroy_user_lock(lck);
3930 if (__kmp_lock_blocks == NULL) {
3938 kmp_user_lock_p *table_ptr = __kmp_user_lock_table.table;
3939 __kmp_user_lock_table.table = NULL;
3940 __kmp_user_lock_table.allocated = 0;
3942 while (table_ptr != NULL) {
3945 kmp_user_lock_p *next = (kmp_user_lock_p *)(table_ptr[0]);
3946 __kmp_free(table_ptr);
3951 kmp_block_of_locks_t *block_ptr = __kmp_lock_blocks;
3952 __kmp_lock_blocks = NULL;
3954 while (block_ptr != NULL) {
3955 kmp_block_of_locks_t *next = block_ptr->next_block;
3956 __kmp_free(block_ptr->locks);
3961 TCW_4(__kmp_init_user_locks, FALSE);
3964 #endif // KMP_USE_DYNAMIC_LOCK