21 #include "kmp_wait_release.h"
22 #include "kmp_wrapper_getpid.h"
25 #include <sys/syscall.h>
43 void __kmp_validate_locks(
void) {
48 x = ~((kmp_uint32)0) - 2;
51 for (i = 0; i < 8; ++i, ++x, ++y) {
52 kmp_uint32 z = (x - y);
56 KMP_ASSERT(offsetof(kmp_base_queuing_lock, tail_id) % 8 == 0);
70 static kmp_int32 __kmp_get_tas_lock_owner(kmp_tas_lock_t *lck) {
71 return KMP_LOCK_STRIP(KMP_ATOMIC_LD_RLX(&lck->lk.poll)) - 1;
74 static inline bool __kmp_is_tas_lock_nestable(kmp_tas_lock_t *lck) {
75 return lck->lk.depth_locked != -1;
78 __forceinline
static int
79 __kmp_acquire_tas_lock_timed_template(kmp_tas_lock_t *lck, kmp_int32 gtid) {
82 #ifdef USE_LOCK_PROFILE
83 kmp_uint32 curr = KMP_LOCK_STRIP(lck->lk.poll);
84 if ((curr != 0) && (curr != gtid + 1))
85 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
89 kmp_int32 tas_free = KMP_LOCK_FREE(tas);
90 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas);
92 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == tas_free &&
93 __kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy)) {
94 KMP_FSYNC_ACQUIRED(lck);
95 return KMP_LOCK_ACQUIRED_FIRST;
100 KMP_FSYNC_PREPARE(lck);
101 KMP_INIT_YIELD(spins);
102 KMP_INIT_BACKOFF(time);
103 kmp_backoff_t backoff = __kmp_spin_backoff_params;
106 __kmp_spin_backoff(&backoff);
108 if (!__kmp_tpause_enabled)
109 __kmp_spin_backoff(&backoff);
111 KMP_YIELD_OVERSUB_ELSE_SPIN(spins, time);
112 }
while (KMP_ATOMIC_LD_RLX(&lck->lk.poll) != tas_free ||
113 !__kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy));
114 KMP_FSYNC_ACQUIRED(lck);
115 return KMP_LOCK_ACQUIRED_FIRST;
118 int __kmp_acquire_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
119 int retval = __kmp_acquire_tas_lock_timed_template(lck, gtid);
123 static int __kmp_acquire_tas_lock_with_checks(kmp_tas_lock_t *lck,
125 char const *
const func =
"omp_set_lock";
126 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
127 __kmp_is_tas_lock_nestable(lck)) {
128 KMP_FATAL(LockNestableUsedAsSimple, func);
130 if ((gtid >= 0) && (__kmp_get_tas_lock_owner(lck) == gtid)) {
131 KMP_FATAL(LockIsAlreadyOwned, func);
133 return __kmp_acquire_tas_lock(lck, gtid);
136 int __kmp_test_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
137 kmp_int32 tas_free = KMP_LOCK_FREE(tas);
138 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas);
139 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == tas_free &&
140 __kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy)) {
141 KMP_FSYNC_ACQUIRED(lck);
147 static int __kmp_test_tas_lock_with_checks(kmp_tas_lock_t *lck,
149 char const *
const func =
"omp_test_lock";
150 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
151 __kmp_is_tas_lock_nestable(lck)) {
152 KMP_FATAL(LockNestableUsedAsSimple, func);
154 return __kmp_test_tas_lock(lck, gtid);
157 int __kmp_release_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
160 KMP_FSYNC_RELEASING(lck);
161 KMP_ATOMIC_ST_REL(&lck->lk.poll, KMP_LOCK_FREE(tas));
165 return KMP_LOCK_RELEASED;
168 static int __kmp_release_tas_lock_with_checks(kmp_tas_lock_t *lck,
170 char const *
const func =
"omp_unset_lock";
172 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
173 __kmp_is_tas_lock_nestable(lck)) {
174 KMP_FATAL(LockNestableUsedAsSimple, func);
176 if (__kmp_get_tas_lock_owner(lck) == -1) {
177 KMP_FATAL(LockUnsettingFree, func);
179 if ((gtid >= 0) && (__kmp_get_tas_lock_owner(lck) >= 0) &&
180 (__kmp_get_tas_lock_owner(lck) != gtid)) {
181 KMP_FATAL(LockUnsettingSetByAnother, func);
183 return __kmp_release_tas_lock(lck, gtid);
186 void __kmp_init_tas_lock(kmp_tas_lock_t *lck) {
187 lck->lk.poll = KMP_LOCK_FREE(tas);
190 void __kmp_destroy_tas_lock(kmp_tas_lock_t *lck) { lck->lk.poll = 0; }
192 static void __kmp_destroy_tas_lock_with_checks(kmp_tas_lock_t *lck) {
193 char const *
const func =
"omp_destroy_lock";
194 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
195 __kmp_is_tas_lock_nestable(lck)) {
196 KMP_FATAL(LockNestableUsedAsSimple, func);
198 if (__kmp_get_tas_lock_owner(lck) != -1) {
199 KMP_FATAL(LockStillOwned, func);
201 __kmp_destroy_tas_lock(lck);
206 int __kmp_acquire_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
207 KMP_DEBUG_ASSERT(gtid >= 0);
209 if (__kmp_get_tas_lock_owner(lck) == gtid) {
210 lck->lk.depth_locked += 1;
211 return KMP_LOCK_ACQUIRED_NEXT;
213 __kmp_acquire_tas_lock_timed_template(lck, gtid);
214 lck->lk.depth_locked = 1;
215 return KMP_LOCK_ACQUIRED_FIRST;
219 static int __kmp_acquire_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
221 char const *
const func =
"omp_set_nest_lock";
222 if (!__kmp_is_tas_lock_nestable(lck)) {
223 KMP_FATAL(LockSimpleUsedAsNestable, func);
225 return __kmp_acquire_nested_tas_lock(lck, gtid);
228 int __kmp_test_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
231 KMP_DEBUG_ASSERT(gtid >= 0);
233 if (__kmp_get_tas_lock_owner(lck) == gtid) {
234 retval = ++lck->lk.depth_locked;
235 }
else if (!__kmp_test_tas_lock(lck, gtid)) {
239 retval = lck->lk.depth_locked = 1;
244 static int __kmp_test_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
246 char const *
const func =
"omp_test_nest_lock";
247 if (!__kmp_is_tas_lock_nestable(lck)) {
248 KMP_FATAL(LockSimpleUsedAsNestable, func);
250 return __kmp_test_nested_tas_lock(lck, gtid);
253 int __kmp_release_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
254 KMP_DEBUG_ASSERT(gtid >= 0);
257 if (--(lck->lk.depth_locked) == 0) {
258 __kmp_release_tas_lock(lck, gtid);
259 return KMP_LOCK_RELEASED;
261 return KMP_LOCK_STILL_HELD;
264 static int __kmp_release_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
266 char const *
const func =
"omp_unset_nest_lock";
268 if (!__kmp_is_tas_lock_nestable(lck)) {
269 KMP_FATAL(LockSimpleUsedAsNestable, func);
271 if (__kmp_get_tas_lock_owner(lck) == -1) {
272 KMP_FATAL(LockUnsettingFree, func);
274 if (__kmp_get_tas_lock_owner(lck) != gtid) {
275 KMP_FATAL(LockUnsettingSetByAnother, func);
277 return __kmp_release_nested_tas_lock(lck, gtid);
280 void __kmp_init_nested_tas_lock(kmp_tas_lock_t *lck) {
281 __kmp_init_tas_lock(lck);
282 lck->lk.depth_locked = 0;
285 void __kmp_destroy_nested_tas_lock(kmp_tas_lock_t *lck) {
286 __kmp_destroy_tas_lock(lck);
287 lck->lk.depth_locked = 0;
290 static void __kmp_destroy_nested_tas_lock_with_checks(kmp_tas_lock_t *lck) {
291 char const *
const func =
"omp_destroy_nest_lock";
292 if (!__kmp_is_tas_lock_nestable(lck)) {
293 KMP_FATAL(LockSimpleUsedAsNestable, func);
295 if (__kmp_get_tas_lock_owner(lck) != -1) {
296 KMP_FATAL(LockStillOwned, func);
298 __kmp_destroy_nested_tas_lock(lck);
311 static kmp_int32 __kmp_get_futex_lock_owner(kmp_futex_lock_t *lck) {
312 return KMP_LOCK_STRIP((TCR_4(lck->lk.poll) >> 1)) - 1;
315 static inline bool __kmp_is_futex_lock_nestable(kmp_futex_lock_t *lck) {
316 return lck->lk.depth_locked != -1;
319 __forceinline
static int
320 __kmp_acquire_futex_lock_timed_template(kmp_futex_lock_t *lck, kmp_int32 gtid) {
321 kmp_int32 gtid_code = (gtid + 1) << 1;
325 #ifdef USE_LOCK_PROFILE
326 kmp_uint32 curr = KMP_LOCK_STRIP(TCR_4(lck->lk.poll));
327 if ((curr != 0) && (curr != gtid_code))
328 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
332 KMP_FSYNC_PREPARE(lck);
333 KA_TRACE(1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d entering\n",
334 lck, lck->lk.poll, gtid));
338 while ((poll_val = KMP_COMPARE_AND_STORE_RET32(
339 &(lck->lk.poll), KMP_LOCK_FREE(futex),
340 KMP_LOCK_BUSY(gtid_code, futex))) != KMP_LOCK_FREE(futex)) {
342 kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1;
345 (
"__kmp_acquire_futex_lock: lck:%p, T#%d poll_val = 0x%x cond = 0x%x\n",
346 lck, gtid, poll_val, cond));
357 if (!KMP_COMPARE_AND_STORE_REL32(&(lck->lk.poll), poll_val,
358 poll_val | KMP_LOCK_BUSY(1, futex))) {
361 (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d can't set bit 0\n",
362 lck, lck->lk.poll, gtid));
365 poll_val |= KMP_LOCK_BUSY(1, futex);
368 (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d bit 0 set\n", lck,
369 lck->lk.poll, gtid));
374 (
"__kmp_acquire_futex_lock: lck:%p, T#%d before futex_wait(0x%x)\n",
375 lck, gtid, poll_val));
378 if ((rc = syscall(__NR_futex, &(lck->lk.poll), FUTEX_WAIT, poll_val, NULL,
380 KA_TRACE(1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d futex_wait(0x%x) "
381 "failed (rc=%ld errno=%d)\n",
382 lck, gtid, poll_val, rc, errno));
387 (
"__kmp_acquire_futex_lock: lck:%p, T#%d after futex_wait(0x%x)\n",
388 lck, gtid, poll_val));
395 KMP_FSYNC_ACQUIRED(lck);
396 KA_TRACE(1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d exiting\n", lck,
397 lck->lk.poll, gtid));
398 return KMP_LOCK_ACQUIRED_FIRST;
401 int __kmp_acquire_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
402 int retval = __kmp_acquire_futex_lock_timed_template(lck, gtid);
406 static int __kmp_acquire_futex_lock_with_checks(kmp_futex_lock_t *lck,
408 char const *
const func =
"omp_set_lock";
409 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
410 __kmp_is_futex_lock_nestable(lck)) {
411 KMP_FATAL(LockNestableUsedAsSimple, func);
413 if ((gtid >= 0) && (__kmp_get_futex_lock_owner(lck) == gtid)) {
414 KMP_FATAL(LockIsAlreadyOwned, func);
416 return __kmp_acquire_futex_lock(lck, gtid);
419 int __kmp_test_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
420 if (KMP_COMPARE_AND_STORE_ACQ32(&(lck->lk.poll), KMP_LOCK_FREE(futex),
421 KMP_LOCK_BUSY((gtid + 1) << 1, futex))) {
422 KMP_FSYNC_ACQUIRED(lck);
428 static int __kmp_test_futex_lock_with_checks(kmp_futex_lock_t *lck,
430 char const *
const func =
"omp_test_lock";
431 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
432 __kmp_is_futex_lock_nestable(lck)) {
433 KMP_FATAL(LockNestableUsedAsSimple, func);
435 return __kmp_test_futex_lock(lck, gtid);
438 int __kmp_release_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
441 KA_TRACE(1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d entering\n",
442 lck, lck->lk.poll, gtid));
444 KMP_FSYNC_RELEASING(lck);
446 kmp_int32 poll_val = KMP_XCHG_FIXED32(&(lck->lk.poll), KMP_LOCK_FREE(futex));
449 (
"__kmp_release_futex_lock: lck:%p, T#%d released poll_val = 0x%x\n",
450 lck, gtid, poll_val));
452 if (KMP_LOCK_STRIP(poll_val) & 1) {
454 (
"__kmp_release_futex_lock: lck:%p, T#%d futex_wake 1 thread\n",
456 syscall(__NR_futex, &(lck->lk.poll), FUTEX_WAKE, KMP_LOCK_BUSY(1, futex),
462 KA_TRACE(1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d exiting\n", lck,
463 lck->lk.poll, gtid));
466 return KMP_LOCK_RELEASED;
469 static int __kmp_release_futex_lock_with_checks(kmp_futex_lock_t *lck,
471 char const *
const func =
"omp_unset_lock";
473 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
474 __kmp_is_futex_lock_nestable(lck)) {
475 KMP_FATAL(LockNestableUsedAsSimple, func);
477 if (__kmp_get_futex_lock_owner(lck) == -1) {
478 KMP_FATAL(LockUnsettingFree, func);
480 if ((gtid >= 0) && (__kmp_get_futex_lock_owner(lck) >= 0) &&
481 (__kmp_get_futex_lock_owner(lck) != gtid)) {
482 KMP_FATAL(LockUnsettingSetByAnother, func);
484 return __kmp_release_futex_lock(lck, gtid);
487 void __kmp_init_futex_lock(kmp_futex_lock_t *lck) {
488 TCW_4(lck->lk.poll, KMP_LOCK_FREE(futex));
491 void __kmp_destroy_futex_lock(kmp_futex_lock_t *lck) { lck->lk.poll = 0; }
493 static void __kmp_destroy_futex_lock_with_checks(kmp_futex_lock_t *lck) {
494 char const *
const func =
"omp_destroy_lock";
495 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
496 __kmp_is_futex_lock_nestable(lck)) {
497 KMP_FATAL(LockNestableUsedAsSimple, func);
499 if (__kmp_get_futex_lock_owner(lck) != -1) {
500 KMP_FATAL(LockStillOwned, func);
502 __kmp_destroy_futex_lock(lck);
507 int __kmp_acquire_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
508 KMP_DEBUG_ASSERT(gtid >= 0);
510 if (__kmp_get_futex_lock_owner(lck) == gtid) {
511 lck->lk.depth_locked += 1;
512 return KMP_LOCK_ACQUIRED_NEXT;
514 __kmp_acquire_futex_lock_timed_template(lck, gtid);
515 lck->lk.depth_locked = 1;
516 return KMP_LOCK_ACQUIRED_FIRST;
520 static int __kmp_acquire_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
522 char const *
const func =
"omp_set_nest_lock";
523 if (!__kmp_is_futex_lock_nestable(lck)) {
524 KMP_FATAL(LockSimpleUsedAsNestable, func);
526 return __kmp_acquire_nested_futex_lock(lck, gtid);
529 int __kmp_test_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
532 KMP_DEBUG_ASSERT(gtid >= 0);
534 if (__kmp_get_futex_lock_owner(lck) == gtid) {
535 retval = ++lck->lk.depth_locked;
536 }
else if (!__kmp_test_futex_lock(lck, gtid)) {
540 retval = lck->lk.depth_locked = 1;
545 static int __kmp_test_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
547 char const *
const func =
"omp_test_nest_lock";
548 if (!__kmp_is_futex_lock_nestable(lck)) {
549 KMP_FATAL(LockSimpleUsedAsNestable, func);
551 return __kmp_test_nested_futex_lock(lck, gtid);
554 int __kmp_release_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
555 KMP_DEBUG_ASSERT(gtid >= 0);
558 if (--(lck->lk.depth_locked) == 0) {
559 __kmp_release_futex_lock(lck, gtid);
560 return KMP_LOCK_RELEASED;
562 return KMP_LOCK_STILL_HELD;
565 static int __kmp_release_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
567 char const *
const func =
"omp_unset_nest_lock";
569 if (!__kmp_is_futex_lock_nestable(lck)) {
570 KMP_FATAL(LockSimpleUsedAsNestable, func);
572 if (__kmp_get_futex_lock_owner(lck) == -1) {
573 KMP_FATAL(LockUnsettingFree, func);
575 if (__kmp_get_futex_lock_owner(lck) != gtid) {
576 KMP_FATAL(LockUnsettingSetByAnother, func);
578 return __kmp_release_nested_futex_lock(lck, gtid);
581 void __kmp_init_nested_futex_lock(kmp_futex_lock_t *lck) {
582 __kmp_init_futex_lock(lck);
583 lck->lk.depth_locked = 0;
586 void __kmp_destroy_nested_futex_lock(kmp_futex_lock_t *lck) {
587 __kmp_destroy_futex_lock(lck);
588 lck->lk.depth_locked = 0;
591 static void __kmp_destroy_nested_futex_lock_with_checks(kmp_futex_lock_t *lck) {
592 char const *
const func =
"omp_destroy_nest_lock";
593 if (!__kmp_is_futex_lock_nestable(lck)) {
594 KMP_FATAL(LockSimpleUsedAsNestable, func);
596 if (__kmp_get_futex_lock_owner(lck) != -1) {
597 KMP_FATAL(LockStillOwned, func);
599 __kmp_destroy_nested_futex_lock(lck);
607 static kmp_int32 __kmp_get_ticket_lock_owner(kmp_ticket_lock_t *lck) {
608 return std::atomic_load_explicit(&lck->lk.owner_id,
609 std::memory_order_relaxed) -
613 static inline bool __kmp_is_ticket_lock_nestable(kmp_ticket_lock_t *lck) {
614 return std::atomic_load_explicit(&lck->lk.depth_locked,
615 std::memory_order_relaxed) != -1;
618 static kmp_uint32 __kmp_bakery_check(
void *now_serving, kmp_uint32 my_ticket) {
619 return std::atomic_load_explicit((std::atomic<unsigned> *)now_serving,
620 std::memory_order_acquire) == my_ticket;
623 __forceinline
static int
624 __kmp_acquire_ticket_lock_timed_template(kmp_ticket_lock_t *lck,
626 kmp_uint32 my_ticket = std::atomic_fetch_add_explicit(
627 &lck->lk.next_ticket, 1U, std::memory_order_relaxed);
629 #ifdef USE_LOCK_PROFILE
630 if (std::atomic_load_explicit(&lck->lk.now_serving,
631 std::memory_order_relaxed) != my_ticket)
632 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
636 if (std::atomic_load_explicit(&lck->lk.now_serving,
637 std::memory_order_acquire) == my_ticket) {
638 return KMP_LOCK_ACQUIRED_FIRST;
640 KMP_WAIT_PTR(&lck->lk.now_serving, my_ticket, __kmp_bakery_check, lck);
641 return KMP_LOCK_ACQUIRED_FIRST;
644 int __kmp_acquire_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
645 int retval = __kmp_acquire_ticket_lock_timed_template(lck, gtid);
649 static int __kmp_acquire_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
651 char const *
const func =
"omp_set_lock";
653 if (!std::atomic_load_explicit(&lck->lk.initialized,
654 std::memory_order_relaxed)) {
655 KMP_FATAL(LockIsUninitialized, func);
657 if (lck->lk.self != lck) {
658 KMP_FATAL(LockIsUninitialized, func);
660 if (__kmp_is_ticket_lock_nestable(lck)) {
661 KMP_FATAL(LockNestableUsedAsSimple, func);
663 if ((gtid >= 0) && (__kmp_get_ticket_lock_owner(lck) == gtid)) {
664 KMP_FATAL(LockIsAlreadyOwned, func);
667 __kmp_acquire_ticket_lock(lck, gtid);
669 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
670 std::memory_order_relaxed);
671 return KMP_LOCK_ACQUIRED_FIRST;
674 int __kmp_test_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
675 kmp_uint32 my_ticket = std::atomic_load_explicit(&lck->lk.next_ticket,
676 std::memory_order_relaxed);
678 if (std::atomic_load_explicit(&lck->lk.now_serving,
679 std::memory_order_relaxed) == my_ticket) {
680 kmp_uint32 next_ticket = my_ticket + 1;
681 if (std::atomic_compare_exchange_strong_explicit(
682 &lck->lk.next_ticket, &my_ticket, next_ticket,
683 std::memory_order_acquire, std::memory_order_acquire)) {
690 static int __kmp_test_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
692 char const *
const func =
"omp_test_lock";
694 if (!std::atomic_load_explicit(&lck->lk.initialized,
695 std::memory_order_relaxed)) {
696 KMP_FATAL(LockIsUninitialized, func);
698 if (lck->lk.self != lck) {
699 KMP_FATAL(LockIsUninitialized, func);
701 if (__kmp_is_ticket_lock_nestable(lck)) {
702 KMP_FATAL(LockNestableUsedAsSimple, func);
705 int retval = __kmp_test_ticket_lock(lck, gtid);
708 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
709 std::memory_order_relaxed);
714 int __kmp_release_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
715 std::atomic_fetch_add_explicit(&lck->lk.now_serving, 1U,
716 std::memory_order_release);
718 return KMP_LOCK_RELEASED;
721 static int __kmp_release_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
723 char const *
const func =
"omp_unset_lock";
725 if (!std::atomic_load_explicit(&lck->lk.initialized,
726 std::memory_order_relaxed)) {
727 KMP_FATAL(LockIsUninitialized, func);
729 if (lck->lk.self != lck) {
730 KMP_FATAL(LockIsUninitialized, func);
732 if (__kmp_is_ticket_lock_nestable(lck)) {
733 KMP_FATAL(LockNestableUsedAsSimple, func);
735 if (__kmp_get_ticket_lock_owner(lck) == -1) {
736 KMP_FATAL(LockUnsettingFree, func);
738 if ((gtid >= 0) && (__kmp_get_ticket_lock_owner(lck) >= 0) &&
739 (__kmp_get_ticket_lock_owner(lck) != gtid)) {
740 KMP_FATAL(LockUnsettingSetByAnother, func);
742 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
743 return __kmp_release_ticket_lock(lck, gtid);
746 void __kmp_init_ticket_lock(kmp_ticket_lock_t *lck) {
747 lck->lk.location = NULL;
749 std::atomic_store_explicit(&lck->lk.next_ticket, 0U,
750 std::memory_order_relaxed);
751 std::atomic_store_explicit(&lck->lk.now_serving, 0U,
752 std::memory_order_relaxed);
753 std::atomic_store_explicit(
754 &lck->lk.owner_id, 0,
755 std::memory_order_relaxed);
756 std::atomic_store_explicit(
757 &lck->lk.depth_locked, -1,
758 std::memory_order_relaxed);
759 std::atomic_store_explicit(&lck->lk.initialized,
true,
760 std::memory_order_release);
763 void __kmp_destroy_ticket_lock(kmp_ticket_lock_t *lck) {
764 std::atomic_store_explicit(&lck->lk.initialized,
false,
765 std::memory_order_release);
767 lck->lk.location = NULL;
768 std::atomic_store_explicit(&lck->lk.next_ticket, 0U,
769 std::memory_order_relaxed);
770 std::atomic_store_explicit(&lck->lk.now_serving, 0U,
771 std::memory_order_relaxed);
772 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
773 std::atomic_store_explicit(&lck->lk.depth_locked, -1,
774 std::memory_order_relaxed);
777 static void __kmp_destroy_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
778 char const *
const func =
"omp_destroy_lock";
780 if (!std::atomic_load_explicit(&lck->lk.initialized,
781 std::memory_order_relaxed)) {
782 KMP_FATAL(LockIsUninitialized, func);
784 if (lck->lk.self != lck) {
785 KMP_FATAL(LockIsUninitialized, func);
787 if (__kmp_is_ticket_lock_nestable(lck)) {
788 KMP_FATAL(LockNestableUsedAsSimple, func);
790 if (__kmp_get_ticket_lock_owner(lck) != -1) {
791 KMP_FATAL(LockStillOwned, func);
793 __kmp_destroy_ticket_lock(lck);
798 int __kmp_acquire_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
799 KMP_DEBUG_ASSERT(gtid >= 0);
801 if (__kmp_get_ticket_lock_owner(lck) == gtid) {
802 std::atomic_fetch_add_explicit(&lck->lk.depth_locked, 1,
803 std::memory_order_relaxed);
804 return KMP_LOCK_ACQUIRED_NEXT;
806 __kmp_acquire_ticket_lock_timed_template(lck, gtid);
807 std::atomic_store_explicit(&lck->lk.depth_locked, 1,
808 std::memory_order_relaxed);
809 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
810 std::memory_order_relaxed);
811 return KMP_LOCK_ACQUIRED_FIRST;
815 static int __kmp_acquire_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
817 char const *
const func =
"omp_set_nest_lock";
819 if (!std::atomic_load_explicit(&lck->lk.initialized,
820 std::memory_order_relaxed)) {
821 KMP_FATAL(LockIsUninitialized, func);
823 if (lck->lk.self != lck) {
824 KMP_FATAL(LockIsUninitialized, func);
826 if (!__kmp_is_ticket_lock_nestable(lck)) {
827 KMP_FATAL(LockSimpleUsedAsNestable, func);
829 return __kmp_acquire_nested_ticket_lock(lck, gtid);
832 int __kmp_test_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
835 KMP_DEBUG_ASSERT(gtid >= 0);
837 if (__kmp_get_ticket_lock_owner(lck) == gtid) {
838 retval = std::atomic_fetch_add_explicit(&lck->lk.depth_locked, 1,
839 std::memory_order_relaxed) +
841 }
else if (!__kmp_test_ticket_lock(lck, gtid)) {
844 std::atomic_store_explicit(&lck->lk.depth_locked, 1,
845 std::memory_order_relaxed);
846 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
847 std::memory_order_relaxed);
853 static int __kmp_test_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
855 char const *
const func =
"omp_test_nest_lock";
857 if (!std::atomic_load_explicit(&lck->lk.initialized,
858 std::memory_order_relaxed)) {
859 KMP_FATAL(LockIsUninitialized, func);
861 if (lck->lk.self != lck) {
862 KMP_FATAL(LockIsUninitialized, func);
864 if (!__kmp_is_ticket_lock_nestable(lck)) {
865 KMP_FATAL(LockSimpleUsedAsNestable, func);
867 return __kmp_test_nested_ticket_lock(lck, gtid);
870 int __kmp_release_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
871 KMP_DEBUG_ASSERT(gtid >= 0);
873 if ((std::atomic_fetch_add_explicit(&lck->lk.depth_locked, -1,
874 std::memory_order_relaxed) -
876 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
877 __kmp_release_ticket_lock(lck, gtid);
878 return KMP_LOCK_RELEASED;
880 return KMP_LOCK_STILL_HELD;
883 static int __kmp_release_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
885 char const *
const func =
"omp_unset_nest_lock";
887 if (!std::atomic_load_explicit(&lck->lk.initialized,
888 std::memory_order_relaxed)) {
889 KMP_FATAL(LockIsUninitialized, func);
891 if (lck->lk.self != lck) {
892 KMP_FATAL(LockIsUninitialized, func);
894 if (!__kmp_is_ticket_lock_nestable(lck)) {
895 KMP_FATAL(LockSimpleUsedAsNestable, func);
897 if (__kmp_get_ticket_lock_owner(lck) == -1) {
898 KMP_FATAL(LockUnsettingFree, func);
900 if (__kmp_get_ticket_lock_owner(lck) != gtid) {
901 KMP_FATAL(LockUnsettingSetByAnother, func);
903 return __kmp_release_nested_ticket_lock(lck, gtid);
906 void __kmp_init_nested_ticket_lock(kmp_ticket_lock_t *lck) {
907 __kmp_init_ticket_lock(lck);
908 std::atomic_store_explicit(&lck->lk.depth_locked, 0,
909 std::memory_order_relaxed);
913 void __kmp_destroy_nested_ticket_lock(kmp_ticket_lock_t *lck) {
914 __kmp_destroy_ticket_lock(lck);
915 std::atomic_store_explicit(&lck->lk.depth_locked, 0,
916 std::memory_order_relaxed);
920 __kmp_destroy_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
921 char const *
const func =
"omp_destroy_nest_lock";
923 if (!std::atomic_load_explicit(&lck->lk.initialized,
924 std::memory_order_relaxed)) {
925 KMP_FATAL(LockIsUninitialized, func);
927 if (lck->lk.self != lck) {
928 KMP_FATAL(LockIsUninitialized, func);
930 if (!__kmp_is_ticket_lock_nestable(lck)) {
931 KMP_FATAL(LockSimpleUsedAsNestable, func);
933 if (__kmp_get_ticket_lock_owner(lck) != -1) {
934 KMP_FATAL(LockStillOwned, func);
936 __kmp_destroy_nested_ticket_lock(lck);
941 static const ident_t *__kmp_get_ticket_lock_location(kmp_ticket_lock_t *lck) {
942 return lck->lk.location;
945 static void __kmp_set_ticket_lock_location(kmp_ticket_lock_t *lck,
947 lck->lk.location = loc;
950 static kmp_lock_flags_t __kmp_get_ticket_lock_flags(kmp_ticket_lock_t *lck) {
951 return lck->lk.flags;
954 static void __kmp_set_ticket_lock_flags(kmp_ticket_lock_t *lck,
955 kmp_lock_flags_t flags) {
956 lck->lk.flags = flags;
1014 #ifdef DEBUG_QUEUING_LOCKS
1017 #define TRACE_BUF_ELE 1024
1018 static char traces[TRACE_BUF_ELE][128] = {0};
1020 #define TRACE_LOCK(X, Y) \
1021 KMP_SNPRINTF(traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s\n", X, Y);
1022 #define TRACE_LOCK_T(X, Y, Z) \
1023 KMP_SNPRINTF(traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s%d\n", X, Y, Z);
1024 #define TRACE_LOCK_HT(X, Y, Z, Q) \
1025 KMP_SNPRINTF(traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s %d,%d\n", X, Y, \
1028 static void __kmp_dump_queuing_lock(kmp_info_t *this_thr, kmp_int32 gtid,
1029 kmp_queuing_lock_t *lck, kmp_int32 head_id,
1030 kmp_int32 tail_id) {
1033 __kmp_printf_no_lock(
"\n__kmp_dump_queuing_lock: TRACE BEGINS HERE! \n");
1035 i = tc % TRACE_BUF_ELE;
1036 __kmp_printf_no_lock(
"%s\n", traces[i]);
1037 i = (i + 1) % TRACE_BUF_ELE;
1038 while (i != (tc % TRACE_BUF_ELE)) {
1039 __kmp_printf_no_lock(
"%s", traces[i]);
1040 i = (i + 1) % TRACE_BUF_ELE;
1042 __kmp_printf_no_lock(
"\n");
1044 __kmp_printf_no_lock(
"\n__kmp_dump_queuing_lock: gtid+1:%d, spin_here:%d, "
1045 "next_wait:%d, head_id:%d, tail_id:%d\n",
1046 gtid + 1, this_thr->th.th_spin_here,
1047 this_thr->th.th_next_waiting, head_id, tail_id);
1049 __kmp_printf_no_lock(
"\t\thead: %d ", lck->lk.head_id);
1051 if (lck->lk.head_id >= 1) {
1052 t = __kmp_threads[lck->lk.head_id - 1]->th.th_next_waiting;
1054 __kmp_printf_no_lock(
"-> %d ", t);
1055 t = __kmp_threads[t - 1]->th.th_next_waiting;
1058 __kmp_printf_no_lock(
"; tail: %d ", lck->lk.tail_id);
1059 __kmp_printf_no_lock(
"\n\n");
1064 static kmp_int32 __kmp_get_queuing_lock_owner(kmp_queuing_lock_t *lck) {
1065 return TCR_4(lck->lk.owner_id) - 1;
1068 static inline bool __kmp_is_queuing_lock_nestable(kmp_queuing_lock_t *lck) {
1069 return lck->lk.depth_locked != -1;
1073 template <
bool takeTime>
1076 __forceinline
static int
1077 __kmp_acquire_queuing_lock_timed_template(kmp_queuing_lock_t *lck,
1079 kmp_info_t *this_thr = __kmp_thread_from_gtid(gtid);
1080 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1081 volatile kmp_int32 *tail_id_p = &lck->lk.tail_id;
1082 volatile kmp_uint32 *spin_here_p;
1085 ompt_state_t prev_state = ompt_state_undefined;
1089 (
"__kmp_acquire_queuing_lock: lck:%p, T#%d entering\n", lck, gtid));
1091 KMP_FSYNC_PREPARE(lck);
1092 KMP_DEBUG_ASSERT(this_thr != NULL);
1093 spin_here_p = &this_thr->th.th_spin_here;
1095 #ifdef DEBUG_QUEUING_LOCKS
1096 TRACE_LOCK(gtid + 1,
"acq ent");
1098 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1099 if (this_thr->th.th_next_waiting != 0)
1100 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1102 KMP_DEBUG_ASSERT(!*spin_here_p);
1103 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
1112 *spin_here_p = TRUE;
1124 #ifdef DEBUG_QUEUING_LOCKS
1126 TRACE_LOCK_HT(gtid + 1,
"acq read: ", head, tail);
1134 enqueued = KMP_COMPARE_AND_STORE_ACQ64((
volatile kmp_int64 *)tail_id_p,
1136 KMP_PACK_64(gtid + 1, gtid + 1));
1137 #ifdef DEBUG_QUEUING_LOCKS
1139 TRACE_LOCK(gtid + 1,
"acq enq: (-1,0)->(tid,tid)");
1145 KMP_DEBUG_ASSERT(tail != gtid + 1);
1147 #ifdef DEBUG_QUEUING_LOCKS
1148 TRACE_LOCK_HT(gtid + 1,
"acq read: ", head, tail);
1155 enqueued = KMP_COMPARE_AND_STORE_ACQ32(tail_id_p, tail, gtid + 1);
1157 #ifdef DEBUG_QUEUING_LOCKS
1159 TRACE_LOCK(gtid + 1,
"acq enq: (h,t)->(h,tid)");
1166 kmp_int32 grabbed_lock;
1168 #ifdef DEBUG_QUEUING_LOCKS
1170 TRACE_LOCK_HT(gtid + 1,
"acq read: ", head, tail);
1176 grabbed_lock = KMP_COMPARE_AND_STORE_ACQ32(head_id_p, 0, -1);
1180 *spin_here_p = FALSE;
1184 (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: no queuing\n",
1186 #ifdef DEBUG_QUEUING_LOCKS
1187 TRACE_LOCK_HT(gtid + 1,
"acq exit: ", head, 0);
1191 if (ompt_enabled.enabled && prev_state != ompt_state_undefined) {
1193 this_thr->th.ompt_thread_info.state = prev_state;
1194 this_thr->th.ompt_thread_info.wait_id = 0;
1198 KMP_FSYNC_ACQUIRED(lck);
1199 return KMP_LOCK_ACQUIRED_FIRST;
1206 if (ompt_enabled.enabled && prev_state == ompt_state_undefined) {
1208 prev_state = this_thr->th.ompt_thread_info.state;
1209 this_thr->th.ompt_thread_info.wait_id = (uint64_t)lck;
1210 this_thr->th.ompt_thread_info.state = ompt_state_wait_lock;
1216 kmp_info_t *tail_thr = __kmp_thread_from_gtid(tail - 1);
1217 KMP_ASSERT(tail_thr != NULL);
1218 tail_thr->th.th_next_waiting = gtid + 1;
1222 (
"__kmp_acquire_queuing_lock: lck:%p, T#%d waiting for lock\n",
1227 KMP_WAIT(spin_here_p, FALSE, KMP_EQ, lck);
1232 #ifdef DEBUG_QUEUING_LOCKS
1233 TRACE_LOCK(gtid + 1,
"acq spin");
1235 if (this_thr->th.th_next_waiting != 0)
1236 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1238 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
1239 KA_TRACE(1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: after "
1240 "waiting on queue\n",
1243 #ifdef DEBUG_QUEUING_LOCKS
1244 TRACE_LOCK(gtid + 1,
"acq exit 2");
1249 this_thr->th.ompt_thread_info.state = prev_state;
1250 this_thr->th.ompt_thread_info.wait_id = 0;
1254 return KMP_LOCK_ACQUIRED_FIRST;
1260 KMP_YIELD_OVERSUB();
1262 #ifdef DEBUG_QUEUING_LOCKS
1263 TRACE_LOCK(gtid + 1,
"acq retry");
1266 KMP_ASSERT2(0,
"should not get here");
1267 return KMP_LOCK_ACQUIRED_FIRST;
1270 int __kmp_acquire_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1271 KMP_DEBUG_ASSERT(gtid >= 0);
1273 int retval = __kmp_acquire_queuing_lock_timed_template<false>(lck, gtid);
1277 static int __kmp_acquire_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1279 char const *
const func =
"omp_set_lock";
1280 if (lck->lk.initialized != lck) {
1281 KMP_FATAL(LockIsUninitialized, func);
1283 if (__kmp_is_queuing_lock_nestable(lck)) {
1284 KMP_FATAL(LockNestableUsedAsSimple, func);
1286 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1287 KMP_FATAL(LockIsAlreadyOwned, func);
1290 __kmp_acquire_queuing_lock(lck, gtid);
1292 lck->lk.owner_id = gtid + 1;
1293 return KMP_LOCK_ACQUIRED_FIRST;
1296 int __kmp_test_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1297 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1300 kmp_info_t *this_thr;
1303 KA_TRACE(1000, (
"__kmp_test_queuing_lock: T#%d entering\n", gtid));
1304 KMP_DEBUG_ASSERT(gtid >= 0);
1306 this_thr = __kmp_thread_from_gtid(gtid);
1307 KMP_DEBUG_ASSERT(this_thr != NULL);
1308 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
1315 if (KMP_COMPARE_AND_STORE_ACQ32(head_id_p, 0, -1)) {
1317 (
"__kmp_test_queuing_lock: T#%d exiting: holding lock\n", gtid));
1318 KMP_FSYNC_ACQUIRED(lck);
1324 (
"__kmp_test_queuing_lock: T#%d exiting: without lock\n", gtid));
1328 static int __kmp_test_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1330 char const *
const func =
"omp_test_lock";
1331 if (lck->lk.initialized != lck) {
1332 KMP_FATAL(LockIsUninitialized, func);
1334 if (__kmp_is_queuing_lock_nestable(lck)) {
1335 KMP_FATAL(LockNestableUsedAsSimple, func);
1338 int retval = __kmp_test_queuing_lock(lck, gtid);
1341 lck->lk.owner_id = gtid + 1;
1346 int __kmp_release_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1347 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1348 volatile kmp_int32 *tail_id_p = &lck->lk.tail_id;
1351 (
"__kmp_release_queuing_lock: lck:%p, T#%d entering\n", lck, gtid));
1352 KMP_DEBUG_ASSERT(gtid >= 0);
1353 #if KMP_DEBUG || DEBUG_QUEUING_LOCKS
1354 kmp_info_t *this_thr = __kmp_thread_from_gtid(gtid);
1356 KMP_DEBUG_ASSERT(this_thr != NULL);
1357 #ifdef DEBUG_QUEUING_LOCKS
1358 TRACE_LOCK(gtid + 1,
"rel ent");
1360 if (this_thr->th.th_spin_here)
1361 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1362 if (this_thr->th.th_next_waiting != 0)
1363 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1365 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
1366 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
1368 KMP_FSYNC_RELEASING(lck);
1377 #ifdef DEBUG_QUEUING_LOCKS
1379 TRACE_LOCK_HT(gtid + 1,
"rel read: ", head, tail);
1381 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1383 KMP_DEBUG_ASSERT(head !=
1388 if (KMP_COMPARE_AND_STORE_REL32(head_id_p, -1, 0)) {
1391 (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: queue empty\n",
1393 #ifdef DEBUG_QUEUING_LOCKS
1394 TRACE_LOCK_HT(gtid + 1,
"rel exit: ", 0, 0);
1400 return KMP_LOCK_RELEASED;
1407 #ifdef DEBUG_QUEUING_LOCKS
1409 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1411 KMP_DEBUG_ASSERT(head > 0);
1414 dequeued = KMP_COMPARE_AND_STORE_REL64(
1415 RCAST(
volatile kmp_int64 *, tail_id_p), KMP_PACK_64(head, head),
1416 KMP_PACK_64(-1, 0));
1417 #ifdef DEBUG_QUEUING_LOCKS
1418 TRACE_LOCK(gtid + 1,
"rel deq: (h,h)->(-1,0)");
1422 volatile kmp_int32 *waiting_id_p;
1423 kmp_info_t *head_thr = __kmp_thread_from_gtid(head - 1);
1424 KMP_DEBUG_ASSERT(head_thr != NULL);
1425 waiting_id_p = &head_thr->th.th_next_waiting;
1428 #ifdef DEBUG_QUEUING_LOCKS
1429 if (head <= 0 || tail <= 0)
1430 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1432 KMP_DEBUG_ASSERT(head > 0 && tail > 0);
1439 KMP_WAIT((
volatile kmp_uint32 *)waiting_id_p, 0, KMP_NEQ, NULL);
1440 #ifdef DEBUG_QUEUING_LOCKS
1441 TRACE_LOCK(gtid + 1,
"rel deq: (h,t)->(h',t)");
1448 kmp_info_t *head_thr = __kmp_thread_from_gtid(head - 1);
1449 KMP_DEBUG_ASSERT(head_thr != NULL);
1452 #ifdef DEBUG_QUEUING_LOCKS
1453 if (head <= 0 || tail <= 0)
1454 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1456 KMP_DEBUG_ASSERT(head > 0 && tail > 0);
1460 head_thr->th.th_next_waiting = 0;
1461 #ifdef DEBUG_QUEUING_LOCKS
1462 TRACE_LOCK_T(gtid + 1,
"rel nw=0 for t=", head);
1467 head_thr->th.th_spin_here = FALSE;
1469 KA_TRACE(1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: after "
1472 #ifdef DEBUG_QUEUING_LOCKS
1473 TRACE_LOCK(gtid + 1,
"rel exit 2");
1475 return KMP_LOCK_RELEASED;
1480 #ifdef DEBUG_QUEUING_LOCKS
1481 TRACE_LOCK(gtid + 1,
"rel retry");
1485 KMP_ASSERT2(0,
"should not get here");
1486 return KMP_LOCK_RELEASED;
1489 static int __kmp_release_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1491 char const *
const func =
"omp_unset_lock";
1493 if (lck->lk.initialized != lck) {
1494 KMP_FATAL(LockIsUninitialized, func);
1496 if (__kmp_is_queuing_lock_nestable(lck)) {
1497 KMP_FATAL(LockNestableUsedAsSimple, func);
1499 if (__kmp_get_queuing_lock_owner(lck) == -1) {
1500 KMP_FATAL(LockUnsettingFree, func);
1502 if (__kmp_get_queuing_lock_owner(lck) != gtid) {
1503 KMP_FATAL(LockUnsettingSetByAnother, func);
1505 lck->lk.owner_id = 0;
1506 return __kmp_release_queuing_lock(lck, gtid);
1509 void __kmp_init_queuing_lock(kmp_queuing_lock_t *lck) {
1510 lck->lk.location = NULL;
1511 lck->lk.head_id = 0;
1512 lck->lk.tail_id = 0;
1513 lck->lk.next_ticket = 0;
1514 lck->lk.now_serving = 0;
1515 lck->lk.owner_id = 0;
1516 lck->lk.depth_locked = -1;
1517 lck->lk.initialized = lck;
1519 KA_TRACE(1000, (
"__kmp_init_queuing_lock: lock %p initialized\n", lck));
1522 void __kmp_destroy_queuing_lock(kmp_queuing_lock_t *lck) {
1523 lck->lk.initialized = NULL;
1524 lck->lk.location = NULL;
1525 lck->lk.head_id = 0;
1526 lck->lk.tail_id = 0;
1527 lck->lk.next_ticket = 0;
1528 lck->lk.now_serving = 0;
1529 lck->lk.owner_id = 0;
1530 lck->lk.depth_locked = -1;
1533 static void __kmp_destroy_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
1534 char const *
const func =
"omp_destroy_lock";
1535 if (lck->lk.initialized != lck) {
1536 KMP_FATAL(LockIsUninitialized, func);
1538 if (__kmp_is_queuing_lock_nestable(lck)) {
1539 KMP_FATAL(LockNestableUsedAsSimple, func);
1541 if (__kmp_get_queuing_lock_owner(lck) != -1) {
1542 KMP_FATAL(LockStillOwned, func);
1544 __kmp_destroy_queuing_lock(lck);
1549 int __kmp_acquire_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1550 KMP_DEBUG_ASSERT(gtid >= 0);
1552 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1553 lck->lk.depth_locked += 1;
1554 return KMP_LOCK_ACQUIRED_NEXT;
1556 __kmp_acquire_queuing_lock_timed_template<false>(lck, gtid);
1558 lck->lk.depth_locked = 1;
1560 lck->lk.owner_id = gtid + 1;
1561 return KMP_LOCK_ACQUIRED_FIRST;
1566 __kmp_acquire_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1568 char const *
const func =
"omp_set_nest_lock";
1569 if (lck->lk.initialized != lck) {
1570 KMP_FATAL(LockIsUninitialized, func);
1572 if (!__kmp_is_queuing_lock_nestable(lck)) {
1573 KMP_FATAL(LockSimpleUsedAsNestable, func);
1575 return __kmp_acquire_nested_queuing_lock(lck, gtid);
1578 int __kmp_test_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1581 KMP_DEBUG_ASSERT(gtid >= 0);
1583 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1584 retval = ++lck->lk.depth_locked;
1585 }
else if (!__kmp_test_queuing_lock(lck, gtid)) {
1589 retval = lck->lk.depth_locked = 1;
1591 lck->lk.owner_id = gtid + 1;
1596 static int __kmp_test_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1598 char const *
const func =
"omp_test_nest_lock";
1599 if (lck->lk.initialized != lck) {
1600 KMP_FATAL(LockIsUninitialized, func);
1602 if (!__kmp_is_queuing_lock_nestable(lck)) {
1603 KMP_FATAL(LockSimpleUsedAsNestable, func);
1605 return __kmp_test_nested_queuing_lock(lck, gtid);
1608 int __kmp_release_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1609 KMP_DEBUG_ASSERT(gtid >= 0);
1612 if (--(lck->lk.depth_locked) == 0) {
1614 lck->lk.owner_id = 0;
1615 __kmp_release_queuing_lock(lck, gtid);
1616 return KMP_LOCK_RELEASED;
1618 return KMP_LOCK_STILL_HELD;
1622 __kmp_release_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1624 char const *
const func =
"omp_unset_nest_lock";
1626 if (lck->lk.initialized != lck) {
1627 KMP_FATAL(LockIsUninitialized, func);
1629 if (!__kmp_is_queuing_lock_nestable(lck)) {
1630 KMP_FATAL(LockSimpleUsedAsNestable, func);
1632 if (__kmp_get_queuing_lock_owner(lck) == -1) {
1633 KMP_FATAL(LockUnsettingFree, func);
1635 if (__kmp_get_queuing_lock_owner(lck) != gtid) {
1636 KMP_FATAL(LockUnsettingSetByAnother, func);
1638 return __kmp_release_nested_queuing_lock(lck, gtid);
1641 void __kmp_init_nested_queuing_lock(kmp_queuing_lock_t *lck) {
1642 __kmp_init_queuing_lock(lck);
1643 lck->lk.depth_locked = 0;
1646 void __kmp_destroy_nested_queuing_lock(kmp_queuing_lock_t *lck) {
1647 __kmp_destroy_queuing_lock(lck);
1648 lck->lk.depth_locked = 0;
1652 __kmp_destroy_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
1653 char const *
const func =
"omp_destroy_nest_lock";
1654 if (lck->lk.initialized != lck) {
1655 KMP_FATAL(LockIsUninitialized, func);
1657 if (!__kmp_is_queuing_lock_nestable(lck)) {
1658 KMP_FATAL(LockSimpleUsedAsNestable, func);
1660 if (__kmp_get_queuing_lock_owner(lck) != -1) {
1661 KMP_FATAL(LockStillOwned, func);
1663 __kmp_destroy_nested_queuing_lock(lck);
1668 static const ident_t *__kmp_get_queuing_lock_location(kmp_queuing_lock_t *lck) {
1669 return lck->lk.location;
1672 static void __kmp_set_queuing_lock_location(kmp_queuing_lock_t *lck,
1674 lck->lk.location = loc;
1677 static kmp_lock_flags_t __kmp_get_queuing_lock_flags(kmp_queuing_lock_t *lck) {
1678 return lck->lk.flags;
1681 static void __kmp_set_queuing_lock_flags(kmp_queuing_lock_t *lck,
1682 kmp_lock_flags_t flags) {
1683 lck->lk.flags = flags;
1686 #if KMP_USE_ADAPTIVE_LOCKS
1690 #if KMP_HAVE_RTM_INTRINSICS
1691 #include <immintrin.h>
1692 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT)
1697 #define _XBEGIN_STARTED (~0u)
1698 #define _XABORT_EXPLICIT (1 << 0)
1699 #define _XABORT_RETRY (1 << 1)
1700 #define _XABORT_CONFLICT (1 << 2)
1701 #define _XABORT_CAPACITY (1 << 3)
1702 #define _XABORT_DEBUG (1 << 4)
1703 #define _XABORT_NESTED (1 << 5)
1704 #define _XABORT_CODE(x) ((unsigned char)(((x) >> 24) & 0xFF))
1707 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT)
1709 #define STRINGIZE_INTERNAL(arg) #arg
1710 #define STRINGIZE(arg) STRINGIZE_INTERNAL(arg)
1716 static __inline
int _xbegin() {
1753 __asm__
volatile(
"1: .byte 0xC7; .byte 0xF8;\n"
1756 "1: movl %%eax,%0\n"
1758 :
"+r"(res)::
"memory",
"%eax");
1764 static __inline
void _xend() {
1772 __asm__
volatile(
".byte 0x0f; .byte 0x01; .byte 0xd5" :::
"memory");
1781 #define _xabort(ARG) _asm _emit 0xc6 _asm _emit 0xf8 _asm _emit ARG
1783 #define _xabort(ARG) \
1784 __asm__ volatile(".byte 0xC6; .byte 0xF8; .byte " STRINGIZE(ARG):::"memory");
1790 #if KMP_DEBUG_ADAPTIVE_LOCKS
1795 static kmp_adaptive_lock_statistics_t destroyedStats;
1798 static kmp_adaptive_lock_info_t liveLocks;
1801 static kmp_bootstrap_lock_t chain_lock =
1802 KMP_BOOTSTRAP_LOCK_INITIALIZER(chain_lock);
1805 void __kmp_init_speculative_stats() {
1806 kmp_adaptive_lock_info_t *lck = &liveLocks;
1808 memset(CCAST(kmp_adaptive_lock_statistics_t *, &(lck->stats)), 0,
1809 sizeof(lck->stats));
1810 lck->stats.next = lck;
1811 lck->stats.prev = lck;
1813 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1814 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1816 __kmp_init_bootstrap_lock(&chain_lock);
1820 static void __kmp_remember_lock(kmp_adaptive_lock_info_t *lck) {
1821 __kmp_acquire_bootstrap_lock(&chain_lock);
1823 lck->stats.next = liveLocks.stats.next;
1824 lck->stats.prev = &liveLocks;
1826 liveLocks.stats.next = lck;
1827 lck->stats.next->stats.prev = lck;
1829 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1830 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1832 __kmp_release_bootstrap_lock(&chain_lock);
1835 static void __kmp_forget_lock(kmp_adaptive_lock_info_t *lck) {
1836 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1837 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1839 kmp_adaptive_lock_info_t *n = lck->stats.next;
1840 kmp_adaptive_lock_info_t *p = lck->stats.prev;
1846 static void __kmp_zero_speculative_stats(kmp_adaptive_lock_info_t *lck) {
1847 memset(CCAST(kmp_adaptive_lock_statistics_t *, &lck->stats), 0,
1848 sizeof(lck->stats));
1849 __kmp_remember_lock(lck);
1852 static void __kmp_add_stats(kmp_adaptive_lock_statistics_t *t,
1853 kmp_adaptive_lock_info_t *lck) {
1854 kmp_adaptive_lock_statistics_t
volatile *s = &lck->stats;
1856 t->nonSpeculativeAcquireAttempts += lck->acquire_attempts;
1857 t->successfulSpeculations += s->successfulSpeculations;
1858 t->hardFailedSpeculations += s->hardFailedSpeculations;
1859 t->softFailedSpeculations += s->softFailedSpeculations;
1860 t->nonSpeculativeAcquires += s->nonSpeculativeAcquires;
1861 t->lemmingYields += s->lemmingYields;
1864 static void __kmp_accumulate_speculative_stats(kmp_adaptive_lock_info_t *lck) {
1865 __kmp_acquire_bootstrap_lock(&chain_lock);
1867 __kmp_add_stats(&destroyedStats, lck);
1868 __kmp_forget_lock(lck);
1870 __kmp_release_bootstrap_lock(&chain_lock);
1873 static float percent(kmp_uint32 count, kmp_uint32 total) {
1874 return (total == 0) ? 0.0 : (100.0 * count) / total;
1877 void __kmp_print_speculative_stats() {
1878 kmp_adaptive_lock_statistics_t total = destroyedStats;
1879 kmp_adaptive_lock_info_t *lck;
1881 for (lck = liveLocks.stats.next; lck != &liveLocks; lck = lck->stats.next) {
1882 __kmp_add_stats(&total, lck);
1884 kmp_adaptive_lock_statistics_t *t = &total;
1885 kmp_uint32 totalSections =
1886 t->nonSpeculativeAcquires + t->successfulSpeculations;
1887 kmp_uint32 totalSpeculations = t->successfulSpeculations +
1888 t->hardFailedSpeculations +
1889 t->softFailedSpeculations;
1890 if (totalSections <= 0)
1894 if (strcmp(__kmp_speculative_statsfile,
"-") == 0) {
1897 size_t buffLen = KMP_STRLEN(__kmp_speculative_statsfile) + 20;
1898 char buffer[buffLen];
1899 KMP_SNPRINTF(&buffer[0], buffLen, __kmp_speculative_statsfile,
1900 (kmp_int32)getpid());
1901 statsFile.
open(buffer,
"w");
1904 fprintf(statsFile,
"Speculative lock statistics (all approximate!)\n");
1906 " Lock parameters: \n"
1907 " max_soft_retries : %10d\n"
1908 " max_badness : %10d\n",
1909 __kmp_adaptive_backoff_params.max_soft_retries,
1910 __kmp_adaptive_backoff_params.max_badness);
1911 fprintf(statsFile,
" Non-speculative acquire attempts : %10d\n",
1912 t->nonSpeculativeAcquireAttempts);
1913 fprintf(statsFile,
" Total critical sections : %10d\n",
1915 fprintf(statsFile,
" Successful speculations : %10d (%5.1f%%)\n",
1916 t->successfulSpeculations,
1917 percent(t->successfulSpeculations, totalSections));
1918 fprintf(statsFile,
" Non-speculative acquires : %10d (%5.1f%%)\n",
1919 t->nonSpeculativeAcquires,
1920 percent(t->nonSpeculativeAcquires, totalSections));
1921 fprintf(statsFile,
" Lemming yields : %10d\n\n",
1924 fprintf(statsFile,
" Speculative acquire attempts : %10d\n",
1926 fprintf(statsFile,
" Successes : %10d (%5.1f%%)\n",
1927 t->successfulSpeculations,
1928 percent(t->successfulSpeculations, totalSpeculations));
1929 fprintf(statsFile,
" Soft failures : %10d (%5.1f%%)\n",
1930 t->softFailedSpeculations,
1931 percent(t->softFailedSpeculations, totalSpeculations));
1932 fprintf(statsFile,
" Hard failures : %10d (%5.1f%%)\n",
1933 t->hardFailedSpeculations,
1934 percent(t->hardFailedSpeculations, totalSpeculations));
1937 #define KMP_INC_STAT(lck, stat) (lck->lk.adaptive.stats.stat++)
1939 #define KMP_INC_STAT(lck, stat)
1943 static inline bool __kmp_is_unlocked_queuing_lock(kmp_queuing_lock_t *lck) {
1946 bool res = lck->lk.head_id == 0;
1950 #if KMP_COMPILER_ICC || KMP_COMPILER_ICX
1953 __sync_synchronize();
1960 static __inline
void
1961 __kmp_update_badness_after_success(kmp_adaptive_lock_t *lck) {
1963 lck->lk.adaptive.badness = 0;
1964 KMP_INC_STAT(lck, successfulSpeculations);
1968 static __inline
void __kmp_step_badness(kmp_adaptive_lock_t *lck) {
1969 kmp_uint32 newBadness = (lck->lk.adaptive.badness << 1) | 1;
1970 if (newBadness > lck->lk.adaptive.max_badness) {
1973 lck->lk.adaptive.badness = newBadness;
1978 KMP_ATTRIBUTE_TARGET_RTM
1979 static __inline
int __kmp_should_speculate(kmp_adaptive_lock_t *lck,
1981 kmp_uint32 badness = lck->lk.adaptive.badness;
1982 kmp_uint32 attempts = lck->lk.adaptive.acquire_attempts;
1983 int res = (attempts & badness) == 0;
1989 KMP_ATTRIBUTE_TARGET_RTM
1990 static int __kmp_test_adaptive_lock_only(kmp_adaptive_lock_t *lck,
1992 int retries = lck->lk.adaptive.max_soft_retries;
1999 kmp_uint32 status = _xbegin();
2004 if (status == _XBEGIN_STARTED) {
2009 if (!__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2013 KMP_ASSERT2(0,
"should not get here");
2018 if (status & SOFT_ABORT_MASK) {
2019 KMP_INC_STAT(lck, softFailedSpeculations);
2022 KMP_INC_STAT(lck, hardFailedSpeculations);
2027 }
while (retries--);
2031 __kmp_step_badness(lck);
2038 static int __kmp_test_adaptive_lock(kmp_adaptive_lock_t *lck, kmp_int32 gtid) {
2040 if (__kmp_should_speculate(lck, gtid) &&
2041 __kmp_test_adaptive_lock_only(lck, gtid))
2046 lck->lk.adaptive.acquire_attempts++;
2049 if (__kmp_test_queuing_lock(GET_QLK_PTR(lck), gtid)) {
2050 KMP_INC_STAT(lck, nonSpeculativeAcquires);
2057 static int __kmp_test_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2059 char const *
const func =
"omp_test_lock";
2060 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2061 KMP_FATAL(LockIsUninitialized, func);
2064 int retval = __kmp_test_adaptive_lock(lck, gtid);
2067 lck->lk.qlk.owner_id = gtid + 1;
2083 static void __kmp_acquire_adaptive_lock(kmp_adaptive_lock_t *lck,
2085 if (__kmp_should_speculate(lck, gtid)) {
2086 if (__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2087 if (__kmp_test_adaptive_lock_only(lck, gtid))
2096 while (!__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2097 KMP_INC_STAT(lck, lemmingYields);
2101 if (__kmp_test_adaptive_lock_only(lck, gtid))
2108 lck->lk.adaptive.acquire_attempts++;
2110 __kmp_acquire_queuing_lock_timed_template<FALSE>(GET_QLK_PTR(lck), gtid);
2112 KMP_INC_STAT(lck, nonSpeculativeAcquires);
2115 static void __kmp_acquire_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2117 char const *
const func =
"omp_set_lock";
2118 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2119 KMP_FATAL(LockIsUninitialized, func);
2121 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) == gtid) {
2122 KMP_FATAL(LockIsAlreadyOwned, func);
2125 __kmp_acquire_adaptive_lock(lck, gtid);
2127 lck->lk.qlk.owner_id = gtid + 1;
2130 KMP_ATTRIBUTE_TARGET_RTM
2131 static int __kmp_release_adaptive_lock(kmp_adaptive_lock_t *lck,
2133 if (__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(
2138 __kmp_update_badness_after_success(lck);
2141 __kmp_release_queuing_lock(GET_QLK_PTR(lck), gtid);
2143 return KMP_LOCK_RELEASED;
2146 static int __kmp_release_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2148 char const *
const func =
"omp_unset_lock";
2150 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2151 KMP_FATAL(LockIsUninitialized, func);
2153 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) == -1) {
2154 KMP_FATAL(LockUnsettingFree, func);
2156 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) != gtid) {
2157 KMP_FATAL(LockUnsettingSetByAnother, func);
2159 lck->lk.qlk.owner_id = 0;
2160 __kmp_release_adaptive_lock(lck, gtid);
2161 return KMP_LOCK_RELEASED;
2164 static void __kmp_init_adaptive_lock(kmp_adaptive_lock_t *lck) {
2165 __kmp_init_queuing_lock(GET_QLK_PTR(lck));
2166 lck->lk.adaptive.badness = 0;
2167 lck->lk.adaptive.acquire_attempts = 0;
2168 lck->lk.adaptive.max_soft_retries =
2169 __kmp_adaptive_backoff_params.max_soft_retries;
2170 lck->lk.adaptive.max_badness = __kmp_adaptive_backoff_params.max_badness;
2171 #if KMP_DEBUG_ADAPTIVE_LOCKS
2172 __kmp_zero_speculative_stats(&lck->lk.adaptive);
2174 KA_TRACE(1000, (
"__kmp_init_adaptive_lock: lock %p initialized\n", lck));
2177 static void __kmp_destroy_adaptive_lock(kmp_adaptive_lock_t *lck) {
2178 #if KMP_DEBUG_ADAPTIVE_LOCKS
2179 __kmp_accumulate_speculative_stats(&lck->lk.adaptive);
2181 __kmp_destroy_queuing_lock(GET_QLK_PTR(lck));
2185 static void __kmp_destroy_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck) {
2186 char const *
const func =
"omp_destroy_lock";
2187 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2188 KMP_FATAL(LockIsUninitialized, func);
2190 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) != -1) {
2191 KMP_FATAL(LockStillOwned, func);
2193 __kmp_destroy_adaptive_lock(lck);
2202 static kmp_int32 __kmp_get_drdpa_lock_owner(kmp_drdpa_lock_t *lck) {
2203 return lck->lk.owner_id - 1;
2206 static inline bool __kmp_is_drdpa_lock_nestable(kmp_drdpa_lock_t *lck) {
2207 return lck->lk.depth_locked != -1;
2210 __forceinline
static int
2211 __kmp_acquire_drdpa_lock_timed_template(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2212 kmp_uint64 ticket = KMP_ATOMIC_INC(&lck->lk.next_ticket);
2213 kmp_uint64 mask = lck->lk.mask;
2214 std::atomic<kmp_uint64> *polls = lck->lk.polls;
2216 #ifdef USE_LOCK_PROFILE
2217 if (polls[ticket & mask] != ticket)
2218 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
2231 KMP_FSYNC_PREPARE(lck);
2232 KMP_INIT_YIELD(spins);
2233 KMP_INIT_BACKOFF(time);
2234 while (polls[ticket & mask] < ticket) {
2235 KMP_YIELD_OVERSUB_ELSE_SPIN(spins, time);
2243 mask = lck->lk.mask;
2244 polls = lck->lk.polls;
2248 KMP_FSYNC_ACQUIRED(lck);
2249 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld acquired lock %p\n",
2251 lck->lk.now_serving = ticket;
2258 if ((lck->lk.old_polls != NULL) && (ticket >= lck->lk.cleanup_ticket)) {
2259 __kmp_free(lck->lk.old_polls);
2260 lck->lk.old_polls = NULL;
2261 lck->lk.cleanup_ticket = 0;
2267 if (lck->lk.old_polls == NULL) {
2268 bool reconfigure =
false;
2269 std::atomic<kmp_uint64> *old_polls = polls;
2270 kmp_uint32 num_polls = TCR_4(lck->lk.num_polls);
2272 if (TCR_4(__kmp_nth) >
2273 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {
2276 if (num_polls > 1) {
2278 num_polls = TCR_4(lck->lk.num_polls);
2281 polls = (std::atomic<kmp_uint64> *)__kmp_allocate(num_polls *
2289 kmp_uint64 num_waiting = TCR_8(lck->lk.next_ticket) - ticket - 1;
2290 if (num_waiting > num_polls) {
2291 kmp_uint32 old_num_polls = num_polls;
2294 mask = (mask << 1) | 1;
2296 }
while (num_polls <= num_waiting);
2302 polls = (std::atomic<kmp_uint64> *)__kmp_allocate(num_polls *
2305 for (i = 0; i < old_num_polls; i++) {
2306 polls[i].store(old_polls[i]);
2321 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld reconfiguring "
2322 "lock %p to %d polls\n",
2323 ticket, lck, num_polls));
2325 lck->lk.old_polls = old_polls;
2326 lck->lk.polls = polls;
2330 lck->lk.num_polls = num_polls;
2331 lck->lk.mask = mask;
2339 lck->lk.cleanup_ticket = lck->lk.next_ticket;
2342 return KMP_LOCK_ACQUIRED_FIRST;
2345 int __kmp_acquire_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2346 int retval = __kmp_acquire_drdpa_lock_timed_template(lck, gtid);
2350 static int __kmp_acquire_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2352 char const *
const func =
"omp_set_lock";
2353 if (lck->lk.initialized != lck) {
2354 KMP_FATAL(LockIsUninitialized, func);
2356 if (__kmp_is_drdpa_lock_nestable(lck)) {
2357 KMP_FATAL(LockNestableUsedAsSimple, func);
2359 if ((gtid >= 0) && (__kmp_get_drdpa_lock_owner(lck) == gtid)) {
2360 KMP_FATAL(LockIsAlreadyOwned, func);
2363 __kmp_acquire_drdpa_lock(lck, gtid);
2365 lck->lk.owner_id = gtid + 1;
2366 return KMP_LOCK_ACQUIRED_FIRST;
2369 int __kmp_test_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2372 kmp_uint64 ticket = lck->lk.next_ticket;
2373 std::atomic<kmp_uint64> *polls = lck->lk.polls;
2374 kmp_uint64 mask = lck->lk.mask;
2375 if (polls[ticket & mask] == ticket) {
2376 kmp_uint64 next_ticket = ticket + 1;
2377 if (__kmp_atomic_compare_store_acq(&lck->lk.next_ticket, ticket,
2379 KMP_FSYNC_ACQUIRED(lck);
2380 KA_TRACE(1000, (
"__kmp_test_drdpa_lock: ticket #%lld acquired lock %p\n",
2382 lck->lk.now_serving = ticket;
2396 static int __kmp_test_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2398 char const *
const func =
"omp_test_lock";
2399 if (lck->lk.initialized != lck) {
2400 KMP_FATAL(LockIsUninitialized, func);
2402 if (__kmp_is_drdpa_lock_nestable(lck)) {
2403 KMP_FATAL(LockNestableUsedAsSimple, func);
2406 int retval = __kmp_test_drdpa_lock(lck, gtid);
2409 lck->lk.owner_id = gtid + 1;
2414 int __kmp_release_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2417 kmp_uint64 ticket = lck->lk.now_serving + 1;
2418 std::atomic<kmp_uint64> *polls = lck->lk.polls;
2419 kmp_uint64 mask = lck->lk.mask;
2420 KA_TRACE(1000, (
"__kmp_release_drdpa_lock: ticket #%lld released lock %p\n",
2422 KMP_FSYNC_RELEASING(lck);
2423 polls[ticket & mask] = ticket;
2424 return KMP_LOCK_RELEASED;
2427 static int __kmp_release_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2429 char const *
const func =
"omp_unset_lock";
2431 if (lck->lk.initialized != lck) {
2432 KMP_FATAL(LockIsUninitialized, func);
2434 if (__kmp_is_drdpa_lock_nestable(lck)) {
2435 KMP_FATAL(LockNestableUsedAsSimple, func);
2437 if (__kmp_get_drdpa_lock_owner(lck) == -1) {
2438 KMP_FATAL(LockUnsettingFree, func);
2440 if ((gtid >= 0) && (__kmp_get_drdpa_lock_owner(lck) >= 0) &&
2441 (__kmp_get_drdpa_lock_owner(lck) != gtid)) {
2442 KMP_FATAL(LockUnsettingSetByAnother, func);
2444 lck->lk.owner_id = 0;
2445 return __kmp_release_drdpa_lock(lck, gtid);
2448 void __kmp_init_drdpa_lock(kmp_drdpa_lock_t *lck) {
2449 lck->lk.location = NULL;
2451 lck->lk.num_polls = 1;
2452 lck->lk.polls = (std::atomic<kmp_uint64> *)__kmp_allocate(
2453 lck->lk.num_polls *
sizeof(*(lck->lk.polls)));
2454 lck->lk.cleanup_ticket = 0;
2455 lck->lk.old_polls = NULL;
2456 lck->lk.next_ticket = 0;
2457 lck->lk.now_serving = 0;
2458 lck->lk.owner_id = 0;
2459 lck->lk.depth_locked = -1;
2460 lck->lk.initialized = lck;
2462 KA_TRACE(1000, (
"__kmp_init_drdpa_lock: lock %p initialized\n", lck));
2465 void __kmp_destroy_drdpa_lock(kmp_drdpa_lock_t *lck) {
2466 lck->lk.initialized = NULL;
2467 lck->lk.location = NULL;
2468 if (lck->lk.polls.load() != NULL) {
2469 __kmp_free(lck->lk.polls.load());
2470 lck->lk.polls = NULL;
2472 if (lck->lk.old_polls != NULL) {
2473 __kmp_free(lck->lk.old_polls);
2474 lck->lk.old_polls = NULL;
2477 lck->lk.num_polls = 0;
2478 lck->lk.cleanup_ticket = 0;
2479 lck->lk.next_ticket = 0;
2480 lck->lk.now_serving = 0;
2481 lck->lk.owner_id = 0;
2482 lck->lk.depth_locked = -1;
2485 static void __kmp_destroy_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
2486 char const *
const func =
"omp_destroy_lock";
2487 if (lck->lk.initialized != lck) {
2488 KMP_FATAL(LockIsUninitialized, func);
2490 if (__kmp_is_drdpa_lock_nestable(lck)) {
2491 KMP_FATAL(LockNestableUsedAsSimple, func);
2493 if (__kmp_get_drdpa_lock_owner(lck) != -1) {
2494 KMP_FATAL(LockStillOwned, func);
2496 __kmp_destroy_drdpa_lock(lck);
2501 int __kmp_acquire_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2502 KMP_DEBUG_ASSERT(gtid >= 0);
2504 if (__kmp_get_drdpa_lock_owner(lck) == gtid) {
2505 lck->lk.depth_locked += 1;
2506 return KMP_LOCK_ACQUIRED_NEXT;
2508 __kmp_acquire_drdpa_lock_timed_template(lck, gtid);
2510 lck->lk.depth_locked = 1;
2512 lck->lk.owner_id = gtid + 1;
2513 return KMP_LOCK_ACQUIRED_FIRST;
2517 static void __kmp_acquire_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2519 char const *
const func =
"omp_set_nest_lock";
2520 if (lck->lk.initialized != lck) {
2521 KMP_FATAL(LockIsUninitialized, func);
2523 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2524 KMP_FATAL(LockSimpleUsedAsNestable, func);
2526 __kmp_acquire_nested_drdpa_lock(lck, gtid);
2529 int __kmp_test_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2532 KMP_DEBUG_ASSERT(gtid >= 0);
2534 if (__kmp_get_drdpa_lock_owner(lck) == gtid) {
2535 retval = ++lck->lk.depth_locked;
2536 }
else if (!__kmp_test_drdpa_lock(lck, gtid)) {
2540 retval = lck->lk.depth_locked = 1;
2542 lck->lk.owner_id = gtid + 1;
2547 static int __kmp_test_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2549 char const *
const func =
"omp_test_nest_lock";
2550 if (lck->lk.initialized != lck) {
2551 KMP_FATAL(LockIsUninitialized, func);
2553 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2554 KMP_FATAL(LockSimpleUsedAsNestable, func);
2556 return __kmp_test_nested_drdpa_lock(lck, gtid);
2559 int __kmp_release_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2560 KMP_DEBUG_ASSERT(gtid >= 0);
2563 if (--(lck->lk.depth_locked) == 0) {
2565 lck->lk.owner_id = 0;
2566 __kmp_release_drdpa_lock(lck, gtid);
2567 return KMP_LOCK_RELEASED;
2569 return KMP_LOCK_STILL_HELD;
2572 static int __kmp_release_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2574 char const *
const func =
"omp_unset_nest_lock";
2576 if (lck->lk.initialized != lck) {
2577 KMP_FATAL(LockIsUninitialized, func);
2579 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2580 KMP_FATAL(LockSimpleUsedAsNestable, func);
2582 if (__kmp_get_drdpa_lock_owner(lck) == -1) {
2583 KMP_FATAL(LockUnsettingFree, func);
2585 if (__kmp_get_drdpa_lock_owner(lck) != gtid) {
2586 KMP_FATAL(LockUnsettingSetByAnother, func);
2588 return __kmp_release_nested_drdpa_lock(lck, gtid);
2591 void __kmp_init_nested_drdpa_lock(kmp_drdpa_lock_t *lck) {
2592 __kmp_init_drdpa_lock(lck);
2593 lck->lk.depth_locked = 0;
2596 void __kmp_destroy_nested_drdpa_lock(kmp_drdpa_lock_t *lck) {
2597 __kmp_destroy_drdpa_lock(lck);
2598 lck->lk.depth_locked = 0;
2601 static void __kmp_destroy_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
2602 char const *
const func =
"omp_destroy_nest_lock";
2603 if (lck->lk.initialized != lck) {
2604 KMP_FATAL(LockIsUninitialized, func);
2606 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2607 KMP_FATAL(LockSimpleUsedAsNestable, func);
2609 if (__kmp_get_drdpa_lock_owner(lck) != -1) {
2610 KMP_FATAL(LockStillOwned, func);
2612 __kmp_destroy_nested_drdpa_lock(lck);
2617 static const ident_t *__kmp_get_drdpa_lock_location(kmp_drdpa_lock_t *lck) {
2618 return lck->lk.location;
2621 static void __kmp_set_drdpa_lock_location(kmp_drdpa_lock_t *lck,
2623 lck->lk.location = loc;
2626 static kmp_lock_flags_t __kmp_get_drdpa_lock_flags(kmp_drdpa_lock_t *lck) {
2627 return lck->lk.flags;
2630 static void __kmp_set_drdpa_lock_flags(kmp_drdpa_lock_t *lck,
2631 kmp_lock_flags_t flags) {
2632 lck->lk.flags = flags;
2636 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
2637 #define __kmp_tsc() __kmp_hardware_timestamp()
2639 kmp_backoff_t __kmp_spin_backoff_params = {1, 4096, 100};
2642 extern kmp_uint64 __kmp_now_nsec();
2643 kmp_backoff_t __kmp_spin_backoff_params = {1, 256, 100};
2644 #define __kmp_tsc() __kmp_now_nsec()
2654 static inline bool before(kmp_uint64 a, kmp_uint64 b) {
2655 return ((kmp_int64)b - (kmp_int64)a) > 0;
2659 void __kmp_spin_backoff(kmp_backoff_t *boff) {
2662 for (i = boff->step; i > 0; i--) {
2663 kmp_uint64 goal = __kmp_tsc() + boff->min_tick;
2665 if (__kmp_umwait_enabled) {
2666 __kmp_tpause(0, boff->min_tick);
2671 }
while (before(__kmp_tsc(), goal));
2676 boff->step = (boff->step << 1 | 1) & (boff->max_backoff - 1);
2679 #if KMP_USE_DYNAMIC_LOCK
2683 static void __kmp_init_direct_lock(kmp_dyna_lock_t *lck,
2684 kmp_dyna_lockseq_t seq) {
2685 TCW_4(((kmp_base_tas_lock_t *)lck)->poll, KMP_GET_D_TAG(seq));
2688 (
"__kmp_init_direct_lock: initialized direct lock with type#%d\n", seq));
2694 #define HLE_ACQUIRE ".byte 0xf2;"
2695 #define HLE_RELEASE ".byte 0xf3;"
2697 static inline kmp_uint32 swap4(kmp_uint32
volatile *p, kmp_uint32 v) {
2698 __asm__
volatile(HLE_ACQUIRE
"xchg %1,%0" :
"+r"(v),
"+m"(*p) : :
"memory");
2702 static void __kmp_destroy_hle_lock(kmp_dyna_lock_t *lck) { TCW_4(*lck, 0); }
2704 static void __kmp_destroy_hle_lock_with_checks(kmp_dyna_lock_t *lck) {
2708 static void __kmp_acquire_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2710 if (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle)) {
2713 while (*(kmp_uint32
volatile *)lck != KMP_LOCK_FREE(hle)) {
2714 for (
int i = delay; i != 0; --i)
2716 delay = ((delay << 1) | 1) & 7;
2718 }
while (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle));
2722 static void __kmp_acquire_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2724 __kmp_acquire_hle_lock(lck, gtid);
2727 static int __kmp_release_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2728 __asm__
volatile(HLE_RELEASE
"movl %1,%0"
2730 :
"r"(KMP_LOCK_FREE(hle))
2732 return KMP_LOCK_RELEASED;
2735 static int __kmp_release_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2737 return __kmp_release_hle_lock(lck, gtid);
2740 static int __kmp_test_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2741 return swap4(lck, KMP_LOCK_BUSY(1, hle)) == KMP_LOCK_FREE(hle);
2744 static int __kmp_test_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2746 return __kmp_test_hle_lock(lck, gtid);
2749 static void __kmp_init_rtm_queuing_lock(kmp_queuing_lock_t *lck) {
2750 __kmp_init_queuing_lock(lck);
2753 static void __kmp_destroy_rtm_queuing_lock(kmp_queuing_lock_t *lck) {
2754 __kmp_destroy_queuing_lock(lck);
2758 __kmp_destroy_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
2759 __kmp_destroy_queuing_lock_with_checks(lck);
2762 KMP_ATTRIBUTE_TARGET_RTM
2763 static void __kmp_acquire_rtm_queuing_lock(kmp_queuing_lock_t *lck,
2765 unsigned retries = 3, status;
2768 if (status == _XBEGIN_STARTED) {
2769 if (__kmp_is_unlocked_queuing_lock(lck))
2773 if ((status & _XABORT_EXPLICIT) && _XABORT_CODE(status) == 0xff) {
2775 while (!__kmp_is_unlocked_queuing_lock(lck)) {
2778 }
else if (!(status & _XABORT_RETRY))
2780 }
while (retries--);
2783 __kmp_acquire_queuing_lock(lck, gtid);
2786 static void __kmp_acquire_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
2788 __kmp_acquire_rtm_queuing_lock(lck, gtid);
2791 KMP_ATTRIBUTE_TARGET_RTM
2792 static int __kmp_release_rtm_queuing_lock(kmp_queuing_lock_t *lck,
2794 if (__kmp_is_unlocked_queuing_lock(lck)) {
2799 __kmp_release_queuing_lock(lck, gtid);
2801 return KMP_LOCK_RELEASED;
2804 static int __kmp_release_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
2806 return __kmp_release_rtm_queuing_lock(lck, gtid);
2809 KMP_ATTRIBUTE_TARGET_RTM
2810 static int __kmp_test_rtm_queuing_lock(kmp_queuing_lock_t *lck,
2812 unsigned retries = 3, status;
2815 if (status == _XBEGIN_STARTED && __kmp_is_unlocked_queuing_lock(lck)) {
2818 if (!(status & _XABORT_RETRY))
2820 }
while (retries--);
2822 return __kmp_test_queuing_lock(lck, gtid);
2825 static int __kmp_test_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
2827 return __kmp_test_rtm_queuing_lock(lck, gtid);
2831 typedef kmp_tas_lock_t kmp_rtm_spin_lock_t;
2833 static void __kmp_destroy_rtm_spin_lock(kmp_rtm_spin_lock_t *lck) {
2834 KMP_ATOMIC_ST_REL(&lck->lk.poll, 0);
2837 static void __kmp_destroy_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck) {
2838 __kmp_destroy_rtm_spin_lock(lck);
2841 KMP_ATTRIBUTE_TARGET_RTM
2842 static int __kmp_acquire_rtm_spin_lock(kmp_rtm_spin_lock_t *lck,
2844 unsigned retries = 3, status;
2845 kmp_int32 lock_free = KMP_LOCK_FREE(rtm_spin);
2846 kmp_int32 lock_busy = KMP_LOCK_BUSY(1, rtm_spin);
2849 if (status == _XBEGIN_STARTED) {
2850 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == lock_free)
2851 return KMP_LOCK_ACQUIRED_FIRST;
2854 if ((status & _XABORT_EXPLICIT) && _XABORT_CODE(status) == 0xff) {
2856 while (KMP_ATOMIC_LD_RLX(&lck->lk.poll) != lock_free) {
2859 }
else if (!(status & _XABORT_RETRY))
2861 }
while (retries--);
2864 KMP_FSYNC_PREPARE(lck);
2865 kmp_backoff_t backoff = __kmp_spin_backoff_params;
2866 while (KMP_ATOMIC_LD_RLX(&lck->lk.poll) != lock_free ||
2867 !__kmp_atomic_compare_store_acq(&lck->lk.poll, lock_free, lock_busy)) {
2868 __kmp_spin_backoff(&backoff);
2870 KMP_FSYNC_ACQUIRED(lck);
2871 return KMP_LOCK_ACQUIRED_FIRST;
2874 static int __kmp_acquire_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck,
2876 return __kmp_acquire_rtm_spin_lock(lck, gtid);
2879 KMP_ATTRIBUTE_TARGET_RTM
2880 static int __kmp_release_rtm_spin_lock(kmp_rtm_spin_lock_t *lck,
2882 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == KMP_LOCK_FREE(rtm_spin)) {
2887 KMP_FSYNC_RELEASING(lck);
2888 KMP_ATOMIC_ST_REL(&lck->lk.poll, KMP_LOCK_FREE(rtm_spin));
2890 return KMP_LOCK_RELEASED;
2893 static int __kmp_release_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck,
2895 return __kmp_release_rtm_spin_lock(lck, gtid);
2898 KMP_ATTRIBUTE_TARGET_RTM
2899 static int __kmp_test_rtm_spin_lock(kmp_rtm_spin_lock_t *lck, kmp_int32 gtid) {
2900 unsigned retries = 3, status;
2901 kmp_int32 lock_free = KMP_LOCK_FREE(rtm_spin);
2902 kmp_int32 lock_busy = KMP_LOCK_BUSY(1, rtm_spin);
2905 if (status == _XBEGIN_STARTED &&
2906 KMP_ATOMIC_LD_RLX(&lck->lk.poll) == lock_free) {
2909 if (!(status & _XABORT_RETRY))
2911 }
while (retries--);
2913 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == lock_free &&
2914 __kmp_atomic_compare_store_acq(&lck->lk.poll, lock_free, lock_busy)) {
2915 KMP_FSYNC_ACQUIRED(lck);
2921 static int __kmp_test_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck,
2923 return __kmp_test_rtm_spin_lock(lck, gtid);
2929 static void __kmp_init_indirect_lock(kmp_dyna_lock_t *l,
2930 kmp_dyna_lockseq_t tag);
2931 static void __kmp_destroy_indirect_lock(kmp_dyna_lock_t *lock);
2932 static int __kmp_set_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32);
2933 static int __kmp_unset_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32);
2934 static int __kmp_test_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32);
2935 static int __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
2937 static int __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
2939 static int __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
2943 #define KMP_FOREACH_LOCK_KIND(m, a) m(ticket, a) m(queuing, a) m(drdpa, a)
2945 #define expand1(lk, op) \
2946 static void __kmp_##op##_##lk##_##lock(kmp_user_lock_p lock) { \
2947 __kmp_##op##_##lk##_##lock(&lock->lk); \
2949 #define expand2(lk, op) \
2950 static int __kmp_##op##_##lk##_##lock(kmp_user_lock_p lock, \
2952 return __kmp_##op##_##lk##_##lock(&lock->lk, gtid); \
2954 #define expand3(lk, op) \
2955 static void __kmp_set_##lk##_##lock_flags(kmp_user_lock_p lock, \
2956 kmp_lock_flags_t flags) { \
2957 __kmp_set_##lk##_lock_flags(&lock->lk, flags); \
2959 #define expand4(lk, op) \
2960 static void __kmp_set_##lk##_##lock_location(kmp_user_lock_p lock, \
2961 const ident_t *loc) { \
2962 __kmp_set_##lk##_lock_location(&lock->lk, loc); \
2965 KMP_FOREACH_LOCK_KIND(expand1, init)
2966 KMP_FOREACH_LOCK_KIND(expand1, init_nested)
2967 KMP_FOREACH_LOCK_KIND(expand1, destroy)
2968 KMP_FOREACH_LOCK_KIND(expand1, destroy_nested)
2969 KMP_FOREACH_LOCK_KIND(expand2, acquire)
2970 KMP_FOREACH_LOCK_KIND(expand2, acquire_nested)
2971 KMP_FOREACH_LOCK_KIND(expand2, release)
2972 KMP_FOREACH_LOCK_KIND(expand2, release_nested)
2973 KMP_FOREACH_LOCK_KIND(expand2, test)
2974 KMP_FOREACH_LOCK_KIND(expand2, test_nested)
2975 KMP_FOREACH_LOCK_KIND(expand3, )
2976 KMP_FOREACH_LOCK_KIND(expand4, )
2987 #define expand(l, op) 0, __kmp_init_direct_lock,
2988 void (*__kmp_direct_init[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t) = {
2989 __kmp_init_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, init)};
2993 #define expand(l, op) 0, (void (*)(kmp_dyna_lock_t *))__kmp_##op##_##l##_lock,
2994 static void (*direct_destroy[])(kmp_dyna_lock_t *) = {
2995 __kmp_destroy_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, destroy)};
2997 #define expand(l, op) \
2998 0, (void (*)(kmp_dyna_lock_t *))__kmp_destroy_##l##_lock_with_checks,
2999 static void (*direct_destroy_check[])(kmp_dyna_lock_t *) = {
3000 __kmp_destroy_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, destroy)};
3004 #define expand(l, op) \
3005 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock,
3006 static int (*direct_set[])(kmp_dyna_lock_t *, kmp_int32) = {
3007 __kmp_set_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, acquire)};
3009 #define expand(l, op) \
3010 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock_with_checks,
3011 static int (*direct_set_check[])(kmp_dyna_lock_t *, kmp_int32) = {
3012 __kmp_set_indirect_lock_with_checks, 0,
3013 KMP_FOREACH_D_LOCK(expand, acquire)};
3017 #define expand(l, op) \
3018 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock,
3019 static int (*direct_unset[])(kmp_dyna_lock_t *, kmp_int32) = {
3020 __kmp_unset_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, release)};
3021 static int (*direct_test[])(kmp_dyna_lock_t *, kmp_int32) = {
3022 __kmp_test_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, test)};
3024 #define expand(l, op) \
3025 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock_with_checks,
3026 static int (*direct_unset_check[])(kmp_dyna_lock_t *, kmp_int32) = {
3027 __kmp_unset_indirect_lock_with_checks, 0,
3028 KMP_FOREACH_D_LOCK(expand, release)};
3029 static int (*direct_test_check[])(kmp_dyna_lock_t *, kmp_int32) = {
3030 __kmp_test_indirect_lock_with_checks, 0, KMP_FOREACH_D_LOCK(expand, test)};
3034 void (**__kmp_direct_destroy)(kmp_dyna_lock_t *) = 0;
3035 int (**__kmp_direct_set)(kmp_dyna_lock_t *, kmp_int32) = 0;
3036 int (**__kmp_direct_unset)(kmp_dyna_lock_t *, kmp_int32) = 0;
3037 int (**__kmp_direct_test)(kmp_dyna_lock_t *, kmp_int32) = 0;
3040 #define expand(l, op) (void (*)(kmp_user_lock_p)) __kmp_##op##_##l##_##lock,
3041 void (*__kmp_indirect_init[])(kmp_user_lock_p) = {
3042 KMP_FOREACH_I_LOCK(expand, init)};
3045 #define expand(l, op) (void (*)(kmp_user_lock_p)) __kmp_##op##_##l##_##lock,
3046 static void (*indirect_destroy[])(kmp_user_lock_p) = {
3047 KMP_FOREACH_I_LOCK(expand, destroy)};
3049 #define expand(l, op) \
3050 (void (*)(kmp_user_lock_p)) __kmp_##op##_##l##_##lock_with_checks,
3051 static void (*indirect_destroy_check[])(kmp_user_lock_p) = {
3052 KMP_FOREACH_I_LOCK(expand, destroy)};
3056 #define expand(l, op) \
3057 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock,
3058 static int (*indirect_set[])(kmp_user_lock_p,
3059 kmp_int32) = {KMP_FOREACH_I_LOCK(expand, acquire)};
3061 #define expand(l, op) \
3062 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock_with_checks,
3063 static int (*indirect_set_check[])(kmp_user_lock_p, kmp_int32) = {
3064 KMP_FOREACH_I_LOCK(expand, acquire)};
3068 #define expand(l, op) \
3069 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock,
3070 static int (*indirect_unset[])(kmp_user_lock_p, kmp_int32) = {
3071 KMP_FOREACH_I_LOCK(expand, release)};
3072 static int (*indirect_test[])(kmp_user_lock_p,
3073 kmp_int32) = {KMP_FOREACH_I_LOCK(expand, test)};
3075 #define expand(l, op) \
3076 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock_with_checks,
3077 static int (*indirect_unset_check[])(kmp_user_lock_p, kmp_int32) = {
3078 KMP_FOREACH_I_LOCK(expand, release)};
3079 static int (*indirect_test_check[])(kmp_user_lock_p, kmp_int32) = {
3080 KMP_FOREACH_I_LOCK(expand, test)};
3084 void (**__kmp_indirect_destroy)(kmp_user_lock_p) = 0;
3085 int (**__kmp_indirect_set)(kmp_user_lock_p, kmp_int32) = 0;
3086 int (**__kmp_indirect_unset)(kmp_user_lock_p, kmp_int32) = 0;
3087 int (**__kmp_indirect_test)(kmp_user_lock_p, kmp_int32) = 0;
3090 kmp_indirect_lock_table_t __kmp_i_lock_table;
3093 static kmp_uint32 __kmp_indirect_lock_size[KMP_NUM_I_LOCKS] = {0};
3096 void (*__kmp_indirect_set_location[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
3098 void (*__kmp_indirect_set_flags[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
3099 kmp_lock_flags_t) = {0};
3100 const ident_t *(*__kmp_indirect_get_location[KMP_NUM_I_LOCKS])(
3101 kmp_user_lock_p) = {0};
3102 kmp_lock_flags_t (*__kmp_indirect_get_flags[KMP_NUM_I_LOCKS])(
3103 kmp_user_lock_p) = {0};
3106 static kmp_indirect_lock_t *__kmp_indirect_lock_pool[KMP_NUM_I_LOCKS] = {0};
3113 kmp_indirect_lock_t *__kmp_allocate_indirect_lock(
void **user_lock,
3115 kmp_indirect_locktag_t tag) {
3116 kmp_indirect_lock_t *lck;
3117 kmp_lock_index_t idx, table_idx;
3119 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3121 if (__kmp_indirect_lock_pool[tag] != NULL) {
3123 lck = __kmp_indirect_lock_pool[tag];
3124 if (OMP_LOCK_T_SIZE <
sizeof(
void *))
3125 idx = lck->lock->pool.index;
3126 __kmp_indirect_lock_pool[tag] = (kmp_indirect_lock_t *)lck->lock->pool.next;
3127 KA_TRACE(20, (
"__kmp_allocate_indirect_lock: reusing an existing lock %p\n",
3130 kmp_uint32 row, col;
3131 kmp_indirect_lock_table_t *lock_table = &__kmp_i_lock_table;
3135 table_idx = lock_table->next;
3136 idx += lock_table->next;
3137 if (table_idx < lock_table->nrow_ptrs * KMP_I_LOCK_CHUNK) {
3138 row = table_idx / KMP_I_LOCK_CHUNK;
3139 col = table_idx % KMP_I_LOCK_CHUNK;
3141 if (!lock_table->table[row]) {
3142 lock_table->table[row] = (kmp_indirect_lock_t *)__kmp_allocate(
3143 sizeof(kmp_indirect_lock_t) * KMP_I_LOCK_CHUNK);
3148 if (!lock_table->next_table) {
3149 kmp_indirect_lock_table_t *next_table =
3150 (kmp_indirect_lock_table_t *)__kmp_allocate(
3151 sizeof(kmp_indirect_lock_table_t));
3152 next_table->table = (kmp_indirect_lock_t **)__kmp_allocate(
3153 sizeof(kmp_indirect_lock_t *) * 2 * lock_table->nrow_ptrs);
3154 next_table->nrow_ptrs = 2 * lock_table->nrow_ptrs;
3155 next_table->next = 0;
3156 next_table->next_table =
nullptr;
3157 lock_table->next_table = next_table;
3159 lock_table = lock_table->next_table;
3160 KMP_ASSERT(lock_table);
3164 lck = &lock_table->table[row][col];
3166 lck->lock = (kmp_user_lock_p)__kmp_allocate(__kmp_indirect_lock_size[tag]);
3168 (
"__kmp_allocate_indirect_lock: allocated a new lock %p\n", lck));
3171 __kmp_release_lock(&__kmp_global_lock, gtid);
3175 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3176 *(kmp_lock_index_t *)&(((kmp_base_tas_lock_t *)user_lock)->poll) =
3179 *((kmp_indirect_lock_t **)user_lock) = lck;
3186 static __forceinline kmp_indirect_lock_t *
3187 __kmp_lookup_indirect_lock(
void **user_lock,
const char *func) {
3188 if (__kmp_env_consistency_check) {
3189 kmp_indirect_lock_t *lck = NULL;
3190 if (user_lock == NULL) {
3191 KMP_FATAL(LockIsUninitialized, func);
3193 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3194 kmp_lock_index_t idx = KMP_EXTRACT_I_INDEX(user_lock);
3195 lck = __kmp_get_i_lock(idx);
3197 lck = *((kmp_indirect_lock_t **)user_lock);
3200 KMP_FATAL(LockIsUninitialized, func);
3204 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3205 return __kmp_get_i_lock(KMP_EXTRACT_I_INDEX(user_lock));
3207 return *((kmp_indirect_lock_t **)user_lock);
3212 static void __kmp_init_indirect_lock(kmp_dyna_lock_t *lock,
3213 kmp_dyna_lockseq_t seq) {
3214 #if KMP_USE_ADAPTIVE_LOCKS
3215 if (seq == lockseq_adaptive && !__kmp_cpuinfo.flags.rtm) {
3216 KMP_WARNING(AdaptiveNotSupported,
"kmp_lockseq_t",
"adaptive");
3217 seq = lockseq_queuing;
3221 if (seq == lockseq_rtm_queuing && !__kmp_cpuinfo.flags.rtm) {
3222 seq = lockseq_queuing;
3225 kmp_indirect_locktag_t tag = KMP_GET_I_TAG(seq);
3226 kmp_indirect_lock_t *l =
3227 __kmp_allocate_indirect_lock((
void **)lock, __kmp_entry_gtid(), tag);
3228 KMP_I_LOCK_FUNC(l, init)(l->lock);
3230 20, (
"__kmp_init_indirect_lock: initialized indirect lock with type#%d\n",
3234 static void __kmp_destroy_indirect_lock(kmp_dyna_lock_t *lock) {
3235 kmp_uint32 gtid = __kmp_entry_gtid();
3236 kmp_indirect_lock_t *l =
3237 __kmp_lookup_indirect_lock((
void **)lock,
"omp_destroy_lock");
3240 KMP_I_LOCK_FUNC(l, destroy)(l->lock);
3241 kmp_indirect_locktag_t tag = l->type;
3243 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3246 l->lock->pool.next = (kmp_user_lock_p)__kmp_indirect_lock_pool[tag];
3247 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3248 l->lock->pool.index = KMP_EXTRACT_I_INDEX(lock);
3250 __kmp_indirect_lock_pool[tag] = l;
3252 __kmp_release_lock(&__kmp_global_lock, gtid);
3255 static int __kmp_set_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) {
3256 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3257 return KMP_I_LOCK_FUNC(l, set)(l->lock, gtid);
3260 static int __kmp_unset_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) {
3261 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3262 return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3265 static int __kmp_test_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) {
3266 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3267 return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid);
3270 static int __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
3272 kmp_indirect_lock_t *l =
3273 __kmp_lookup_indirect_lock((
void **)lock,
"omp_set_lock");
3274 return KMP_I_LOCK_FUNC(l, set)(l->lock, gtid);
3277 static int __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
3279 kmp_indirect_lock_t *l =
3280 __kmp_lookup_indirect_lock((
void **)lock,
"omp_unset_lock");
3281 return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3284 static int __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
3286 kmp_indirect_lock_t *l =
3287 __kmp_lookup_indirect_lock((
void **)lock,
"omp_test_lock");
3288 return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid);
3291 kmp_dyna_lockseq_t __kmp_user_lock_seq = lockseq_queuing;
3294 kmp_int32 __kmp_get_user_lock_owner(kmp_user_lock_p lck, kmp_uint32 seq) {
3297 case lockseq_nested_tas:
3298 return __kmp_get_tas_lock_owner((kmp_tas_lock_t *)lck);
3301 case lockseq_nested_futex:
3302 return __kmp_get_futex_lock_owner((kmp_futex_lock_t *)lck);
3304 case lockseq_ticket:
3305 case lockseq_nested_ticket:
3306 return __kmp_get_ticket_lock_owner((kmp_ticket_lock_t *)lck);
3307 case lockseq_queuing:
3308 case lockseq_nested_queuing:
3309 #if KMP_USE_ADAPTIVE_LOCKS
3310 case lockseq_adaptive:
3312 return __kmp_get_queuing_lock_owner((kmp_queuing_lock_t *)lck);
3314 case lockseq_nested_drdpa:
3315 return __kmp_get_drdpa_lock_owner((kmp_drdpa_lock_t *)lck);
3322 void __kmp_init_dynamic_user_locks() {
3324 if (__kmp_env_consistency_check) {
3325 __kmp_direct_set = direct_set_check;
3326 __kmp_direct_unset = direct_unset_check;
3327 __kmp_direct_test = direct_test_check;
3328 __kmp_direct_destroy = direct_destroy_check;
3329 __kmp_indirect_set = indirect_set_check;
3330 __kmp_indirect_unset = indirect_unset_check;
3331 __kmp_indirect_test = indirect_test_check;
3332 __kmp_indirect_destroy = indirect_destroy_check;
3334 __kmp_direct_set = direct_set;
3335 __kmp_direct_unset = direct_unset;
3336 __kmp_direct_test = direct_test;
3337 __kmp_direct_destroy = direct_destroy;
3338 __kmp_indirect_set = indirect_set;
3339 __kmp_indirect_unset = indirect_unset;
3340 __kmp_indirect_test = indirect_test;
3341 __kmp_indirect_destroy = indirect_destroy;
3346 if (__kmp_init_user_locks)
3350 __kmp_i_lock_table.nrow_ptrs = KMP_I_LOCK_TABLE_INIT_NROW_PTRS;
3351 __kmp_i_lock_table.table = (kmp_indirect_lock_t **)__kmp_allocate(
3352 sizeof(kmp_indirect_lock_t *) * KMP_I_LOCK_TABLE_INIT_NROW_PTRS);
3353 *(__kmp_i_lock_table.table) = (kmp_indirect_lock_t *)__kmp_allocate(
3354 KMP_I_LOCK_CHUNK *
sizeof(kmp_indirect_lock_t));
3355 __kmp_i_lock_table.next = 0;
3356 __kmp_i_lock_table.next_table =
nullptr;
3359 __kmp_indirect_lock_size[locktag_ticket] =
sizeof(kmp_ticket_lock_t);
3360 __kmp_indirect_lock_size[locktag_queuing] =
sizeof(kmp_queuing_lock_t);
3361 #if KMP_USE_ADAPTIVE_LOCKS
3362 __kmp_indirect_lock_size[locktag_adaptive] =
sizeof(kmp_adaptive_lock_t);
3364 __kmp_indirect_lock_size[locktag_drdpa] =
sizeof(kmp_drdpa_lock_t);
3366 __kmp_indirect_lock_size[locktag_rtm_queuing] =
sizeof(kmp_queuing_lock_t);
3368 __kmp_indirect_lock_size[locktag_nested_tas] =
sizeof(kmp_tas_lock_t);
3370 __kmp_indirect_lock_size[locktag_nested_futex] =
sizeof(kmp_futex_lock_t);
3372 __kmp_indirect_lock_size[locktag_nested_ticket] =
sizeof(kmp_ticket_lock_t);
3373 __kmp_indirect_lock_size[locktag_nested_queuing] =
sizeof(kmp_queuing_lock_t);
3374 __kmp_indirect_lock_size[locktag_nested_drdpa] =
sizeof(kmp_drdpa_lock_t);
3377 #define fill_jumps(table, expand, sep) \
3379 table[locktag##sep##ticket] = expand(ticket); \
3380 table[locktag##sep##queuing] = expand(queuing); \
3381 table[locktag##sep##drdpa] = expand(drdpa); \
3384 #if KMP_USE_ADAPTIVE_LOCKS
3385 #define fill_table(table, expand) \
3387 fill_jumps(table, expand, _); \
3388 table[locktag_adaptive] = expand(queuing); \
3389 fill_jumps(table, expand, _nested_); \
3392 #define fill_table(table, expand) \
3394 fill_jumps(table, expand, _); \
3395 fill_jumps(table, expand, _nested_); \
3400 (void (*)(kmp_user_lock_p, const ident_t *)) __kmp_set_##l##_lock_location
3401 fill_table(__kmp_indirect_set_location, expand);
3404 (void (*)(kmp_user_lock_p, kmp_lock_flags_t)) __kmp_set_##l##_lock_flags
3405 fill_table(__kmp_indirect_set_flags, expand);
3408 (const ident_t *(*)(kmp_user_lock_p)) __kmp_get_##l##_lock_location
3409 fill_table(__kmp_indirect_get_location, expand);
3412 (kmp_lock_flags_t(*)(kmp_user_lock_p)) __kmp_get_##l##_lock_flags
3413 fill_table(__kmp_indirect_get_flags, expand);
3416 __kmp_init_user_locks = TRUE;
3420 void __kmp_cleanup_indirect_user_locks() {
3425 for (k = 0; k < KMP_NUM_I_LOCKS; ++k) {
3426 kmp_indirect_lock_t *l = __kmp_indirect_lock_pool[k];
3428 kmp_indirect_lock_t *ll = l;
3429 l = (kmp_indirect_lock_t *)l->lock->pool.next;
3430 KA_TRACE(20, (
"__kmp_cleanup_indirect_user_locks: freeing %p from pool\n",
3432 __kmp_free(ll->lock);
3435 __kmp_indirect_lock_pool[k] = NULL;
3438 kmp_indirect_lock_table_t *ptr = &__kmp_i_lock_table;
3440 for (kmp_uint32 row = 0; row < ptr->nrow_ptrs; ++row) {
3441 if (!ptr->table[row])
3443 for (kmp_uint32 col = 0; col < KMP_I_LOCK_CHUNK; ++col) {
3444 kmp_indirect_lock_t *l = &ptr->table[row][col];
3447 KMP_I_LOCK_FUNC(l, destroy)(l->lock);
3448 KA_TRACE(20, (
"__kmp_cleanup_indirect_user_locks: destroy/freeing %p "
3451 __kmp_free(l->lock);
3454 __kmp_free(ptr->table[row]);
3456 __kmp_free(ptr->table);
3457 kmp_indirect_lock_table_t *next_table = ptr->next_table;
3458 if (ptr != &__kmp_i_lock_table)
3463 __kmp_init_user_locks = FALSE;
3466 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3467 int __kmp_num_locks_in_block = 1;
3471 static void __kmp_init_tas_lock_with_checks(kmp_tas_lock_t *lck) {
3472 __kmp_init_tas_lock(lck);
3475 static void __kmp_init_nested_tas_lock_with_checks(kmp_tas_lock_t *lck) {
3476 __kmp_init_nested_tas_lock(lck);
3480 static void __kmp_init_futex_lock_with_checks(kmp_futex_lock_t *lck) {
3481 __kmp_init_futex_lock(lck);
3484 static void __kmp_init_nested_futex_lock_with_checks(kmp_futex_lock_t *lck) {
3485 __kmp_init_nested_futex_lock(lck);
3489 static int __kmp_is_ticket_lock_initialized(kmp_ticket_lock_t *lck) {
3490 return lck == lck->lk.self;
3493 static void __kmp_init_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
3494 __kmp_init_ticket_lock(lck);
3497 static void __kmp_init_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
3498 __kmp_init_nested_ticket_lock(lck);
3501 static int __kmp_is_queuing_lock_initialized(kmp_queuing_lock_t *lck) {
3502 return lck == lck->lk.initialized;
3505 static void __kmp_init_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
3506 __kmp_init_queuing_lock(lck);
3510 __kmp_init_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
3511 __kmp_init_nested_queuing_lock(lck);
3514 #if KMP_USE_ADAPTIVE_LOCKS
3515 static void __kmp_init_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck) {
3516 __kmp_init_adaptive_lock(lck);
3520 static int __kmp_is_drdpa_lock_initialized(kmp_drdpa_lock_t *lck) {
3521 return lck == lck->lk.initialized;
3524 static void __kmp_init_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
3525 __kmp_init_drdpa_lock(lck);
3528 static void __kmp_init_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
3529 __kmp_init_nested_drdpa_lock(lck);
3536 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3538 size_t __kmp_base_user_lock_size = 0;
3539 size_t __kmp_user_lock_size = 0;
3541 kmp_int32 (*__kmp_get_user_lock_owner_)(kmp_user_lock_p lck) = NULL;
3542 int (*__kmp_acquire_user_lock_with_checks_)(kmp_user_lock_p lck,
3543 kmp_int32 gtid) = NULL;
3545 int (*__kmp_test_user_lock_with_checks_)(kmp_user_lock_p lck,
3546 kmp_int32 gtid) = NULL;
3547 int (*__kmp_release_user_lock_with_checks_)(kmp_user_lock_p lck,
3548 kmp_int32 gtid) = NULL;
3549 void (*__kmp_init_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3550 void (*__kmp_destroy_user_lock_)(kmp_user_lock_p lck) = NULL;
3551 void (*__kmp_destroy_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3552 int (*__kmp_acquire_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3553 kmp_int32 gtid) = NULL;
3555 int (*__kmp_test_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3556 kmp_int32 gtid) = NULL;
3557 int (*__kmp_release_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3558 kmp_int32 gtid) = NULL;
3559 void (*__kmp_init_nested_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3560 void (*__kmp_destroy_nested_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3562 int (*__kmp_is_user_lock_initialized_)(kmp_user_lock_p lck) = NULL;
3563 const ident_t *(*__kmp_get_user_lock_location_)(kmp_user_lock_p lck) = NULL;
3564 void (*__kmp_set_user_lock_location_)(kmp_user_lock_p lck,
3566 kmp_lock_flags_t (*__kmp_get_user_lock_flags_)(kmp_user_lock_p lck) = NULL;
3567 void (*__kmp_set_user_lock_flags_)(kmp_user_lock_p lck,
3568 kmp_lock_flags_t flags) = NULL;
3570 void __kmp_set_user_lock_vptrs(kmp_lock_kind_t user_lock_kind) {
3571 switch (user_lock_kind) {
3577 __kmp_base_user_lock_size =
sizeof(kmp_base_tas_lock_t);
3578 __kmp_user_lock_size =
sizeof(kmp_tas_lock_t);
3580 __kmp_get_user_lock_owner_ =
3581 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_tas_lock_owner);
3583 if (__kmp_env_consistency_check) {
3584 KMP_BIND_USER_LOCK_WITH_CHECKS(tas);
3585 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(tas);
3587 KMP_BIND_USER_LOCK(tas);
3588 KMP_BIND_NESTED_USER_LOCK(tas);
3591 __kmp_destroy_user_lock_ =
3592 (void (*)(kmp_user_lock_p))(&__kmp_destroy_tas_lock);
3594 __kmp_is_user_lock_initialized_ = (int (*)(kmp_user_lock_p))NULL;
3596 __kmp_get_user_lock_location_ = (
const ident_t *(*)(kmp_user_lock_p))NULL;
3598 __kmp_set_user_lock_location_ =
3599 (void (*)(kmp_user_lock_p,
const ident_t *))NULL;
3601 __kmp_get_user_lock_flags_ = (kmp_lock_flags_t(*)(kmp_user_lock_p))NULL;
3603 __kmp_set_user_lock_flags_ =
3604 (void (*)(kmp_user_lock_p, kmp_lock_flags_t))NULL;
3610 __kmp_base_user_lock_size =
sizeof(kmp_base_futex_lock_t);
3611 __kmp_user_lock_size =
sizeof(kmp_futex_lock_t);
3613 __kmp_get_user_lock_owner_ =
3614 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_futex_lock_owner);
3616 if (__kmp_env_consistency_check) {
3617 KMP_BIND_USER_LOCK_WITH_CHECKS(futex);
3618 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(futex);
3620 KMP_BIND_USER_LOCK(futex);
3621 KMP_BIND_NESTED_USER_LOCK(futex);
3624 __kmp_destroy_user_lock_ =
3625 (void (*)(kmp_user_lock_p))(&__kmp_destroy_futex_lock);
3627 __kmp_is_user_lock_initialized_ = (int (*)(kmp_user_lock_p))NULL;
3629 __kmp_get_user_lock_location_ = (
const ident_t *(*)(kmp_user_lock_p))NULL;
3631 __kmp_set_user_lock_location_ =
3632 (void (*)(kmp_user_lock_p,
const ident_t *))NULL;
3634 __kmp_get_user_lock_flags_ = (kmp_lock_flags_t(*)(kmp_user_lock_p))NULL;
3636 __kmp_set_user_lock_flags_ =
3637 (void (*)(kmp_user_lock_p, kmp_lock_flags_t))NULL;
3643 __kmp_base_user_lock_size =
sizeof(kmp_base_ticket_lock_t);
3644 __kmp_user_lock_size =
sizeof(kmp_ticket_lock_t);
3646 __kmp_get_user_lock_owner_ =
3647 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_ticket_lock_owner);
3649 if (__kmp_env_consistency_check) {
3650 KMP_BIND_USER_LOCK_WITH_CHECKS(ticket);
3651 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(ticket);
3653 KMP_BIND_USER_LOCK(ticket);
3654 KMP_BIND_NESTED_USER_LOCK(ticket);
3657 __kmp_destroy_user_lock_ =
3658 (void (*)(kmp_user_lock_p))(&__kmp_destroy_ticket_lock);
3660 __kmp_is_user_lock_initialized_ =
3661 (int (*)(kmp_user_lock_p))(&__kmp_is_ticket_lock_initialized);
3663 __kmp_get_user_lock_location_ =
3664 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_ticket_lock_location);
3666 __kmp_set_user_lock_location_ = (void (*)(
3667 kmp_user_lock_p,
const ident_t *))(&__kmp_set_ticket_lock_location);
3669 __kmp_get_user_lock_flags_ =
3670 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_ticket_lock_flags);
3672 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3673 &__kmp_set_ticket_lock_flags);
3677 __kmp_base_user_lock_size =
sizeof(kmp_base_queuing_lock_t);
3678 __kmp_user_lock_size =
sizeof(kmp_queuing_lock_t);
3680 __kmp_get_user_lock_owner_ =
3681 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_owner);
3683 if (__kmp_env_consistency_check) {
3684 KMP_BIND_USER_LOCK_WITH_CHECKS(queuing);
3685 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(queuing);
3687 KMP_BIND_USER_LOCK(queuing);
3688 KMP_BIND_NESTED_USER_LOCK(queuing);
3691 __kmp_destroy_user_lock_ =
3692 (void (*)(kmp_user_lock_p))(&__kmp_destroy_queuing_lock);
3694 __kmp_is_user_lock_initialized_ =
3695 (int (*)(kmp_user_lock_p))(&__kmp_is_queuing_lock_initialized);
3697 __kmp_get_user_lock_location_ =
3698 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_location);
3700 __kmp_set_user_lock_location_ = (void (*)(
3701 kmp_user_lock_p,
const ident_t *))(&__kmp_set_queuing_lock_location);
3703 __kmp_get_user_lock_flags_ =
3704 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_flags);
3706 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3707 &__kmp_set_queuing_lock_flags);
3710 #if KMP_USE_ADAPTIVE_LOCKS
3712 __kmp_base_user_lock_size =
sizeof(kmp_base_adaptive_lock_t);
3713 __kmp_user_lock_size =
sizeof(kmp_adaptive_lock_t);
3715 __kmp_get_user_lock_owner_ =
3716 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_owner);
3718 if (__kmp_env_consistency_check) {
3719 KMP_BIND_USER_LOCK_WITH_CHECKS(adaptive);
3721 KMP_BIND_USER_LOCK(adaptive);
3724 __kmp_destroy_user_lock_ =
3725 (void (*)(kmp_user_lock_p))(&__kmp_destroy_adaptive_lock);
3727 __kmp_is_user_lock_initialized_ =
3728 (int (*)(kmp_user_lock_p))(&__kmp_is_queuing_lock_initialized);
3730 __kmp_get_user_lock_location_ =
3731 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_location);
3733 __kmp_set_user_lock_location_ = (void (*)(
3734 kmp_user_lock_p,
const ident_t *))(&__kmp_set_queuing_lock_location);
3736 __kmp_get_user_lock_flags_ =
3737 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_flags);
3739 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3740 &__kmp_set_queuing_lock_flags);
3746 __kmp_base_user_lock_size =
sizeof(kmp_base_drdpa_lock_t);
3747 __kmp_user_lock_size =
sizeof(kmp_drdpa_lock_t);
3749 __kmp_get_user_lock_owner_ =
3750 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_drdpa_lock_owner);
3752 if (__kmp_env_consistency_check) {
3753 KMP_BIND_USER_LOCK_WITH_CHECKS(drdpa);
3754 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(drdpa);
3756 KMP_BIND_USER_LOCK(drdpa);
3757 KMP_BIND_NESTED_USER_LOCK(drdpa);
3760 __kmp_destroy_user_lock_ =
3761 (void (*)(kmp_user_lock_p))(&__kmp_destroy_drdpa_lock);
3763 __kmp_is_user_lock_initialized_ =
3764 (int (*)(kmp_user_lock_p))(&__kmp_is_drdpa_lock_initialized);
3766 __kmp_get_user_lock_location_ =
3767 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_drdpa_lock_location);
3769 __kmp_set_user_lock_location_ = (void (*)(
3770 kmp_user_lock_p,
const ident_t *))(&__kmp_set_drdpa_lock_location);
3772 __kmp_get_user_lock_flags_ =
3773 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_drdpa_lock_flags);
3775 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3776 &__kmp_set_drdpa_lock_flags);
3784 kmp_lock_table_t __kmp_user_lock_table = {1, 0, NULL};
3785 kmp_user_lock_p __kmp_lock_pool = NULL;
3788 kmp_block_of_locks *__kmp_lock_blocks = NULL;
3789 int __kmp_num_locks_in_block = 1;
3791 static kmp_lock_index_t __kmp_lock_table_insert(kmp_user_lock_p lck) {
3793 kmp_lock_index_t index;
3794 if (__kmp_user_lock_table.used >= __kmp_user_lock_table.allocated) {
3795 kmp_lock_index_t size;
3796 kmp_user_lock_p *table;
3798 if (__kmp_user_lock_table.allocated == 0) {
3801 size = __kmp_user_lock_table.allocated * 2;
3803 table = (kmp_user_lock_p *)__kmp_allocate(
sizeof(kmp_user_lock_p) * size);
3804 KMP_MEMCPY(table + 1, __kmp_user_lock_table.table + 1,
3805 sizeof(kmp_user_lock_p) * (__kmp_user_lock_table.used - 1));
3806 table[0] = (kmp_user_lock_p)__kmp_user_lock_table.table;
3811 __kmp_user_lock_table.table = table;
3812 __kmp_user_lock_table.allocated = size;
3814 KMP_DEBUG_ASSERT(__kmp_user_lock_table.used <
3815 __kmp_user_lock_table.allocated);
3816 index = __kmp_user_lock_table.used;
3817 __kmp_user_lock_table.table[index] = lck;
3818 ++__kmp_user_lock_table.used;
3822 static kmp_user_lock_p __kmp_lock_block_allocate() {
3824 static int last_index = 0;
3825 if ((last_index >= __kmp_num_locks_in_block) || (__kmp_lock_blocks == NULL)) {
3829 KMP_DEBUG_ASSERT(__kmp_user_lock_size > 0);
3830 size_t space_for_locks = __kmp_user_lock_size * __kmp_num_locks_in_block;
3832 (
char *)__kmp_allocate(space_for_locks +
sizeof(kmp_block_of_locks));
3834 kmp_block_of_locks *new_block =
3835 (kmp_block_of_locks *)(&buffer[space_for_locks]);
3836 new_block->next_block = __kmp_lock_blocks;
3837 new_block->locks = (
void *)buffer;
3840 __kmp_lock_blocks = new_block;
3842 kmp_user_lock_p ret = (kmp_user_lock_p)(&(
3843 ((
char *)(__kmp_lock_blocks->locks))[last_index * __kmp_user_lock_size]));
3850 kmp_user_lock_p __kmp_user_lock_allocate(
void **user_lock, kmp_int32 gtid,
3851 kmp_lock_flags_t flags) {
3852 kmp_user_lock_p lck;
3853 kmp_lock_index_t index;
3854 KMP_DEBUG_ASSERT(user_lock);
3856 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3858 if (__kmp_lock_pool == NULL) {
3861 if (__kmp_num_locks_in_block <= 1) {
3862 lck = (kmp_user_lock_p)__kmp_allocate(__kmp_user_lock_size);
3864 lck = __kmp_lock_block_allocate();
3869 index = __kmp_lock_table_insert(lck);
3872 lck = __kmp_lock_pool;
3873 index = __kmp_lock_pool->pool.index;
3874 __kmp_lock_pool = __kmp_lock_pool->pool.next;
3879 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3880 *((kmp_lock_index_t *)user_lock) = index;
3882 *((kmp_user_lock_p *)user_lock) = lck;
3886 __kmp_set_user_lock_flags(lck, flags);
3888 __kmp_release_lock(&__kmp_global_lock, gtid);
3894 void __kmp_user_lock_free(
void **user_lock, kmp_int32 gtid,
3895 kmp_user_lock_p lck) {
3896 KMP_DEBUG_ASSERT(user_lock != NULL);
3897 KMP_DEBUG_ASSERT(lck != NULL);
3899 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3901 lck->pool.next = __kmp_lock_pool;
3902 __kmp_lock_pool = lck;
3903 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3904 kmp_lock_index_t index = *((kmp_lock_index_t *)user_lock);
3905 KMP_DEBUG_ASSERT(0 < index && index <= __kmp_user_lock_table.used);
3906 lck->pool.index = index;
3909 __kmp_release_lock(&__kmp_global_lock, gtid);
3912 kmp_user_lock_p __kmp_lookup_user_lock(
void **user_lock,
char const *func) {
3913 kmp_user_lock_p lck = NULL;
3915 if (__kmp_env_consistency_check) {
3916 if (user_lock == NULL) {
3917 KMP_FATAL(LockIsUninitialized, func);
3921 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3922 kmp_lock_index_t index = *((kmp_lock_index_t *)user_lock);
3923 if (__kmp_env_consistency_check) {
3924 if (!(0 < index && index < __kmp_user_lock_table.used)) {
3925 KMP_FATAL(LockIsUninitialized, func);
3928 KMP_DEBUG_ASSERT(0 < index && index < __kmp_user_lock_table.used);
3929 KMP_DEBUG_ASSERT(__kmp_user_lock_size > 0);
3930 lck = __kmp_user_lock_table.table[index];
3932 lck = *((kmp_user_lock_p *)user_lock);
3935 if (__kmp_env_consistency_check) {
3937 KMP_FATAL(LockIsUninitialized, func);
3944 void __kmp_cleanup_user_locks(
void) {
3947 __kmp_lock_pool = NULL;
3949 #define IS_CRITICAL(lck) \
3950 ((__kmp_get_user_lock_flags_ != NULL) && \
3951 ((*__kmp_get_user_lock_flags_)(lck)&kmp_lf_critical_section))
3976 while (__kmp_user_lock_table.used > 1) {
3981 kmp_user_lock_p lck =
3982 __kmp_user_lock_table.table[--__kmp_user_lock_table.used];
3984 if ((__kmp_is_user_lock_initialized_ != NULL) &&
3985 (*__kmp_is_user_lock_initialized_)(lck)) {
3989 if (__kmp_env_consistency_check && (!IS_CRITICAL(lck)) &&
3990 ((loc = __kmp_get_user_lock_location(lck)) != NULL) &&
3992 kmp_str_loc_t str_loc = __kmp_str_loc_init(loc->
psource,
false);
3993 KMP_WARNING(CnsLockNotDestroyed, str_loc.file, str_loc.line);
3994 __kmp_str_loc_free(&str_loc);
3998 if (IS_CRITICAL(lck)) {
4001 (
"__kmp_cleanup_user_locks: free critical section lock %p (%p)\n",
4002 lck, *(
void **)lck));
4004 KA_TRACE(20, (
"__kmp_cleanup_user_locks: free lock %p (%p)\n", lck,
4010 __kmp_destroy_user_lock(lck);
4014 if (__kmp_lock_blocks == NULL) {
4022 kmp_user_lock_p *table_ptr = __kmp_user_lock_table.table;
4023 __kmp_user_lock_table.table = NULL;
4024 __kmp_user_lock_table.allocated = 0;
4026 while (table_ptr != NULL) {
4029 kmp_user_lock_p *next = (kmp_user_lock_p *)(table_ptr[0]);
4030 __kmp_free(table_ptr);
4035 kmp_block_of_locks_t *block_ptr = __kmp_lock_blocks;
4036 __kmp_lock_blocks = NULL;
4038 while (block_ptr != NULL) {
4039 kmp_block_of_locks_t *next = block_ptr->next_block;
4040 __kmp_free(block_ptr->locks);
4045 TCW_4(__kmp_init_user_locks, FALSE);
void open(const char *filename, const char *mode, const char *env_var=nullptr)