13#ifndef KMP_WAIT_RELEASE_H
14#define KMP_WAIT_RELEASE_H
20#include "ompt-specific.h"
36struct flag_properties {
37 unsigned int type : 16;
38 unsigned int reserved : 16;
41template <enum flag_type FlagType>
struct flag_traits {};
43template <>
struct flag_traits<flag32> {
44 typedef kmp_uint32 flag_t;
45 static const flag_type t = flag32;
46 static inline flag_t tcr(flag_t f) {
return TCR_4(f); }
47 static inline flag_t test_then_add4(
volatile flag_t *f) {
48 return KMP_TEST_THEN_ADD4_32(RCAST(
volatile kmp_int32 *, f));
50 static inline flag_t test_then_or(
volatile flag_t *f, flag_t v) {
51 return KMP_TEST_THEN_OR32(f, v);
53 static inline flag_t test_then_and(
volatile flag_t *f, flag_t v) {
54 return KMP_TEST_THEN_AND32(f, v);
58template <>
struct flag_traits<atomic_flag64> {
59 typedef kmp_uint64 flag_t;
60 static const flag_type t = atomic_flag64;
61 static inline flag_t tcr(flag_t f) {
return TCR_8(f); }
62 static inline flag_t test_then_add4(
volatile flag_t *f) {
63 return KMP_TEST_THEN_ADD4_64(RCAST(
volatile kmp_int64 *, f));
65 static inline flag_t test_then_or(
volatile flag_t *f, flag_t v) {
66 return KMP_TEST_THEN_OR64(f, v);
68 static inline flag_t test_then_and(
volatile flag_t *f, flag_t v) {
69 return KMP_TEST_THEN_AND64(f, v);
73template <>
struct flag_traits<flag64> {
74 typedef kmp_uint64 flag_t;
75 static const flag_type t = flag64;
76 static inline flag_t tcr(flag_t f) {
return TCR_8(f); }
77 static inline flag_t test_then_add4(
volatile flag_t *f) {
78 return KMP_TEST_THEN_ADD4_64(RCAST(
volatile kmp_int64 *, f));
80 static inline flag_t test_then_or(
volatile flag_t *f, flag_t v) {
81 return KMP_TEST_THEN_OR64(f, v);
83 static inline flag_t test_then_and(
volatile flag_t *f, flag_t v) {
84 return KMP_TEST_THEN_AND64(f, v);
88template <>
struct flag_traits<flag_oncore> {
89 typedef kmp_uint64 flag_t;
90 static const flag_type t = flag_oncore;
91 static inline flag_t tcr(flag_t f) {
return TCR_8(f); }
92 static inline flag_t test_then_add4(
volatile flag_t *f) {
93 return KMP_TEST_THEN_ADD4_64(RCAST(
volatile kmp_int64 *, f));
95 static inline flag_t test_then_or(
volatile flag_t *f, flag_t v) {
96 return KMP_TEST_THEN_OR64(f, v);
98 static inline flag_t test_then_and(
volatile flag_t *f, flag_t v) {
99 return KMP_TEST_THEN_AND64(f, v);
108 kmp_info_t *waiting_threads[1] = {
nullptr};
110 std::atomic<bool> *sleepLoc;
113 typedef flag_traits<FlagType> traits_type;
126 return waiting_threads[i];
133 waiting_threads[0] = thr;
136 enum barrier_type get_bt() {
return bs_last_barrier; }
140template <
typename PtrType, flag_type FlagType,
bool Sleepable>
143 volatile PtrType *loc;
145 typedef flag_traits<FlagType> traits_type;
148 typedef PtrType flag_t;
152 this->waiting_threads[0] = thr;
156 kmp_flag_native(
volatile PtrType *p, PtrType c, std::atomic<bool> *sloc)
159 void *
operator new(
size_t size) {
return __kmp_allocate(size); }
160 void operator delete(
void *p) { __kmp_free(p); }
161 volatile PtrType *get() {
return loc; }
162 void *get_void_p() {
return RCAST(
void *, CCAST(PtrType *, loc)); }
163 void set(
volatile PtrType *new_loc) { loc = new_loc; }
164 PtrType load() {
return *loc; }
165 void store(PtrType val) { *loc = val; }
168 if (Sleepable && !(this->sleepLoc))
169 return (traits_type::tcr(*(this->get())) & ~KMP_BARRIER_SLEEP_STATE) ==
172 return traits_type::tcr(*(this->get())) ==
checker;
183 return traits_type::tcr(*(this->get())) !=
checker;
188 (void)traits_type::test_then_add4((
volatile PtrType *)this->get());
194 if (this->sleepLoc) {
195 this->sleepLoc->store(
true);
196 return *(this->get());
198 return traits_type::test_then_or((
volatile PtrType *)this->get(),
199 KMP_BARRIER_SLEEP_STATE);
205 if (this->sleepLoc) {
206 this->sleepLoc->store(
false);
209 traits_type::test_then_and((
volatile PtrType *)this->get(),
210 ~KMP_BARRIER_SLEEP_STATE);
216 return this->sleepLoc->load();
217 return old_loc & KMP_BARRIER_SLEEP_STATE;
222 return this->sleepLoc->load();
225 bool is_any_sleeping() {
227 return this->sleepLoc->load();
230 kmp_uint8 *get_stolen() {
return NULL; }
234template <
typename PtrType, flag_type FlagType,
bool Sleepable>
237 std::atomic<PtrType> *
loc;
240 typedef flag_traits<FlagType> traits_type;
241 typedef PtrType flag_t;
245 this->waiting_threads[0] = thr;
249 kmp_flag_atomic(std::atomic<PtrType> *p, PtrType c, std::atomic<bool> *sloc)
252 std::atomic<PtrType> *
get() {
return loc; }
256 void set(std::atomic<PtrType> *new_loc) {
loc = new_loc; }
258 PtrType
load() {
return loc->load(std::memory_order_acquire); }
260 void store(PtrType val) {
loc->store(val, std::memory_order_release); }
263 if (Sleepable && !(this->sleepLoc))
264 return (this->
load() & ~KMP_BARRIER_SLEEP_STATE) ==
checker;
284 if (this->sleepLoc) {
285 this->sleepLoc->store(
true);
286 return *(this->
get());
288 return KMP_ATOMIC_OR(this->
get(), KMP_BARRIER_SLEEP_STATE);
294 if (this->sleepLoc) {
295 this->sleepLoc->store(
false);
298 KMP_ATOMIC_AND(this->
get(), ~KMP_BARRIER_SLEEP_STATE);
304 return this->sleepLoc->load();
305 return old_loc & KMP_BARRIER_SLEEP_STATE;
310 return this->sleepLoc->load();
313 bool is_any_sleeping() {
315 return this->sleepLoc->load();
318 kmp_uint8 *get_stolen() {
return NULL; }
323static void __ompt_implicit_task_end(kmp_info_t *this_thr,
324 ompt_state_t ompt_state,
326 int ds_tid = this_thr->th.th_info.ds.ds_tid;
327 if (ompt_state == ompt_state_wait_barrier_implicit_parallel ||
328 ompt_state == ompt_state_wait_barrier_teams) {
329 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
331 void *codeptr = NULL;
332 ompt_sync_region_t sync_kind = ompt_sync_region_barrier_implicit_parallel;
333 if (this_thr->th.ompt_thread_info.parallel_flags & ompt_parallel_league)
334 sync_kind = ompt_sync_region_barrier_teams;
335 if (ompt_enabled.ompt_callback_sync_region_wait) {
336 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
337 sync_kind, ompt_scope_end, NULL, tId, codeptr);
339 if (ompt_enabled.ompt_callback_sync_region) {
340 ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
341 sync_kind, ompt_scope_end, NULL, tId, codeptr);
344 if (!KMP_MASTER_TID(ds_tid)) {
345 if (ompt_enabled.ompt_callback_implicit_task) {
346 int flags = this_thr->th.ompt_thread_info.parallel_flags;
347 flags = (flags & ompt_parallel_league) ? ompt_task_initial
348 : ompt_task_implicit;
349 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
350 ompt_scope_end, NULL, tId, 0, ds_tid, flags);
353 this_thr->th.ompt_thread_info.state = ompt_state_idle;
355 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
366template <
class C,
bool final_spin,
bool Cancellable =
false,
367 bool Sleepable =
true>
369__kmp_wait_template(kmp_info_t *this_thr,
370 C *flag USE_ITT_BUILD_ARG(
void *itt_sync_obj)) {
371#if USE_ITT_BUILD && USE_ITT_NOTIFY
372 volatile void *spin = flag->get();
376 int tasks_completed = FALSE;
378 kmp_uint64 poll_count;
379 kmp_uint64 hibernate_goal;
381 kmp_uint32 hibernate;
385 KMP_FSYNC_SPIN_INIT(spin, NULL);
386 if (flag->done_check()) {
387 KMP_FSYNC_SPIN_ACQUIRED(CCAST(
void *, spin));
390 th_gtid = this_thr->th.th_info.ds.ds_gtid;
392 kmp_team_t *team = this_thr->th.th_team;
393 if (team && team->t.t_cancel_request == cancel_parallel)
398 KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking,
true);
401 (
"__kmp_wait_sleep: T#%d waiting for flag(%p)\n", th_gtid, flag));
457 ompt_state_t ompt_entry_state;
459 if (ompt_enabled.enabled) {
460 ompt_entry_state = this_thr->th.ompt_thread_info.state;
462 (ompt_entry_state != ompt_state_wait_barrier_implicit_parallel &&
463 ompt_entry_state != ompt_state_wait_barrier_teams) ||
464 KMP_MASTER_TID(this_thr->th.th_info.ds.ds_tid)) {
465 ompt_lw_taskteam_t *team = NULL;
466 if (this_thr->th.th_team)
467 team = this_thr->th.th_team->t.ompt_serialized_team_info;
469 tId = &(team->ompt_task_info.task_data);
471 tId = OMPT_CUR_TASK_DATA(this_thr);
474 tId = &(this_thr->th.ompt_thread_info.task_data);
476 if (final_spin && (__kmp_tasking_mode == tskm_immediate_exec ||
477 this_thr->th.th_task_team == NULL)) {
479 __ompt_implicit_task_end(this_thr, ompt_entry_state, tId);
484 KMP_INIT_YIELD(spins);
485 KMP_INIT_BACKOFF(time);
487 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME ||
488 __kmp_pause_status == kmp_soft_paused) {
492#ifdef KMP_ADJUST_BLOCKTIME
493 if (__kmp_pause_status == kmp_soft_paused ||
494 (__kmp_zero_bt && !this_thr->th.th_team_bt_set))
499 hibernate = this_thr->th.th_team_bt_intervals;
501 hibernate = this_thr->th.th_team_bt_intervals;
512 hibernate += TCR_4(__kmp_global.g.g_time.dt.t_value);
513 KF_TRACE(20, (
"__kmp_wait_sleep: T#%d now=%d, hibernate=%d, intervals=%d\n",
514 th_gtid, __kmp_global.g.g_time.dt.t_value, hibernate,
515 hibernate - __kmp_global.g.g_time.dt.t_value));
517 if (__kmp_pause_status == kmp_soft_paused) {
519 hibernate_goal = KMP_NOW();
521 hibernate_goal = KMP_NOW() + this_thr->th.th_team_bt_intervals;
530 while (flag->notdone_check()) {
531 kmp_task_team_t *task_team = NULL;
532 if (__kmp_tasking_mode != tskm_immediate_exec) {
533 task_team = this_thr->th.th_task_team;
541 if (task_team != NULL) {
542 if (TCR_SYNC_4(task_team->tt.tt_active)) {
543 if (KMP_TASKING_ENABLED(task_team)) {
545 this_thr, th_gtid, final_spin,
546 &tasks_completed USE_ITT_BUILD_ARG(itt_sync_obj), 0);
548 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;
550 KMP_DEBUG_ASSERT(!KMP_MASTER_TID(this_thr->th.th_info.ds.ds_tid));
553 if (final_spin && ompt_enabled.enabled)
554 __ompt_implicit_task_end(this_thr, ompt_entry_state, tId);
556 this_thr->th.th_task_team = NULL;
557 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;
560 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;
564 KMP_FSYNC_SPIN_PREPARE(CCAST(
void *, spin));
565 if (TCR_4(__kmp_global.g.g_done)) {
566 if (__kmp_global.g.g_abort)
567 __kmp_abort_thread();
573 KMP_YIELD_OVERSUB_ELSE_SPIN(spins, time);
578 if (this_thr->th.th_stats->isIdle() &&
579 KMP_GET_THREAD_STATE() == FORK_JOIN_BARRIER) {
580 KMP_SET_THREAD_STATE(IDLE);
581 KMP_PUSH_PARTITIONED_TIMER(OMP_idle);
586 kmp_team_t *team = this_thr->th.th_team;
587 if (team && team->t.t_cancel_request == cancel_parallel)
601 if (task_team && KMP_HIDDEN_HELPER_WORKER_THREAD(th_gtid) &&
602 !TCR_4(__kmp_hidden_helper_team_done)) {
605 if (KMP_ATOMIC_LD_ACQ(&__kmp_unexecuted_hidden_helper_tasks) == 0) {
606 __kmp_hidden_helper_worker_thread_wait();
612 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
613 __kmp_pause_status != kmp_soft_paused)
617 if (task_team != NULL && TCR_4(task_team->tt.tt_found_tasks) &&
618 !__kmp_wpolicy_passive)
623 if (TCR_4(__kmp_global.g.g_time.dt.t_value) < hibernate)
626 if (KMP_BLOCKING(hibernate_goal, poll_count++))
634#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
635 if (__kmp_mwait_enabled || __kmp_umwait_enabled) {
636 KF_TRACE(50, (
"__kmp_wait_sleep: T#%d using monitor/mwait\n", th_gtid));
637 flag->mwait(th_gtid);
640 KF_TRACE(50, (
"__kmp_wait_sleep: T#%d suspend time reached\n", th_gtid));
643 KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking,
false);
645 flag->suspend(th_gtid);
648 KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking,
true);
650#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
654 if (TCR_4(__kmp_global.g.g_done)) {
655 if (__kmp_global.g.g_abort)
656 __kmp_abort_thread();
658 }
else if (__kmp_tasking_mode != tskm_immediate_exec &&
659 this_thr->th.th_reap_state == KMP_SAFE_TO_REAP) {
660 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP;
666 ompt_state_t ompt_exit_state = this_thr->th.ompt_thread_info.state;
667 if (ompt_enabled.enabled && ompt_exit_state != ompt_state_undefined) {
670 __ompt_implicit_task_end(this_thr, ompt_exit_state, tId);
671 ompt_exit_state = this_thr->th.ompt_thread_info.state;
674 if (ompt_exit_state == ompt_state_idle) {
675 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
681 if (KMP_GET_THREAD_STATE() == IDLE) {
682 KMP_POP_PARTITIONED_TIMER();
683 KMP_SET_THREAD_STATE(thread_state);
684 this_thr->th.th_stats->resetIdleFlag();
690 KMP_ATOMIC_ST_REL(&this_thr->th.th_blocking,
false);
692 KMP_FSYNC_SPIN_ACQUIRED(CCAST(
void *, spin));
694 kmp_team_t *team = this_thr->th.th_team;
695 if (team && team->t.t_cancel_request == cancel_parallel) {
696 if (tasks_completed) {
699 kmp_task_team_t *task_team = this_thr->th.th_task_team;
700 std::atomic<kmp_int32> *unfinished_threads =
701 &(task_team->tt.tt_unfinished_threads);
702 KMP_ATOMIC_INC(unfinished_threads);
710#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
714static inline void __kmp_mwait_template(
int th_gtid, C *flag) {
715 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_mwait);
716 kmp_info_t *th = __kmp_threads[th_gtid];
718 KF_TRACE(30, (
"__kmp_mwait_template: T#%d enter for flag = %p\n", th_gtid,
722 KMP_DEBUG_ASSERT(__kmp_mwait_enabled || __kmp_umwait_enabled);
724 __kmp_suspend_initialize_thread(th);
725 __kmp_lock_suspend_mx(th);
727 volatile void *spin = flag->get();
728 void *cacheline = (
void *)(kmp_uintptr_t(spin) & ~(CACHE_LINE - 1));
730 if (!flag->done_check()) {
732 th->th.th_active = FALSE;
733 if (th->th.th_active_in_pool) {
734 th->th.th_active_in_pool = FALSE;
735 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
736 KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
738 flag->set_sleeping();
739 KF_TRACE(50, (
"__kmp_mwait_template: T#%d calling monitor\n", th_gtid));
741 if (__kmp_umwait_enabled) {
742 __kmp_umonitor(cacheline);
745 if (__kmp_mwait_enabled) {
746 __kmp_mm_monitor(cacheline, 0, 0);
752 if (flag->done_check())
753 flag->unset_sleeping();
756 TCW_PTR(th->th.th_sleep_loc, (
void *)flag);
757 th->th.th_sleep_loc_type = flag->get_type();
758 __kmp_unlock_suspend_mx(th);
759 KF_TRACE(50, (
"__kmp_mwait_template: T#%d calling mwait\n", th_gtid));
761 if (__kmp_umwait_enabled) {
762 __kmp_umwait(1, 100);
765 if (__kmp_mwait_enabled) {
766 __kmp_mm_mwait(0, __kmp_mwait_hints);
769 KF_TRACE(50, (
"__kmp_mwait_template: T#%d mwait done\n", th_gtid));
770 __kmp_lock_suspend_mx(th);
772 if (flag->is_sleeping())
773 flag->unset_sleeping();
774 TCW_PTR(th->th.th_sleep_loc, NULL);
775 th->th.th_sleep_loc_type = flag_unset;
778 th->th.th_active = TRUE;
779 if (TCR_4(th->th.th_in_pool)) {
780 KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
781 th->th.th_active_in_pool = TRUE;
784 __kmp_unlock_suspend_mx(th);
785 KF_TRACE(30, (
"__kmp_mwait_template: T#%d exit\n", th_gtid));
793template <
class C>
static inline void __kmp_release_template(C *flag) {
795 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
797 KF_TRACE(20, (
"__kmp_release: T#%d releasing flag(%x)\n", gtid, flag->get()));
798 KMP_DEBUG_ASSERT(flag->get());
799 KMP_FSYNC_RELEASING(flag->get_void_p());
801 flag->internal_release();
803 KF_TRACE(100, (
"__kmp_release: T#%d set new spin=%d\n", gtid, flag->get(),
806 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
809 if (flag->is_any_sleeping()) {
810 for (
unsigned int i = 0; i < flag->get_num_waiters(); ++i) {
812 kmp_info_t *waiter = flag->get_waiter(i);
814 int wait_gtid = waiter->th.th_info.ds.ds_gtid;
816 KF_TRACE(50, (
"__kmp_release: T#%d waking up thread T#%d since sleep "
818 gtid, wait_gtid, flag->get()));
819 flag->resume(wait_gtid);
826template <
bool Cancellable,
bool Sleepable>
827class kmp_flag_32 :
public kmp_flag_atomic<kmp_uint32, flag32, Sleepable> {
829 kmp_flag_32(std::atomic<kmp_uint32> *p)
831 kmp_flag_32(std::atomic<kmp_uint32> *p, kmp_info_t *thr)
833 kmp_flag_32(std::atomic<kmp_uint32> *p, kmp_uint32 c)
835 void suspend(
int th_gtid) { __kmp_suspend_32(th_gtid,
this); }
836#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
837 void mwait(
int th_gtid) { __kmp_mwait_32(th_gtid,
this); }
839 void resume(
int th_gtid) { __kmp_resume_32(th_gtid,
this); }
840 int execute_tasks(kmp_info_t *this_thr, kmp_int32 gtid,
int final_spin,
841 int *thread_finished USE_ITT_BUILD_ARG(
void *itt_sync_obj),
842 kmp_int32 is_constrained) {
843 return __kmp_execute_tasks_32(
844 this_thr, gtid,
this, final_spin,
845 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
847 bool wait(kmp_info_t *this_thr,
848 int final_spin USE_ITT_BUILD_ARG(
void *itt_sync_obj)) {
850 return __kmp_wait_template<kmp_flag_32, TRUE, Cancellable, Sleepable>(
851 this_thr,
this USE_ITT_BUILD_ARG(itt_sync_obj));
853 return __kmp_wait_template<kmp_flag_32, FALSE, Cancellable, Sleepable>(
854 this_thr,
this USE_ITT_BUILD_ARG(itt_sync_obj));
856 void release() { __kmp_release_template(
this); }
857 flag_type get_ptr_type() {
return flag32; }
860template <
bool Cancellable,
bool Sleepable>
861class kmp_flag_64 :
public kmp_flag_native<kmp_uint64, flag64, Sleepable> {
863 kmp_flag_64(
volatile kmp_uint64 *p)
865 kmp_flag_64(
volatile kmp_uint64 *p, kmp_info_t *thr)
867 kmp_flag_64(
volatile kmp_uint64 *p, kmp_uint64 c)
869 kmp_flag_64(
volatile kmp_uint64 *p, kmp_uint64 c, std::atomic<bool> *loc)
871 void suspend(
int th_gtid) { __kmp_suspend_64(th_gtid,
this); }
872#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
873 void mwait(
int th_gtid) { __kmp_mwait_64(th_gtid,
this); }
875 void resume(
int th_gtid) { __kmp_resume_64(th_gtid,
this); }
876 int execute_tasks(kmp_info_t *this_thr, kmp_int32 gtid,
int final_spin,
877 int *thread_finished USE_ITT_BUILD_ARG(
void *itt_sync_obj),
878 kmp_int32 is_constrained) {
879 return __kmp_execute_tasks_64(
880 this_thr, gtid,
this, final_spin,
881 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
883 bool wait(kmp_info_t *this_thr,
884 int final_spin USE_ITT_BUILD_ARG(
void *itt_sync_obj)) {
886 return __kmp_wait_template<kmp_flag_64, TRUE, Cancellable, Sleepable>(
887 this_thr,
this USE_ITT_BUILD_ARG(itt_sync_obj));
889 return __kmp_wait_template<kmp_flag_64, FALSE, Cancellable, Sleepable>(
890 this_thr,
this USE_ITT_BUILD_ARG(itt_sync_obj));
892 void release() { __kmp_release_template(
this); }
893 flag_type get_ptr_type() {
return flag64; }
896template <
bool Cancellable,
bool Sleepable>
897class kmp_atomic_flag_64
900 kmp_atomic_flag_64(std::atomic<kmp_uint64> *p)
902 kmp_atomic_flag_64(std::atomic<kmp_uint64> *p, kmp_info_t *thr)
904 kmp_atomic_flag_64(std::atomic<kmp_uint64> *p, kmp_uint64 c)
906 kmp_atomic_flag_64(std::atomic<kmp_uint64> *p, kmp_uint64 c,
907 std::atomic<bool> *loc)
909 void suspend(
int th_gtid) { __kmp_atomic_suspend_64(th_gtid,
this); }
910 void mwait(
int th_gtid) { __kmp_atomic_mwait_64(th_gtid,
this); }
911 void resume(
int th_gtid) { __kmp_atomic_resume_64(th_gtid,
this); }
912 int execute_tasks(kmp_info_t *this_thr, kmp_int32 gtid,
int final_spin,
913 int *thread_finished USE_ITT_BUILD_ARG(
void *itt_sync_obj),
914 kmp_int32 is_constrained) {
915 return __kmp_atomic_execute_tasks_64(
916 this_thr, gtid,
this, final_spin,
917 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
919 bool wait(kmp_info_t *this_thr,
920 int final_spin USE_ITT_BUILD_ARG(
void *itt_sync_obj)) {
922 return __kmp_wait_template<kmp_atomic_flag_64, TRUE, Cancellable,
924 this_thr,
this USE_ITT_BUILD_ARG(itt_sync_obj));
926 return __kmp_wait_template<kmp_atomic_flag_64, FALSE, Cancellable,
928 this_thr,
this USE_ITT_BUILD_ARG(itt_sync_obj));
930 void release() { __kmp_release_template(
this); }
931 flag_type get_ptr_type() {
return atomic_flag64; }
935class kmp_flag_oncore :
public kmp_flag_native<kmp_uint64, flag_oncore, false> {
938 enum barrier_type bt;
940 kmp_info_t *this_thr =
nullptr;
944 unsigned char &byteref(
volatile kmp_uint64 *loc,
size_t offset) {
945 return (RCAST(
unsigned char *, CCAST(kmp_uint64 *, loc)))[offset];
949 kmp_flag_oncore(
volatile kmp_uint64 *p)
950 :
kmp_flag_native<kmp_uint64, flag_oncore, false>(p), flag_switch(false) {
952 kmp_flag_oncore(
volatile kmp_uint64 *p, kmp_uint32 idx)
955 bt(bs_last_barrier) USE_ITT_BUILD_ARG(itt_sync_obj(nullptr)) {}
956 kmp_flag_oncore(
volatile kmp_uint64 *p, kmp_uint64 c, kmp_uint32 idx,
957 enum barrier_type bar_t,
958 kmp_info_t *thr USE_ITT_BUILD_ARG(
void *itt))
960 flag_switch(false), bt(bar_t),
961 this_thr(thr) USE_ITT_BUILD_ARG(itt_sync_obj(itt)) {}
962 virtual ~kmp_flag_oncore()
override {}
963 void *
operator new(
size_t size) {
return __kmp_allocate(size); }
964 void operator delete(
void *p) { __kmp_free(p); }
966 return byteref(&old_loc, offset) ==
checker;
971 if (this_thr->th.th_bar[bt].bb.wait_flag == KMP_BARRIER_SWITCH_TO_OWN_FLAG)
973 if (byteref(get(), offset) != 1 && !flag_switch)
975 else if (flag_switch) {
976 this_thr->th.th_bar[bt].bb.wait_flag = KMP_BARRIER_SWITCHING;
977 kmp_flag_64<> flag(&this_thr->th.th_bar[bt].bb.b_go,
978 (kmp_uint64)KMP_BARRIER_STATE_BUMP);
979 __kmp_wait_64(this_thr, &flag, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
985 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
986 byteref(get(), offset) = 1;
989 byteref(&mask, offset) = 1;
990 KMP_TEST_THEN_OR64(get(), mask);
993 void wait(kmp_info_t *this_thr,
int final_spin) {
995 __kmp_wait_template<kmp_flag_oncore, TRUE>(
996 this_thr,
this USE_ITT_BUILD_ARG(itt_sync_obj));
998 __kmp_wait_template<kmp_flag_oncore, FALSE>(
999 this_thr,
this USE_ITT_BUILD_ARG(itt_sync_obj));
1001 void release() { __kmp_release_template(
this); }
1002 void suspend(
int th_gtid) { __kmp_suspend_oncore(th_gtid,
this); }
1003#if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
1004 void mwait(
int th_gtid) { __kmp_mwait_oncore(th_gtid,
this); }
1006 void resume(
int th_gtid) { __kmp_resume_oncore(th_gtid,
this); }
1007 int execute_tasks(kmp_info_t *this_thr, kmp_int32 gtid,
int final_spin,
1008 int *thread_finished USE_ITT_BUILD_ARG(
void *itt_sync_obj),
1009 kmp_int32 is_constrained) {
1011 int ret = __kmp_execute_tasks_oncore(
1012 this_thr, gtid,
this, final_spin,
1013 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
1014 if (ompd_state & OMPD_ENABLE_BP)
1018 return __kmp_execute_tasks_oncore(
1019 this_thr, gtid,
this, final_spin,
1020 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
1023 enum barrier_type get_bt() {
return bt; }
1024 flag_type get_ptr_type() {
return flag_oncore; }
1027static inline void __kmp_null_resume_wrapper(kmp_info_t *thr) {
1028 int gtid = __kmp_gtid_from_thread(thr);
1029 void *flag = CCAST(
void *, thr->th.th_sleep_loc);
1030 flag_type type = thr->th.th_sleep_loc_type;
1036 __kmp_resume_32(gtid, RCAST(kmp_flag_32<> *, flag));
1039 __kmp_resume_64(gtid, RCAST(kmp_flag_64<> *, flag));
1042 __kmp_atomic_resume_64(gtid, RCAST(kmp_atomic_flag_64<> *, flag));
1045 __kmp_resume_oncore(gtid, RCAST(kmp_flag_oncore *, flag));
1048 KF_TRACE(100, (
"__kmp_null_resume_wrapper: flag type %d is unset\n", type));
std::atomic< PtrType > * loc
bool is_sleeping_val(PtrType old_loc)
bool done_check_val(PtrType old_loc)
void set(std::atomic< PtrType > *new_loc)
std::atomic< PtrType > * get()
bool is_sleeping_val(PtrType old_loc)
virtual bool notdone_check()
virtual bool done_check_val(PtrType old_loc)
virtual bool done_check()
kmp_uint32 num_waiting_threads
kmp_uint32 get_num_waiters()
kmp_info_t * get_waiter(kmp_uint32 i)
void set_waiter(kmp_info_t *thr)
stats_state_e
the states which a thread can be in