14 #include "kmp_affinity.h"
15 #include "kmp_atomic.h"
16 #include "kmp_environment.h"
17 #include "kmp_error.h"
21 #include "kmp_settings.h"
22 #include "kmp_stats.h"
24 #include "kmp_wait_release.h"
25 #include "kmp_wrapper_getpid.h"
26 #include "kmp_dispatch.h"
27 #if KMP_USE_HIER_SCHED
28 #include "kmp_dispatch_hier.h"
32 #include "ompt-specific.h"
35 #if OMP_PROFILING_SUPPORT
36 #include "llvm/Support/TimeProfiler.h"
37 static char *ProfileTraceFile =
nullptr;
41 #define KMP_USE_PRCTL 0
47 #include "tsan_annotations.h"
58 #if defined(KMP_GOMP_COMPAT)
59 char const __kmp_version_alt_comp[] =
60 KMP_VERSION_PREFIX
"alternative compiler support: yes";
63 char const __kmp_version_omp_api[] =
64 KMP_VERSION_PREFIX
"API version: 5.0 (201611)";
67 char const __kmp_version_lock[] =
68 KMP_VERSION_PREFIX
"lock type: run time selectable";
71 #define KMP_MIN(x, y) ((x) < (y) ? (x) : (y))
76 kmp_info_t __kmp_monitor;
81 void __kmp_cleanup(
void);
83 static void __kmp_initialize_info(kmp_info_t *, kmp_team_t *,
int tid,
85 static void __kmp_initialize_team(kmp_team_t *team,
int new_nproc,
86 kmp_internal_control_t *new_icvs,
88 #if KMP_AFFINITY_SUPPORTED
89 static void __kmp_partition_places(kmp_team_t *team,
90 int update_master_only = 0);
92 static void __kmp_do_serial_initialize(
void);
93 void __kmp_fork_barrier(
int gtid,
int tid);
94 void __kmp_join_barrier(
int gtid);
95 void __kmp_setup_icv_copy(kmp_team_t *team,
int new_nproc,
96 kmp_internal_control_t *new_icvs,
ident_t *loc);
98 #ifdef USE_LOAD_BALANCE
99 static int __kmp_load_balance_nproc(kmp_root_t *root,
int set_nproc);
102 static int __kmp_expand_threads(
int nNeed);
104 static int __kmp_unregister_root_other_thread(
int gtid);
106 static void __kmp_reap_thread(kmp_info_t *thread,
int is_root);
107 kmp_info_t *__kmp_thread_pool_insert_pt = NULL;
112 int __kmp_get_global_thread_id() {
114 kmp_info_t **other_threads;
122 (
"*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n",
123 __kmp_nth, __kmp_all_nth));
130 if (!TCR_4(__kmp_init_gtid))
133 #ifdef KMP_TDATA_GTID
134 if (TCR_4(__kmp_gtid_mode) >= 3) {
135 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id: using TDATA\n"));
139 if (TCR_4(__kmp_gtid_mode) >= 2) {
140 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id: using keyed TLS\n"));
141 return __kmp_gtid_get_specific();
143 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id: using internal alg.\n"));
145 stack_addr = (
char *)&stack_data;
146 other_threads = __kmp_threads;
159 for (i = 0; i < __kmp_threads_capacity; i++) {
161 kmp_info_t *thr = (kmp_info_t *)TCR_SYNC_PTR(other_threads[i]);
165 stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize);
166 stack_base = (
char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase);
170 if (stack_addr <= stack_base) {
171 size_t stack_diff = stack_base - stack_addr;
173 if (stack_diff <= stack_size) {
176 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == i);
184 (
"*** __kmp_get_global_thread_id: internal alg. failed to find "
185 "thread, using TLS\n"));
186 i = __kmp_gtid_get_specific();
196 if (!TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow)) {
197 KMP_FATAL(StackOverflow, i);
200 stack_base = (
char *)other_threads[i]->th.th_info.ds.ds_stackbase;
201 if (stack_addr > stack_base) {
202 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr);
203 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,
204 other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr -
207 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,
208 stack_base - stack_addr);
212 if (__kmp_storage_map) {
213 char *stack_end = (
char *)other_threads[i]->th.th_info.ds.ds_stackbase;
214 char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize;
215 __kmp_print_storage_map_gtid(i, stack_beg, stack_end,
216 other_threads[i]->th.th_info.ds.ds_stacksize,
217 "th_%d stack (refinement)", i);
222 int __kmp_get_global_thread_id_reg() {
225 if (!__kmp_init_serial) {
228 #ifdef KMP_TDATA_GTID
229 if (TCR_4(__kmp_gtid_mode) >= 3) {
230 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id_reg: using TDATA\n"));
234 if (TCR_4(__kmp_gtid_mode) >= 2) {
235 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id_reg: using keyed TLS\n"));
236 gtid = __kmp_gtid_get_specific();
239 (
"*** __kmp_get_global_thread_id_reg: using internal alg.\n"));
240 gtid = __kmp_get_global_thread_id();
244 if (gtid == KMP_GTID_DNE) {
246 (
"__kmp_get_global_thread_id_reg: Encountered new root thread. "
247 "Registering a new gtid.\n"));
248 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
249 if (!__kmp_init_serial) {
250 __kmp_do_serial_initialize();
251 gtid = __kmp_gtid_get_specific();
253 gtid = __kmp_register_root(FALSE);
255 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
259 KMP_DEBUG_ASSERT(gtid >= 0);
265 void __kmp_check_stack_overlap(kmp_info_t *th) {
267 char *stack_beg = NULL;
268 char *stack_end = NULL;
271 KA_TRACE(10, (
"__kmp_check_stack_overlap: called\n"));
272 if (__kmp_storage_map) {
273 stack_end = (
char *)th->th.th_info.ds.ds_stackbase;
274 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
276 gtid = __kmp_gtid_from_thread(th);
278 if (gtid == KMP_GTID_MONITOR) {
279 __kmp_print_storage_map_gtid(
280 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
281 "th_%s stack (%s)",
"mon",
282 (th->th.th_info.ds.ds_stackgrow) ?
"initial" :
"actual");
284 __kmp_print_storage_map_gtid(
285 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
286 "th_%d stack (%s)", gtid,
287 (th->th.th_info.ds.ds_stackgrow) ?
"initial" :
"actual");
293 gtid = __kmp_gtid_from_thread(th);
294 if (__kmp_env_checks == TRUE && !KMP_UBER_GTID(gtid)) {
296 (
"__kmp_check_stack_overlap: performing extensive checking\n"));
297 if (stack_beg == NULL) {
298 stack_end = (
char *)th->th.th_info.ds.ds_stackbase;
299 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
302 for (f = 0; f < __kmp_threads_capacity; f++) {
303 kmp_info_t *f_th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[f]);
305 if (f_th && f_th != th) {
306 char *other_stack_end =
307 (
char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase);
308 char *other_stack_beg =
309 other_stack_end - (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize);
310 if ((stack_beg > other_stack_beg && stack_beg < other_stack_end) ||
311 (stack_end > other_stack_beg && stack_end < other_stack_end)) {
314 if (__kmp_storage_map)
315 __kmp_print_storage_map_gtid(
316 -1, other_stack_beg, other_stack_end,
317 (
size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize),
318 "th_%d stack (overlapped)", __kmp_gtid_from_thread(f_th));
320 __kmp_fatal(KMP_MSG(StackOverlap), KMP_HNT(ChangeStackLimit),
326 KA_TRACE(10, (
"__kmp_check_stack_overlap: returning\n"));
331 void __kmp_infinite_loop(
void) {
332 static int done = FALSE;
339 #define MAX_MESSAGE 512
341 void __kmp_print_storage_map_gtid(
int gtid,
void *p1,
void *p2,
size_t size,
342 char const *format, ...) {
343 char buffer[MAX_MESSAGE];
346 va_start(ap, format);
347 KMP_SNPRINTF(buffer,
sizeof(buffer),
"OMP storage map: %p %p%8lu %s\n", p1,
348 p2, (
unsigned long)size, format);
349 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
350 __kmp_vprintf(kmp_err, buffer, ap);
351 #if KMP_PRINT_DATA_PLACEMENT
354 if (p1 <= p2 && (
char *)p2 - (
char *)p1 == size) {
355 if (__kmp_storage_map_verbose) {
356 node = __kmp_get_host_node(p1);
358 __kmp_storage_map_verbose = FALSE;
362 int localProc = __kmp_get_cpu_from_gtid(gtid);
364 const int page_size = KMP_GET_PAGE_SIZE();
366 p1 = (
void *)((
size_t)p1 & ~((size_t)page_size - 1));
367 p2 = (
void *)(((
size_t)p2 - 1) & ~((
size_t)page_size - 1));
369 __kmp_printf_no_lock(
" GTID %d localNode %d\n", gtid,
372 __kmp_printf_no_lock(
" GTID %d\n", gtid);
381 (
char *)p1 += page_size;
382 }
while (p1 <= p2 && (node = __kmp_get_host_node(p1)) == lastNode);
383 __kmp_printf_no_lock(
" %p-%p memNode %d\n", last, (
char *)p1 - 1,
387 __kmp_printf_no_lock(
" %p-%p memNode %d\n", p1,
388 (
char *)p1 + (page_size - 1),
389 __kmp_get_host_node(p1));
391 __kmp_printf_no_lock(
" %p-%p memNode %d\n", p2,
392 (
char *)p2 + (page_size - 1),
393 __kmp_get_host_node(p2));
399 __kmp_printf_no_lock(
" %s\n", KMP_I18N_STR(StorageMapWarning));
402 __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
405 void __kmp_warn(
char const *format, ...) {
406 char buffer[MAX_MESSAGE];
409 if (__kmp_generate_warnings == kmp_warnings_off) {
413 va_start(ap, format);
415 KMP_SNPRINTF(buffer,
sizeof(buffer),
"OMP warning: %s\n", format);
416 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
417 __kmp_vprintf(kmp_err, buffer, ap);
418 __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
423 void __kmp_abort_process() {
425 __kmp_acquire_bootstrap_lock(&__kmp_exit_lock);
427 if (__kmp_debug_buf) {
428 __kmp_dump_debug_buffer();
431 if (KMP_OS_WINDOWS) {
434 __kmp_global.g.g_abort = SIGABRT;
448 __kmp_unregister_library();
452 __kmp_infinite_loop();
453 __kmp_release_bootstrap_lock(&__kmp_exit_lock);
457 void __kmp_abort_thread(
void) {
460 __kmp_infinite_loop();
466 static void __kmp_print_thread_storage_map(kmp_info_t *thr,
int gtid) {
467 __kmp_print_storage_map_gtid(gtid, thr, thr + 1,
sizeof(kmp_info_t),
"th_%d",
470 __kmp_print_storage_map_gtid(gtid, &thr->th.th_info, &thr->th.th_team,
471 sizeof(kmp_desc_t),
"th_%d.th_info", gtid);
473 __kmp_print_storage_map_gtid(gtid, &thr->th.th_local, &thr->th.th_pri_head,
474 sizeof(kmp_local_t),
"th_%d.th_local", gtid);
476 __kmp_print_storage_map_gtid(
477 gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier],
478 sizeof(kmp_balign_t) * bs_last_barrier,
"th_%d.th_bar", gtid);
480 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_plain_barrier],
481 &thr->th.th_bar[bs_plain_barrier + 1],
482 sizeof(kmp_balign_t),
"th_%d.th_bar[plain]",
485 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_forkjoin_barrier],
486 &thr->th.th_bar[bs_forkjoin_barrier + 1],
487 sizeof(kmp_balign_t),
"th_%d.th_bar[forkjoin]",
490 #if KMP_FAST_REDUCTION_BARRIER
491 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_reduction_barrier],
492 &thr->th.th_bar[bs_reduction_barrier + 1],
493 sizeof(kmp_balign_t),
"th_%d.th_bar[reduction]",
495 #endif // KMP_FAST_REDUCTION_BARRIER
501 static void __kmp_print_team_storage_map(
const char *header, kmp_team_t *team,
502 int team_id,
int num_thr) {
503 int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2;
504 __kmp_print_storage_map_gtid(-1, team, team + 1,
sizeof(kmp_team_t),
"%s_%d",
507 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[0],
508 &team->t.t_bar[bs_last_barrier],
509 sizeof(kmp_balign_team_t) * bs_last_barrier,
510 "%s_%d.t_bar", header, team_id);
512 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_plain_barrier],
513 &team->t.t_bar[bs_plain_barrier + 1],
514 sizeof(kmp_balign_team_t),
"%s_%d.t_bar[plain]",
517 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_forkjoin_barrier],
518 &team->t.t_bar[bs_forkjoin_barrier + 1],
519 sizeof(kmp_balign_team_t),
520 "%s_%d.t_bar[forkjoin]", header, team_id);
522 #if KMP_FAST_REDUCTION_BARRIER
523 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_reduction_barrier],
524 &team->t.t_bar[bs_reduction_barrier + 1],
525 sizeof(kmp_balign_team_t),
526 "%s_%d.t_bar[reduction]", header, team_id);
527 #endif // KMP_FAST_REDUCTION_BARRIER
529 __kmp_print_storage_map_gtid(
530 -1, &team->t.t_dispatch[0], &team->t.t_dispatch[num_thr],
531 sizeof(kmp_disp_t) * num_thr,
"%s_%d.t_dispatch", header, team_id);
533 __kmp_print_storage_map_gtid(
534 -1, &team->t.t_threads[0], &team->t.t_threads[num_thr],
535 sizeof(kmp_info_t *) * num_thr,
"%s_%d.t_threads", header, team_id);
537 __kmp_print_storage_map_gtid(-1, &team->t.t_disp_buffer[0],
538 &team->t.t_disp_buffer[num_disp_buff],
539 sizeof(dispatch_shared_info_t) * num_disp_buff,
540 "%s_%d.t_disp_buffer", header, team_id);
543 static void __kmp_init_allocator() { __kmp_init_memkind(); }
544 static void __kmp_fini_allocator() { __kmp_fini_memkind(); }
551 static void __kmp_reset_lock(kmp_bootstrap_lock_t *lck) {
553 __kmp_init_bootstrap_lock(lck);
556 static void __kmp_reset_locks_on_process_detach(
int gtid_req) {
574 for (i = 0; i < __kmp_threads_capacity; ++i) {
577 kmp_info_t *th = __kmp_threads[i];
580 int gtid = th->th.th_info.ds.ds_gtid;
581 if (gtid == gtid_req)
586 int alive = __kmp_is_thread_alive(th, &exit_val);
591 if (thread_count == 0)
597 __kmp_reset_lock(&__kmp_forkjoin_lock);
599 __kmp_reset_lock(&__kmp_stdio_lock);
603 BOOL WINAPI DllMain(HINSTANCE hInstDLL, DWORD fdwReason, LPVOID lpReserved) {
608 case DLL_PROCESS_ATTACH:
609 KA_TRACE(10, (
"DllMain: PROCESS_ATTACH\n"));
613 case DLL_PROCESS_DETACH:
614 KA_TRACE(10, (
"DllMain: PROCESS_DETACH T#%d\n", __kmp_gtid_get_specific()));
616 if (lpReserved != NULL) {
642 __kmp_reset_locks_on_process_detach(__kmp_gtid_get_specific());
645 __kmp_internal_end_library(__kmp_gtid_get_specific());
649 case DLL_THREAD_ATTACH:
650 KA_TRACE(10, (
"DllMain: THREAD_ATTACH\n"));
656 case DLL_THREAD_DETACH:
657 KA_TRACE(10, (
"DllMain: THREAD_DETACH T#%d\n", __kmp_gtid_get_specific()));
659 __kmp_internal_end_thread(__kmp_gtid_get_specific());
670 void __kmp_parallel_deo(
int *gtid_ref,
int *cid_ref,
ident_t *loc_ref) {
671 int gtid = *gtid_ref;
672 #ifdef BUILD_PARALLEL_ORDERED
673 kmp_team_t *team = __kmp_team_from_gtid(gtid);
676 if (__kmp_env_consistency_check) {
677 if (__kmp_threads[gtid]->th.th_root->r.r_active)
678 #if KMP_USE_DYNAMIC_LOCK
679 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL, 0);
681 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL);
684 #ifdef BUILD_PARALLEL_ORDERED
685 if (!team->t.t_serialized) {
687 KMP_WAIT(&team->t.t_ordered.dt.t_value, __kmp_tid_from_gtid(gtid), KMP_EQ,
695 void __kmp_parallel_dxo(
int *gtid_ref,
int *cid_ref,
ident_t *loc_ref) {
696 int gtid = *gtid_ref;
697 #ifdef BUILD_PARALLEL_ORDERED
698 int tid = __kmp_tid_from_gtid(gtid);
699 kmp_team_t *team = __kmp_team_from_gtid(gtid);
702 if (__kmp_env_consistency_check) {
703 if (__kmp_threads[gtid]->th.th_root->r.r_active)
704 __kmp_pop_sync(gtid, ct_ordered_in_parallel, loc_ref);
706 #ifdef BUILD_PARALLEL_ORDERED
707 if (!team->t.t_serialized) {
712 team->t.t_ordered.dt.t_value = ((tid + 1) % team->t.t_nproc);
722 int __kmp_enter_single(
int gtid,
ident_t *id_ref,
int push_ws) {
727 if (!TCR_4(__kmp_init_parallel))
728 __kmp_parallel_initialize();
729 __kmp_resume_if_soft_paused();
731 th = __kmp_threads[gtid];
732 team = th->th.th_team;
735 th->th.th_ident = id_ref;
737 if (team->t.t_serialized) {
740 kmp_int32 old_this = th->th.th_local.this_construct;
742 ++th->th.th_local.this_construct;
746 if (team->t.t_construct == old_this) {
747 status = __kmp_atomic_compare_store_acq(&team->t.t_construct, old_this,
748 th->th.th_local.this_construct);
751 if (__itt_metadata_add_ptr && __kmp_forkjoin_frames_mode == 3 &&
752 KMP_MASTER_GTID(gtid) && th->th.th_teams_microtask == NULL &&
753 team->t.t_active_level ==
755 __kmp_itt_metadata_single(id_ref);
760 if (__kmp_env_consistency_check) {
761 if (status && push_ws) {
762 __kmp_push_workshare(gtid, ct_psingle, id_ref);
764 __kmp_check_workshare(gtid, ct_psingle, id_ref);
769 __kmp_itt_single_start(gtid);
775 void __kmp_exit_single(
int gtid) {
777 __kmp_itt_single_end(gtid);
779 if (__kmp_env_consistency_check)
780 __kmp_pop_workshare(gtid, ct_psingle, NULL);
789 static int __kmp_reserve_threads(kmp_root_t *root, kmp_team_t *parent_team,
790 int master_tid,
int set_nthreads,
794 KMP_DEBUG_ASSERT(__kmp_init_serial);
795 KMP_DEBUG_ASSERT(root && parent_team);
796 kmp_info_t *this_thr = parent_team->t.t_threads[master_tid];
800 new_nthreads = set_nthreads;
801 if (!get__dynamic_2(parent_team, master_tid)) {
804 #ifdef USE_LOAD_BALANCE
805 else if (__kmp_global.g.g_dynamic_mode == dynamic_load_balance) {
806 new_nthreads = __kmp_load_balance_nproc(root, set_nthreads);
807 if (new_nthreads == 1) {
808 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d load balance reduced "
809 "reservation to 1 thread\n",
813 if (new_nthreads < set_nthreads) {
814 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d load balance reduced "
815 "reservation to %d threads\n",
816 master_tid, new_nthreads));
820 else if (__kmp_global.g.g_dynamic_mode == dynamic_thread_limit) {
821 new_nthreads = __kmp_avail_proc - __kmp_nth +
822 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
823 if (new_nthreads <= 1) {
824 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d thread limit reduced "
825 "reservation to 1 thread\n",
829 if (new_nthreads < set_nthreads) {
830 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d thread limit reduced "
831 "reservation to %d threads\n",
832 master_tid, new_nthreads));
834 new_nthreads = set_nthreads;
836 }
else if (__kmp_global.g.g_dynamic_mode == dynamic_random) {
837 if (set_nthreads > 2) {
838 new_nthreads = __kmp_get_random(parent_team->t.t_threads[master_tid]);
839 new_nthreads = (new_nthreads % set_nthreads) + 1;
840 if (new_nthreads == 1) {
841 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d dynamic random reduced "
842 "reservation to 1 thread\n",
846 if (new_nthreads < set_nthreads) {
847 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d dynamic random reduced "
848 "reservation to %d threads\n",
849 master_tid, new_nthreads));
857 if (__kmp_nth + new_nthreads -
858 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
860 int tl_nthreads = __kmp_max_nth - __kmp_nth +
861 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
862 if (tl_nthreads <= 0) {
867 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
868 __kmp_reserve_warn = 1;
869 __kmp_msg(kmp_ms_warning,
870 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads),
871 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
873 if (tl_nthreads == 1) {
874 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT "
875 "reduced reservation to 1 thread\n",
879 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced "
880 "reservation to %d threads\n",
881 master_tid, tl_nthreads));
882 new_nthreads = tl_nthreads;
886 int cg_nthreads = this_thr->th.th_cg_roots->cg_nthreads;
887 int max_cg_threads = this_thr->th.th_cg_roots->cg_thread_limit;
888 if (cg_nthreads + new_nthreads -
889 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
891 int tl_nthreads = max_cg_threads - cg_nthreads +
892 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
893 if (tl_nthreads <= 0) {
898 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
899 __kmp_reserve_warn = 1;
900 __kmp_msg(kmp_ms_warning,
901 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads),
902 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
904 if (tl_nthreads == 1) {
905 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT "
906 "reduced reservation to 1 thread\n",
910 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced "
911 "reservation to %d threads\n",
912 master_tid, tl_nthreads));
913 new_nthreads = tl_nthreads;
919 capacity = __kmp_threads_capacity;
920 if (TCR_PTR(__kmp_threads[0]) == NULL) {
923 if (__kmp_nth + new_nthreads -
924 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
927 int slotsRequired = __kmp_nth + new_nthreads -
928 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) -
930 int slotsAdded = __kmp_expand_threads(slotsRequired);
931 if (slotsAdded < slotsRequired) {
933 new_nthreads -= (slotsRequired - slotsAdded);
934 KMP_ASSERT(new_nthreads >= 1);
937 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
938 __kmp_reserve_warn = 1;
939 if (__kmp_tp_cached) {
940 __kmp_msg(kmp_ms_warning,
941 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads),
942 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity),
943 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null);
945 __kmp_msg(kmp_ms_warning,
946 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads),
947 KMP_HNT(SystemLimitOnThreads), __kmp_msg_null);
954 if (new_nthreads == 1) {
956 (
"__kmp_reserve_threads: T#%d serializing team after reclaiming "
957 "dead roots and rechecking; requested %d threads\n",
958 __kmp_get_gtid(), set_nthreads));
960 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d allocating %d threads; requested"
962 __kmp_get_gtid(), new_nthreads, set_nthreads));
971 static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team,
972 kmp_info_t *master_th,
int master_gtid) {
976 KA_TRACE(10, (
"__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc));
977 KMP_DEBUG_ASSERT(master_gtid == __kmp_get_gtid());
981 master_th->th.th_info.ds.ds_tid = 0;
982 master_th->th.th_team = team;
983 master_th->th.th_team_nproc = team->t.t_nproc;
984 master_th->th.th_team_master = master_th;
985 master_th->th.th_team_serialized = FALSE;
986 master_th->th.th_dispatch = &team->t.t_dispatch[0];
989 #if KMP_NESTED_HOT_TEAMS
991 kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams;
994 int level = team->t.t_active_level - 1;
995 if (master_th->th.th_teams_microtask) {
996 if (master_th->th.th_teams_size.nteams > 1) {
1000 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
1001 master_th->th.th_teams_level == team->t.t_level) {
1006 if (level < __kmp_hot_teams_max_level) {
1007 if (hot_teams[level].hot_team) {
1009 KMP_DEBUG_ASSERT(hot_teams[level].hot_team == team);
1013 hot_teams[level].hot_team = team;
1014 hot_teams[level].hot_team_nth = team->t.t_nproc;
1021 use_hot_team = team == root->r.r_hot_team;
1023 if (!use_hot_team) {
1026 team->t.t_threads[0] = master_th;
1027 __kmp_initialize_info(master_th, team, 0, master_gtid);
1030 for (i = 1; i < team->t.t_nproc; i++) {
1033 kmp_info_t *thr = __kmp_allocate_thread(root, team, i);
1034 team->t.t_threads[i] = thr;
1035 KMP_DEBUG_ASSERT(thr);
1036 KMP_DEBUG_ASSERT(thr->th.th_team == team);
1038 KA_TRACE(20, (
"__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
1039 "T#%d(%d:%d) join =%llu, plain=%llu\n",
1040 __kmp_gtid_from_tid(0, team), team->t.t_id, 0,
1041 __kmp_gtid_from_tid(i, team), team->t.t_id, i,
1042 team->t.t_bar[bs_forkjoin_barrier].b_arrived,
1043 team->t.t_bar[bs_plain_barrier].b_arrived));
1044 thr->th.th_teams_microtask = master_th->th.th_teams_microtask;
1045 thr->th.th_teams_level = master_th->th.th_teams_level;
1046 thr->th.th_teams_size = master_th->th.th_teams_size;
1049 kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar;
1050 for (b = 0; b < bs_last_barrier; ++b) {
1051 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
1052 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
1054 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
1060 #if KMP_AFFINITY_SUPPORTED
1061 __kmp_partition_places(team);
1065 if (__kmp_display_affinity && team->t.t_display_affinity != 1) {
1066 for (i = 0; i < team->t.t_nproc; i++) {
1067 kmp_info_t *thr = team->t.t_threads[i];
1068 if (thr->th.th_prev_num_threads != team->t.t_nproc ||
1069 thr->th.th_prev_level != team->t.t_level) {
1070 team->t.t_display_affinity = 1;
1079 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1083 inline static void propagateFPControl(kmp_team_t *team) {
1084 if (__kmp_inherit_fp_control) {
1085 kmp_int16 x87_fpu_control_word;
1089 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
1090 __kmp_store_mxcsr(&mxcsr);
1091 mxcsr &= KMP_X86_MXCSR_MASK;
1102 KMP_CHECK_UPDATE(team->t.t_x87_fpu_control_word, x87_fpu_control_word);
1103 KMP_CHECK_UPDATE(team->t.t_mxcsr, mxcsr);
1106 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, TRUE);
1110 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, FALSE);
1116 inline static void updateHWFPControl(kmp_team_t *team) {
1117 if (__kmp_inherit_fp_control && team->t.t_fp_control_saved) {
1120 kmp_int16 x87_fpu_control_word;
1122 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
1123 __kmp_store_mxcsr(&mxcsr);
1124 mxcsr &= KMP_X86_MXCSR_MASK;
1126 if (team->t.t_x87_fpu_control_word != x87_fpu_control_word) {
1127 __kmp_clear_x87_fpu_status_word();
1128 __kmp_load_x87_fpu_control_word(&team->t.t_x87_fpu_control_word);
1131 if (team->t.t_mxcsr != mxcsr) {
1132 __kmp_load_mxcsr(&team->t.t_mxcsr);
1137 #define propagateFPControl(x) ((void)0)
1138 #define updateHWFPControl(x) ((void)0)
1141 static void __kmp_alloc_argv_entries(
int argc, kmp_team_t *team,
1146 void __kmp_serialized_parallel(
ident_t *loc, kmp_int32 global_tid) {
1147 kmp_info_t *this_thr;
1148 kmp_team_t *serial_team;
1150 KC_TRACE(10, (
"__kmpc_serialized_parallel: called by T#%d\n", global_tid));
1157 if (!TCR_4(__kmp_init_parallel))
1158 __kmp_parallel_initialize();
1159 __kmp_resume_if_soft_paused();
1161 this_thr = __kmp_threads[global_tid];
1162 serial_team = this_thr->th.th_serial_team;
1165 KMP_DEBUG_ASSERT(serial_team);
1168 if (__kmp_tasking_mode != tskm_immediate_exec) {
1170 this_thr->th.th_task_team ==
1171 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]);
1172 KMP_DEBUG_ASSERT(serial_team->t.t_task_team[this_thr->th.th_task_state] ==
1174 KA_TRACE(20, (
"__kmpc_serialized_parallel: T#%d pushing task_team %p / "
1175 "team %p, new task_team = NULL\n",
1176 global_tid, this_thr->th.th_task_team, this_thr->th.th_team));
1177 this_thr->th.th_task_team = NULL;
1180 kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind;
1181 if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
1182 proc_bind = proc_bind_false;
1183 }
else if (proc_bind == proc_bind_default) {
1186 proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind;
1189 this_thr->th.th_set_proc_bind = proc_bind_default;
1192 ompt_data_t ompt_parallel_data = ompt_data_none;
1193 ompt_data_t *implicit_task_data;
1194 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid);
1195 if (ompt_enabled.enabled &&
1196 this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
1198 ompt_task_info_t *parent_task_info;
1199 parent_task_info = OMPT_CUR_TASK_INFO(this_thr);
1201 parent_task_info->frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1202 if (ompt_enabled.ompt_callback_parallel_begin) {
1205 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)(
1206 &(parent_task_info->task_data), &(parent_task_info->frame),
1207 &ompt_parallel_data, team_size,
1208 ompt_parallel_invoker_program | ompt_parallel_team, codeptr);
1211 #endif // OMPT_SUPPORT
1213 if (this_thr->th.th_team != serial_team) {
1215 int level = this_thr->th.th_team->t.t_level;
1217 if (serial_team->t.t_serialized) {
1220 kmp_team_t *new_team;
1222 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
1225 __kmp_allocate_team(this_thr->th.th_root, 1, 1,
1229 proc_bind, &this_thr->th.th_current_task->td_icvs,
1230 0 USE_NESTED_HOT_ARG(NULL));
1231 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1232 KMP_ASSERT(new_team);
1235 new_team->t.t_threads[0] = this_thr;
1236 new_team->t.t_parent = this_thr->th.th_team;
1237 serial_team = new_team;
1238 this_thr->th.th_serial_team = serial_team;
1242 (
"__kmpc_serialized_parallel: T#%d allocated new serial team %p\n",
1243 global_tid, serial_team));
1251 (
"__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n",
1252 global_tid, serial_team));
1256 KMP_DEBUG_ASSERT(serial_team->t.t_threads);
1257 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
1258 KMP_DEBUG_ASSERT(this_thr->th.th_team != serial_team);
1259 serial_team->t.t_ident = loc;
1260 serial_team->t.t_serialized = 1;
1261 serial_team->t.t_nproc = 1;
1262 serial_team->t.t_parent = this_thr->th.th_team;
1263 serial_team->t.t_sched.sched = this_thr->th.th_team->t.t_sched.sched;
1264 this_thr->th.th_team = serial_team;
1265 serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid;
1267 KF_TRACE(10, (
"__kmpc_serialized_parallel: T#d curtask=%p\n", global_tid,
1268 this_thr->th.th_current_task));
1269 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 1);
1270 this_thr->th.th_current_task->td_flags.executing = 0;
1272 __kmp_push_current_task_to_thread(this_thr, serial_team, 0);
1277 copy_icvs(&this_thr->th.th_current_task->td_icvs,
1278 &this_thr->th.th_current_task->td_parent->td_icvs);
1282 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
1283 this_thr->th.th_current_task->td_icvs.nproc =
1284 __kmp_nested_nth.nth[level + 1];
1287 if (__kmp_nested_proc_bind.used &&
1288 (level + 1 < __kmp_nested_proc_bind.used)) {
1289 this_thr->th.th_current_task->td_icvs.proc_bind =
1290 __kmp_nested_proc_bind.bind_types[level + 1];
1294 serial_team->t.t_pkfn = (microtask_t)(~0);
1296 this_thr->th.th_info.ds.ds_tid = 0;
1299 this_thr->th.th_team_nproc = 1;
1300 this_thr->th.th_team_master = this_thr;
1301 this_thr->th.th_team_serialized = 1;
1303 serial_team->t.t_level = serial_team->t.t_parent->t.t_level + 1;
1304 serial_team->t.t_active_level = serial_team->t.t_parent->t.t_active_level;
1305 serial_team->t.t_def_allocator = this_thr->th.th_def_allocator;
1307 propagateFPControl(serial_team);
1310 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);
1311 if (!serial_team->t.t_dispatch->th_disp_buffer) {
1312 serial_team->t.t_dispatch->th_disp_buffer =
1313 (dispatch_private_info_t *)__kmp_allocate(
1314 sizeof(dispatch_private_info_t));
1316 this_thr->th.th_dispatch = serial_team->t.t_dispatch;
1323 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team);
1324 KMP_DEBUG_ASSERT(serial_team->t.t_threads);
1325 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
1326 ++serial_team->t.t_serialized;
1327 this_thr->th.th_team_serialized = serial_team->t.t_serialized;
1330 int level = this_thr->th.th_team->t.t_level;
1333 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
1334 this_thr->th.th_current_task->td_icvs.nproc =
1335 __kmp_nested_nth.nth[level + 1];
1337 serial_team->t.t_level++;
1338 KF_TRACE(10, (
"__kmpc_serialized_parallel: T#%d increasing nesting level "
1339 "of serial team %p to %d\n",
1340 global_tid, serial_team, serial_team->t.t_level));
1343 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);
1345 dispatch_private_info_t *disp_buffer =
1346 (dispatch_private_info_t *)__kmp_allocate(
1347 sizeof(dispatch_private_info_t));
1348 disp_buffer->next = serial_team->t.t_dispatch->th_disp_buffer;
1349 serial_team->t.t_dispatch->th_disp_buffer = disp_buffer;
1351 this_thr->th.th_dispatch = serial_team->t.t_dispatch;
1355 KMP_CHECK_UPDATE(serial_team->t.t_cancel_request, cancel_noreq);
1359 if (__kmp_display_affinity) {
1360 if (this_thr->th.th_prev_level != serial_team->t.t_level ||
1361 this_thr->th.th_prev_num_threads != 1) {
1363 __kmp_aux_display_affinity(global_tid, NULL);
1364 this_thr->th.th_prev_level = serial_team->t.t_level;
1365 this_thr->th.th_prev_num_threads = 1;
1369 if (__kmp_env_consistency_check)
1370 __kmp_push_parallel(global_tid, NULL);
1372 serial_team->t.ompt_team_info.master_return_address = codeptr;
1373 if (ompt_enabled.enabled &&
1374 this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
1375 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1377 ompt_lw_taskteam_t lw_taskteam;
1378 __ompt_lw_taskteam_init(&lw_taskteam, this_thr, global_tid,
1379 &ompt_parallel_data, codeptr);
1381 __ompt_lw_taskteam_link(&lw_taskteam, this_thr, 1);
1385 implicit_task_data = OMPT_CUR_TASK_DATA(this_thr);
1386 if (ompt_enabled.ompt_callback_implicit_task) {
1387 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1388 ompt_scope_begin, OMPT_CUR_TEAM_DATA(this_thr),
1389 OMPT_CUR_TASK_DATA(this_thr), 1, __kmp_tid_from_gtid(global_tid), ompt_task_implicit);
1390 OMPT_CUR_TASK_INFO(this_thr)
1391 ->thread_num = __kmp_tid_from_gtid(global_tid);
1395 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
1396 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1403 int __kmp_fork_call(
ident_t *loc,
int gtid,
1404 enum fork_context_e call_context,
1405 kmp_int32 argc, microtask_t microtask, launch_t invoker,
1410 int master_this_cons;
1412 kmp_team_t *parent_team;
1413 kmp_info_t *master_th;
1417 int master_set_numthreads;
1421 #if KMP_NESTED_HOT_TEAMS
1422 kmp_hot_team_ptr_t **p_hot_teams;
1425 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_fork_call);
1428 KA_TRACE(20, (
"__kmp_fork_call: enter T#%d\n", gtid));
1429 if (__kmp_stkpadding > 0 && __kmp_root[gtid] != NULL) {
1432 void *dummy = KMP_ALLOCA(__kmp_stkpadding);
1434 if (__kmp_stkpadding > KMP_MAX_STKPADDING)
1435 __kmp_stkpadding += (short)((kmp_int64)dummy);
1441 if (!TCR_4(__kmp_init_parallel))
1442 __kmp_parallel_initialize();
1443 __kmp_resume_if_soft_paused();
1446 master_th = __kmp_threads[gtid];
1448 parent_team = master_th->th.th_team;
1449 master_tid = master_th->th.th_info.ds.ds_tid;
1450 master_this_cons = master_th->th.th_local.this_construct;
1451 root = master_th->th.th_root;
1452 master_active = root->r.r_active;
1453 master_set_numthreads = master_th->th.th_set_nproc;
1456 ompt_data_t ompt_parallel_data = ompt_data_none;
1457 ompt_data_t *parent_task_data;
1458 ompt_frame_t *ompt_frame;
1459 ompt_data_t *implicit_task_data;
1460 void *return_address = NULL;
1462 if (ompt_enabled.enabled) {
1463 __ompt_get_task_info_internal(0, NULL, &parent_task_data, &ompt_frame,
1465 return_address = OMPT_LOAD_RETURN_ADDRESS(gtid);
1470 level = parent_team->t.t_level;
1472 active_level = parent_team->t.t_active_level;
1474 teams_level = master_th->th.th_teams_level;
1475 #if KMP_NESTED_HOT_TEAMS
1476 p_hot_teams = &master_th->th.th_hot_teams;
1477 if (*p_hot_teams == NULL && __kmp_hot_teams_max_level > 0) {
1478 *p_hot_teams = (kmp_hot_team_ptr_t *)__kmp_allocate(
1479 sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level);
1480 (*p_hot_teams)[0].hot_team = root->r.r_hot_team;
1482 (*p_hot_teams)[0].hot_team_nth = 1;
1487 if (ompt_enabled.enabled) {
1488 if (ompt_enabled.ompt_callback_parallel_begin) {
1489 int team_size = master_set_numthreads
1490 ? master_set_numthreads
1491 : get__nproc_2(parent_team, master_tid);
1492 int flags = OMPT_INVOKER(call_context) |
1493 ((microtask == (microtask_t)__kmp_teams_master)
1494 ? ompt_parallel_league
1495 : ompt_parallel_team);
1496 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)(
1497 parent_task_data, ompt_frame, &ompt_parallel_data, team_size, flags,
1500 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1504 master_th->th.th_ident = loc;
1506 if (master_th->th.th_teams_microtask && ap &&
1507 microtask != (microtask_t)__kmp_teams_master && level == teams_level) {
1511 parent_team->t.t_ident = loc;
1512 __kmp_alloc_argv_entries(argc, parent_team, TRUE);
1513 parent_team->t.t_argc = argc;
1514 argv = (
void **)parent_team->t.t_argv;
1515 for (i = argc - 1; i >= 0; --i)
1516 *argv++ = va_arg(kmp_va_deref(ap),
void *);
1518 if (parent_team == master_th->th.th_serial_team) {
1521 KMP_DEBUG_ASSERT(parent_team->t.t_serialized > 1);
1523 if (call_context == fork_context_gnu) {
1526 parent_team->t.t_serialized--;
1532 void **exit_frame_p;
1534 ompt_lw_taskteam_t lw_taskteam;
1536 if (ompt_enabled.enabled) {
1537 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1538 &ompt_parallel_data, return_address);
1539 exit_frame_p = &(lw_taskteam.ompt_task_info.frame.exit_frame.ptr);
1541 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1545 implicit_task_data = OMPT_CUR_TASK_DATA(master_th);
1546 if (ompt_enabled.ompt_callback_implicit_task) {
1547 OMPT_CUR_TASK_INFO(master_th)
1548 ->thread_num = __kmp_tid_from_gtid(gtid);
1549 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1550 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th),
1551 implicit_task_data, 1,
1552 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit);
1556 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
1558 exit_frame_p = &dummy;
1563 parent_team->t.t_serialized--;
1566 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1567 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1568 __kmp_invoke_microtask(microtask, gtid, 0, argc, parent_team->t.t_argv
1577 if (ompt_enabled.enabled) {
1578 *exit_frame_p = NULL;
1579 OMPT_CUR_TASK_INFO(master_th)->frame.exit_frame = ompt_data_none;
1580 if (ompt_enabled.ompt_callback_implicit_task) {
1581 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1582 ompt_scope_end, NULL, implicit_task_data, 1,
1583 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit);
1585 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th);
1586 __ompt_lw_taskteam_unlink(master_th);
1587 if (ompt_enabled.ompt_callback_parallel_end) {
1588 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1589 &ompt_parallel_data, OMPT_CUR_TASK_DATA(master_th),
1590 OMPT_INVOKER(call_context) | ompt_parallel_team,
1593 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1599 parent_team->t.t_pkfn = microtask;
1600 parent_team->t.t_invoke = invoker;
1601 KMP_ATOMIC_INC(&root->r.r_in_parallel);
1602 parent_team->t.t_active_level++;
1603 parent_team->t.t_level++;
1604 parent_team->t.t_def_allocator = master_th->th.th_def_allocator;
1607 if (ompt_enabled.enabled) {
1608 ompt_lw_taskteam_t lw_taskteam;
1609 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1610 &ompt_parallel_data, return_address);
1611 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 1,
true);
1616 if (master_set_numthreads) {
1617 if (master_set_numthreads < master_th->th.th_teams_size.nth) {
1619 kmp_info_t **other_threads = parent_team->t.t_threads;
1620 parent_team->t.t_nproc = master_set_numthreads;
1621 for (i = 0; i < master_set_numthreads; ++i) {
1622 other_threads[i]->th.th_team_nproc = master_set_numthreads;
1626 master_th->th.th_set_nproc = 0;
1630 if (__kmp_debugging) {
1631 int nth = __kmp_omp_num_threads(loc);
1633 master_set_numthreads = nth;
1638 #if USE_ITT_BUILD && USE_ITT_NOTIFY
1639 if (((__itt_frame_submit_v3_ptr && __itt_get_timestamp_ptr) ||
1641 __kmp_forkjoin_frames_mode == 3 &&
1642 parent_team->t.t_active_level == 1
1643 && master_th->th.th_teams_size.nteams == 1) {
1644 kmp_uint64 tmp_time = __itt_get_timestamp();
1645 master_th->th.th_frame_time = tmp_time;
1646 parent_team->t.t_region_time = tmp_time;
1648 if (__itt_stack_caller_create_ptr) {
1650 parent_team->t.t_stack_id = __kmp_itt_stack_caller_create();
1654 KF_TRACE(10, (
"__kmp_fork_call: before internal fork: root=%p, team=%p, "
1655 "master_th=%p, gtid=%d\n",
1656 root, parent_team, master_th, gtid));
1657 __kmp_internal_fork(loc, gtid, parent_team);
1658 KF_TRACE(10, (
"__kmp_fork_call: after internal fork: root=%p, team=%p, "
1659 "master_th=%p, gtid=%d\n",
1660 root, parent_team, master_th, gtid));
1662 if (call_context == fork_context_gnu)
1666 KA_TRACE(20, (
"__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,
1667 parent_team->t.t_id, parent_team->t.t_pkfn));
1669 if (!parent_team->t.t_invoke(gtid)) {
1670 KMP_ASSERT2(0,
"cannot invoke microtask for MASTER thread");
1672 KA_TRACE(20, (
"__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,
1673 parent_team->t.t_id, parent_team->t.t_pkfn));
1676 KA_TRACE(20, (
"__kmp_fork_call: parallel exit T#%d\n", gtid));
1682 if (__kmp_tasking_mode != tskm_immediate_exec) {
1683 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
1684 parent_team->t.t_task_team[master_th->th.th_task_state]);
1688 if (parent_team->t.t_active_level >=
1689 master_th->th.th_current_task->td_icvs.max_active_levels) {
1692 int enter_teams = ((ap == NULL && active_level == 0) ||
1693 (ap && teams_level > 0 && teams_level == level));
1695 master_set_numthreads
1696 ? master_set_numthreads
1705 if ((get__max_active_levels(master_th) == 1 &&
1706 (root->r.r_in_parallel && !enter_teams)) ||
1707 (__kmp_library == library_serial)) {
1708 KC_TRACE(10, (
"__kmp_fork_call: T#%d serializing team; requested %d"
1716 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
1721 nthreads = __kmp_reserve_threads(root, parent_team, master_tid,
1722 nthreads, enter_teams);
1723 if (nthreads == 1) {
1727 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1731 KMP_DEBUG_ASSERT(nthreads > 0);
1734 master_th->th.th_set_nproc = 0;
1737 if (nthreads == 1) {
1739 #if KMP_OS_LINUX && \
1740 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
1743 void **args = (
void **)KMP_ALLOCA(argc *
sizeof(
void *));
1748 (
"__kmp_fork_call: T#%d serializing parallel region\n", gtid));
1752 if (call_context == fork_context_intel) {
1754 master_th->th.th_serial_team->t.t_ident = loc;
1757 master_th->th.th_serial_team->t.t_level--;
1762 void **exit_frame_p;
1763 ompt_task_info_t *task_info;
1765 ompt_lw_taskteam_t lw_taskteam;
1767 if (ompt_enabled.enabled) {
1768 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1769 &ompt_parallel_data, return_address);
1771 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1774 task_info = OMPT_CUR_TASK_INFO(master_th);
1775 exit_frame_p = &(task_info->frame.exit_frame.ptr);
1776 if (ompt_enabled.ompt_callback_implicit_task) {
1777 OMPT_CUR_TASK_INFO(master_th)
1778 ->thread_num = __kmp_tid_from_gtid(gtid);
1779 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1780 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th),
1781 &(task_info->task_data), 1,
1782 OMPT_CUR_TASK_INFO(master_th)->thread_num,
1783 ompt_task_implicit);
1787 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
1789 exit_frame_p = &dummy;
1794 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1795 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1796 __kmp_invoke_microtask(microtask, gtid, 0, argc,
1797 parent_team->t.t_argv
1806 if (ompt_enabled.enabled) {
1807 *exit_frame_p = NULL;
1808 if (ompt_enabled.ompt_callback_implicit_task) {
1809 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1810 ompt_scope_end, NULL, &(task_info->task_data), 1,
1811 OMPT_CUR_TASK_INFO(master_th)->thread_num,
1812 ompt_task_implicit);
1814 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th);
1815 __ompt_lw_taskteam_unlink(master_th);
1816 if (ompt_enabled.ompt_callback_parallel_end) {
1817 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1818 &ompt_parallel_data, parent_task_data,
1819 OMPT_INVOKER(call_context) | ompt_parallel_team,
1822 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1825 }
else if (microtask == (microtask_t)__kmp_teams_master) {
1826 KMP_DEBUG_ASSERT(master_th->th.th_team ==
1827 master_th->th.th_serial_team);
1828 team = master_th->th.th_team;
1830 team->t.t_invoke = invoker;
1831 __kmp_alloc_argv_entries(argc, team, TRUE);
1832 team->t.t_argc = argc;
1833 argv = (
void **)team->t.t_argv;
1835 for (i = argc - 1; i >= 0; --i)
1836 *argv++ = va_arg(kmp_va_deref(ap),
void *);
1838 for (i = 0; i < argc; ++i)
1840 argv[i] = parent_team->t.t_argv[i];
1848 if (ompt_enabled.enabled) {
1849 ompt_task_info_t *task_info = OMPT_CUR_TASK_INFO(master_th);
1850 if (ompt_enabled.ompt_callback_implicit_task) {
1851 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1852 ompt_scope_end, NULL, &(task_info->task_data), 0,
1853 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_initial);
1855 if (ompt_enabled.ompt_callback_parallel_end) {
1856 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1857 &ompt_parallel_data, parent_task_data,
1858 OMPT_INVOKER(call_context) | ompt_parallel_league,
1861 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1866 for (i = argc - 1; i >= 0; --i)
1867 *argv++ = va_arg(kmp_va_deref(ap),
void *);
1872 void **exit_frame_p;
1873 ompt_task_info_t *task_info;
1875 ompt_lw_taskteam_t lw_taskteam;
1877 if (ompt_enabled.enabled) {
1878 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1879 &ompt_parallel_data, return_address);
1880 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1882 task_info = OMPT_CUR_TASK_INFO(master_th);
1883 exit_frame_p = &(task_info->frame.exit_frame.ptr);
1886 implicit_task_data = OMPT_CUR_TASK_DATA(master_th);
1887 if (ompt_enabled.ompt_callback_implicit_task) {
1888 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1889 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th),
1890 implicit_task_data, 1, __kmp_tid_from_gtid(gtid),
1891 ompt_task_implicit);
1892 OMPT_CUR_TASK_INFO(master_th)
1893 ->thread_num = __kmp_tid_from_gtid(gtid);
1897 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
1899 exit_frame_p = &dummy;
1904 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1905 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1906 __kmp_invoke_microtask(microtask, gtid, 0, argc, args
1915 if (ompt_enabled.enabled) {
1916 *exit_frame_p = NULL;
1917 if (ompt_enabled.ompt_callback_implicit_task) {
1918 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1919 ompt_scope_end, NULL, &(task_info->task_data), 1,
1920 OMPT_CUR_TASK_INFO(master_th)->thread_num,
1921 ompt_task_implicit);
1924 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th);
1925 __ompt_lw_taskteam_unlink(master_th);
1926 if (ompt_enabled.ompt_callback_parallel_end) {
1927 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1928 &ompt_parallel_data, parent_task_data,
1929 OMPT_INVOKER(call_context) | ompt_parallel_team,
1932 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1936 }
else if (call_context == fork_context_gnu) {
1938 ompt_lw_taskteam_t lwt;
1939 __ompt_lw_taskteam_init(&lwt, master_th, gtid, &ompt_parallel_data,
1942 lwt.ompt_task_info.frame.exit_frame = ompt_data_none;
1943 __ompt_lw_taskteam_link(&lwt, master_th, 1);
1948 KA_TRACE(20, (
"__kmp_fork_call: T#%d serial exit\n", gtid));
1951 KMP_ASSERT2(call_context < fork_context_last,
1952 "__kmp_fork_call: unknown fork_context parameter");
1955 KA_TRACE(20, (
"__kmp_fork_call: T#%d serial exit\n", gtid));
1962 KF_TRACE(10, (
"__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
1963 "curtask=%p, curtask_max_aclevel=%d\n",
1964 parent_team->t.t_active_level, master_th,
1965 master_th->th.th_current_task,
1966 master_th->th.th_current_task->td_icvs.max_active_levels));
1970 master_th->th.th_current_task->td_flags.executing = 0;
1972 if (!master_th->th.th_teams_microtask || level > teams_level) {
1974 KMP_ATOMIC_INC(&root->r.r_in_parallel);
1978 int nthreads_icv = master_th->th.th_current_task->td_icvs.nproc;
1979 if ((level + 1 < __kmp_nested_nth.used) &&
1980 (__kmp_nested_nth.nth[level + 1] != nthreads_icv)) {
1981 nthreads_icv = __kmp_nested_nth.nth[level + 1];
1987 kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind;
1988 kmp_proc_bind_t proc_bind_icv =
1990 if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
1991 proc_bind = proc_bind_false;
1993 if (proc_bind == proc_bind_default) {
1996 proc_bind = master_th->th.th_current_task->td_icvs.proc_bind;
2002 if ((level + 1 < __kmp_nested_proc_bind.used) &&
2003 (__kmp_nested_proc_bind.bind_types[level + 1] !=
2004 master_th->th.th_current_task->td_icvs.proc_bind)) {
2005 proc_bind_icv = __kmp_nested_proc_bind.bind_types[level + 1];
2010 master_th->th.th_set_proc_bind = proc_bind_default;
2012 if ((nthreads_icv > 0) || (proc_bind_icv != proc_bind_default)) {
2013 kmp_internal_control_t new_icvs;
2014 copy_icvs(&new_icvs, &master_th->th.th_current_task->td_icvs);
2015 new_icvs.next = NULL;
2016 if (nthreads_icv > 0) {
2017 new_icvs.nproc = nthreads_icv;
2019 if (proc_bind_icv != proc_bind_default) {
2020 new_icvs.proc_bind = proc_bind_icv;
2024 KF_TRACE(10, (
"__kmp_fork_call: before __kmp_allocate_team\n"));
2025 team = __kmp_allocate_team(root, nthreads, nthreads,
2029 proc_bind, &new_icvs,
2030 argc USE_NESTED_HOT_ARG(master_th));
2033 KF_TRACE(10, (
"__kmp_fork_call: before __kmp_allocate_team\n"));
2034 team = __kmp_allocate_team(root, nthreads, nthreads,
2039 &master_th->th.th_current_task->td_icvs,
2040 argc USE_NESTED_HOT_ARG(master_th));
2043 10, (
"__kmp_fork_call: after __kmp_allocate_team - team = %p\n", team));
2046 KMP_CHECK_UPDATE(team->t.t_master_tid, master_tid);
2047 KMP_CHECK_UPDATE(team->t.t_master_this_cons, master_this_cons);
2048 KMP_CHECK_UPDATE(team->t.t_ident, loc);
2049 KMP_CHECK_UPDATE(team->t.t_parent, parent_team);
2050 KMP_CHECK_UPDATE_SYNC(team->t.t_pkfn, microtask);
2052 KMP_CHECK_UPDATE_SYNC(team->t.ompt_team_info.master_return_address,
2055 KMP_CHECK_UPDATE(team->t.t_invoke, invoker);
2057 if (!master_th->th.th_teams_microtask || level > teams_level) {
2058 int new_level = parent_team->t.t_level + 1;
2059 KMP_CHECK_UPDATE(team->t.t_level, new_level);
2060 new_level = parent_team->t.t_active_level + 1;
2061 KMP_CHECK_UPDATE(team->t.t_active_level, new_level);
2064 int new_level = parent_team->t.t_level;
2065 KMP_CHECK_UPDATE(team->t.t_level, new_level);
2066 new_level = parent_team->t.t_active_level;
2067 KMP_CHECK_UPDATE(team->t.t_active_level, new_level);
2069 kmp_r_sched_t new_sched = get__sched_2(parent_team, master_tid);
2071 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched);
2073 KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq);
2074 KMP_CHECK_UPDATE(team->t.t_def_allocator, master_th->th.th_def_allocator);
2077 propagateFPControl(team);
2079 if (__kmp_tasking_mode != tskm_immediate_exec) {
2082 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
2083 parent_team->t.t_task_team[master_th->th.th_task_state]);
2084 KA_TRACE(20, (
"__kmp_fork_call: Master T#%d pushing task_team %p / team "
2085 "%p, new task_team %p / team %p\n",
2086 __kmp_gtid_from_thread(master_th),
2087 master_th->th.th_task_team, parent_team,
2088 team->t.t_task_team[master_th->th.th_task_state], team));
2090 if (active_level || master_th->th.th_task_team) {
2092 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
2093 if (master_th->th.th_task_state_top >=
2094 master_th->th.th_task_state_stack_sz) {
2095 kmp_uint32 new_size = 2 * master_th->th.th_task_state_stack_sz;
2096 kmp_uint8 *old_stack, *new_stack;
2098 new_stack = (kmp_uint8 *)__kmp_allocate(new_size);
2099 for (i = 0; i < master_th->th.th_task_state_stack_sz; ++i) {
2100 new_stack[i] = master_th->th.th_task_state_memo_stack[i];
2102 for (i = master_th->th.th_task_state_stack_sz; i < new_size;
2106 old_stack = master_th->th.th_task_state_memo_stack;
2107 master_th->th.th_task_state_memo_stack = new_stack;
2108 master_th->th.th_task_state_stack_sz = new_size;
2109 __kmp_free(old_stack);
2113 .th_task_state_memo_stack[master_th->th.th_task_state_top] =
2114 master_th->th.th_task_state;
2115 master_th->th.th_task_state_top++;
2116 #if KMP_NESTED_HOT_TEAMS
2117 if (master_th->th.th_hot_teams &&
2118 active_level < __kmp_hot_teams_max_level &&
2119 team == master_th->th.th_hot_teams[active_level].hot_team) {
2121 master_th->th.th_task_state =
2123 .th_task_state_memo_stack[master_th->th.th_task_state_top];
2126 master_th->th.th_task_state = 0;
2127 #if KMP_NESTED_HOT_TEAMS
2131 #if !KMP_NESTED_HOT_TEAMS
2132 KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) ||
2133 (team == root->r.r_hot_team));
2139 (
"__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n",
2140 gtid, parent_team->t.t_id, team->t.t_master_tid, team->t.t_id,
2142 KMP_DEBUG_ASSERT(team != root->r.r_hot_team ||
2143 (team->t.t_master_tid == 0 &&
2144 (team->t.t_parent == root->r.r_root_team ||
2145 team->t.t_parent->t.t_serialized)));
2149 argv = (
void **)team->t.t_argv;
2151 for (i = argc - 1; i >= 0; --i) {
2152 void *new_argv = va_arg(kmp_va_deref(ap),
void *);
2153 KMP_CHECK_UPDATE(*argv, new_argv);
2157 for (i = 0; i < argc; ++i) {
2159 KMP_CHECK_UPDATE(argv[i], team->t.t_parent->t.t_argv[i]);
2164 KMP_CHECK_UPDATE(team->t.t_master_active, master_active);
2165 if (!root->r.r_active)
2166 root->r.r_active = TRUE;
2168 __kmp_fork_team_threads(root, team, master_th, gtid);
2169 __kmp_setup_icv_copy(team, nthreads,
2170 &master_th->th.th_current_task->td_icvs, loc);
2173 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
2176 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2179 if (team->t.t_active_level == 1
2180 && !master_th->th.th_teams_microtask) {
2182 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) &&
2183 (__kmp_forkjoin_frames_mode == 3 ||
2184 __kmp_forkjoin_frames_mode == 1)) {
2185 kmp_uint64 tmp_time = 0;
2186 if (__itt_get_timestamp_ptr)
2187 tmp_time = __itt_get_timestamp();
2189 master_th->th.th_frame_time = tmp_time;
2190 if (__kmp_forkjoin_frames_mode == 3)
2191 team->t.t_region_time = tmp_time;
2195 if ((__itt_frame_begin_v3_ptr || KMP_ITT_DEBUG) &&
2196 __kmp_forkjoin_frames && !__kmp_forkjoin_frames_mode) {
2198 __kmp_itt_region_forking(gtid, team->t.t_nproc, 0);
2204 KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team);
2207 (
"__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n",
2208 root, team, master_th, gtid));
2211 if (__itt_stack_caller_create_ptr) {
2212 team->t.t_stack_id =
2213 __kmp_itt_stack_caller_create();
2221 __kmp_internal_fork(loc, gtid, team);
2222 KF_TRACE(10, (
"__kmp_internal_fork : after : root=%p, team=%p, "
2223 "master_th=%p, gtid=%d\n",
2224 root, team, master_th, gtid));
2227 if (call_context == fork_context_gnu) {
2228 KA_TRACE(20, (
"__kmp_fork_call: parallel exit T#%d\n", gtid));
2233 KA_TRACE(20, (
"__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,
2234 team->t.t_id, team->t.t_pkfn));
2237 #if KMP_STATS_ENABLED
2241 KMP_SET_THREAD_STATE(stats_state_e::TEAMS_REGION);
2245 if (!team->t.t_invoke(gtid)) {
2246 KMP_ASSERT2(0,
"cannot invoke microtask for MASTER thread");
2249 #if KMP_STATS_ENABLED
2252 KMP_SET_THREAD_STATE(previous_state);
2256 KA_TRACE(20, (
"__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,
2257 team->t.t_id, team->t.t_pkfn));
2260 KA_TRACE(20, (
"__kmp_fork_call: parallel exit T#%d\n", gtid));
2263 if (ompt_enabled.enabled) {
2264 master_th->th.ompt_thread_info.state = ompt_state_overhead;
2272 static inline void __kmp_join_restore_state(kmp_info_t *thread,
2275 thread->th.ompt_thread_info.state =
2276 ((team->t.t_serialized) ? ompt_state_work_serial
2277 : ompt_state_work_parallel);
2280 static inline void __kmp_join_ompt(
int gtid, kmp_info_t *thread,
2281 kmp_team_t *team, ompt_data_t *parallel_data,
2282 int flags,
void *codeptr) {
2283 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2284 if (ompt_enabled.ompt_callback_parallel_end) {
2285 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
2286 parallel_data, &(task_info->task_data), flags, codeptr);
2289 task_info->frame.enter_frame = ompt_data_none;
2290 __kmp_join_restore_state(thread, team);
2294 void __kmp_join_call(
ident_t *loc,
int gtid
2297 enum fork_context_e fork_context
2301 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_join_call);
2303 kmp_team_t *parent_team;
2304 kmp_info_t *master_th;
2308 KA_TRACE(20, (
"__kmp_join_call: enter T#%d\n", gtid));
2311 master_th = __kmp_threads[gtid];
2312 root = master_th->th.th_root;
2313 team = master_th->th.th_team;
2314 parent_team = team->t.t_parent;
2316 master_th->th.th_ident = loc;
2319 void *team_microtask = (
void *)team->t.t_pkfn;
2323 if (ompt_enabled.enabled &&
2324 !(team->t.t_serialized && fork_context == fork_context_gnu)) {
2325 master_th->th.ompt_thread_info.state = ompt_state_overhead;
2330 if (__kmp_tasking_mode != tskm_immediate_exec && !exit_teams) {
2331 KA_TRACE(20, (
"__kmp_join_call: T#%d, old team = %p old task_team = %p, "
2332 "th_task_team = %p\n",
2333 __kmp_gtid_from_thread(master_th), team,
2334 team->t.t_task_team[master_th->th.th_task_state],
2335 master_th->th.th_task_team));
2336 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
2337 team->t.t_task_team[master_th->th.th_task_state]);
2341 if (team->t.t_serialized) {
2342 if (master_th->th.th_teams_microtask) {
2344 int level = team->t.t_level;
2345 int tlevel = master_th->th.th_teams_level;
2346 if (level == tlevel) {
2350 }
else if (level == tlevel + 1) {
2354 team->t.t_serialized++;
2360 if (ompt_enabled.enabled) {
2361 __kmp_join_restore_state(master_th, parent_team);
2368 master_active = team->t.t_master_active;
2373 __kmp_internal_join(loc, gtid, team);
2375 master_th->th.th_task_state =
2382 ompt_data_t *parallel_data = &(team->t.ompt_team_info.parallel_data);
2383 void *codeptr = team->t.ompt_team_info.master_return_address;
2387 if (__itt_stack_caller_create_ptr) {
2389 __kmp_itt_stack_caller_destroy((__itt_caller)team->t.t_stack_id);
2392 if (team->t.t_active_level == 1 &&
2393 (!master_th->th.th_teams_microtask ||
2394 master_th->th.th_teams_size.nteams == 1)) {
2395 master_th->th.th_ident = loc;
2398 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) &&
2399 __kmp_forkjoin_frames_mode == 3)
2400 __kmp_itt_frame_submit(gtid, team->t.t_region_time,
2401 master_th->th.th_frame_time, 0, loc,
2402 master_th->th.th_team_nproc, 1);
2403 else if ((__itt_frame_end_v3_ptr || KMP_ITT_DEBUG) &&
2404 !__kmp_forkjoin_frames_mode && __kmp_forkjoin_frames)
2405 __kmp_itt_region_joined(gtid);
2409 if (master_th->th.th_teams_microtask && !exit_teams &&
2410 team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
2411 team->t.t_level == master_th->th.th_teams_level + 1) {
2416 ompt_data_t ompt_parallel_data = ompt_data_none;
2417 if (ompt_enabled.enabled) {
2418 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2419 if (ompt_enabled.ompt_callback_implicit_task) {
2420 int ompt_team_size = team->t.t_nproc;
2421 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
2422 ompt_scope_end, NULL, &(task_info->task_data), ompt_team_size,
2423 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit);
2425 task_info->frame.exit_frame = ompt_data_none;
2426 task_info->task_data = ompt_data_none;
2427 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th);
2428 __ompt_lw_taskteam_unlink(master_th);
2433 team->t.t_active_level--;
2434 KMP_ATOMIC_DEC(&root->r.r_in_parallel);
2440 if (master_th->th.th_team_nproc < master_th->th.th_teams_size.nth) {
2441 int old_num = master_th->th.th_team_nproc;
2442 int new_num = master_th->th.th_teams_size.nth;
2443 kmp_info_t **other_threads = team->t.t_threads;
2444 team->t.t_nproc = new_num;
2445 for (
int i = 0; i < old_num; ++i) {
2446 other_threads[i]->th.th_team_nproc = new_num;
2449 for (
int i = old_num; i < new_num; ++i) {
2451 KMP_DEBUG_ASSERT(other_threads[i]);
2452 kmp_balign_t *balign = other_threads[i]->th.th_bar;
2453 for (
int b = 0; b < bs_last_barrier; ++b) {
2454 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
2455 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
2457 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
2460 if (__kmp_tasking_mode != tskm_immediate_exec) {
2462 other_threads[i]->th.th_task_state = master_th->th.th_task_state;
2468 if (ompt_enabled.enabled) {
2469 __kmp_join_ompt(gtid, master_th, parent_team, &ompt_parallel_data,
2470 OMPT_INVOKER(fork_context) | ompt_parallel_team, codeptr);
2478 master_th->th.th_info.ds.ds_tid = team->t.t_master_tid;
2479 master_th->th.th_local.this_construct = team->t.t_master_this_cons;
2481 master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid];
2486 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
2488 if (!master_th->th.th_teams_microtask ||
2489 team->t.t_level > master_th->th.th_teams_level) {
2491 KMP_ATOMIC_DEC(&root->r.r_in_parallel);
2493 KMP_DEBUG_ASSERT(root->r.r_in_parallel >= 0);
2496 if (ompt_enabled.enabled) {
2497 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2498 if (ompt_enabled.ompt_callback_implicit_task) {
2499 int flags = (team_microtask == (
void *)__kmp_teams_master)
2501 : ompt_task_implicit;
2502 int ompt_team_size = (flags == ompt_task_initial) ? 0 : team->t.t_nproc;
2503 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
2504 ompt_scope_end, NULL, &(task_info->task_data), ompt_team_size,
2505 OMPT_CUR_TASK_INFO(master_th)->thread_num, flags);
2507 task_info->frame.exit_frame = ompt_data_none;
2508 task_info->task_data = ompt_data_none;
2512 KF_TRACE(10, (
"__kmp_join_call1: T#%d, this_thread=%p team=%p\n", 0,
2514 __kmp_pop_current_task_from_thread(master_th);
2516 #if KMP_AFFINITY_SUPPORTED
2518 master_th->th.th_first_place = team->t.t_first_place;
2519 master_th->th.th_last_place = team->t.t_last_place;
2520 #endif // KMP_AFFINITY_SUPPORTED
2521 master_th->th.th_def_allocator = team->t.t_def_allocator;
2523 updateHWFPControl(team);
2525 if (root->r.r_active != master_active)
2526 root->r.r_active = master_active;
2528 __kmp_free_team(root, team USE_NESTED_HOT_ARG(
2536 master_th->th.th_team = parent_team;
2537 master_th->th.th_team_nproc = parent_team->t.t_nproc;
2538 master_th->th.th_team_master = parent_team->t.t_threads[0];
2539 master_th->th.th_team_serialized = parent_team->t.t_serialized;
2542 if (parent_team->t.t_serialized &&
2543 parent_team != master_th->th.th_serial_team &&
2544 parent_team != root->r.r_root_team) {
2545 __kmp_free_team(root,
2546 master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL));
2547 master_th->th.th_serial_team = parent_team;
2550 if (__kmp_tasking_mode != tskm_immediate_exec) {
2551 if (master_th->th.th_task_state_top >
2553 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
2555 master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] =
2556 master_th->th.th_task_state;
2557 --master_th->th.th_task_state_top;
2559 master_th->th.th_task_state =
2561 .th_task_state_memo_stack[master_th->th.th_task_state_top];
2564 master_th->th.th_task_team =
2565 parent_team->t.t_task_team[master_th->th.th_task_state];
2567 (
"__kmp_join_call: Master T#%d restoring task_team %p / team %p\n",
2568 __kmp_gtid_from_thread(master_th), master_th->th.th_task_team,
2575 master_th->th.th_current_task->td_flags.executing = 1;
2577 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2581 OMPT_INVOKER(fork_context) |
2582 ((team_microtask == (
void *)__kmp_teams_master) ? ompt_parallel_league
2583 : ompt_parallel_team);
2584 if (ompt_enabled.enabled) {
2585 __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, flags,
2591 KA_TRACE(20, (
"__kmp_join_call: exit T#%d\n", gtid));
2596 void __kmp_save_internal_controls(kmp_info_t *thread) {
2598 if (thread->th.th_team != thread->th.th_serial_team) {
2601 if (thread->th.th_team->t.t_serialized > 1) {
2604 if (thread->th.th_team->t.t_control_stack_top == NULL) {
2607 if (thread->th.th_team->t.t_control_stack_top->serial_nesting_level !=
2608 thread->th.th_team->t.t_serialized) {
2613 kmp_internal_control_t *control =
2614 (kmp_internal_control_t *)__kmp_allocate(
2615 sizeof(kmp_internal_control_t));
2617 copy_icvs(control, &thread->th.th_current_task->td_icvs);
2619 control->serial_nesting_level = thread->th.th_team->t.t_serialized;
2621 control->next = thread->th.th_team->t.t_control_stack_top;
2622 thread->th.th_team->t.t_control_stack_top = control;
2628 void __kmp_set_num_threads(
int new_nth,
int gtid) {
2632 KF_TRACE(10, (
"__kmp_set_num_threads: new __kmp_nth = %d\n", new_nth));
2633 KMP_DEBUG_ASSERT(__kmp_init_serial);
2637 else if (new_nth > __kmp_max_nth)
2638 new_nth = __kmp_max_nth;
2641 thread = __kmp_threads[gtid];
2642 if (thread->th.th_current_task->td_icvs.nproc == new_nth)
2645 __kmp_save_internal_controls(thread);
2647 set__nproc(thread, new_nth);
2652 root = thread->th.th_root;
2653 if (__kmp_init_parallel && (!root->r.r_active) &&
2654 (root->r.r_hot_team->t.t_nproc > new_nth)
2655 #
if KMP_NESTED_HOT_TEAMS
2656 && __kmp_hot_teams_max_level && !__kmp_hot_teams_mode
2659 kmp_team_t *hot_team = root->r.r_hot_team;
2662 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
2665 for (f = new_nth; f < hot_team->t.t_nproc; f++) {
2666 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL);
2667 if (__kmp_tasking_mode != tskm_immediate_exec) {
2670 hot_team->t.t_threads[f]->th.th_task_team = NULL;
2672 __kmp_free_thread(hot_team->t.t_threads[f]);
2673 hot_team->t.t_threads[f] = NULL;
2675 hot_team->t.t_nproc = new_nth;
2676 #if KMP_NESTED_HOT_TEAMS
2677 if (thread->th.th_hot_teams) {
2678 KMP_DEBUG_ASSERT(hot_team == thread->th.th_hot_teams[0].hot_team);
2679 thread->th.th_hot_teams[0].hot_team_nth = new_nth;
2683 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2686 for (f = 0; f < new_nth; f++) {
2687 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL);
2688 hot_team->t.t_threads[f]->th.th_team_nproc = new_nth;
2691 hot_team->t.t_size_changed = -1;
2696 void __kmp_set_max_active_levels(
int gtid,
int max_active_levels) {
2699 KF_TRACE(10, (
"__kmp_set_max_active_levels: new max_active_levels for thread "
2701 gtid, max_active_levels));
2702 KMP_DEBUG_ASSERT(__kmp_init_serial);
2705 if (max_active_levels < 0) {
2706 KMP_WARNING(ActiveLevelsNegative, max_active_levels);
2711 KF_TRACE(10, (
"__kmp_set_max_active_levels: the call is ignored: new "
2712 "max_active_levels for thread %d = (%d)\n",
2713 gtid, max_active_levels));
2716 if (max_active_levels <= KMP_MAX_ACTIVE_LEVELS_LIMIT) {
2721 KMP_WARNING(ActiveLevelsExceedLimit, max_active_levels,
2722 KMP_MAX_ACTIVE_LEVELS_LIMIT);
2723 max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT;
2729 KF_TRACE(10, (
"__kmp_set_max_active_levels: after validation: new "
2730 "max_active_levels for thread %d = (%d)\n",
2731 gtid, max_active_levels));
2733 thread = __kmp_threads[gtid];
2735 __kmp_save_internal_controls(thread);
2737 set__max_active_levels(thread, max_active_levels);
2741 int __kmp_get_max_active_levels(
int gtid) {
2744 KF_TRACE(10, (
"__kmp_get_max_active_levels: thread %d\n", gtid));
2745 KMP_DEBUG_ASSERT(__kmp_init_serial);
2747 thread = __kmp_threads[gtid];
2748 KMP_DEBUG_ASSERT(thread->th.th_current_task);
2749 KF_TRACE(10, (
"__kmp_get_max_active_levels: thread %d, curtask=%p, "
2750 "curtask_maxaclevel=%d\n",
2751 gtid, thread->th.th_current_task,
2752 thread->th.th_current_task->td_icvs.max_active_levels));
2753 return thread->th.th_current_task->td_icvs.max_active_levels;
2756 KMP_BUILD_ASSERT(
sizeof(kmp_sched_t) ==
sizeof(
int));
2757 KMP_BUILD_ASSERT(
sizeof(
enum sched_type) ==
sizeof(
int));
2760 void __kmp_set_schedule(
int gtid, kmp_sched_t kind,
int chunk) {
2762 kmp_sched_t orig_kind;
2765 KF_TRACE(10, (
"__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n",
2766 gtid, (
int)kind, chunk));
2767 KMP_DEBUG_ASSERT(__kmp_init_serial);
2774 kind = __kmp_sched_without_mods(kind);
2776 if (kind <= kmp_sched_lower || kind >= kmp_sched_upper ||
2777 (kind <= kmp_sched_lower_ext && kind >= kmp_sched_upper_std)) {
2779 __kmp_msg(kmp_ms_warning, KMP_MSG(ScheduleKindOutOfRange, kind),
2780 KMP_HNT(DefaultScheduleKindUsed,
"static, no chunk"),
2782 kind = kmp_sched_default;
2786 thread = __kmp_threads[gtid];
2788 __kmp_save_internal_controls(thread);
2790 if (kind < kmp_sched_upper_std) {
2791 if (kind == kmp_sched_static && chunk < KMP_DEFAULT_CHUNK) {
2794 thread->th.th_current_task->td_icvs.sched.r_sched_type =
kmp_sch_static;
2796 thread->th.th_current_task->td_icvs.sched.r_sched_type =
2797 __kmp_sch_map[kind - kmp_sched_lower - 1];
2802 thread->th.th_current_task->td_icvs.sched.r_sched_type =
2803 __kmp_sch_map[kind - kmp_sched_lower_ext + kmp_sched_upper_std -
2804 kmp_sched_lower - 2];
2806 __kmp_sched_apply_mods_intkind(
2807 orig_kind, &(thread->th.th_current_task->td_icvs.sched.r_sched_type));
2808 if (kind == kmp_sched_auto || chunk < 1) {
2810 thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK;
2812 thread->th.th_current_task->td_icvs.sched.chunk = chunk;
2817 void __kmp_get_schedule(
int gtid, kmp_sched_t *kind,
int *chunk) {
2821 KF_TRACE(10, (
"__kmp_get_schedule: thread %d\n", gtid));
2822 KMP_DEBUG_ASSERT(__kmp_init_serial);
2824 thread = __kmp_threads[gtid];
2826 th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type;
2827 switch (SCHEDULE_WITHOUT_MODIFIERS(th_type)) {
2829 case kmp_sch_static_greedy:
2830 case kmp_sch_static_balanced:
2831 *kind = kmp_sched_static;
2832 __kmp_sched_apply_mods_stdkind(kind, th_type);
2835 case kmp_sch_static_chunked:
2836 *kind = kmp_sched_static;
2838 case kmp_sch_dynamic_chunked:
2839 *kind = kmp_sched_dynamic;
2842 case kmp_sch_guided_iterative_chunked:
2843 case kmp_sch_guided_analytical_chunked:
2844 *kind = kmp_sched_guided;
2847 *kind = kmp_sched_auto;
2849 case kmp_sch_trapezoidal:
2850 *kind = kmp_sched_trapezoidal;
2852 #if KMP_STATIC_STEAL_ENABLED
2853 case kmp_sch_static_steal:
2854 *kind = kmp_sched_static_steal;
2858 KMP_FATAL(UnknownSchedulingType, th_type);
2861 __kmp_sched_apply_mods_stdkind(kind, th_type);
2862 *chunk = thread->th.th_current_task->td_icvs.sched.chunk;
2865 int __kmp_get_ancestor_thread_num(
int gtid,
int level) {
2871 KF_TRACE(10, (
"__kmp_get_ancestor_thread_num: thread %d %d\n", gtid, level));
2872 KMP_DEBUG_ASSERT(__kmp_init_serial);
2879 thr = __kmp_threads[gtid];
2880 team = thr->th.th_team;
2881 ii = team->t.t_level;
2885 if (thr->th.th_teams_microtask) {
2887 int tlevel = thr->th.th_teams_level;
2890 KMP_DEBUG_ASSERT(ii >= tlevel);
2902 return __kmp_tid_from_gtid(gtid);
2904 dd = team->t.t_serialized;
2906 while (ii > level) {
2907 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
2909 if ((team->t.t_serialized) && (!dd)) {
2910 team = team->t.t_parent;
2914 team = team->t.t_parent;
2915 dd = team->t.t_serialized;
2920 return (dd > 1) ? (0) : (team->t.t_master_tid);
2923 int __kmp_get_team_size(
int gtid,
int level) {
2929 KF_TRACE(10, (
"__kmp_get_team_size: thread %d %d\n", gtid, level));
2930 KMP_DEBUG_ASSERT(__kmp_init_serial);
2937 thr = __kmp_threads[gtid];
2938 team = thr->th.th_team;
2939 ii = team->t.t_level;
2943 if (thr->th.th_teams_microtask) {
2945 int tlevel = thr->th.th_teams_level;
2948 KMP_DEBUG_ASSERT(ii >= tlevel);
2959 while (ii > level) {
2960 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
2962 if (team->t.t_serialized && (!dd)) {
2963 team = team->t.t_parent;
2967 team = team->t.t_parent;
2972 return team->t.t_nproc;
2975 kmp_r_sched_t __kmp_get_schedule_global() {
2980 kmp_r_sched_t r_sched;
2986 enum sched_type s = SCHEDULE_WITHOUT_MODIFIERS(__kmp_sched);
2987 enum sched_type sched_modifiers = SCHEDULE_GET_MODIFIERS(__kmp_sched);
2990 r_sched.r_sched_type = __kmp_static;
2993 r_sched.r_sched_type = __kmp_guided;
2995 r_sched.r_sched_type = __kmp_sched;
2997 SCHEDULE_SET_MODIFIERS(r_sched.r_sched_type, sched_modifiers);
2999 if (__kmp_chunk < KMP_DEFAULT_CHUNK) {
3001 r_sched.chunk = KMP_DEFAULT_CHUNK;
3003 r_sched.chunk = __kmp_chunk;
3011 static void __kmp_alloc_argv_entries(
int argc, kmp_team_t *team,
int realloc) {
3013 KMP_DEBUG_ASSERT(team);
3014 if (!realloc || argc > team->t.t_max_argc) {
3016 KA_TRACE(100, (
"__kmp_alloc_argv_entries: team %d: needed entries=%d, "
3017 "current entries=%d\n",
3018 team->t.t_id, argc, (realloc) ? team->t.t_max_argc : 0));
3020 if (realloc && team->t.t_argv != &team->t.t_inline_argv[0])
3021 __kmp_free((
void *)team->t.t_argv);
3023 if (argc <= KMP_INLINE_ARGV_ENTRIES) {
3025 team->t.t_max_argc = KMP_INLINE_ARGV_ENTRIES;
3026 KA_TRACE(100, (
"__kmp_alloc_argv_entries: team %d: inline allocate %d "
3028 team->t.t_id, team->t.t_max_argc));
3029 team->t.t_argv = &team->t.t_inline_argv[0];
3030 if (__kmp_storage_map) {
3031 __kmp_print_storage_map_gtid(
3032 -1, &team->t.t_inline_argv[0],
3033 &team->t.t_inline_argv[KMP_INLINE_ARGV_ENTRIES],
3034 (
sizeof(
void *) * KMP_INLINE_ARGV_ENTRIES),
"team_%d.t_inline_argv",
3039 team->t.t_max_argc = (argc <= (KMP_MIN_MALLOC_ARGV_ENTRIES >> 1))
3040 ? KMP_MIN_MALLOC_ARGV_ENTRIES
3042 KA_TRACE(100, (
"__kmp_alloc_argv_entries: team %d: dynamic allocate %d "
3044 team->t.t_id, team->t.t_max_argc));
3046 (
void **)__kmp_page_allocate(
sizeof(
void *) * team->t.t_max_argc);
3047 if (__kmp_storage_map) {
3048 __kmp_print_storage_map_gtid(-1, &team->t.t_argv[0],
3049 &team->t.t_argv[team->t.t_max_argc],
3050 sizeof(
void *) * team->t.t_max_argc,
3051 "team_%d.t_argv", team->t.t_id);
3057 static void __kmp_allocate_team_arrays(kmp_team_t *team,
int max_nth) {
3059 int num_disp_buff = max_nth > 1 ? __kmp_dispatch_num_buffers : 2;
3061 (kmp_info_t **)__kmp_allocate(
sizeof(kmp_info_t *) * max_nth);
3062 team->t.t_disp_buffer = (dispatch_shared_info_t *)__kmp_allocate(
3063 sizeof(dispatch_shared_info_t) * num_disp_buff);
3064 team->t.t_dispatch =
3065 (kmp_disp_t *)__kmp_allocate(
sizeof(kmp_disp_t) * max_nth);
3066 team->t.t_implicit_task_taskdata =
3067 (kmp_taskdata_t *)__kmp_allocate(
sizeof(kmp_taskdata_t) * max_nth);
3068 team->t.t_max_nproc = max_nth;
3071 for (i = 0; i < num_disp_buff; ++i) {
3072 team->t.t_disp_buffer[i].buffer_index = i;
3073 team->t.t_disp_buffer[i].doacross_buf_idx = i;
3077 static void __kmp_free_team_arrays(kmp_team_t *team) {
3080 for (i = 0; i < team->t.t_max_nproc; ++i) {
3081 if (team->t.t_dispatch[i].th_disp_buffer != NULL) {
3082 __kmp_free(team->t.t_dispatch[i].th_disp_buffer);
3083 team->t.t_dispatch[i].th_disp_buffer = NULL;
3086 #if KMP_USE_HIER_SCHED
3087 __kmp_dispatch_free_hierarchies(team);
3089 __kmp_free(team->t.t_threads);
3090 __kmp_free(team->t.t_disp_buffer);
3091 __kmp_free(team->t.t_dispatch);
3092 __kmp_free(team->t.t_implicit_task_taskdata);
3093 team->t.t_threads = NULL;
3094 team->t.t_disp_buffer = NULL;
3095 team->t.t_dispatch = NULL;
3096 team->t.t_implicit_task_taskdata = 0;
3099 static void __kmp_reallocate_team_arrays(kmp_team_t *team,
int max_nth) {
3100 kmp_info_t **oldThreads = team->t.t_threads;
3102 __kmp_free(team->t.t_disp_buffer);
3103 __kmp_free(team->t.t_dispatch);
3104 __kmp_free(team->t.t_implicit_task_taskdata);
3105 __kmp_allocate_team_arrays(team, max_nth);
3107 KMP_MEMCPY(team->t.t_threads, oldThreads,
3108 team->t.t_nproc *
sizeof(kmp_info_t *));
3110 __kmp_free(oldThreads);
3113 static kmp_internal_control_t __kmp_get_global_icvs(
void) {
3115 kmp_r_sched_t r_sched =
3116 __kmp_get_schedule_global();
3118 KMP_DEBUG_ASSERT(__kmp_nested_proc_bind.used > 0);
3120 kmp_internal_control_t g_icvs = {
3122 (kmp_int8)__kmp_global.g.g_dynamic,
3124 (kmp_int8)__kmp_env_blocktime,
3126 __kmp_dflt_blocktime,
3131 __kmp_dflt_team_nth,
3135 __kmp_dflt_max_active_levels,
3139 __kmp_nested_proc_bind.bind_types[0],
3140 __kmp_default_device,
3147 static kmp_internal_control_t __kmp_get_x_global_icvs(
const kmp_team_t *team) {
3149 kmp_internal_control_t gx_icvs;
3150 gx_icvs.serial_nesting_level =
3152 copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs);
3153 gx_icvs.next = NULL;
3158 static void __kmp_initialize_root(kmp_root_t *root) {
3160 kmp_team_t *root_team;
3161 kmp_team_t *hot_team;
3162 int hot_team_max_nth;
3163 kmp_r_sched_t r_sched =
3164 __kmp_get_schedule_global();
3165 kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
3166 KMP_DEBUG_ASSERT(root);
3167 KMP_ASSERT(!root->r.r_begin);
3170 __kmp_init_lock(&root->r.r_begin_lock);
3171 root->r.r_begin = FALSE;
3172 root->r.r_active = FALSE;
3173 root->r.r_in_parallel = 0;
3174 root->r.r_blocktime = __kmp_dflt_blocktime;
3178 KF_TRACE(10, (
"__kmp_initialize_root: before root_team\n"));
3181 __kmp_allocate_team(root,
3187 __kmp_nested_proc_bind.bind_types[0], &r_icvs,
3189 USE_NESTED_HOT_ARG(NULL)
3194 TCW_SYNC_PTR(root_team->t.t_pkfn, (microtask_t)(~0));
3197 KF_TRACE(10, (
"__kmp_initialize_root: after root_team = %p\n", root_team));
3199 root->r.r_root_team = root_team;
3200 root_team->t.t_control_stack_top = NULL;
3203 root_team->t.t_threads[0] = NULL;
3204 root_team->t.t_nproc = 1;
3205 root_team->t.t_serialized = 1;
3207 root_team->t.t_sched.sched = r_sched.sched;
3210 (
"__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n",
3211 root_team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
3215 KF_TRACE(10, (
"__kmp_initialize_root: before hot_team\n"));
3218 __kmp_allocate_team(root,
3220 __kmp_dflt_team_nth_ub * 2,
3224 __kmp_nested_proc_bind.bind_types[0], &r_icvs,
3226 USE_NESTED_HOT_ARG(NULL)
3228 KF_TRACE(10, (
"__kmp_initialize_root: after hot_team = %p\n", hot_team));
3230 root->r.r_hot_team = hot_team;
3231 root_team->t.t_control_stack_top = NULL;
3234 hot_team->t.t_parent = root_team;
3237 hot_team_max_nth = hot_team->t.t_max_nproc;
3238 for (f = 0; f < hot_team_max_nth; ++f) {
3239 hot_team->t.t_threads[f] = NULL;
3241 hot_team->t.t_nproc = 1;
3243 hot_team->t.t_sched.sched = r_sched.sched;
3244 hot_team->t.t_size_changed = 0;
3249 typedef struct kmp_team_list_item {
3250 kmp_team_p
const *entry;
3251 struct kmp_team_list_item *next;
3252 } kmp_team_list_item_t;
3253 typedef kmp_team_list_item_t *kmp_team_list_t;
3255 static void __kmp_print_structure_team_accum(
3256 kmp_team_list_t list,
3257 kmp_team_p
const *team
3267 KMP_DEBUG_ASSERT(list != NULL);
3272 __kmp_print_structure_team_accum(list, team->t.t_parent);
3273 __kmp_print_structure_team_accum(list, team->t.t_next_pool);
3277 while (l->next != NULL && l->entry != team) {
3280 if (l->next != NULL) {
3286 while (l->next != NULL && l->entry->t.t_id <= team->t.t_id) {
3292 kmp_team_list_item_t *item = (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(
3293 sizeof(kmp_team_list_item_t));
3300 static void __kmp_print_structure_team(
char const *title, kmp_team_p
const *team
3303 __kmp_printf(
"%s", title);
3305 __kmp_printf(
"%2x %p\n", team->t.t_id, team);
3307 __kmp_printf(
" - (nil)\n");
3311 static void __kmp_print_structure_thread(
char const *title,
3312 kmp_info_p
const *thread) {
3313 __kmp_printf(
"%s", title);
3314 if (thread != NULL) {
3315 __kmp_printf(
"%2d %p\n", thread->th.th_info.ds.ds_gtid, thread);
3317 __kmp_printf(
" - (nil)\n");
3321 void __kmp_print_structure(
void) {
3323 kmp_team_list_t list;
3327 (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(
sizeof(kmp_team_list_item_t));
3331 __kmp_printf(
"\n------------------------------\nGlobal Thread "
3332 "Table\n------------------------------\n");
3335 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3336 __kmp_printf(
"%2d", gtid);
3337 if (__kmp_threads != NULL) {
3338 __kmp_printf(
" %p", __kmp_threads[gtid]);
3340 if (__kmp_root != NULL) {
3341 __kmp_printf(
" %p", __kmp_root[gtid]);
3348 __kmp_printf(
"\n------------------------------\nThreads\n--------------------"
3350 if (__kmp_threads != NULL) {
3352 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3353 kmp_info_t
const *thread = __kmp_threads[gtid];
3354 if (thread != NULL) {
3355 __kmp_printf(
"GTID %2d %p:\n", gtid, thread);
3356 __kmp_printf(
" Our Root: %p\n", thread->th.th_root);
3357 __kmp_print_structure_team(
" Our Team: ", thread->th.th_team);
3358 __kmp_print_structure_team(
" Serial Team: ",
3359 thread->th.th_serial_team);
3360 __kmp_printf(
" Threads: %2d\n", thread->th.th_team_nproc);
3361 __kmp_print_structure_thread(
" Master: ",
3362 thread->th.th_team_master);
3363 __kmp_printf(
" Serialized?: %2d\n", thread->th.th_team_serialized);
3364 __kmp_printf(
" Set NProc: %2d\n", thread->th.th_set_nproc);
3365 __kmp_printf(
" Set Proc Bind: %2d\n", thread->th.th_set_proc_bind);
3366 __kmp_print_structure_thread(
" Next in pool: ",
3367 thread->th.th_next_pool);
3369 __kmp_print_structure_team_accum(list, thread->th.th_team);
3370 __kmp_print_structure_team_accum(list, thread->th.th_serial_team);
3374 __kmp_printf(
"Threads array is not allocated.\n");
3378 __kmp_printf(
"\n------------------------------\nUbers\n----------------------"
3380 if (__kmp_root != NULL) {
3382 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3383 kmp_root_t
const *root = __kmp_root[gtid];
3385 __kmp_printf(
"GTID %2d %p:\n", gtid, root);
3386 __kmp_print_structure_team(
" Root Team: ", root->r.r_root_team);
3387 __kmp_print_structure_team(
" Hot Team: ", root->r.r_hot_team);
3388 __kmp_print_structure_thread(
" Uber Thread: ",
3389 root->r.r_uber_thread);
3390 __kmp_printf(
" Active?: %2d\n", root->r.r_active);
3391 __kmp_printf(
" In Parallel: %2d\n",
3392 KMP_ATOMIC_LD_RLX(&root->r.r_in_parallel));
3394 __kmp_print_structure_team_accum(list, root->r.r_root_team);
3395 __kmp_print_structure_team_accum(list, root->r.r_hot_team);
3399 __kmp_printf(
"Ubers array is not allocated.\n");
3402 __kmp_printf(
"\n------------------------------\nTeams\n----------------------"
3404 while (list->next != NULL) {
3405 kmp_team_p
const *team = list->entry;
3407 __kmp_printf(
"Team %2x %p:\n", team->t.t_id, team);
3408 __kmp_print_structure_team(
" Parent Team: ", team->t.t_parent);
3409 __kmp_printf(
" Master TID: %2d\n", team->t.t_master_tid);
3410 __kmp_printf(
" Max threads: %2d\n", team->t.t_max_nproc);
3411 __kmp_printf(
" Levels of serial: %2d\n", team->t.t_serialized);
3412 __kmp_printf(
" Number threads: %2d\n", team->t.t_nproc);
3413 for (i = 0; i < team->t.t_nproc; ++i) {
3414 __kmp_printf(
" Thread %2d: ", i);
3415 __kmp_print_structure_thread(
"", team->t.t_threads[i]);
3417 __kmp_print_structure_team(
" Next in pool: ", team->t.t_next_pool);
3423 __kmp_printf(
"\n------------------------------\nPools\n----------------------"
3425 __kmp_print_structure_thread(
"Thread pool: ",
3426 CCAST(kmp_info_t *, __kmp_thread_pool));
3427 __kmp_print_structure_team(
"Team pool: ",
3428 CCAST(kmp_team_t *, __kmp_team_pool));
3432 while (list != NULL) {
3433 kmp_team_list_item_t *item = list;
3435 KMP_INTERNAL_FREE(item);
3444 static const unsigned __kmp_primes[] = {
3445 0x9e3779b1, 0xffe6cc59, 0x2109f6dd, 0x43977ab5, 0xba5703f5, 0xb495a877,
3446 0xe1626741, 0x79695e6b, 0xbc98c09f, 0xd5bee2b3, 0x287488f9, 0x3af18231,
3447 0x9677cd4d, 0xbe3a6929, 0xadc6a877, 0xdcf0674b, 0xbe4d6fe9, 0x5f15e201,
3448 0x99afc3fd, 0xf3f16801, 0xe222cfff, 0x24ba5fdb, 0x0620452d, 0x79f149e3,
3449 0xc8b93f49, 0x972702cd, 0xb07dd827, 0x6c97d5ed, 0x085a3d61, 0x46eb5ea7,
3450 0x3d9910ed, 0x2e687b5b, 0x29609227, 0x6eb081f1, 0x0954c4e1, 0x9d114db9,
3451 0x542acfa9, 0xb3e6bd7b, 0x0742d917, 0xe9f3ffa7, 0x54581edb, 0xf2480f45,
3452 0x0bb9288f, 0xef1affc7, 0x85fa0ca7, 0x3ccc14db, 0xe6baf34b, 0x343377f7,
3453 0x5ca19031, 0xe6d9293b, 0xf0a9f391, 0x5d2e980b, 0xfc411073, 0xc3749363,
3454 0xb892d829, 0x3549366b, 0x629750ad, 0xb98294e5, 0x892d9483, 0xc235baf3,
3455 0x3d2402a3, 0x6bdef3c9, 0xbec333cd, 0x40c9520f};
3459 unsigned short __kmp_get_random(kmp_info_t *thread) {
3460 unsigned x = thread->th.th_x;
3461 unsigned short r = (
unsigned short)(x >> 16);
3463 thread->th.th_x = x * thread->th.th_a + 1;
3465 KA_TRACE(30, (
"__kmp_get_random: THREAD: %d, RETURN: %u\n",
3466 thread->th.th_info.ds.ds_tid, r));
3472 void __kmp_init_random(kmp_info_t *thread) {
3473 unsigned seed = thread->th.th_info.ds.ds_tid;
3476 __kmp_primes[seed % (
sizeof(__kmp_primes) /
sizeof(__kmp_primes[0]))];
3477 thread->th.th_x = (seed + 1) * thread->th.th_a + 1;
3479 (
"__kmp_init_random: THREAD: %u; A: %u\n", seed, thread->th.th_a));
3485 static int __kmp_reclaim_dead_roots(
void) {
3488 for (i = 0; i < __kmp_threads_capacity; ++i) {
3489 if (KMP_UBER_GTID(i) &&
3490 !__kmp_still_running((kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[i])) &&
3493 r += __kmp_unregister_root_other_thread(i);
3518 static int __kmp_expand_threads(
int nNeed) {
3520 int minimumRequiredCapacity;
3522 kmp_info_t **newThreads;
3523 kmp_root_t **newRoot;
3529 #if KMP_OS_WINDOWS && !KMP_DYNAMIC_LIB
3532 added = __kmp_reclaim_dead_roots();
3561 KMP_DEBUG_ASSERT(__kmp_sys_max_nth >= __kmp_threads_capacity);
3564 if (__kmp_sys_max_nth - __kmp_threads_capacity < nNeed) {
3568 minimumRequiredCapacity = __kmp_threads_capacity + nNeed;
3570 newCapacity = __kmp_threads_capacity;
3572 newCapacity = newCapacity <= (__kmp_sys_max_nth >> 1) ? (newCapacity << 1)
3573 : __kmp_sys_max_nth;
3574 }
while (newCapacity < minimumRequiredCapacity);
3575 newThreads = (kmp_info_t **)__kmp_allocate(
3576 (
sizeof(kmp_info_t *) +
sizeof(kmp_root_t *)) * newCapacity + CACHE_LINE);
3578 (kmp_root_t **)((
char *)newThreads +
sizeof(kmp_info_t *) * newCapacity);
3579 KMP_MEMCPY(newThreads, __kmp_threads,
3580 __kmp_threads_capacity *
sizeof(kmp_info_t *));
3581 KMP_MEMCPY(newRoot, __kmp_root,
3582 __kmp_threads_capacity *
sizeof(kmp_root_t *));
3584 kmp_info_t **temp_threads = __kmp_threads;
3585 *(kmp_info_t * *
volatile *)&__kmp_threads = newThreads;
3586 *(kmp_root_t * *
volatile *)&__kmp_root = newRoot;
3587 __kmp_free(temp_threads);
3588 added += newCapacity - __kmp_threads_capacity;
3589 *(
volatile int *)&__kmp_threads_capacity = newCapacity;
3591 if (newCapacity > __kmp_tp_capacity) {
3592 __kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock);
3593 if (__kmp_tp_cached && newCapacity > __kmp_tp_capacity) {
3594 __kmp_threadprivate_resize_cache(newCapacity);
3596 *(
volatile int *)&__kmp_tp_capacity = newCapacity;
3598 __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock);
3607 int __kmp_register_root(
int initial_thread) {
3608 kmp_info_t *root_thread;
3612 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
3613 KA_TRACE(20, (
"__kmp_register_root: entered\n"));
3630 capacity = __kmp_threads_capacity;
3631 if (!initial_thread && TCR_PTR(__kmp_threads[0]) == NULL) {
3636 if (__kmp_all_nth >= capacity && !__kmp_expand_threads(1)) {
3637 if (__kmp_tp_cached) {
3638 __kmp_fatal(KMP_MSG(CantRegisterNewThread),
3639 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity),
3640 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null);
3642 __kmp_fatal(KMP_MSG(CantRegisterNewThread), KMP_HNT(SystemLimitOnThreads),
3652 if (TCR_4(__kmp_init_hidden_helper_threads)) {
3655 for (gtid = 1; TCR_PTR(__kmp_threads[gtid]) != NULL &&
3656 gtid <= __kmp_hidden_helper_threads_num;
3659 KMP_ASSERT(gtid <= __kmp_hidden_helper_threads_num);
3660 KA_TRACE(1, (
"__kmp_register_root: found slot in threads array for "
3661 "hidden helper thread: T#%d\n",
3667 if (initial_thread && __kmp_threads[0] == NULL) {
3670 for (gtid = __kmp_hidden_helper_threads_num + 1;
3671 TCR_PTR(__kmp_threads[gtid]) != NULL; gtid++)
3675 1, (
"__kmp_register_root: found slot in threads array: T#%d\n", gtid));
3676 KMP_ASSERT(gtid < __kmp_threads_capacity);
3681 TCW_4(__kmp_nth, __kmp_nth + 1);
3685 if (__kmp_adjust_gtid_mode) {
3686 if (__kmp_all_nth >= __kmp_tls_gtid_min) {
3687 if (TCR_4(__kmp_gtid_mode) != 2) {
3688 TCW_4(__kmp_gtid_mode, 2);
3691 if (TCR_4(__kmp_gtid_mode) != 1) {
3692 TCW_4(__kmp_gtid_mode, 1);
3697 #ifdef KMP_ADJUST_BLOCKTIME
3700 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
3701 if (__kmp_nth > __kmp_avail_proc) {
3702 __kmp_zero_bt = TRUE;
3708 if (!(root = __kmp_root[gtid])) {
3709 root = __kmp_root[gtid] = (kmp_root_t *)__kmp_allocate(
sizeof(kmp_root_t));
3710 KMP_DEBUG_ASSERT(!root->r.r_root_team);
3713 #if KMP_STATS_ENABLED
3715 __kmp_stats_thread_ptr = __kmp_stats_list->push_back(gtid);
3716 __kmp_stats_thread_ptr->startLife();
3717 KMP_SET_THREAD_STATE(SERIAL_REGION);
3720 __kmp_initialize_root(root);
3723 if (root->r.r_uber_thread) {
3724 root_thread = root->r.r_uber_thread;
3726 root_thread = (kmp_info_t *)__kmp_allocate(
sizeof(kmp_info_t));
3727 if (__kmp_storage_map) {
3728 __kmp_print_thread_storage_map(root_thread, gtid);
3730 root_thread->th.th_info.ds.ds_gtid = gtid;
3732 root_thread->th.ompt_thread_info.thread_data = ompt_data_none;
3734 root_thread->th.th_root = root;
3735 if (__kmp_env_consistency_check) {
3736 root_thread->th.th_cons = __kmp_allocate_cons_stack(gtid);
3739 __kmp_initialize_fast_memory(root_thread);
3743 KMP_DEBUG_ASSERT(root_thread->th.th_local.bget_data == NULL);
3744 __kmp_initialize_bget(root_thread);
3746 __kmp_init_random(root_thread);
3750 if (!root_thread->th.th_serial_team) {
3751 kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
3752 KF_TRACE(10, (
"__kmp_register_root: before serial_team\n"));
3753 root_thread->th.th_serial_team = __kmp_allocate_team(
3758 proc_bind_default, &r_icvs, 0 USE_NESTED_HOT_ARG(NULL));
3760 KMP_ASSERT(root_thread->th.th_serial_team);
3761 KF_TRACE(10, (
"__kmp_register_root: after serial_team = %p\n",
3762 root_thread->th.th_serial_team));
3765 TCW_SYNC_PTR(__kmp_threads[gtid], root_thread);
3767 root->r.r_root_team->t.t_threads[0] = root_thread;
3768 root->r.r_hot_team->t.t_threads[0] = root_thread;
3769 root_thread->th.th_serial_team->t.t_threads[0] = root_thread;
3771 root_thread->th.th_serial_team->t.t_serialized = 0;
3772 root->r.r_uber_thread = root_thread;
3775 __kmp_initialize_info(root_thread, root->r.r_root_team, 0, gtid);
3776 TCW_4(__kmp_init_gtid, TRUE);
3779 __kmp_gtid_set_specific(gtid);
3782 __kmp_itt_thread_name(gtid);
3785 #ifdef KMP_TDATA_GTID
3788 __kmp_create_worker(gtid, root_thread, __kmp_stksize);
3789 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == gtid);
3791 KA_TRACE(20, (
"__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
3793 gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team),
3794 root->r.r_hot_team->t.t_id, 0, KMP_INIT_BARRIER_STATE,
3795 KMP_INIT_BARRIER_STATE));
3798 for (b = 0; b < bs_last_barrier; ++b) {
3799 root_thread->th.th_bar[b].bb.b_arrived = KMP_INIT_BARRIER_STATE;
3801 root_thread->th.th_bar[b].bb.b_worker_arrived = 0;
3805 KMP_DEBUG_ASSERT(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived ==
3806 KMP_INIT_BARRIER_STATE);
3808 #if KMP_AFFINITY_SUPPORTED
3809 root_thread->th.th_current_place = KMP_PLACE_UNDEFINED;
3810 root_thread->th.th_new_place = KMP_PLACE_UNDEFINED;
3811 root_thread->th.th_first_place = KMP_PLACE_UNDEFINED;
3812 root_thread->th.th_last_place = KMP_PLACE_UNDEFINED;
3813 if (TCR_4(__kmp_init_middle)) {
3814 __kmp_affinity_set_init_mask(gtid, TRUE);
3817 root_thread->th.th_def_allocator = __kmp_def_allocator;
3818 root_thread->th.th_prev_level = 0;
3819 root_thread->th.th_prev_num_threads = 1;
3821 kmp_cg_root_t *tmp = (kmp_cg_root_t *)__kmp_allocate(
sizeof(kmp_cg_root_t));
3822 tmp->cg_root = root_thread;
3823 tmp->cg_thread_limit = __kmp_cg_max_nth;
3824 tmp->cg_nthreads = 1;
3825 KA_TRACE(100, (
"__kmp_register_root: Thread %p created node %p with"
3826 " cg_nthreads init to 1\n",
3829 root_thread->th.th_cg_roots = tmp;
3831 __kmp_root_counter++;
3834 if (!initial_thread && ompt_enabled.enabled) {
3836 kmp_info_t *root_thread = ompt_get_thread();
3838 ompt_set_thread_state(root_thread, ompt_state_overhead);
3840 if (ompt_enabled.ompt_callback_thread_begin) {
3841 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)(
3842 ompt_thread_initial, __ompt_get_thread_data_internal());
3844 ompt_data_t *task_data;
3845 ompt_data_t *parallel_data;
3846 __ompt_get_task_info_internal(0, NULL, &task_data, NULL, ¶llel_data, NULL);
3847 if (ompt_enabled.ompt_callback_implicit_task) {
3848 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
3849 ompt_scope_begin, parallel_data, task_data, 1, 1, ompt_task_initial);
3852 ompt_set_thread_state(root_thread, ompt_state_work_serial);
3857 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
3862 #if KMP_NESTED_HOT_TEAMS
3863 static int __kmp_free_hot_teams(kmp_root_t *root, kmp_info_t *thr,
int level,
3864 const int max_level) {
3866 kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams;
3867 if (!hot_teams || !hot_teams[level].hot_team) {
3870 KMP_DEBUG_ASSERT(level < max_level);
3871 kmp_team_t *team = hot_teams[level].hot_team;
3872 nth = hot_teams[level].hot_team_nth;
3874 if (level < max_level - 1) {
3875 for (i = 0; i < nth; ++i) {
3876 kmp_info_t *th = team->t.t_threads[i];
3877 n += __kmp_free_hot_teams(root, th, level + 1, max_level);
3878 if (i > 0 && th->th.th_hot_teams) {
3879 __kmp_free(th->th.th_hot_teams);
3880 th->th.th_hot_teams = NULL;
3884 __kmp_free_team(root, team, NULL);
3891 static int __kmp_reset_root(
int gtid, kmp_root_t *root) {
3892 kmp_team_t *root_team = root->r.r_root_team;
3893 kmp_team_t *hot_team = root->r.r_hot_team;
3894 int n = hot_team->t.t_nproc;
3897 KMP_DEBUG_ASSERT(!root->r.r_active);
3899 root->r.r_root_team = NULL;
3900 root->r.r_hot_team = NULL;
3903 __kmp_free_team(root, root_team USE_NESTED_HOT_ARG(NULL));
3904 #if KMP_NESTED_HOT_TEAMS
3905 if (__kmp_hot_teams_max_level >
3907 for (i = 0; i < hot_team->t.t_nproc; ++i) {
3908 kmp_info_t *th = hot_team->t.t_threads[i];
3909 if (__kmp_hot_teams_max_level > 1) {
3910 n += __kmp_free_hot_teams(root, th, 1, __kmp_hot_teams_max_level);
3912 if (th->th.th_hot_teams) {
3913 __kmp_free(th->th.th_hot_teams);
3914 th->th.th_hot_teams = NULL;
3919 __kmp_free_team(root, hot_team USE_NESTED_HOT_ARG(NULL));
3924 if (__kmp_tasking_mode != tskm_immediate_exec) {
3925 __kmp_wait_to_unref_task_teams();
3931 10, (
"__kmp_reset_root: free handle, th = %p, handle = %" KMP_UINTPTR_SPEC
3933 (LPVOID) & (root->r.r_uber_thread->th),
3934 root->r.r_uber_thread->th.th_info.ds.ds_thread));
3935 __kmp_free_handle(root->r.r_uber_thread->th.th_info.ds.ds_thread);
3939 ompt_data_t *task_data;
3940 ompt_data_t *parallel_data;
3941 __ompt_get_task_info_internal(0, NULL, &task_data, NULL, ¶llel_data, NULL);
3942 if (ompt_enabled.ompt_callback_implicit_task) {
3943 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
3944 ompt_scope_end, parallel_data, task_data, 0, 1, ompt_task_initial);
3946 if (ompt_enabled.ompt_callback_thread_end) {
3947 ompt_callbacks.ompt_callback(ompt_callback_thread_end)(
3948 &(root->r.r_uber_thread->th.ompt_thread_info.thread_data));
3954 i = root->r.r_uber_thread->th.th_cg_roots->cg_nthreads--;
3955 KA_TRACE(100, (
"__kmp_reset_root: Thread %p decrement cg_nthreads on node %p"
3957 root->r.r_uber_thread, root->r.r_uber_thread->th.th_cg_roots,
3958 root->r.r_uber_thread->th.th_cg_roots->cg_nthreads));
3961 KMP_DEBUG_ASSERT(root->r.r_uber_thread ==
3962 root->r.r_uber_thread->th.th_cg_roots->cg_root);
3963 KMP_DEBUG_ASSERT(root->r.r_uber_thread->th.th_cg_roots->up == NULL);
3964 __kmp_free(root->r.r_uber_thread->th.th_cg_roots);
3965 root->r.r_uber_thread->th.th_cg_roots = NULL;
3967 __kmp_reap_thread(root->r.r_uber_thread, 1);
3971 root->r.r_uber_thread = NULL;
3973 root->r.r_begin = FALSE;
3978 void __kmp_unregister_root_current_thread(
int gtid) {
3979 KA_TRACE(1, (
"__kmp_unregister_root_current_thread: enter T#%d\n", gtid));
3983 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
3984 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
3985 KC_TRACE(10, (
"__kmp_unregister_root_current_thread: already finished, "
3988 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
3991 kmp_root_t *root = __kmp_root[gtid];
3993 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]);
3994 KMP_ASSERT(KMP_UBER_GTID(gtid));
3995 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root);
3996 KMP_ASSERT(root->r.r_active == FALSE);
4000 kmp_info_t *thread = __kmp_threads[gtid];
4001 kmp_team_t *team = thread->th.th_team;
4002 kmp_task_team_t *task_team = thread->th.th_task_team;
4005 if (task_team != NULL && task_team->tt.tt_found_proxy_tasks) {
4008 thread->th.ompt_thread_info.state = ompt_state_undefined;
4010 __kmp_task_team_wait(thread, team USE_ITT_BUILD_ARG(NULL));
4013 __kmp_reset_root(gtid, root);
4017 (
"__kmp_unregister_root_current_thread: T#%d unregistered\n", gtid));
4019 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
4026 static int __kmp_unregister_root_other_thread(
int gtid) {
4027 kmp_root_t *root = __kmp_root[gtid];
4030 KA_TRACE(1, (
"__kmp_unregister_root_other_thread: enter T#%d\n", gtid));
4031 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]);
4032 KMP_ASSERT(KMP_UBER_GTID(gtid));
4033 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root);
4034 KMP_ASSERT(root->r.r_active == FALSE);
4036 r = __kmp_reset_root(gtid, root);
4038 (
"__kmp_unregister_root_other_thread: T#%d unregistered\n", gtid));
4044 void __kmp_task_info() {
4046 kmp_int32 gtid = __kmp_entry_gtid();
4047 kmp_int32 tid = __kmp_tid_from_gtid(gtid);
4048 kmp_info_t *this_thr = __kmp_threads[gtid];
4049 kmp_team_t *steam = this_thr->th.th_serial_team;
4050 kmp_team_t *team = this_thr->th.th_team;
4053 "__kmp_task_info: gtid=%d tid=%d t_thread=%p team=%p steam=%p curtask=%p "
4055 gtid, tid, this_thr, team, steam, this_thr->th.th_current_task,
4056 team->t.t_implicit_task_taskdata[tid].td_parent);
4063 static void __kmp_initialize_info(kmp_info_t *this_thr, kmp_team_t *team,
4064 int tid,
int gtid) {
4068 kmp_info_t *master = team->t.t_threads[0];
4069 KMP_DEBUG_ASSERT(this_thr != NULL);
4070 KMP_DEBUG_ASSERT(this_thr->th.th_serial_team);
4071 KMP_DEBUG_ASSERT(team);
4072 KMP_DEBUG_ASSERT(team->t.t_threads);
4073 KMP_DEBUG_ASSERT(team->t.t_dispatch);
4074 KMP_DEBUG_ASSERT(master);
4075 KMP_DEBUG_ASSERT(master->th.th_root);
4079 TCW_SYNC_PTR(this_thr->th.th_team, team);
4081 this_thr->th.th_info.ds.ds_tid = tid;
4082 this_thr->th.th_set_nproc = 0;
4083 if (__kmp_tasking_mode != tskm_immediate_exec)
4086 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP;
4088 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;
4089 this_thr->th.th_set_proc_bind = proc_bind_default;
4090 #if KMP_AFFINITY_SUPPORTED
4091 this_thr->th.th_new_place = this_thr->th.th_current_place;
4093 this_thr->th.th_root = master->th.th_root;
4096 this_thr->th.th_team_nproc = team->t.t_nproc;
4097 this_thr->th.th_team_master = master;
4098 this_thr->th.th_team_serialized = team->t.t_serialized;
4099 TCW_PTR(this_thr->th.th_sleep_loc, NULL);
4101 KMP_DEBUG_ASSERT(team->t.t_implicit_task_taskdata);
4103 KF_TRACE(10, (
"__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n",
4104 tid, gtid, this_thr, this_thr->th.th_current_task));
4106 __kmp_init_implicit_task(this_thr->th.th_team_master->th.th_ident, this_thr,
4109 KF_TRACE(10, (
"__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n",
4110 tid, gtid, this_thr, this_thr->th.th_current_task));
4115 this_thr->th.th_dispatch = &team->t.t_dispatch[tid];
4117 this_thr->th.th_local.this_construct = 0;
4119 if (!this_thr->th.th_pri_common) {
4120 this_thr->th.th_pri_common =
4121 (
struct common_table *)__kmp_allocate(
sizeof(
struct common_table));
4122 if (__kmp_storage_map) {
4123 __kmp_print_storage_map_gtid(
4124 gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1,
4125 sizeof(
struct common_table),
"th_%d.th_pri_common\n", gtid);
4127 this_thr->th.th_pri_head = NULL;
4130 if (this_thr != master &&
4131 this_thr->th.th_cg_roots != master->th.th_cg_roots) {
4133 KMP_DEBUG_ASSERT(master->th.th_cg_roots);
4134 kmp_cg_root_t *tmp = this_thr->th.th_cg_roots;
4137 int i = tmp->cg_nthreads--;
4138 KA_TRACE(100, (
"__kmp_initialize_info: Thread %p decrement cg_nthreads"
4139 " on node %p of thread %p to %d\n",
4140 this_thr, tmp, tmp->cg_root, tmp->cg_nthreads));
4145 this_thr->th.th_cg_roots = master->th.th_cg_roots;
4147 this_thr->th.th_cg_roots->cg_nthreads++;
4148 KA_TRACE(100, (
"__kmp_initialize_info: Thread %p increment cg_nthreads on"
4149 " node %p of thread %p to %d\n",
4150 this_thr, this_thr->th.th_cg_roots,
4151 this_thr->th.th_cg_roots->cg_root,
4152 this_thr->th.th_cg_roots->cg_nthreads));
4153 this_thr->th.th_current_task->td_icvs.thread_limit =
4154 this_thr->th.th_cg_roots->cg_thread_limit;
4159 volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch;
4162 sizeof(dispatch_private_info_t) *
4163 (team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers);
4164 KD_TRACE(10, (
"__kmp_initialize_info: T#%d max_nproc: %d\n", gtid,
4165 team->t.t_max_nproc));
4166 KMP_ASSERT(dispatch);
4167 KMP_DEBUG_ASSERT(team->t.t_dispatch);
4168 KMP_DEBUG_ASSERT(dispatch == &team->t.t_dispatch[tid]);
4170 dispatch->th_disp_index = 0;
4171 dispatch->th_doacross_buf_idx = 0;
4172 if (!dispatch->th_disp_buffer) {
4173 dispatch->th_disp_buffer =
4174 (dispatch_private_info_t *)__kmp_allocate(disp_size);
4176 if (__kmp_storage_map) {
4177 __kmp_print_storage_map_gtid(
4178 gtid, &dispatch->th_disp_buffer[0],
4179 &dispatch->th_disp_buffer[team->t.t_max_nproc == 1
4181 : __kmp_dispatch_num_buffers],
4182 disp_size,
"th_%d.th_dispatch.th_disp_buffer "
4183 "(team_%d.t_dispatch[%d].th_disp_buffer)",
4184 gtid, team->t.t_id, gtid);
4187 memset(&dispatch->th_disp_buffer[0],
'\0', disp_size);
4190 dispatch->th_dispatch_pr_current = 0;
4191 dispatch->th_dispatch_sh_current = 0;
4193 dispatch->th_deo_fcn = 0;
4194 dispatch->th_dxo_fcn = 0;
4197 this_thr->th.th_next_pool = NULL;
4199 if (!this_thr->th.th_task_state_memo_stack) {
4201 this_thr->th.th_task_state_memo_stack =
4202 (kmp_uint8 *)__kmp_allocate(4 *
sizeof(kmp_uint8));
4203 this_thr->th.th_task_state_top = 0;
4204 this_thr->th.th_task_state_stack_sz = 4;
4205 for (i = 0; i < this_thr->th.th_task_state_stack_sz;
4207 this_thr->th.th_task_state_memo_stack[i] = 0;
4210 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
4211 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
4221 kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team,
4223 kmp_team_t *serial_team;
4224 kmp_info_t *new_thr;
4227 KA_TRACE(20, (
"__kmp_allocate_thread: T#%d\n", __kmp_get_gtid()));
4228 KMP_DEBUG_ASSERT(root && team);
4229 #if !KMP_NESTED_HOT_TEAMS
4230 KMP_DEBUG_ASSERT(KMP_MASTER_GTID(__kmp_get_gtid()));
4235 if (__kmp_thread_pool) {
4236 new_thr = CCAST(kmp_info_t *, __kmp_thread_pool);
4237 __kmp_thread_pool = (
volatile kmp_info_t *)new_thr->th.th_next_pool;
4238 if (new_thr == __kmp_thread_pool_insert_pt) {
4239 __kmp_thread_pool_insert_pt = NULL;
4241 TCW_4(new_thr->th.th_in_pool, FALSE);
4242 __kmp_suspend_initialize_thread(new_thr);
4243 __kmp_lock_suspend_mx(new_thr);
4244 if (new_thr->th.th_active_in_pool == TRUE) {
4245 KMP_DEBUG_ASSERT(new_thr->th.th_active == TRUE);
4246 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
4247 new_thr->th.th_active_in_pool = FALSE;
4249 __kmp_unlock_suspend_mx(new_thr);
4251 KA_TRACE(20, (
"__kmp_allocate_thread: T#%d using thread T#%d\n",
4252 __kmp_get_gtid(), new_thr->th.th_info.ds.ds_gtid));
4253 KMP_ASSERT(!new_thr->th.th_team);
4254 KMP_DEBUG_ASSERT(__kmp_nth < __kmp_threads_capacity);
4257 __kmp_initialize_info(new_thr, team, new_tid,
4258 new_thr->th.th_info.ds.ds_gtid);
4259 KMP_DEBUG_ASSERT(new_thr->th.th_serial_team);
4261 TCW_4(__kmp_nth, __kmp_nth + 1);
4263 new_thr->th.th_task_state = 0;
4264 new_thr->th.th_task_state_top = 0;
4265 new_thr->th.th_task_state_stack_sz = 4;
4267 #ifdef KMP_ADJUST_BLOCKTIME
4270 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
4271 if (__kmp_nth > __kmp_avail_proc) {
4272 __kmp_zero_bt = TRUE;
4281 kmp_balign_t *balign = new_thr->th.th_bar;
4282 for (b = 0; b < bs_last_barrier; ++b)
4283 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
4286 KF_TRACE(10, (
"__kmp_allocate_thread: T#%d using thread %p T#%d\n",
4287 __kmp_get_gtid(), new_thr, new_thr->th.th_info.ds.ds_gtid));
4294 KMP_ASSERT(__kmp_nth == __kmp_all_nth);
4295 KMP_ASSERT(__kmp_all_nth < __kmp_threads_capacity);
4300 if (!TCR_4(__kmp_init_monitor)) {
4301 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
4302 if (!TCR_4(__kmp_init_monitor)) {
4303 KF_TRACE(10, (
"before __kmp_create_monitor\n"));
4304 TCW_4(__kmp_init_monitor, 1);
4305 __kmp_create_monitor(&__kmp_monitor);
4306 KF_TRACE(10, (
"after __kmp_create_monitor\n"));
4317 while (TCR_4(__kmp_init_monitor) < 2) {
4320 KF_TRACE(10, (
"after monitor thread has started\n"));
4323 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
4330 int new_start_gtid = TCR_4(__kmp_init_hidden_helper_threads)
4332 : __kmp_hidden_helper_threads_num + 1;
4334 for (new_gtid = new_start_gtid; TCR_PTR(__kmp_threads[new_gtid]) != NULL;
4336 KMP_DEBUG_ASSERT(new_gtid < __kmp_threads_capacity);
4339 if (TCR_4(__kmp_init_hidden_helper_threads)) {
4340 KMP_DEBUG_ASSERT(new_gtid <= __kmp_hidden_helper_threads_num);
4345 new_thr = (kmp_info_t *)__kmp_allocate(
sizeof(kmp_info_t));
4347 TCW_SYNC_PTR(__kmp_threads[new_gtid], new_thr);
4349 #if USE_ITT_BUILD && USE_ITT_NOTIFY && KMP_DEBUG
4352 __itt_suppress_mark_range(
4353 __itt_suppress_range, __itt_suppress_threading_errors,
4354 &new_thr->th.th_sleep_loc,
sizeof(new_thr->th.th_sleep_loc));
4355 __itt_suppress_mark_range(
4356 __itt_suppress_range, __itt_suppress_threading_errors,
4357 &new_thr->th.th_reap_state,
sizeof(new_thr->th.th_reap_state));
4359 __itt_suppress_mark_range(
4360 __itt_suppress_range, __itt_suppress_threading_errors,
4361 &new_thr->th.th_suspend_init,
sizeof(new_thr->th.th_suspend_init));
4363 __itt_suppress_mark_range(__itt_suppress_range,
4364 __itt_suppress_threading_errors,
4365 &new_thr->th.th_suspend_init_count,
4366 sizeof(new_thr->th.th_suspend_init_count));
4369 __itt_suppress_mark_range(__itt_suppress_range,
4370 __itt_suppress_threading_errors,
4371 CCAST(kmp_uint64 *, &new_thr->th.th_bar[0].bb.b_go),
4372 sizeof(new_thr->th.th_bar[0].bb.b_go));
4373 __itt_suppress_mark_range(__itt_suppress_range,
4374 __itt_suppress_threading_errors,
4375 CCAST(kmp_uint64 *, &new_thr->th.th_bar[1].bb.b_go),
4376 sizeof(new_thr->th.th_bar[1].bb.b_go));
4377 __itt_suppress_mark_range(__itt_suppress_range,
4378 __itt_suppress_threading_errors,
4379 CCAST(kmp_uint64 *, &new_thr->th.th_bar[2].bb.b_go),
4380 sizeof(new_thr->th.th_bar[2].bb.b_go));
4382 if (__kmp_storage_map) {
4383 __kmp_print_thread_storage_map(new_thr, new_gtid);
4388 kmp_internal_control_t r_icvs = __kmp_get_x_global_icvs(team);
4389 KF_TRACE(10, (
"__kmp_allocate_thread: before th_serial/serial_team\n"));
4390 new_thr->th.th_serial_team = serial_team =
4391 (kmp_team_t *)__kmp_allocate_team(root, 1, 1,
4395 proc_bind_default, &r_icvs,
4396 0 USE_NESTED_HOT_ARG(NULL));
4398 KMP_ASSERT(serial_team);
4399 serial_team->t.t_serialized = 0;
4401 serial_team->t.t_threads[0] = new_thr;
4403 (
"__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n",
4407 __kmp_initialize_info(new_thr, team, new_tid, new_gtid);
4410 __kmp_initialize_fast_memory(new_thr);
4414 KMP_DEBUG_ASSERT(new_thr->th.th_local.bget_data == NULL);
4415 __kmp_initialize_bget(new_thr);
4418 __kmp_init_random(new_thr);
4422 (
"__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n",
4423 __kmp_get_gtid(), KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
4426 kmp_balign_t *balign = new_thr->th.th_bar;
4427 for (b = 0; b < bs_last_barrier; ++b) {
4428 balign[b].bb.b_go = KMP_INIT_BARRIER_STATE;
4429 balign[b].bb.team = NULL;
4430 balign[b].bb.wait_flag = KMP_BARRIER_NOT_WAITING;
4431 balign[b].bb.use_oncore_barrier = 0;
4434 new_thr->th.th_spin_here = FALSE;
4435 new_thr->th.th_next_waiting = 0;
4437 new_thr->th.th_blocking =
false;
4440 #if KMP_AFFINITY_SUPPORTED
4441 new_thr->th.th_current_place = KMP_PLACE_UNDEFINED;
4442 new_thr->th.th_new_place = KMP_PLACE_UNDEFINED;
4443 new_thr->th.th_first_place = KMP_PLACE_UNDEFINED;
4444 new_thr->th.th_last_place = KMP_PLACE_UNDEFINED;
4446 new_thr->th.th_def_allocator = __kmp_def_allocator;
4447 new_thr->th.th_prev_level = 0;
4448 new_thr->th.th_prev_num_threads = 1;
4450 TCW_4(new_thr->th.th_in_pool, FALSE);
4451 new_thr->th.th_active_in_pool = FALSE;
4452 TCW_4(new_thr->th.th_active, TRUE);
4460 if (__kmp_adjust_gtid_mode) {
4461 if (__kmp_all_nth >= __kmp_tls_gtid_min) {
4462 if (TCR_4(__kmp_gtid_mode) != 2) {
4463 TCW_4(__kmp_gtid_mode, 2);
4466 if (TCR_4(__kmp_gtid_mode) != 1) {
4467 TCW_4(__kmp_gtid_mode, 1);
4472 #ifdef KMP_ADJUST_BLOCKTIME
4475 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
4476 if (__kmp_nth > __kmp_avail_proc) {
4477 __kmp_zero_bt = TRUE;
4484 10, (
"__kmp_allocate_thread: before __kmp_create_worker: %p\n", new_thr));
4485 __kmp_create_worker(new_gtid, new_thr, __kmp_stksize);
4487 (
"__kmp_allocate_thread: after __kmp_create_worker: %p\n", new_thr));
4489 KA_TRACE(20, (
"__kmp_allocate_thread: T#%d forked T#%d\n", __kmp_get_gtid(),
4500 static void __kmp_reinitialize_team(kmp_team_t *team,
4501 kmp_internal_control_t *new_icvs,
4503 KF_TRACE(10, (
"__kmp_reinitialize_team: enter this_thread=%p team=%p\n",
4504 team->t.t_threads[0], team));
4505 KMP_DEBUG_ASSERT(team && new_icvs);
4506 KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc);
4507 KMP_CHECK_UPDATE(team->t.t_ident, loc);
4509 KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID());
4511 __kmp_init_implicit_task(loc, team->t.t_threads[0], team, 0, FALSE);
4512 copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs);
4514 KF_TRACE(10, (
"__kmp_reinitialize_team: exit this_thread=%p team=%p\n",
4515 team->t.t_threads[0], team));
4521 static void __kmp_initialize_team(kmp_team_t *team,
int new_nproc,
4522 kmp_internal_control_t *new_icvs,
4524 KF_TRACE(10, (
"__kmp_initialize_team: enter: team=%p\n", team));
4527 KMP_DEBUG_ASSERT(team);
4528 KMP_DEBUG_ASSERT(new_nproc <= team->t.t_max_nproc);
4529 KMP_DEBUG_ASSERT(team->t.t_threads);
4532 team->t.t_master_tid = 0;
4534 team->t.t_serialized = new_nproc > 1 ? 0 : 1;
4535 team->t.t_nproc = new_nproc;
4538 team->t.t_next_pool = NULL;
4542 TCW_SYNC_PTR(team->t.t_pkfn, NULL);
4543 team->t.t_invoke = NULL;
4546 team->t.t_sched.sched = new_icvs->sched.sched;
4548 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
4549 team->t.t_fp_control_saved = FALSE;
4550 team->t.t_x87_fpu_control_word = 0;
4551 team->t.t_mxcsr = 0;
4554 team->t.t_construct = 0;
4556 team->t.t_ordered.dt.t_value = 0;
4557 team->t.t_master_active = FALSE;
4560 team->t.t_copypriv_data = NULL;
4563 team->t.t_copyin_counter = 0;
4566 team->t.t_control_stack_top = NULL;
4568 __kmp_reinitialize_team(team, new_icvs, loc);
4571 KF_TRACE(10, (
"__kmp_initialize_team: exit: team=%p\n", team));
4574 #if (KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED
4577 __kmp_set_thread_affinity_mask_full_tmp(kmp_affin_mask_t *old_mask) {
4578 if (KMP_AFFINITY_CAPABLE()) {
4580 if (old_mask != NULL) {
4581 status = __kmp_get_system_affinity(old_mask, TRUE);
4584 __kmp_fatal(KMP_MSG(ChangeThreadAffMaskError), KMP_ERR(error),
4588 __kmp_set_system_affinity(__kmp_affin_fullMask, TRUE);
4593 #if KMP_AFFINITY_SUPPORTED
4599 static void __kmp_partition_places(kmp_team_t *team,
int update_master_only) {
4601 kmp_info_t *master_th = team->t.t_threads[0];
4602 KMP_DEBUG_ASSERT(master_th != NULL);
4603 kmp_proc_bind_t proc_bind = team->t.t_proc_bind;
4604 int first_place = master_th->th.th_first_place;
4605 int last_place = master_th->th.th_last_place;
4606 int masters_place = master_th->th.th_current_place;
4607 team->t.t_first_place = first_place;
4608 team->t.t_last_place = last_place;
4610 KA_TRACE(20, (
"__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "
4611 "bound to place %d partition = [%d,%d]\n",
4612 proc_bind, __kmp_gtid_from_thread(team->t.t_threads[0]),
4613 team->t.t_id, masters_place, first_place, last_place));
4615 switch (proc_bind) {
4617 case proc_bind_default:
4620 KMP_DEBUG_ASSERT(team->t.t_nproc == 1);
4623 case proc_bind_master: {
4625 int n_th = team->t.t_nproc;
4626 for (f = 1; f < n_th; f++) {
4627 kmp_info_t *th = team->t.t_threads[f];
4628 KMP_DEBUG_ASSERT(th != NULL);
4629 th->th.th_first_place = first_place;
4630 th->th.th_last_place = last_place;
4631 th->th.th_new_place = masters_place;
4632 if (__kmp_display_affinity && masters_place != th->th.th_current_place &&
4633 team->t.t_display_affinity != 1) {
4634 team->t.t_display_affinity = 1;
4637 KA_TRACE(100, (
"__kmp_partition_places: master: T#%d(%d:%d) place %d "
4638 "partition = [%d,%d]\n",
4639 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,
4640 f, masters_place, first_place, last_place));
4644 case proc_bind_close: {
4646 int n_th = team->t.t_nproc;
4648 if (first_place <= last_place) {
4649 n_places = last_place - first_place + 1;
4651 n_places = __kmp_affinity_num_masks - first_place + last_place + 1;
4653 if (n_th <= n_places) {
4654 int place = masters_place;
4655 for (f = 1; f < n_th; f++) {
4656 kmp_info_t *th = team->t.t_threads[f];
4657 KMP_DEBUG_ASSERT(th != NULL);
4659 if (place == last_place) {
4660 place = first_place;
4661 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4666 th->th.th_first_place = first_place;
4667 th->th.th_last_place = last_place;
4668 th->th.th_new_place = place;
4669 if (__kmp_display_affinity && place != th->th.th_current_place &&
4670 team->t.t_display_affinity != 1) {
4671 team->t.t_display_affinity = 1;
4674 KA_TRACE(100, (
"__kmp_partition_places: close: T#%d(%d:%d) place %d "
4675 "partition = [%d,%d]\n",
4676 __kmp_gtid_from_thread(team->t.t_threads[f]),
4677 team->t.t_id, f, place, first_place, last_place));
4680 int S, rem, gap, s_count;
4681 S = n_th / n_places;
4683 rem = n_th - (S * n_places);
4684 gap = rem > 0 ? n_places / rem : n_places;
4685 int place = masters_place;
4687 for (f = 0; f < n_th; f++) {
4688 kmp_info_t *th = team->t.t_threads[f];
4689 KMP_DEBUG_ASSERT(th != NULL);
4691 th->th.th_first_place = first_place;
4692 th->th.th_last_place = last_place;
4693 th->th.th_new_place = place;
4694 if (__kmp_display_affinity && place != th->th.th_current_place &&
4695 team->t.t_display_affinity != 1) {
4696 team->t.t_display_affinity = 1;
4700 if ((s_count == S) && rem && (gap_ct == gap)) {
4702 }
else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
4704 if (place == last_place) {
4705 place = first_place;
4706 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4714 }
else if (s_count == S) {
4715 if (place == last_place) {
4716 place = first_place;
4717 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4727 (
"__kmp_partition_places: close: T#%d(%d:%d) place %d "
4728 "partition = [%d,%d]\n",
4729 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, f,
4730 th->th.th_new_place, first_place, last_place));
4732 KMP_DEBUG_ASSERT(place == masters_place);
4736 case proc_bind_spread: {
4738 int n_th = team->t.t_nproc;
4741 if (first_place <= last_place) {
4742 n_places = last_place - first_place + 1;
4744 n_places = __kmp_affinity_num_masks - first_place + last_place + 1;
4746 if (n_th <= n_places) {
4749 if (n_places !=
static_cast<int>(__kmp_affinity_num_masks)) {
4750 int S = n_places / n_th;
4751 int s_count, rem, gap, gap_ct;
4753 place = masters_place;
4754 rem = n_places - n_th * S;
4755 gap = rem ? n_th / rem : 1;
4758 if (update_master_only == 1)
4760 for (f = 0; f < thidx; f++) {
4761 kmp_info_t *th = team->t.t_threads[f];
4762 KMP_DEBUG_ASSERT(th != NULL);
4764 th->th.th_first_place = place;
4765 th->th.th_new_place = place;
4766 if (__kmp_display_affinity && place != th->th.th_current_place &&
4767 team->t.t_display_affinity != 1) {
4768 team->t.t_display_affinity = 1;
4771 while (s_count < S) {
4772 if (place == last_place) {
4773 place = first_place;
4774 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4781 if (rem && (gap_ct == gap)) {
4782 if (place == last_place) {
4783 place = first_place;
4784 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4792 th->th.th_last_place = place;
4795 if (place == last_place) {
4796 place = first_place;
4797 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4804 (
"__kmp_partition_places: spread: T#%d(%d:%d) place %d "
4805 "partition = [%d,%d], __kmp_affinity_num_masks: %u\n",
4806 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,
4807 f, th->th.th_new_place, th->th.th_first_place,
4808 th->th.th_last_place, __kmp_affinity_num_masks));
4814 double current =
static_cast<double>(masters_place);
4816 (
static_cast<double>(n_places + 1) /
static_cast<double>(n_th));
4821 if (update_master_only == 1)
4823 for (f = 0; f < thidx; f++) {
4824 first =
static_cast<int>(current);
4825 last =
static_cast<int>(current + spacing) - 1;
4826 KMP_DEBUG_ASSERT(last >= first);
4827 if (first >= n_places) {
4828 if (masters_place) {
4831 if (first == (masters_place + 1)) {
4832 KMP_DEBUG_ASSERT(f == n_th);
4835 if (last == masters_place) {
4836 KMP_DEBUG_ASSERT(f == (n_th - 1));
4840 KMP_DEBUG_ASSERT(f == n_th);
4845 if (last >= n_places) {
4846 last = (n_places - 1);
4851 KMP_DEBUG_ASSERT(0 <= first);
4852 KMP_DEBUG_ASSERT(n_places > first);
4853 KMP_DEBUG_ASSERT(0 <= last);
4854 KMP_DEBUG_ASSERT(n_places > last);
4855 KMP_DEBUG_ASSERT(last_place >= first_place);
4856 th = team->t.t_threads[f];
4857 KMP_DEBUG_ASSERT(th);
4858 th->th.th_first_place = first;
4859 th->th.th_new_place = place;
4860 th->th.th_last_place = last;
4861 if (__kmp_display_affinity && place != th->th.th_current_place &&
4862 team->t.t_display_affinity != 1) {
4863 team->t.t_display_affinity = 1;
4866 (
"__kmp_partition_places: spread: T#%d(%d:%d) place %d "
4867 "partition = [%d,%d], spacing = %.4f\n",
4868 __kmp_gtid_from_thread(team->t.t_threads[f]),
4869 team->t.t_id, f, th->th.th_new_place,
4870 th->th.th_first_place, th->th.th_last_place, spacing));
4874 KMP_DEBUG_ASSERT(update_master_only || place == masters_place);
4876 int S, rem, gap, s_count;
4877 S = n_th / n_places;
4879 rem = n_th - (S * n_places);
4880 gap = rem > 0 ? n_places / rem : n_places;
4881 int place = masters_place;
4884 if (update_master_only == 1)
4886 for (f = 0; f < thidx; f++) {
4887 kmp_info_t *th = team->t.t_threads[f];
4888 KMP_DEBUG_ASSERT(th != NULL);
4890 th->th.th_first_place = place;
4891 th->th.th_last_place = place;
4892 th->th.th_new_place = place;
4893 if (__kmp_display_affinity && place != th->th.th_current_place &&
4894 team->t.t_display_affinity != 1) {
4895 team->t.t_display_affinity = 1;
4899 if ((s_count == S) && rem && (gap_ct == gap)) {
4901 }
else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
4903 if (place == last_place) {
4904 place = first_place;
4905 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4913 }
else if (s_count == S) {
4914 if (place == last_place) {
4915 place = first_place;
4916 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4925 KA_TRACE(100, (
"__kmp_partition_places: spread: T#%d(%d:%d) place %d "
4926 "partition = [%d,%d]\n",
4927 __kmp_gtid_from_thread(team->t.t_threads[f]),
4928 team->t.t_id, f, th->th.th_new_place,
4929 th->th.th_first_place, th->th.th_last_place));
4931 KMP_DEBUG_ASSERT(update_master_only || place == masters_place);
4939 KA_TRACE(20, (
"__kmp_partition_places: exit T#%d\n", team->t.t_id));
4942 #endif // KMP_AFFINITY_SUPPORTED
4947 __kmp_allocate_team(kmp_root_t *root,
int new_nproc,
int max_nproc,
4949 ompt_data_t ompt_parallel_data,
4951 kmp_proc_bind_t new_proc_bind,
4952 kmp_internal_control_t *new_icvs,
4953 int argc USE_NESTED_HOT_ARG(kmp_info_t *master)) {
4954 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_allocate_team);
4957 int use_hot_team = !root->r.r_active;
4960 KA_TRACE(20, (
"__kmp_allocate_team: called\n"));
4961 KMP_DEBUG_ASSERT(new_nproc >= 1 && argc >= 0);
4962 KMP_DEBUG_ASSERT(max_nproc >= new_nproc);
4965 #if KMP_NESTED_HOT_TEAMS
4966 kmp_hot_team_ptr_t *hot_teams;
4968 team = master->th.th_team;
4969 level = team->t.t_active_level;
4970 if (master->th.th_teams_microtask) {
4971 if (master->th.th_teams_size.nteams > 1 &&
4974 (microtask_t)__kmp_teams_master ||
4975 master->th.th_teams_level <
4981 hot_teams = master->th.th_hot_teams;
4982 if (level < __kmp_hot_teams_max_level && hot_teams &&
4983 hot_teams[level].hot_team) {
4991 KMP_DEBUG_ASSERT(new_nproc == 1);
4995 if (use_hot_team && new_nproc > 1) {
4996 KMP_DEBUG_ASSERT(new_nproc <= max_nproc);
4997 #if KMP_NESTED_HOT_TEAMS
4998 team = hot_teams[level].hot_team;
5000 team = root->r.r_hot_team;
5003 if (__kmp_tasking_mode != tskm_immediate_exec) {
5004 KA_TRACE(20, (
"__kmp_allocate_team: hot team task_team[0] = %p "
5005 "task_team[1] = %p before reinit\n",
5006 team->t.t_task_team[0], team->t.t_task_team[1]));
5013 if (team->t.t_nproc == new_nproc) {
5014 KA_TRACE(20, (
"__kmp_allocate_team: reusing hot team\n"));
5017 if (team->t.t_size_changed == -1) {
5018 team->t.t_size_changed = 1;
5020 KMP_CHECK_UPDATE(team->t.t_size_changed, 0);
5024 kmp_r_sched_t new_sched = new_icvs->sched;
5026 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched);
5028 __kmp_reinitialize_team(team, new_icvs,
5029 root->r.r_uber_thread->th.th_ident);
5031 KF_TRACE(10, (
"__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n", 0,
5032 team->t.t_threads[0], team));
5033 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);
5035 #if KMP_AFFINITY_SUPPORTED
5036 if ((team->t.t_size_changed == 0) &&
5037 (team->t.t_proc_bind == new_proc_bind)) {
5038 if (new_proc_bind == proc_bind_spread) {
5039 __kmp_partition_places(
5042 KA_TRACE(200, (
"__kmp_allocate_team: reusing hot team #%d bindings: "
5043 "proc_bind = %d, partition = [%d,%d]\n",
5044 team->t.t_id, new_proc_bind, team->t.t_first_place,
5045 team->t.t_last_place));
5047 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5048 __kmp_partition_places(team);
5051 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5053 }
else if (team->t.t_nproc > new_nproc) {
5055 (
"__kmp_allocate_team: decreasing hot team thread count to %d\n",
5058 team->t.t_size_changed = 1;
5059 #if KMP_NESTED_HOT_TEAMS
5060 if (__kmp_hot_teams_mode == 0) {
5063 KMP_DEBUG_ASSERT(hot_teams[level].hot_team_nth == team->t.t_nproc);
5064 hot_teams[level].hot_team_nth = new_nproc;
5065 #endif // KMP_NESTED_HOT_TEAMS
5067 for (f = new_nproc; f < team->t.t_nproc; f++) {
5068 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5069 if (__kmp_tasking_mode != tskm_immediate_exec) {
5072 team->t.t_threads[f]->th.th_task_team = NULL;
5074 __kmp_free_thread(team->t.t_threads[f]);
5075 team->t.t_threads[f] = NULL;
5077 #if KMP_NESTED_HOT_TEAMS
5082 for (f = new_nproc; f < team->t.t_nproc; ++f) {
5083 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5084 kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar;
5085 for (
int b = 0; b < bs_last_barrier; ++b) {
5086 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG) {
5087 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG;
5089 KMP_CHECK_UPDATE(balign[b].bb.leaf_kids, 0);
5093 #endif // KMP_NESTED_HOT_TEAMS
5094 team->t.t_nproc = new_nproc;
5096 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_icvs->sched.sched);
5097 __kmp_reinitialize_team(team, new_icvs,
5098 root->r.r_uber_thread->th.th_ident);
5101 for (f = 0; f < new_nproc; ++f) {
5102 team->t.t_threads[f]->th.th_team_nproc = new_nproc;
5107 KF_TRACE(10, (
"__kmp_allocate_team: T#%d, this_thread=%p team=%p\n", 0,
5108 team->t.t_threads[0], team));
5110 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);
5113 for (f = 0; f < team->t.t_nproc; f++) {
5114 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
5115 team->t.t_threads[f]->th.th_team_nproc ==
5120 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5121 #if KMP_AFFINITY_SUPPORTED
5122 __kmp_partition_places(team);
5125 #if (KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED
5126 kmp_affin_mask_t *old_mask;
5127 if (KMP_AFFINITY_CAPABLE()) {
5128 KMP_CPU_ALLOC(old_mask);
5133 (
"__kmp_allocate_team: increasing hot team thread count to %d\n",
5136 team->t.t_size_changed = 1;
5138 #if KMP_NESTED_HOT_TEAMS
5139 int avail_threads = hot_teams[level].hot_team_nth;
5140 if (new_nproc < avail_threads)
5141 avail_threads = new_nproc;
5142 kmp_info_t **other_threads = team->t.t_threads;
5143 for (f = team->t.t_nproc; f < avail_threads; ++f) {
5147 kmp_balign_t *balign = other_threads[f]->th.th_bar;
5148 for (b = 0; b < bs_last_barrier; ++b) {
5149 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5150 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
5152 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5156 if (hot_teams[level].hot_team_nth >= new_nproc) {
5159 KMP_DEBUG_ASSERT(__kmp_hot_teams_mode == 1);
5160 team->t.t_nproc = new_nproc;
5166 hot_teams[level].hot_team_nth = new_nproc;
5167 #endif // KMP_NESTED_HOT_TEAMS
5168 if (team->t.t_max_nproc < new_nproc) {
5170 __kmp_reallocate_team_arrays(team, new_nproc);
5171 __kmp_reinitialize_team(team, new_icvs, NULL);
5174 #if (KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED
5179 __kmp_set_thread_affinity_mask_full_tmp(old_mask);
5183 for (f = team->t.t_nproc; f < new_nproc; f++) {
5184 kmp_info_t *new_worker = __kmp_allocate_thread(root, team, f);
5185 KMP_DEBUG_ASSERT(new_worker);
5186 team->t.t_threads[f] = new_worker;
5189 (
"__kmp_allocate_team: team %d init T#%d arrived: "
5190 "join=%llu, plain=%llu\n",
5191 team->t.t_id, __kmp_gtid_from_tid(f, team), team->t.t_id, f,
5192 team->t.t_bar[bs_forkjoin_barrier].b_arrived,
5193 team->t.t_bar[bs_plain_barrier].b_arrived));
5197 kmp_balign_t *balign = new_worker->th.th_bar;
5198 for (b = 0; b < bs_last_barrier; ++b) {
5199 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5200 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag !=
5201 KMP_BARRIER_PARENT_FLAG);
5203 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5209 #if (KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED
5210 if (KMP_AFFINITY_CAPABLE()) {
5212 __kmp_set_system_affinity(old_mask, TRUE);
5213 KMP_CPU_FREE(old_mask);
5216 #if KMP_NESTED_HOT_TEAMS
5218 #endif // KMP_NESTED_HOT_TEAMS
5220 int old_nproc = team->t.t_nproc;
5222 __kmp_initialize_team(team, new_nproc, new_icvs,
5223 root->r.r_uber_thread->th.th_ident);
5226 KMP_DEBUG_ASSERT(team->t.t_nproc == new_nproc);
5227 for (f = 0; f < team->t.t_nproc; ++f)
5228 __kmp_initialize_info(team->t.t_threads[f], team, f,
5229 __kmp_gtid_from_tid(f, team));
5237 for (f = old_nproc; f < team->t.t_nproc; ++f)
5238 team->t.t_threads[f]->th.th_task_state =
5239 team->t.t_threads[0]->th.th_task_state_memo_stack[level];
5241 kmp_uint8 old_state =
5242 team->t.t_threads[0]->th.th_task_state;
5243 for (f = old_nproc; f < team->t.t_nproc; ++f)
5244 team->t.t_threads[f]->th.th_task_state = old_state;
5248 for (f = 0; f < team->t.t_nproc; ++f) {
5249 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
5250 team->t.t_threads[f]->th.th_team_nproc ==
5255 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5256 #if KMP_AFFINITY_SUPPORTED
5257 __kmp_partition_places(team);
5261 kmp_info_t *master = team->t.t_threads[0];
5262 if (master->th.th_teams_microtask) {
5263 for (f = 1; f < new_nproc; ++f) {
5265 kmp_info_t *thr = team->t.t_threads[f];
5266 thr->th.th_teams_microtask = master->th.th_teams_microtask;
5267 thr->th.th_teams_level = master->th.th_teams_level;
5268 thr->th.th_teams_size = master->th.th_teams_size;
5271 #if KMP_NESTED_HOT_TEAMS
5275 for (f = 1; f < new_nproc; ++f) {
5276 kmp_info_t *thr = team->t.t_threads[f];
5278 kmp_balign_t *balign = thr->th.th_bar;
5279 for (b = 0; b < bs_last_barrier; ++b) {
5280 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5281 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
5283 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5288 #endif // KMP_NESTED_HOT_TEAMS
5291 __kmp_alloc_argv_entries(argc, team, TRUE);
5292 KMP_CHECK_UPDATE(team->t.t_argc, argc);
5296 KF_TRACE(10, (
" hot_team = %p\n", team));
5299 if (__kmp_tasking_mode != tskm_immediate_exec) {
5300 KA_TRACE(20, (
"__kmp_allocate_team: hot team task_team[0] = %p "
5301 "task_team[1] = %p after reinit\n",
5302 team->t.t_task_team[0], team->t.t_task_team[1]));
5307 __ompt_team_assign_id(team, ompt_parallel_data);
5317 for (team = CCAST(kmp_team_t *, __kmp_team_pool); (team);) {
5320 if (team->t.t_max_nproc >= max_nproc) {
5322 __kmp_team_pool = team->t.t_next_pool;
5325 __kmp_initialize_team(team, new_nproc, new_icvs, NULL);
5327 KA_TRACE(20, (
"__kmp_allocate_team: setting task_team[0] %p and "
5328 "task_team[1] %p to NULL\n",
5329 &team->t.t_task_team[0], &team->t.t_task_team[1]));
5330 team->t.t_task_team[0] = NULL;
5331 team->t.t_task_team[1] = NULL;
5334 __kmp_alloc_argv_entries(argc, team, TRUE);
5335 KMP_CHECK_UPDATE(team->t.t_argc, argc);
5338 20, (
"__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",
5339 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
5342 for (b = 0; b < bs_last_barrier; ++b) {
5343 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE;
5345 team->t.t_bar[b].b_master_arrived = 0;
5346 team->t.t_bar[b].b_team_arrived = 0;
5351 team->t.t_proc_bind = new_proc_bind;
5353 KA_TRACE(20, (
"__kmp_allocate_team: using team from pool %d.\n",
5357 __ompt_team_assign_id(team, ompt_parallel_data);
5369 team = __kmp_reap_team(team);
5370 __kmp_team_pool = team;
5375 team = (kmp_team_t *)__kmp_allocate(
sizeof(kmp_team_t));
5378 team->t.t_max_nproc = max_nproc;
5381 __kmp_allocate_team_arrays(team, max_nproc);
5383 KA_TRACE(20, (
"__kmp_allocate_team: making a new team\n"));
5384 __kmp_initialize_team(team, new_nproc, new_icvs, NULL);
5386 KA_TRACE(20, (
"__kmp_allocate_team: setting task_team[0] %p and task_team[1] "
5388 &team->t.t_task_team[0], &team->t.t_task_team[1]));
5389 team->t.t_task_team[0] = NULL;
5391 team->t.t_task_team[1] = NULL;
5394 if (__kmp_storage_map) {
5395 __kmp_print_team_storage_map(
"team", team, team->t.t_id, new_nproc);
5399 __kmp_alloc_argv_entries(argc, team, FALSE);
5400 team->t.t_argc = argc;
5403 (
"__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",
5404 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
5407 for (b = 0; b < bs_last_barrier; ++b) {
5408 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE;
5410 team->t.t_bar[b].b_master_arrived = 0;
5411 team->t.t_bar[b].b_team_arrived = 0;
5416 team->t.t_proc_bind = new_proc_bind;
5419 __ompt_team_assign_id(team, ompt_parallel_data);
5420 team->t.ompt_serialized_team_info = NULL;
5425 KA_TRACE(20, (
"__kmp_allocate_team: done creating a new team %d.\n",
5436 void __kmp_free_team(kmp_root_t *root,
5437 kmp_team_t *team USE_NESTED_HOT_ARG(kmp_info_t *master)) {
5439 KA_TRACE(20, (
"__kmp_free_team: T#%d freeing team %d\n", __kmp_get_gtid(),
5443 KMP_DEBUG_ASSERT(root);
5444 KMP_DEBUG_ASSERT(team);
5445 KMP_DEBUG_ASSERT(team->t.t_nproc <= team->t.t_max_nproc);
5446 KMP_DEBUG_ASSERT(team->t.t_threads);
5448 int use_hot_team = team == root->r.r_hot_team;
5449 #if KMP_NESTED_HOT_TEAMS
5451 kmp_hot_team_ptr_t *hot_teams;
5453 level = team->t.t_active_level - 1;
5454 if (master->th.th_teams_microtask) {
5455 if (master->th.th_teams_size.nteams > 1) {
5459 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
5460 master->th.th_teams_level == team->t.t_level) {
5465 hot_teams = master->th.th_hot_teams;
5466 if (level < __kmp_hot_teams_max_level) {
5467 KMP_DEBUG_ASSERT(team == hot_teams[level].hot_team);
5471 #endif // KMP_NESTED_HOT_TEAMS
5474 TCW_SYNC_PTR(team->t.t_pkfn,
5477 team->t.t_copyin_counter = 0;
5482 if (!use_hot_team) {
5483 if (__kmp_tasking_mode != tskm_immediate_exec) {
5485 for (f = 1; f < team->t.t_nproc; ++f) {
5486 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5487 kmp_info_t *th = team->t.t_threads[f];
5488 volatile kmp_uint32 *state = &th->th.th_reap_state;
5489 while (*state != KMP_SAFE_TO_REAP) {
5493 if (!__kmp_is_thread_alive(th, &ecode)) {
5494 *state = KMP_SAFE_TO_REAP;
5499 kmp_flag_64<> fl(&th->th.th_bar[bs_forkjoin_barrier].bb.b_go, th);
5500 if (fl.is_sleeping())
5501 fl.resume(__kmp_gtid_from_thread(th));
5508 for (tt_idx = 0; tt_idx < 2; ++tt_idx) {
5509 kmp_task_team_t *task_team = team->t.t_task_team[tt_idx];
5510 if (task_team != NULL) {
5511 for (f = 0; f < team->t.t_nproc; ++f) {
5512 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5513 team->t.t_threads[f]->th.th_task_team = NULL;
5517 (
"__kmp_free_team: T#%d deactivating task_team %p on team %d\n",
5518 __kmp_get_gtid(), task_team, team->t.t_id));
5519 #if KMP_NESTED_HOT_TEAMS
5520 __kmp_free_task_team(master, task_team);
5522 team->t.t_task_team[tt_idx] = NULL;
5528 team->t.t_parent = NULL;
5529 team->t.t_level = 0;
5530 team->t.t_active_level = 0;
5533 for (f = 1; f < team->t.t_nproc; ++f) {
5534 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5535 __kmp_free_thread(team->t.t_threads[f]);
5536 team->t.t_threads[f] = NULL;
5541 team->t.t_next_pool = CCAST(kmp_team_t *, __kmp_team_pool);
5542 __kmp_team_pool = (
volatile kmp_team_t *)team;
5545 KMP_DEBUG_ASSERT(team->t.t_threads[1] &&
5546 team->t.t_threads[1]->th.th_cg_roots);
5547 if (team->t.t_threads[1]->th.th_cg_roots->cg_root == team->t.t_threads[1]) {
5549 for (f = 1; f < team->t.t_nproc; ++f) {
5550 kmp_info_t *thr = team->t.t_threads[f];
5551 KMP_DEBUG_ASSERT(thr && thr->th.th_cg_roots &&
5552 thr->th.th_cg_roots->cg_root == thr);
5554 kmp_cg_root_t *tmp = thr->th.th_cg_roots;
5555 thr->th.th_cg_roots = tmp->up;
5556 KA_TRACE(100, (
"__kmp_free_team: Thread %p popping node %p and moving"
5557 " up to node %p. cg_nthreads was %d\n",
5558 thr, tmp, thr->th.th_cg_roots, tmp->cg_nthreads));
5559 int i = tmp->cg_nthreads--;
5564 if (thr->th.th_cg_roots)
5565 thr->th.th_current_task->td_icvs.thread_limit =
5566 thr->th.th_cg_roots->cg_thread_limit;
5575 kmp_team_t *__kmp_reap_team(kmp_team_t *team) {
5576 kmp_team_t *next_pool = team->t.t_next_pool;
5578 KMP_DEBUG_ASSERT(team);
5579 KMP_DEBUG_ASSERT(team->t.t_dispatch);
5580 KMP_DEBUG_ASSERT(team->t.t_disp_buffer);
5581 KMP_DEBUG_ASSERT(team->t.t_threads);
5582 KMP_DEBUG_ASSERT(team->t.t_argv);
5587 __kmp_free_team_arrays(team);
5588 if (team->t.t_argv != &team->t.t_inline_argv[0])
5589 __kmp_free((
void *)team->t.t_argv);
5621 void __kmp_free_thread(kmp_info_t *this_th) {
5625 KA_TRACE(20, (
"__kmp_free_thread: T#%d putting T#%d back on free pool.\n",
5626 __kmp_get_gtid(), this_th->th.th_info.ds.ds_gtid));
5628 KMP_DEBUG_ASSERT(this_th);
5633 kmp_balign_t *balign = this_th->th.th_bar;
5634 for (b = 0; b < bs_last_barrier; ++b) {
5635 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG)
5636 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG;
5637 balign[b].bb.team = NULL;
5638 balign[b].bb.leaf_kids = 0;
5640 this_th->th.th_task_state = 0;
5641 this_th->th.th_reap_state = KMP_SAFE_TO_REAP;
5644 TCW_PTR(this_th->th.th_team, NULL);
5645 TCW_PTR(this_th->th.th_root, NULL);
5646 TCW_PTR(this_th->th.th_dispatch, NULL);
5648 while (this_th->th.th_cg_roots) {
5649 this_th->th.th_cg_roots->cg_nthreads--;
5650 KA_TRACE(100, (
"__kmp_free_thread: Thread %p decrement cg_nthreads on node"
5651 " %p of thread %p to %d\n",
5652 this_th, this_th->th.th_cg_roots,
5653 this_th->th.th_cg_roots->cg_root,
5654 this_th->th.th_cg_roots->cg_nthreads));
5655 kmp_cg_root_t *tmp = this_th->th.th_cg_roots;
5656 if (tmp->cg_root == this_th) {
5657 KMP_DEBUG_ASSERT(tmp->cg_nthreads == 0);
5659 5, (
"__kmp_free_thread: Thread %p freeing node %p\n", this_th, tmp));
5660 this_th->th.th_cg_roots = tmp->up;
5663 if (tmp->cg_nthreads == 0) {
5666 this_th->th.th_cg_roots = NULL;
5676 __kmp_free_implicit_task(this_th);
5677 this_th->th.th_current_task = NULL;
5681 gtid = this_th->th.th_info.ds.ds_gtid;
5682 if (__kmp_thread_pool_insert_pt != NULL) {
5683 KMP_DEBUG_ASSERT(__kmp_thread_pool != NULL);
5684 if (__kmp_thread_pool_insert_pt->th.th_info.ds.ds_gtid > gtid) {
5685 __kmp_thread_pool_insert_pt = NULL;
5694 if (__kmp_thread_pool_insert_pt != NULL) {
5695 scan = &(__kmp_thread_pool_insert_pt->th.th_next_pool);
5697 scan = CCAST(kmp_info_t **, &__kmp_thread_pool);
5699 for (; (*scan != NULL) && ((*scan)->th.th_info.ds.ds_gtid < gtid);
5700 scan = &((*scan)->th.th_next_pool))
5705 TCW_PTR(this_th->th.th_next_pool, *scan);
5706 __kmp_thread_pool_insert_pt = *scan = this_th;
5707 KMP_DEBUG_ASSERT((this_th->th.th_next_pool == NULL) ||
5708 (this_th->th.th_info.ds.ds_gtid <
5709 this_th->th.th_next_pool->th.th_info.ds.ds_gtid));
5710 TCW_4(this_th->th.th_in_pool, TRUE);
5711 __kmp_suspend_initialize_thread(this_th);
5712 __kmp_lock_suspend_mx(this_th);
5713 if (this_th->th.th_active == TRUE) {
5714 KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
5715 this_th->th.th_active_in_pool = TRUE;
5719 KMP_DEBUG_ASSERT(this_th->th.th_active_in_pool == FALSE);
5722 __kmp_unlock_suspend_mx(this_th);
5724 TCW_4(__kmp_nth, __kmp_nth - 1);
5726 #ifdef KMP_ADJUST_BLOCKTIME
5729 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
5730 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
5731 if (__kmp_nth <= __kmp_avail_proc) {
5732 __kmp_zero_bt = FALSE;
5742 void *__kmp_launch_thread(kmp_info_t *this_thr) {
5743 #if OMP_PROFILING_SUPPORT
5744 ProfileTraceFile = getenv(
"LIBOMPTARGET_PROFILE");
5746 if (ProfileTraceFile)
5747 llvm::timeTraceProfilerInitialize(500 ,
"libomptarget");
5750 int gtid = this_thr->th.th_info.ds.ds_gtid;
5752 kmp_team_t **
volatile pteam;
5755 KA_TRACE(10, (
"__kmp_launch_thread: T#%d start\n", gtid));
5757 if (__kmp_env_consistency_check) {
5758 this_thr->th.th_cons = __kmp_allocate_cons_stack(gtid);
5762 ompt_data_t *thread_data;
5763 if (ompt_enabled.enabled) {
5764 thread_data = &(this_thr->th.ompt_thread_info.thread_data);
5765 *thread_data = ompt_data_none;
5767 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
5768 this_thr->th.ompt_thread_info.wait_id = 0;
5769 this_thr->th.ompt_thread_info.idle_frame = OMPT_GET_FRAME_ADDRESS(0);
5770 this_thr->th.ompt_thread_info.parallel_flags = 0;
5771 if (ompt_enabled.ompt_callback_thread_begin) {
5772 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)(
5773 ompt_thread_worker, thread_data);
5775 this_thr->th.ompt_thread_info.state = ompt_state_idle;
5780 while (!TCR_4(__kmp_global.g.g_done)) {
5781 KMP_DEBUG_ASSERT(this_thr == __kmp_threads[gtid]);
5785 KA_TRACE(20, (
"__kmp_launch_thread: T#%d waiting for work\n", gtid));
5788 __kmp_fork_barrier(gtid, KMP_GTID_DNE);
5791 if (ompt_enabled.enabled) {
5792 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
5796 pteam = &this_thr->th.th_team;
5799 if (TCR_SYNC_PTR(*pteam) && !TCR_4(__kmp_global.g.g_done)) {
5801 if (TCR_SYNC_PTR((*pteam)->t.t_pkfn) != NULL) {
5804 (
"__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n",
5805 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid),
5806 (*pteam)->t.t_pkfn));
5808 updateHWFPControl(*pteam);
5811 if (ompt_enabled.enabled) {
5812 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
5816 rc = (*pteam)->t.t_invoke(gtid);
5820 KA_TRACE(20, (
"__kmp_launch_thread: T#%d(%d:%d) done microtask = %p\n",
5821 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid),
5822 (*pteam)->t.t_pkfn));
5825 if (ompt_enabled.enabled) {
5827 __ompt_get_task_info_object(0)->frame.exit_frame = ompt_data_none;
5829 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
5833 __kmp_join_barrier(gtid);
5836 TCR_SYNC_PTR((intptr_t)__kmp_global.g.g_done);
5839 if (ompt_enabled.ompt_callback_thread_end) {
5840 ompt_callbacks.ompt_callback(ompt_callback_thread_end)(thread_data);
5844 this_thr->th.th_task_team = NULL;
5846 __kmp_common_destroy_gtid(gtid);
5848 KA_TRACE(10, (
"__kmp_launch_thread: T#%d done\n", gtid));
5851 #if OMP_PROFILING_SUPPORT
5852 llvm::timeTraceProfilerFinishThread();
5859 void __kmp_internal_end_dest(
void *specific_gtid) {
5862 __kmp_type_convert((kmp_intptr_t)specific_gtid - 1, >id);
5864 KA_TRACE(30, (
"__kmp_internal_end_dest: T#%d\n", gtid));
5868 __kmp_internal_end_thread(gtid);
5871 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB
5873 __attribute__((destructor))
void __kmp_internal_end_dtor(
void) {
5874 __kmp_internal_end_atexit();
5881 void __kmp_internal_end_atexit(
void) {
5882 KA_TRACE(30, (
"__kmp_internal_end_atexit\n"));
5906 __kmp_internal_end_library(-1);
5908 __kmp_close_console();
5912 static void __kmp_reap_thread(kmp_info_t *thread,
int is_root) {
5917 KMP_DEBUG_ASSERT(thread != NULL);
5919 gtid = thread->th.th_info.ds.ds_gtid;
5922 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
5925 20, (
"__kmp_reap_thread: releasing T#%d from fork barrier for reap\n",
5929 ANNOTATE_HAPPENS_BEFORE(thread);
5930 kmp_flag_64<> flag(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go,
5932 __kmp_release_64(&flag);
5936 __kmp_reap_worker(thread);
5948 if (thread->th.th_active_in_pool) {
5949 thread->th.th_active_in_pool = FALSE;
5950 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
5951 KMP_DEBUG_ASSERT(__kmp_thread_pool_active_nth >= 0);
5955 __kmp_free_implicit_task(thread);
5959 __kmp_free_fast_memory(thread);
5962 __kmp_suspend_uninitialize_thread(thread);
5964 KMP_DEBUG_ASSERT(__kmp_threads[gtid] == thread);
5965 TCW_SYNC_PTR(__kmp_threads[gtid], NULL);
5970 #ifdef KMP_ADJUST_BLOCKTIME
5973 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
5974 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
5975 if (__kmp_nth <= __kmp_avail_proc) {
5976 __kmp_zero_bt = FALSE;
5982 if (__kmp_env_consistency_check) {
5983 if (thread->th.th_cons) {
5984 __kmp_free_cons_stack(thread->th.th_cons);
5985 thread->th.th_cons = NULL;
5989 if (thread->th.th_pri_common != NULL) {
5990 __kmp_free(thread->th.th_pri_common);
5991 thread->th.th_pri_common = NULL;
5994 if (thread->th.th_task_state_memo_stack != NULL) {
5995 __kmp_free(thread->th.th_task_state_memo_stack);
5996 thread->th.th_task_state_memo_stack = NULL;
6000 if (thread->th.th_local.bget_data != NULL) {
6001 __kmp_finalize_bget(thread);
6005 #if KMP_AFFINITY_SUPPORTED
6006 if (thread->th.th_affin_mask != NULL) {
6007 KMP_CPU_FREE(thread->th.th_affin_mask);
6008 thread->th.th_affin_mask = NULL;
6012 #if KMP_USE_HIER_SCHED
6013 if (thread->th.th_hier_bar_data != NULL) {
6014 __kmp_free(thread->th.th_hier_bar_data);
6015 thread->th.th_hier_bar_data = NULL;
6019 __kmp_reap_team(thread->th.th_serial_team);
6020 thread->th.th_serial_team = NULL;
6027 static void __kmp_internal_end(
void) {
6031 __kmp_unregister_library();
6038 __kmp_reclaim_dead_roots();
6042 for (i = 0; i < __kmp_threads_capacity; i++)
6044 if (__kmp_root[i]->r.r_active)
6047 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
6049 if (i < __kmp_threads_capacity) {
6061 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
6062 if (TCR_4(__kmp_init_monitor)) {
6063 __kmp_reap_monitor(&__kmp_monitor);
6064 TCW_4(__kmp_init_monitor, 0);
6066 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
6067 KA_TRACE(10, (
"__kmp_internal_end: monitor reaped\n"));
6068 #endif // KMP_USE_MONITOR
6073 for (i = 0; i < __kmp_threads_capacity; i++) {
6074 if (__kmp_root[i]) {
6077 KMP_ASSERT(!__kmp_root[i]->r.r_active);
6086 while (__kmp_thread_pool != NULL) {
6088 kmp_info_t *thread = CCAST(kmp_info_t *, __kmp_thread_pool);
6089 __kmp_thread_pool = thread->th.th_next_pool;
6091 KMP_DEBUG_ASSERT(thread->th.th_reap_state == KMP_SAFE_TO_REAP);
6092 thread->th.th_next_pool = NULL;
6093 thread->th.th_in_pool = FALSE;
6094 __kmp_reap_thread(thread, 0);
6096 __kmp_thread_pool_insert_pt = NULL;
6099 while (__kmp_team_pool != NULL) {
6101 kmp_team_t *team = CCAST(kmp_team_t *, __kmp_team_pool);
6102 __kmp_team_pool = team->t.t_next_pool;
6104 team->t.t_next_pool = NULL;
6105 __kmp_reap_team(team);
6108 __kmp_reap_task_teams();
6115 for (i = 0; i < __kmp_threads_capacity; i++) {
6116 kmp_info_t *thr = __kmp_threads[i];
6117 while (thr && KMP_ATOMIC_LD_ACQ(&thr->th.th_blocking))
6122 for (i = 0; i < __kmp_threads_capacity; ++i) {
6129 TCW_SYNC_4(__kmp_init_common, FALSE);
6131 KA_TRACE(10, (
"__kmp_internal_end: all workers reaped\n"));
6139 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
6140 if (TCR_4(__kmp_init_monitor)) {
6141 __kmp_reap_monitor(&__kmp_monitor);
6142 TCW_4(__kmp_init_monitor, 0);
6144 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
6145 KA_TRACE(10, (
"__kmp_internal_end: monitor reaped\n"));
6148 TCW_4(__kmp_init_gtid, FALSE);
6157 void __kmp_internal_end_library(
int gtid_req) {
6164 if (__kmp_global.g.g_abort) {
6165 KA_TRACE(11, (
"__kmp_internal_end_library: abort, exiting\n"));
6169 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6170 KA_TRACE(10, (
"__kmp_internal_end_library: already finished\n"));
6177 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific();
6179 10, (
"__kmp_internal_end_library: enter T#%d (%d)\n", gtid, gtid_req));
6180 if (gtid == KMP_GTID_SHUTDOWN) {
6181 KA_TRACE(10, (
"__kmp_internal_end_library: !__kmp_init_runtime, system "
6182 "already shutdown\n"));
6184 }
else if (gtid == KMP_GTID_MONITOR) {
6185 KA_TRACE(10, (
"__kmp_internal_end_library: monitor thread, gtid not "
6186 "registered, or system shutdown\n"));
6188 }
else if (gtid == KMP_GTID_DNE) {
6189 KA_TRACE(10, (
"__kmp_internal_end_library: gtid not registered or system "
6192 }
else if (KMP_UBER_GTID(gtid)) {
6194 if (__kmp_root[gtid]->r.r_active) {
6195 __kmp_global.g.g_abort = -1;
6196 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
6197 __kmp_unregister_library();
6199 (
"__kmp_internal_end_library: root still active, abort T#%d\n",
6205 (
"__kmp_internal_end_library: unregistering sibling T#%d\n", gtid));
6206 __kmp_unregister_root_current_thread(gtid);
6213 #ifdef DUMP_DEBUG_ON_EXIT
6214 if (__kmp_debug_buf)
6215 __kmp_dump_debug_buffer();
6220 __kmp_unregister_library();
6225 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6228 if (__kmp_global.g.g_abort) {
6229 KA_TRACE(10, (
"__kmp_internal_end_library: abort, exiting\n"));
6231 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6234 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6235 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6244 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
6247 __kmp_internal_end();
6249 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6250 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6252 KA_TRACE(10, (
"__kmp_internal_end_library: exit\n"));
6254 #ifdef DUMP_DEBUG_ON_EXIT
6255 if (__kmp_debug_buf)
6256 __kmp_dump_debug_buffer();
6260 __kmp_close_console();
6263 __kmp_fini_allocator();
6267 void __kmp_internal_end_thread(
int gtid_req) {
6276 if (__kmp_global.g.g_abort) {
6277 KA_TRACE(11, (
"__kmp_internal_end_thread: abort, exiting\n"));
6281 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6282 KA_TRACE(10, (
"__kmp_internal_end_thread: already finished\n"));
6287 if (TCR_4(__kmp_init_hidden_helper)) {
6288 TCW_SYNC_4(__kmp_hidden_helper_team_done, TRUE);
6290 __kmp_hidden_helper_main_thread_release();
6292 __kmp_hidden_helper_threads_deinitz_wait();
6299 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific();
6301 (
"__kmp_internal_end_thread: enter T#%d (%d)\n", gtid, gtid_req));
6302 if (gtid == KMP_GTID_SHUTDOWN) {
6303 KA_TRACE(10, (
"__kmp_internal_end_thread: !__kmp_init_runtime, system "
6304 "already shutdown\n"));
6306 }
else if (gtid == KMP_GTID_MONITOR) {
6307 KA_TRACE(10, (
"__kmp_internal_end_thread: monitor thread, gtid not "
6308 "registered, or system shutdown\n"));
6310 }
else if (gtid == KMP_GTID_DNE) {
6311 KA_TRACE(10, (
"__kmp_internal_end_thread: gtid not registered or system "
6315 }
else if (KMP_UBER_GTID(gtid)) {
6317 if (__kmp_root[gtid]->r.r_active) {
6318 __kmp_global.g.g_abort = -1;
6319 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
6321 (
"__kmp_internal_end_thread: root still active, abort T#%d\n",
6325 KA_TRACE(10, (
"__kmp_internal_end_thread: unregistering sibling T#%d\n",
6327 __kmp_unregister_root_current_thread(gtid);
6331 KA_TRACE(10, (
"__kmp_internal_end_thread: worker thread T#%d\n", gtid));
6334 __kmp_threads[gtid]->th.th_task_team = NULL;
6338 (
"__kmp_internal_end_thread: worker thread done, exiting T#%d\n",
6344 if (__kmp_pause_status != kmp_hard_paused)
6348 KA_TRACE(10, (
"__kmp_internal_end_thread: exiting T#%d\n", gtid_req));
6353 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6356 if (__kmp_global.g.g_abort) {
6357 KA_TRACE(10, (
"__kmp_internal_end_thread: abort, exiting\n"));
6359 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6362 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6363 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6374 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
6376 for (i = 0; i < __kmp_threads_capacity; ++i) {
6377 if (KMP_UBER_GTID(i)) {
6380 (
"__kmp_internal_end_thread: remaining sibling task: gtid==%d\n", i));
6381 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6382 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6389 __kmp_internal_end();
6391 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6392 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6394 KA_TRACE(10, (
"__kmp_internal_end_thread: exit T#%d\n", gtid_req));
6396 #ifdef DUMP_DEBUG_ON_EXIT
6397 if (__kmp_debug_buf)
6398 __kmp_dump_debug_buffer();
6405 static long __kmp_registration_flag = 0;
6407 static char *__kmp_registration_str = NULL;
6410 static inline char *__kmp_reg_status_name() {
6416 #if KMP_OS_UNIX && !KMP_OS_DARWIN && KMP_DYNAMIC_LIB
6417 return __kmp_str_format(
"__KMP_REGISTERED_LIB_%d_%d", (
int)getpid(),
6420 return __kmp_str_format(
"__KMP_REGISTERED_LIB_%d", (
int)getpid());
6424 void __kmp_register_library_startup(
void) {
6426 char *name = __kmp_reg_status_name();
6432 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
6433 __kmp_initialize_system_tick();
6435 __kmp_read_system_time(&time.dtime);
6436 __kmp_registration_flag = 0xCAFE0000L | (time.ltime & 0x0000FFFFL);
6437 __kmp_registration_str =
6438 __kmp_str_format(
"%p-%lx-%s", &__kmp_registration_flag,
6439 __kmp_registration_flag, KMP_LIBRARY_FILE);
6441 KA_TRACE(50, (
"__kmp_register_library_startup: %s=\"%s\"\n", name,
6442 __kmp_registration_str));
6448 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB // shared memory is with dynamic library
6449 char *shm_name = __kmp_str_format(
"/%s", name);
6450 int shm_preexist = 0;
6452 int fd1 = shm_open(shm_name, O_CREAT | O_EXCL | O_RDWR, 0666);
6453 if ((fd1 == -1) && (errno == EEXIST)) {
6456 fd1 = shm_open(shm_name, O_RDWR, 0666);
6459 __kmp_fatal(KMP_MSG(FunctionError,
"Can't open SHM"), KMP_ERR(0),
6465 }
else if (fd1 == -1) {
6468 __kmp_fatal(KMP_MSG(FunctionError,
"Can't open SHM2"), KMP_ERR(errno),
6471 if (shm_preexist == 0) {
6473 if (ftruncate(fd1, SHM_SIZE) == -1) {
6475 __kmp_fatal(KMP_MSG(FunctionError,
"Can't set size of SHM"),
6476 KMP_ERR(errno), __kmp_msg_null);
6480 (
char *)mmap(0, SHM_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd1, 0);
6481 if (data1 == MAP_FAILED) {
6483 __kmp_fatal(KMP_MSG(FunctionError,
"Can't map SHM"), KMP_ERR(errno),
6486 if (shm_preexist == 0) {
6487 KMP_STRCPY_S(data1, SHM_SIZE, __kmp_registration_str);
6490 value = __kmp_str_format(
"%s", data1);
6491 munmap(data1, SHM_SIZE);
6493 #else // Windows and unix with static library
6495 __kmp_env_set(name, __kmp_registration_str, 0);
6497 value = __kmp_env_get(name);
6500 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) {
6507 char *flag_addr_str = NULL;
6508 char *flag_val_str = NULL;
6509 char const *file_name = NULL;
6510 __kmp_str_split(tail,
'-', &flag_addr_str, &tail);
6511 __kmp_str_split(tail,
'-', &flag_val_str, &tail);
6514 long *flag_addr = 0;
6516 KMP_SSCANF(flag_addr_str,
"%p", RCAST(
void**, &flag_addr));
6517 KMP_SSCANF(flag_val_str,
"%lx", &flag_val);
6518 if (flag_addr != 0 && flag_val != 0 && strcmp(file_name,
"") != 0) {
6522 if (__kmp_is_address_mapped(flag_addr) && *flag_addr == flag_val) {
6536 file_name =
"unknown library";
6541 char *duplicate_ok = __kmp_env_get(
"KMP_DUPLICATE_LIB_OK");
6542 if (!__kmp_str_match_true(duplicate_ok)) {
6544 __kmp_fatal(KMP_MSG(DuplicateLibrary, KMP_LIBRARY_FILE, file_name),
6545 KMP_HNT(DuplicateLibrary), __kmp_msg_null);
6547 KMP_INTERNAL_FREE(duplicate_ok);
6548 __kmp_duplicate_library_ok = 1;
6553 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB // shared memory is with dynamic library
6555 shm_unlink(shm_name);
6558 __kmp_env_unset(name);
6561 default: { KMP_DEBUG_ASSERT(0); }
break;
6564 KMP_INTERNAL_FREE((
void *)value);
6565 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB // shared memory is with dynamic library
6566 KMP_INTERNAL_FREE((
void *)shm_name);
6569 KMP_INTERNAL_FREE((
void *)name);
6573 void __kmp_unregister_library(
void) {
6575 char *name = __kmp_reg_status_name();
6578 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB // shared memory is with dynamic library
6579 char *shm_name = __kmp_str_format(
"/%s", name);
6580 int fd1 = shm_open(shm_name, O_RDONLY, 0666);
6585 char *data1 = (
char *)mmap(0, SHM_SIZE, PROT_READ, MAP_SHARED, fd1, 0);
6586 if (data1 != MAP_FAILED) {
6587 value = __kmp_str_format(
"%s", data1);
6588 munmap(data1, SHM_SIZE);
6592 value = __kmp_env_get(name);
6595 KMP_DEBUG_ASSERT(__kmp_registration_flag != 0);
6596 KMP_DEBUG_ASSERT(__kmp_registration_str != NULL);
6597 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) {
6599 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB // shared memory is with dynamic library
6600 shm_unlink(shm_name);
6602 __kmp_env_unset(name);
6606 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB // shared memory is with dynamic library
6607 KMP_INTERNAL_FREE(shm_name);
6610 KMP_INTERNAL_FREE(__kmp_registration_str);
6611 KMP_INTERNAL_FREE(value);
6612 KMP_INTERNAL_FREE(name);
6614 __kmp_registration_flag = 0;
6615 __kmp_registration_str = NULL;
6622 #if KMP_MIC_SUPPORTED
6624 static void __kmp_check_mic_type() {
6625 kmp_cpuid_t cpuid_state = {0};
6626 kmp_cpuid_t *cs_p = &cpuid_state;
6627 __kmp_x86_cpuid(1, 0, cs_p);
6629 if ((cs_p->eax & 0xff0) == 0xB10) {
6630 __kmp_mic_type = mic2;
6631 }
else if ((cs_p->eax & 0xf0ff0) == 0x50670) {
6632 __kmp_mic_type = mic3;
6634 __kmp_mic_type = non_mic;
6641 static void __kmp_user_level_mwait_init() {
6642 struct kmp_cpuid buf;
6643 __kmp_x86_cpuid(7, 0, &buf);
6644 __kmp_umwait_enabled = ((buf.ecx >> 5) & 1) && __kmp_user_level_mwait;
6645 KF_TRACE(30, (
"__kmp_user_level_mwait_init: __kmp_umwait_enabled = %d\n",
6646 __kmp_umwait_enabled));
6648 #elif KMP_HAVE_MWAIT
6649 #ifndef AT_INTELPHIUSERMWAIT
6652 #define AT_INTELPHIUSERMWAIT 10000
6657 unsigned long getauxval(
unsigned long) KMP_WEAK_ATTRIBUTE_EXTERNAL;
6658 unsigned long getauxval(
unsigned long) {
return 0; }
6660 static void __kmp_user_level_mwait_init() {
6665 if (__kmp_mic_type == mic3) {
6666 unsigned long res = getauxval(AT_INTELPHIUSERMWAIT);
6667 if ((res & 0x1) || __kmp_user_level_mwait) {
6668 __kmp_mwait_enabled = TRUE;
6669 if (__kmp_user_level_mwait) {
6670 KMP_INFORM(EnvMwaitWarn);
6673 __kmp_mwait_enabled = FALSE;
6676 KF_TRACE(30, (
"__kmp_user_level_mwait_init: __kmp_mic_type = %d, "
6677 "__kmp_mwait_enabled = %d\n",
6678 __kmp_mic_type, __kmp_mwait_enabled));
6682 static void __kmp_do_serial_initialize(
void) {
6686 KA_TRACE(10, (
"__kmp_do_serial_initialize: enter\n"));
6688 KMP_DEBUG_ASSERT(
sizeof(kmp_int32) == 4);
6689 KMP_DEBUG_ASSERT(
sizeof(kmp_uint32) == 4);
6690 KMP_DEBUG_ASSERT(
sizeof(kmp_int64) == 8);
6691 KMP_DEBUG_ASSERT(
sizeof(kmp_uint64) == 8);
6692 KMP_DEBUG_ASSERT(
sizeof(kmp_intptr_t) ==
sizeof(
void *));
6698 __kmp_validate_locks();
6701 __kmp_init_allocator();
6706 __kmp_register_library_startup();
6709 if (TCR_4(__kmp_global.g.g_done)) {
6710 KA_TRACE(10, (
"__kmp_do_serial_initialize: reinitialization of library\n"));
6713 __kmp_global.g.g_abort = 0;
6714 TCW_SYNC_4(__kmp_global.g.g_done, FALSE);
6717 #if KMP_USE_ADAPTIVE_LOCKS
6718 #if KMP_DEBUG_ADAPTIVE_LOCKS
6719 __kmp_init_speculative_stats();
6722 #if KMP_STATS_ENABLED
6725 __kmp_init_lock(&__kmp_global_lock);
6726 __kmp_init_queuing_lock(&__kmp_dispatch_lock);
6727 __kmp_init_lock(&__kmp_debug_lock);
6728 __kmp_init_atomic_lock(&__kmp_atomic_lock);
6729 __kmp_init_atomic_lock(&__kmp_atomic_lock_1i);
6730 __kmp_init_atomic_lock(&__kmp_atomic_lock_2i);
6731 __kmp_init_atomic_lock(&__kmp_atomic_lock_4i);
6732 __kmp_init_atomic_lock(&__kmp_atomic_lock_4r);
6733 __kmp_init_atomic_lock(&__kmp_atomic_lock_8i);
6734 __kmp_init_atomic_lock(&__kmp_atomic_lock_8r);
6735 __kmp_init_atomic_lock(&__kmp_atomic_lock_8c);
6736 __kmp_init_atomic_lock(&__kmp_atomic_lock_10r);
6737 __kmp_init_atomic_lock(&__kmp_atomic_lock_16r);
6738 __kmp_init_atomic_lock(&__kmp_atomic_lock_16c);
6739 __kmp_init_atomic_lock(&__kmp_atomic_lock_20c);
6740 __kmp_init_atomic_lock(&__kmp_atomic_lock_32c);
6741 __kmp_init_bootstrap_lock(&__kmp_forkjoin_lock);
6742 __kmp_init_bootstrap_lock(&__kmp_exit_lock);
6744 __kmp_init_bootstrap_lock(&__kmp_monitor_lock);
6746 __kmp_init_bootstrap_lock(&__kmp_tp_cached_lock);
6750 __kmp_runtime_initialize();
6752 #if KMP_MIC_SUPPORTED
6753 __kmp_check_mic_type();
6760 __kmp_abort_delay = 0;
6764 __kmp_dflt_team_nth_ub = __kmp_xproc;
6765 if (__kmp_dflt_team_nth_ub < KMP_MIN_NTH) {
6766 __kmp_dflt_team_nth_ub = KMP_MIN_NTH;
6768 if (__kmp_dflt_team_nth_ub > __kmp_sys_max_nth) {
6769 __kmp_dflt_team_nth_ub = __kmp_sys_max_nth;
6771 __kmp_max_nth = __kmp_sys_max_nth;
6772 __kmp_cg_max_nth = __kmp_sys_max_nth;
6773 __kmp_teams_max_nth = __kmp_xproc;
6774 if (__kmp_teams_max_nth > __kmp_sys_max_nth) {
6775 __kmp_teams_max_nth = __kmp_sys_max_nth;
6780 __kmp_dflt_blocktime = KMP_DEFAULT_BLOCKTIME;
6782 __kmp_monitor_wakeups =
6783 KMP_WAKEUPS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups);
6784 __kmp_bt_intervals =
6785 KMP_INTERVALS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups);
6788 __kmp_library = library_throughput;
6790 __kmp_static = kmp_sch_static_balanced;
6797 #if KMP_FAST_REDUCTION_BARRIER
6798 #define kmp_reduction_barrier_gather_bb ((int)1)
6799 #define kmp_reduction_barrier_release_bb ((int)1)
6800 #define kmp_reduction_barrier_gather_pat bp_hyper_bar
6801 #define kmp_reduction_barrier_release_pat bp_hyper_bar
6802 #endif // KMP_FAST_REDUCTION_BARRIER
6803 for (i = bs_plain_barrier; i < bs_last_barrier; i++) {
6804 __kmp_barrier_gather_branch_bits[i] = __kmp_barrier_gather_bb_dflt;
6805 __kmp_barrier_release_branch_bits[i] = __kmp_barrier_release_bb_dflt;
6806 __kmp_barrier_gather_pattern[i] = __kmp_barrier_gather_pat_dflt;
6807 __kmp_barrier_release_pattern[i] = __kmp_barrier_release_pat_dflt;
6808 #if KMP_FAST_REDUCTION_BARRIER
6809 if (i == bs_reduction_barrier) {
6811 __kmp_barrier_gather_branch_bits[i] = kmp_reduction_barrier_gather_bb;
6812 __kmp_barrier_release_branch_bits[i] = kmp_reduction_barrier_release_bb;
6813 __kmp_barrier_gather_pattern[i] = kmp_reduction_barrier_gather_pat;
6814 __kmp_barrier_release_pattern[i] = kmp_reduction_barrier_release_pat;
6816 #endif // KMP_FAST_REDUCTION_BARRIER
6818 #if KMP_FAST_REDUCTION_BARRIER
6819 #undef kmp_reduction_barrier_release_pat
6820 #undef kmp_reduction_barrier_gather_pat
6821 #undef kmp_reduction_barrier_release_bb
6822 #undef kmp_reduction_barrier_gather_bb
6823 #endif // KMP_FAST_REDUCTION_BARRIER
6824 #if KMP_MIC_SUPPORTED
6825 if (__kmp_mic_type == mic2) {
6827 __kmp_barrier_gather_branch_bits[bs_plain_barrier] = 3;
6828 __kmp_barrier_release_branch_bits[bs_forkjoin_barrier] =
6830 __kmp_barrier_gather_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar;
6831 __kmp_barrier_release_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar;
6833 #if KMP_FAST_REDUCTION_BARRIER
6834 if (__kmp_mic_type == mic2) {
6835 __kmp_barrier_gather_pattern[bs_reduction_barrier] = bp_hierarchical_bar;
6836 __kmp_barrier_release_pattern[bs_reduction_barrier] = bp_hierarchical_bar;
6838 #endif // KMP_FAST_REDUCTION_BARRIER
6839 #endif // KMP_MIC_SUPPORTED
6843 __kmp_env_checks = TRUE;
6845 __kmp_env_checks = FALSE;
6849 __kmp_foreign_tp = TRUE;
6851 __kmp_global.g.g_dynamic = FALSE;
6852 __kmp_global.g.g_dynamic_mode = dynamic_default;
6854 __kmp_env_initialize(NULL);
6856 #if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
6857 __kmp_user_level_mwait_init();
6861 char const *val = __kmp_env_get(
"KMP_DUMP_CATALOG");
6862 if (__kmp_str_match_true(val)) {
6863 kmp_str_buf_t buffer;
6864 __kmp_str_buf_init(&buffer);
6865 __kmp_i18n_dump_catalog(&buffer);
6866 __kmp_printf(
"%s", buffer.str);
6867 __kmp_str_buf_free(&buffer);
6869 __kmp_env_free(&val);
6872 __kmp_threads_capacity =
6873 __kmp_initial_threads_capacity(__kmp_dflt_team_nth_ub);
6875 __kmp_tp_capacity = __kmp_default_tp_capacity(
6876 __kmp_dflt_team_nth_ub, __kmp_max_nth, __kmp_allThreadsSpecified);
6881 KMP_DEBUG_ASSERT(__kmp_thread_pool == NULL);
6882 KMP_DEBUG_ASSERT(__kmp_thread_pool_insert_pt == NULL);
6883 KMP_DEBUG_ASSERT(__kmp_team_pool == NULL);
6884 __kmp_thread_pool = NULL;
6885 __kmp_thread_pool_insert_pt = NULL;
6886 __kmp_team_pool = NULL;
6893 (
sizeof(kmp_info_t *) +
sizeof(kmp_root_t *)) * __kmp_threads_capacity +
6895 __kmp_threads = (kmp_info_t **)__kmp_allocate(size);
6896 __kmp_root = (kmp_root_t **)((
char *)__kmp_threads +
6897 sizeof(kmp_info_t *) * __kmp_threads_capacity);
6900 KMP_DEBUG_ASSERT(__kmp_all_nth ==
6902 KMP_DEBUG_ASSERT(__kmp_nth == 0);
6907 gtid = __kmp_register_root(TRUE);
6908 KA_TRACE(10, (
"__kmp_do_serial_initialize T#%d\n", gtid));
6909 KMP_ASSERT(KMP_UBER_GTID(gtid));
6910 KMP_ASSERT(KMP_INITIAL_GTID(gtid));
6914 __kmp_common_initialize();
6918 __kmp_register_atfork();
6921 #if !KMP_DYNAMIC_LIB
6925 int rc = atexit(__kmp_internal_end_atexit);
6927 __kmp_fatal(KMP_MSG(FunctionError,
"atexit()"), KMP_ERR(rc),
6933 #if KMP_HANDLE_SIGNALS
6939 __kmp_install_signals(FALSE);
6942 __kmp_install_signals(TRUE);
6947 __kmp_init_counter++;
6949 __kmp_init_serial = TRUE;
6951 if (__kmp_settings) {
6955 if (__kmp_display_env || __kmp_display_env_verbose) {
6956 __kmp_env_print_2();
6965 KA_TRACE(10, (
"__kmp_do_serial_initialize: exit\n"));
6968 void __kmp_serial_initialize(
void) {
6969 if (__kmp_init_serial) {
6972 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6973 if (__kmp_init_serial) {
6974 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6977 __kmp_do_serial_initialize();
6978 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6981 static void __kmp_do_middle_initialize(
void) {
6983 int prev_dflt_team_nth;
6985 if (!__kmp_init_serial) {
6986 __kmp_do_serial_initialize();
6989 KA_TRACE(10, (
"__kmp_middle_initialize: enter\n"));
6993 prev_dflt_team_nth = __kmp_dflt_team_nth;
6995 #if KMP_AFFINITY_SUPPORTED
6998 __kmp_affinity_initialize();
7002 for (i = 0; i < __kmp_threads_capacity; i++) {
7003 if (TCR_PTR(__kmp_threads[i]) != NULL) {
7004 __kmp_affinity_set_init_mask(i, TRUE);
7009 KMP_ASSERT(__kmp_xproc > 0);
7010 if (__kmp_avail_proc == 0) {
7011 __kmp_avail_proc = __kmp_xproc;
7017 while ((j < __kmp_nested_nth.used) && !__kmp_nested_nth.nth[j]) {
7018 __kmp_nested_nth.nth[j] = __kmp_dflt_team_nth = __kmp_dflt_team_nth_ub =
7023 if (__kmp_dflt_team_nth == 0) {
7024 #ifdef KMP_DFLT_NTH_CORES
7026 __kmp_dflt_team_nth = __kmp_ncores;
7027 KA_TRACE(20, (
"__kmp_middle_initialize: setting __kmp_dflt_team_nth = "
7028 "__kmp_ncores (%d)\n",
7029 __kmp_dflt_team_nth));
7032 __kmp_dflt_team_nth = __kmp_avail_proc;
7033 KA_TRACE(20, (
"__kmp_middle_initialize: setting __kmp_dflt_team_nth = "
7034 "__kmp_avail_proc(%d)\n",
7035 __kmp_dflt_team_nth));
7039 if (__kmp_dflt_team_nth < KMP_MIN_NTH) {
7040 __kmp_dflt_team_nth = KMP_MIN_NTH;
7042 if (__kmp_dflt_team_nth > __kmp_sys_max_nth) {
7043 __kmp_dflt_team_nth = __kmp_sys_max_nth;
7048 KMP_DEBUG_ASSERT(__kmp_dflt_team_nth <= __kmp_dflt_team_nth_ub);
7050 if (__kmp_dflt_team_nth != prev_dflt_team_nth) {
7055 for (i = 0; i < __kmp_threads_capacity; i++) {
7056 kmp_info_t *thread = __kmp_threads[i];
7059 if (thread->th.th_current_task->td_icvs.nproc != 0)
7062 set__nproc(__kmp_threads[i], __kmp_dflt_team_nth);
7067 (
"__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n",
7068 __kmp_dflt_team_nth));
7070 #ifdef KMP_ADJUST_BLOCKTIME
7072 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
7073 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
7074 if (__kmp_nth > __kmp_avail_proc) {
7075 __kmp_zero_bt = TRUE;
7081 TCW_SYNC_4(__kmp_init_middle, TRUE);
7083 KA_TRACE(10, (
"__kmp_do_middle_initialize: exit\n"));
7086 void __kmp_middle_initialize(
void) {
7087 if (__kmp_init_middle) {
7090 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
7091 if (__kmp_init_middle) {
7092 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7095 __kmp_do_middle_initialize();
7096 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7099 void __kmp_parallel_initialize(
void) {
7100 int gtid = __kmp_entry_gtid();
7103 if (TCR_4(__kmp_init_parallel))
7105 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
7106 if (TCR_4(__kmp_init_parallel)) {
7107 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7112 if (TCR_4(__kmp_global.g.g_done)) {
7115 (
"__kmp_parallel_initialize: attempt to init while shutting down\n"));
7116 __kmp_infinite_loop();
7122 if (!__kmp_init_middle) {
7123 __kmp_do_middle_initialize();
7125 __kmp_resume_if_hard_paused();
7128 KA_TRACE(10, (
"__kmp_parallel_initialize: enter\n"));
7129 KMP_ASSERT(KMP_UBER_GTID(gtid));
7131 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
7134 __kmp_store_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
7135 __kmp_store_mxcsr(&__kmp_init_mxcsr);
7136 __kmp_init_mxcsr &= KMP_X86_MXCSR_MASK;
7140 #if KMP_HANDLE_SIGNALS
7142 __kmp_install_signals(TRUE);
7146 __kmp_suspend_initialize();
7148 #if defined(USE_LOAD_BALANCE)
7149 if (__kmp_global.g.g_dynamic_mode == dynamic_default) {
7150 __kmp_global.g.g_dynamic_mode = dynamic_load_balance;
7153 if (__kmp_global.g.g_dynamic_mode == dynamic_default) {
7154 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit;
7158 if (__kmp_version) {
7159 __kmp_print_version_2();
7163 TCW_SYNC_4(__kmp_init_parallel, TRUE);
7166 KA_TRACE(10, (
"__kmp_parallel_initialize: exit\n"));
7168 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7171 void __kmp_hidden_helper_initialize() {
7172 if (TCR_4(__kmp_init_hidden_helper))
7176 if (!TCR_4(__kmp_init_parallel))
7177 __kmp_parallel_initialize();
7181 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
7182 if (TCR_4(__kmp_init_hidden_helper)) {
7183 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7188 KMP_ATOMIC_ST_REL(&__kmp_unexecuted_hidden_helper_tasks, 0);
7192 TCW_SYNC_4(__kmp_init_hidden_helper_threads, TRUE);
7195 __kmp_do_initialize_hidden_helper_threads();
7198 __kmp_hidden_helper_threads_initz_wait();
7201 TCW_SYNC_4(__kmp_init_hidden_helper, TRUE);
7203 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7208 void __kmp_run_before_invoked_task(
int gtid,
int tid, kmp_info_t *this_thr,
7210 kmp_disp_t *dispatch;
7215 this_thr->th.th_local.this_construct = 0;
7216 #if KMP_CACHE_MANAGE
7217 KMP_CACHE_PREFETCH(&this_thr->th.th_bar[bs_forkjoin_barrier].bb.b_arrived);
7219 dispatch = (kmp_disp_t *)TCR_PTR(this_thr->th.th_dispatch);
7220 KMP_DEBUG_ASSERT(dispatch);
7221 KMP_DEBUG_ASSERT(team->t.t_dispatch);
7225 dispatch->th_disp_index = 0;
7226 dispatch->th_doacross_buf_idx = 0;
7227 if (__kmp_env_consistency_check)
7228 __kmp_push_parallel(gtid, team->t.t_ident);
7233 void __kmp_run_after_invoked_task(
int gtid,
int tid, kmp_info_t *this_thr,
7235 if (__kmp_env_consistency_check)
7236 __kmp_pop_parallel(gtid, team->t.t_ident);
7238 __kmp_finish_implicit_task(this_thr);
7241 int __kmp_invoke_task_func(
int gtid) {
7243 int tid = __kmp_tid_from_gtid(gtid);
7244 kmp_info_t *this_thr = __kmp_threads[gtid];
7245 kmp_team_t *team = this_thr->th.th_team;
7247 __kmp_run_before_invoked_task(gtid, tid, this_thr, team);
7249 if (__itt_stack_caller_create_ptr) {
7250 __kmp_itt_stack_callee_enter(
7252 team->t.t_stack_id);
7255 #if INCLUDE_SSC_MARKS
7256 SSC_MARK_INVOKING();
7261 void **exit_frame_p;
7262 ompt_data_t *my_task_data;
7263 ompt_data_t *my_parallel_data;
7266 if (ompt_enabled.enabled) {
7268 team->t.t_implicit_task_taskdata[tid].ompt_task_info.frame.exit_frame.ptr);
7270 exit_frame_p = &dummy;
7274 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data);
7275 my_parallel_data = &(team->t.ompt_team_info.parallel_data);
7276 if (ompt_enabled.ompt_callback_implicit_task) {
7277 ompt_team_size = team->t.t_nproc;
7278 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
7279 ompt_scope_begin, my_parallel_data, my_task_data, ompt_team_size,
7280 __kmp_tid_from_gtid(gtid), ompt_task_implicit);
7281 OMPT_CUR_TASK_INFO(this_thr)->thread_num = __kmp_tid_from_gtid(gtid);
7285 #if KMP_STATS_ENABLED
7287 if (previous_state == stats_state_e::TEAMS_REGION) {
7288 KMP_PUSH_PARTITIONED_TIMER(OMP_teams);
7290 KMP_PUSH_PARTITIONED_TIMER(OMP_parallel);
7292 KMP_SET_THREAD_STATE(IMPLICIT_TASK);
7295 rc = __kmp_invoke_microtask((microtask_t)TCR_SYNC_PTR(team->t.t_pkfn), gtid,
7296 tid, (
int)team->t.t_argc, (
void **)team->t.t_argv
7303 *exit_frame_p = NULL;
7304 this_thr->th.ompt_thread_info.parallel_flags |= ompt_parallel_team;
7307 #if KMP_STATS_ENABLED
7308 if (previous_state == stats_state_e::TEAMS_REGION) {
7309 KMP_SET_THREAD_STATE(previous_state);
7311 KMP_POP_PARTITIONED_TIMER();
7315 if (__itt_stack_caller_create_ptr) {
7316 __kmp_itt_stack_callee_leave(
7318 team->t.t_stack_id);
7321 __kmp_run_after_invoked_task(gtid, tid, this_thr, team);
7326 void __kmp_teams_master(
int gtid) {
7328 kmp_info_t *thr = __kmp_threads[gtid];
7329 kmp_team_t *team = thr->th.th_team;
7330 ident_t *loc = team->t.t_ident;
7331 thr->th.th_set_nproc = thr->th.th_teams_size.nth;
7332 KMP_DEBUG_ASSERT(thr->th.th_teams_microtask);
7333 KMP_DEBUG_ASSERT(thr->th.th_set_nproc);
7334 KA_TRACE(20, (
"__kmp_teams_master: T#%d, Tid %d, microtask %p\n", gtid,
7335 __kmp_tid_from_gtid(gtid), thr->th.th_teams_microtask));
7338 kmp_cg_root_t *tmp = (kmp_cg_root_t *)__kmp_allocate(
sizeof(kmp_cg_root_t));
7341 tmp->cg_thread_limit = thr->th.th_current_task->td_icvs.thread_limit;
7342 tmp->cg_nthreads = 1;
7343 KA_TRACE(100, (
"__kmp_teams_master: Thread %p created node %p and init"
7344 " cg_nthreads to 1\n",
7346 tmp->up = thr->th.th_cg_roots;
7347 thr->th.th_cg_roots = tmp;
7351 #if INCLUDE_SSC_MARKS
7354 __kmp_fork_call(loc, gtid, fork_context_intel, team->t.t_argc,
7355 (microtask_t)thr->th.th_teams_microtask,
7356 VOLATILE_CAST(launch_t) __kmp_invoke_task_func, NULL);
7357 #if INCLUDE_SSC_MARKS
7361 if (thr->th.th_team_nproc < thr->th.th_teams_size.nth)
7362 thr->th.th_teams_size.nth = thr->th.th_team_nproc;
7365 __kmp_join_call(loc, gtid
7374 int __kmp_invoke_teams_master(
int gtid) {
7375 kmp_info_t *this_thr = __kmp_threads[gtid];
7376 kmp_team_t *team = this_thr->th.th_team;
7378 if (!__kmp_threads[gtid]->th.th_team->t.t_serialized)
7379 KMP_DEBUG_ASSERT((
void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn ==
7380 (
void *)__kmp_teams_master);
7382 __kmp_run_before_invoked_task(gtid, 0, this_thr, team);
7384 int tid = __kmp_tid_from_gtid(gtid);
7385 ompt_data_t *task_data =
7386 &team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data;
7387 ompt_data_t *parallel_data = &team->t.ompt_team_info.parallel_data;
7388 if (ompt_enabled.ompt_callback_implicit_task) {
7389 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
7390 ompt_scope_begin, parallel_data, task_data, team->t.t_nproc, tid,
7392 OMPT_CUR_TASK_INFO(this_thr)->thread_num = tid;
7395 __kmp_teams_master(gtid);
7397 this_thr->th.ompt_thread_info.parallel_flags |= ompt_parallel_league;
7399 __kmp_run_after_invoked_task(gtid, 0, this_thr, team);
7408 void __kmp_push_num_threads(
ident_t *
id,
int gtid,
int num_threads) {
7409 kmp_info_t *thr = __kmp_threads[gtid];
7411 if (num_threads > 0)
7412 thr->th.th_set_nproc = num_threads;
7417 void __kmp_push_num_teams(
ident_t *
id,
int gtid,
int num_teams,
7419 kmp_info_t *thr = __kmp_threads[gtid];
7420 KMP_DEBUG_ASSERT(num_teams >= 0);
7421 KMP_DEBUG_ASSERT(num_threads >= 0);
7425 if (num_teams > __kmp_teams_max_nth) {
7426 if (!__kmp_reserve_warn) {
7427 __kmp_reserve_warn = 1;
7428 __kmp_msg(kmp_ms_warning,
7429 KMP_MSG(CantFormThrTeam, num_teams, __kmp_teams_max_nth),
7430 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
7432 num_teams = __kmp_teams_max_nth;
7436 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams;
7439 if (!TCR_4(__kmp_init_middle))
7440 __kmp_middle_initialize();
7441 KMP_DEBUG_ASSERT(__kmp_avail_proc);
7442 KMP_DEBUG_ASSERT(__kmp_dflt_team_nth);
7443 if (num_threads == 0) {
7444 num_threads = __kmp_avail_proc / num_teams;
7448 if (num_threads > __kmp_dflt_team_nth) {
7449 num_threads = __kmp_dflt_team_nth;
7451 if (num_threads > thr->th.th_current_task->td_icvs.thread_limit) {
7452 num_threads = thr->th.th_current_task->td_icvs.thread_limit;
7454 if (num_teams * num_threads > __kmp_teams_max_nth) {
7455 num_threads = __kmp_teams_max_nth / num_teams;
7460 thr->th.th_current_task->td_icvs.thread_limit = num_threads;
7462 if (num_threads > __kmp_dflt_team_nth) {
7463 num_threads = __kmp_dflt_team_nth;
7465 if (num_teams * num_threads > __kmp_teams_max_nth) {
7466 int new_threads = __kmp_teams_max_nth / num_teams;
7467 if (!__kmp_reserve_warn) {
7468 __kmp_reserve_warn = 1;
7469 __kmp_msg(kmp_ms_warning,
7470 KMP_MSG(CantFormThrTeam, num_threads, new_threads),
7471 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
7473 num_threads = new_threads;
7476 thr->th.th_teams_size.nth = num_threads;
7480 void __kmp_push_proc_bind(
ident_t *
id,
int gtid, kmp_proc_bind_t proc_bind) {
7481 kmp_info_t *thr = __kmp_threads[gtid];
7482 thr->th.th_set_proc_bind = proc_bind;
7487 void __kmp_internal_fork(
ident_t *
id,
int gtid, kmp_team_t *team) {
7488 kmp_info_t *this_thr = __kmp_threads[gtid];
7494 KMP_DEBUG_ASSERT(team);
7495 KMP_DEBUG_ASSERT(this_thr->th.th_team == team);
7496 KMP_ASSERT(KMP_MASTER_GTID(gtid));
7499 team->t.t_construct = 0;
7500 team->t.t_ordered.dt.t_value =
7504 KMP_DEBUG_ASSERT(team->t.t_disp_buffer);
7505 if (team->t.t_max_nproc > 1) {
7507 for (i = 0; i < __kmp_dispatch_num_buffers; ++i) {
7508 team->t.t_disp_buffer[i].buffer_index = i;
7509 team->t.t_disp_buffer[i].doacross_buf_idx = i;
7512 team->t.t_disp_buffer[0].buffer_index = 0;
7513 team->t.t_disp_buffer[0].doacross_buf_idx = 0;
7517 KMP_ASSERT(this_thr->th.th_team == team);
7520 for (f = 0; f < team->t.t_nproc; f++) {
7521 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
7522 team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc);
7527 __kmp_fork_barrier(gtid, 0);
7530 void __kmp_internal_join(
ident_t *
id,
int gtid, kmp_team_t *team) {
7531 kmp_info_t *this_thr = __kmp_threads[gtid];
7533 KMP_DEBUG_ASSERT(team);
7534 KMP_DEBUG_ASSERT(this_thr->th.th_team == team);
7535 KMP_ASSERT(KMP_MASTER_GTID(gtid));
7541 if (__kmp_threads[gtid] &&
7542 __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc) {
7543 __kmp_printf(
"GTID: %d, __kmp_threads[%d]=%p\n", gtid, gtid,
7544 __kmp_threads[gtid]);
7545 __kmp_printf(
"__kmp_threads[%d]->th.th_team_nproc=%d, TEAM: %p, "
7546 "team->t.t_nproc=%d\n",
7547 gtid, __kmp_threads[gtid]->th.th_team_nproc, team,
7549 __kmp_print_structure();
7551 KMP_DEBUG_ASSERT(__kmp_threads[gtid] &&
7552 __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc);
7555 __kmp_join_barrier(gtid);
7557 if (ompt_enabled.enabled &&
7558 this_thr->th.ompt_thread_info.state == ompt_state_wait_barrier_implicit) {
7559 int ds_tid = this_thr->th.th_info.ds.ds_tid;
7560 ompt_data_t *task_data = OMPT_CUR_TASK_DATA(this_thr);
7561 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
7563 void *codeptr = NULL;
7564 if (KMP_MASTER_TID(ds_tid) &&
7565 (ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait) ||
7566 ompt_callbacks.ompt_callback(ompt_callback_sync_region)))
7567 codeptr = OMPT_CUR_TEAM_INFO(this_thr)->master_return_address;
7569 if (ompt_enabled.ompt_callback_sync_region_wait) {
7570 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
7571 ompt_sync_region_barrier_implicit, ompt_scope_end, NULL, task_data,
7574 if (ompt_enabled.ompt_callback_sync_region) {
7575 ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
7576 ompt_sync_region_barrier_implicit, ompt_scope_end, NULL, task_data,
7580 if (!KMP_MASTER_TID(ds_tid) && ompt_enabled.ompt_callback_implicit_task) {
7581 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
7582 ompt_scope_end, NULL, task_data, 0, ds_tid, ompt_task_implicit);
7588 KMP_ASSERT(this_thr->th.th_team == team);
7593 #ifdef USE_LOAD_BALANCE
7597 static int __kmp_active_hot_team_nproc(kmp_root_t *root) {
7600 kmp_team_t *hot_team;
7602 if (root->r.r_active) {
7605 hot_team = root->r.r_hot_team;
7606 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
7607 return hot_team->t.t_nproc - 1;
7612 for (i = 1; i < hot_team->t.t_nproc; i++) {
7613 if (hot_team->t.t_threads[i]->th.th_active) {
7622 static int __kmp_load_balance_nproc(kmp_root_t *root,
int set_nproc) {
7625 int hot_team_active;
7626 int team_curr_active;
7629 KB_TRACE(20, (
"__kmp_load_balance_nproc: called root:%p set_nproc:%d\n", root,
7631 KMP_DEBUG_ASSERT(root);
7632 KMP_DEBUG_ASSERT(root->r.r_root_team->t.t_threads[0]
7633 ->th.th_current_task->td_icvs.dynamic == TRUE);
7634 KMP_DEBUG_ASSERT(set_nproc > 1);
7636 if (set_nproc == 1) {
7637 KB_TRACE(20, (
"__kmp_load_balance_nproc: serial execution.\n"));
7646 pool_active = __kmp_thread_pool_active_nth;
7647 hot_team_active = __kmp_active_hot_team_nproc(root);
7648 team_curr_active = pool_active + hot_team_active + 1;
7651 system_active = __kmp_get_load_balance(__kmp_avail_proc + team_curr_active);
7652 KB_TRACE(30, (
"__kmp_load_balance_nproc: system active = %d pool active = %d "
7653 "hot team active = %d\n",
7654 system_active, pool_active, hot_team_active));
7656 if (system_active < 0) {
7660 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit;
7661 KMP_WARNING(CantLoadBalUsing,
"KMP_DYNAMIC_MODE=thread limit");
7664 retval = __kmp_avail_proc - __kmp_nth +
7665 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
7666 if (retval > set_nproc) {
7669 if (retval < KMP_MIN_NTH) {
7670 retval = KMP_MIN_NTH;
7673 KB_TRACE(20, (
"__kmp_load_balance_nproc: thread limit exit. retval:%d\n",
7681 if (system_active < team_curr_active) {
7682 system_active = team_curr_active;
7684 retval = __kmp_avail_proc - system_active + team_curr_active;
7685 if (retval > set_nproc) {
7688 if (retval < KMP_MIN_NTH) {
7689 retval = KMP_MIN_NTH;
7692 KB_TRACE(20, (
"__kmp_load_balance_nproc: exit. retval:%d\n", retval));
7701 void __kmp_cleanup(
void) {
7704 KA_TRACE(10, (
"__kmp_cleanup: enter\n"));
7706 if (TCR_4(__kmp_init_parallel)) {
7707 #if KMP_HANDLE_SIGNALS
7708 __kmp_remove_signals();
7710 TCW_4(__kmp_init_parallel, FALSE);
7713 if (TCR_4(__kmp_init_middle)) {
7714 #if KMP_AFFINITY_SUPPORTED
7715 __kmp_affinity_uninitialize();
7717 __kmp_cleanup_hierarchy();
7718 TCW_4(__kmp_init_middle, FALSE);
7721 KA_TRACE(10, (
"__kmp_cleanup: go serial cleanup\n"));
7723 if (__kmp_init_serial) {
7724 __kmp_runtime_destroy();
7725 __kmp_init_serial = FALSE;
7728 __kmp_cleanup_threadprivate_caches();
7730 for (f = 0; f < __kmp_threads_capacity; f++) {
7731 if (__kmp_root[f] != NULL) {
7732 __kmp_free(__kmp_root[f]);
7733 __kmp_root[f] = NULL;
7736 __kmp_free(__kmp_threads);
7739 __kmp_threads = NULL;
7741 __kmp_threads_capacity = 0;
7743 #if KMP_USE_DYNAMIC_LOCK
7744 __kmp_cleanup_indirect_user_locks();
7746 __kmp_cleanup_user_locks();
7749 #if KMP_AFFINITY_SUPPORTED
7750 KMP_INTERNAL_FREE(CCAST(
char *, __kmp_cpuinfo_file));
7751 __kmp_cpuinfo_file = NULL;
7754 #if KMP_USE_ADAPTIVE_LOCKS
7755 #if KMP_DEBUG_ADAPTIVE_LOCKS
7756 __kmp_print_speculative_stats();
7759 KMP_INTERNAL_FREE(__kmp_nested_nth.nth);
7760 __kmp_nested_nth.nth = NULL;
7761 __kmp_nested_nth.size = 0;
7762 __kmp_nested_nth.used = 0;
7763 KMP_INTERNAL_FREE(__kmp_nested_proc_bind.bind_types);
7764 __kmp_nested_proc_bind.bind_types = NULL;
7765 __kmp_nested_proc_bind.size = 0;
7766 __kmp_nested_proc_bind.used = 0;
7767 if (__kmp_affinity_format) {
7768 KMP_INTERNAL_FREE(__kmp_affinity_format);
7769 __kmp_affinity_format = NULL;
7772 __kmp_i18n_catclose();
7774 #if KMP_USE_HIER_SCHED
7775 __kmp_hier_scheds.deallocate();
7778 #if KMP_STATS_ENABLED
7782 KA_TRACE(10, (
"__kmp_cleanup: exit\n"));
7787 int __kmp_ignore_mppbeg(
void) {
7790 if ((env = getenv(
"KMP_IGNORE_MPPBEG")) != NULL) {
7791 if (__kmp_str_match_false(env))
7798 int __kmp_ignore_mppend(
void) {
7801 if ((env = getenv(
"KMP_IGNORE_MPPEND")) != NULL) {
7802 if (__kmp_str_match_false(env))
7809 void __kmp_internal_begin(
void) {
7815 gtid = __kmp_entry_gtid();
7816 root = __kmp_threads[gtid]->th.th_root;
7817 KMP_ASSERT(KMP_UBER_GTID(gtid));
7819 if (root->r.r_begin)
7821 __kmp_acquire_lock(&root->r.r_begin_lock, gtid);
7822 if (root->r.r_begin) {
7823 __kmp_release_lock(&root->r.r_begin_lock, gtid);
7827 root->r.r_begin = TRUE;
7829 __kmp_release_lock(&root->r.r_begin_lock, gtid);
7834 void __kmp_user_set_library(
enum library_type arg) {
7841 gtid = __kmp_entry_gtid();
7842 thread = __kmp_threads[gtid];
7844 root = thread->th.th_root;
7846 KA_TRACE(20, (
"__kmp_user_set_library: enter T#%d, arg: %d, %d\n", gtid, arg,
7848 if (root->r.r_in_parallel) {
7850 KMP_WARNING(SetLibraryIncorrectCall);
7855 case library_serial:
7856 thread->th.th_set_nproc = 0;
7857 set__nproc(thread, 1);
7859 case library_turnaround:
7860 thread->th.th_set_nproc = 0;
7861 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth
7862 : __kmp_dflt_team_nth_ub);
7864 case library_throughput:
7865 thread->th.th_set_nproc = 0;
7866 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth
7867 : __kmp_dflt_team_nth_ub);
7870 KMP_FATAL(UnknownLibraryType, arg);
7873 __kmp_aux_set_library(arg);
7876 void __kmp_aux_set_stacksize(
size_t arg) {
7877 if (!__kmp_init_serial)
7878 __kmp_serial_initialize();
7881 if (arg & (0x1000 - 1)) {
7882 arg &= ~(0x1000 - 1);
7887 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
7890 if (!TCR_4(__kmp_init_parallel)) {
7893 if (value < __kmp_sys_min_stksize)
7894 value = __kmp_sys_min_stksize;
7895 else if (value > KMP_MAX_STKSIZE)
7896 value = KMP_MAX_STKSIZE;
7898 __kmp_stksize = value;
7900 __kmp_env_stksize = TRUE;
7903 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7908 void __kmp_aux_set_library(
enum library_type arg) {
7909 __kmp_library = arg;
7911 switch (__kmp_library) {
7912 case library_serial: {
7913 KMP_INFORM(LibraryIsSerial);
7915 case library_turnaround:
7916 if (__kmp_use_yield == 1 && !__kmp_use_yield_exp_set)
7917 __kmp_use_yield = 2;
7919 case library_throughput:
7920 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME)
7921 __kmp_dflt_blocktime = 200;
7924 KMP_FATAL(UnknownLibraryType, arg);
7930 static kmp_team_t *__kmp_aux_get_team_info(
int &teams_serialized) {
7931 kmp_info_t *thr = __kmp_entry_thread();
7932 teams_serialized = 0;
7933 if (thr->th.th_teams_microtask) {
7934 kmp_team_t *team = thr->th.th_team;
7935 int tlevel = thr->th.th_teams_level;
7936 int ii = team->t.t_level;
7937 teams_serialized = team->t.t_serialized;
7938 int level = tlevel + 1;
7939 KMP_DEBUG_ASSERT(ii >= tlevel);
7940 while (ii > level) {
7941 for (teams_serialized = team->t.t_serialized;
7942 (teams_serialized > 0) && (ii > level); teams_serialized--, ii--) {
7944 if (team->t.t_serialized && (!teams_serialized)) {
7945 team = team->t.t_parent;
7949 team = team->t.t_parent;
7958 int __kmp_aux_get_team_num() {
7960 kmp_team_t *team = __kmp_aux_get_team_info(serialized);
7962 if (serialized > 1) {
7965 return team->t.t_master_tid;
7971 int __kmp_aux_get_num_teams() {
7973 kmp_team_t *team = __kmp_aux_get_team_info(serialized);
7975 if (serialized > 1) {
7978 return team->t.t_parent->t.t_nproc;
8017 typedef struct kmp_affinity_format_field_t {
8019 const char *long_name;
8022 } kmp_affinity_format_field_t;
8024 static const kmp_affinity_format_field_t __kmp_affinity_format_table[] = {
8025 #if KMP_AFFINITY_SUPPORTED
8026 {
'A',
"thread_affinity",
's'},
8028 {
't',
"team_num",
'd'},
8029 {
'T',
"num_teams",
'd'},
8030 {
'L',
"nesting_level",
'd'},
8031 {
'n',
"thread_num",
'd'},
8032 {
'N',
"num_threads",
'd'},
8033 {
'a',
"ancestor_tnum",
'd'},
8035 {
'P',
"process_id",
'd'},
8036 {
'i',
"native_thread_id",
'd'}};
8039 static int __kmp_aux_capture_affinity_field(
int gtid,
const kmp_info_t *th,
8041 kmp_str_buf_t *field_buffer) {
8042 int rc, format_index, field_value;
8043 const char *width_left, *width_right;
8044 bool pad_zeros, right_justify, parse_long_name, found_valid_name;
8045 static const int FORMAT_SIZE = 20;
8046 char format[FORMAT_SIZE] = {0};
8047 char absolute_short_name = 0;
8049 KMP_DEBUG_ASSERT(gtid >= 0);
8050 KMP_DEBUG_ASSERT(th);
8051 KMP_DEBUG_ASSERT(**ptr ==
'%');
8052 KMP_DEBUG_ASSERT(field_buffer);
8054 __kmp_str_buf_clear(field_buffer);
8061 __kmp_str_buf_cat(field_buffer,
"%", 1);
8072 right_justify =
false;
8074 right_justify =
true;
8078 width_left = width_right = NULL;
8079 if (**ptr >=
'0' && **ptr <=
'9') {
8087 format[format_index++] =
'%';
8089 format[format_index++] =
'-';
8091 format[format_index++] =
'0';
8092 if (width_left && width_right) {
8096 while (i < 8 && width_left < width_right) {
8097 format[format_index++] = *width_left;
8105 found_valid_name =
false;
8106 parse_long_name = (**ptr ==
'{');
8107 if (parse_long_name)
8109 for (
size_t i = 0; i <
sizeof(__kmp_affinity_format_table) /
8110 sizeof(__kmp_affinity_format_table[0]);
8112 char short_name = __kmp_affinity_format_table[i].short_name;
8113 const char *long_name = __kmp_affinity_format_table[i].long_name;
8114 char field_format = __kmp_affinity_format_table[i].field_format;
8115 if (parse_long_name) {
8116 size_t length = KMP_STRLEN(long_name);
8117 if (strncmp(*ptr, long_name, length) == 0) {
8118 found_valid_name =
true;
8121 }
else if (**ptr == short_name) {
8122 found_valid_name =
true;
8125 if (found_valid_name) {
8126 format[format_index++] = field_format;
8127 format[format_index++] =
'\0';
8128 absolute_short_name = short_name;
8132 if (parse_long_name) {
8134 absolute_short_name = 0;
8142 switch (absolute_short_name) {
8144 rc = __kmp_str_buf_print(field_buffer, format, __kmp_aux_get_team_num());
8147 rc = __kmp_str_buf_print(field_buffer, format, __kmp_aux_get_num_teams());
8150 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_level);
8153 rc = __kmp_str_buf_print(field_buffer, format, __kmp_tid_from_gtid(gtid));
8156 static const int BUFFER_SIZE = 256;
8157 char buf[BUFFER_SIZE];
8158 __kmp_expand_host_name(buf, BUFFER_SIZE);
8159 rc = __kmp_str_buf_print(field_buffer, format, buf);
8162 rc = __kmp_str_buf_print(field_buffer, format, getpid());
8165 rc = __kmp_str_buf_print(field_buffer, format, __kmp_gettid());
8168 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_nproc);
8172 __kmp_get_ancestor_thread_num(gtid, th->th.th_team->t.t_level - 1);
8173 rc = __kmp_str_buf_print(field_buffer, format, field_value);
8175 #if KMP_AFFINITY_SUPPORTED
8178 __kmp_str_buf_init(&buf);
8179 __kmp_affinity_str_buf_mask(&buf, th->th.th_affin_mask);
8180 rc = __kmp_str_buf_print(field_buffer, format, buf.str);
8181 __kmp_str_buf_free(&buf);
8187 rc = __kmp_str_buf_print(field_buffer,
"%s",
"undefined");
8189 if (parse_long_name) {
8198 KMP_ASSERT(format_index <= FORMAT_SIZE);
8208 size_t __kmp_aux_capture_affinity(
int gtid,
const char *format,
8209 kmp_str_buf_t *buffer) {
8210 const char *parse_ptr;
8212 const kmp_info_t *th;
8213 kmp_str_buf_t field;
8215 KMP_DEBUG_ASSERT(buffer);
8216 KMP_DEBUG_ASSERT(gtid >= 0);
8218 __kmp_str_buf_init(&field);
8219 __kmp_str_buf_clear(buffer);
8221 th = __kmp_threads[gtid];
8227 if (parse_ptr == NULL || *parse_ptr ==
'\0') {
8228 parse_ptr = __kmp_affinity_format;
8230 KMP_DEBUG_ASSERT(parse_ptr);
8232 while (*parse_ptr !=
'\0') {
8234 if (*parse_ptr ==
'%') {
8236 int rc = __kmp_aux_capture_affinity_field(gtid, th, &parse_ptr, &field);
8237 __kmp_str_buf_catbuf(buffer, &field);
8241 __kmp_str_buf_cat(buffer, parse_ptr, 1);
8246 __kmp_str_buf_free(&field);
8251 void __kmp_aux_display_affinity(
int gtid,
const char *format) {
8253 __kmp_str_buf_init(&buf);
8254 __kmp_aux_capture_affinity(gtid, format, &buf);
8255 __kmp_fprintf(kmp_out,
"%s" KMP_END_OF_LINE, buf.str);
8256 __kmp_str_buf_free(&buf);
8261 void __kmp_aux_set_blocktime(
int arg, kmp_info_t *thread,
int tid) {
8262 int blocktime = arg;
8268 __kmp_save_internal_controls(thread);
8271 if (blocktime < KMP_MIN_BLOCKTIME)
8272 blocktime = KMP_MIN_BLOCKTIME;
8273 else if (blocktime > KMP_MAX_BLOCKTIME)
8274 blocktime = KMP_MAX_BLOCKTIME;
8276 set__blocktime_team(thread->th.th_team, tid, blocktime);
8277 set__blocktime_team(thread->th.th_serial_team, 0, blocktime);
8281 bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME(blocktime, __kmp_monitor_wakeups);
8283 set__bt_intervals_team(thread->th.th_team, tid, bt_intervals);
8284 set__bt_intervals_team(thread->th.th_serial_team, 0, bt_intervals);
8290 set__bt_set_team(thread->th.th_team, tid, bt_set);
8291 set__bt_set_team(thread->th.th_serial_team, 0, bt_set);
8293 KF_TRACE(10, (
"kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, "
8294 "bt_intervals=%d, monitor_updates=%d\n",
8295 __kmp_gtid_from_tid(tid, thread->th.th_team),
8296 thread->th.th_team->t.t_id, tid, blocktime, bt_intervals,
8297 __kmp_monitor_wakeups));
8299 KF_TRACE(10, (
"kmp_set_blocktime: T#%d(%d:%d), blocktime=%d\n",
8300 __kmp_gtid_from_tid(tid, thread->th.th_team),
8301 thread->th.th_team->t.t_id, tid, blocktime));
8305 void __kmp_aux_set_defaults(
char const *str,
size_t len) {
8306 if (!__kmp_init_serial) {
8307 __kmp_serial_initialize();
8309 __kmp_env_initialize(str);
8311 if (__kmp_settings || __kmp_display_env || __kmp_display_env_verbose) {
8319 PACKED_REDUCTION_METHOD_T
8320 __kmp_determine_reduction_method(
8321 ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars,
size_t reduce_size,
8322 void *reduce_data,
void (*reduce_func)(
void *lhs_data,
void *rhs_data),
8323 kmp_critical_name *lck) {
8334 PACKED_REDUCTION_METHOD_T retval;
8338 KMP_DEBUG_ASSERT(loc);
8339 KMP_DEBUG_ASSERT(lck);
8341 #define FAST_REDUCTION_ATOMIC_METHOD_GENERATED \
8342 ((loc->flags & (KMP_IDENT_ATOMIC_REDUCE)) == (KMP_IDENT_ATOMIC_REDUCE))
8343 #define FAST_REDUCTION_TREE_METHOD_GENERATED ((reduce_data) && (reduce_func))
8345 retval = critical_reduce_block;
8348 team_size = __kmp_get_team_num_threads(global_tid);
8349 if (team_size == 1) {
8351 retval = empty_reduce_block;
8355 int atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;
8357 #if KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || \
8358 KMP_ARCH_MIPS64 || KMP_ARCH_RISCV64
8360 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
8361 KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN || KMP_OS_HURD
8363 int teamsize_cutoff = 4;
8365 #if KMP_MIC_SUPPORTED
8366 if (__kmp_mic_type != non_mic) {
8367 teamsize_cutoff = 8;
8370 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
8371 if (tree_available) {
8372 if (team_size <= teamsize_cutoff) {
8373 if (atomic_available) {
8374 retval = atomic_reduce_block;
8377 retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER;
8379 }
else if (atomic_available) {
8380 retval = atomic_reduce_block;
8383 #error "Unknown or unsupported OS"
8384 #endif // KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD ||
8387 #elif KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_AARCH || KMP_ARCH_MIPS
8389 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_WINDOWS || KMP_OS_HURD
8393 if (atomic_available) {
8394 if (num_vars <= 2) {
8395 retval = atomic_reduce_block;
8401 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
8402 if (atomic_available && (num_vars <= 3)) {
8403 retval = atomic_reduce_block;
8404 }
else if (tree_available) {
8405 if ((reduce_size > (9 *
sizeof(kmp_real64))) &&
8406 (reduce_size < (2000 *
sizeof(kmp_real64)))) {
8407 retval = TREE_REDUCE_BLOCK_WITH_PLAIN_BARRIER;
8412 #error "Unknown or unsupported OS"
8416 #error "Unknown or unsupported architecture"
8424 if (__kmp_force_reduction_method != reduction_method_not_defined &&
8427 PACKED_REDUCTION_METHOD_T forced_retval = critical_reduce_block;
8429 int atomic_available, tree_available;
8431 switch ((forced_retval = __kmp_force_reduction_method)) {
8432 case critical_reduce_block:
8436 case atomic_reduce_block:
8437 atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;
8438 if (!atomic_available) {
8439 KMP_WARNING(RedMethodNotSupported,
"atomic");
8440 forced_retval = critical_reduce_block;
8444 case tree_reduce_block:
8445 tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
8446 if (!tree_available) {
8447 KMP_WARNING(RedMethodNotSupported,
"tree");
8448 forced_retval = critical_reduce_block;
8450 #if KMP_FAST_REDUCTION_BARRIER
8451 forced_retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER;
8460 retval = forced_retval;
8463 KA_TRACE(10, (
"reduction method selected=%08x\n", retval));
8465 #undef FAST_REDUCTION_TREE_METHOD_GENERATED
8466 #undef FAST_REDUCTION_ATOMIC_METHOD_GENERATED
8471 kmp_int32 __kmp_get_reduce_method(
void) {
8472 return ((__kmp_entry_thread()->th.th_local.packed_reduction_method) >> 8);
8477 void __kmp_soft_pause() { __kmp_pause_status = kmp_soft_paused; }
8481 void __kmp_hard_pause() {
8482 __kmp_pause_status = kmp_hard_paused;
8483 __kmp_internal_end_thread(-1);
8487 void __kmp_resume_if_soft_paused() {
8488 if (__kmp_pause_status == kmp_soft_paused) {
8489 __kmp_pause_status = kmp_not_paused;
8491 for (
int gtid = 1; gtid < __kmp_threads_capacity; ++gtid) {
8492 kmp_info_t *thread = __kmp_threads[gtid];
8494 kmp_flag_64<> fl(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go,
8496 if (fl.is_sleeping())
8498 else if (__kmp_try_suspend_mx(thread)) {
8499 __kmp_unlock_suspend_mx(thread);
8502 if (fl.is_sleeping()) {
8505 }
else if (__kmp_try_suspend_mx(thread)) {
8506 __kmp_unlock_suspend_mx(thread);
8518 int __kmp_pause_resource(kmp_pause_status_t level) {
8519 if (level == kmp_not_paused) {
8520 if (__kmp_pause_status == kmp_not_paused) {
8524 KMP_DEBUG_ASSERT(__kmp_pause_status == kmp_soft_paused ||
8525 __kmp_pause_status == kmp_hard_paused);
8526 __kmp_pause_status = kmp_not_paused;
8529 }
else if (level == kmp_soft_paused) {
8530 if (__kmp_pause_status != kmp_not_paused) {
8537 }
else if (level == kmp_hard_paused) {
8538 if (__kmp_pause_status != kmp_not_paused) {
8551 void __kmp_omp_display_env(
int verbose) {
8552 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
8553 if (__kmp_init_serial == 0)
8554 __kmp_do_serial_initialize();
8555 __kmp_display_env_impl(!verbose, verbose);
8556 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
8560 kmp_info_t **__kmp_hidden_helper_threads;
8561 kmp_info_t *__kmp_hidden_helper_main_thread;
8562 kmp_int32 __kmp_hidden_helper_threads_num = 8;
8563 std::atomic<kmp_int32> __kmp_unexecuted_hidden_helper_tasks;
8565 kmp_int32 __kmp_enable_hidden_helper = TRUE;
8567 kmp_int32 __kmp_enable_hidden_helper = FALSE;
8571 std::atomic<kmp_int32> __kmp_hit_hidden_helper_threads_num;
8573 void __kmp_hidden_helper_wrapper_fn(
int *gtid,
int *, ...) {
8578 KMP_ATOMIC_INC(&__kmp_hit_hidden_helper_threads_num);
8579 while (KMP_ATOMIC_LD_ACQ(&__kmp_hit_hidden_helper_threads_num) !=
8580 __kmp_hidden_helper_threads_num)
8586 TCW_4(__kmp_init_hidden_helper_threads, FALSE);
8587 __kmp_hidden_helper_initz_release();
8588 __kmp_hidden_helper_main_thread_wait();
8590 for (
int i = 1; i < __kmp_hit_hidden_helper_threads_num; ++i) {
8591 __kmp_hidden_helper_worker_thread_signal();
8597 void __kmp_hidden_helper_threads_initz_routine() {
8599 const int gtid = __kmp_register_root(TRUE);
8600 __kmp_hidden_helper_main_thread = __kmp_threads[gtid];
8601 __kmp_hidden_helper_threads = &__kmp_threads[gtid];
8602 __kmp_hidden_helper_main_thread->th.th_set_nproc =
8603 __kmp_hidden_helper_threads_num;
8605 KMP_ATOMIC_ST_REL(&__kmp_hit_hidden_helper_threads_num, 0);
8610 TCW_SYNC_4(__kmp_init_hidden_helper, FALSE);
8612 __kmp_hidden_helper_threads_deinitz_release();