16 #include "kmp_stats.h"
17 #include "kmp_wait_release.h"
18 #include "kmp_taskdeps.h"
21 #include "ompt-specific.h"
24 #if ENABLE_LIBOMPTARGET
25 static void (*tgt_target_nowait_query)(
void **);
27 void __kmp_init_target_task() {
28 *(
void **)(&tgt_target_nowait_query) = KMP_DLSYM(
"__tgt_target_nowait_query");
33 static void __kmp_enable_tasking(kmp_task_team_t *task_team,
34 kmp_info_t *this_thr);
35 static void __kmp_alloc_task_deque(kmp_info_t *thread,
36 kmp_thread_data_t *thread_data);
37 static int __kmp_realloc_task_threads_data(kmp_info_t *thread,
38 kmp_task_team_t *task_team);
39 static void __kmp_bottom_half_finish_proxy(kmp_int32 gtid, kmp_task_t *ptask);
41 static kmp_tdg_info_t *__kmp_find_tdg(kmp_int32 tdg_id);
42 int __kmp_taskloop_task(
int gtid,
void *ptask);
48 static bool __kmp_task_is_allowed(
int gtid,
const kmp_int32 is_constrained,
49 const kmp_taskdata_t *tasknew,
50 const kmp_taskdata_t *taskcurr) {
51 if (is_constrained && (tasknew->td_flags.tiedness == TASK_TIED)) {
55 kmp_taskdata_t *current = taskcurr->td_last_tied;
56 KMP_DEBUG_ASSERT(current != NULL);
58 if (current->td_flags.tasktype == TASK_EXPLICIT ||
59 current->td_taskwait_thread > 0) {
60 kmp_int32 level = current->td_level;
61 kmp_taskdata_t *parent = tasknew->td_parent;
62 while (parent != current && parent->td_level > level) {
64 parent = parent->td_parent;
65 KMP_DEBUG_ASSERT(parent != NULL);
67 if (parent != current)
72 kmp_depnode_t *node = tasknew->td_depnode;
74 if (!tasknew->is_taskgraph && UNLIKELY(node && (node->dn.mtx_num_locks > 0))) {
76 if (UNLIKELY(node && (node->dn.mtx_num_locks > 0))) {
78 for (
int i = 0; i < node->dn.mtx_num_locks; ++i) {
79 KMP_DEBUG_ASSERT(node->dn.mtx_locks[i] != NULL);
80 if (__kmp_test_lock(node->dn.mtx_locks[i], gtid))
83 for (
int j = i - 1; j >= 0; --j)
84 __kmp_release_lock(node->dn.mtx_locks[j], gtid);
88 node->dn.mtx_num_locks = -node->dn.mtx_num_locks;
97 static void __kmp_realloc_task_deque(kmp_info_t *thread,
98 kmp_thread_data_t *thread_data) {
99 kmp_int32 size = TASK_DEQUE_SIZE(thread_data->td);
100 KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) == size);
101 kmp_int32 new_size = 2 * size;
103 KE_TRACE(10, (
"__kmp_realloc_task_deque: T#%d reallocating deque[from %d to "
104 "%d] for thread_data %p\n",
105 __kmp_gtid_from_thread(thread), size, new_size, thread_data));
107 kmp_taskdata_t **new_deque =
108 (kmp_taskdata_t **)__kmp_allocate(new_size *
sizeof(kmp_taskdata_t *));
111 for (i = thread_data->td.td_deque_head, j = 0; j < size;
112 i = (i + 1) & TASK_DEQUE_MASK(thread_data->td), j++)
113 new_deque[j] = thread_data->td.td_deque[i];
115 __kmp_free(thread_data->td.td_deque);
117 thread_data->td.td_deque_head = 0;
118 thread_data->td.td_deque_tail = size;
119 thread_data->td.td_deque = new_deque;
120 thread_data->td.td_deque_size = new_size;
123 static kmp_task_pri_t *__kmp_alloc_task_pri_list() {
124 kmp_task_pri_t *l = (kmp_task_pri_t *)__kmp_allocate(
sizeof(kmp_task_pri_t));
125 kmp_thread_data_t *thread_data = &l->td;
126 __kmp_init_bootstrap_lock(&thread_data->td.td_deque_lock);
127 thread_data->td.td_deque_last_stolen = -1;
128 KE_TRACE(20, (
"__kmp_alloc_task_pri_list: T#%d allocating deque[%d] "
129 "for thread_data %p\n",
130 __kmp_get_gtid(), INITIAL_TASK_DEQUE_SIZE, thread_data));
131 thread_data->td.td_deque = (kmp_taskdata_t **)__kmp_allocate(
132 INITIAL_TASK_DEQUE_SIZE *
sizeof(kmp_taskdata_t *));
133 thread_data->td.td_deque_size = INITIAL_TASK_DEQUE_SIZE;
142 static kmp_thread_data_t *
143 __kmp_get_priority_deque_data(kmp_task_team_t *task_team, kmp_int32 pri) {
144 kmp_thread_data_t *thread_data;
145 kmp_task_pri_t *lst = task_team->tt.tt_task_pri_list;
146 if (lst->priority == pri) {
148 thread_data = &lst->td;
149 }
else if (lst->priority < pri) {
152 kmp_task_pri_t *list = __kmp_alloc_task_pri_list();
153 thread_data = &list->td;
154 list->priority = pri;
156 task_team->tt.tt_task_pri_list = list;
158 kmp_task_pri_t *next_queue = lst->next;
159 while (next_queue && next_queue->priority > pri) {
161 next_queue = lst->next;
164 if (next_queue == NULL) {
166 kmp_task_pri_t *list = __kmp_alloc_task_pri_list();
167 thread_data = &list->td;
168 list->priority = pri;
171 }
else if (next_queue->priority == pri) {
173 thread_data = &next_queue->td;
176 kmp_task_pri_t *list = __kmp_alloc_task_pri_list();
177 thread_data = &list->td;
178 list->priority = pri;
179 list->next = next_queue;
187 static kmp_int32 __kmp_push_priority_task(kmp_int32 gtid, kmp_info_t *thread,
188 kmp_taskdata_t *taskdata,
189 kmp_task_team_t *task_team,
191 kmp_thread_data_t *thread_data = NULL;
193 (
"__kmp_push_priority_task: T#%d trying to push task %p, pri %d.\n",
194 gtid, taskdata, pri));
197 kmp_task_pri_t *lst = task_team->tt.tt_task_pri_list;
198 if (UNLIKELY(lst == NULL)) {
199 __kmp_acquire_bootstrap_lock(&task_team->tt.tt_task_pri_lock);
200 if (task_team->tt.tt_task_pri_list == NULL) {
202 kmp_task_pri_t *list = __kmp_alloc_task_pri_list();
203 thread_data = &list->td;
204 list->priority = pri;
206 task_team->tt.tt_task_pri_list = list;
209 thread_data = __kmp_get_priority_deque_data(task_team, pri);
211 __kmp_release_bootstrap_lock(&task_team->tt.tt_task_pri_lock);
213 if (lst->priority == pri) {
215 thread_data = &lst->td;
217 __kmp_acquire_bootstrap_lock(&task_team->tt.tt_task_pri_lock);
218 thread_data = __kmp_get_priority_deque_data(task_team, pri);
219 __kmp_release_bootstrap_lock(&task_team->tt.tt_task_pri_lock);
222 KMP_DEBUG_ASSERT(thread_data);
224 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
226 if (TCR_4(thread_data->td.td_deque_ntasks) >=
227 TASK_DEQUE_SIZE(thread_data->td)) {
228 if (__kmp_enable_task_throttling &&
229 __kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata,
230 thread->th.th_current_task)) {
231 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
232 KA_TRACE(20, (
"__kmp_push_priority_task: T#%d deque is full; returning "
233 "TASK_NOT_PUSHED for task %p\n",
235 return TASK_NOT_PUSHED;
238 __kmp_realloc_task_deque(thread, thread_data);
241 KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) <
242 TASK_DEQUE_SIZE(thread_data->td));
244 thread_data->td.td_deque[thread_data->td.td_deque_tail] = taskdata;
246 thread_data->td.td_deque_tail =
247 (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td);
248 TCW_4(thread_data->td.td_deque_ntasks,
249 TCR_4(thread_data->td.td_deque_ntasks) + 1);
250 KMP_FSYNC_RELEASING(thread->th.th_current_task);
251 KMP_FSYNC_RELEASING(taskdata);
252 KA_TRACE(20, (
"__kmp_push_priority_task: T#%d returning "
253 "TASK_SUCCESSFULLY_PUSHED: task=%p ntasks=%d head=%u tail=%u\n",
254 gtid, taskdata, thread_data->td.td_deque_ntasks,
255 thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
256 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
257 task_team->tt.tt_num_task_pri++;
258 return TASK_SUCCESSFULLY_PUSHED;
262 static kmp_int32 __kmp_push_task(kmp_int32 gtid, kmp_task_t *task) {
263 kmp_info_t *thread = __kmp_threads[gtid];
264 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
269 if (UNLIKELY(taskdata->td_flags.hidden_helper &&
270 !KMP_HIDDEN_HELPER_THREAD(gtid))) {
271 kmp_int32 shadow_gtid = KMP_GTID_TO_SHADOW_GTID(gtid);
272 __kmpc_give_task(task, __kmp_tid_from_gtid(shadow_gtid));
274 __kmp_hidden_helper_worker_thread_signal();
275 return TASK_SUCCESSFULLY_PUSHED;
278 kmp_task_team_t *task_team = thread->th.th_task_team;
279 kmp_int32 tid = __kmp_tid_from_gtid(gtid);
280 kmp_thread_data_t *thread_data;
283 (
"__kmp_push_task: T#%d trying to push task %p.\n", gtid, taskdata));
285 if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) {
288 kmp_int32 counter = 1 + KMP_ATOMIC_INC(&taskdata->td_untied_count);
289 KMP_DEBUG_USE_VAR(counter);
292 (
"__kmp_push_task: T#%d untied_count (%d) incremented for task %p\n",
293 gtid, counter, taskdata));
297 if (UNLIKELY(taskdata->td_flags.task_serial)) {
298 KA_TRACE(20, (
"__kmp_push_task: T#%d team serialized; returning "
299 "TASK_NOT_PUSHED for task %p\n",
301 return TASK_NOT_PUSHED;
306 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
307 if (UNLIKELY(!KMP_TASKING_ENABLED(task_team))) {
308 __kmp_enable_tasking(task_team, thread);
310 KMP_DEBUG_ASSERT(TCR_4(task_team->tt.tt_found_tasks) == TRUE);
311 KMP_DEBUG_ASSERT(TCR_PTR(task_team->tt.tt_threads_data) != NULL);
313 if (taskdata->td_flags.priority_specified && task->data2.priority > 0 &&
314 __kmp_max_task_priority > 0) {
315 int pri = KMP_MIN(task->data2.priority, __kmp_max_task_priority);
316 return __kmp_push_priority_task(gtid, thread, taskdata, task_team, pri);
320 thread_data = &task_team->tt.tt_threads_data[tid];
325 if (UNLIKELY(thread_data->td.td_deque == NULL)) {
326 __kmp_alloc_task_deque(thread, thread_data);
331 if (TCR_4(thread_data->td.td_deque_ntasks) >=
332 TASK_DEQUE_SIZE(thread_data->td)) {
333 if (__kmp_enable_task_throttling &&
334 __kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata,
335 thread->th.th_current_task)) {
336 KA_TRACE(20, (
"__kmp_push_task: T#%d deque is full; returning "
337 "TASK_NOT_PUSHED for task %p\n",
339 return TASK_NOT_PUSHED;
341 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
343 if (TCR_4(thread_data->td.td_deque_ntasks) >=
344 TASK_DEQUE_SIZE(thread_data->td)) {
346 __kmp_realloc_task_deque(thread, thread_data);
352 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
354 if (TCR_4(thread_data->td.td_deque_ntasks) >=
355 TASK_DEQUE_SIZE(thread_data->td)) {
356 if (__kmp_enable_task_throttling &&
357 __kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata,
358 thread->th.th_current_task)) {
359 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
360 KA_TRACE(20, (
"__kmp_push_task: T#%d deque is full on 2nd check; "
361 "returning TASK_NOT_PUSHED for task %p\n",
363 return TASK_NOT_PUSHED;
366 __kmp_realloc_task_deque(thread, thread_data);
371 KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) <
372 TASK_DEQUE_SIZE(thread_data->td));
374 thread_data->td.td_deque[thread_data->td.td_deque_tail] =
377 thread_data->td.td_deque_tail =
378 (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td);
379 TCW_4(thread_data->td.td_deque_ntasks,
380 TCR_4(thread_data->td.td_deque_ntasks) + 1);
381 KMP_FSYNC_RELEASING(thread->th.th_current_task);
382 KMP_FSYNC_RELEASING(taskdata);
383 KA_TRACE(20, (
"__kmp_push_task: T#%d returning TASK_SUCCESSFULLY_PUSHED: "
384 "task=%p ntasks=%d head=%u tail=%u\n",
385 gtid, taskdata, thread_data->td.td_deque_ntasks,
386 thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
388 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
390 return TASK_SUCCESSFULLY_PUSHED;
397 void __kmp_pop_current_task_from_thread(kmp_info_t *this_thr) {
398 KF_TRACE(10, (
"__kmp_pop_current_task_from_thread(enter): T#%d "
399 "this_thread=%p, curtask=%p, "
400 "curtask_parent=%p\n",
401 0, this_thr, this_thr->th.th_current_task,
402 this_thr->th.th_current_task->td_parent));
404 this_thr->th.th_current_task = this_thr->th.th_current_task->td_parent;
406 KF_TRACE(10, (
"__kmp_pop_current_task_from_thread(exit): T#%d "
407 "this_thread=%p, curtask=%p, "
408 "curtask_parent=%p\n",
409 0, this_thr, this_thr->th.th_current_task,
410 this_thr->th.th_current_task->td_parent));
419 void __kmp_push_current_task_to_thread(kmp_info_t *this_thr, kmp_team_t *team,
423 KF_TRACE(10, (
"__kmp_push_current_task_to_thread(enter): T#%d this_thread=%p "
426 tid, this_thr, this_thr->th.th_current_task,
427 team->t.t_implicit_task_taskdata[tid].td_parent));
429 KMP_DEBUG_ASSERT(this_thr != NULL);
432 if (this_thr->th.th_current_task != &team->t.t_implicit_task_taskdata[0]) {
433 team->t.t_implicit_task_taskdata[0].td_parent =
434 this_thr->th.th_current_task;
435 this_thr->th.th_current_task = &team->t.t_implicit_task_taskdata[0];
438 team->t.t_implicit_task_taskdata[tid].td_parent =
439 team->t.t_implicit_task_taskdata[0].td_parent;
440 this_thr->th.th_current_task = &team->t.t_implicit_task_taskdata[tid];
443 KF_TRACE(10, (
"__kmp_push_current_task_to_thread(exit): T#%d this_thread=%p "
446 tid, this_thr, this_thr->th.th_current_task,
447 team->t.t_implicit_task_taskdata[tid].td_parent));
455 static void __kmp_task_start(kmp_int32 gtid, kmp_task_t *task,
456 kmp_taskdata_t *current_task) {
457 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
458 kmp_info_t *thread = __kmp_threads[gtid];
461 (
"__kmp_task_start(enter): T#%d starting task %p: current_task=%p\n",
462 gtid, taskdata, current_task));
464 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
469 current_task->td_flags.executing = 0;
472 thread->th.th_current_task = taskdata;
474 KMP_DEBUG_ASSERT(taskdata->td_flags.started == 0 ||
475 taskdata->td_flags.tiedness == TASK_UNTIED);
476 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0 ||
477 taskdata->td_flags.tiedness == TASK_UNTIED);
478 taskdata->td_flags.started = 1;
479 taskdata->td_flags.executing = 1;
480 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0);
481 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0);
488 KA_TRACE(10, (
"__kmp_task_start(exit): T#%d task=%p\n", gtid, taskdata));
498 static inline void __ompt_task_start(kmp_task_t *task,
499 kmp_taskdata_t *current_task,
501 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
502 ompt_task_status_t status = ompt_task_switch;
503 if (__kmp_threads[gtid]->th.ompt_thread_info.ompt_task_yielded) {
504 status = ompt_task_yield;
505 __kmp_threads[gtid]->th.ompt_thread_info.ompt_task_yielded = 0;
508 if (ompt_enabled.ompt_callback_task_schedule) {
509 ompt_callbacks.ompt_callback(ompt_callback_task_schedule)(
510 &(current_task->ompt_task_info.task_data), status,
511 &(taskdata->ompt_task_info.task_data));
513 taskdata->ompt_task_info.scheduling_parent = current_task;
518 static inline void __ompt_task_finish(kmp_task_t *task,
519 kmp_taskdata_t *resumed_task,
520 ompt_task_status_t status) {
521 if (ompt_enabled.ompt_callback_task_schedule) {
522 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
523 if (__kmp_omp_cancellation && taskdata->td_taskgroup &&
524 taskdata->td_taskgroup->cancel_request == cancel_taskgroup) {
525 status = ompt_task_cancel;
529 ompt_callbacks.ompt_callback(ompt_callback_task_schedule)(
530 &(taskdata->ompt_task_info.task_data), status,
531 (resumed_task ? &(resumed_task->ompt_task_info.task_data) : NULL));
537 static void __kmpc_omp_task_begin_if0_template(
ident_t *loc_ref, kmp_int32 gtid,
540 void *return_address) {
541 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
542 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task;
544 KA_TRACE(10, (
"__kmpc_omp_task_begin_if0(enter): T#%d loc=%p task=%p "
546 gtid, loc_ref, taskdata, current_task));
548 if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) {
551 kmp_int32 counter = 1 + KMP_ATOMIC_INC(&taskdata->td_untied_count);
552 KMP_DEBUG_USE_VAR(counter);
553 KA_TRACE(20, (
"__kmpc_omp_task_begin_if0: T#%d untied_count (%d) "
554 "incremented for task %p\n",
555 gtid, counter, taskdata));
558 taskdata->td_flags.task_serial =
560 __kmp_task_start(gtid, task, current_task);
564 if (current_task->ompt_task_info.frame.enter_frame.ptr == NULL) {
565 current_task->ompt_task_info.frame.enter_frame.ptr =
566 taskdata->ompt_task_info.frame.exit_frame.ptr = frame_address;
567 current_task->ompt_task_info.frame.enter_frame_flags =
568 taskdata->ompt_task_info.frame.exit_frame_flags =
569 OMPT_FRAME_FLAGS_APP;
571 if (ompt_enabled.ompt_callback_task_create) {
572 ompt_task_info_t *parent_info = &(current_task->ompt_task_info);
573 ompt_callbacks.ompt_callback(ompt_callback_task_create)(
574 &(parent_info->task_data), &(parent_info->frame),
575 &(taskdata->ompt_task_info.task_data),
576 TASK_TYPE_DETAILS_FORMAT(taskdata), 0, return_address);
578 __ompt_task_start(task, current_task, gtid);
582 KA_TRACE(10, (
"__kmpc_omp_task_begin_if0(exit): T#%d loc=%p task=%p,\n", gtid,
588 static void __kmpc_omp_task_begin_if0_ompt(
ident_t *loc_ref, kmp_int32 gtid,
591 void *return_address) {
592 __kmpc_omp_task_begin_if0_template<true>(loc_ref, gtid, task, frame_address,
609 __attribute__((target(
"backchain")))
611 void __kmpc_omp_task_begin_if0(
ident_t *loc_ref, kmp_int32 gtid,
614 if (UNLIKELY(ompt_enabled.enabled)) {
615 OMPT_STORE_RETURN_ADDRESS(gtid);
616 __kmpc_omp_task_begin_if0_ompt(loc_ref, gtid, task,
617 OMPT_GET_FRAME_ADDRESS(1),
618 OMPT_LOAD_RETURN_ADDRESS(gtid));
622 __kmpc_omp_task_begin_if0_template<false>(loc_ref, gtid, task, NULL, NULL);
628 void __kmpc_omp_task_begin(
ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *task) {
629 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task;
633 (
"__kmpc_omp_task_begin(enter): T#%d loc=%p task=%p current_task=%p\n",
634 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task), current_task));
636 __kmp_task_start(gtid, task, current_task);
638 KA_TRACE(10, (
"__kmpc_omp_task_begin(exit): T#%d loc=%p task=%p,\n", gtid,
639 loc_ref, KMP_TASK_TO_TASKDATA(task)));
649 static void __kmp_free_task(kmp_int32 gtid, kmp_taskdata_t *taskdata,
650 kmp_info_t *thread) {
651 KA_TRACE(30, (
"__kmp_free_task: T#%d freeing data from task %p\n", gtid,
655 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
656 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0);
657 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 1);
658 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0);
659 KMP_DEBUG_ASSERT(taskdata->td_allocated_child_tasks == 0 ||
660 taskdata->td_flags.task_serial == 1);
661 KMP_DEBUG_ASSERT(taskdata->td_incomplete_child_tasks == 0);
662 kmp_task_t *task = KMP_TASKDATA_TO_TASK(taskdata);
664 task->data1.destructors = NULL;
665 task->data2.priority = 0;
667 taskdata->td_flags.freed = 1;
670 if (!taskdata->is_taskgraph) {
674 __kmp_fast_free(thread, taskdata);
676 __kmp_thread_free(thread, taskdata);
680 taskdata->td_flags.complete = 0;
681 taskdata->td_flags.started = 0;
682 taskdata->td_flags.freed = 0;
683 taskdata->td_flags.executing = 0;
684 taskdata->td_flags.task_serial =
685 (taskdata->td_parent->td_flags.final ||
686 taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser);
689 KMP_ATOMIC_ST_RLX(&taskdata->td_untied_count, 0);
690 KMP_ATOMIC_ST_RLX(&taskdata->td_incomplete_child_tasks, 0);
692 KMP_ATOMIC_ST_RLX(&taskdata->td_allocated_child_tasks, 1);
696 KA_TRACE(20, (
"__kmp_free_task: T#%d freed task %p\n", gtid, taskdata));
705 static void __kmp_free_task_and_ancestors(kmp_int32 gtid,
706 kmp_taskdata_t *taskdata,
707 kmp_info_t *thread) {
710 kmp_int32 team_serial =
711 (taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser) &&
712 !taskdata->td_flags.proxy;
713 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
715 kmp_int32 children = KMP_ATOMIC_DEC(&taskdata->td_allocated_child_tasks) - 1;
716 KMP_DEBUG_ASSERT(children >= 0);
719 while (children == 0) {
720 kmp_taskdata_t *parent_taskdata = taskdata->td_parent;
722 KA_TRACE(20, (
"__kmp_free_task_and_ancestors(enter): T#%d task %p complete "
723 "and freeing itself\n",
727 __kmp_free_task(gtid, taskdata, thread);
729 taskdata = parent_taskdata;
735 if (taskdata->td_flags.tasktype == TASK_IMPLICIT) {
736 if (taskdata->td_dephash) {
737 int children = KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks);
738 kmp_tasking_flags_t flags_old = taskdata->td_flags;
739 if (children == 0 && flags_old.complete == 1) {
740 kmp_tasking_flags_t flags_new = flags_old;
741 flags_new.complete = 0;
742 if (KMP_COMPARE_AND_STORE_ACQ32(
743 RCAST(kmp_int32 *, &taskdata->td_flags),
744 *RCAST(kmp_int32 *, &flags_old),
745 *RCAST(kmp_int32 *, &flags_new))) {
746 KA_TRACE(100, (
"__kmp_free_task_and_ancestors: T#%d cleans "
747 "dephash of implicit task %p\n",
750 __kmp_dephash_free_entries(thread, taskdata->td_dephash);
757 children = KMP_ATOMIC_DEC(&taskdata->td_allocated_child_tasks) - 1;
758 KMP_DEBUG_ASSERT(children >= 0);
762 20, (
"__kmp_free_task_and_ancestors(exit): T#%d task %p has %d children; "
763 "not freeing it yet\n",
764 gtid, taskdata, children));
775 static bool __kmp_track_children_task(kmp_taskdata_t *taskdata) {
776 kmp_tasking_flags_t flags = taskdata->td_flags;
777 bool ret = !(flags.team_serial || flags.tasking_ser);
778 ret = ret || flags.proxy == TASK_PROXY ||
779 flags.detachable == TASK_DETACHABLE || flags.hidden_helper;
781 KMP_ATOMIC_LD_ACQ(&taskdata->td_parent->td_incomplete_child_tasks) > 0;
783 if (taskdata->td_taskgroup && taskdata->is_taskgraph)
784 ret = ret || KMP_ATOMIC_LD_ACQ(&taskdata->td_taskgroup->count) > 0;
799 static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
800 kmp_taskdata_t *resumed_task) {
801 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
802 kmp_info_t *thread = __kmp_threads[gtid];
803 kmp_task_team_t *task_team =
804 thread->th.th_task_team;
810 kmp_int32 children = 0;
812 KA_TRACE(10, (
"__kmp_task_finish(enter): T#%d finishing task %p and resuming "
814 gtid, taskdata, resumed_task));
816 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
819 is_taskgraph = taskdata->is_taskgraph;
822 if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) {
825 kmp_int32 counter = KMP_ATOMIC_DEC(&taskdata->td_untied_count) - 1;
828 (
"__kmp_task_finish: T#%d untied_count (%d) decremented for task %p\n",
829 gtid, counter, taskdata));
833 if (resumed_task == NULL) {
834 KMP_DEBUG_ASSERT(taskdata->td_flags.task_serial);
835 resumed_task = taskdata->td_parent;
838 thread->th.th_current_task = resumed_task;
839 resumed_task->td_flags.executing = 1;
840 KA_TRACE(10, (
"__kmp_task_finish(exit): T#%d partially done task %p, "
841 "resuming task %p\n",
842 gtid, taskdata, resumed_task));
850 (taskdata->td_flags.tasking_ser || taskdata->td_flags.task_serial) ==
851 taskdata->td_flags.task_serial);
852 if (taskdata->td_flags.task_serial) {
853 if (resumed_task == NULL) {
854 resumed_task = taskdata->td_parent;
858 KMP_DEBUG_ASSERT(resumed_task !=
868 if (UNLIKELY(taskdata->td_flags.destructors_thunk)) {
869 kmp_routine_entry_t destr_thunk = task->data1.destructors;
870 KMP_ASSERT(destr_thunk);
871 destr_thunk(gtid, task);
874 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0);
875 KMP_DEBUG_ASSERT(taskdata->td_flags.started == 1);
876 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0);
878 bool completed =
true;
879 if (UNLIKELY(taskdata->td_flags.detachable == TASK_DETACHABLE)) {
880 if (taskdata->td_allow_completion_event.type ==
881 KMP_EVENT_ALLOW_COMPLETION) {
883 __kmp_acquire_tas_lock(&taskdata->td_allow_completion_event.lock, gtid);
884 if (taskdata->td_allow_completion_event.type ==
885 KMP_EVENT_ALLOW_COMPLETION) {
887 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 1);
888 taskdata->td_flags.executing = 0;
895 __ompt_task_finish(task, resumed_task, ompt_task_detach);
901 taskdata->td_flags.proxy = TASK_PROXY;
904 __kmp_release_tas_lock(&taskdata->td_allow_completion_event.lock, gtid);
909 if (taskdata->td_target_data.async_handle != NULL) {
915 __ompt_task_finish(task, resumed_task, ompt_task_switch);
918 __kmpc_give_task(task, __kmp_tid_from_gtid(gtid));
919 if (KMP_HIDDEN_HELPER_THREAD(gtid))
920 __kmp_hidden_helper_worker_thread_signal();
925 taskdata->td_flags.complete = 1;
927 taskdata->td_flags.onced = 1;
933 __ompt_task_finish(task, resumed_task, ompt_task_complete);
937 if (__kmp_track_children_task(taskdata)) {
938 __kmp_release_deps(gtid, taskdata);
943 KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks);
944 KMP_DEBUG_ASSERT(children >= 0);
946 if (taskdata->td_taskgroup && !taskdata->is_taskgraph)
948 if (taskdata->td_taskgroup)
950 KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count);
951 }
else if (task_team && (task_team->tt.tt_found_proxy_tasks ||
952 task_team->tt.tt_hidden_helper_task_encountered)) {
955 __kmp_release_deps(gtid, taskdata);
961 KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 1);
962 taskdata->td_flags.executing = 0;
965 if (taskdata->td_flags.hidden_helper) {
967 KMP_ASSERT(KMP_HIDDEN_HELPER_THREAD(gtid));
968 KMP_ATOMIC_DEC(&__kmp_unexecuted_hidden_helper_tasks);
973 20, (
"__kmp_task_finish: T#%d finished task %p, %d incomplete children\n",
974 gtid, taskdata, children));
980 thread->th.th_current_task = resumed_task;
982 __kmp_free_task_and_ancestors(gtid, taskdata, thread);
986 resumed_task->td_flags.executing = 1;
989 if (is_taskgraph && __kmp_track_children_task(taskdata) &&
990 taskdata->td_taskgroup) {
997 KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count);
1002 10, (
"__kmp_task_finish(exit): T#%d finished task %p, resuming task %p\n",
1003 gtid, taskdata, resumed_task));
1008 template <
bool ompt>
1009 static void __kmpc_omp_task_complete_if0_template(
ident_t *loc_ref,
1012 KA_TRACE(10, (
"__kmpc_omp_task_complete_if0(enter): T#%d loc=%p task=%p\n",
1013 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task)));
1014 KMP_DEBUG_ASSERT(gtid >= 0);
1016 __kmp_task_finish<ompt>(gtid, task, NULL);
1018 KA_TRACE(10, (
"__kmpc_omp_task_complete_if0(exit): T#%d loc=%p task=%p\n",
1019 gtid, loc_ref, KMP_TASK_TO_TASKDATA(task)));
1023 ompt_frame_t *ompt_frame;
1024 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
1025 ompt_frame->enter_frame = ompt_data_none;
1026 ompt_frame->enter_frame_flags = OMPT_FRAME_FLAGS_RUNTIME;
1035 void __kmpc_omp_task_complete_if0_ompt(
ident_t *loc_ref, kmp_int32 gtid,
1037 __kmpc_omp_task_complete_if0_template<true>(loc_ref, gtid, task);
1046 void __kmpc_omp_task_complete_if0(
ident_t *loc_ref, kmp_int32 gtid,
1049 if (UNLIKELY(ompt_enabled.enabled)) {
1050 __kmpc_omp_task_complete_if0_ompt(loc_ref, gtid, task);
1054 __kmpc_omp_task_complete_if0_template<false>(loc_ref, gtid, task);
1060 void __kmpc_omp_task_complete(
ident_t *loc_ref, kmp_int32 gtid,
1062 KA_TRACE(10, (
"__kmpc_omp_task_complete(enter): T#%d loc=%p task=%p\n", gtid,
1063 loc_ref, KMP_TASK_TO_TASKDATA(task)));
1065 __kmp_task_finish<false>(gtid, task,
1068 KA_TRACE(10, (
"__kmpc_omp_task_complete(exit): T#%d loc=%p task=%p\n", gtid,
1069 loc_ref, KMP_TASK_TO_TASKDATA(task)));
1085 void __kmp_init_implicit_task(
ident_t *loc_ref, kmp_info_t *this_thr,
1086 kmp_team_t *team,
int tid,
int set_curr_task) {
1087 kmp_taskdata_t *task = &team->t.t_implicit_task_taskdata[tid];
1091 (
"__kmp_init_implicit_task(enter): T#:%d team=%p task=%p, reinit=%s\n",
1092 tid, team, task, set_curr_task ?
"TRUE" :
"FALSE"));
1094 task->td_task_id = KMP_GEN_TASK_ID();
1095 task->td_team = team;
1098 task->td_ident = loc_ref;
1099 task->td_taskwait_ident = NULL;
1100 task->td_taskwait_counter = 0;
1101 task->td_taskwait_thread = 0;
1103 task->td_flags.tiedness = TASK_TIED;
1104 task->td_flags.tasktype = TASK_IMPLICIT;
1105 task->td_flags.proxy = TASK_FULL;
1108 task->td_flags.task_serial = 1;
1109 task->td_flags.tasking_ser = (__kmp_tasking_mode == tskm_immediate_exec);
1110 task->td_flags.team_serial = (team->t.t_serialized) ? 1 : 0;
1112 task->td_flags.started = 1;
1113 task->td_flags.executing = 1;
1114 task->td_flags.complete = 0;
1115 task->td_flags.freed = 0;
1117 task->td_flags.onced = 0;
1120 task->td_depnode = NULL;
1121 task->td_last_tied = task;
1122 task->td_allow_completion_event.type = KMP_EVENT_UNINITIALIZED;
1124 if (set_curr_task) {
1125 KMP_ATOMIC_ST_REL(&task->td_incomplete_child_tasks, 0);
1127 KMP_ATOMIC_ST_REL(&task->td_allocated_child_tasks, 0);
1128 task->td_taskgroup = NULL;
1129 task->td_dephash = NULL;
1130 __kmp_push_current_task_to_thread(this_thr, team, tid);
1132 KMP_DEBUG_ASSERT(task->td_incomplete_child_tasks == 0);
1133 KMP_DEBUG_ASSERT(task->td_allocated_child_tasks == 0);
1137 if (UNLIKELY(ompt_enabled.enabled))
1138 __ompt_task_init(task, tid);
1141 KF_TRACE(10, (
"__kmp_init_implicit_task(exit): T#:%d team=%p task=%p\n", tid,
1150 void __kmp_finish_implicit_task(kmp_info_t *thread) {
1151 kmp_taskdata_t *task = thread->th.th_current_task;
1152 if (task->td_dephash) {
1154 task->td_flags.complete = 1;
1156 task->td_flags.onced = 1;
1158 children = KMP_ATOMIC_LD_ACQ(&task->td_incomplete_child_tasks);
1159 kmp_tasking_flags_t flags_old = task->td_flags;
1160 if (children == 0 && flags_old.complete == 1) {
1161 kmp_tasking_flags_t flags_new = flags_old;
1162 flags_new.complete = 0;
1163 if (KMP_COMPARE_AND_STORE_ACQ32(RCAST(kmp_int32 *, &task->td_flags),
1164 *RCAST(kmp_int32 *, &flags_old),
1165 *RCAST(kmp_int32 *, &flags_new))) {
1166 KA_TRACE(100, (
"__kmp_finish_implicit_task: T#%d cleans "
1167 "dephash of implicit task %p\n",
1168 thread->th.th_info.ds.ds_gtid, task));
1169 __kmp_dephash_free_entries(thread, task->td_dephash);
1179 void __kmp_free_implicit_task(kmp_info_t *thread) {
1180 kmp_taskdata_t *task = thread->th.th_current_task;
1181 if (task && task->td_dephash) {
1182 __kmp_dephash_free(thread, task->td_dephash);
1183 task->td_dephash = NULL;
1189 static size_t __kmp_round_up_to_val(
size_t size,
size_t val) {
1190 if (size & (val - 1)) {
1192 if (size <= KMP_SIZE_T_MAX - val) {
1211 kmp_task_t *__kmp_task_alloc(
ident_t *loc_ref, kmp_int32 gtid,
1212 kmp_tasking_flags_t *flags,
1213 size_t sizeof_kmp_task_t,
size_t sizeof_shareds,
1214 kmp_routine_entry_t task_entry) {
1216 kmp_taskdata_t *taskdata;
1217 kmp_info_t *thread = __kmp_threads[gtid];
1218 kmp_team_t *team = thread->th.th_team;
1219 kmp_taskdata_t *parent_task = thread->th.th_current_task;
1220 size_t shareds_offset;
1222 if (UNLIKELY(!TCR_4(__kmp_init_middle)))
1223 __kmp_middle_initialize();
1225 if (flags->hidden_helper) {
1226 if (__kmp_enable_hidden_helper) {
1227 if (!TCR_4(__kmp_init_hidden_helper))
1228 __kmp_hidden_helper_initialize();
1231 flags->hidden_helper = FALSE;
1235 KA_TRACE(10, (
"__kmp_task_alloc(enter): T#%d loc=%p, flags=(0x%x) "
1236 "sizeof_task=%ld sizeof_shared=%ld entry=%p\n",
1237 gtid, loc_ref, *((kmp_int32 *)flags), sizeof_kmp_task_t,
1238 sizeof_shareds, task_entry));
1240 KMP_DEBUG_ASSERT(parent_task);
1241 if (parent_task->td_flags.final) {
1242 if (flags->merged_if0) {
1247 if (flags->tiedness == TASK_UNTIED && !team->t.t_serialized) {
1251 KMP_CHECK_UPDATE(thread->th.th_task_team->tt.tt_untied_task_encountered, 1);
1257 if (UNLIKELY(flags->proxy == TASK_PROXY ||
1258 flags->detachable == TASK_DETACHABLE || flags->hidden_helper)) {
1259 if (flags->proxy == TASK_PROXY) {
1260 flags->tiedness = TASK_UNTIED;
1261 flags->merged_if0 = 1;
1265 if ((thread->th.th_task_team) == NULL) {
1268 KMP_DEBUG_ASSERT(team->t.t_serialized);
1270 (
"T#%d creating task team in __kmp_task_alloc for proxy task\n",
1272 __kmp_task_team_setup(thread, team);
1273 thread->th.th_task_team = team->t.t_task_team[thread->th.th_task_state];
1275 kmp_task_team_t *task_team = thread->th.th_task_team;
1278 if (!KMP_TASKING_ENABLED(task_team)) {
1281 (
"T#%d enabling tasking in __kmp_task_alloc for proxy task\n", gtid));
1282 __kmp_enable_tasking(task_team, thread);
1283 kmp_int32 tid = thread->th.th_info.ds.ds_tid;
1284 kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[tid];
1286 if (thread_data->td.td_deque == NULL) {
1287 __kmp_alloc_task_deque(thread, thread_data);
1291 if ((flags->proxy == TASK_PROXY || flags->detachable == TASK_DETACHABLE) &&
1292 task_team->tt.tt_found_proxy_tasks == FALSE)
1293 TCW_4(task_team->tt.tt_found_proxy_tasks, TRUE);
1294 if (flags->hidden_helper &&
1295 task_team->tt.tt_hidden_helper_task_encountered == FALSE)
1296 TCW_4(task_team->tt.tt_hidden_helper_task_encountered, TRUE);
1301 shareds_offset =
sizeof(kmp_taskdata_t) + sizeof_kmp_task_t;
1302 shareds_offset = __kmp_round_up_to_val(shareds_offset,
sizeof(kmp_uint64));
1305 KA_TRACE(30, (
"__kmp_task_alloc: T#%d First malloc size: %ld\n", gtid,
1307 KA_TRACE(30, (
"__kmp_task_alloc: T#%d Second malloc size: %ld\n", gtid,
1312 taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(thread, shareds_offset +
1315 taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(thread, shareds_offset +
1319 task = KMP_TASKDATA_TO_TASK(taskdata);
1322 #if KMP_ARCH_X86 || KMP_ARCH_PPC64 || KMP_ARCH_S390X || !KMP_HAVE_QUAD
1323 KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (
sizeof(
double) - 1)) == 0);
1324 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task) & (
sizeof(
double) - 1)) == 0);
1326 KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (
sizeof(_Quad) - 1)) == 0);
1327 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task) & (
sizeof(_Quad) - 1)) == 0);
1329 if (sizeof_shareds > 0) {
1331 task->shareds = &((
char *)taskdata)[shareds_offset];
1333 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task->shareds) & (
sizeof(
void *) - 1)) ==
1336 task->shareds = NULL;
1338 task->routine = task_entry;
1341 taskdata->td_task_id = KMP_GEN_TASK_ID();
1342 taskdata->td_team = thread->th.th_team;
1343 taskdata->td_alloc_thread = thread;
1344 taskdata->td_parent = parent_task;
1345 taskdata->td_level = parent_task->td_level + 1;
1346 KMP_ATOMIC_ST_RLX(&taskdata->td_untied_count, 0);
1347 taskdata->td_ident = loc_ref;
1348 taskdata->td_taskwait_ident = NULL;
1349 taskdata->td_taskwait_counter = 0;
1350 taskdata->td_taskwait_thread = 0;
1351 KMP_DEBUG_ASSERT(taskdata->td_parent != NULL);
1353 if (flags->proxy == TASK_FULL)
1354 copy_icvs(&taskdata->td_icvs, &taskdata->td_parent->td_icvs);
1356 taskdata->td_flags = *flags;
1357 taskdata->td_task_team = thread->th.th_task_team;
1358 taskdata->td_size_alloc = shareds_offset + sizeof_shareds;
1359 taskdata->td_flags.tasktype = TASK_EXPLICIT;
1362 if (flags->hidden_helper) {
1363 kmp_info_t *shadow_thread = __kmp_threads[KMP_GTID_TO_SHADOW_GTID(gtid)];
1364 taskdata->td_team = shadow_thread->th.th_team;
1365 taskdata->td_task_team = shadow_thread->th.th_task_team;
1369 taskdata->td_flags.tasking_ser = (__kmp_tasking_mode == tskm_immediate_exec);
1372 taskdata->td_flags.team_serial = (team->t.t_serialized) ? 1 : 0;
1378 taskdata->td_flags.task_serial =
1379 (parent_task->td_flags.final || taskdata->td_flags.team_serial ||
1380 taskdata->td_flags.tasking_ser || flags->merged_if0);
1382 taskdata->td_flags.started = 0;
1383 taskdata->td_flags.executing = 0;
1384 taskdata->td_flags.complete = 0;
1385 taskdata->td_flags.freed = 0;
1387 taskdata->td_flags.onced = 0;
1388 taskdata->is_taskgraph = 0;
1389 taskdata->tdg =
nullptr;
1391 KMP_ATOMIC_ST_RLX(&taskdata->td_incomplete_child_tasks, 0);
1393 KMP_ATOMIC_ST_RLX(&taskdata->td_allocated_child_tasks, 1);
1394 taskdata->td_taskgroup =
1395 parent_task->td_taskgroup;
1396 taskdata->td_dephash = NULL;
1397 taskdata->td_depnode = NULL;
1398 taskdata->td_target_data.async_handle = NULL;
1399 if (flags->tiedness == TASK_UNTIED)
1400 taskdata->td_last_tied = NULL;
1402 taskdata->td_last_tied = taskdata;
1403 taskdata->td_allow_completion_event.type = KMP_EVENT_UNINITIALIZED;
1405 if (UNLIKELY(ompt_enabled.enabled))
1406 __ompt_task_init(taskdata, gtid);
1410 if (__kmp_track_children_task(taskdata)) {
1411 KMP_ATOMIC_INC(&parent_task->td_incomplete_child_tasks);
1412 if (parent_task->td_taskgroup)
1413 KMP_ATOMIC_INC(&parent_task->td_taskgroup->count);
1416 if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT) {
1417 KMP_ATOMIC_INC(&taskdata->td_parent->td_allocated_child_tasks);
1419 if (flags->hidden_helper) {
1420 taskdata->td_flags.task_serial = FALSE;
1422 KMP_ATOMIC_INC(&__kmp_unexecuted_hidden_helper_tasks);
1427 kmp_tdg_info_t *tdg = __kmp_find_tdg(__kmp_curr_tdg_idx);
1428 if (tdg && __kmp_tdg_is_recording(tdg->tdg_status) &&
1429 (task_entry != (kmp_routine_entry_t)__kmp_taskloop_task)) {
1430 taskdata->is_taskgraph = 1;
1431 taskdata->tdg = __kmp_global_tdgs[__kmp_curr_tdg_idx];
1432 taskdata->td_task_id = KMP_GEN_TASK_ID();
1433 taskdata->td_tdg_task_id = KMP_ATOMIC_INC(&__kmp_tdg_task_id);
1436 KA_TRACE(20, (
"__kmp_task_alloc(exit): T#%d created task %p parent=%p\n",
1437 gtid, taskdata, taskdata->td_parent));
1442 kmp_task_t *__kmpc_omp_task_alloc(
ident_t *loc_ref, kmp_int32 gtid,
1443 kmp_int32 flags,
size_t sizeof_kmp_task_t,
1444 size_t sizeof_shareds,
1445 kmp_routine_entry_t task_entry) {
1447 kmp_tasking_flags_t *input_flags = (kmp_tasking_flags_t *)&flags;
1448 __kmp_assert_valid_gtid(gtid);
1449 input_flags->native = FALSE;
1451 KA_TRACE(10, (
"__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s %s %s) "
1452 "sizeof_task=%ld sizeof_shared=%ld entry=%p\n",
1453 gtid, loc_ref, input_flags->tiedness ?
"tied " :
"untied",
1454 input_flags->proxy ?
"proxy" :
"",
1455 input_flags->detachable ?
"detachable" :
"", sizeof_kmp_task_t,
1456 sizeof_shareds, task_entry));
1458 retval = __kmp_task_alloc(loc_ref, gtid, input_flags, sizeof_kmp_task_t,
1459 sizeof_shareds, task_entry);
1461 KA_TRACE(20, (
"__kmpc_omp_task_alloc(exit): T#%d retval %p\n", gtid, retval));
1466 kmp_task_t *__kmpc_omp_target_task_alloc(
ident_t *loc_ref, kmp_int32 gtid,
1468 size_t sizeof_kmp_task_t,
1469 size_t sizeof_shareds,
1470 kmp_routine_entry_t task_entry,
1471 kmp_int64 device_id) {
1472 auto &input_flags =
reinterpret_cast<kmp_tasking_flags_t &
>(flags);
1474 input_flags.tiedness = TASK_UNTIED;
1475 input_flags.target = 1;
1477 if (__kmp_enable_hidden_helper)
1478 input_flags.hidden_helper = TRUE;
1480 return __kmpc_omp_task_alloc(loc_ref, gtid, flags, sizeof_kmp_task_t,
1481 sizeof_shareds, task_entry);
1499 kmp_task_t *new_task, kmp_int32 naffins,
1500 kmp_task_affinity_info_t *affin_list) {
1510 __attribute__((target(
"backchain")))
1513 __kmp_invoke_task(kmp_int32 gtid, kmp_task_t *task,
1514 kmp_taskdata_t *current_task) {
1515 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
1519 30, (
"__kmp_invoke_task(enter): T#%d invoking task %p, current_task=%p\n",
1520 gtid, taskdata, current_task));
1521 KMP_DEBUG_ASSERT(task);
1522 if (UNLIKELY(taskdata->td_flags.proxy == TASK_PROXY &&
1523 taskdata->td_flags.complete == 1)) {
1528 (
"__kmp_invoke_task: T#%d running bottom finish for proxy task %p\n",
1531 __kmp_bottom_half_finish_proxy(gtid, task);
1533 KA_TRACE(30, (
"__kmp_invoke_task(exit): T#%d completed bottom finish for "
1534 "proxy task %p, resuming task %p\n",
1535 gtid, taskdata, current_task));
1543 ompt_thread_info_t oldInfo;
1544 if (UNLIKELY(ompt_enabled.enabled)) {
1546 thread = __kmp_threads[gtid];
1547 oldInfo = thread->th.ompt_thread_info;
1548 thread->th.ompt_thread_info.wait_id = 0;
1549 thread->th.ompt_thread_info.state = (thread->th.th_team_serialized)
1550 ? ompt_state_work_serial
1551 : ompt_state_work_parallel;
1552 taskdata->ompt_task_info.frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1557 if (taskdata->td_flags.proxy != TASK_PROXY) {
1558 __kmp_task_start(gtid, task, current_task);
1564 if (UNLIKELY(__kmp_omp_cancellation)) {
1565 thread = __kmp_threads[gtid];
1566 kmp_team_t *this_team = thread->th.th_team;
1567 kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup;
1568 if ((taskgroup && taskgroup->cancel_request) ||
1569 (this_team->t.t_cancel_request == cancel_parallel)) {
1570 #if OMPT_SUPPORT && OMPT_OPTIONAL
1571 ompt_data_t *task_data;
1572 if (UNLIKELY(ompt_enabled.ompt_callback_cancel)) {
1573 __ompt_get_task_info_internal(0, NULL, &task_data, NULL, NULL, NULL);
1574 ompt_callbacks.ompt_callback(ompt_callback_cancel)(
1576 ((taskgroup && taskgroup->cancel_request) ? ompt_cancel_taskgroup
1577 : ompt_cancel_parallel) |
1578 ompt_cancel_discarded_task,
1591 if (taskdata->td_flags.tiedness == TASK_UNTIED) {
1592 taskdata->td_last_tied = current_task->td_last_tied;
1593 KMP_DEBUG_ASSERT(taskdata->td_last_tied);
1595 #if KMP_STATS_ENABLED
1597 switch (KMP_GET_THREAD_STATE()) {
1598 case FORK_JOIN_BARRIER:
1599 KMP_PUSH_PARTITIONED_TIMER(OMP_task_join_bar);
1602 KMP_PUSH_PARTITIONED_TIMER(OMP_task_plain_bar);
1605 KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskyield);
1608 KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskwait);
1611 KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskgroup);
1614 KMP_PUSH_PARTITIONED_TIMER(OMP_task_immediate);
1621 if (UNLIKELY(ompt_enabled.enabled))
1622 __ompt_task_start(task, current_task, gtid);
1624 #if OMPT_SUPPORT && OMPT_OPTIONAL
1625 if (UNLIKELY(ompt_enabled.ompt_callback_dispatch &&
1626 taskdata->ompt_task_info.dispatch_chunk.iterations > 0)) {
1627 ompt_data_t instance = ompt_data_none;
1628 instance.ptr = &(taskdata->ompt_task_info.dispatch_chunk);
1629 ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
1630 ompt_callbacks.ompt_callback(ompt_callback_dispatch)(
1631 &(team_info->parallel_data), &(taskdata->ompt_task_info.task_data),
1632 ompt_dispatch_taskloop_chunk, instance);
1633 taskdata->ompt_task_info.dispatch_chunk = {0, 0};
1638 if (ompd_state & OMPD_ENABLE_BP)
1639 ompd_bp_task_begin();
1642 #if USE_ITT_BUILD && USE_ITT_NOTIFY
1643 kmp_uint64 cur_time;
1644 kmp_int32 kmp_itt_count_task =
1645 __kmp_forkjoin_frames_mode == 3 && !taskdata->td_flags.task_serial &&
1646 current_task->td_flags.tasktype == TASK_IMPLICIT;
1647 if (kmp_itt_count_task) {
1648 thread = __kmp_threads[gtid];
1650 if (thread->th.th_bar_arrive_time)
1651 cur_time = __itt_get_timestamp();
1653 kmp_itt_count_task = 0;
1655 KMP_FSYNC_ACQUIRED(taskdata);
1658 #if ENABLE_LIBOMPTARGET
1659 if (taskdata->td_target_data.async_handle != NULL) {
1663 KMP_ASSERT(tgt_target_nowait_query);
1664 tgt_target_nowait_query(&taskdata->td_target_data.async_handle);
1667 if (task->routine != NULL) {
1668 #ifdef KMP_GOMP_COMPAT
1669 if (taskdata->td_flags.native) {
1670 ((void (*)(
void *))(*(task->routine)))(task->shareds);
1674 (*(task->routine))(gtid, task);
1677 KMP_POP_PARTITIONED_TIMER();
1679 #if USE_ITT_BUILD && USE_ITT_NOTIFY
1680 if (kmp_itt_count_task) {
1682 thread->th.th_bar_arrive_time += (__itt_get_timestamp() - cur_time);
1684 KMP_FSYNC_CANCEL(taskdata);
1685 KMP_FSYNC_RELEASING(taskdata->td_parent);
1690 if (ompd_state & OMPD_ENABLE_BP)
1695 if (taskdata->td_flags.proxy != TASK_PROXY) {
1697 if (UNLIKELY(ompt_enabled.enabled)) {
1698 thread->th.ompt_thread_info = oldInfo;
1699 if (taskdata->td_flags.tiedness == TASK_TIED) {
1700 taskdata->ompt_task_info.frame.exit_frame = ompt_data_none;
1702 __kmp_task_finish<true>(gtid, task, current_task);
1705 __kmp_task_finish<false>(gtid, task, current_task);
1708 else if (UNLIKELY(ompt_enabled.enabled && taskdata->td_flags.target)) {
1709 __ompt_task_finish(task, current_task, ompt_task_switch);
1715 (
"__kmp_invoke_task(exit): T#%d completed task %p, resuming task %p\n",
1716 gtid, taskdata, current_task));
1730 kmp_int32 __kmpc_omp_task_parts(
ident_t *loc_ref, kmp_int32 gtid,
1731 kmp_task_t *new_task) {
1732 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
1734 KA_TRACE(10, (
"__kmpc_omp_task_parts(enter): T#%d loc=%p task=%p\n", gtid,
1735 loc_ref, new_taskdata));
1738 kmp_taskdata_t *parent;
1739 if (UNLIKELY(ompt_enabled.enabled)) {
1740 parent = new_taskdata->td_parent;
1741 if (ompt_enabled.ompt_callback_task_create) {
1742 ompt_callbacks.ompt_callback(ompt_callback_task_create)(
1743 &(parent->ompt_task_info.task_data), &(parent->ompt_task_info.frame),
1744 &(new_taskdata->ompt_task_info.task_data),
1745 TASK_TYPE_DETAILS_FORMAT(new_taskdata), 0,
1746 OMPT_GET_RETURN_ADDRESS(0));
1754 if (__kmp_push_task(gtid, new_task) == TASK_NOT_PUSHED)
1756 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task;
1757 new_taskdata->td_flags.task_serial = 1;
1758 __kmp_invoke_task(gtid, new_task, current_task);
1763 (
"__kmpc_omp_task_parts(exit): T#%d returning TASK_CURRENT_NOT_QUEUED: "
1764 "loc=%p task=%p, return: TASK_CURRENT_NOT_QUEUED\n",
1765 gtid, loc_ref, new_taskdata));
1768 if (UNLIKELY(ompt_enabled.enabled)) {
1769 parent->ompt_task_info.frame.enter_frame = ompt_data_none;
1770 parent->ompt_task_info.frame.enter_frame_flags = OMPT_FRAME_FLAGS_RUNTIME;
1773 return TASK_CURRENT_NOT_QUEUED;
1787 kmp_int32 __kmp_omp_task(kmp_int32 gtid, kmp_task_t *new_task,
1788 bool serialize_immediate) {
1789 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
1792 if (new_taskdata->is_taskgraph &&
1793 __kmp_tdg_is_recording(new_taskdata->tdg->tdg_status)) {
1794 kmp_tdg_info_t *tdg = new_taskdata->tdg;
1796 if (new_taskdata->td_tdg_task_id >= new_taskdata->tdg->map_size) {
1797 __kmp_acquire_bootstrap_lock(&tdg->graph_lock);
1800 if (new_taskdata->td_tdg_task_id >= tdg->map_size) {
1801 kmp_uint old_size = tdg->map_size;
1802 kmp_uint new_size = old_size * 2;
1803 kmp_node_info_t *old_record = tdg->record_map;
1804 kmp_node_info_t *new_record = (kmp_node_info_t *)__kmp_allocate(
1805 new_size *
sizeof(kmp_node_info_t));
1807 KMP_MEMCPY(new_record, old_record, old_size *
sizeof(kmp_node_info_t));
1808 tdg->record_map = new_record;
1810 __kmp_free(old_record);
1812 for (kmp_int i = old_size; i < new_size; i++) {
1813 kmp_int32 *successorsList = (kmp_int32 *)__kmp_allocate(
1814 __kmp_successors_size *
sizeof(kmp_int32));
1815 new_record[i].task =
nullptr;
1816 new_record[i].successors = successorsList;
1817 new_record[i].nsuccessors = 0;
1818 new_record[i].npredecessors = 0;
1819 new_record[i].successors_size = __kmp_successors_size;
1820 KMP_ATOMIC_ST_REL(&new_record[i].npredecessors_counter, 0);
1824 tdg->map_size = new_size;
1826 __kmp_release_bootstrap_lock(&tdg->graph_lock);
1829 if (tdg->record_map[new_taskdata->td_tdg_task_id].task ==
nullptr) {
1830 tdg->record_map[new_taskdata->td_tdg_task_id].task = new_task;
1831 tdg->record_map[new_taskdata->td_tdg_task_id].parent_task =
1832 new_taskdata->td_parent;
1833 KMP_ATOMIC_INC(&tdg->num_tasks);
1840 if (new_taskdata->td_flags.proxy == TASK_PROXY ||
1841 __kmp_push_task(gtid, new_task) == TASK_NOT_PUSHED)
1843 kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task;
1844 if (serialize_immediate)
1845 new_taskdata->td_flags.task_serial = 1;
1846 __kmp_invoke_task(gtid, new_task, current_task);
1847 }
else if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME &&
1848 __kmp_wpolicy_passive) {
1849 kmp_info_t *this_thr = __kmp_threads[gtid];
1850 kmp_team_t *team = this_thr->th.th_team;
1851 kmp_int32 nthreads = this_thr->th.th_team_nproc;
1852 for (
int i = 0; i < nthreads; ++i) {
1853 kmp_info_t *thread = team->t.t_threads[i];
1854 if (thread == this_thr)
1856 if (thread->th.th_sleep_loc != NULL) {
1857 __kmp_null_resume_wrapper(thread);
1862 return TASK_CURRENT_NOT_QUEUED;
1877 kmp_int32 __kmpc_omp_task(
ident_t *loc_ref, kmp_int32 gtid,
1878 kmp_task_t *new_task) {
1880 KMP_SET_THREAD_STATE_BLOCK(EXPLICIT_TASK);
1882 #if KMP_DEBUG || OMPT_SUPPORT
1883 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
1885 KA_TRACE(10, (
"__kmpc_omp_task(enter): T#%d loc=%p task=%p\n", gtid, loc_ref,
1887 __kmp_assert_valid_gtid(gtid);
1890 kmp_taskdata_t *parent = NULL;
1891 if (UNLIKELY(ompt_enabled.enabled)) {
1892 if (!new_taskdata->td_flags.started) {
1893 OMPT_STORE_RETURN_ADDRESS(gtid);
1894 parent = new_taskdata->td_parent;
1895 if (!parent->ompt_task_info.frame.enter_frame.ptr) {
1896 parent->ompt_task_info.frame.enter_frame.ptr =
1897 OMPT_GET_FRAME_ADDRESS(0);
1899 if (ompt_enabled.ompt_callback_task_create) {
1900 ompt_callbacks.ompt_callback(ompt_callback_task_create)(
1901 &(parent->ompt_task_info.task_data),
1902 &(parent->ompt_task_info.frame),
1903 &(new_taskdata->ompt_task_info.task_data),
1904 TASK_TYPE_DETAILS_FORMAT(new_taskdata), 0,
1905 OMPT_LOAD_RETURN_ADDRESS(gtid));
1910 __ompt_task_finish(new_task,
1911 new_taskdata->ompt_task_info.scheduling_parent,
1913 new_taskdata->ompt_task_info.frame.exit_frame = ompt_data_none;
1918 res = __kmp_omp_task(gtid, new_task,
true);
1920 KA_TRACE(10, (
"__kmpc_omp_task(exit): T#%d returning "
1921 "TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n",
1922 gtid, loc_ref, new_taskdata));
1924 if (UNLIKELY(ompt_enabled.enabled && parent != NULL)) {
1925 parent->ompt_task_info.frame.enter_frame = ompt_data_none;
1944 kmp_int32 __kmp_omp_taskloop_task(
ident_t *loc_ref, kmp_int32 gtid,
1945 kmp_task_t *new_task,
void *codeptr_ra) {
1947 KMP_SET_THREAD_STATE_BLOCK(EXPLICIT_TASK);
1949 #if KMP_DEBUG || OMPT_SUPPORT
1950 kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
1952 KA_TRACE(10, (
"__kmpc_omp_task(enter): T#%d loc=%p task=%p\n", gtid, loc_ref,
1956 kmp_taskdata_t *parent = NULL;
1957 if (UNLIKELY(ompt_enabled.enabled && !new_taskdata->td_flags.started)) {
1958 parent = new_taskdata->td_parent;
1959 if (!parent->ompt_task_info.frame.enter_frame.ptr)
1960 parent->ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1961 if (ompt_enabled.ompt_callback_task_create) {
1962 ompt_callbacks.ompt_callback(ompt_callback_task_create)(
1963 &(parent->ompt_task_info.task_data), &(parent->ompt_task_info.frame),
1964 &(new_taskdata->ompt_task_info.task_data),
1965 TASK_TYPE_DETAILS_FORMAT(new_taskdata), 0, codeptr_ra);
1970 res = __kmp_omp_task(gtid, new_task,
true);
1972 KA_TRACE(10, (
"__kmpc_omp_task(exit): T#%d returning "
1973 "TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n",
1974 gtid, loc_ref, new_taskdata));
1976 if (UNLIKELY(ompt_enabled.enabled && parent != NULL)) {
1977 parent->ompt_task_info.frame.enter_frame = ompt_data_none;
1983 template <
bool ompt>
1984 static kmp_int32 __kmpc_omp_taskwait_template(
ident_t *loc_ref, kmp_int32 gtid,
1985 void *frame_address,
1986 void *return_address) {
1987 kmp_taskdata_t *taskdata =
nullptr;
1989 int thread_finished = FALSE;
1990 KMP_SET_THREAD_STATE_BLOCK(TASKWAIT);
1992 KA_TRACE(10, (
"__kmpc_omp_taskwait(enter): T#%d loc=%p\n", gtid, loc_ref));
1993 KMP_DEBUG_ASSERT(gtid >= 0);
1995 if (__kmp_tasking_mode != tskm_immediate_exec) {
1996 thread = __kmp_threads[gtid];
1997 taskdata = thread->th.th_current_task;
1999 #if OMPT_SUPPORT && OMPT_OPTIONAL
2000 ompt_data_t *my_task_data;
2001 ompt_data_t *my_parallel_data;
2004 my_task_data = &(taskdata->ompt_task_info.task_data);
2005 my_parallel_data = OMPT_CUR_TEAM_DATA(thread);
2007 taskdata->ompt_task_info.frame.enter_frame.ptr = frame_address;
2009 if (ompt_enabled.ompt_callback_sync_region) {
2010 ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
2011 ompt_sync_region_taskwait, ompt_scope_begin, my_parallel_data,
2012 my_task_data, return_address);
2015 if (ompt_enabled.ompt_callback_sync_region_wait) {
2016 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
2017 ompt_sync_region_taskwait, ompt_scope_begin, my_parallel_data,
2018 my_task_data, return_address);
2028 taskdata->td_taskwait_counter += 1;
2029 taskdata->td_taskwait_ident = loc_ref;
2030 taskdata->td_taskwait_thread = gtid + 1;
2033 void *itt_sync_obj = NULL;
2035 KMP_ITT_TASKWAIT_STARTING(itt_sync_obj);
2040 !taskdata->td_flags.team_serial && !taskdata->td_flags.final;
2042 must_wait = must_wait || (thread->th.th_task_team != NULL &&
2043 thread->th.th_task_team->tt.tt_found_proxy_tasks);
2047 (__kmp_enable_hidden_helper && thread->th.th_task_team != NULL &&
2048 thread->th.th_task_team->tt.tt_hidden_helper_task_encountered);
2051 kmp_flag_32<false, false> flag(
2052 RCAST(std::atomic<kmp_uint32> *,
2053 &(taskdata->td_incomplete_child_tasks)),
2055 while (KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks) != 0) {
2056 flag.execute_tasks(thread, gtid, FALSE,
2057 &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj),
2058 __kmp_task_stealing_constraint);
2062 KMP_ITT_TASKWAIT_FINISHED(itt_sync_obj);
2063 KMP_FSYNC_ACQUIRED(taskdata);
2068 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread;
2070 #if OMPT_SUPPORT && OMPT_OPTIONAL
2072 if (ompt_enabled.ompt_callback_sync_region_wait) {
2073 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
2074 ompt_sync_region_taskwait, ompt_scope_end, my_parallel_data,
2075 my_task_data, return_address);
2077 if (ompt_enabled.ompt_callback_sync_region) {
2078 ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
2079 ompt_sync_region_taskwait, ompt_scope_end, my_parallel_data,
2080 my_task_data, return_address);
2082 taskdata->ompt_task_info.frame.enter_frame = ompt_data_none;
2087 KA_TRACE(10, (
"__kmpc_omp_taskwait(exit): T#%d task %p finished waiting, "
2088 "returning TASK_CURRENT_NOT_QUEUED\n",
2091 return TASK_CURRENT_NOT_QUEUED;
2094 #if OMPT_SUPPORT && OMPT_OPTIONAL
2096 static kmp_int32 __kmpc_omp_taskwait_ompt(
ident_t *loc_ref, kmp_int32 gtid,
2097 void *frame_address,
2098 void *return_address) {
2099 return __kmpc_omp_taskwait_template<true>(loc_ref, gtid, frame_address,
2106 kmp_int32 __kmpc_omp_taskwait(
ident_t *loc_ref, kmp_int32 gtid) {
2107 #if OMPT_SUPPORT && OMPT_OPTIONAL
2108 if (UNLIKELY(ompt_enabled.enabled)) {
2109 OMPT_STORE_RETURN_ADDRESS(gtid);
2110 return __kmpc_omp_taskwait_ompt(loc_ref, gtid, OMPT_GET_FRAME_ADDRESS(0),
2111 OMPT_LOAD_RETURN_ADDRESS(gtid));
2114 return __kmpc_omp_taskwait_template<false>(loc_ref, gtid, NULL, NULL);
2118 kmp_int32 __kmpc_omp_taskyield(
ident_t *loc_ref, kmp_int32 gtid,
int end_part) {
2119 kmp_taskdata_t *taskdata = NULL;
2121 int thread_finished = FALSE;
2124 KMP_SET_THREAD_STATE_BLOCK(TASKYIELD);
2126 KA_TRACE(10, (
"__kmpc_omp_taskyield(enter): T#%d loc=%p end_part = %d\n",
2127 gtid, loc_ref, end_part));
2128 __kmp_assert_valid_gtid(gtid);
2130 if (__kmp_tasking_mode != tskm_immediate_exec && __kmp_init_parallel) {
2131 thread = __kmp_threads[gtid];
2132 taskdata = thread->th.th_current_task;
2139 taskdata->td_taskwait_counter += 1;
2140 taskdata->td_taskwait_ident = loc_ref;
2141 taskdata->td_taskwait_thread = gtid + 1;
2144 void *itt_sync_obj = NULL;
2146 KMP_ITT_TASKWAIT_STARTING(itt_sync_obj);
2149 if (!taskdata->td_flags.team_serial) {
2150 kmp_task_team_t *task_team = thread->th.th_task_team;
2151 if (task_team != NULL) {
2152 if (KMP_TASKING_ENABLED(task_team)) {
2154 if (UNLIKELY(ompt_enabled.enabled))
2155 thread->th.ompt_thread_info.ompt_task_yielded = 1;
2157 __kmp_execute_tasks_32(
2158 thread, gtid, (kmp_flag_32<> *)NULL, FALSE,
2159 &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj),
2160 __kmp_task_stealing_constraint);
2162 if (UNLIKELY(ompt_enabled.enabled))
2163 thread->th.ompt_thread_info.ompt_task_yielded = 0;
2169 KMP_ITT_TASKWAIT_FINISHED(itt_sync_obj);
2174 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread;
2177 KA_TRACE(10, (
"__kmpc_omp_taskyield(exit): T#%d task %p resuming, "
2178 "returning TASK_CURRENT_NOT_QUEUED\n",
2181 return TASK_CURRENT_NOT_QUEUED;
2202 unsigned reserved31 : 31;
2282 template <
typename T>
2283 void *__kmp_task_reduction_init(
int gtid,
int num, T *data) {
2284 __kmp_assert_valid_gtid(gtid);
2285 kmp_info_t *thread = __kmp_threads[gtid];
2286 kmp_taskgroup_t *tg = thread->th.th_current_task->td_taskgroup;
2287 kmp_uint32 nth = thread->th.th_team_nproc;
2291 KMP_ASSERT(tg != NULL);
2292 KMP_ASSERT(data != NULL);
2293 KMP_ASSERT(num > 0);
2294 if (nth == 1 && !__kmp_enable_hidden_helper) {
2295 KA_TRACE(10, (
"__kmpc_task_reduction_init: T#%d, tg %p, exiting nth=1\n",
2299 KA_TRACE(10, (
"__kmpc_task_reduction_init: T#%d, taskgroup %p, #items %d\n",
2303 for (
int i = 0; i < num; ++i) {
2304 size_t size = data[i].reduce_size - 1;
2306 size += CACHE_LINE - size % CACHE_LINE;
2307 KMP_ASSERT(data[i].reduce_comb != NULL);
2310 arr[i].
flags = data[i].flags;
2314 __kmp_assign_orig<T>(arr[i], data[i]);
2315 if (!arr[i].flags.lazy_priv) {
2318 arr[i].
reduce_pend = (
char *)(arr[i].reduce_priv) + nth * size;
2319 if (arr[i].reduce_init != NULL) {
2321 for (
size_t j = 0; j < nth; ++j) {
2322 __kmp_call_init<T>(arr[i], j * size);
2329 arr[i].
reduce_priv = __kmp_allocate(nth *
sizeof(
void *));
2332 tg->reduce_data = (
void *)arr;
2333 tg->reduce_num_data = num;
2353 kmp_tdg_info_t *tdg = __kmp_find_tdg(__kmp_curr_tdg_idx);
2354 if (tdg && __kmp_tdg_is_recording(tdg->tdg_status)) {
2355 kmp_tdg_info_t *this_tdg = __kmp_global_tdgs[__kmp_curr_tdg_idx];
2356 this_tdg->rec_taskred_data =
2358 this_tdg->rec_num_taskred = num;
2359 KMP_MEMCPY(this_tdg->rec_taskred_data, data,
2380 kmp_tdg_info_t *tdg = __kmp_find_tdg(__kmp_curr_tdg_idx);
2381 if (tdg && __kmp_tdg_is_recording(tdg->tdg_status)) {
2382 kmp_tdg_info_t *this_tdg = __kmp_global_tdgs[__kmp_curr_tdg_idx];
2383 this_tdg->rec_taskred_data =
2385 this_tdg->rec_num_taskred = num;
2386 KMP_MEMCPY(this_tdg->rec_taskred_data, data,
2394 template <
typename T>
2395 void __kmp_task_reduction_init_copy(kmp_info_t *thr,
int num, T *data,
2396 kmp_taskgroup_t *tg,
void *reduce_data) {
2398 KA_TRACE(20, (
"__kmp_task_reduction_init_copy: Th %p, init taskgroup %p,"
2400 thr, tg, reduce_data));
2405 for (
int i = 0; i < num; ++i) {
2408 tg->reduce_data = (
void *)arr;
2409 tg->reduce_num_data = num;
2422 __kmp_assert_valid_gtid(gtid);
2423 kmp_info_t *thread = __kmp_threads[gtid];
2424 kmp_int32 nth = thread->th.th_team_nproc;
2428 kmp_taskgroup_t *tg = (kmp_taskgroup_t *)tskgrp;
2430 tg = thread->th.th_current_task->td_taskgroup;
2431 KMP_ASSERT(tg != NULL);
2434 kmp_int32 tid = thread->th.th_info.ds.ds_tid;
2437 if ((thread->th.th_current_task->is_taskgraph) &&
2438 (!__kmp_tdg_is_recording(
2439 __kmp_global_tdgs[__kmp_curr_tdg_idx]->tdg_status))) {
2440 tg = thread->th.th_current_task->td_taskgroup;
2441 KMP_ASSERT(tg != NULL);
2442 KMP_ASSERT(tg->reduce_data != NULL);
2444 num = tg->reduce_num_data;
2448 KMP_ASSERT(data != NULL);
2449 while (tg != NULL) {
2451 num = tg->reduce_num_data;
2452 for (
int i = 0; i < num; ++i) {
2453 if (!arr[i].flags.lazy_priv) {
2454 if (data == arr[i].reduce_shar ||
2455 (data >= arr[i].reduce_priv && data < arr[i].reduce_pend))
2456 return (
char *)(arr[i].
reduce_priv) + tid * arr[i].reduce_size;
2459 void **p_priv = (
void **)(arr[i].reduce_priv);
2460 if (data == arr[i].reduce_shar)
2463 for (
int j = 0; j < nth; ++j)
2464 if (data == p_priv[j])
2468 if (p_priv[tid] == NULL) {
2470 p_priv[tid] = __kmp_allocate(arr[i].reduce_size);
2471 if (arr[i].reduce_init != NULL) {
2472 if (arr[i].reduce_orig != NULL) {
2474 p_priv[tid], arr[i].reduce_orig);
2476 ((void (*)(
void *))arr[i].
reduce_init)(p_priv[tid]);
2483 KMP_ASSERT(tg->parent);
2486 KMP_ASSERT2(0,
"Unknown task reduction item");
2492 static void __kmp_task_reduction_fini(kmp_info_t *th, kmp_taskgroup_t *tg) {
2493 kmp_int32 nth = th->th.th_team_nproc;
2496 __kmp_enable_hidden_helper);
2499 kmp_int32 num = tg->reduce_num_data;
2500 for (
int i = 0; i < num; ++i) {
2502 void (*f_fini)(
void *) = (
void (*)(
void *))(arr[i].
reduce_fini);
2503 void (*f_comb)(
void *,
void *) =
2505 if (!arr[i].flags.lazy_priv) {
2508 for (
int j = 0; j < nth; ++j) {
2509 void *priv_data = (
char *)pr_data + j * size;
2510 f_comb(sh_data, priv_data);
2515 void **pr_data = (
void **)(arr[i].reduce_priv);
2516 for (
int j = 0; j < nth; ++j) {
2517 if (pr_data[j] != NULL) {
2518 f_comb(sh_data, pr_data[j]);
2521 __kmp_free(pr_data[j]);
2525 __kmp_free(arr[i].reduce_priv);
2527 __kmp_thread_free(th, arr);
2528 tg->reduce_data = NULL;
2529 tg->reduce_num_data = 0;
2535 static void __kmp_task_reduction_clean(kmp_info_t *th, kmp_taskgroup_t *tg) {
2536 __kmp_thread_free(th, tg->reduce_data);
2537 tg->reduce_data = NULL;
2538 tg->reduce_num_data = 0;
2541 template <
typename T>
2542 void *__kmp_task_reduction_modifier_init(
ident_t *loc,
int gtid,
int is_ws,
2544 __kmp_assert_valid_gtid(gtid);
2545 kmp_info_t *thr = __kmp_threads[gtid];
2546 kmp_int32 nth = thr->th.th_team_nproc;
2547 __kmpc_taskgroup(loc, gtid);
2550 (
"__kmpc_reduction_modifier_init: T#%d, tg %p, exiting nth=1\n",
2551 gtid, thr->th.th_current_task->td_taskgroup));
2552 return (
void *)thr->th.th_current_task->td_taskgroup;
2554 kmp_team_t *team = thr->th.th_team;
2556 kmp_taskgroup_t *tg;
2557 reduce_data = KMP_ATOMIC_LD_RLX(&team->t.t_tg_reduce_data[is_ws]);
2558 if (reduce_data == NULL &&
2559 __kmp_atomic_compare_store(&team->t.t_tg_reduce_data[is_ws], reduce_data,
2562 KMP_DEBUG_ASSERT(reduce_data == NULL);
2564 tg = (kmp_taskgroup_t *)__kmp_task_reduction_init<T>(gtid, num, data);
2568 KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&team->t.t_tg_fini_counter[0]) == 0);
2569 KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&team->t.t_tg_fini_counter[1]) == 0);
2570 KMP_ATOMIC_ST_REL(&team->t.t_tg_reduce_data[is_ws], reduce_data);
2573 (reduce_data = KMP_ATOMIC_LD_ACQ(&team->t.t_tg_reduce_data[is_ws])) ==
2577 KMP_DEBUG_ASSERT(reduce_data > (
void *)1);
2578 tg = thr->th.th_current_task->td_taskgroup;
2579 __kmp_task_reduction_init_copy<T>(thr, num, data, tg, reduce_data);
2601 int num,
void *data) {
2602 return __kmp_task_reduction_modifier_init(loc, gtid, is_ws, num,
2622 return __kmp_task_reduction_modifier_init(loc, gtid, is_ws, num,
2635 __kmpc_end_taskgroup(loc, gtid);
2639 void __kmpc_taskgroup(
ident_t *loc,
int gtid) {
2640 __kmp_assert_valid_gtid(gtid);
2641 kmp_info_t *thread = __kmp_threads[gtid];
2642 kmp_taskdata_t *taskdata = thread->th.th_current_task;
2643 kmp_taskgroup_t *tg_new =
2644 (kmp_taskgroup_t *)__kmp_thread_malloc(thread,
sizeof(kmp_taskgroup_t));
2645 KA_TRACE(10, (
"__kmpc_taskgroup: T#%d loc=%p group=%p\n", gtid, loc, tg_new));
2646 KMP_ATOMIC_ST_RLX(&tg_new->count, 0);
2647 KMP_ATOMIC_ST_RLX(&tg_new->cancel_request, cancel_noreq);
2648 tg_new->parent = taskdata->td_taskgroup;
2649 tg_new->reduce_data = NULL;
2650 tg_new->reduce_num_data = 0;
2651 tg_new->gomp_data = NULL;
2652 taskdata->td_taskgroup = tg_new;
2654 #if OMPT_SUPPORT && OMPT_OPTIONAL
2655 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region)) {
2656 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2658 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2659 kmp_team_t *team = thread->th.th_team;
2660 ompt_data_t my_task_data = taskdata->ompt_task_info.task_data;
2662 ompt_data_t my_parallel_data = team->t.ompt_team_info.parallel_data;
2664 ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
2665 ompt_sync_region_taskgroup, ompt_scope_begin, &(my_parallel_data),
2666 &(my_task_data), codeptr);
2673 void __kmpc_end_taskgroup(
ident_t *loc,
int gtid) {
2674 __kmp_assert_valid_gtid(gtid);
2675 kmp_info_t *thread = __kmp_threads[gtid];
2676 kmp_taskdata_t *taskdata = thread->th.th_current_task;
2677 kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup;
2678 int thread_finished = FALSE;
2680 #if OMPT_SUPPORT && OMPT_OPTIONAL
2682 ompt_data_t my_task_data;
2683 ompt_data_t my_parallel_data;
2684 void *codeptr =
nullptr;
2685 if (UNLIKELY(ompt_enabled.enabled)) {
2686 team = thread->th.th_team;
2687 my_task_data = taskdata->ompt_task_info.task_data;
2689 my_parallel_data = team->t.ompt_team_info.parallel_data;
2690 codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2692 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2696 KA_TRACE(10, (
"__kmpc_end_taskgroup(enter): T#%d loc=%p\n", gtid, loc));
2697 KMP_DEBUG_ASSERT(taskgroup != NULL);
2698 KMP_SET_THREAD_STATE_BLOCK(TASKGROUP);
2700 if (__kmp_tasking_mode != tskm_immediate_exec) {
2702 taskdata->td_taskwait_counter += 1;
2703 taskdata->td_taskwait_ident = loc;
2704 taskdata->td_taskwait_thread = gtid + 1;
2708 void *itt_sync_obj = NULL;
2710 KMP_ITT_TASKWAIT_STARTING(itt_sync_obj);
2714 #if OMPT_SUPPORT && OMPT_OPTIONAL
2715 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region_wait)) {
2716 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
2717 ompt_sync_region_taskgroup, ompt_scope_begin, &(my_parallel_data),
2718 &(my_task_data), codeptr);
2722 if (!taskdata->td_flags.team_serial ||
2723 (thread->th.th_task_team != NULL &&
2724 (thread->th.th_task_team->tt.tt_found_proxy_tasks ||
2725 thread->th.th_task_team->tt.tt_hidden_helper_task_encountered))) {
2726 kmp_flag_32<false, false> flag(
2727 RCAST(std::atomic<kmp_uint32> *, &(taskgroup->count)), 0U);
2728 while (KMP_ATOMIC_LD_ACQ(&taskgroup->count) != 0) {
2729 flag.execute_tasks(thread, gtid, FALSE,
2730 &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj),
2731 __kmp_task_stealing_constraint);
2734 taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread;
2736 #if OMPT_SUPPORT && OMPT_OPTIONAL
2737 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region_wait)) {
2738 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
2739 ompt_sync_region_taskgroup, ompt_scope_end, &(my_parallel_data),
2740 &(my_task_data), codeptr);
2745 KMP_ITT_TASKWAIT_FINISHED(itt_sync_obj);
2746 KMP_FSYNC_ACQUIRED(taskdata);
2749 KMP_DEBUG_ASSERT(taskgroup->count == 0);
2751 if (taskgroup->reduce_data != NULL &&
2752 !taskgroup->gomp_data) {
2755 kmp_team_t *t = thread->th.th_team;
2759 if ((reduce_data = KMP_ATOMIC_LD_ACQ(&t->t.t_tg_reduce_data[0])) != NULL &&
2762 cnt = KMP_ATOMIC_INC(&t->t.t_tg_fini_counter[0]);
2763 if (cnt == thread->th.th_team_nproc - 1) {
2766 __kmp_task_reduction_fini(thread, taskgroup);
2769 __kmp_thread_free(thread, reduce_data);
2770 KMP_ATOMIC_ST_REL(&t->t.t_tg_reduce_data[0], NULL);
2771 KMP_ATOMIC_ST_REL(&t->t.t_tg_fini_counter[0], 0);
2775 __kmp_task_reduction_clean(thread, taskgroup);
2777 }
else if ((reduce_data = KMP_ATOMIC_LD_ACQ(&t->t.t_tg_reduce_data[1])) !=
2781 cnt = KMP_ATOMIC_INC(&t->t.t_tg_fini_counter[1]);
2782 if (cnt == thread->th.th_team_nproc - 1) {
2784 __kmp_task_reduction_fini(thread, taskgroup);
2787 __kmp_thread_free(thread, reduce_data);
2788 KMP_ATOMIC_ST_REL(&t->t.t_tg_reduce_data[1], NULL);
2789 KMP_ATOMIC_ST_REL(&t->t.t_tg_fini_counter[1], 0);
2793 __kmp_task_reduction_clean(thread, taskgroup);
2797 __kmp_task_reduction_fini(thread, taskgroup);
2801 taskdata->td_taskgroup = taskgroup->parent;
2802 __kmp_thread_free(thread, taskgroup);
2804 KA_TRACE(10, (
"__kmpc_end_taskgroup(exit): T#%d task %p finished waiting\n",
2807 #if OMPT_SUPPORT && OMPT_OPTIONAL
2808 if (UNLIKELY(ompt_enabled.ompt_callback_sync_region)) {
2809 ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
2810 ompt_sync_region_taskgroup, ompt_scope_end, &(my_parallel_data),
2811 &(my_task_data), codeptr);
2816 static kmp_task_t *__kmp_get_priority_task(kmp_int32 gtid,
2817 kmp_task_team_t *task_team,
2818 kmp_int32 is_constrained) {
2819 kmp_task_t *task = NULL;
2820 kmp_taskdata_t *taskdata;
2821 kmp_taskdata_t *current;
2822 kmp_thread_data_t *thread_data;
2823 int ntasks = task_team->tt.tt_num_task_pri;
2826 20, (
"__kmp_get_priority_task(exit #1): T#%d No tasks to get\n", gtid));
2831 if (__kmp_atomic_compare_store(&task_team->tt.tt_num_task_pri, ntasks,
2834 ntasks = task_team->tt.tt_num_task_pri;
2835 }
while (ntasks > 0);
2837 KA_TRACE(20, (
"__kmp_get_priority_task(exit #2): T#%d No tasks to get\n",
2843 kmp_task_pri_t *list = task_team->tt.tt_task_pri_list;
2845 KMP_ASSERT(list != NULL);
2846 thread_data = &list->td;
2847 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
2848 deque_ntasks = thread_data->td.td_deque_ntasks;
2849 if (deque_ntasks == 0) {
2850 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
2851 KA_TRACE(20, (
"__kmp_get_priority_task: T#%d No tasks to get from %p\n",
2852 __kmp_get_gtid(), thread_data));
2855 }
while (deque_ntasks == 0);
2856 KMP_DEBUG_ASSERT(deque_ntasks);
2857 int target = thread_data->td.td_deque_head;
2858 current = __kmp_threads[gtid]->th.th_current_task;
2859 taskdata = thread_data->td.td_deque[target];
2860 if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) {
2862 thread_data->td.td_deque_head =
2863 (target + 1) & TASK_DEQUE_MASK(thread_data->td);
2865 if (!task_team->tt.tt_untied_task_encountered) {
2867 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
2868 KA_TRACE(20, (
"__kmp_get_priority_task(exit #3): T#%d could not get task "
2869 "from %p: task_team=%p ntasks=%d head=%u tail=%u\n",
2870 gtid, thread_data, task_team, deque_ntasks, target,
2871 thread_data->td.td_deque_tail));
2872 task_team->tt.tt_num_task_pri++;
2878 for (i = 1; i < deque_ntasks; ++i) {
2879 target = (target + 1) & TASK_DEQUE_MASK(thread_data->td);
2880 taskdata = thread_data->td.td_deque[target];
2881 if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) {
2887 if (taskdata == NULL) {
2889 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
2891 10, (
"__kmp_get_priority_task(exit #4): T#%d could not get task from "
2892 "%p: task_team=%p ntasks=%d head=%u tail=%u\n",
2893 gtid, thread_data, task_team, deque_ntasks,
2894 thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
2895 task_team->tt.tt_num_task_pri++;
2899 for (i = i + 1; i < deque_ntasks; ++i) {
2901 target = (target + 1) & TASK_DEQUE_MASK(thread_data->td);
2902 thread_data->td.td_deque[prev] = thread_data->td.td_deque[target];
2906 thread_data->td.td_deque_tail ==
2907 (kmp_uint32)((target + 1) & TASK_DEQUE_MASK(thread_data->td)));
2908 thread_data->td.td_deque_tail = target;
2910 thread_data->td.td_deque_ntasks = deque_ntasks - 1;
2911 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
2912 task = KMP_TASKDATA_TO_TASK(taskdata);
2917 static kmp_task_t *__kmp_remove_my_task(kmp_info_t *thread, kmp_int32 gtid,
2918 kmp_task_team_t *task_team,
2919 kmp_int32 is_constrained) {
2921 kmp_taskdata_t *taskdata;
2922 kmp_thread_data_t *thread_data;
2925 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
2926 KMP_DEBUG_ASSERT(task_team->tt.tt_threads_data !=
2929 thread_data = &task_team->tt.tt_threads_data[__kmp_tid_from_gtid(gtid)];
2931 KA_TRACE(10, (
"__kmp_remove_my_task(enter): T#%d ntasks=%d head=%u tail=%u\n",
2932 gtid, thread_data->td.td_deque_ntasks,
2933 thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
2935 if (TCR_4(thread_data->td.td_deque_ntasks) == 0) {
2937 (
"__kmp_remove_my_task(exit #1): T#%d No tasks to remove: "
2938 "ntasks=%d head=%u tail=%u\n",
2939 gtid, thread_data->td.td_deque_ntasks,
2940 thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
2944 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
2946 if (TCR_4(thread_data->td.td_deque_ntasks) == 0) {
2947 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
2949 (
"__kmp_remove_my_task(exit #2): T#%d No tasks to remove: "
2950 "ntasks=%d head=%u tail=%u\n",
2951 gtid, thread_data->td.td_deque_ntasks,
2952 thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
2956 tail = (thread_data->td.td_deque_tail - 1) &
2957 TASK_DEQUE_MASK(thread_data->td);
2958 taskdata = thread_data->td.td_deque[tail];
2960 if (!__kmp_task_is_allowed(gtid, is_constrained, taskdata,
2961 thread->th.th_current_task)) {
2963 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
2965 (
"__kmp_remove_my_task(exit #3): T#%d TSC blocks tail task: "
2966 "ntasks=%d head=%u tail=%u\n",
2967 gtid, thread_data->td.td_deque_ntasks,
2968 thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
2972 thread_data->td.td_deque_tail = tail;
2973 TCW_4(thread_data->td.td_deque_ntasks, thread_data->td.td_deque_ntasks - 1);
2975 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
2977 KA_TRACE(10, (
"__kmp_remove_my_task(exit #4): T#%d task %p removed: "
2978 "ntasks=%d head=%u tail=%u\n",
2979 gtid, taskdata, thread_data->td.td_deque_ntasks,
2980 thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
2982 task = KMP_TASKDATA_TO_TASK(taskdata);
2989 static kmp_task_t *__kmp_steal_task(kmp_int32 victim_tid, kmp_int32 gtid,
2990 kmp_task_team_t *task_team,
2991 std::atomic<kmp_int32> *unfinished_threads,
2992 int *thread_finished,
2993 kmp_int32 is_constrained) {
2995 kmp_taskdata_t *taskdata;
2996 kmp_taskdata_t *current;
2997 kmp_thread_data_t *victim_td, *threads_data;
2999 kmp_info_t *victim_thr;
3001 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
3003 threads_data = task_team->tt.tt_threads_data;
3004 KMP_DEBUG_ASSERT(threads_data != NULL);
3005 KMP_DEBUG_ASSERT(victim_tid >= 0);
3006 KMP_DEBUG_ASSERT(victim_tid < task_team->tt.tt_max_threads);
3008 victim_td = &threads_data[victim_tid];
3009 victim_thr = victim_td->td.td_thr;
3012 KA_TRACE(10, (
"__kmp_steal_task(enter): T#%d try to steal from T#%d: "
3013 "task_team=%p ntasks=%d head=%u tail=%u\n",
3014 gtid, __kmp_gtid_from_thread(victim_thr), task_team,
3015 victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head,
3016 victim_td->td.td_deque_tail));
3018 if (TCR_4(victim_td->td.td_deque_ntasks) == 0) {
3019 KA_TRACE(10, (
"__kmp_steal_task(exit #1): T#%d could not steal from T#%d: "
3020 "task_team=%p ntasks=%d head=%u tail=%u\n",
3021 gtid, __kmp_gtid_from_thread(victim_thr), task_team,
3022 victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head,
3023 victim_td->td.td_deque_tail));
3027 __kmp_acquire_bootstrap_lock(&victim_td->td.td_deque_lock);
3029 int ntasks = TCR_4(victim_td->td.td_deque_ntasks);
3032 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock);
3033 KA_TRACE(10, (
"__kmp_steal_task(exit #2): T#%d could not steal from T#%d: "
3034 "task_team=%p ntasks=%d head=%u tail=%u\n",
3035 gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks,
3036 victim_td->td.td_deque_head, victim_td->td.td_deque_tail));
3040 KMP_DEBUG_ASSERT(victim_td->td.td_deque != NULL);
3041 current = __kmp_threads[gtid]->th.th_current_task;
3042 taskdata = victim_td->td.td_deque[victim_td->td.td_deque_head];
3043 if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) {
3045 victim_td->td.td_deque_head =
3046 (victim_td->td.td_deque_head + 1) & TASK_DEQUE_MASK(victim_td->td);
3048 if (!task_team->tt.tt_untied_task_encountered) {
3050 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock);
3051 KA_TRACE(10, (
"__kmp_steal_task(exit #3): T#%d could not steal from "
3052 "T#%d: task_team=%p ntasks=%d head=%u tail=%u\n",
3053 gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks,
3054 victim_td->td.td_deque_head, victim_td->td.td_deque_tail));
3059 target = victim_td->td.td_deque_head;
3061 for (i = 1; i < ntasks; ++i) {
3062 target = (target + 1) & TASK_DEQUE_MASK(victim_td->td);
3063 taskdata = victim_td->td.td_deque[target];
3064 if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) {
3070 if (taskdata == NULL) {
3072 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock);
3073 KA_TRACE(10, (
"__kmp_steal_task(exit #4): T#%d could not steal from "
3074 "T#%d: task_team=%p ntasks=%d head=%u tail=%u\n",
3075 gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks,
3076 victim_td->td.td_deque_head, victim_td->td.td_deque_tail));
3080 for (i = i + 1; i < ntasks; ++i) {
3082 target = (target + 1) & TASK_DEQUE_MASK(victim_td->td);
3083 victim_td->td.td_deque[prev] = victim_td->td.td_deque[target];
3087 victim_td->td.td_deque_tail ==
3088 (kmp_uint32)((target + 1) & TASK_DEQUE_MASK(victim_td->td)));
3089 victim_td->td.td_deque_tail = target;
3091 if (*thread_finished) {
3098 KMP_ATOMIC_INC(unfinished_threads);
3101 (
"__kmp_steal_task: T#%d inc unfinished_threads to %d: task_team=%p\n",
3102 gtid, count + 1, task_team));
3103 *thread_finished = FALSE;
3105 TCW_4(victim_td->td.td_deque_ntasks, ntasks - 1);
3107 __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock);
3111 (
"__kmp_steal_task(exit #5): T#%d stole task %p from T#%d: "
3112 "task_team=%p ntasks=%d head=%u tail=%u\n",
3113 gtid, taskdata, __kmp_gtid_from_thread(victim_thr), task_team,
3114 ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail));
3116 task = KMP_TASKDATA_TO_TASK(taskdata);
3130 static inline int __kmp_execute_tasks_template(
3131 kmp_info_t *thread, kmp_int32 gtid, C *flag,
int final_spin,
3132 int *thread_finished USE_ITT_BUILD_ARG(
void *itt_sync_obj),
3133 kmp_int32 is_constrained) {
3134 kmp_task_team_t *task_team = thread->th.th_task_team;
3135 kmp_thread_data_t *threads_data;
3137 kmp_info_t *other_thread;
3138 kmp_taskdata_t *current_task = thread->th.th_current_task;
3139 std::atomic<kmp_int32> *unfinished_threads;
3140 kmp_int32 nthreads, victim_tid = -2, use_own_tasks = 1, new_victim = 0,
3141 tid = thread->th.th_info.ds.ds_tid;
3143 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
3144 KMP_DEBUG_ASSERT(thread == __kmp_threads[gtid]);
3146 if (task_team == NULL || current_task == NULL)
3149 KA_TRACE(15, (
"__kmp_execute_tasks_template(enter): T#%d final_spin=%d "
3150 "*thread_finished=%d\n",
3151 gtid, final_spin, *thread_finished));
3153 thread->th.th_reap_state = KMP_NOT_SAFE_TO_REAP;
3154 threads_data = (kmp_thread_data_t *)TCR_PTR(task_team->tt.tt_threads_data);
3156 KMP_DEBUG_ASSERT(threads_data != NULL);
3158 nthreads = task_team->tt.tt_nproc;
3159 unfinished_threads = &(task_team->tt.tt_unfinished_threads);
3160 KMP_DEBUG_ASSERT(*unfinished_threads >= 0);
3166 if (task_team->tt.tt_num_task_pri) {
3167 task = __kmp_get_priority_task(gtid, task_team, is_constrained);
3169 if (task == NULL && use_own_tasks) {
3170 task = __kmp_remove_my_task(thread, gtid, task_team, is_constrained);
3172 if ((task == NULL) && (nthreads > 1)) {
3176 if (victim_tid == -2) {
3177 victim_tid = threads_data[tid].td.td_deque_last_stolen;
3180 other_thread = threads_data[victim_tid].td.td_thr;
3182 if (victim_tid != -1) {
3184 }
else if (!new_victim) {
3190 victim_tid = __kmp_get_random(thread) % (nthreads - 1);
3191 if (victim_tid >= tid) {
3195 other_thread = threads_data[victim_tid].td.td_thr;
3205 if ((__kmp_tasking_mode == tskm_task_teams) &&
3206 (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) &&
3207 (TCR_PTR(CCAST(
void *, other_thread->th.th_sleep_loc)) !=
3210 __kmp_null_resume_wrapper(other_thread);
3224 __kmp_steal_task(victim_tid, gtid, task_team, unfinished_threads,
3225 thread_finished, is_constrained);
3228 if (threads_data[tid].td.td_deque_last_stolen != victim_tid) {
3229 threads_data[tid].td.td_deque_last_stolen = victim_tid;
3236 KMP_CHECK_UPDATE(threads_data[tid].td.td_deque_last_stolen, -1);
3245 #if USE_ITT_BUILD && USE_ITT_NOTIFY
3246 if (__itt_sync_create_ptr || KMP_ITT_DEBUG) {
3247 if (itt_sync_obj == NULL) {
3249 itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier);
3251 __kmp_itt_task_starting(itt_sync_obj);
3254 __kmp_invoke_task(gtid, task, current_task);
3256 if (itt_sync_obj != NULL)
3257 __kmp_itt_task_finished(itt_sync_obj);
3264 if (flag == NULL || (!final_spin && flag->done_check())) {
3267 (
"__kmp_execute_tasks_template: T#%d spin condition satisfied\n",
3271 if (thread->th.th_task_team == NULL) {
3274 KMP_YIELD(__kmp_library == library_throughput);
3277 if (!use_own_tasks && TCR_4(threads_data[tid].td.td_deque_ntasks) != 0) {
3278 KA_TRACE(20, (
"__kmp_execute_tasks_template: T#%d stolen task spawned "
3279 "other tasks, restart\n",
3290 KMP_ATOMIC_LD_ACQ(¤t_task->td_incomplete_child_tasks) == 0) {
3294 if (!*thread_finished) {
3296 kmp_int32 count = -1 +
3298 KMP_ATOMIC_DEC(unfinished_threads);
3299 KA_TRACE(20, (
"__kmp_execute_tasks_template: T#%d dec "
3300 "unfinished_threads to %d task_team=%p\n",
3301 gtid, count, task_team));
3302 *thread_finished = TRUE;
3310 if (flag != NULL && flag->done_check()) {
3313 (
"__kmp_execute_tasks_template: T#%d spin condition satisfied\n",
3321 if (thread->th.th_task_team == NULL) {
3323 (
"__kmp_execute_tasks_template: T#%d no more tasks\n", gtid));
3332 if (flag == NULL || (!final_spin && flag->done_check())) {
3334 (
"__kmp_execute_tasks_template: T#%d spin condition satisfied\n",
3341 if (nthreads == 1 &&
3342 KMP_ATOMIC_LD_ACQ(¤t_task->td_incomplete_child_tasks))
3346 (
"__kmp_execute_tasks_template: T#%d can't find work\n", gtid));
3352 template <
bool C,
bool S>
3353 int __kmp_execute_tasks_32(
3354 kmp_info_t *thread, kmp_int32 gtid, kmp_flag_32<C, S> *flag,
int final_spin,
3355 int *thread_finished USE_ITT_BUILD_ARG(
void *itt_sync_obj),
3356 kmp_int32 is_constrained) {
3357 return __kmp_execute_tasks_template(
3358 thread, gtid, flag, final_spin,
3359 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
3362 template <
bool C,
bool S>
3363 int __kmp_execute_tasks_64(
3364 kmp_info_t *thread, kmp_int32 gtid, kmp_flag_64<C, S> *flag,
int final_spin,
3365 int *thread_finished USE_ITT_BUILD_ARG(
void *itt_sync_obj),
3366 kmp_int32 is_constrained) {
3367 return __kmp_execute_tasks_template(
3368 thread, gtid, flag, final_spin,
3369 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
3372 template <
bool C,
bool S>
3373 int __kmp_atomic_execute_tasks_64(
3374 kmp_info_t *thread, kmp_int32 gtid, kmp_atomic_flag_64<C, S> *flag,
3375 int final_spin,
int *thread_finished USE_ITT_BUILD_ARG(
void *itt_sync_obj),
3376 kmp_int32 is_constrained) {
3377 return __kmp_execute_tasks_template(
3378 thread, gtid, flag, final_spin,
3379 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
3382 int __kmp_execute_tasks_oncore(
3383 kmp_info_t *thread, kmp_int32 gtid, kmp_flag_oncore *flag,
int final_spin,
3384 int *thread_finished USE_ITT_BUILD_ARG(
void *itt_sync_obj),
3385 kmp_int32 is_constrained) {
3386 return __kmp_execute_tasks_template(
3387 thread, gtid, flag, final_spin,
3388 thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
3392 __kmp_execute_tasks_32<false, false>(kmp_info_t *, kmp_int32,
3393 kmp_flag_32<false, false> *,
int,
3394 int *USE_ITT_BUILD_ARG(
void *), kmp_int32);
3396 template int __kmp_execute_tasks_64<false, true>(kmp_info_t *, kmp_int32,
3397 kmp_flag_64<false, true> *,
3399 int *USE_ITT_BUILD_ARG(
void *),
3402 template int __kmp_execute_tasks_64<true, false>(kmp_info_t *, kmp_int32,
3403 kmp_flag_64<true, false> *,
3405 int *USE_ITT_BUILD_ARG(
void *),
3408 template int __kmp_atomic_execute_tasks_64<false, true>(
3409 kmp_info_t *, kmp_int32, kmp_atomic_flag_64<false, true> *,
int,
3410 int *USE_ITT_BUILD_ARG(
void *), kmp_int32);
3412 template int __kmp_atomic_execute_tasks_64<true, false>(
3413 kmp_info_t *, kmp_int32, kmp_atomic_flag_64<true, false> *,
int,
3414 int *USE_ITT_BUILD_ARG(
void *), kmp_int32);
3419 static void __kmp_enable_tasking(kmp_task_team_t *task_team,
3420 kmp_info_t *this_thr) {
3421 kmp_thread_data_t *threads_data;
3422 int nthreads, i, is_init_thread;
3424 KA_TRACE(10, (
"__kmp_enable_tasking(enter): T#%d\n",
3425 __kmp_gtid_from_thread(this_thr)));
3427 KMP_DEBUG_ASSERT(task_team != NULL);
3428 KMP_DEBUG_ASSERT(this_thr->th.th_team != NULL);
3430 nthreads = task_team->tt.tt_nproc;
3431 KMP_DEBUG_ASSERT(nthreads > 0);
3432 KMP_DEBUG_ASSERT(nthreads == this_thr->th.th_team->t.t_nproc);
3435 is_init_thread = __kmp_realloc_task_threads_data(this_thr, task_team);
3437 if (!is_init_thread) {
3441 (
"__kmp_enable_tasking(exit): T#%d: threads array already set up.\n",
3442 __kmp_gtid_from_thread(this_thr)));
3445 threads_data = (kmp_thread_data_t *)TCR_PTR(task_team->tt.tt_threads_data);
3446 KMP_DEBUG_ASSERT(threads_data != NULL);
3448 if (__kmp_tasking_mode == tskm_task_teams &&
3449 (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME)) {
3453 for (i = 0; i < nthreads; i++) {
3455 kmp_info_t *thread = threads_data[i].td.td_thr;
3457 if (i == this_thr->th.th_info.ds.ds_tid) {
3466 if ((sleep_loc = TCR_PTR(CCAST(
void *, thread->th.th_sleep_loc))) !=
3468 KF_TRACE(50, (
"__kmp_enable_tasking: T#%d waking up thread T#%d\n",
3469 __kmp_gtid_from_thread(this_thr),
3470 __kmp_gtid_from_thread(thread)));
3471 __kmp_null_resume_wrapper(thread);
3473 KF_TRACE(50, (
"__kmp_enable_tasking: T#%d don't wake up thread T#%d\n",
3474 __kmp_gtid_from_thread(this_thr),
3475 __kmp_gtid_from_thread(thread)));
3480 KA_TRACE(10, (
"__kmp_enable_tasking(exit): T#%d\n",
3481 __kmp_gtid_from_thread(this_thr)));
3514 static kmp_task_team_t *__kmp_free_task_teams =
3517 kmp_bootstrap_lock_t __kmp_task_team_lock =
3518 KMP_BOOTSTRAP_LOCK_INITIALIZER(__kmp_task_team_lock);
3525 static void __kmp_alloc_task_deque(kmp_info_t *thread,
3526 kmp_thread_data_t *thread_data) {
3527 __kmp_init_bootstrap_lock(&thread_data->td.td_deque_lock);
3528 KMP_DEBUG_ASSERT(thread_data->td.td_deque == NULL);
3531 thread_data->td.td_deque_last_stolen = -1;
3533 KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) == 0);
3534 KMP_DEBUG_ASSERT(thread_data->td.td_deque_head == 0);
3535 KMP_DEBUG_ASSERT(thread_data->td.td_deque_tail == 0);
3539 (
"__kmp_alloc_task_deque: T#%d allocating deque[%d] for thread_data %p\n",
3540 __kmp_gtid_from_thread(thread), INITIAL_TASK_DEQUE_SIZE, thread_data));
3544 thread_data->td.td_deque = (kmp_taskdata_t **)__kmp_allocate(
3545 INITIAL_TASK_DEQUE_SIZE *
sizeof(kmp_taskdata_t *));
3546 thread_data->td.td_deque_size = INITIAL_TASK_DEQUE_SIZE;
3552 static void __kmp_free_task_deque(kmp_thread_data_t *thread_data) {
3553 if (thread_data->td.td_deque != NULL) {
3554 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
3555 TCW_4(thread_data->td.td_deque_ntasks, 0);
3556 __kmp_free(thread_data->td.td_deque);
3557 thread_data->td.td_deque = NULL;
3558 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
3569 static int __kmp_realloc_task_threads_data(kmp_info_t *thread,
3570 kmp_task_team_t *task_team) {
3571 kmp_thread_data_t **threads_data_p;
3572 kmp_int32 nthreads, maxthreads;
3573 int is_init_thread = FALSE;
3575 if (TCR_4(task_team->tt.tt_found_tasks)) {
3580 threads_data_p = &task_team->tt.tt_threads_data;
3581 nthreads = task_team->tt.tt_nproc;
3582 maxthreads = task_team->tt.tt_max_threads;
3587 __kmp_acquire_bootstrap_lock(&task_team->tt.tt_threads_lock);
3589 if (!TCR_4(task_team->tt.tt_found_tasks)) {
3591 kmp_team_t *team = thread->th.th_team;
3594 is_init_thread = TRUE;
3595 if (maxthreads < nthreads) {
3597 if (*threads_data_p != NULL) {
3598 kmp_thread_data_t *old_data = *threads_data_p;
3599 kmp_thread_data_t *new_data = NULL;
3603 (
"__kmp_realloc_task_threads_data: T#%d reallocating "
3604 "threads data for task_team %p, new_size = %d, old_size = %d\n",
3605 __kmp_gtid_from_thread(thread), task_team, nthreads, maxthreads));
3610 new_data = (kmp_thread_data_t *)__kmp_allocate(
3611 nthreads *
sizeof(kmp_thread_data_t));
3613 KMP_MEMCPY_S((
void *)new_data, nthreads *
sizeof(kmp_thread_data_t),
3614 (
void *)old_data, maxthreads *
sizeof(kmp_thread_data_t));
3617 (*threads_data_p) = new_data;
3618 __kmp_free(old_data);
3620 KE_TRACE(10, (
"__kmp_realloc_task_threads_data: T#%d allocating "
3621 "threads data for task_team %p, size = %d\n",
3622 __kmp_gtid_from_thread(thread), task_team, nthreads));
3626 *threads_data_p = (kmp_thread_data_t *)__kmp_allocate(
3627 nthreads *
sizeof(kmp_thread_data_t));
3629 task_team->tt.tt_max_threads = nthreads;
3632 KMP_DEBUG_ASSERT(*threads_data_p != NULL);
3636 for (i = 0; i < nthreads; i++) {
3637 kmp_thread_data_t *thread_data = &(*threads_data_p)[i];
3638 thread_data->td.td_thr = team->t.t_threads[i];
3640 if (thread_data->td.td_deque_last_stolen >= nthreads) {
3644 thread_data->td.td_deque_last_stolen = -1;
3649 TCW_SYNC_4(task_team->tt.tt_found_tasks, TRUE);
3652 __kmp_release_bootstrap_lock(&task_team->tt.tt_threads_lock);
3653 return is_init_thread;
3659 static void __kmp_free_task_threads_data(kmp_task_team_t *task_team) {
3660 __kmp_acquire_bootstrap_lock(&task_team->tt.tt_threads_lock);
3661 if (task_team->tt.tt_threads_data != NULL) {
3663 for (i = 0; i < task_team->tt.tt_max_threads; i++) {
3664 __kmp_free_task_deque(&task_team->tt.tt_threads_data[i]);
3666 __kmp_free(task_team->tt.tt_threads_data);
3667 task_team->tt.tt_threads_data = NULL;
3669 __kmp_release_bootstrap_lock(&task_team->tt.tt_threads_lock);
3675 static void __kmp_free_task_pri_list(kmp_task_team_t *task_team) {
3676 __kmp_acquire_bootstrap_lock(&task_team->tt.tt_task_pri_lock);
3677 if (task_team->tt.tt_task_pri_list != NULL) {
3678 kmp_task_pri_t *list = task_team->tt.tt_task_pri_list;
3679 while (list != NULL) {
3680 kmp_task_pri_t *next = list->next;
3681 __kmp_free_task_deque(&list->td);
3685 task_team->tt.tt_task_pri_list = NULL;
3687 __kmp_release_bootstrap_lock(&task_team->tt.tt_task_pri_lock);
3690 static inline void __kmp_task_team_init(kmp_task_team_t *task_team,
3692 int team_nth = team->t.t_nproc;
3694 if (!task_team->tt.tt_active || team_nth != task_team->tt.tt_nproc) {
3695 TCW_4(task_team->tt.tt_found_tasks, FALSE);
3696 TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE);
3697 TCW_4(task_team->tt.tt_hidden_helper_task_encountered, FALSE);
3698 TCW_4(task_team->tt.tt_nproc, team_nth);
3699 KMP_ATOMIC_ST_REL(&task_team->tt.tt_unfinished_threads, team_nth);
3700 TCW_4(task_team->tt.tt_active, TRUE);
3708 static kmp_task_team_t *__kmp_allocate_task_team(kmp_info_t *thread,
3710 kmp_task_team_t *task_team = NULL;
3712 KA_TRACE(20, (
"__kmp_allocate_task_team: T#%d entering; team = %p\n",
3713 (thread ? __kmp_gtid_from_thread(thread) : -1), team));
3715 if (TCR_PTR(__kmp_free_task_teams) != NULL) {
3717 __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock);
3718 if (__kmp_free_task_teams != NULL) {
3719 task_team = __kmp_free_task_teams;
3720 TCW_PTR(__kmp_free_task_teams, task_team->tt.tt_next);
3721 task_team->tt.tt_next = NULL;
3723 __kmp_release_bootstrap_lock(&__kmp_task_team_lock);
3726 if (task_team == NULL) {
3727 KE_TRACE(10, (
"__kmp_allocate_task_team: T#%d allocating "
3728 "task team for team %p\n",
3729 __kmp_gtid_from_thread(thread), team));
3732 task_team = (kmp_task_team_t *)__kmp_allocate(
sizeof(kmp_task_team_t));
3733 __kmp_init_bootstrap_lock(&task_team->tt.tt_threads_lock);
3734 __kmp_init_bootstrap_lock(&task_team->tt.tt_task_pri_lock);
3735 #if USE_ITT_BUILD && USE_ITT_NOTIFY && KMP_DEBUG
3738 __itt_suppress_mark_range(
3739 __itt_suppress_range, __itt_suppress_threading_errors,
3740 &task_team->tt.tt_found_tasks,
sizeof(task_team->tt.tt_found_tasks));
3741 __itt_suppress_mark_range(__itt_suppress_range,
3742 __itt_suppress_threading_errors,
3743 CCAST(kmp_uint32 *, &task_team->tt.tt_active),
3744 sizeof(task_team->tt.tt_active));
3752 __kmp_task_team_init(task_team, team);
3754 KA_TRACE(20, (
"__kmp_allocate_task_team: T#%d exiting; task_team = %p "
3755 "unfinished_threads init'd to %d\n",
3756 (thread ? __kmp_gtid_from_thread(thread) : -1), task_team,
3757 KMP_ATOMIC_LD_RLX(&task_team->tt.tt_unfinished_threads)));
3764 void __kmp_free_task_team(kmp_info_t *thread, kmp_task_team_t *task_team) {
3765 KA_TRACE(20, (
"__kmp_free_task_team: T#%d task_team = %p\n",
3766 thread ? __kmp_gtid_from_thread(thread) : -1, task_team));
3769 __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock);
3771 KMP_DEBUG_ASSERT(task_team->tt.tt_next == NULL);
3772 task_team->tt.tt_next = __kmp_free_task_teams;
3773 TCW_PTR(__kmp_free_task_teams, task_team);
3775 __kmp_release_bootstrap_lock(&__kmp_task_team_lock);
3783 void __kmp_reap_task_teams(
void) {
3784 kmp_task_team_t *task_team;
3786 if (TCR_PTR(__kmp_free_task_teams) != NULL) {
3788 __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock);
3789 while ((task_team = __kmp_free_task_teams) != NULL) {
3790 __kmp_free_task_teams = task_team->tt.tt_next;
3791 task_team->tt.tt_next = NULL;
3794 if (task_team->tt.tt_threads_data != NULL) {
3795 __kmp_free_task_threads_data(task_team);
3797 if (task_team->tt.tt_task_pri_list != NULL) {
3798 __kmp_free_task_pri_list(task_team);
3800 __kmp_free(task_team);
3802 __kmp_release_bootstrap_lock(&__kmp_task_team_lock);
3810 void __kmp_push_task_team_node(kmp_info_t *thread, kmp_team_t *team) {
3811 KMP_DEBUG_ASSERT(team->t.t_nproc == 1);
3812 kmp_task_team_list_t *current =
3813 (kmp_task_team_list_t *)(&team->t.t_task_team[0]);
3814 kmp_task_team_list_t *node =
3815 (kmp_task_team_list_t *)__kmp_allocate(
sizeof(kmp_task_team_list_t));
3816 node->task_team = current->task_team;
3817 node->next = current->next;
3818 thread->th.th_task_team = current->task_team = NULL;
3819 current->next = node;
3823 void __kmp_pop_task_team_node(kmp_info_t *thread, kmp_team_t *team) {
3824 KMP_DEBUG_ASSERT(team->t.t_nproc == 1);
3825 kmp_task_team_list_t *current =
3826 (kmp_task_team_list_t *)(&team->t.t_task_team[0]);
3827 if (current->task_team) {
3828 __kmp_free_task_team(thread, current->task_team);
3830 kmp_task_team_list_t *next = current->next;
3832 current->task_team = next->task_team;
3833 current->next = next->next;
3834 KMP_DEBUG_ASSERT(next != current);
3836 thread->th.th_task_team = current->task_team;
3843 void __kmp_wait_to_unref_task_teams(
void) {
3849 KMP_INIT_YIELD(spins);
3850 KMP_INIT_BACKOFF(time);
3858 for (thread = CCAST(kmp_info_t *, __kmp_thread_pool); thread != NULL;
3859 thread = thread->th.th_next_pool) {
3863 if (TCR_PTR(thread->th.th_task_team) == NULL) {
3864 KA_TRACE(10, (
"__kmp_wait_to_unref_task_team: T#%d task_team == NULL\n",
3865 __kmp_gtid_from_thread(thread)));
3870 if (!__kmp_is_thread_alive(thread, &exit_val)) {
3871 thread->th.th_task_team = NULL;
3878 KA_TRACE(10, (
"__kmp_wait_to_unref_task_team: Waiting for T#%d to "
3879 "unreference task_team\n",
3880 __kmp_gtid_from_thread(thread)));
3882 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
3885 if ((sleep_loc = TCR_PTR(CCAST(
void *, thread->th.th_sleep_loc))) !=
3889 (
"__kmp_wait_to_unref_task_team: T#%d waking up thread T#%d\n",
3890 __kmp_gtid_from_thread(thread), __kmp_gtid_from_thread(thread)));
3891 __kmp_null_resume_wrapper(thread);
3900 KMP_YIELD_OVERSUB_ELSE_SPIN(spins, time);
3906 void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team) {
3907 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
3912 if (team == this_thr->th.th_serial_team ||
3913 team == this_thr->th.th_root->r.r_root_team) {
3914 KMP_DEBUG_ASSERT(team->t.t_nproc == 1);
3915 if (team->t.t_task_team[0] == NULL) {
3916 team->t.t_task_team[0] = __kmp_allocate_task_team(this_thr, team);
3918 20, (
"__kmp_task_team_setup: Primary T#%d created new task_team %p"
3919 " for serial/root team %p\n",
3920 __kmp_gtid_from_thread(this_thr), team->t.t_task_team[0], team));
3923 __kmp_task_team_init(team->t.t_task_team[0], team);
3931 if (team->t.t_task_team[this_thr->th.th_task_state] == NULL) {
3932 team->t.t_task_team[this_thr->th.th_task_state] =
3933 __kmp_allocate_task_team(this_thr, team);
3934 KA_TRACE(20, (
"__kmp_task_team_setup: Primary T#%d created new task_team %p"
3935 " for team %d at parity=%d\n",
3936 __kmp_gtid_from_thread(this_thr),
3937 team->t.t_task_team[this_thr->th.th_task_state], team->t.t_id,
3938 this_thr->th.th_task_state));
3947 int other_team = 1 - this_thr->th.th_task_state;
3948 KMP_DEBUG_ASSERT(other_team >= 0 && other_team < 2);
3949 if (team->t.t_task_team[other_team] == NULL) {
3950 team->t.t_task_team[other_team] = __kmp_allocate_task_team(this_thr, team);
3951 KA_TRACE(20, (
"__kmp_task_team_setup: Primary T#%d created second new "
3952 "task_team %p for team %d at parity=%d\n",
3953 __kmp_gtid_from_thread(this_thr),
3954 team->t.t_task_team[other_team], team->t.t_id, other_team));
3957 kmp_task_team_t *task_team = team->t.t_task_team[other_team];
3958 __kmp_task_team_init(task_team, team);
3961 KA_TRACE(20, (
"__kmp_task_team_setup: Primary T#%d reset next task_team "
3962 "%p for team %d at parity=%d\n",
3963 __kmp_gtid_from_thread(this_thr),
3964 team->t.t_task_team[other_team], team->t.t_id, other_team));
3971 if (this_thr == __kmp_hidden_helper_main_thread) {
3972 for (
int i = 0; i < 2; ++i) {
3973 kmp_task_team_t *task_team = team->t.t_task_team[i];
3974 if (KMP_TASKING_ENABLED(task_team)) {
3977 __kmp_enable_tasking(task_team, this_thr);
3978 for (
int j = 0; j < task_team->tt.tt_nproc; ++j) {
3979 kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[j];
3980 if (thread_data->td.td_deque == NULL) {
3981 __kmp_alloc_task_deque(__kmp_hidden_helper_threads[j], thread_data);
3991 void __kmp_task_team_sync(kmp_info_t *this_thr, kmp_team_t *team) {
3992 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
3993 KMP_DEBUG_ASSERT(team != this_thr->th.th_serial_team);
3994 KMP_DEBUG_ASSERT(team != this_thr->th.th_root->r.r_root_team);
3998 this_thr->th.th_task_state = (kmp_uint8)(1 - this_thr->th.th_task_state);
4002 TCW_PTR(this_thr->th.th_task_team,
4003 team->t.t_task_team[this_thr->th.th_task_state]);
4005 (
"__kmp_task_team_sync: Thread T#%d task team switched to task_team "
4006 "%p from Team #%d (parity=%d)\n",
4007 __kmp_gtid_from_thread(this_thr), this_thr->th.th_task_team,
4008 team->t.t_id, this_thr->th.th_task_state));
4017 void __kmp_task_team_wait(
4018 kmp_info_t *this_thr,
4019 kmp_team_t *team USE_ITT_BUILD_ARG(
void *itt_sync_obj),
int wait) {
4020 kmp_task_team_t *task_team = team->t.t_task_team[this_thr->th.th_task_state];
4022 KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
4023 KMP_DEBUG_ASSERT(task_team == this_thr->th.th_task_team);
4025 if ((task_team != NULL) && KMP_TASKING_ENABLED(task_team)) {
4027 KA_TRACE(20, (
"__kmp_task_team_wait: Primary T#%d waiting for all tasks "
4028 "(for unfinished_threads to reach 0) on task_team = %p\n",
4029 __kmp_gtid_from_thread(this_thr), task_team));
4033 kmp_flag_32<false, false> flag(
4034 RCAST(std::atomic<kmp_uint32> *,
4035 &task_team->tt.tt_unfinished_threads),
4037 flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
4043 (
"__kmp_task_team_wait: Primary T#%d deactivating task_team %p: "
4044 "setting active to false, setting local and team's pointer to NULL\n",
4045 __kmp_gtid_from_thread(this_thr), task_team));
4046 TCW_SYNC_4(task_team->tt.tt_found_proxy_tasks, FALSE);
4047 TCW_SYNC_4(task_team->tt.tt_hidden_helper_task_encountered, FALSE);
4048 KMP_CHECK_UPDATE(task_team->tt.tt_untied_task_encountered, 0);
4049 TCW_SYNC_4(task_team->tt.tt_active, FALSE);
4052 TCW_PTR(this_thr->th.th_task_team, NULL);
4061 void __kmp_tasking_barrier(kmp_team_t *team, kmp_info_t *thread,
int gtid) {
4062 std::atomic<kmp_uint32> *spin = RCAST(
4063 std::atomic<kmp_uint32> *,
4064 &team->t.t_task_team[thread->th.th_task_state]->tt.tt_unfinished_threads);
4066 KMP_DEBUG_ASSERT(__kmp_tasking_mode == tskm_extra_barrier);
4069 KMP_FSYNC_SPIN_INIT(spin, NULL);
4071 kmp_flag_32<false, false> spin_flag(spin, 0U);
4072 while (!spin_flag.execute_tasks(thread, gtid, TRUE,
4073 &flag USE_ITT_BUILD_ARG(NULL), 0)) {
4076 KMP_FSYNC_SPIN_PREPARE(RCAST(
void *, spin));
4079 if (TCR_4(__kmp_global.g.g_done)) {
4080 if (__kmp_global.g.g_abort)
4081 __kmp_abort_thread();
4087 KMP_FSYNC_SPIN_ACQUIRED(RCAST(
void *, spin));
4096 static bool __kmp_give_task(kmp_info_t *thread, kmp_int32 tid, kmp_task_t *task,
4098 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
4099 kmp_task_team_t *task_team = taskdata->td_task_team;
4101 KA_TRACE(20, (
"__kmp_give_task: trying to give task %p to thread %d.\n",
4105 KMP_DEBUG_ASSERT(task_team != NULL);
4107 bool result =
false;
4108 kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[tid];
4110 if (thread_data->td.td_deque == NULL) {
4114 (
"__kmp_give_task: thread %d has no queue while giving task %p.\n",
4119 if (TCR_4(thread_data->td.td_deque_ntasks) >=
4120 TASK_DEQUE_SIZE(thread_data->td)) {
4123 (
"__kmp_give_task: queue is full while giving task %p to thread %d.\n",
4128 if (TASK_DEQUE_SIZE(thread_data->td) / INITIAL_TASK_DEQUE_SIZE >= pass)
4131 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
4132 if (TCR_4(thread_data->td.td_deque_ntasks) >=
4133 TASK_DEQUE_SIZE(thread_data->td)) {
4135 __kmp_realloc_task_deque(thread, thread_data);
4140 __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
4142 if (TCR_4(thread_data->td.td_deque_ntasks) >=
4143 TASK_DEQUE_SIZE(thread_data->td)) {
4144 KA_TRACE(30, (
"__kmp_give_task: queue is full while giving task %p to "
4150 if (TASK_DEQUE_SIZE(thread_data->td) / INITIAL_TASK_DEQUE_SIZE >= pass)
4151 goto release_and_exit;
4153 __kmp_realloc_task_deque(thread, thread_data);
4159 thread_data->td.td_deque[thread_data->td.td_deque_tail] = taskdata;
4161 thread_data->td.td_deque_tail =
4162 (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td);
4163 TCW_4(thread_data->td.td_deque_ntasks,
4164 TCR_4(thread_data->td.td_deque_ntasks) + 1);
4167 KA_TRACE(30, (
"__kmp_give_task: successfully gave task %p to thread %d.\n",
4171 __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
4176 #define PROXY_TASK_FLAG 0x40000000
4193 static void __kmp_first_top_half_finish_proxy(kmp_taskdata_t *taskdata) {
4194 KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
4195 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY);
4196 KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0);
4197 KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0);
4199 taskdata->td_flags.complete = 1;
4201 taskdata->td_flags.onced = 1;
4204 if (taskdata->td_taskgroup)
4205 KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count);
4209 KMP_ATOMIC_OR(&taskdata->td_incomplete_child_tasks, PROXY_TASK_FLAG);
4212 static void __kmp_second_top_half_finish_proxy(kmp_taskdata_t *taskdata) {
4214 kmp_int32 children = 0;
4218 KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks);
4219 KMP_DEBUG_ASSERT(children >= 0);
4222 KMP_ATOMIC_AND(&taskdata->td_incomplete_child_tasks, ~PROXY_TASK_FLAG);
4225 static void __kmp_bottom_half_finish_proxy(kmp_int32 gtid, kmp_task_t *ptask) {
4226 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask);
4227 kmp_info_t *thread = __kmp_threads[gtid];
4229 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY);
4230 KMP_DEBUG_ASSERT(taskdata->td_flags.complete ==
4235 while ((KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks) &
4236 PROXY_TASK_FLAG) > 0)
4239 __kmp_release_deps(gtid, taskdata);
4240 __kmp_free_task_and_ancestors(gtid, taskdata, thread);
4252 KMP_DEBUG_ASSERT(ptask != NULL);
4253 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask);
4255 10, (
"__kmp_proxy_task_completed(enter): T#%d proxy task %p completing\n",
4257 __kmp_assert_valid_gtid(gtid);
4258 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY);
4260 __kmp_first_top_half_finish_proxy(taskdata);
4261 __kmp_second_top_half_finish_proxy(taskdata);
4262 __kmp_bottom_half_finish_proxy(gtid, ptask);
4265 (
"__kmp_proxy_task_completed(exit): T#%d proxy task %p completing\n",
4269 void __kmpc_give_task(kmp_task_t *ptask, kmp_int32 start = 0) {
4270 KMP_DEBUG_ASSERT(ptask != NULL);
4271 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask);
4275 kmp_team_t *team = taskdata->td_team;
4276 kmp_int32 nthreads = team->t.t_nproc;
4281 kmp_int32 start_k = start % nthreads;
4283 kmp_int32 k = start_k;
4287 thread = team->t.t_threads[k];
4288 k = (k + 1) % nthreads;
4294 }
while (!__kmp_give_task(thread, k, ptask, pass));
4296 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME && __kmp_wpolicy_passive) {
4298 for (
int i = 0; i < nthreads; ++i) {
4299 thread = team->t.t_threads[i];
4300 if (thread->th.th_sleep_loc != NULL) {
4301 __kmp_null_resume_wrapper(thread);
4316 KMP_DEBUG_ASSERT(ptask != NULL);
4317 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask);
4321 (
"__kmp_proxy_task_completed_ooo(enter): proxy task completing ooo %p\n",
4324 KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY);
4326 __kmp_first_top_half_finish_proxy(taskdata);
4328 __kmpc_give_task(ptask);
4330 __kmp_second_top_half_finish_proxy(taskdata);
4334 (
"__kmp_proxy_task_completed_ooo(exit): proxy task completing ooo %p\n",
4338 kmp_event_t *__kmpc_task_allow_completion_event(
ident_t *loc_ref,
int gtid,
4340 kmp_taskdata_t *td = KMP_TASK_TO_TASKDATA(task);
4341 if (td->td_allow_completion_event.type == KMP_EVENT_UNINITIALIZED) {
4342 td->td_allow_completion_event.type = KMP_EVENT_ALLOW_COMPLETION;
4343 td->td_allow_completion_event.ed.task = task;
4344 __kmp_init_tas_lock(&td->td_allow_completion_event.lock);
4346 return &td->td_allow_completion_event;
4349 void __kmp_fulfill_event(kmp_event_t *event) {
4350 if (event->type == KMP_EVENT_ALLOW_COMPLETION) {
4351 kmp_task_t *ptask = event->ed.task;
4352 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask);
4353 bool detached =
false;
4354 int gtid = __kmp_get_gtid();
4359 __kmp_acquire_tas_lock(&event->lock, gtid);
4360 if (taskdata->td_flags.proxy == TASK_PROXY) {
4366 if (UNLIKELY(ompt_enabled.enabled))
4367 __ompt_task_finish(ptask, NULL, ompt_task_early_fulfill);
4370 event->type = KMP_EVENT_UNINITIALIZED;
4371 __kmp_release_tas_lock(&event->lock, gtid);
4377 if (UNLIKELY(ompt_enabled.enabled))
4378 __ompt_task_finish(ptask, NULL, ompt_task_late_fulfill);
4382 kmp_team_t *team = taskdata->td_team;
4383 kmp_info_t *thread = __kmp_get_thread();
4384 if (thread->th.th_team == team) {
4404 kmp_task_t *__kmp_task_dup_alloc(kmp_info_t *thread, kmp_task_t *task_src
4406 ,
int taskloop_recur
4410 kmp_taskdata_t *taskdata;
4411 kmp_taskdata_t *taskdata_src = KMP_TASK_TO_TASKDATA(task_src);
4412 kmp_taskdata_t *parent_task = taskdata_src->td_parent;
4413 size_t shareds_offset;
4416 KA_TRACE(10, (
"__kmp_task_dup_alloc(enter): Th %p, source task %p\n", thread,
4418 KMP_DEBUG_ASSERT(taskdata_src->td_flags.proxy ==
4420 KMP_DEBUG_ASSERT(taskdata_src->td_flags.tasktype == TASK_EXPLICIT);
4421 task_size = taskdata_src->td_size_alloc;
4424 KA_TRACE(30, (
"__kmp_task_dup_alloc: Th %p, malloc size %ld\n", thread,
4427 taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(thread, task_size);
4429 taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(thread, task_size);
4431 KMP_MEMCPY(taskdata, taskdata_src, task_size);
4433 task = KMP_TASKDATA_TO_TASK(taskdata);
4437 if (taskdata->is_taskgraph && !taskloop_recur &&
4438 __kmp_tdg_is_recording(taskdata_src->tdg->tdg_status))
4439 taskdata->td_tdg_task_id = KMP_ATOMIC_INC(&__kmp_tdg_task_id);
4441 taskdata->td_task_id = KMP_GEN_TASK_ID();
4442 if (task->shareds != NULL) {
4443 shareds_offset = (
char *)task_src->shareds - (
char *)taskdata_src;
4444 task->shareds = &((
char *)taskdata)[shareds_offset];
4445 KMP_DEBUG_ASSERT((((kmp_uintptr_t)task->shareds) & (
sizeof(
void *) - 1)) ==
4448 taskdata->td_alloc_thread = thread;
4449 taskdata->td_parent = parent_task;
4451 taskdata->td_taskgroup = parent_task->td_taskgroup;
4454 if (taskdata->td_flags.tiedness == TASK_TIED)
4455 taskdata->td_last_tied = taskdata;
4459 if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) {
4460 KMP_ATOMIC_INC(&parent_task->td_incomplete_child_tasks);
4461 if (parent_task->td_taskgroup)
4462 KMP_ATOMIC_INC(&parent_task->td_taskgroup->count);
4465 if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT)
4466 KMP_ATOMIC_INC(&taskdata->td_parent->td_allocated_child_tasks);
4470 (
"__kmp_task_dup_alloc(exit): Th %p, created task %p, parent=%p\n",
4471 thread, taskdata, taskdata->td_parent));
4473 if (UNLIKELY(ompt_enabled.enabled))
4474 __ompt_task_init(taskdata, thread->th.th_info.ds.ds_gtid);
4483 typedef void (*p_task_dup_t)(kmp_task_t *, kmp_task_t *, kmp_int32);
4485 KMP_BUILD_ASSERT(
sizeof(
long) == 4 ||
sizeof(
long) == 8);
4490 class kmp_taskloop_bounds_t {
4492 const kmp_taskdata_t *taskdata;
4493 size_t lower_offset;
4494 size_t upper_offset;
4497 kmp_taskloop_bounds_t(kmp_task_t *_task, kmp_uint64 *lb, kmp_uint64 *ub)
4498 : task(_task), taskdata(KMP_TASK_TO_TASKDATA(task)),
4499 lower_offset((char *)lb - (char *)task),
4500 upper_offset((char *)ub - (char *)task) {
4501 KMP_DEBUG_ASSERT((
char *)lb > (
char *)_task);
4502 KMP_DEBUG_ASSERT((
char *)ub > (
char *)_task);
4504 kmp_taskloop_bounds_t(kmp_task_t *_task,
const kmp_taskloop_bounds_t &bounds)
4505 : task(_task), taskdata(KMP_TASK_TO_TASKDATA(_task)),
4506 lower_offset(bounds.lower_offset), upper_offset(bounds.upper_offset) {}
4507 size_t get_lower_offset()
const {
return lower_offset; }
4508 size_t get_upper_offset()
const {
return upper_offset; }
4509 kmp_uint64 get_lb()
const {
4511 #if defined(KMP_GOMP_COMPAT)
4513 if (!taskdata->td_flags.native) {
4514 retval = *(kmp_int64 *)((
char *)task + lower_offset);
4517 if (taskdata->td_size_loop_bounds == 4) {
4518 kmp_int32 *lb = RCAST(kmp_int32 *, task->shareds);
4519 retval = (kmp_int64)*lb;
4521 kmp_int64 *lb = RCAST(kmp_int64 *, task->shareds);
4522 retval = (kmp_int64)*lb;
4527 retval = *(kmp_int64 *)((
char *)task + lower_offset);
4531 kmp_uint64 get_ub()
const {
4533 #if defined(KMP_GOMP_COMPAT)
4535 if (!taskdata->td_flags.native) {
4536 retval = *(kmp_int64 *)((
char *)task + upper_offset);
4539 if (taskdata->td_size_loop_bounds == 4) {
4540 kmp_int32 *ub = RCAST(kmp_int32 *, task->shareds) + 1;
4541 retval = (kmp_int64)*ub;
4543 kmp_int64 *ub = RCAST(kmp_int64 *, task->shareds) + 1;
4544 retval = (kmp_int64)*ub;
4548 retval = *(kmp_int64 *)((
char *)task + upper_offset);
4552 void set_lb(kmp_uint64 lb) {
4553 #if defined(KMP_GOMP_COMPAT)
4555 if (!taskdata->td_flags.native) {
4556 *(kmp_uint64 *)((
char *)task + lower_offset) = lb;
4559 if (taskdata->td_size_loop_bounds == 4) {
4560 kmp_uint32 *lower = RCAST(kmp_uint32 *, task->shareds);
4561 *lower = (kmp_uint32)lb;
4563 kmp_uint64 *lower = RCAST(kmp_uint64 *, task->shareds);
4564 *lower = (kmp_uint64)lb;
4568 *(kmp_uint64 *)((
char *)task + lower_offset) = lb;
4571 void set_ub(kmp_uint64 ub) {
4572 #if defined(KMP_GOMP_COMPAT)
4574 if (!taskdata->td_flags.native) {
4575 *(kmp_uint64 *)((
char *)task + upper_offset) = ub;
4578 if (taskdata->td_size_loop_bounds == 4) {
4579 kmp_uint32 *upper = RCAST(kmp_uint32 *, task->shareds) + 1;
4580 *upper = (kmp_uint32)ub;
4582 kmp_uint64 *upper = RCAST(kmp_uint64 *, task->shareds) + 1;
4583 *upper = (kmp_uint64)ub;
4587 *(kmp_uint64 *)((
char *)task + upper_offset) = ub;
4608 void __kmp_taskloop_linear(
ident_t *loc,
int gtid, kmp_task_t *task,
4609 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
4610 kmp_uint64 ub_glob, kmp_uint64 num_tasks,
4611 kmp_uint64 grainsize, kmp_uint64 extras,
4612 kmp_int64 last_chunk, kmp_uint64 tc,
4618 KMP_TIME_PARTITIONED_BLOCK(OMP_taskloop_scheduling);
4619 p_task_dup_t ptask_dup = (p_task_dup_t)task_dup;
4621 kmp_taskloop_bounds_t task_bounds(task, lb, ub);
4622 kmp_uint64 lower = task_bounds.get_lb();
4623 kmp_uint64 upper = task_bounds.get_ub();
4625 kmp_info_t *thread = __kmp_threads[gtid];
4626 kmp_taskdata_t *current_task = thread->th.th_current_task;
4627 kmp_task_t *next_task;
4628 kmp_int32 lastpriv = 0;
4630 KMP_DEBUG_ASSERT(tc == num_tasks * grainsize +
4631 (last_chunk < 0 ? last_chunk : extras));
4632 KMP_DEBUG_ASSERT(num_tasks > extras);
4633 KMP_DEBUG_ASSERT(num_tasks > 0);
4634 KA_TRACE(20, (
"__kmp_taskloop_linear: T#%d: %lld tasks, grainsize %lld, "
4635 "extras %lld, last_chunk %lld, i=%lld,%lld(%d)%lld, dup %p\n",
4636 gtid, num_tasks, grainsize, extras, last_chunk, lower, upper,
4637 ub_glob, st, task_dup));
4640 for (i = 0; i < num_tasks; ++i) {
4641 kmp_uint64 chunk_minus_1;
4643 chunk_minus_1 = grainsize - 1;
4645 chunk_minus_1 = grainsize;
4648 upper = lower + st * chunk_minus_1;
4652 if (i == num_tasks - 1) {
4655 KMP_DEBUG_ASSERT(upper == *ub);
4656 if (upper == ub_glob)
4658 }
else if (st > 0) {
4659 KMP_DEBUG_ASSERT((kmp_uint64)st > *ub - upper);
4660 if ((kmp_uint64)st > ub_glob - upper)
4663 KMP_DEBUG_ASSERT(upper + st < *ub);
4664 if (upper - ub_glob < (kmp_uint64)(-st))
4670 next_task = __kmp_task_dup_alloc(thread, task, 0);
4672 next_task = __kmp_task_dup_alloc(thread, task);
4675 kmp_taskdata_t *next_taskdata = KMP_TASK_TO_TASKDATA(next_task);
4676 kmp_taskloop_bounds_t next_task_bounds =
4677 kmp_taskloop_bounds_t(next_task, task_bounds);
4680 next_task_bounds.set_lb(lower);
4681 if (next_taskdata->td_flags.native) {
4682 next_task_bounds.set_ub(upper + (st > 0 ? 1 : -1));
4684 next_task_bounds.set_ub(upper);
4686 if (ptask_dup != NULL)
4688 ptask_dup(next_task, task, lastpriv);
4690 (
"__kmp_taskloop_linear: T#%d; task #%llu: task %p: lower %lld, "
4691 "upper %lld stride %lld, (offsets %p %p)\n",
4692 gtid, i, next_task, lower, upper, st,
4693 next_task_bounds.get_lower_offset(),
4694 next_task_bounds.get_upper_offset()));
4696 __kmp_omp_taskloop_task(NULL, gtid, next_task,
4699 if (ompt_enabled.ompt_callback_dispatch) {
4700 OMPT_GET_DISPATCH_CHUNK(next_taskdata->ompt_task_info.dispatch_chunk,
4705 __kmp_omp_task(gtid, next_task,
true);
4710 __kmp_task_start(gtid, task, current_task);
4712 __kmp_task_finish<false>(gtid, task, current_task);
4717 typedef struct __taskloop_params {
4724 kmp_uint64 num_tasks;
4725 kmp_uint64 grainsize;
4727 kmp_int64 last_chunk;
4729 kmp_uint64 num_t_min;
4733 } __taskloop_params_t;
4735 void __kmp_taskloop_recur(
ident_t *,
int, kmp_task_t *, kmp_uint64 *,
4736 kmp_uint64 *, kmp_int64, kmp_uint64, kmp_uint64,
4737 kmp_uint64, kmp_uint64, kmp_int64, kmp_uint64,
4745 int __kmp_taskloop_task(
int gtid,
void *ptask) {
4746 __taskloop_params_t *p =
4747 (__taskloop_params_t *)((kmp_task_t *)ptask)->shareds;
4748 kmp_task_t *task = p->task;
4749 kmp_uint64 *lb = p->lb;
4750 kmp_uint64 *ub = p->ub;
4751 void *task_dup = p->task_dup;
4753 kmp_int64 st = p->st;
4754 kmp_uint64 ub_glob = p->ub_glob;
4755 kmp_uint64 num_tasks = p->num_tasks;
4756 kmp_uint64 grainsize = p->grainsize;
4757 kmp_uint64 extras = p->extras;
4758 kmp_int64 last_chunk = p->last_chunk;
4759 kmp_uint64 tc = p->tc;
4760 kmp_uint64 num_t_min = p->num_t_min;
4762 void *codeptr_ra = p->codeptr_ra;
4765 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
4766 KMP_DEBUG_ASSERT(task != NULL);
4768 (
"__kmp_taskloop_task: T#%d, task %p: %lld tasks, grainsize"
4769 " %lld, extras %lld, last_chunk %lld, i=%lld,%lld(%d), dup %p\n",
4770 gtid, taskdata, num_tasks, grainsize, extras, last_chunk, *lb, *ub,
4773 KMP_DEBUG_ASSERT(num_tasks * 2 + 1 > num_t_min);
4774 if (num_tasks > num_t_min)
4775 __kmp_taskloop_recur(NULL, gtid, task, lb, ub, st, ub_glob, num_tasks,
4776 grainsize, extras, last_chunk, tc, num_t_min,
4782 __kmp_taskloop_linear(NULL, gtid, task, lb, ub, st, ub_glob, num_tasks,
4783 grainsize, extras, last_chunk, tc,
4789 KA_TRACE(40, (
"__kmp_taskloop_task(exit): T#%d\n", gtid));
4811 void __kmp_taskloop_recur(
ident_t *loc,
int gtid, kmp_task_t *task,
4812 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
4813 kmp_uint64 ub_glob, kmp_uint64 num_tasks,
4814 kmp_uint64 grainsize, kmp_uint64 extras,
4815 kmp_int64 last_chunk, kmp_uint64 tc,
4816 kmp_uint64 num_t_min,
4821 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
4822 KMP_DEBUG_ASSERT(task != NULL);
4823 KMP_DEBUG_ASSERT(num_tasks > num_t_min);
4825 (
"__kmp_taskloop_recur: T#%d, task %p: %lld tasks, grainsize"
4826 " %lld, extras %lld, last_chunk %lld, i=%lld,%lld(%d), dup %p\n",
4827 gtid, taskdata, num_tasks, grainsize, extras, last_chunk, *lb, *ub,
4829 p_task_dup_t ptask_dup = (p_task_dup_t)task_dup;
4830 kmp_uint64 lower = *lb;
4831 kmp_info_t *thread = __kmp_threads[gtid];
4833 kmp_task_t *next_task;
4834 size_t lower_offset =
4835 (
char *)lb - (
char *)task;
4836 size_t upper_offset =
4837 (
char *)ub - (
char *)task;
4839 KMP_DEBUG_ASSERT(tc == num_tasks * grainsize +
4840 (last_chunk < 0 ? last_chunk : extras));
4841 KMP_DEBUG_ASSERT(num_tasks > extras);
4842 KMP_DEBUG_ASSERT(num_tasks > 0);
4845 kmp_uint64 lb1, ub0, tc0, tc1, ext0, ext1;
4846 kmp_int64 last_chunk0 = 0, last_chunk1 = 0;
4847 kmp_uint64 gr_size0 = grainsize;
4848 kmp_uint64 n_tsk0 = num_tasks >> 1;
4849 kmp_uint64 n_tsk1 = num_tasks - n_tsk0;
4850 if (last_chunk < 0) {
4852 last_chunk1 = last_chunk;
4853 tc0 = grainsize * n_tsk0;
4855 }
else if (n_tsk0 <= extras) {
4858 ext1 = extras - n_tsk0;
4859 tc0 = gr_size0 * n_tsk0;
4864 tc1 = grainsize * n_tsk1;
4867 ub0 = lower + st * (tc0 - 1);
4872 next_task = __kmp_task_dup_alloc(thread, task,
4875 next_task = __kmp_task_dup_alloc(thread, task);
4878 *(kmp_uint64 *)((
char *)next_task + lower_offset) = lb1;
4879 if (ptask_dup != NULL)
4880 ptask_dup(next_task, task, 0);
4885 kmp_taskdata_t *current_task = thread->th.th_current_task;
4886 thread->th.th_current_task = taskdata->td_parent;
4887 kmp_task_t *new_task =
4888 __kmpc_omp_task_alloc(loc, gtid, 1, 3 *
sizeof(
void *),
4889 sizeof(__taskloop_params_t), &__kmp_taskloop_task);
4891 thread->th.th_current_task = current_task;
4892 __taskloop_params_t *p = (__taskloop_params_t *)new_task->shareds;
4893 p->task = next_task;
4894 p->lb = (kmp_uint64 *)((
char *)next_task + lower_offset);
4895 p->ub = (kmp_uint64 *)((
char *)next_task + upper_offset);
4896 p->task_dup = task_dup;
4898 p->ub_glob = ub_glob;
4899 p->num_tasks = n_tsk1;
4900 p->grainsize = grainsize;
4902 p->last_chunk = last_chunk1;
4904 p->num_t_min = num_t_min;
4906 p->codeptr_ra = codeptr_ra;
4910 kmp_taskdata_t *new_task_data = KMP_TASK_TO_TASKDATA(new_task);
4911 new_task_data->tdg = taskdata->tdg;
4912 new_task_data->is_taskgraph = 0;
4917 __kmp_omp_taskloop_task(NULL, gtid, new_task, codeptr_ra);
4919 __kmp_omp_task(gtid, new_task,
true);
4923 if (n_tsk0 > num_t_min)
4924 __kmp_taskloop_recur(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0, gr_size0,
4925 ext0, last_chunk0, tc0, num_t_min,
4931 __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0,
4932 gr_size0, ext0, last_chunk0, tc0,
4938 KA_TRACE(40, (
"__kmp_taskloop_recur(exit): T#%d\n", gtid));
4941 static void __kmp_taskloop(
ident_t *loc,
int gtid, kmp_task_t *task,
int if_val,
4942 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
4943 int nogroup,
int sched, kmp_uint64 grainsize,
4944 int modifier,
void *task_dup) {
4945 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
4946 KMP_DEBUG_ASSERT(task != NULL);
4948 #if OMPT_SUPPORT && OMPT_OPTIONAL
4949 OMPT_STORE_RETURN_ADDRESS(gtid);
4951 __kmpc_taskgroup(loc, gtid);
4955 KMP_ATOMIC_DEC(&__kmp_tdg_task_id);
4959 kmp_taskloop_bounds_t task_bounds(task, lb, ub);
4962 kmp_uint64 lower = task_bounds.get_lb();
4963 kmp_uint64 upper = task_bounds.get_ub();
4964 kmp_uint64 ub_glob = upper;
4965 kmp_uint64 num_tasks = 0, extras = 0;
4966 kmp_int64 last_chunk =
4968 kmp_uint64 num_tasks_min = __kmp_taskloop_min_tasks;
4969 kmp_info_t *thread = __kmp_threads[gtid];
4970 kmp_taskdata_t *current_task = thread->th.th_current_task;
4972 KA_TRACE(20, (
"__kmp_taskloop: T#%d, task %p, lb %lld, ub %lld, st %lld, "
4973 "grain %llu(%d, %d), dup %p\n",
4974 gtid, taskdata, lower, upper, st, grainsize, sched, modifier,
4979 tc = upper - lower + 1;
4980 }
else if (st < 0) {
4981 tc = (lower - upper) / (-st) + 1;
4983 tc = (upper - lower) / st + 1;
4986 KA_TRACE(20, (
"__kmp_taskloop(exit): T#%d zero-trip loop\n", gtid));
4988 __kmp_task_start(gtid, task, current_task);
4990 __kmp_task_finish<false>(gtid, task, current_task);
4994 #if OMPT_SUPPORT && OMPT_OPTIONAL
4995 ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
4996 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
4997 if (ompt_enabled.ompt_callback_work) {
4998 ompt_callbacks.ompt_callback(ompt_callback_work)(
4999 ompt_work_taskloop, ompt_scope_begin, &(team_info->parallel_data),
5000 &(task_info->task_data), tc, OMPT_GET_RETURN_ADDRESS(0));
5004 if (num_tasks_min == 0)
5007 KMP_MIN(thread->th.th_team_nproc * 10, INITIAL_TASK_DEQUE_SIZE);
5013 grainsize = thread->th.th_team_nproc *
static_cast<kmp_uint64
>(10);
5016 if (grainsize > tc) {
5021 num_tasks = grainsize;
5022 grainsize = tc / num_tasks;
5023 extras = tc % num_tasks;
5027 if (grainsize > tc) {
5033 num_tasks = (tc + grainsize - 1) / grainsize;
5034 last_chunk = tc - (num_tasks * grainsize);
5037 num_tasks = tc / grainsize;
5039 grainsize = tc / num_tasks;
5040 extras = tc % num_tasks;
5045 KMP_ASSERT2(0,
"unknown scheduling of taskloop");
5048 KMP_DEBUG_ASSERT(tc == num_tasks * grainsize +
5049 (last_chunk < 0 ? last_chunk : extras));
5050 KMP_DEBUG_ASSERT(num_tasks > extras);
5051 KMP_DEBUG_ASSERT(num_tasks > 0);
5057 taskdata->td_flags.task_serial = 1;
5058 taskdata->td_flags.tiedness = TASK_TIED;
5060 __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
5061 grainsize, extras, last_chunk, tc,
5063 OMPT_GET_RETURN_ADDRESS(0),
5068 }
else if (num_tasks > num_tasks_min && !taskdata->td_flags.native) {
5069 KA_TRACE(20, (
"__kmp_taskloop: T#%d, go recursive: tc %llu, #tasks %llu"
5070 "(%lld), grain %llu, extras %llu, last_chunk %lld\n",
5071 gtid, tc, num_tasks, num_tasks_min, grainsize, extras,
5073 __kmp_taskloop_recur(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
5074 grainsize, extras, last_chunk, tc, num_tasks_min,
5076 OMPT_GET_RETURN_ADDRESS(0),
5080 KA_TRACE(20, (
"__kmp_taskloop: T#%d, go linear: tc %llu, #tasks %llu"
5081 "(%lld), grain %llu, extras %llu, last_chunk %lld\n",
5082 gtid, tc, num_tasks, num_tasks_min, grainsize, extras,
5084 __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
5085 grainsize, extras, last_chunk, tc,
5087 OMPT_GET_RETURN_ADDRESS(0),
5092 #if OMPT_SUPPORT && OMPT_OPTIONAL
5093 if (ompt_enabled.ompt_callback_work) {
5094 ompt_callbacks.ompt_callback(ompt_callback_work)(
5095 ompt_work_taskloop, ompt_scope_end, &(team_info->parallel_data),
5096 &(task_info->task_data), tc, OMPT_GET_RETURN_ADDRESS(0));
5101 #if OMPT_SUPPORT && OMPT_OPTIONAL
5102 OMPT_STORE_RETURN_ADDRESS(gtid);
5104 __kmpc_end_taskgroup(loc, gtid);
5106 KA_TRACE(20, (
"__kmp_taskloop(exit): T#%d\n", gtid));
5126 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
int nogroup,
5127 int sched, kmp_uint64 grainsize,
void *task_dup) {
5128 __kmp_assert_valid_gtid(gtid);
5129 KA_TRACE(20, (
"__kmpc_taskloop(enter): T#%d\n", gtid));
5130 __kmp_taskloop(loc, gtid, task, if_val, lb, ub, st, nogroup, sched, grainsize,
5132 KA_TRACE(20, (
"__kmpc_taskloop(exit): T#%d\n", gtid));
5153 kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
5154 int nogroup,
int sched, kmp_uint64 grainsize,
5155 int modifier,
void *task_dup) {
5156 __kmp_assert_valid_gtid(gtid);
5157 KA_TRACE(20, (
"__kmpc_taskloop_5(enter): T#%d\n", gtid));
5158 __kmp_taskloop(loc, gtid, task, if_val, lb, ub, st, nogroup, sched, grainsize,
5159 modifier, task_dup);
5160 KA_TRACE(20, (
"__kmpc_taskloop_5(exit): T#%d\n", gtid));
5172 if (gtid == KMP_GTID_DNE)
5175 kmp_info_t *thread = __kmp_thread_from_gtid(gtid);
5176 kmp_taskdata_t *taskdata = thread->th.th_current_task;
5181 return &taskdata->td_target_data.async_handle;
5193 if (gtid == KMP_GTID_DNE)
5196 kmp_info_t *thread = __kmp_thread_from_gtid(gtid);
5197 kmp_taskdata_t *taskdata = thread->th.th_current_task;
5202 return taskdata->td_task_team != NULL;
5210 static kmp_tdg_info_t *__kmp_find_tdg(kmp_int32 tdg_id) {
5211 kmp_tdg_info_t *res =
nullptr;
5212 if (__kmp_max_tdgs == 0)
5215 if (__kmp_global_tdgs == NULL)
5216 __kmp_global_tdgs = (kmp_tdg_info_t **)__kmp_allocate(
5217 sizeof(kmp_tdg_info_t *) * __kmp_max_tdgs);
5219 if ((__kmp_global_tdgs[tdg_id]) &&
5220 (__kmp_global_tdgs[tdg_id]->tdg_status != KMP_TDG_NONE))
5221 res = __kmp_global_tdgs[tdg_id];
5228 void __kmp_print_tdg_dot(kmp_tdg_info_t *tdg, kmp_int32 gtid) {
5229 kmp_int32 tdg_id = tdg->tdg_id;
5230 KA_TRACE(10, (
"__kmp_print_tdg_dot(enter): T#%d tdg_id=%d \n", gtid, tdg_id));
5233 sprintf(file_name,
"tdg_%d.dot", tdg_id);
5236 kmp_int32 num_tasks = KMP_ATOMIC_LD_RLX(&tdg->num_tasks);
5240 " subgraph cluster {\n"
5243 for (kmp_int32 i = 0; i < num_tasks; i++) {
5244 fprintf(tdg_file,
" %d[style=bold]\n", i);
5246 fprintf(tdg_file,
" }\n");
5247 for (kmp_int32 i = 0; i < num_tasks; i++) {
5248 kmp_int32 nsuccessors = tdg->record_map[i].nsuccessors;
5249 kmp_int32 *successors = tdg->record_map[i].successors;
5250 if (nsuccessors > 0) {
5251 for (kmp_int32 j = 0; j < nsuccessors; j++)
5252 fprintf(tdg_file,
" %d -> %d \n", i, successors[j]);
5255 fprintf(tdg_file,
"}");
5256 KA_TRACE(10, (
"__kmp_print_tdg_dot(exit): T#%d tdg_id=%d \n", gtid, tdg_id));
5263 void __kmp_exec_tdg(kmp_int32 gtid, kmp_tdg_info_t *tdg) {
5264 KMP_DEBUG_ASSERT(tdg->tdg_status == KMP_TDG_READY);
5265 KA_TRACE(10, (
"__kmp_exec_tdg(enter): T#%d tdg_id=%d num_roots=%d\n", gtid,
5266 tdg->tdg_id, tdg->num_roots));
5267 kmp_node_info_t *this_record_map = tdg->record_map;
5268 kmp_int32 *this_root_tasks = tdg->root_tasks;
5269 kmp_int32 this_num_roots = tdg->num_roots;
5270 kmp_int32 this_num_tasks = KMP_ATOMIC_LD_RLX(&tdg->num_tasks);
5272 kmp_info_t *thread = __kmp_threads[gtid];
5273 kmp_taskdata_t *parent_task = thread->th.th_current_task;
5275 if (tdg->rec_taskred_data) {
5279 for (kmp_int32 j = 0; j < this_num_tasks; j++) {
5280 kmp_taskdata_t *td = KMP_TASK_TO_TASKDATA(this_record_map[j].task);
5282 td->td_parent = parent_task;
5283 this_record_map[j].parent_task = parent_task;
5285 kmp_taskgroup_t *parent_taskgroup =
5286 this_record_map[j].parent_task->td_taskgroup;
5288 KMP_ATOMIC_ST_RLX(&this_record_map[j].npredecessors_counter,
5289 this_record_map[j].npredecessors);
5290 KMP_ATOMIC_INC(&this_record_map[j].parent_task->td_incomplete_child_tasks);
5292 if (parent_taskgroup) {
5293 KMP_ATOMIC_INC(&parent_taskgroup->count);
5295 td->td_taskgroup = parent_taskgroup;
5296 }
else if (td->td_taskgroup !=
nullptr) {
5298 td->td_taskgroup =
nullptr;
5300 if (this_record_map[j].parent_task->td_flags.tasktype == TASK_EXPLICIT)
5301 KMP_ATOMIC_INC(&this_record_map[j].parent_task->td_allocated_child_tasks);
5304 for (kmp_int32 j = 0; j < this_num_roots; ++j) {
5305 __kmp_omp_task(gtid, this_record_map[this_root_tasks[j]].task,
true);
5307 KA_TRACE(10, (
"__kmp_exec_tdg(exit): T#%d tdg_id=%d num_roots=%d\n", gtid,
5308 tdg->tdg_id, tdg->num_roots));
5316 static inline void __kmp_start_record(kmp_int32 gtid,
5317 kmp_taskgraph_flags_t *flags,
5319 kmp_tdg_info_t *tdg =
5320 (kmp_tdg_info_t *)__kmp_allocate(
sizeof(kmp_tdg_info_t));
5321 __kmp_global_tdgs[__kmp_curr_tdg_idx] = tdg;
5323 tdg->tdg_id = tdg_id;
5324 tdg->map_size = INIT_MAPSIZE;
5325 tdg->num_roots = -1;
5326 tdg->root_tasks =
nullptr;
5327 tdg->tdg_status = KMP_TDG_RECORDING;
5328 tdg->rec_num_taskred = 0;
5329 tdg->rec_taskred_data =
nullptr;
5330 KMP_ATOMIC_ST_RLX(&tdg->num_tasks, 0);
5333 kmp_node_info_t *this_record_map =
5334 (kmp_node_info_t *)__kmp_allocate(INIT_MAPSIZE *
sizeof(kmp_node_info_t));
5335 for (kmp_int32 i = 0; i < INIT_MAPSIZE; i++) {
5336 kmp_int32 *successorsList =
5337 (kmp_int32 *)__kmp_allocate(__kmp_successors_size *
sizeof(kmp_int32));
5338 this_record_map[i].task =
nullptr;
5339 this_record_map[i].successors = successorsList;
5340 this_record_map[i].nsuccessors = 0;
5341 this_record_map[i].npredecessors = 0;
5342 this_record_map[i].successors_size = __kmp_successors_size;
5343 KMP_ATOMIC_ST_RLX(&this_record_map[i].npredecessors_counter, 0);
5346 __kmp_global_tdgs[__kmp_curr_tdg_idx]->record_map = this_record_map;
5356 kmp_int32 __kmpc_start_record_task(
ident_t *loc_ref, kmp_int32 gtid,
5357 kmp_int32 input_flags, kmp_int32 tdg_id) {
5360 kmp_taskgraph_flags_t *flags = (kmp_taskgraph_flags_t *)&input_flags;
5362 (
"__kmpc_start_record_task(enter): T#%d loc=%p flags=%d tdg_id=%d\n",
5363 gtid, loc_ref, input_flags, tdg_id));
5365 if (__kmp_max_tdgs == 0) {
5368 (
"__kmpc_start_record_task(abandon): T#%d loc=%p flags=%d tdg_id = %d, "
5369 "__kmp_max_tdgs = 0\n",
5370 gtid, loc_ref, input_flags, tdg_id));
5374 __kmpc_taskgroup(loc_ref, gtid);
5375 if (kmp_tdg_info_t *tdg = __kmp_find_tdg(tdg_id)) {
5377 __kmp_exec_tdg(gtid, tdg);
5380 __kmp_curr_tdg_idx = tdg_id;
5381 KMP_DEBUG_ASSERT(__kmp_curr_tdg_idx < __kmp_max_tdgs);
5382 __kmp_start_record(gtid, flags, tdg_id);
5386 KA_TRACE(10, (
"__kmpc_start_record_task(exit): T#%d TDG %d starts to %s\n",
5387 gtid, tdg_id, res ?
"record" :
"execute"));
5394 void __kmp_end_record(kmp_int32 gtid, kmp_tdg_info_t *tdg) {
5396 kmp_node_info_t *this_record_map = tdg->record_map;
5397 kmp_int32 this_num_tasks = KMP_ATOMIC_LD_RLX(&tdg->num_tasks);
5398 kmp_int32 *this_root_tasks =
5399 (kmp_int32 *)__kmp_allocate(this_num_tasks *
sizeof(kmp_int32));
5400 kmp_int32 this_map_size = tdg->map_size;
5401 kmp_int32 this_num_roots = 0;
5402 kmp_info_t *thread = __kmp_threads[gtid];
5404 for (kmp_int32 i = 0; i < this_num_tasks; i++) {
5405 if (this_record_map[i].npredecessors == 0) {
5406 this_root_tasks[this_num_roots++] = i;
5411 tdg->map_size = this_map_size;
5412 tdg->num_roots = this_num_roots;
5413 tdg->root_tasks = this_root_tasks;
5414 KMP_DEBUG_ASSERT(tdg->tdg_status == KMP_TDG_RECORDING);
5415 tdg->tdg_status = KMP_TDG_READY;
5417 if (thread->th.th_current_task->td_dephash) {
5418 __kmp_dephash_free(thread, thread->th.th_current_task->td_dephash);
5419 thread->th.th_current_task->td_dephash = NULL;
5423 for (kmp_int32 i = 0; i < this_num_tasks; i++) {
5424 KMP_ATOMIC_ST_RLX(&this_record_map[i].npredecessors_counter,
5425 this_record_map[i].npredecessors);
5427 KMP_ATOMIC_ST_RLX(&__kmp_tdg_task_id, 0);
5430 __kmp_print_tdg_dot(tdg, gtid);
5440 void __kmpc_end_record_task(
ident_t *loc_ref, kmp_int32 gtid,
5441 kmp_int32 input_flags, kmp_int32 tdg_id) {
5442 kmp_tdg_info_t *tdg = __kmp_find_tdg(tdg_id);
5444 KA_TRACE(10, (
"__kmpc_end_record_task(enter): T#%d loc=%p finishes recording"
5445 " tdg=%d with flags=%d\n",
5446 gtid, loc_ref, tdg_id, input_flags));
5447 if (__kmp_max_tdgs) {
5449 __kmpc_end_taskgroup(loc_ref, gtid);
5450 if (__kmp_tdg_is_recording(tdg->tdg_status))
5451 __kmp_end_record(gtid, tdg);
5453 KA_TRACE(10, (
"__kmpc_end_record_task(exit): T#%d loc=%p finished recording"
5454 " tdg=%d, its status is now READY\n",
5455 gtid, loc_ref, tdg_id));
struct kmp_taskred_data kmp_taskred_data_t
struct kmp_task_red_input kmp_task_red_input_t
struct kmp_taskred_flags kmp_taskred_flags_t
struct kmp_taskred_input kmp_taskred_input_t
#define KMP_COUNT_BLOCK(name)
Increments specified counter (name).
void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int sched, kmp_uint64 grainsize, void *task_dup)
void * __kmpc_taskred_modifier_init(ident_t *loc, int gtid, int is_ws, int num, void *data)
void * __kmpc_taskred_init(int gtid, int num, void *data)
void * __kmpc_task_reduction_init(int gtid, int num, void *data)
bool __kmpc_omp_has_task_team(kmp_int32 gtid)
void __kmpc_proxy_task_completed_ooo(kmp_task_t *ptask)
void __kmpc_task_reduction_modifier_fini(ident_t *loc, int gtid, int is_ws)
kmp_int32 __kmpc_omp_reg_task_with_affinity(ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 naffins, kmp_task_affinity_info_t *affin_list)
void * __kmpc_task_reduction_modifier_init(ident_t *loc, int gtid, int is_ws, int num, void *data)
void __kmpc_taskloop_5(ident_t *loc, int gtid, kmp_task_t *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int sched, kmp_uint64 grainsize, int modifier, void *task_dup)
void __kmpc_proxy_task_completed(kmp_int32 gtid, kmp_task_t *ptask)
void * __kmpc_task_reduction_get_th_data(int gtid, void *tskgrp, void *data)
void ** __kmpc_omp_get_target_async_handle_ptr(kmp_int32 gtid)
kmp_taskred_flags_t flags