14 #include "kmp_error.h"
18 #define MAX_MESSAGE 512
22 #define KMP_DEBUG_REF_CTS(x) KF_TRACE(1, x);
24 #define THREAD_ALLOC_FOR_TASKQ
26 static int in_parallel_context(kmp_team_t *team) {
27 return !team->t.t_serialized;
30 static void __kmp_taskq_eo(
int *gtid_ref,
int *cid_ref,
ident_t *loc_ref) {
32 int tid = __kmp_tid_from_gtid(gtid);
34 kmpc_task_queue_t *taskq;
35 kmp_taskq_t *tq = &__kmp_threads[gtid]->th.th_team->t.t_taskq;
37 if (__kmp_env_consistency_check)
38 #if KMP_USE_DYNAMIC_LOCK
39 __kmp_push_sync(gtid, ct_ordered_in_taskq, loc_ref, NULL, 0);
41 __kmp_push_sync(gtid, ct_ordered_in_taskq, loc_ref, NULL);
44 if (!__kmp_threads[gtid]->th.th_team->t.t_serialized) {
50 my_token = tq->tq_curr_thunk[tid]->th_tasknum;
52 taskq = tq->tq_curr_thunk[tid]->th.th_shareds->sv_queue;
54 KMP_WAIT_YIELD(&taskq->tq_tasknum_serving, my_token, KMP_EQ, NULL);
59 static void __kmp_taskq_xo(
int *gtid_ref,
int *cid_ref,
ident_t *loc_ref) {
61 int tid = __kmp_tid_from_gtid(gtid);
63 kmp_taskq_t *tq = &__kmp_threads[gtid]->th.th_team->t.t_taskq;
65 if (__kmp_env_consistency_check)
66 __kmp_pop_sync(gtid, ct_ordered_in_taskq, loc_ref);
68 if (!__kmp_threads[gtid]->th.th_team->t.t_serialized) {
74 my_token = tq->tq_curr_thunk[tid]->th_tasknum;
78 tq->tq_curr_thunk[tid]->th.th_shareds->sv_queue->tq_tasknum_serving =
85 static void __kmp_taskq_check_ordered(kmp_int32 gtid, kmpc_thunk_t *thunk) {
87 kmpc_task_queue_t *taskq;
93 my_token = thunk->th_tasknum;
95 taskq = thunk->th.th_shareds->sv_queue;
97 if (taskq->tq_tasknum_serving <= my_token) {
98 KMP_WAIT_YIELD(&taskq->tq_tasknum_serving, my_token, KMP_GE, NULL);
100 taskq->tq_tasknum_serving = my_token + 1;
107 static void __kmp_dump_TQF(kmp_int32 flags) {
108 if (flags & TQF_IS_ORDERED)
109 __kmp_printf(
"ORDERED ");
110 if (flags & TQF_IS_LASTPRIVATE)
111 __kmp_printf(
"LAST_PRIV ");
112 if (flags & TQF_IS_NOWAIT)
113 __kmp_printf(
"NOWAIT ");
114 if (flags & TQF_HEURISTICS)
115 __kmp_printf(
"HEURIST ");
116 if (flags & TQF_INTERFACE_RESERVED1)
117 __kmp_printf(
"RESERV1 ");
118 if (flags & TQF_INTERFACE_RESERVED2)
119 __kmp_printf(
"RESERV2 ");
120 if (flags & TQF_INTERFACE_RESERVED3)
121 __kmp_printf(
"RESERV3 ");
122 if (flags & TQF_INTERFACE_RESERVED4)
123 __kmp_printf(
"RESERV4 ");
124 if (flags & TQF_IS_LAST_TASK)
125 __kmp_printf(
"LAST_TASK ");
126 if (flags & TQF_TASKQ_TASK)
127 __kmp_printf(
"TASKQ_TASK ");
128 if (flags & TQF_RELEASE_WORKERS)
129 __kmp_printf(
"RELEASE ");
130 if (flags & TQF_ALL_TASKS_QUEUED)
131 __kmp_printf(
"ALL_QUEUED ");
132 if (flags & TQF_PARALLEL_CONTEXT)
133 __kmp_printf(
"PARALLEL ");
134 if (flags & TQF_DEALLOCATED)
135 __kmp_printf(
"DEALLOC ");
136 if (!(flags & (TQF_INTERNAL_FLAGS | TQF_INTERFACE_FLAGS)))
137 __kmp_printf(
"(NONE)");
140 static void __kmp_dump_thunk(kmp_taskq_t *tq, kmpc_thunk_t *thunk,
141 kmp_int32 global_tid) {
143 int nproc = __kmp_threads[global_tid]->th.th_team->t.t_nproc;
145 __kmp_printf(
"\tThunk at %p on (%d): ", thunk, global_tid);
148 for (i = 0; i < nproc; i++) {
149 if (tq->tq_curr_thunk[i] == thunk) {
150 __kmp_printf(
"[%i] ", i);
153 __kmp_printf(
"th_shareds=%p, ", thunk->th.th_shareds);
154 __kmp_printf(
"th_task=%p, ", thunk->th_task);
155 __kmp_printf(
"th_encl_thunk=%p, ", thunk->th_encl_thunk);
156 __kmp_printf(
"th_status=%d, ", thunk->th_status);
157 __kmp_printf(
"th_tasknum=%u, ", thunk->th_tasknum);
158 __kmp_printf(
"th_flags=");
159 __kmp_dump_TQF(thunk->th_flags);
165 static void __kmp_dump_thunk_stack(kmpc_thunk_t *thunk, kmp_int32 thread_num) {
168 __kmp_printf(
" Thunk stack for T#%d: ", thread_num);
170 for (th = thunk; th != NULL; th = th->th_encl_thunk)
171 __kmp_printf(
"%p ", th);
176 static void __kmp_dump_task_queue(kmp_taskq_t *tq, kmpc_task_queue_t *queue,
177 kmp_int32 global_tid) {
180 kmpc_task_queue_t *taskq;
182 __kmp_printf(
"Task Queue at %p on (%d):\n", queue, global_tid);
185 int in_parallel = queue->tq_flags & TQF_PARALLEL_CONTEXT;
187 if (__kmp_env_consistency_check) {
188 __kmp_printf(
" tq_loc : ");
201 __kmp_printf(
" tq_parent : %p\n", queue->tq.tq_parent);
202 __kmp_printf(
" tq_first_child : %p\n", queue->tq_first_child);
203 __kmp_printf(
" tq_next_child : %p\n", queue->tq_next_child);
204 __kmp_printf(
" tq_prev_child : %p\n", queue->tq_prev_child);
205 __kmp_printf(
" tq_ref_count : %d\n", queue->tq_ref_count);
220 __kmp_printf(
" tq_shareds : ");
221 for (i = 0; i < ((queue == tq->tq_root) ? queue->tq_nproc : 1); i++)
222 __kmp_printf(
"%p ", queue->tq_shareds[i].ai_data);
226 __kmp_printf(
" tq_tasknum_queuing : %u\n", queue->tq_tasknum_queuing);
227 __kmp_printf(
" tq_tasknum_serving : %u\n", queue->tq_tasknum_serving);
230 __kmp_printf(
" tq_queue : %p\n", queue->tq_queue);
231 __kmp_printf(
" tq_thunk_space : %p\n", queue->tq_thunk_space);
232 __kmp_printf(
" tq_taskq_slot : %p\n", queue->tq_taskq_slot);
234 __kmp_printf(
" tq_free_thunks : ");
235 for (thunk = queue->tq_free_thunks; thunk != NULL;
236 thunk = thunk->th.th_next_free)
237 __kmp_printf(
"%p ", thunk);
240 __kmp_printf(
" tq_nslots : %d\n", queue->tq_nslots);
241 __kmp_printf(
" tq_head : %d\n", queue->tq_head);
242 __kmp_printf(
" tq_tail : %d\n", queue->tq_tail);
243 __kmp_printf(
" tq_nfull : %d\n", queue->tq_nfull);
244 __kmp_printf(
" tq_hiwat : %d\n", queue->tq_hiwat);
245 __kmp_printf(
" tq_flags : ");
246 __kmp_dump_TQF(queue->tq_flags);
250 __kmp_printf(
" tq_th_thunks : ");
251 for (i = 0; i < queue->tq_nproc; i++) {
252 __kmp_printf(
"%d ", queue->tq_th_thunks[i].ai_data);
258 __kmp_printf(
" Queue slots:\n");
261 for (count = 0; count < queue->tq_nfull; ++count) {
262 __kmp_printf(
"(%d)", qs);
263 __kmp_dump_thunk(tq, queue->tq_queue[qs].qs_thunk, global_tid);
264 qs = (qs + 1) % queue->tq_nslots;
270 if (queue->tq_taskq_slot != NULL) {
271 __kmp_printf(
" TaskQ slot:\n");
272 __kmp_dump_thunk(tq, CCAST(kmpc_thunk_t *, queue->tq_taskq_slot),
281 __kmp_printf(
" Taskq freelist: ");
289 for (taskq = tq->tq_freelist; taskq != NULL; taskq = taskq->tq.tq_next_free)
290 __kmp_printf(
"%p ", taskq);
294 __kmp_printf(
"\n\n");
297 static void __kmp_aux_dump_task_queue_tree(kmp_taskq_t *tq,
298 kmpc_task_queue_t *curr_queue,
300 kmp_int32 global_tid) {
302 int nproc = __kmp_threads[global_tid]->th.th_team->t.t_nproc;
303 kmpc_task_queue_t *queue = curr_queue;
305 if (curr_queue == NULL)
310 for (i = 0; i < level; i++)
313 __kmp_printf(
"%p", curr_queue);
315 for (i = 0; i < nproc; i++) {
316 if (tq->tq_curr_thunk[i] &&
317 tq->tq_curr_thunk[i]->th.th_shareds->sv_queue == curr_queue) {
318 __kmp_printf(
" [%i]", i);
330 qs = curr_queue->tq_tail;
332 for (count = 0; count < curr_queue->tq_nfull; ++count) {
333 __kmp_printf(
"%p ", curr_queue->tq_queue[qs].qs_thunk);
334 qs = (qs + 1) % curr_queue->tq_nslots;
341 if (curr_queue->tq_first_child) {
348 if (curr_queue->tq_first_child) {
349 for (queue = CCAST(kmpc_task_queue_t *, curr_queue->tq_first_child);
350 queue != NULL; queue = queue->tq_next_child) {
351 __kmp_aux_dump_task_queue_tree(tq, queue, level + 1, global_tid);
359 static void __kmp_dump_task_queue_tree(kmp_taskq_t *tq,
360 kmpc_task_queue_t *tqroot,
361 kmp_int32 global_tid) {
362 __kmp_printf(
"TaskQ Tree at root %p on (%d):\n", tqroot, global_tid);
364 __kmp_aux_dump_task_queue_tree(tq, tqroot, 0, global_tid);
372 static void *__kmp_taskq_allocate(
size_t size, kmp_int32 global_tid) {
373 void *addr, *orig_addr;
376 KB_TRACE(5, (
"__kmp_taskq_allocate: called size=%d, gtid=%d\n", (
int)size,
379 bytes =
sizeof(
void *) + CACHE_LINE + size;
381 #ifdef THREAD_ALLOC_FOR_TASKQ
383 (
void *)__kmp_thread_malloc(__kmp_thread_from_gtid(global_tid), bytes);
385 KE_TRACE(10, (
"%%%%%% MALLOC( %d )\n", bytes));
386 orig_addr = (
void *)KMP_INTERNAL_MALLOC(bytes);
390 KMP_FATAL(OutOfHeapMemory);
394 if (((kmp_uintptr_t)addr & (CACHE_LINE - 1)) != 0) {
395 KB_TRACE(50, (
"__kmp_taskq_allocate: adjust for cache alignment\n"));
396 addr = (
void *)(((kmp_uintptr_t)addr + CACHE_LINE) & ~(CACHE_LINE - 1));
399 (*(
void **)addr) = orig_addr;
402 (
"__kmp_taskq_allocate: allocate: %p, use: %p - %p, size: %d, "
404 orig_addr, ((
void **)addr) + 1,
405 ((
char *)(((
void **)addr) + 1)) + size - 1, (
int)size, global_tid));
407 return (((
void **)addr) + 1);
410 static void __kmpc_taskq_free(
void *p, kmp_int32 global_tid) {
411 KB_TRACE(5, (
"__kmpc_taskq_free: called addr=%p, gtid=%d\n", p, global_tid));
413 KB_TRACE(10, (
"__kmpc_taskq_free: freeing: %p, gtid: %d\n",
414 (*(((
void **)p) - 1)), global_tid));
416 #ifdef THREAD_ALLOC_FOR_TASKQ
417 __kmp_thread_free(__kmp_thread_from_gtid(global_tid), *(((
void **)p) - 1));
419 KMP_INTERNAL_FREE(*(((
void **)p) - 1));
426 static kmpc_task_queue_t *
427 __kmp_alloc_taskq(kmp_taskq_t *tq,
int in_parallel, kmp_int32 nslots,
428 kmp_int32 nthunks, kmp_int32 nshareds, kmp_int32 nproc,
429 size_t sizeof_thunk,
size_t sizeof_shareds,
430 kmpc_thunk_t **new_taskq_thunk, kmp_int32 global_tid) {
433 kmpc_task_queue_t *new_queue;
434 kmpc_aligned_shared_vars_t *shared_var_array;
435 char *shared_var_storage;
438 __kmp_acquire_lock(&tq->tq_freelist_lck, global_tid);
444 if (tq->tq_freelist) {
445 new_queue = tq->tq_freelist;
446 tq->tq_freelist = tq->tq_freelist->tq.tq_next_free;
448 KMP_DEBUG_ASSERT(new_queue->tq_flags & TQF_DEALLOCATED);
450 new_queue->tq_flags = 0;
452 __kmp_release_lock(&tq->tq_freelist_lck, global_tid);
454 __kmp_release_lock(&tq->tq_freelist_lck, global_tid);
456 new_queue = (kmpc_task_queue_t *)__kmp_taskq_allocate(
457 sizeof(kmpc_task_queue_t), global_tid);
458 new_queue->tq_flags = 0;
465 (CACHE_LINE - (sizeof_thunk % CACHE_LINE));
466 pt = (
char *)__kmp_taskq_allocate(nthunks * sizeof_thunk, global_tid);
467 new_queue->tq_thunk_space = (kmpc_thunk_t *)pt;
468 *new_taskq_thunk = (kmpc_thunk_t *)(pt + (nthunks - 1) * sizeof_thunk);
472 new_queue->tq_free_thunks = (kmpc_thunk_t *)pt;
474 for (i = 0; i < (nthunks - 2); i++) {
475 ((kmpc_thunk_t *)(pt + i * sizeof_thunk))->th.th_next_free =
476 (kmpc_thunk_t *)(pt + (i + 1) * sizeof_thunk);
478 ((kmpc_thunk_t *)(pt + i * sizeof_thunk))->th_flags = TQF_DEALLOCATED;
482 ((kmpc_thunk_t *)(pt + (nthunks - 2) * sizeof_thunk))->th.th_next_free = NULL;
484 ((kmpc_thunk_t *)(pt + (nthunks - 2) * sizeof_thunk))->th_flags =
491 __kmp_init_lock(&new_queue->tq_link_lck);
492 __kmp_init_lock(&new_queue->tq_free_thunks_lck);
493 __kmp_init_lock(&new_queue->tq_queue_lck);
498 bytes = nslots *
sizeof(kmpc_aligned_queue_slot_t);
499 new_queue->tq_queue =
500 (kmpc_aligned_queue_slot_t *)__kmp_taskq_allocate(bytes, global_tid);
503 sizeof_shareds +=
sizeof(kmpc_task_queue_t *);
505 (CACHE_LINE - (sizeof_shareds % CACHE_LINE));
507 bytes = nshareds *
sizeof(kmpc_aligned_shared_vars_t);
509 (kmpc_aligned_shared_vars_t *)__kmp_taskq_allocate(bytes, global_tid);
511 bytes = nshareds * sizeof_shareds;
512 shared_var_storage = (
char *)__kmp_taskq_allocate(bytes, global_tid);
514 for (i = 0; i < nshareds; i++) {
515 shared_var_array[i].ai_data =
516 (kmpc_shared_vars_t *)(shared_var_storage + i * sizeof_shareds);
517 shared_var_array[i].ai_data->sv_queue = new_queue;
519 new_queue->tq_shareds = shared_var_array;
524 bytes = nproc *
sizeof(kmpc_aligned_int32_t);
525 new_queue->tq_th_thunks =
526 (kmpc_aligned_int32_t *)__kmp_taskq_allocate(bytes, global_tid);
527 new_queue->tq_nproc = nproc;
529 for (i = 0; i < nproc; i++)
530 new_queue->tq_th_thunks[i].ai_data = 0;
536 static void __kmp_free_taskq(kmp_taskq_t *tq, kmpc_task_queue_t *p,
537 int in_parallel, kmp_int32 global_tid) {
538 __kmpc_taskq_free(p->tq_thunk_space, global_tid);
539 __kmpc_taskq_free(p->tq_queue, global_tid);
542 __kmpc_taskq_free(CCAST(kmpc_shared_vars_t *, p->tq_shareds[0].ai_data),
545 __kmpc_taskq_free(p->tq_shareds, global_tid);
548 p->tq_first_child = NULL;
549 p->tq_next_child = NULL;
550 p->tq_prev_child = NULL;
551 p->tq_ref_count = -10;
552 p->tq_shareds = NULL;
553 p->tq_tasknum_queuing = 0;
554 p->tq_tasknum_serving = 0;
556 p->tq_thunk_space = NULL;
557 p->tq_taskq_slot = NULL;
558 p->tq_free_thunks = NULL;
568 for (i = 0; i < p->tq_nproc; i++)
569 p->tq_th_thunks[i].ai_data = 0;
571 if (__kmp_env_consistency_check)
573 KMP_DEBUG_ASSERT(p->tq_flags & TQF_DEALLOCATED);
574 p->tq_flags = TQF_DEALLOCATED;
578 __kmpc_taskq_free(p->tq_th_thunks, global_tid);
579 __kmp_destroy_lock(&p->tq_link_lck);
580 __kmp_destroy_lock(&p->tq_queue_lck);
581 __kmp_destroy_lock(&p->tq_free_thunks_lck);
584 p->tq_th_thunks = NULL;
591 __kmp_acquire_lock(&tq->tq_freelist_lck, global_tid);
592 p->tq.tq_next_free = tq->tq_freelist;
595 __kmp_release_lock(&tq->tq_freelist_lck, global_tid);
602 static kmpc_thunk_t *__kmp_alloc_thunk(kmpc_task_queue_t *queue,
603 int in_parallel, kmp_int32 global_tid) {
607 __kmp_acquire_lock(&queue->tq_free_thunks_lck, global_tid);
613 fl = queue->tq_free_thunks;
615 KMP_DEBUG_ASSERT(fl != NULL);
617 queue->tq_free_thunks = fl->th.th_next_free;
621 __kmp_release_lock(&queue->tq_free_thunks_lck, global_tid);
626 static void __kmp_free_thunk(kmpc_task_queue_t *queue, kmpc_thunk_t *p,
627 int in_parallel, kmp_int32 global_tid) {
630 p->th_encl_thunk = 0;
637 __kmp_acquire_lock(&queue->tq_free_thunks_lck, global_tid);
643 p->th.th_next_free = queue->tq_free_thunks;
644 queue->tq_free_thunks = p;
647 p->th_flags = TQF_DEALLOCATED;
651 __kmp_release_lock(&queue->tq_free_thunks_lck, global_tid);
655 static kmp_int32 __kmp_enqueue_task(kmp_taskq_t *tq, kmp_int32 global_tid,
656 kmpc_task_queue_t *queue,
657 kmpc_thunk_t *thunk,
int in_parallel) {
663 __kmp_acquire_lock(&queue->tq_queue_lck, global_tid);
669 KMP_DEBUG_ASSERT(queue->tq_nfull < queue->tq_nslots);
671 queue->tq_queue[(queue->tq_head)++].qs_thunk = thunk;
673 if (queue->tq_head >= queue->tq_nslots)
681 ret = (in_parallel) ? (queue->tq_nfull == queue->tq_nslots) : FALSE;
685 __kmp_release_lock(&queue->tq_queue_lck, global_tid);
687 if (tq->tq_global_flags & TQF_RELEASE_WORKERS) {
692 tq->tq_global_flags &= ~TQF_RELEASE_WORKERS;
704 static kmpc_thunk_t *__kmp_dequeue_task(kmp_int32 global_tid,
705 kmpc_task_queue_t *queue,
708 int tid = __kmp_tid_from_gtid(global_tid);
710 KMP_DEBUG_ASSERT(queue->tq_nfull > 0);
712 if (queue->tq.tq_parent != NULL && in_parallel) {
714 __kmp_acquire_lock(&queue->tq.tq_parent->tq_link_lck, global_tid);
715 ct = ++(queue->tq_ref_count);
716 __kmp_release_lock(&queue->tq.tq_parent->tq_link_lck, global_tid);
718 (
"line %d gtid %d: Q %p inc %d\n", __LINE__, global_tid, queue, ct));
721 pt = queue->tq_queue[(queue->tq_tail)++].qs_thunk;
723 if (queue->tq_tail >= queue->tq_nslots)
727 queue->tq_th_thunks[tid].ai_data++;
732 KF_TRACE(200, (
"__kmp_dequeue_task: T#%d(:%d) now has %d outstanding "
733 "thunks from queue %p\n",
734 global_tid, tid, queue->tq_th_thunks[tid].ai_data, queue));
745 KMP_DEBUG_ASSERT(queue->tq_nfull >= 0);
748 KMP_DEBUG_ASSERT(queue->tq_th_thunks[tid].ai_data <=
749 __KMP_TASKQ_THUNKS_PER_TH);
779 static kmpc_thunk_t *__kmp_find_task_in_queue(kmp_int32 global_tid,
780 kmpc_task_queue_t *queue) {
781 kmpc_thunk_t *pt = NULL;
782 int tid = __kmp_tid_from_gtid(global_tid);
785 if (!(queue->tq_flags & TQF_DEALLOCATED)) {
787 __kmp_acquire_lock(&queue->tq_queue_lck, global_tid);
790 if (!(queue->tq_flags & TQF_DEALLOCATED)) {
795 if ((queue->tq_taskq_slot != NULL) &&
796 (queue->tq_nfull <= queue->tq_hiwat)) {
799 pt = CCAST(kmpc_thunk_t *, queue->tq_taskq_slot);
800 queue->tq_taskq_slot = NULL;
801 }
else if (queue->tq_nfull == 0 ||
802 queue->tq_th_thunks[tid].ai_data >=
803 __KMP_TASKQ_THUNKS_PER_TH) {
807 }
else if (queue->tq_nfull > 1) {
810 pt = __kmp_dequeue_task(global_tid, queue, TRUE);
811 }
else if (!(queue->tq_flags & TQF_IS_LASTPRIVATE)) {
813 pt = __kmp_dequeue_task(global_tid, queue, TRUE);
814 }
else if (queue->tq_flags & TQF_IS_LAST_TASK) {
818 pt = __kmp_dequeue_task(global_tid, queue, TRUE);
825 __kmp_release_lock(&queue->tq_queue_lck, global_tid);
834 static kmpc_thunk_t *
835 __kmp_find_task_in_descendant_queue(kmp_int32 global_tid,
836 kmpc_task_queue_t *curr_queue) {
837 kmpc_thunk_t *pt = NULL;
838 kmpc_task_queue_t *queue = curr_queue;
840 if (curr_queue->tq_first_child != NULL) {
841 __kmp_acquire_lock(&curr_queue->tq_link_lck, global_tid);
846 queue = CCAST(kmpc_task_queue_t *, curr_queue->tq_first_child);
848 __kmp_release_lock(&curr_queue->tq_link_lck, global_tid);
852 while (queue != NULL) {
854 kmpc_task_queue_t *next;
856 ct = ++(queue->tq_ref_count);
857 __kmp_release_lock(&curr_queue->tq_link_lck, global_tid);
859 (
"line %d gtid %d: Q %p inc %d\n", __LINE__, global_tid, queue, ct));
861 pt = __kmp_find_task_in_queue(global_tid, queue);
866 __kmp_acquire_lock(&curr_queue->tq_link_lck, global_tid);
871 ct = --(queue->tq_ref_count);
872 KMP_DEBUG_REF_CTS((
"line %d gtid %d: Q %p dec %d\n", __LINE__,
873 global_tid, queue, ct));
874 KMP_DEBUG_ASSERT(queue->tq_ref_count >= 0);
876 __kmp_release_lock(&curr_queue->tq_link_lck, global_tid);
885 pt = __kmp_find_task_in_descendant_queue(global_tid, queue);
890 __kmp_acquire_lock(&curr_queue->tq_link_lck, global_tid);
895 ct = --(queue->tq_ref_count);
896 KMP_DEBUG_REF_CTS((
"line %d gtid %d: Q %p dec %d\n", __LINE__,
897 global_tid, queue, ct));
898 KMP_DEBUG_ASSERT(ct >= 0);
900 __kmp_release_lock(&curr_queue->tq_link_lck, global_tid);
905 __kmp_acquire_lock(&curr_queue->tq_link_lck, global_tid);
910 next = queue->tq_next_child;
912 ct = --(queue->tq_ref_count);
914 (
"line %d gtid %d: Q %p dec %d\n", __LINE__, global_tid, queue, ct));
915 KMP_DEBUG_ASSERT(ct >= 0);
920 __kmp_release_lock(&curr_queue->tq_link_lck, global_tid);
929 static kmpc_thunk_t *
930 __kmp_find_task_in_ancestor_queue(kmp_taskq_t *tq, kmp_int32 global_tid,
931 kmpc_task_queue_t *curr_queue) {
932 kmpc_task_queue_t *queue;
937 if (curr_queue->tq.tq_parent != NULL) {
938 queue = curr_queue->tq.tq_parent;
940 while (queue != NULL) {
941 if (queue->tq.tq_parent != NULL) {
943 __kmp_acquire_lock(&queue->tq.tq_parent->tq_link_lck, global_tid);
948 ct = ++(queue->tq_ref_count);
949 __kmp_release_lock(&queue->tq.tq_parent->tq_link_lck, global_tid);
950 KMP_DEBUG_REF_CTS((
"line %d gtid %d: Q %p inc %d\n", __LINE__,
951 global_tid, queue, ct));
954 pt = __kmp_find_task_in_queue(global_tid, queue);
956 if (queue->tq.tq_parent != NULL) {
958 __kmp_acquire_lock(&queue->tq.tq_parent->tq_link_lck, global_tid);
963 ct = --(queue->tq_ref_count);
964 KMP_DEBUG_REF_CTS((
"line %d gtid %d: Q %p dec %d\n", __LINE__,
965 global_tid, queue, ct));
966 KMP_DEBUG_ASSERT(ct >= 0);
968 __kmp_release_lock(&queue->tq.tq_parent->tq_link_lck, global_tid);
974 if (queue->tq.tq_parent != NULL) {
976 __kmp_acquire_lock(&queue->tq.tq_parent->tq_link_lck, global_tid);
981 ct = --(queue->tq_ref_count);
982 KMP_DEBUG_REF_CTS((
"line %d gtid %d: Q %p dec %d\n", __LINE__,
983 global_tid, queue, ct));
984 KMP_DEBUG_ASSERT(ct >= 0);
986 queue = queue->tq.tq_parent;
989 __kmp_release_lock(&queue->tq_link_lck, global_tid);
993 pt = __kmp_find_task_in_descendant_queue(global_tid, tq->tq_root);
998 static int __kmp_taskq_tasks_finished(kmpc_task_queue_t *queue) {
1003 for (i = 0; i < queue->tq_nproc; i++) {
1004 if (queue->tq_th_thunks[i].ai_data != 0)
1011 static int __kmp_taskq_has_any_children(kmpc_task_queue_t *queue) {
1012 return (queue->tq_first_child != NULL);
1015 static void __kmp_remove_queue_from_tree(kmp_taskq_t *tq, kmp_int32 global_tid,
1016 kmpc_task_queue_t *queue,
1020 kmpc_thunk_t *thunk;
1024 (
"Before Deletion of TaskQ at %p on (%d):\n", queue, global_tid));
1025 KF_DUMP(50, __kmp_dump_task_queue(tq, queue, global_tid));
1028 KMP_DEBUG_ASSERT(queue->tq.tq_parent != NULL);
1031 __kmp_acquire_lock(&queue->tq.tq_parent->tq_link_lck, global_tid);
1037 KMP_DEBUG_ASSERT(queue->tq_first_child == NULL);
1040 if (queue->tq_prev_child != NULL)
1041 queue->tq_prev_child->tq_next_child = queue->tq_next_child;
1042 if (queue->tq_next_child != NULL)
1043 queue->tq_next_child->tq_prev_child = queue->tq_prev_child;
1044 if (queue->tq.tq_parent->tq_first_child == queue)
1045 queue->tq.tq_parent->tq_first_child = queue->tq_next_child;
1047 queue->tq_prev_child = NULL;
1048 queue->tq_next_child = NULL;
1052 (
"line %d gtid %d: Q %p waiting for ref_count of %d to reach 1\n",
1053 __LINE__, global_tid, queue, queue->tq_ref_count));
1056 while (queue->tq_ref_count > 1) {
1057 __kmp_release_lock(&queue->tq.tq_parent->tq_link_lck, global_tid);
1059 KMP_WAIT_YIELD((
volatile kmp_uint32 *)&queue->tq_ref_count, 1, KMP_LE,
1062 __kmp_acquire_lock(&queue->tq.tq_parent->tq_link_lck, global_tid);
1068 __kmp_release_lock(&queue->tq.tq_parent->tq_link_lck, global_tid);
1072 (
"line %d gtid %d: Q %p freeing queue\n", __LINE__, global_tid, queue));
1075 KMP_DEBUG_ASSERT(queue->tq_flags & TQF_ALL_TASKS_QUEUED);
1076 KMP_DEBUG_ASSERT(queue->tq_nfull == 0);
1078 for (i = 0; i < queue->tq_nproc; i++) {
1079 KMP_DEBUG_ASSERT(queue->tq_th_thunks[i].ai_data == 0);
1083 for (thunk = queue->tq_free_thunks; thunk != NULL;
1084 thunk = thunk->th.th_next_free)
1088 queue->tq_nslots + (queue->tq_nproc * __KMP_TASKQ_THUNKS_PER_TH));
1092 __kmp_free_taskq(tq, queue, TRUE, global_tid);
1094 KF_TRACE(50, (
"After Deletion of TaskQ at %p on (%d):\n", queue, global_tid));
1095 KF_DUMP(50, __kmp_dump_task_queue_tree(tq, tq->tq_root, global_tid));
1102 static void __kmp_find_and_remove_finished_child_taskq(
1103 kmp_taskq_t *tq, kmp_int32 global_tid, kmpc_task_queue_t *curr_queue) {
1104 kmpc_task_queue_t *queue = curr_queue;
1106 if (curr_queue->tq_first_child != NULL) {
1107 __kmp_acquire_lock(&curr_queue->tq_link_lck, global_tid);
1112 queue = CCAST(kmpc_task_queue_t *, curr_queue->tq_first_child);
1113 if (queue != NULL) {
1114 __kmp_release_lock(&curr_queue->tq_link_lck, global_tid);
1118 while (queue != NULL) {
1119 kmpc_task_queue_t *next;
1120 int ct = ++(queue->tq_ref_count);
1122 (
"line %d gtid %d: Q %p inc %d\n", __LINE__, global_tid, queue, ct));
1128 if (queue->tq_flags & TQF_IS_NOWAIT) {
1129 __kmp_find_and_remove_finished_child_taskq(tq, global_tid, queue);
1131 if ((queue->tq_flags & TQF_ALL_TASKS_QUEUED) &&
1132 (queue->tq_nfull == 0) && __kmp_taskq_tasks_finished(queue) &&
1133 !__kmp_taskq_has_any_children(queue)) {
1138 if (__kmp_test_lock(&queue->tq_queue_lck, global_tid)) {
1139 if (!(queue->tq_flags & TQF_DEALLOCATED)) {
1140 queue->tq_flags |= TQF_DEALLOCATED;
1141 __kmp_release_lock(&queue->tq_queue_lck, global_tid);
1143 __kmp_remove_queue_from_tree(tq, global_tid, queue, TRUE);
1149 __kmp_release_lock(&queue->tq_queue_lck, global_tid);
1156 __kmp_acquire_lock(&curr_queue->tq_link_lck, global_tid);
1161 next = queue->tq_next_child;
1163 ct = --(queue->tq_ref_count);
1165 (
"line %d gtid %d: Q %p dec %d\n", __LINE__, global_tid, queue, ct));
1166 KMP_DEBUG_ASSERT(ct >= 0);
1171 __kmp_release_lock(&curr_queue->tq_link_lck, global_tid);
1178 static void __kmp_remove_all_child_taskq(kmp_taskq_t *tq, kmp_int32 global_tid,
1179 kmpc_task_queue_t *queue) {
1180 kmpc_task_queue_t *next_child;
1182 queue = CCAST(kmpc_task_queue_t *, queue->tq_first_child);
1184 while (queue != NULL) {
1185 __kmp_remove_all_child_taskq(tq, global_tid, queue);
1187 next_child = queue->tq_next_child;
1188 queue->tq_flags |= TQF_DEALLOCATED;
1189 __kmp_remove_queue_from_tree(tq, global_tid, queue, FALSE);
1194 static void __kmp_execute_task_from_queue(kmp_taskq_t *tq,
ident_t *loc,
1195 kmp_int32 global_tid,
1196 kmpc_thunk_t *thunk,
1198 kmpc_task_queue_t *queue = thunk->th.th_shareds->sv_queue;
1199 kmp_int32 tid = __kmp_tid_from_gtid(global_tid);
1201 KF_TRACE(100, (
"After dequeueing this Task on (%d):\n", global_tid));
1202 KF_DUMP(100, __kmp_dump_thunk(tq, thunk, global_tid));
1203 KF_TRACE(100, (
"Task Queue: %p looks like this (%d):\n", queue, global_tid));
1204 KF_DUMP(100, __kmp_dump_task_queue(tq, queue, global_tid));
1220 if (!(thunk->th_flags & TQF_TASKQ_TASK)) {
1221 kmp_int32 index = (queue == tq->tq_root) ? tid : 0;
1222 thunk->th.th_shareds =
1223 CCAST(kmpc_shared_vars_t *, queue->tq_shareds[index].ai_data);
1225 if (__kmp_env_consistency_check) {
1226 __kmp_push_workshare(global_tid,
1227 (queue->tq_flags & TQF_IS_ORDERED) ? ct_task_ordered
1232 if (__kmp_env_consistency_check)
1233 __kmp_push_workshare(global_tid, ct_taskq, queue->tq_loc);
1237 thunk->th_encl_thunk = tq->tq_curr_thunk[tid];
1238 tq->tq_curr_thunk[tid] = thunk;
1240 KF_DUMP(200, __kmp_dump_thunk_stack(tq->tq_curr_thunk[tid], global_tid));
1243 KF_TRACE(50, (
"Begin Executing Thunk %p from queue %p on (%d)\n", thunk,
1244 queue, global_tid));
1245 thunk->th_task(global_tid, thunk);
1246 KF_TRACE(50, (
"End Executing Thunk %p from queue %p on (%d)\n", thunk, queue,
1249 if (!(thunk->th_flags & TQF_TASKQ_TASK)) {
1250 if (__kmp_env_consistency_check)
1251 __kmp_pop_workshare(global_tid,
1252 (queue->tq_flags & TQF_IS_ORDERED) ? ct_task_ordered
1257 tq->tq_curr_thunk[tid] = thunk->th_encl_thunk;
1258 thunk->th_encl_thunk = NULL;
1259 KF_DUMP(200, __kmp_dump_thunk_stack(tq->tq_curr_thunk[tid], global_tid));
1262 if ((thunk->th_flags & TQF_IS_ORDERED) && in_parallel) {
1263 __kmp_taskq_check_ordered(global_tid, thunk);
1266 __kmp_free_thunk(queue, thunk, in_parallel, global_tid);
1268 KF_TRACE(100, (
"T#%d After freeing thunk: %p, TaskQ looks like this:\n",
1269 global_tid, thunk));
1270 KF_DUMP(100, __kmp_dump_task_queue(tq, queue, global_tid));
1276 KMP_DEBUG_ASSERT(queue->tq_th_thunks[tid].ai_data >= 1);
1280 (
"__kmp_execute_task_from_queue: T#%d has %d thunks in queue %p\n",
1281 global_tid, queue->tq_th_thunks[tid].ai_data - 1, queue));
1283 queue->tq_th_thunks[tid].ai_data--;
1288 if (queue->tq.tq_parent != NULL && in_parallel) {
1290 __kmp_acquire_lock(&queue->tq.tq_parent->tq_link_lck, global_tid);
1291 ct = --(queue->tq_ref_count);
1292 __kmp_release_lock(&queue->tq.tq_parent->tq_link_lck, global_tid);
1294 (
"line %d gtid %d: Q %p dec %d\n", __LINE__, global_tid, queue, ct));
1295 KMP_DEBUG_ASSERT(ct >= 0);
1302 kmpc_thunk_t *__kmpc_taskq(
ident_t *loc, kmp_int32 global_tid,
1303 kmpc_task_t taskq_task,
size_t sizeof_thunk,
1304 size_t sizeof_shareds, kmp_int32 flags,
1305 kmpc_shared_vars_t **shareds) {
1307 kmp_int32 nslots, nthunks, nshareds, nproc;
1308 kmpc_task_queue_t *new_queue, *curr_queue;
1309 kmpc_thunk_t *new_taskq_thunk;
1315 KE_TRACE(10, (
"__kmpc_taskq called (%d)\n", global_tid));
1317 th = __kmp_threads[global_tid];
1318 team = th->th.th_team;
1319 tq = &team->t.t_taskq;
1320 nproc = team->t.t_nproc;
1321 tid = __kmp_tid_from_gtid(global_tid);
1324 in_parallel = in_parallel_context(team);
1329 th->th.th_dispatch->th_deo_fcn = __kmp_taskq_eo;
1332 th->th.th_dispatch->th_dxo_fcn = __kmp_taskq_xo;
1338 if (__kmp_barrier(bs_plain_barrier, global_tid, TRUE, 0, NULL, NULL)) {
1345 CCAST(kmpc_shared_vars_t *, tq->tq_root->tq_shareds[tid].ai_data);
1346 KE_TRACE(10, (
"__kmpc_taskq return (%d)\n", global_tid));
1353 if (tq->tq_curr_thunk_capacity < nproc) {
1354 if (tq->tq_curr_thunk)
1355 __kmp_free(tq->tq_curr_thunk);
1359 __kmp_init_lock(&tq->tq_freelist_lck);
1363 (kmpc_thunk_t **)__kmp_allocate(nproc *
sizeof(kmpc_thunk_t *));
1364 tq->tq_curr_thunk_capacity = nproc;
1368 tq->tq_global_flags = TQF_RELEASE_WORKERS;
1373 nslots = (in_parallel) ? (2 * nproc) : 1;
1377 nthunks = (in_parallel) ? (nslots + (nproc * __KMP_TASKQ_THUNKS_PER_TH) + 1)
1382 nshareds = (!tq->tq_root && in_parallel) ? nproc : 1;
1386 new_queue = __kmp_alloc_taskq(tq, in_parallel, nslots, nthunks, nshareds,
1387 nproc, sizeof_thunk, sizeof_shareds,
1388 &new_taskq_thunk, global_tid);
1391 new_queue->tq_flags = flags & TQF_INTERFACE_FLAGS;
1394 new_queue->tq_tasknum_queuing = 0;
1395 new_queue->tq_tasknum_serving = 0;
1396 new_queue->tq_flags |= TQF_PARALLEL_CONTEXT;
1399 new_queue->tq_taskq_slot = NULL;
1400 new_queue->tq_nslots = nslots;
1401 new_queue->tq_hiwat = HIGH_WATER_MARK(nslots);
1402 new_queue->tq_nfull = 0;
1403 new_queue->tq_head = 0;
1404 new_queue->tq_tail = 0;
1405 new_queue->tq_loc = loc;
1407 if ((new_queue->tq_flags & TQF_IS_ORDERED) && in_parallel) {
1409 new_queue->tq_tasknum_serving = 1;
1412 th->th.th_dispatch->th_deo_fcn = __kmp_taskq_eo;
1415 th->th.th_dispatch->th_dxo_fcn = __kmp_taskq_xo;
1419 *shareds = CCAST(kmpc_shared_vars_t *, new_queue->tq_shareds[0].ai_data);
1421 new_taskq_thunk->th.th_shareds = *shareds;
1422 new_taskq_thunk->th_task = taskq_task;
1423 new_taskq_thunk->th_flags = new_queue->tq_flags | TQF_TASKQ_TASK;
1424 new_taskq_thunk->th_status = 0;
1426 KMP_DEBUG_ASSERT(new_taskq_thunk->th_flags & TQF_TASKQ_TASK);
1436 new_queue->tq.tq_parent = NULL;
1437 new_queue->tq_first_child = NULL;
1438 new_queue->tq_next_child = NULL;
1439 new_queue->tq_prev_child = NULL;
1440 new_queue->tq_ref_count = 1;
1441 tq->tq_root = new_queue;
1443 curr_queue = tq->tq_curr_thunk[tid]->th.th_shareds->sv_queue;
1444 new_queue->tq.tq_parent = curr_queue;
1445 new_queue->tq_first_child = NULL;
1446 new_queue->tq_prev_child = NULL;
1447 new_queue->tq_ref_count =
1450 KMP_DEBUG_REF_CTS((
"line %d gtid %d: Q %p alloc %d\n", __LINE__,
1451 global_tid, new_queue, new_queue->tq_ref_count));
1453 __kmp_acquire_lock(&curr_queue->tq_link_lck, global_tid);
1459 new_queue->tq_next_child =
1460 CCAST(
struct kmpc_task_queue_t *, curr_queue->tq_first_child);
1462 if (curr_queue->tq_first_child != NULL)
1463 curr_queue->tq_first_child->tq_prev_child = new_queue;
1465 curr_queue->tq_first_child = new_queue;
1467 __kmp_release_lock(&curr_queue->tq_link_lck, global_tid);
1471 new_taskq_thunk->th_encl_thunk = tq->tq_curr_thunk[tid];
1472 tq->tq_curr_thunk[tid] = new_taskq_thunk;
1474 KF_DUMP(200, __kmp_dump_thunk_stack(tq->tq_curr_thunk[tid], global_tid));
1476 new_taskq_thunk->th_encl_thunk = 0;
1477 new_queue->tq.tq_parent = NULL;
1478 new_queue->tq_first_child = NULL;
1479 new_queue->tq_next_child = NULL;
1480 new_queue->tq_prev_child = NULL;
1481 new_queue->tq_ref_count = 1;
1485 KF_TRACE(150, (
"Creating TaskQ Task on (%d):\n", global_tid));
1486 KF_DUMP(150, __kmp_dump_thunk(tq, new_taskq_thunk, global_tid));
1490 (
"After TaskQ at %p Creation on (%d):\n", new_queue, global_tid));
1492 KF_TRACE(25, (
"After Serial TaskQ at %p Creation on (%d):\n", new_queue,
1496 KF_DUMP(25, __kmp_dump_task_queue(tq, new_queue, global_tid));
1499 KF_DUMP(50, __kmp_dump_task_queue_tree(tq, tq->tq_root, global_tid));
1503 if (__kmp_env_consistency_check)
1504 __kmp_push_workshare(global_tid, ct_taskq, new_queue->tq_loc);
1506 KE_TRACE(10, (
"__kmpc_taskq return (%d)\n", global_tid));
1508 return new_taskq_thunk;
1513 void __kmpc_end_taskq(
ident_t *loc, kmp_int32 global_tid,
1514 kmpc_thunk_t *taskq_thunk) {
1521 kmp_int32 is_outermost;
1522 kmpc_task_queue_t *queue;
1523 kmpc_thunk_t *thunk;
1526 KE_TRACE(10, (
"__kmpc_end_taskq called (%d)\n", global_tid));
1528 tq = &__kmp_threads[global_tid]->th.th_team->t.t_taskq;
1529 nproc = __kmp_threads[global_tid]->th.th_team->t.t_nproc;
1533 queue = (taskq_thunk == NULL) ? tq->tq_root
1534 : taskq_thunk->th.th_shareds->sv_queue;
1536 KE_TRACE(50, (
"__kmpc_end_taskq queue=%p (%d) \n", queue, global_tid));
1537 is_outermost = (queue == tq->tq_root);
1538 in_parallel = (queue->tq_flags & TQF_PARALLEL_CONTEXT);
1546 if (is_outermost && (KMP_MASTER_GTID(global_tid))) {
1547 if (tq->tq_global_flags & TQF_RELEASE_WORKERS) {
1549 tq->tq_global_flags &= ~TQF_RELEASE_WORKERS;
1551 __kmp_end_split_barrier(bs_plain_barrier, global_tid);
1559 KMP_INIT_YIELD(spins);
1561 while ((queue->tq_nfull == 0) && (queue->tq_taskq_slot == NULL) &&
1562 (!__kmp_taskq_has_any_children(queue)) &&
1563 (!(queue->tq_flags & TQF_ALL_TASKS_QUEUED))) {
1564 KMP_YIELD_WHEN(TRUE, spins);
1568 while (((queue->tq_nfull != 0) || (queue->tq_taskq_slot != NULL)) &&
1569 (thunk = __kmp_find_task_in_queue(global_tid, queue)) != NULL) {
1570 KF_TRACE(50, (
"Found thunk: %p in primary queue %p (%d)\n", thunk,
1571 queue, global_tid));
1572 __kmp_execute_task_from_queue(tq, loc, global_tid, thunk, in_parallel);
1576 if ((__kmp_taskq_has_any_children(queue)) &&
1577 (thunk = __kmp_find_task_in_descendant_queue(global_tid, queue)) !=
1581 (
"Stole thunk: %p in descendant queue: %p while waiting in "
1583 thunk, thunk->th.th_shareds->sv_queue, queue, global_tid));
1585 __kmp_execute_task_from_queue(tq, loc, global_tid, thunk, in_parallel);
1588 }
while ((!(queue->tq_flags & TQF_ALL_TASKS_QUEUED)) ||
1589 (queue->tq_nfull != 0));
1591 KF_TRACE(50, (
"All tasks queued and dequeued in queue: %p (%d)\n", queue,
1597 while ((!__kmp_taskq_tasks_finished(queue)) &&
1598 (thunk = __kmp_find_task_in_descendant_queue(global_tid, queue)) !=
1601 KF_TRACE(50, (
"Stole thunk: %p in descendant queue: %p while waiting in "
1603 thunk, thunk->th.th_shareds->sv_queue, queue, global_tid));
1605 __kmp_execute_task_from_queue(tq, loc, global_tid, thunk, in_parallel);
1608 KF_TRACE(50, (
"No work found in descendent queues or all work finished in "
1610 queue, global_tid));
1612 if (!is_outermost) {
1615 if (queue->tq_flags & TQF_IS_NOWAIT) {
1616 __kmp_acquire_lock(&queue->tq.tq_parent->tq_link_lck, global_tid);
1617 queue->tq_ref_count--;
1618 KMP_DEBUG_ASSERT(queue->tq_ref_count >= 0);
1619 __kmp_release_lock(&queue->tq.tq_parent->tq_link_lck, global_tid);
1622 10, (
"__kmpc_end_taskq return for nowait case (%d)\n", global_tid));
1627 __kmp_find_and_remove_finished_child_taskq(tq, global_tid, queue);
1631 KMP_INIT_YIELD(spins);
1633 while (!__kmp_taskq_tasks_finished(queue) ||
1634 __kmp_taskq_has_any_children(queue)) {
1635 thunk = __kmp_find_task_in_ancestor_queue(tq, global_tid, queue);
1637 if (thunk != NULL) {
1639 (
"Stole thunk: %p in ancestor queue: %p while waiting in "
1641 thunk, thunk->th.th_shareds->sv_queue, queue, global_tid));
1642 __kmp_execute_task_from_queue(tq, loc, global_tid, thunk,
1646 KMP_YIELD_WHEN(thunk == NULL, spins);
1648 __kmp_find_and_remove_finished_child_taskq(tq, global_tid, queue);
1651 __kmp_acquire_lock(&queue->tq_queue_lck, global_tid);
1652 if (!(queue->tq_flags & TQF_DEALLOCATED)) {
1653 queue->tq_flags |= TQF_DEALLOCATED;
1655 __kmp_release_lock(&queue->tq_queue_lck, global_tid);
1658 if (taskq_thunk != NULL) {
1659 __kmp_remove_queue_from_tree(tq, global_tid, queue, TRUE);
1664 (
"__kmpc_end_taskq return for non_outermost queue, wait case (%d)\n",
1672 KMP_INIT_YIELD(spins);
1674 while (!__kmp_taskq_tasks_finished(queue)) {
1675 thunk = __kmp_find_task_in_descendant_queue(global_tid, queue);
1677 if (thunk != NULL) {
1679 (
"Stole thunk: %p in descendant queue: %p while waiting in "
1681 thunk, thunk->th.th_shareds->sv_queue, queue, global_tid));
1683 __kmp_execute_task_from_queue(tq, loc, global_tid, thunk, in_parallel);
1686 KMP_YIELD_WHEN(thunk == NULL, spins);
1694 if (!__kmp_barrier(bs_plain_barrier, global_tid, TRUE, 0, NULL, NULL)) {
1699 __kmp_remove_all_child_taskq(tq, global_tid, queue);
1702 KF_TRACE(100, (
"T#%d Before Deletion of top-level TaskQ at %p:\n",
1703 global_tid, queue));
1704 KF_DUMP(100, __kmp_dump_task_queue(tq, queue, global_tid));
1708 KMP_DEBUG_ASSERT((queue->tq.tq_parent == NULL) &&
1709 (queue->tq_next_child == NULL));
1712 KMP_DEBUG_ASSERT(queue->tq_first_child == NULL);
1714 for (i = 0; i < nproc; i++) {
1715 KMP_DEBUG_ASSERT(queue->tq_th_thunks[i].ai_data == 0);
1718 for (i = 0, thunk = queue->tq_free_thunks; thunk != NULL;
1719 i++, thunk = thunk->th.th_next_free)
1722 KMP_DEBUG_ASSERT(i ==
1723 queue->tq_nslots + (nproc * __KMP_TASKQ_THUNKS_PER_TH));
1725 for (i = 0; i < nproc; i++) {
1726 KMP_DEBUG_ASSERT(!tq->tq_curr_thunk[i]);
1733 KF_TRACE(50, (
"After Deletion of top-level TaskQ at %p on (%d):\n", queue,
1736 queue->tq_flags |= TQF_DEALLOCATED;
1737 __kmp_free_taskq(tq, queue, in_parallel, global_tid);
1739 KF_DUMP(50, __kmp_dump_task_queue_tree(tq, tq->tq_root, global_tid));
1742 __kmp_end_split_barrier(bs_plain_barrier, global_tid);
1745 th = __kmp_threads[global_tid];
1748 th->th.th_dispatch->th_deo_fcn = 0;
1751 th->th.th_dispatch->th_dxo_fcn = 0;
1756 if (queue->tq_nfull > 0) {
1757 KMP_DEBUG_ASSERT(queue->tq_nfull == 1);
1759 thunk = __kmp_dequeue_task(global_tid, queue, in_parallel);
1761 if (queue->tq_flags & TQF_IS_LAST_TASK) {
1767 thunk->th_flags |= TQF_IS_LAST_TASK;
1770 KF_TRACE(50, (
"T#%d found thunk: %p in serial queue: %p\n", global_tid,
1773 __kmp_execute_task_from_queue(tq, loc, global_tid, thunk, in_parallel);
1777 KF_TRACE(100, (
"Before Deletion of Serialized TaskQ at %p on (%d):\n",
1778 queue, global_tid));
1779 KF_DUMP(100, __kmp_dump_task_queue(tq, queue, global_tid));
1783 for (thunk = queue->tq_free_thunks; thunk != NULL;
1784 thunk = thunk->th.th_next_free)
1786 KMP_DEBUG_ASSERT(i == queue->tq_nslots + 1);
1790 (
"Serialized TaskQ at %p deleted on (%d).\n", queue, global_tid));
1792 queue->tq_flags |= TQF_DEALLOCATED;
1793 __kmp_free_taskq(tq, queue, in_parallel, global_tid);
1796 KE_TRACE(10, (
"__kmpc_end_taskq return (%d)\n", global_tid));
1802 kmp_int32 __kmpc_task(
ident_t *loc, kmp_int32 global_tid, kmpc_thunk_t *thunk) {
1804 kmpc_task_queue_t *queue;
1808 KE_TRACE(10, (
"__kmpc_task called (%d)\n", global_tid));
1810 KMP_DEBUG_ASSERT(!(thunk->th_flags &
1813 tq = &__kmp_threads[global_tid]->th.th_team->t.t_taskq;
1814 queue = thunk->th.th_shareds->sv_queue;
1815 in_parallel = (queue->tq_flags & TQF_PARALLEL_CONTEXT);
1817 if (in_parallel && (thunk->th_flags & TQF_IS_ORDERED))
1818 thunk->th_tasknum = ++queue->tq_tasknum_queuing;
1824 if (!in_parallel && queue->tq_nfull > 0) {
1825 kmpc_thunk_t *prev_thunk;
1827 KMP_DEBUG_ASSERT(queue->tq_nfull == 1);
1829 prev_thunk = __kmp_dequeue_task(global_tid, queue, in_parallel);
1831 KF_TRACE(50, (
"T#%d found thunk: %p in serial queue: %p\n", global_tid,
1832 prev_thunk, queue));
1834 __kmp_execute_task_from_queue(tq, loc, global_tid, prev_thunk, in_parallel);
1843 KF_TRACE(100, (
"After enqueueing this Task on (%d):\n", global_tid));
1844 KF_DUMP(100, __kmp_dump_thunk(tq, thunk, global_tid));
1846 ret = __kmp_enqueue_task(tq, global_tid, queue, thunk, in_parallel);
1848 KF_TRACE(100, (
"Task Queue looks like this on (%d):\n", global_tid));
1849 KF_DUMP(100, __kmp_dump_task_queue(tq, queue, global_tid));
1851 KE_TRACE(10, (
"__kmpc_task return (%d)\n", global_tid));
1859 void __kmpc_taskq_task(
ident_t *loc, kmp_int32 global_tid, kmpc_thunk_t *thunk,
1861 kmpc_task_queue_t *queue;
1862 kmp_taskq_t *tq = &__kmp_threads[global_tid]->th.th_team->t.t_taskq;
1863 int tid = __kmp_tid_from_gtid(global_tid);
1865 KE_TRACE(10, (
"__kmpc_taskq_task called (%d)\n", global_tid));
1866 KF_TRACE(100, (
"TaskQ Task argument thunk on (%d):\n", global_tid));
1867 KF_DUMP(100, __kmp_dump_thunk(tq, thunk, global_tid));
1869 queue = thunk->th.th_shareds->sv_queue;
1871 if (__kmp_env_consistency_check)
1872 __kmp_pop_workshare(global_tid, ct_taskq, loc);
1875 KMP_DEBUG_ASSERT(thunk->th_flags & TQF_TASKQ_TASK);
1878 KMP_DEBUG_ASSERT(queue->tq_taskq_slot == NULL);
1881 tq->tq_curr_thunk[tid] = thunk->th_encl_thunk;
1882 thunk->th_encl_thunk = NULL;
1884 KF_DUMP(200, __kmp_dump_thunk_stack(tq->tq_curr_thunk[tid], global_tid));
1886 thunk->th_status = status;
1895 queue->tq_taskq_slot = thunk;
1897 KE_TRACE(10, (
"__kmpc_taskq_task return (%d)\n", global_tid));
1902 void __kmpc_end_taskq_task(
ident_t *loc, kmp_int32 global_tid,
1903 kmpc_thunk_t *thunk) {
1905 kmpc_task_queue_t *queue;
1909 KE_TRACE(10, (
"__kmpc_end_taskq_task called (%d)\n", global_tid));
1911 tq = &__kmp_threads[global_tid]->th.th_team->t.t_taskq;
1912 queue = thunk->th.th_shareds->sv_queue;
1913 in_parallel = (queue->tq_flags & TQF_PARALLEL_CONTEXT);
1914 tid = __kmp_tid_from_gtid(global_tid);
1916 if (__kmp_env_consistency_check)
1917 __kmp_pop_workshare(global_tid, ct_taskq, loc);
1920 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1921 KMP_TEST_THEN_OR32(RCAST(
volatile kmp_uint32 *, &queue->tq_flags),
1922 TQF_ALL_TASKS_QUEUED);
1925 __kmp_acquire_lock(&queue->tq_queue_lck, global_tid);
1931 queue->tq_flags |= TQF_ALL_TASKS_QUEUED;
1932 __kmp_release_lock(&queue->tq_queue_lck, global_tid);
1937 if (thunk->th_flags & TQF_IS_LASTPRIVATE) {
1949 queue->tq_flags |= TQF_IS_LAST_TASK;
1951 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1952 KMP_TEST_THEN_OR32(RCAST(
volatile kmp_uint32 *, &queue->tq_flags),
1956 __kmp_acquire_lock(&queue->tq_queue_lck, global_tid);
1962 queue->tq_flags |= TQF_IS_LAST_TASK;
1963 __kmp_release_lock(&queue->tq_queue_lck, global_tid);
1974 tq->tq_curr_thunk[tid] = thunk->th_encl_thunk;
1975 thunk->th_encl_thunk = NULL;
1977 KF_DUMP(200, __kmp_dump_thunk_stack(tq->tq_curr_thunk[tid], global_tid));
1980 KE_TRACE(10, (
"__kmpc_end_taskq_task return (%d)\n", global_tid));
1986 kmpc_thunk_t *__kmpc_task_buffer(
ident_t *loc, kmp_int32 global_tid,
1987 kmpc_thunk_t *taskq_thunk, kmpc_task_t task) {
1989 kmpc_task_queue_t *queue;
1990 kmpc_thunk_t *new_thunk;
1993 KE_TRACE(10, (
"__kmpc_task_buffer called (%d)\n", global_tid));
1996 taskq_thunk->th_flags &
1999 tq = &__kmp_threads[global_tid]->th.th_team->t.t_taskq;
2000 queue = taskq_thunk->th.th_shareds->sv_queue;
2001 in_parallel = (queue->tq_flags & TQF_PARALLEL_CONTEXT);
2009 new_thunk = __kmp_alloc_thunk(queue, in_parallel, global_tid);
2010 new_thunk->th.th_shareds =
2011 CCAST(kmpc_shared_vars_t *, queue->tq_shareds[0].ai_data);
2012 new_thunk->th_encl_thunk = NULL;
2013 new_thunk->th_task = task;
2016 new_thunk->th_flags = queue->tq_flags & TQF_INTERFACE_FLAGS;
2018 new_thunk->th_status = 0;
2020 KMP_DEBUG_ASSERT(!(new_thunk->th_flags & TQF_TASKQ_TASK));
2022 KF_TRACE(100, (
"Creating Regular Task on (%d):\n", global_tid));
2023 KF_DUMP(100, __kmp_dump_thunk(tq, new_thunk, global_tid));
2025 KE_TRACE(10, (
"__kmpc_task_buffer return (%d)\n", global_tid));
KMP_EXPORT void __kmpc_end_barrier_master(ident_t *, kmp_int32 global_tid)