LLVM OpenMP* Runtime Library
kmp_tasking.cpp
1 /*
2  * kmp_tasking.cpp -- OpenMP 3.0 tasking support.
3  */
4 
5 //===----------------------------------------------------------------------===//
6 //
7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8 // See https://llvm.org/LICENSE.txt for license information.
9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "kmp.h"
14 #include "kmp_i18n.h"
15 #include "kmp_itt.h"
16 #include "kmp_stats.h"
17 #include "kmp_wait_release.h"
18 #include "kmp_taskdeps.h"
19 
20 #if OMPT_SUPPORT
21 #include "ompt-specific.h"
22 #endif
23 
24 #if ENABLE_LIBOMPTARGET
25 // Declaration of synchronization function from libomptarget.
26 extern "C" void __tgt_target_nowait_query(void **) KMP_WEAK_ATTRIBUTE_INTERNAL;
27 #endif
28 
29 /* forward declaration */
30 static void __kmp_enable_tasking(kmp_task_team_t *task_team,
31  kmp_info_t *this_thr);
32 static void __kmp_alloc_task_deque(kmp_info_t *thread,
33  kmp_thread_data_t *thread_data);
34 static int __kmp_realloc_task_threads_data(kmp_info_t *thread,
35  kmp_task_team_t *task_team);
36 static void __kmp_bottom_half_finish_proxy(kmp_int32 gtid, kmp_task_t *ptask);
37 
38 #ifdef BUILD_TIED_TASK_STACK
39 
40 // __kmp_trace_task_stack: print the tied tasks from the task stack in order
41 // from top do bottom
42 //
43 // gtid: global thread identifier for thread containing stack
44 // thread_data: thread data for task team thread containing stack
45 // threshold: value above which the trace statement triggers
46 // location: string identifying call site of this function (for trace)
47 static void __kmp_trace_task_stack(kmp_int32 gtid,
48  kmp_thread_data_t *thread_data,
49  int threshold, char *location) {
50  kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks;
51  kmp_taskdata_t **stack_top = task_stack->ts_top;
52  kmp_int32 entries = task_stack->ts_entries;
53  kmp_taskdata_t *tied_task;
54 
55  KA_TRACE(
56  threshold,
57  ("__kmp_trace_task_stack(start): location = %s, gtid = %d, entries = %d, "
58  "first_block = %p, stack_top = %p \n",
59  location, gtid, entries, task_stack->ts_first_block, stack_top));
60 
61  KMP_DEBUG_ASSERT(stack_top != NULL);
62  KMP_DEBUG_ASSERT(entries > 0);
63 
64  while (entries != 0) {
65  KMP_DEBUG_ASSERT(stack_top != &task_stack->ts_first_block.sb_block[0]);
66  // fix up ts_top if we need to pop from previous block
67  if (entries & TASK_STACK_INDEX_MASK == 0) {
68  kmp_stack_block_t *stack_block = (kmp_stack_block_t *)(stack_top);
69 
70  stack_block = stack_block->sb_prev;
71  stack_top = &stack_block->sb_block[TASK_STACK_BLOCK_SIZE];
72  }
73 
74  // finish bookkeeping
75  stack_top--;
76  entries--;
77 
78  tied_task = *stack_top;
79 
80  KMP_DEBUG_ASSERT(tied_task != NULL);
81  KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED);
82 
83  KA_TRACE(threshold,
84  ("__kmp_trace_task_stack(%s): gtid=%d, entry=%d, "
85  "stack_top=%p, tied_task=%p\n",
86  location, gtid, entries, stack_top, tied_task));
87  }
88  KMP_DEBUG_ASSERT(stack_top == &task_stack->ts_first_block.sb_block[0]);
89 
90  KA_TRACE(threshold,
91  ("__kmp_trace_task_stack(exit): location = %s, gtid = %d\n",
92  location, gtid));
93 }
94 
95 // __kmp_init_task_stack: initialize the task stack for the first time
96 // after a thread_data structure is created.
97 // It should not be necessary to do this again (assuming the stack works).
98 //
99 // gtid: global thread identifier of calling thread
100 // thread_data: thread data for task team thread containing stack
101 static void __kmp_init_task_stack(kmp_int32 gtid,
102  kmp_thread_data_t *thread_data) {
103  kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks;
104  kmp_stack_block_t *first_block;
105 
106  // set up the first block of the stack
107  first_block = &task_stack->ts_first_block;
108  task_stack->ts_top = (kmp_taskdata_t **)first_block;
109  memset((void *)first_block, '\0',
110  TASK_STACK_BLOCK_SIZE * sizeof(kmp_taskdata_t *));
111 
112  // initialize the stack to be empty
113  task_stack->ts_entries = TASK_STACK_EMPTY;
114  first_block->sb_next = NULL;
115  first_block->sb_prev = NULL;
116 }
117 
118 // __kmp_free_task_stack: free the task stack when thread_data is destroyed.
119 //
120 // gtid: global thread identifier for calling thread
121 // thread_data: thread info for thread containing stack
122 static void __kmp_free_task_stack(kmp_int32 gtid,
123  kmp_thread_data_t *thread_data) {
124  kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks;
125  kmp_stack_block_t *stack_block = &task_stack->ts_first_block;
126 
127  KMP_DEBUG_ASSERT(task_stack->ts_entries == TASK_STACK_EMPTY);
128  // free from the second block of the stack
129  while (stack_block != NULL) {
130  kmp_stack_block_t *next_block = (stack_block) ? stack_block->sb_next : NULL;
131 
132  stack_block->sb_next = NULL;
133  stack_block->sb_prev = NULL;
134  if (stack_block != &task_stack->ts_first_block) {
135  __kmp_thread_free(thread,
136  stack_block); // free the block, if not the first
137  }
138  stack_block = next_block;
139  }
140  // initialize the stack to be empty
141  task_stack->ts_entries = 0;
142  task_stack->ts_top = NULL;
143 }
144 
145 // __kmp_push_task_stack: Push the tied task onto the task stack.
146 // Grow the stack if necessary by allocating another block.
147 //
148 // gtid: global thread identifier for calling thread
149 // thread: thread info for thread containing stack
150 // tied_task: the task to push on the stack
151 static void __kmp_push_task_stack(kmp_int32 gtid, kmp_info_t *thread,
152  kmp_taskdata_t *tied_task) {
153  // GEH - need to consider what to do if tt_threads_data not allocated yet
154  kmp_thread_data_t *thread_data =
155  &thread->th.th_task_team->tt.tt_threads_data[__kmp_tid_from_gtid(gtid)];
156  kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks;
157 
158  if (tied_task->td_flags.team_serial || tied_task->td_flags.tasking_ser) {
159  return; // Don't push anything on stack if team or team tasks are serialized
160  }
161 
162  KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED);
163  KMP_DEBUG_ASSERT(task_stack->ts_top != NULL);
164 
165  KA_TRACE(20,
166  ("__kmp_push_task_stack(enter): GTID: %d; THREAD: %p; TASK: %p\n",
167  gtid, thread, tied_task));
168  // Store entry
169  *(task_stack->ts_top) = tied_task;
170 
171  // Do bookkeeping for next push
172  task_stack->ts_top++;
173  task_stack->ts_entries++;
174 
175  if (task_stack->ts_entries & TASK_STACK_INDEX_MASK == 0) {
176  // Find beginning of this task block
177  kmp_stack_block_t *stack_block =
178  (kmp_stack_block_t *)(task_stack->ts_top - TASK_STACK_BLOCK_SIZE);
179 
180  // Check if we already have a block
181  if (stack_block->sb_next !=
182  NULL) { // reset ts_top to beginning of next block
183  task_stack->ts_top = &stack_block->sb_next->sb_block[0];
184  } else { // Alloc new block and link it up
185  kmp_stack_block_t *new_block = (kmp_stack_block_t *)__kmp_thread_calloc(
186  thread, sizeof(kmp_stack_block_t));
187 
188  task_stack->ts_top = &new_block->sb_block[0];
189  stack_block->sb_next = new_block;
190  new_block->sb_prev = stack_block;
191  new_block->sb_next = NULL;
192 
193  KA_TRACE(
194  30,
195  ("__kmp_push_task_stack(): GTID: %d; TASK: %p; Alloc new block: %p\n",
196  gtid, tied_task, new_block));
197  }
198  }
199  KA_TRACE(20, ("__kmp_push_task_stack(exit): GTID: %d; TASK: %p\n", gtid,
200  tied_task));
201 }
202 
203 // __kmp_pop_task_stack: Pop the tied task from the task stack. Don't return
204 // the task, just check to make sure it matches the ending task passed in.
205 //
206 // gtid: global thread identifier for the calling thread
207 // thread: thread info structure containing stack
208 // tied_task: the task popped off the stack
209 // ending_task: the task that is ending (should match popped task)
210 static void __kmp_pop_task_stack(kmp_int32 gtid, kmp_info_t *thread,
211  kmp_taskdata_t *ending_task) {
212  // GEH - need to consider what to do if tt_threads_data not allocated yet
213  kmp_thread_data_t *thread_data =
214  &thread->th.th_task_team->tt_threads_data[__kmp_tid_from_gtid(gtid)];
215  kmp_task_stack_t *task_stack = &thread_data->td.td_susp_tied_tasks;
216  kmp_taskdata_t *tied_task;
217 
218  if (ending_task->td_flags.team_serial || ending_task->td_flags.tasking_ser) {
219  // Don't pop anything from stack if team or team tasks are serialized
220  return;
221  }
222 
223  KMP_DEBUG_ASSERT(task_stack->ts_top != NULL);
224  KMP_DEBUG_ASSERT(task_stack->ts_entries > 0);
225 
226  KA_TRACE(20, ("__kmp_pop_task_stack(enter): GTID: %d; THREAD: %p\n", gtid,
227  thread));
228 
229  // fix up ts_top if we need to pop from previous block
230  if (task_stack->ts_entries & TASK_STACK_INDEX_MASK == 0) {
231  kmp_stack_block_t *stack_block = (kmp_stack_block_t *)(task_stack->ts_top);
232 
233  stack_block = stack_block->sb_prev;
234  task_stack->ts_top = &stack_block->sb_block[TASK_STACK_BLOCK_SIZE];
235  }
236 
237  // finish bookkeeping
238  task_stack->ts_top--;
239  task_stack->ts_entries--;
240 
241  tied_task = *(task_stack->ts_top);
242 
243  KMP_DEBUG_ASSERT(tied_task != NULL);
244  KMP_DEBUG_ASSERT(tied_task->td_flags.tasktype == TASK_TIED);
245  KMP_DEBUG_ASSERT(tied_task == ending_task); // If we built the stack correctly
246 
247  KA_TRACE(20, ("__kmp_pop_task_stack(exit): GTID: %d; TASK: %p\n", gtid,
248  tied_task));
249  return;
250 }
251 #endif /* BUILD_TIED_TASK_STACK */
252 
253 // returns 1 if new task is allowed to execute, 0 otherwise
254 // checks Task Scheduling constraint (if requested) and
255 // mutexinoutset dependencies if any
256 static bool __kmp_task_is_allowed(int gtid, const kmp_int32 is_constrained,
257  const kmp_taskdata_t *tasknew,
258  const kmp_taskdata_t *taskcurr) {
259  if (is_constrained && (tasknew->td_flags.tiedness == TASK_TIED)) {
260  // Check if the candidate obeys the Task Scheduling Constraints (TSC)
261  // only descendant of all deferred tied tasks can be scheduled, checking
262  // the last one is enough, as it in turn is the descendant of all others
263  kmp_taskdata_t *current = taskcurr->td_last_tied;
264  KMP_DEBUG_ASSERT(current != NULL);
265  // check if the task is not suspended on barrier
266  if (current->td_flags.tasktype == TASK_EXPLICIT ||
267  current->td_taskwait_thread > 0) { // <= 0 on barrier
268  kmp_int32 level = current->td_level;
269  kmp_taskdata_t *parent = tasknew->td_parent;
270  while (parent != current && parent->td_level > level) {
271  // check generation up to the level of the current task
272  parent = parent->td_parent;
273  KMP_DEBUG_ASSERT(parent != NULL);
274  }
275  if (parent != current)
276  return false;
277  }
278  }
279  // Check mutexinoutset dependencies, acquire locks
280  kmp_depnode_t *node = tasknew->td_depnode;
281  if (UNLIKELY(node && (node->dn.mtx_num_locks > 0))) {
282  for (int i = 0; i < node->dn.mtx_num_locks; ++i) {
283  KMP_DEBUG_ASSERT(node->dn.mtx_locks[i] != NULL);
284  if (__kmp_test_lock(node->dn.mtx_locks[i], gtid))
285  continue;
286  // could not get the lock, release previous locks
287  for (int j = i - 1; j >= 0; --j)
288  __kmp_release_lock(node->dn.mtx_locks[j], gtid);
289  return false;
290  }
291  // negative num_locks means all locks acquired successfully
292  node->dn.mtx_num_locks = -node->dn.mtx_num_locks;
293  }
294  return true;
295 }
296 
297 // __kmp_realloc_task_deque:
298 // Re-allocates a task deque for a particular thread, copies the content from
299 // the old deque and adjusts the necessary data structures relating to the
300 // deque. This operation must be done with the deque_lock being held
301 static void __kmp_realloc_task_deque(kmp_info_t *thread,
302  kmp_thread_data_t *thread_data) {
303  kmp_int32 size = TASK_DEQUE_SIZE(thread_data->td);
304  KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) == size);
305  kmp_int32 new_size = 2 * size;
306 
307  KE_TRACE(10, ("__kmp_realloc_task_deque: T#%d reallocating deque[from %d to "
308  "%d] for thread_data %p\n",
309  __kmp_gtid_from_thread(thread), size, new_size, thread_data));
310 
311  kmp_taskdata_t **new_deque =
312  (kmp_taskdata_t **)__kmp_allocate(new_size * sizeof(kmp_taskdata_t *));
313 
314  int i, j;
315  for (i = thread_data->td.td_deque_head, j = 0; j < size;
316  i = (i + 1) & TASK_DEQUE_MASK(thread_data->td), j++)
317  new_deque[j] = thread_data->td.td_deque[i];
318 
319  __kmp_free(thread_data->td.td_deque);
320 
321  thread_data->td.td_deque_head = 0;
322  thread_data->td.td_deque_tail = size;
323  thread_data->td.td_deque = new_deque;
324  thread_data->td.td_deque_size = new_size;
325 }
326 
327 static kmp_task_pri_t *__kmp_alloc_task_pri_list() {
328  kmp_task_pri_t *l = (kmp_task_pri_t *)__kmp_allocate(sizeof(kmp_task_pri_t));
329  kmp_thread_data_t *thread_data = &l->td;
330  __kmp_init_bootstrap_lock(&thread_data->td.td_deque_lock);
331  thread_data->td.td_deque_last_stolen = -1;
332  KE_TRACE(20, ("__kmp_alloc_task_pri_list: T#%d allocating deque[%d] "
333  "for thread_data %p\n",
334  __kmp_get_gtid(), INITIAL_TASK_DEQUE_SIZE, thread_data));
335  thread_data->td.td_deque = (kmp_taskdata_t **)__kmp_allocate(
336  INITIAL_TASK_DEQUE_SIZE * sizeof(kmp_taskdata_t *));
337  thread_data->td.td_deque_size = INITIAL_TASK_DEQUE_SIZE;
338  return l;
339 }
340 
341 // The function finds the deque of priority tasks with given priority, or
342 // allocates a new deque and put it into sorted (high -> low) list of deques.
343 // Deques of non-default priority tasks are shared between all threads in team,
344 // as opposed to per-thread deques of tasks with default priority.
345 // The function is called under the lock task_team->tt.tt_task_pri_lock.
346 static kmp_thread_data_t *
347 __kmp_get_priority_deque_data(kmp_task_team_t *task_team, kmp_int32 pri) {
348  kmp_thread_data_t *thread_data;
349  kmp_task_pri_t *lst = task_team->tt.tt_task_pri_list;
350  if (lst->priority == pri) {
351  // Found queue of tasks with given priority.
352  thread_data = &lst->td;
353  } else if (lst->priority < pri) {
354  // All current priority queues contain tasks with lower priority.
355  // Allocate new one for given priority tasks.
356  kmp_task_pri_t *list = __kmp_alloc_task_pri_list();
357  thread_data = &list->td;
358  list->priority = pri;
359  list->next = lst;
360  task_team->tt.tt_task_pri_list = list;
361  } else { // task_team->tt.tt_task_pri_list->priority > pri
362  kmp_task_pri_t *next_queue = lst->next;
363  while (next_queue && next_queue->priority > pri) {
364  lst = next_queue;
365  next_queue = lst->next;
366  }
367  // lst->priority > pri && (next == NULL || pri >= next->priority)
368  if (next_queue == NULL) {
369  // No queue with pri priority, need to allocate new one.
370  kmp_task_pri_t *list = __kmp_alloc_task_pri_list();
371  thread_data = &list->td;
372  list->priority = pri;
373  list->next = NULL;
374  lst->next = list;
375  } else if (next_queue->priority == pri) {
376  // Found queue of tasks with given priority.
377  thread_data = &next_queue->td;
378  } else { // lst->priority > pri > next->priority
379  // insert newly allocated between existed queues
380  kmp_task_pri_t *list = __kmp_alloc_task_pri_list();
381  thread_data = &list->td;
382  list->priority = pri;
383  list->next = next_queue;
384  lst->next = list;
385  }
386  }
387  return thread_data;
388 }
389 
390 // __kmp_push_priority_task: Add a task to the team's priority task deque
391 static kmp_int32 __kmp_push_priority_task(kmp_int32 gtid, kmp_info_t *thread,
392  kmp_taskdata_t *taskdata,
393  kmp_task_team_t *task_team,
394  kmp_int32 pri) {
395  kmp_thread_data_t *thread_data = NULL;
396  KA_TRACE(20,
397  ("__kmp_push_priority_task: T#%d trying to push task %p, pri %d.\n",
398  gtid, taskdata, pri));
399 
400  // Find task queue specific to priority value
401  kmp_task_pri_t *lst = task_team->tt.tt_task_pri_list;
402  if (UNLIKELY(lst == NULL)) {
403  __kmp_acquire_bootstrap_lock(&task_team->tt.tt_task_pri_lock);
404  if (task_team->tt.tt_task_pri_list == NULL) {
405  // List of queues is still empty, allocate one.
406  kmp_task_pri_t *list = __kmp_alloc_task_pri_list();
407  thread_data = &list->td;
408  list->priority = pri;
409  list->next = NULL;
410  task_team->tt.tt_task_pri_list = list;
411  } else {
412  // Other thread initialized a queue. Check if it fits and get thread_data.
413  thread_data = __kmp_get_priority_deque_data(task_team, pri);
414  }
415  __kmp_release_bootstrap_lock(&task_team->tt.tt_task_pri_lock);
416  } else {
417  if (lst->priority == pri) {
418  // Found queue of tasks with given priority.
419  thread_data = &lst->td;
420  } else {
421  __kmp_acquire_bootstrap_lock(&task_team->tt.tt_task_pri_lock);
422  thread_data = __kmp_get_priority_deque_data(task_team, pri);
423  __kmp_release_bootstrap_lock(&task_team->tt.tt_task_pri_lock);
424  }
425  }
426  KMP_DEBUG_ASSERT(thread_data);
427 
428  __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
429  // Check if deque is full
430  if (TCR_4(thread_data->td.td_deque_ntasks) >=
431  TASK_DEQUE_SIZE(thread_data->td)) {
432  if (__kmp_enable_task_throttling &&
433  __kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata,
434  thread->th.th_current_task)) {
435  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
436  KA_TRACE(20, ("__kmp_push_priority_task: T#%d deque is full; returning "
437  "TASK_NOT_PUSHED for task %p\n",
438  gtid, taskdata));
439  return TASK_NOT_PUSHED;
440  } else {
441  // expand deque to push the task which is not allowed to execute
442  __kmp_realloc_task_deque(thread, thread_data);
443  }
444  }
445  KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) <
446  TASK_DEQUE_SIZE(thread_data->td));
447  // Push taskdata.
448  thread_data->td.td_deque[thread_data->td.td_deque_tail] = taskdata;
449  // Wrap index.
450  thread_data->td.td_deque_tail =
451  (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td);
452  TCW_4(thread_data->td.td_deque_ntasks,
453  TCR_4(thread_data->td.td_deque_ntasks) + 1); // Adjust task count
454  KMP_FSYNC_RELEASING(thread->th.th_current_task); // releasing self
455  KMP_FSYNC_RELEASING(taskdata); // releasing child
456  KA_TRACE(20, ("__kmp_push_priority_task: T#%d returning "
457  "TASK_SUCCESSFULLY_PUSHED: task=%p ntasks=%d head=%u tail=%u\n",
458  gtid, taskdata, thread_data->td.td_deque_ntasks,
459  thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
460  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
461  task_team->tt.tt_num_task_pri++; // atomic inc
462  return TASK_SUCCESSFULLY_PUSHED;
463 }
464 
465 // __kmp_push_task: Add a task to the thread's deque
466 static kmp_int32 __kmp_push_task(kmp_int32 gtid, kmp_task_t *task) {
467  kmp_info_t *thread = __kmp_threads[gtid];
468  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
469 
470  // If we encounter a hidden helper task, and the current thread is not a
471  // hidden helper thread, we have to give the task to any hidden helper thread
472  // starting from its shadow one.
473  if (UNLIKELY(taskdata->td_flags.hidden_helper &&
474  !KMP_HIDDEN_HELPER_THREAD(gtid))) {
475  kmp_int32 shadow_gtid = KMP_GTID_TO_SHADOW_GTID(gtid);
476  __kmpc_give_task(task, __kmp_tid_from_gtid(shadow_gtid));
477  // Signal the hidden helper threads.
478  __kmp_hidden_helper_worker_thread_signal();
479  return TASK_SUCCESSFULLY_PUSHED;
480  }
481 
482  kmp_task_team_t *task_team = thread->th.th_task_team;
483  kmp_int32 tid = __kmp_tid_from_gtid(gtid);
484  kmp_thread_data_t *thread_data;
485 
486  KA_TRACE(20,
487  ("__kmp_push_task: T#%d trying to push task %p.\n", gtid, taskdata));
488 
489  if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) {
490  // untied task needs to increment counter so that the task structure is not
491  // freed prematurely
492  kmp_int32 counter = 1 + KMP_ATOMIC_INC(&taskdata->td_untied_count);
493  KMP_DEBUG_USE_VAR(counter);
494  KA_TRACE(
495  20,
496  ("__kmp_push_task: T#%d untied_count (%d) incremented for task %p\n",
497  gtid, counter, taskdata));
498  }
499 
500  // The first check avoids building task_team thread data if serialized
501  if (UNLIKELY(taskdata->td_flags.task_serial)) {
502  KA_TRACE(20, ("__kmp_push_task: T#%d team serialized; returning "
503  "TASK_NOT_PUSHED for task %p\n",
504  gtid, taskdata));
505  return TASK_NOT_PUSHED;
506  }
507 
508  // Now that serialized tasks have returned, we can assume that we are not in
509  // immediate exec mode
510  KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
511  if (UNLIKELY(!KMP_TASKING_ENABLED(task_team))) {
512  __kmp_enable_tasking(task_team, thread);
513  }
514  KMP_DEBUG_ASSERT(TCR_4(task_team->tt.tt_found_tasks) == TRUE);
515  KMP_DEBUG_ASSERT(TCR_PTR(task_team->tt.tt_threads_data) != NULL);
516 
517  if (taskdata->td_flags.priority_specified && task->data2.priority > 0 &&
518  __kmp_max_task_priority > 0) {
519  int pri = KMP_MIN(task->data2.priority, __kmp_max_task_priority);
520  return __kmp_push_priority_task(gtid, thread, taskdata, task_team, pri);
521  }
522 
523  // Find tasking deque specific to encountering thread
524  thread_data = &task_team->tt.tt_threads_data[tid];
525 
526  // No lock needed since only owner can allocate. If the task is hidden_helper,
527  // we don't need it either because we have initialized the dequeue for hidden
528  // helper thread data.
529  if (UNLIKELY(thread_data->td.td_deque == NULL)) {
530  __kmp_alloc_task_deque(thread, thread_data);
531  }
532 
533  int locked = 0;
534  // Check if deque is full
535  if (TCR_4(thread_data->td.td_deque_ntasks) >=
536  TASK_DEQUE_SIZE(thread_data->td)) {
537  if (__kmp_enable_task_throttling &&
538  __kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata,
539  thread->th.th_current_task)) {
540  KA_TRACE(20, ("__kmp_push_task: T#%d deque is full; returning "
541  "TASK_NOT_PUSHED for task %p\n",
542  gtid, taskdata));
543  return TASK_NOT_PUSHED;
544  } else {
545  __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
546  locked = 1;
547  if (TCR_4(thread_data->td.td_deque_ntasks) >=
548  TASK_DEQUE_SIZE(thread_data->td)) {
549  // expand deque to push the task which is not allowed to execute
550  __kmp_realloc_task_deque(thread, thread_data);
551  }
552  }
553  }
554  // Lock the deque for the task push operation
555  if (!locked) {
556  __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
557  // Need to recheck as we can get a proxy task from thread outside of OpenMP
558  if (TCR_4(thread_data->td.td_deque_ntasks) >=
559  TASK_DEQUE_SIZE(thread_data->td)) {
560  if (__kmp_enable_task_throttling &&
561  __kmp_task_is_allowed(gtid, __kmp_task_stealing_constraint, taskdata,
562  thread->th.th_current_task)) {
563  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
564  KA_TRACE(20, ("__kmp_push_task: T#%d deque is full on 2nd check; "
565  "returning TASK_NOT_PUSHED for task %p\n",
566  gtid, taskdata));
567  return TASK_NOT_PUSHED;
568  } else {
569  // expand deque to push the task which is not allowed to execute
570  __kmp_realloc_task_deque(thread, thread_data);
571  }
572  }
573  }
574  // Must have room since no thread can add tasks but calling thread
575  KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) <
576  TASK_DEQUE_SIZE(thread_data->td));
577 
578  thread_data->td.td_deque[thread_data->td.td_deque_tail] =
579  taskdata; // Push taskdata
580  // Wrap index.
581  thread_data->td.td_deque_tail =
582  (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td);
583  TCW_4(thread_data->td.td_deque_ntasks,
584  TCR_4(thread_data->td.td_deque_ntasks) + 1); // Adjust task count
585  KMP_FSYNC_RELEASING(thread->th.th_current_task); // releasing self
586  KMP_FSYNC_RELEASING(taskdata); // releasing child
587  KA_TRACE(20, ("__kmp_push_task: T#%d returning TASK_SUCCESSFULLY_PUSHED: "
588  "task=%p ntasks=%d head=%u tail=%u\n",
589  gtid, taskdata, thread_data->td.td_deque_ntasks,
590  thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
591 
592  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
593 
594  return TASK_SUCCESSFULLY_PUSHED;
595 }
596 
597 // __kmp_pop_current_task_from_thread: set up current task from called thread
598 // when team ends
599 //
600 // this_thr: thread structure to set current_task in.
601 void __kmp_pop_current_task_from_thread(kmp_info_t *this_thr) {
602  KF_TRACE(10, ("__kmp_pop_current_task_from_thread(enter): T#%d "
603  "this_thread=%p, curtask=%p, "
604  "curtask_parent=%p\n",
605  0, this_thr, this_thr->th.th_current_task,
606  this_thr->th.th_current_task->td_parent));
607 
608  this_thr->th.th_current_task = this_thr->th.th_current_task->td_parent;
609 
610  KF_TRACE(10, ("__kmp_pop_current_task_from_thread(exit): T#%d "
611  "this_thread=%p, curtask=%p, "
612  "curtask_parent=%p\n",
613  0, this_thr, this_thr->th.th_current_task,
614  this_thr->th.th_current_task->td_parent));
615 }
616 
617 // __kmp_push_current_task_to_thread: set up current task in called thread for a
618 // new team
619 //
620 // this_thr: thread structure to set up
621 // team: team for implicit task data
622 // tid: thread within team to set up
623 void __kmp_push_current_task_to_thread(kmp_info_t *this_thr, kmp_team_t *team,
624  int tid) {
625  // current task of the thread is a parent of the new just created implicit
626  // tasks of new team
627  KF_TRACE(10, ("__kmp_push_current_task_to_thread(enter): T#%d this_thread=%p "
628  "curtask=%p "
629  "parent_task=%p\n",
630  tid, this_thr, this_thr->th.th_current_task,
631  team->t.t_implicit_task_taskdata[tid].td_parent));
632 
633  KMP_DEBUG_ASSERT(this_thr != NULL);
634 
635  if (tid == 0) {
636  if (this_thr->th.th_current_task != &team->t.t_implicit_task_taskdata[0]) {
637  team->t.t_implicit_task_taskdata[0].td_parent =
638  this_thr->th.th_current_task;
639  this_thr->th.th_current_task = &team->t.t_implicit_task_taskdata[0];
640  }
641  } else {
642  team->t.t_implicit_task_taskdata[tid].td_parent =
643  team->t.t_implicit_task_taskdata[0].td_parent;
644  this_thr->th.th_current_task = &team->t.t_implicit_task_taskdata[tid];
645  }
646 
647  KF_TRACE(10, ("__kmp_push_current_task_to_thread(exit): T#%d this_thread=%p "
648  "curtask=%p "
649  "parent_task=%p\n",
650  tid, this_thr, this_thr->th.th_current_task,
651  team->t.t_implicit_task_taskdata[tid].td_parent));
652 }
653 
654 // __kmp_task_start: bookkeeping for a task starting execution
655 //
656 // GTID: global thread id of calling thread
657 // task: task starting execution
658 // current_task: task suspending
659 static void __kmp_task_start(kmp_int32 gtid, kmp_task_t *task,
660  kmp_taskdata_t *current_task) {
661  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
662  kmp_info_t *thread = __kmp_threads[gtid];
663 
664  KA_TRACE(10,
665  ("__kmp_task_start(enter): T#%d starting task %p: current_task=%p\n",
666  gtid, taskdata, current_task));
667 
668  KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
669 
670  // mark currently executing task as suspended
671  // TODO: GEH - make sure root team implicit task is initialized properly.
672  // KMP_DEBUG_ASSERT( current_task -> td_flags.executing == 1 );
673  current_task->td_flags.executing = 0;
674 
675 // Add task to stack if tied
676 #ifdef BUILD_TIED_TASK_STACK
677  if (taskdata->td_flags.tiedness == TASK_TIED) {
678  __kmp_push_task_stack(gtid, thread, taskdata);
679  }
680 #endif /* BUILD_TIED_TASK_STACK */
681 
682  // mark starting task as executing and as current task
683  thread->th.th_current_task = taskdata;
684 
685  KMP_DEBUG_ASSERT(taskdata->td_flags.started == 0 ||
686  taskdata->td_flags.tiedness == TASK_UNTIED);
687  KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0 ||
688  taskdata->td_flags.tiedness == TASK_UNTIED);
689  taskdata->td_flags.started = 1;
690  taskdata->td_flags.executing = 1;
691  KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0);
692  KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0);
693 
694  // GEH TODO: shouldn't we pass some sort of location identifier here?
695  // APT: yes, we will pass location here.
696  // need to store current thread state (in a thread or taskdata structure)
697  // before setting work_state, otherwise wrong state is set after end of task
698 
699  KA_TRACE(10, ("__kmp_task_start(exit): T#%d task=%p\n", gtid, taskdata));
700 
701  return;
702 }
703 
704 #if OMPT_SUPPORT
705 //------------------------------------------------------------------------------
706 // __ompt_task_init:
707 // Initialize OMPT fields maintained by a task. This will only be called after
708 // ompt_start_tool, so we already know whether ompt is enabled or not.
709 
710 static inline void __ompt_task_init(kmp_taskdata_t *task, int tid) {
711  // The calls to __ompt_task_init already have the ompt_enabled condition.
712  task->ompt_task_info.task_data.value = 0;
713  task->ompt_task_info.frame.exit_frame = ompt_data_none;
714  task->ompt_task_info.frame.enter_frame = ompt_data_none;
715  task->ompt_task_info.frame.exit_frame_flags =
716  ompt_frame_runtime | ompt_frame_framepointer;
717  task->ompt_task_info.frame.enter_frame_flags =
718  ompt_frame_runtime | ompt_frame_framepointer;
719  task->ompt_task_info.dispatch_chunk.start = 0;
720  task->ompt_task_info.dispatch_chunk.iterations = 0;
721 }
722 
723 // __ompt_task_start:
724 // Build and trigger task-begin event
725 static inline void __ompt_task_start(kmp_task_t *task,
726  kmp_taskdata_t *current_task,
727  kmp_int32 gtid) {
728  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
729  ompt_task_status_t status = ompt_task_switch;
730  if (__kmp_threads[gtid]->th.ompt_thread_info.ompt_task_yielded) {
731  status = ompt_task_yield;
732  __kmp_threads[gtid]->th.ompt_thread_info.ompt_task_yielded = 0;
733  }
734  /* let OMPT know that we're about to run this task */
735  if (ompt_enabled.ompt_callback_task_schedule) {
736  ompt_callbacks.ompt_callback(ompt_callback_task_schedule)(
737  &(current_task->ompt_task_info.task_data), status,
738  &(taskdata->ompt_task_info.task_data));
739  }
740  taskdata->ompt_task_info.scheduling_parent = current_task;
741 }
742 
743 // __ompt_task_finish:
744 // Build and trigger final task-schedule event
745 static inline void __ompt_task_finish(kmp_task_t *task,
746  kmp_taskdata_t *resumed_task,
747  ompt_task_status_t status) {
748  if (ompt_enabled.ompt_callback_task_schedule) {
749  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
750  if (__kmp_omp_cancellation && taskdata->td_taskgroup &&
751  taskdata->td_taskgroup->cancel_request == cancel_taskgroup) {
752  status = ompt_task_cancel;
753  }
754 
755  /* let OMPT know that we're returning to the callee task */
756  ompt_callbacks.ompt_callback(ompt_callback_task_schedule)(
757  &(taskdata->ompt_task_info.task_data), status,
758  (resumed_task ? &(resumed_task->ompt_task_info.task_data) : NULL));
759  }
760 }
761 #endif
762 
763 template <bool ompt>
764 static void __kmpc_omp_task_begin_if0_template(ident_t *loc_ref, kmp_int32 gtid,
765  kmp_task_t *task,
766  void *frame_address,
767  void *return_address) {
768  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
769  kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task;
770 
771  KA_TRACE(10, ("__kmpc_omp_task_begin_if0(enter): T#%d loc=%p task=%p "
772  "current_task=%p\n",
773  gtid, loc_ref, taskdata, current_task));
774 
775  if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) {
776  // untied task needs to increment counter so that the task structure is not
777  // freed prematurely
778  kmp_int32 counter = 1 + KMP_ATOMIC_INC(&taskdata->td_untied_count);
779  KMP_DEBUG_USE_VAR(counter);
780  KA_TRACE(20, ("__kmpc_omp_task_begin_if0: T#%d untied_count (%d) "
781  "incremented for task %p\n",
782  gtid, counter, taskdata));
783  }
784 
785  taskdata->td_flags.task_serial =
786  1; // Execute this task immediately, not deferred.
787  __kmp_task_start(gtid, task, current_task);
788 
789 #if OMPT_SUPPORT
790  if (ompt) {
791  if (current_task->ompt_task_info.frame.enter_frame.ptr == NULL) {
792  current_task->ompt_task_info.frame.enter_frame.ptr =
793  taskdata->ompt_task_info.frame.exit_frame.ptr = frame_address;
794  current_task->ompt_task_info.frame.enter_frame_flags =
795  taskdata->ompt_task_info.frame.exit_frame_flags =
796  ompt_frame_application | ompt_frame_framepointer;
797  }
798  if (ompt_enabled.ompt_callback_task_create) {
799  ompt_task_info_t *parent_info = &(current_task->ompt_task_info);
800  ompt_callbacks.ompt_callback(ompt_callback_task_create)(
801  &(parent_info->task_data), &(parent_info->frame),
802  &(taskdata->ompt_task_info.task_data),
803  ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(taskdata), 0,
804  return_address);
805  }
806  __ompt_task_start(task, current_task, gtid);
807  }
808 #endif // OMPT_SUPPORT
809 
810  KA_TRACE(10, ("__kmpc_omp_task_begin_if0(exit): T#%d loc=%p task=%p,\n", gtid,
811  loc_ref, taskdata));
812 }
813 
814 #if OMPT_SUPPORT
815 OMPT_NOINLINE
816 static void __kmpc_omp_task_begin_if0_ompt(ident_t *loc_ref, kmp_int32 gtid,
817  kmp_task_t *task,
818  void *frame_address,
819  void *return_address) {
820  __kmpc_omp_task_begin_if0_template<true>(loc_ref, gtid, task, frame_address,
821  return_address);
822 }
823 #endif // OMPT_SUPPORT
824 
825 // __kmpc_omp_task_begin_if0: report that a given serialized task has started
826 // execution
827 //
828 // loc_ref: source location information; points to beginning of task block.
829 // gtid: global thread number.
830 // task: task thunk for the started task.
831 void __kmpc_omp_task_begin_if0(ident_t *loc_ref, kmp_int32 gtid,
832  kmp_task_t *task) {
833 #if OMPT_SUPPORT
834  if (UNLIKELY(ompt_enabled.enabled)) {
835  OMPT_STORE_RETURN_ADDRESS(gtid);
836  __kmpc_omp_task_begin_if0_ompt(loc_ref, gtid, task,
837  OMPT_GET_FRAME_ADDRESS(1),
838  OMPT_LOAD_RETURN_ADDRESS(gtid));
839  return;
840  }
841 #endif
842  __kmpc_omp_task_begin_if0_template<false>(loc_ref, gtid, task, NULL, NULL);
843 }
844 
845 #ifdef TASK_UNUSED
846 // __kmpc_omp_task_begin: report that a given task has started execution
847 // NEVER GENERATED BY COMPILER, DEPRECATED!!!
848 void __kmpc_omp_task_begin(ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *task) {
849  kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task;
850 
851  KA_TRACE(
852  10,
853  ("__kmpc_omp_task_begin(enter): T#%d loc=%p task=%p current_task=%p\n",
854  gtid, loc_ref, KMP_TASK_TO_TASKDATA(task), current_task));
855 
856  __kmp_task_start(gtid, task, current_task);
857 
858  KA_TRACE(10, ("__kmpc_omp_task_begin(exit): T#%d loc=%p task=%p,\n", gtid,
859  loc_ref, KMP_TASK_TO_TASKDATA(task)));
860  return;
861 }
862 #endif // TASK_UNUSED
863 
864 // __kmp_free_task: free the current task space and the space for shareds
865 //
866 // gtid: Global thread ID of calling thread
867 // taskdata: task to free
868 // thread: thread data structure of caller
869 static void __kmp_free_task(kmp_int32 gtid, kmp_taskdata_t *taskdata,
870  kmp_info_t *thread) {
871  KA_TRACE(30, ("__kmp_free_task: T#%d freeing data from task %p\n", gtid,
872  taskdata));
873 
874  // Check to make sure all flags and counters have the correct values
875  KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
876  KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 0);
877  KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 1);
878  KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0);
879  KMP_DEBUG_ASSERT(taskdata->td_allocated_child_tasks == 0 ||
880  taskdata->td_flags.task_serial == 1);
881  KMP_DEBUG_ASSERT(taskdata->td_incomplete_child_tasks == 0);
882  kmp_task_t *task = KMP_TASKDATA_TO_TASK(taskdata);
883  // Clear data to not be re-used later by mistake.
884  task->data1.destructors = NULL;
885  task->data2.priority = 0;
886 
887  taskdata->td_flags.freed = 1;
888 // deallocate the taskdata and shared variable blocks associated with this task
889 #if USE_FAST_MEMORY
890  __kmp_fast_free(thread, taskdata);
891 #else /* ! USE_FAST_MEMORY */
892  __kmp_thread_free(thread, taskdata);
893 #endif
894  KA_TRACE(20, ("__kmp_free_task: T#%d freed task %p\n", gtid, taskdata));
895 }
896 
897 // __kmp_free_task_and_ancestors: free the current task and ancestors without
898 // children
899 //
900 // gtid: Global thread ID of calling thread
901 // taskdata: task to free
902 // thread: thread data structure of caller
903 static void __kmp_free_task_and_ancestors(kmp_int32 gtid,
904  kmp_taskdata_t *taskdata,
905  kmp_info_t *thread) {
906  // Proxy tasks must always be allowed to free their parents
907  // because they can be run in background even in serial mode.
908  kmp_int32 team_serial =
909  (taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser) &&
910  !taskdata->td_flags.proxy;
911  KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
912 
913  kmp_int32 children = KMP_ATOMIC_DEC(&taskdata->td_allocated_child_tasks) - 1;
914  KMP_DEBUG_ASSERT(children >= 0);
915 
916  // Now, go up the ancestor tree to see if any ancestors can now be freed.
917  while (children == 0) {
918  kmp_taskdata_t *parent_taskdata = taskdata->td_parent;
919 
920  KA_TRACE(20, ("__kmp_free_task_and_ancestors(enter): T#%d task %p complete "
921  "and freeing itself\n",
922  gtid, taskdata));
923 
924  // --- Deallocate my ancestor task ---
925  __kmp_free_task(gtid, taskdata, thread);
926 
927  taskdata = parent_taskdata;
928 
929  if (team_serial)
930  return;
931  // Stop checking ancestors at implicit task instead of walking up ancestor
932  // tree to avoid premature deallocation of ancestors.
933  if (taskdata->td_flags.tasktype == TASK_IMPLICIT) {
934  if (taskdata->td_dephash) { // do we need to cleanup dephash?
935  int children = KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks);
936  kmp_tasking_flags_t flags_old = taskdata->td_flags;
937  if (children == 0 && flags_old.complete == 1) {
938  kmp_tasking_flags_t flags_new = flags_old;
939  flags_new.complete = 0;
940  if (KMP_COMPARE_AND_STORE_ACQ32(
941  RCAST(kmp_int32 *, &taskdata->td_flags),
942  *RCAST(kmp_int32 *, &flags_old),
943  *RCAST(kmp_int32 *, &flags_new))) {
944  KA_TRACE(100, ("__kmp_free_task_and_ancestors: T#%d cleans "
945  "dephash of implicit task %p\n",
946  gtid, taskdata));
947  // cleanup dephash of finished implicit task
948  __kmp_dephash_free_entries(thread, taskdata->td_dephash);
949  }
950  }
951  }
952  return;
953  }
954  // Predecrement simulated by "- 1" calculation
955  children = KMP_ATOMIC_DEC(&taskdata->td_allocated_child_tasks) - 1;
956  KMP_DEBUG_ASSERT(children >= 0);
957  }
958 
959  KA_TRACE(
960  20, ("__kmp_free_task_and_ancestors(exit): T#%d task %p has %d children; "
961  "not freeing it yet\n",
962  gtid, taskdata, children));
963 }
964 
965 // Only need to keep track of child task counts if any of the following:
966 // 1. team parallel and tasking not serialized;
967 // 2. it is a proxy or detachable or hidden helper task
968 // 3. the children counter of its parent task is greater than 0.
969 // The reason for the 3rd one is for serialized team that found detached task,
970 // hidden helper task, T. In this case, the execution of T is still deferred,
971 // and it is also possible that a regular task depends on T. In this case, if we
972 // don't track the children, task synchronization will be broken.
973 static bool __kmp_track_children_task(kmp_taskdata_t *taskdata) {
974  kmp_tasking_flags_t flags = taskdata->td_flags;
975  bool ret = !(flags.team_serial || flags.tasking_ser);
976  ret = ret || flags.proxy == TASK_PROXY ||
977  flags.detachable == TASK_DETACHABLE || flags.hidden_helper;
978  ret = ret ||
979  KMP_ATOMIC_LD_ACQ(&taskdata->td_parent->td_incomplete_child_tasks) > 0;
980  return ret;
981 }
982 
983 // __kmp_task_finish: bookkeeping to do when a task finishes execution
984 //
985 // gtid: global thread ID for calling thread
986 // task: task to be finished
987 // resumed_task: task to be resumed. (may be NULL if task is serialized)
988 //
989 // template<ompt>: effectively ompt_enabled.enabled!=0
990 // the version with ompt=false is inlined, allowing to optimize away all ompt
991 // code in this case
992 template <bool ompt>
993 static void __kmp_task_finish(kmp_int32 gtid, kmp_task_t *task,
994  kmp_taskdata_t *resumed_task) {
995  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
996  kmp_info_t *thread = __kmp_threads[gtid];
997  kmp_task_team_t *task_team =
998  thread->th.th_task_team; // might be NULL for serial teams...
999 #if KMP_DEBUG
1000  kmp_int32 children = 0;
1001 #endif
1002  KA_TRACE(10, ("__kmp_task_finish(enter): T#%d finishing task %p and resuming "
1003  "task %p\n",
1004  gtid, taskdata, resumed_task));
1005 
1006  KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
1007 
1008 // Pop task from stack if tied
1009 #ifdef BUILD_TIED_TASK_STACK
1010  if (taskdata->td_flags.tiedness == TASK_TIED) {
1011  __kmp_pop_task_stack(gtid, thread, taskdata);
1012  }
1013 #endif /* BUILD_TIED_TASK_STACK */
1014 
1015  if (UNLIKELY(taskdata->td_flags.tiedness == TASK_UNTIED)) {
1016  // untied task needs to check the counter so that the task structure is not
1017  // freed prematurely
1018  kmp_int32 counter = KMP_ATOMIC_DEC(&taskdata->td_untied_count) - 1;
1019  KA_TRACE(
1020  20,
1021  ("__kmp_task_finish: T#%d untied_count (%d) decremented for task %p\n",
1022  gtid, counter, taskdata));
1023  if (counter > 0) {
1024  // untied task is not done, to be continued possibly by other thread, do
1025  // not free it now
1026  if (resumed_task == NULL) {
1027  KMP_DEBUG_ASSERT(taskdata->td_flags.task_serial);
1028  resumed_task = taskdata->td_parent; // In a serialized task, the resumed
1029  // task is the parent
1030  }
1031  thread->th.th_current_task = resumed_task; // restore current_task
1032  resumed_task->td_flags.executing = 1; // resume previous task
1033  KA_TRACE(10, ("__kmp_task_finish(exit): T#%d partially done task %p, "
1034  "resuming task %p\n",
1035  gtid, taskdata, resumed_task));
1036  return;
1037  }
1038  }
1039 
1040  // bookkeeping for resuming task:
1041  // GEH - note tasking_ser => task_serial
1042  KMP_DEBUG_ASSERT(
1043  (taskdata->td_flags.tasking_ser || taskdata->td_flags.task_serial) ==
1044  taskdata->td_flags.task_serial);
1045  if (taskdata->td_flags.task_serial) {
1046  if (resumed_task == NULL) {
1047  resumed_task = taskdata->td_parent; // In a serialized task, the resumed
1048  // task is the parent
1049  }
1050  } else {
1051  KMP_DEBUG_ASSERT(resumed_task !=
1052  NULL); // verify that resumed task is passed as argument
1053  }
1054 
1055  /* If the tasks' destructor thunk flag has been set, we need to invoke the
1056  destructor thunk that has been generated by the compiler. The code is
1057  placed here, since at this point other tasks might have been released
1058  hence overlapping the destructor invocations with some other work in the
1059  released tasks. The OpenMP spec is not specific on when the destructors
1060  are invoked, so we should be free to choose. */
1061  if (UNLIKELY(taskdata->td_flags.destructors_thunk)) {
1062  kmp_routine_entry_t destr_thunk = task->data1.destructors;
1063  KMP_ASSERT(destr_thunk);
1064  destr_thunk(gtid, task);
1065  }
1066 
1067  KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0);
1068  KMP_DEBUG_ASSERT(taskdata->td_flags.started == 1);
1069  KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0);
1070 
1071  bool completed = true;
1072  if (UNLIKELY(taskdata->td_flags.detachable == TASK_DETACHABLE)) {
1073  if (taskdata->td_allow_completion_event.type ==
1074  KMP_EVENT_ALLOW_COMPLETION) {
1075  // event hasn't been fulfilled yet. Try to detach task.
1076  __kmp_acquire_tas_lock(&taskdata->td_allow_completion_event.lock, gtid);
1077  if (taskdata->td_allow_completion_event.type ==
1078  KMP_EVENT_ALLOW_COMPLETION) {
1079  // task finished execution
1080  KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 1);
1081  taskdata->td_flags.executing = 0; // suspend the finishing task
1082 
1083 #if OMPT_SUPPORT
1084  // For a detached task, which is not completed, we switch back
1085  // the omp_fulfill_event signals completion
1086  // locking is necessary to avoid a race with ompt_task_late_fulfill
1087  if (ompt)
1088  __ompt_task_finish(task, resumed_task, ompt_task_detach);
1089 #endif
1090 
1091  // no access to taskdata after this point!
1092  // __kmp_fulfill_event might free taskdata at any time from now
1093 
1094  taskdata->td_flags.proxy = TASK_PROXY; // proxify!
1095  completed = false;
1096  }
1097  __kmp_release_tas_lock(&taskdata->td_allow_completion_event.lock, gtid);
1098  }
1099  }
1100 
1101  // Tasks with valid target async handles must be re-enqueued.
1102  if (taskdata->td_target_data.async_handle != NULL) {
1103  // Note: no need to translate gtid to its shadow. If the current thread is a
1104  // hidden helper one, then the gtid is already correct. Otherwise, hidden
1105  // helper threads are disabled, and gtid refers to a OpenMP thread.
1106  __kmpc_give_task(task, __kmp_tid_from_gtid(gtid));
1107  if (KMP_HIDDEN_HELPER_THREAD(gtid))
1108  __kmp_hidden_helper_worker_thread_signal();
1109  completed = false;
1110  }
1111 
1112  if (completed) {
1113  taskdata->td_flags.complete = 1; // mark the task as completed
1114 
1115 #if OMPT_SUPPORT
1116  // This is not a detached task, we are done here
1117  if (ompt)
1118  __ompt_task_finish(task, resumed_task, ompt_task_complete);
1119 #endif
1120  // TODO: What would be the balance between the conditions in the function
1121  // and an atomic operation?
1122  if (__kmp_track_children_task(taskdata)) {
1123  __kmp_release_deps(gtid, taskdata);
1124  // Predecrement simulated by "- 1" calculation
1125 #if KMP_DEBUG
1126  children = -1 +
1127 #endif
1128  KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks);
1129  KMP_DEBUG_ASSERT(children >= 0);
1130  if (taskdata->td_taskgroup)
1131  KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count);
1132  } else if (task_team && (task_team->tt.tt_found_proxy_tasks ||
1133  task_team->tt.tt_hidden_helper_task_encountered)) {
1134  // if we found proxy or hidden helper tasks there could exist a dependency
1135  // chain with the proxy task as origin
1136  __kmp_release_deps(gtid, taskdata);
1137  }
1138  // td_flags.executing must be marked as 0 after __kmp_release_deps has been
1139  // called. Othertwise, if a task is executed immediately from the
1140  // release_deps code, the flag will be reset to 1 again by this same
1141  // function
1142  KMP_DEBUG_ASSERT(taskdata->td_flags.executing == 1);
1143  taskdata->td_flags.executing = 0; // suspend the finishing task
1144 
1145  // Decrement the counter of hidden helper tasks to be executed.
1146  if (taskdata->td_flags.hidden_helper) {
1147  // Hidden helper tasks can only be executed by hidden helper threads.
1148  KMP_ASSERT(KMP_HIDDEN_HELPER_THREAD(gtid));
1149  KMP_ATOMIC_DEC(&__kmp_unexecuted_hidden_helper_tasks);
1150  }
1151  }
1152 
1153  KA_TRACE(
1154  20, ("__kmp_task_finish: T#%d finished task %p, %d incomplete children\n",
1155  gtid, taskdata, children));
1156 
1157  // Free this task and then ancestor tasks if they have no children.
1158  // Restore th_current_task first as suggested by John:
1159  // johnmc: if an asynchronous inquiry peers into the runtime system
1160  // it doesn't see the freed task as the current task.
1161  thread->th.th_current_task = resumed_task;
1162  if (completed)
1163  __kmp_free_task_and_ancestors(gtid, taskdata, thread);
1164 
1165  // TODO: GEH - make sure root team implicit task is initialized properly.
1166  // KMP_DEBUG_ASSERT( resumed_task->td_flags.executing == 0 );
1167  resumed_task->td_flags.executing = 1; // resume previous task
1168 
1169  KA_TRACE(
1170  10, ("__kmp_task_finish(exit): T#%d finished task %p, resuming task %p\n",
1171  gtid, taskdata, resumed_task));
1172 
1173  return;
1174 }
1175 
1176 template <bool ompt>
1177 static void __kmpc_omp_task_complete_if0_template(ident_t *loc_ref,
1178  kmp_int32 gtid,
1179  kmp_task_t *task) {
1180  KA_TRACE(10, ("__kmpc_omp_task_complete_if0(enter): T#%d loc=%p task=%p\n",
1181  gtid, loc_ref, KMP_TASK_TO_TASKDATA(task)));
1182  KMP_DEBUG_ASSERT(gtid >= 0);
1183  // this routine will provide task to resume
1184  __kmp_task_finish<ompt>(gtid, task, NULL);
1185 
1186  KA_TRACE(10, ("__kmpc_omp_task_complete_if0(exit): T#%d loc=%p task=%p\n",
1187  gtid, loc_ref, KMP_TASK_TO_TASKDATA(task)));
1188 
1189 #if OMPT_SUPPORT
1190  if (ompt) {
1191  ompt_frame_t *ompt_frame;
1192  __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
1193  ompt_frame->enter_frame = ompt_data_none;
1194  ompt_frame->enter_frame_flags =
1195  ompt_frame_runtime | ompt_frame_framepointer;
1196  }
1197 #endif
1198 
1199  return;
1200 }
1201 
1202 #if OMPT_SUPPORT
1203 OMPT_NOINLINE
1204 void __kmpc_omp_task_complete_if0_ompt(ident_t *loc_ref, kmp_int32 gtid,
1205  kmp_task_t *task) {
1206  __kmpc_omp_task_complete_if0_template<true>(loc_ref, gtid, task);
1207 }
1208 #endif // OMPT_SUPPORT
1209 
1210 // __kmpc_omp_task_complete_if0: report that a task has completed execution
1211 //
1212 // loc_ref: source location information; points to end of task block.
1213 // gtid: global thread number.
1214 // task: task thunk for the completed task.
1215 void __kmpc_omp_task_complete_if0(ident_t *loc_ref, kmp_int32 gtid,
1216  kmp_task_t *task) {
1217 #if OMPT_SUPPORT
1218  if (UNLIKELY(ompt_enabled.enabled)) {
1219  __kmpc_omp_task_complete_if0_ompt(loc_ref, gtid, task);
1220  return;
1221  }
1222 #endif
1223  __kmpc_omp_task_complete_if0_template<false>(loc_ref, gtid, task);
1224 }
1225 
1226 #ifdef TASK_UNUSED
1227 // __kmpc_omp_task_complete: report that a task has completed execution
1228 // NEVER GENERATED BY COMPILER, DEPRECATED!!!
1229 void __kmpc_omp_task_complete(ident_t *loc_ref, kmp_int32 gtid,
1230  kmp_task_t *task) {
1231  KA_TRACE(10, ("__kmpc_omp_task_complete(enter): T#%d loc=%p task=%p\n", gtid,
1232  loc_ref, KMP_TASK_TO_TASKDATA(task)));
1233 
1234  __kmp_task_finish<false>(gtid, task,
1235  NULL); // Not sure how to find task to resume
1236 
1237  KA_TRACE(10, ("__kmpc_omp_task_complete(exit): T#%d loc=%p task=%p\n", gtid,
1238  loc_ref, KMP_TASK_TO_TASKDATA(task)));
1239  return;
1240 }
1241 #endif // TASK_UNUSED
1242 
1243 // __kmp_init_implicit_task: Initialize the appropriate fields in the implicit
1244 // task for a given thread
1245 //
1246 // loc_ref: reference to source location of parallel region
1247 // this_thr: thread data structure corresponding to implicit task
1248 // team: team for this_thr
1249 // tid: thread id of given thread within team
1250 // set_curr_task: TRUE if need to push current task to thread
1251 // NOTE: Routine does not set up the implicit task ICVS. This is assumed to
1252 // have already been done elsewhere.
1253 // TODO: Get better loc_ref. Value passed in may be NULL
1254 void __kmp_init_implicit_task(ident_t *loc_ref, kmp_info_t *this_thr,
1255  kmp_team_t *team, int tid, int set_curr_task) {
1256  kmp_taskdata_t *task = &team->t.t_implicit_task_taskdata[tid];
1257 
1258  KF_TRACE(
1259  10,
1260  ("__kmp_init_implicit_task(enter): T#:%d team=%p task=%p, reinit=%s\n",
1261  tid, team, task, set_curr_task ? "TRUE" : "FALSE"));
1262 
1263  task->td_task_id = KMP_GEN_TASK_ID();
1264  task->td_team = team;
1265  // task->td_parent = NULL; // fix for CQ230101 (broken parent task info
1266  // in debugger)
1267  task->td_ident = loc_ref;
1268  task->td_taskwait_ident = NULL;
1269  task->td_taskwait_counter = 0;
1270  task->td_taskwait_thread = 0;
1271 
1272  task->td_flags.tiedness = TASK_TIED;
1273  task->td_flags.tasktype = TASK_IMPLICIT;
1274  task->td_flags.proxy = TASK_FULL;
1275 
1276  // All implicit tasks are executed immediately, not deferred
1277  task->td_flags.task_serial = 1;
1278  task->td_flags.tasking_ser = (__kmp_tasking_mode == tskm_immediate_exec);
1279  task->td_flags.team_serial = (team->t.t_serialized) ? 1 : 0;
1280 
1281  task->td_flags.started = 1;
1282  task->td_flags.executing = 1;
1283  task->td_flags.complete = 0;
1284  task->td_flags.freed = 0;
1285 
1286  task->td_depnode = NULL;
1287  task->td_last_tied = task;
1288  task->td_allow_completion_event.type = KMP_EVENT_UNINITIALIZED;
1289 
1290  if (set_curr_task) { // only do this init first time thread is created
1291  KMP_ATOMIC_ST_REL(&task->td_incomplete_child_tasks, 0);
1292  // Not used: don't need to deallocate implicit task
1293  KMP_ATOMIC_ST_REL(&task->td_allocated_child_tasks, 0);
1294  task->td_taskgroup = NULL; // An implicit task does not have taskgroup
1295  task->td_dephash = NULL;
1296  __kmp_push_current_task_to_thread(this_thr, team, tid);
1297  } else {
1298  KMP_DEBUG_ASSERT(task->td_incomplete_child_tasks == 0);
1299  KMP_DEBUG_ASSERT(task->td_allocated_child_tasks == 0);
1300  }
1301 
1302 #if OMPT_SUPPORT
1303  if (UNLIKELY(ompt_enabled.enabled))
1304  __ompt_task_init(task, tid);
1305 #endif
1306 
1307  KF_TRACE(10, ("__kmp_init_implicit_task(exit): T#:%d team=%p task=%p\n", tid,
1308  team, task));
1309 }
1310 
1311 // __kmp_finish_implicit_task: Release resources associated to implicit tasks
1312 // at the end of parallel regions. Some resources are kept for reuse in the next
1313 // parallel region.
1314 //
1315 // thread: thread data structure corresponding to implicit task
1316 void __kmp_finish_implicit_task(kmp_info_t *thread) {
1317  kmp_taskdata_t *task = thread->th.th_current_task;
1318  if (task->td_dephash) {
1319  int children;
1320  task->td_flags.complete = 1;
1321  children = KMP_ATOMIC_LD_ACQ(&task->td_incomplete_child_tasks);
1322  kmp_tasking_flags_t flags_old = task->td_flags;
1323  if (children == 0 && flags_old.complete == 1) {
1324  kmp_tasking_flags_t flags_new = flags_old;
1325  flags_new.complete = 0;
1326  if (KMP_COMPARE_AND_STORE_ACQ32(RCAST(kmp_int32 *, &task->td_flags),
1327  *RCAST(kmp_int32 *, &flags_old),
1328  *RCAST(kmp_int32 *, &flags_new))) {
1329  KA_TRACE(100, ("__kmp_finish_implicit_task: T#%d cleans "
1330  "dephash of implicit task %p\n",
1331  thread->th.th_info.ds.ds_gtid, task));
1332  __kmp_dephash_free_entries(thread, task->td_dephash);
1333  }
1334  }
1335  }
1336 }
1337 
1338 // __kmp_free_implicit_task: Release resources associated to implicit tasks
1339 // when these are destroyed regions
1340 //
1341 // thread: thread data structure corresponding to implicit task
1342 void __kmp_free_implicit_task(kmp_info_t *thread) {
1343  kmp_taskdata_t *task = thread->th.th_current_task;
1344  if (task && task->td_dephash) {
1345  __kmp_dephash_free(thread, task->td_dephash);
1346  task->td_dephash = NULL;
1347  }
1348 }
1349 
1350 // Round up a size to a power of two specified by val: Used to insert padding
1351 // between structures co-allocated using a single malloc() call
1352 static size_t __kmp_round_up_to_val(size_t size, size_t val) {
1353  if (size & (val - 1)) {
1354  size &= ~(val - 1);
1355  if (size <= KMP_SIZE_T_MAX - val) {
1356  size += val; // Round up if there is no overflow.
1357  }
1358  }
1359  return size;
1360 } // __kmp_round_up_to_va
1361 
1362 // __kmp_task_alloc: Allocate the taskdata and task data structures for a task
1363 //
1364 // loc_ref: source location information
1365 // gtid: global thread number.
1366 // flags: include tiedness & task type (explicit vs. implicit) of the ''new''
1367 // task encountered. Converted from kmp_int32 to kmp_tasking_flags_t in routine.
1368 // sizeof_kmp_task_t: Size in bytes of kmp_task_t data structure including
1369 // private vars accessed in task.
1370 // sizeof_shareds: Size in bytes of array of pointers to shared vars accessed
1371 // in task.
1372 // task_entry: Pointer to task code entry point generated by compiler.
1373 // returns: a pointer to the allocated kmp_task_t structure (task).
1374 kmp_task_t *__kmp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
1375  kmp_tasking_flags_t *flags,
1376  size_t sizeof_kmp_task_t, size_t sizeof_shareds,
1377  kmp_routine_entry_t task_entry) {
1378  kmp_task_t *task;
1379  kmp_taskdata_t *taskdata;
1380  kmp_info_t *thread = __kmp_threads[gtid];
1381  kmp_team_t *team = thread->th.th_team;
1382  kmp_taskdata_t *parent_task = thread->th.th_current_task;
1383  size_t shareds_offset;
1384 
1385  if (UNLIKELY(!TCR_4(__kmp_init_middle)))
1386  __kmp_middle_initialize();
1387 
1388  if (flags->hidden_helper) {
1389  if (__kmp_enable_hidden_helper) {
1390  if (!TCR_4(__kmp_init_hidden_helper))
1391  __kmp_hidden_helper_initialize();
1392  } else {
1393  // If the hidden helper task is not enabled, reset the flag to FALSE.
1394  flags->hidden_helper = FALSE;
1395  }
1396  }
1397 
1398  KA_TRACE(10, ("__kmp_task_alloc(enter): T#%d loc=%p, flags=(0x%x) "
1399  "sizeof_task=%ld sizeof_shared=%ld entry=%p\n",
1400  gtid, loc_ref, *((kmp_int32 *)flags), sizeof_kmp_task_t,
1401  sizeof_shareds, task_entry));
1402 
1403  KMP_DEBUG_ASSERT(parent_task);
1404  if (parent_task->td_flags.final) {
1405  if (flags->merged_if0) {
1406  }
1407  flags->final = 1;
1408  }
1409 
1410  if (flags->tiedness == TASK_UNTIED && !team->t.t_serialized) {
1411  // Untied task encountered causes the TSC algorithm to check entire deque of
1412  // the victim thread. If no untied task encountered, then checking the head
1413  // of the deque should be enough.
1414  KMP_CHECK_UPDATE(thread->th.th_task_team->tt.tt_untied_task_encountered, 1);
1415  }
1416 
1417  // Detachable tasks are not proxy tasks yet but could be in the future. Doing
1418  // the tasking setup
1419  // when that happens is too late.
1420  if (UNLIKELY(flags->proxy == TASK_PROXY ||
1421  flags->detachable == TASK_DETACHABLE || flags->hidden_helper)) {
1422  if (flags->proxy == TASK_PROXY) {
1423  flags->tiedness = TASK_UNTIED;
1424  flags->merged_if0 = 1;
1425  }
1426  /* are we running in a sequential parallel or tskm_immediate_exec... we need
1427  tasking support enabled */
1428  if ((thread->th.th_task_team) == NULL) {
1429  /* This should only happen if the team is serialized
1430  setup a task team and propagate it to the thread */
1431  KMP_DEBUG_ASSERT(team->t.t_serialized);
1432  KA_TRACE(30,
1433  ("T#%d creating task team in __kmp_task_alloc for proxy task\n",
1434  gtid));
1435  // 1 indicates setup the current team regardless of nthreads
1436  __kmp_task_team_setup(thread, team, 1);
1437  thread->th.th_task_team = team->t.t_task_team[thread->th.th_task_state];
1438  }
1439  kmp_task_team_t *task_team = thread->th.th_task_team;
1440 
1441  /* tasking must be enabled now as the task might not be pushed */
1442  if (!KMP_TASKING_ENABLED(task_team)) {
1443  KA_TRACE(
1444  30,
1445  ("T#%d enabling tasking in __kmp_task_alloc for proxy task\n", gtid));
1446  __kmp_enable_tasking(task_team, thread);
1447  kmp_int32 tid = thread->th.th_info.ds.ds_tid;
1448  kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[tid];
1449  // No lock needed since only owner can allocate
1450  if (thread_data->td.td_deque == NULL) {
1451  __kmp_alloc_task_deque(thread, thread_data);
1452  }
1453  }
1454 
1455  if ((flags->proxy == TASK_PROXY || flags->detachable == TASK_DETACHABLE) &&
1456  task_team->tt.tt_found_proxy_tasks == FALSE)
1457  TCW_4(task_team->tt.tt_found_proxy_tasks, TRUE);
1458  if (flags->hidden_helper &&
1459  task_team->tt.tt_hidden_helper_task_encountered == FALSE)
1460  TCW_4(task_team->tt.tt_hidden_helper_task_encountered, TRUE);
1461  }
1462 
1463  // Calculate shared structure offset including padding after kmp_task_t struct
1464  // to align pointers in shared struct
1465  shareds_offset = sizeof(kmp_taskdata_t) + sizeof_kmp_task_t;
1466  shareds_offset = __kmp_round_up_to_val(shareds_offset, sizeof(void *));
1467 
1468  // Allocate a kmp_taskdata_t block and a kmp_task_t block.
1469  KA_TRACE(30, ("__kmp_task_alloc: T#%d First malloc size: %ld\n", gtid,
1470  shareds_offset));
1471  KA_TRACE(30, ("__kmp_task_alloc: T#%d Second malloc size: %ld\n", gtid,
1472  sizeof_shareds));
1473 
1474  // Avoid double allocation here by combining shareds with taskdata
1475 #if USE_FAST_MEMORY
1476  taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(thread, shareds_offset +
1477  sizeof_shareds);
1478 #else /* ! USE_FAST_MEMORY */
1479  taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(thread, shareds_offset +
1480  sizeof_shareds);
1481 #endif /* USE_FAST_MEMORY */
1482 
1483  task = KMP_TASKDATA_TO_TASK(taskdata);
1484 
1485 // Make sure task & taskdata are aligned appropriately
1486 #if KMP_ARCH_X86 || KMP_ARCH_PPC64 || !KMP_HAVE_QUAD
1487  KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (sizeof(double) - 1)) == 0);
1488  KMP_DEBUG_ASSERT((((kmp_uintptr_t)task) & (sizeof(double) - 1)) == 0);
1489 #else
1490  KMP_DEBUG_ASSERT((((kmp_uintptr_t)taskdata) & (sizeof(_Quad) - 1)) == 0);
1491  KMP_DEBUG_ASSERT((((kmp_uintptr_t)task) & (sizeof(_Quad) - 1)) == 0);
1492 #endif
1493  if (sizeof_shareds > 0) {
1494  // Avoid double allocation here by combining shareds with taskdata
1495  task->shareds = &((char *)taskdata)[shareds_offset];
1496  // Make sure shareds struct is aligned to pointer size
1497  KMP_DEBUG_ASSERT((((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1)) ==
1498  0);
1499  } else {
1500  task->shareds = NULL;
1501  }
1502  task->routine = task_entry;
1503  task->part_id = 0; // AC: Always start with 0 part id
1504 
1505  taskdata->td_task_id = KMP_GEN_TASK_ID();
1506  taskdata->td_team = thread->th.th_team;
1507  taskdata->td_alloc_thread = thread;
1508  taskdata->td_parent = parent_task;
1509  taskdata->td_level = parent_task->td_level + 1; // increment nesting level
1510  KMP_ATOMIC_ST_RLX(&taskdata->td_untied_count, 0);
1511  taskdata->td_ident = loc_ref;
1512  taskdata->td_taskwait_ident = NULL;
1513  taskdata->td_taskwait_counter = 0;
1514  taskdata->td_taskwait_thread = 0;
1515  KMP_DEBUG_ASSERT(taskdata->td_parent != NULL);
1516  // avoid copying icvs for proxy tasks
1517  if (flags->proxy == TASK_FULL)
1518  copy_icvs(&taskdata->td_icvs, &taskdata->td_parent->td_icvs);
1519 
1520  taskdata->td_flags = *flags;
1521  taskdata->td_task_team = thread->th.th_task_team;
1522  taskdata->td_size_alloc = shareds_offset + sizeof_shareds;
1523  taskdata->td_flags.tasktype = TASK_EXPLICIT;
1524  // If it is hidden helper task, we need to set the team and task team
1525  // correspondingly.
1526  if (flags->hidden_helper) {
1527  kmp_info_t *shadow_thread = __kmp_threads[KMP_GTID_TO_SHADOW_GTID(gtid)];
1528  taskdata->td_team = shadow_thread->th.th_team;
1529  taskdata->td_task_team = shadow_thread->th.th_task_team;
1530  }
1531 
1532  // GEH - TODO: fix this to copy parent task's value of tasking_ser flag
1533  taskdata->td_flags.tasking_ser = (__kmp_tasking_mode == tskm_immediate_exec);
1534 
1535  // GEH - TODO: fix this to copy parent task's value of team_serial flag
1536  taskdata->td_flags.team_serial = (team->t.t_serialized) ? 1 : 0;
1537 
1538  // GEH - Note we serialize the task if the team is serialized to make sure
1539  // implicit parallel region tasks are not left until program termination to
1540  // execute. Also, it helps locality to execute immediately.
1541 
1542  taskdata->td_flags.task_serial =
1543  (parent_task->td_flags.final || taskdata->td_flags.team_serial ||
1544  taskdata->td_flags.tasking_ser || flags->merged_if0);
1545 
1546  taskdata->td_flags.started = 0;
1547  taskdata->td_flags.executing = 0;
1548  taskdata->td_flags.complete = 0;
1549  taskdata->td_flags.freed = 0;
1550 
1551  KMP_ATOMIC_ST_RLX(&taskdata->td_incomplete_child_tasks, 0);
1552  // start at one because counts current task and children
1553  KMP_ATOMIC_ST_RLX(&taskdata->td_allocated_child_tasks, 1);
1554  taskdata->td_taskgroup =
1555  parent_task->td_taskgroup; // task inherits taskgroup from the parent task
1556  taskdata->td_dephash = NULL;
1557  taskdata->td_depnode = NULL;
1558  taskdata->td_target_data.async_handle = NULL;
1559  if (flags->tiedness == TASK_UNTIED)
1560  taskdata->td_last_tied = NULL; // will be set when the task is scheduled
1561  else
1562  taskdata->td_last_tied = taskdata;
1563  taskdata->td_allow_completion_event.type = KMP_EVENT_UNINITIALIZED;
1564 #if OMPT_SUPPORT
1565  if (UNLIKELY(ompt_enabled.enabled))
1566  __ompt_task_init(taskdata, gtid);
1567 #endif
1568  // TODO: What would be the balance between the conditions in the function and
1569  // an atomic operation?
1570  if (__kmp_track_children_task(taskdata)) {
1571  KMP_ATOMIC_INC(&parent_task->td_incomplete_child_tasks);
1572  if (parent_task->td_taskgroup)
1573  KMP_ATOMIC_INC(&parent_task->td_taskgroup->count);
1574  // Only need to keep track of allocated child tasks for explicit tasks since
1575  // implicit not deallocated
1576  if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT) {
1577  KMP_ATOMIC_INC(&taskdata->td_parent->td_allocated_child_tasks);
1578  }
1579  if (flags->hidden_helper) {
1580  taskdata->td_flags.task_serial = FALSE;
1581  // Increment the number of hidden helper tasks to be executed
1582  KMP_ATOMIC_INC(&__kmp_unexecuted_hidden_helper_tasks);
1583  }
1584  }
1585 
1586  KA_TRACE(20, ("__kmp_task_alloc(exit): T#%d created task %p parent=%p\n",
1587  gtid, taskdata, taskdata->td_parent));
1588 
1589  return task;
1590 }
1591 
1592 kmp_task_t *__kmpc_omp_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
1593  kmp_int32 flags, size_t sizeof_kmp_task_t,
1594  size_t sizeof_shareds,
1595  kmp_routine_entry_t task_entry) {
1596  kmp_task_t *retval;
1597  kmp_tasking_flags_t *input_flags = (kmp_tasking_flags_t *)&flags;
1598  __kmp_assert_valid_gtid(gtid);
1599  input_flags->native = FALSE;
1600  // __kmp_task_alloc() sets up all other runtime flags
1601  KA_TRACE(10, ("__kmpc_omp_task_alloc(enter): T#%d loc=%p, flags=(%s %s %s) "
1602  "sizeof_task=%ld sizeof_shared=%ld entry=%p\n",
1603  gtid, loc_ref, input_flags->tiedness ? "tied " : "untied",
1604  input_flags->proxy ? "proxy" : "",
1605  input_flags->detachable ? "detachable" : "", sizeof_kmp_task_t,
1606  sizeof_shareds, task_entry));
1607 
1608  retval = __kmp_task_alloc(loc_ref, gtid, input_flags, sizeof_kmp_task_t,
1609  sizeof_shareds, task_entry);
1610 
1611  KA_TRACE(20, ("__kmpc_omp_task_alloc(exit): T#%d retval %p\n", gtid, retval));
1612 
1613  return retval;
1614 }
1615 
1616 kmp_task_t *__kmpc_omp_target_task_alloc(ident_t *loc_ref, kmp_int32 gtid,
1617  kmp_int32 flags,
1618  size_t sizeof_kmp_task_t,
1619  size_t sizeof_shareds,
1620  kmp_routine_entry_t task_entry,
1621  kmp_int64 device_id) {
1622  auto &input_flags = reinterpret_cast<kmp_tasking_flags_t &>(flags);
1623  // target task is untied defined in the specification
1624  input_flags.tiedness = TASK_UNTIED;
1625 
1626  if (__kmp_enable_hidden_helper)
1627  input_flags.hidden_helper = TRUE;
1628 
1629  return __kmpc_omp_task_alloc(loc_ref, gtid, flags, sizeof_kmp_task_t,
1630  sizeof_shareds, task_entry);
1631 }
1632 
1646 kmp_int32
1648  kmp_task_t *new_task, kmp_int32 naffins,
1649  kmp_task_affinity_info_t *affin_list) {
1650  return 0;
1651 }
1652 
1653 // __kmp_invoke_task: invoke the specified task
1654 //
1655 // gtid: global thread ID of caller
1656 // task: the task to invoke
1657 // current_task: the task to resume after task invocation
1658 static void __kmp_invoke_task(kmp_int32 gtid, kmp_task_t *task,
1659  kmp_taskdata_t *current_task) {
1660  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
1661  kmp_info_t *thread;
1662  int discard = 0 /* false */;
1663  KA_TRACE(
1664  30, ("__kmp_invoke_task(enter): T#%d invoking task %p, current_task=%p\n",
1665  gtid, taskdata, current_task));
1666  KMP_DEBUG_ASSERT(task);
1667  if (UNLIKELY(taskdata->td_flags.proxy == TASK_PROXY &&
1668  taskdata->td_flags.complete == 1)) {
1669  // This is a proxy task that was already completed but it needs to run
1670  // its bottom-half finish
1671  KA_TRACE(
1672  30,
1673  ("__kmp_invoke_task: T#%d running bottom finish for proxy task %p\n",
1674  gtid, taskdata));
1675 
1676  __kmp_bottom_half_finish_proxy(gtid, task);
1677 
1678  KA_TRACE(30, ("__kmp_invoke_task(exit): T#%d completed bottom finish for "
1679  "proxy task %p, resuming task %p\n",
1680  gtid, taskdata, current_task));
1681 
1682  return;
1683  }
1684 
1685 #if OMPT_SUPPORT
1686  // For untied tasks, the first task executed only calls __kmpc_omp_task and
1687  // does not execute code.
1688  ompt_thread_info_t oldInfo;
1689  if (UNLIKELY(ompt_enabled.enabled)) {
1690  // Store the threads states and restore them after the task
1691  thread = __kmp_threads[gtid];
1692  oldInfo = thread->th.ompt_thread_info;
1693  thread->th.ompt_thread_info.wait_id = 0;
1694  thread->th.ompt_thread_info.state = (thread->th.th_team_serialized)
1695  ? ompt_state_work_serial
1696  : ompt_state_work_parallel;
1697  taskdata->ompt_task_info.frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1698  }
1699 #endif
1700 
1701  // Proxy tasks are not handled by the runtime
1702  if (taskdata->td_flags.proxy != TASK_PROXY) {
1703  __kmp_task_start(gtid, task, current_task); // OMPT only if not discarded
1704  }
1705 
1706  // TODO: cancel tasks if the parallel region has also been cancelled
1707  // TODO: check if this sequence can be hoisted above __kmp_task_start
1708  // if cancellation has been enabled for this run ...
1709  if (UNLIKELY(__kmp_omp_cancellation)) {
1710  thread = __kmp_threads[gtid];
1711  kmp_team_t *this_team = thread->th.th_team;
1712  kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup;
1713  if ((taskgroup && taskgroup->cancel_request) ||
1714  (this_team->t.t_cancel_request == cancel_parallel)) {
1715 #if OMPT_SUPPORT && OMPT_OPTIONAL
1716  ompt_data_t *task_data;
1717  if (UNLIKELY(ompt_enabled.ompt_callback_cancel)) {
1718  __ompt_get_task_info_internal(0, NULL, &task_data, NULL, NULL, NULL);
1719  ompt_callbacks.ompt_callback(ompt_callback_cancel)(
1720  task_data,
1721  ((taskgroup && taskgroup->cancel_request) ? ompt_cancel_taskgroup
1722  : ompt_cancel_parallel) |
1723  ompt_cancel_discarded_task,
1724  NULL);
1725  }
1726 #endif
1727  KMP_COUNT_BLOCK(TASK_cancelled);
1728  // this task belongs to a task group and we need to cancel it
1729  discard = 1 /* true */;
1730  }
1731  }
1732 
1733  // Invoke the task routine and pass in relevant data.
1734  // Thunks generated by gcc take a different argument list.
1735  if (!discard) {
1736  if (taskdata->td_flags.tiedness == TASK_UNTIED) {
1737  taskdata->td_last_tied = current_task->td_last_tied;
1738  KMP_DEBUG_ASSERT(taskdata->td_last_tied);
1739  }
1740 #if KMP_STATS_ENABLED
1741  KMP_COUNT_BLOCK(TASK_executed);
1742  switch (KMP_GET_THREAD_STATE()) {
1743  case FORK_JOIN_BARRIER:
1744  KMP_PUSH_PARTITIONED_TIMER(OMP_task_join_bar);
1745  break;
1746  case PLAIN_BARRIER:
1747  KMP_PUSH_PARTITIONED_TIMER(OMP_task_plain_bar);
1748  break;
1749  case TASKYIELD:
1750  KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskyield);
1751  break;
1752  case TASKWAIT:
1753  KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskwait);
1754  break;
1755  case TASKGROUP:
1756  KMP_PUSH_PARTITIONED_TIMER(OMP_task_taskgroup);
1757  break;
1758  default:
1759  KMP_PUSH_PARTITIONED_TIMER(OMP_task_immediate);
1760  break;
1761  }
1762 #endif // KMP_STATS_ENABLED
1763 
1764 // OMPT task begin
1765 #if OMPT_SUPPORT
1766  if (UNLIKELY(ompt_enabled.enabled))
1767  __ompt_task_start(task, current_task, gtid);
1768 #endif
1769 #if OMPT_SUPPORT && OMPT_OPTIONAL
1770  if (UNLIKELY(ompt_enabled.ompt_callback_dispatch &&
1771  taskdata->ompt_task_info.dispatch_chunk.iterations > 0)) {
1772  ompt_data_t instance = ompt_data_none;
1773  instance.ptr = &(taskdata->ompt_task_info.dispatch_chunk);
1774  ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
1775  ompt_callbacks.ompt_callback(ompt_callback_dispatch)(
1776  &(team_info->parallel_data), &(taskdata->ompt_task_info.task_data),
1777  ompt_dispatch_taskloop_chunk, instance);
1778  taskdata->ompt_task_info.dispatch_chunk = {0, 0};
1779  }
1780 #endif // OMPT_SUPPORT && OMPT_OPTIONAL
1781 
1782 #if OMPD_SUPPORT
1783  if (ompd_state & OMPD_ENABLE_BP)
1784  ompd_bp_task_begin();
1785 #endif
1786 
1787 #if USE_ITT_BUILD && USE_ITT_NOTIFY
1788  kmp_uint64 cur_time;
1789  kmp_int32 kmp_itt_count_task =
1790  __kmp_forkjoin_frames_mode == 3 && !taskdata->td_flags.task_serial &&
1791  current_task->td_flags.tasktype == TASK_IMPLICIT;
1792  if (kmp_itt_count_task) {
1793  thread = __kmp_threads[gtid];
1794  // Time outer level explicit task on barrier for adjusting imbalance time
1795  if (thread->th.th_bar_arrive_time)
1796  cur_time = __itt_get_timestamp();
1797  else
1798  kmp_itt_count_task = 0; // thread is not on a barrier - skip timing
1799  }
1800  KMP_FSYNC_ACQUIRED(taskdata); // acquired self (new task)
1801 #endif
1802 
1803 #if ENABLE_LIBOMPTARGET
1804  if (taskdata->td_target_data.async_handle != NULL) {
1805  // If we have a valid target async handle, that means that we have already
1806  // executed the task routine once. We must query for the handle completion
1807  // instead of re-executing the routine.
1808  __tgt_target_nowait_query(&taskdata->td_target_data.async_handle);
1809  } else
1810 #endif
1811  if (task->routine != NULL) {
1812 #ifdef KMP_GOMP_COMPAT
1813  if (taskdata->td_flags.native) {
1814  ((void (*)(void *))(*(task->routine)))(task->shareds);
1815  } else
1816 #endif /* KMP_GOMP_COMPAT */
1817  {
1818  (*(task->routine))(gtid, task);
1819  }
1820  }
1821  KMP_POP_PARTITIONED_TIMER();
1822 
1823 #if USE_ITT_BUILD && USE_ITT_NOTIFY
1824  if (kmp_itt_count_task) {
1825  // Barrier imbalance - adjust arrive time with the task duration
1826  thread->th.th_bar_arrive_time += (__itt_get_timestamp() - cur_time);
1827  }
1828  KMP_FSYNC_CANCEL(taskdata); // destroy self (just executed)
1829  KMP_FSYNC_RELEASING(taskdata->td_parent); // releasing parent
1830 #endif
1831  }
1832 
1833 #if OMPD_SUPPORT
1834  if (ompd_state & OMPD_ENABLE_BP)
1835  ompd_bp_task_end();
1836 #endif
1837 
1838  // Proxy tasks are not handled by the runtime
1839  if (taskdata->td_flags.proxy != TASK_PROXY) {
1840 #if OMPT_SUPPORT
1841  if (UNLIKELY(ompt_enabled.enabled)) {
1842  thread->th.ompt_thread_info = oldInfo;
1843  if (taskdata->td_flags.tiedness == TASK_TIED) {
1844  taskdata->ompt_task_info.frame.exit_frame = ompt_data_none;
1845  }
1846  __kmp_task_finish<true>(gtid, task, current_task);
1847  } else
1848 #endif
1849  __kmp_task_finish<false>(gtid, task, current_task);
1850  }
1851 
1852  KA_TRACE(
1853  30,
1854  ("__kmp_invoke_task(exit): T#%d completed task %p, resuming task %p\n",
1855  gtid, taskdata, current_task));
1856  return;
1857 }
1858 
1859 // __kmpc_omp_task_parts: Schedule a thread-switchable task for execution
1860 //
1861 // loc_ref: location of original task pragma (ignored)
1862 // gtid: Global Thread ID of encountering thread
1863 // new_task: task thunk allocated by __kmp_omp_task_alloc() for the ''new task''
1864 // Returns:
1865 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to
1866 // be resumed later.
1867 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be
1868 // resumed later.
1869 kmp_int32 __kmpc_omp_task_parts(ident_t *loc_ref, kmp_int32 gtid,
1870  kmp_task_t *new_task) {
1871  kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
1872 
1873  KA_TRACE(10, ("__kmpc_omp_task_parts(enter): T#%d loc=%p task=%p\n", gtid,
1874  loc_ref, new_taskdata));
1875 
1876 #if OMPT_SUPPORT
1877  kmp_taskdata_t *parent;
1878  if (UNLIKELY(ompt_enabled.enabled)) {
1879  parent = new_taskdata->td_parent;
1880  if (ompt_enabled.ompt_callback_task_create) {
1881  ompt_callbacks.ompt_callback(ompt_callback_task_create)(
1882  &(parent->ompt_task_info.task_data), &(parent->ompt_task_info.frame),
1883  &(new_taskdata->ompt_task_info.task_data), ompt_task_explicit, 0,
1884  OMPT_GET_RETURN_ADDRESS(0));
1885  }
1886  }
1887 #endif
1888 
1889  /* Should we execute the new task or queue it? For now, let's just always try
1890  to queue it. If the queue fills up, then we'll execute it. */
1891 
1892  if (__kmp_push_task(gtid, new_task) == TASK_NOT_PUSHED) // if cannot defer
1893  { // Execute this task immediately
1894  kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task;
1895  new_taskdata->td_flags.task_serial = 1;
1896  __kmp_invoke_task(gtid, new_task, current_task);
1897  }
1898 
1899  KA_TRACE(
1900  10,
1901  ("__kmpc_omp_task_parts(exit): T#%d returning TASK_CURRENT_NOT_QUEUED: "
1902  "loc=%p task=%p, return: TASK_CURRENT_NOT_QUEUED\n",
1903  gtid, loc_ref, new_taskdata));
1904 
1905 #if OMPT_SUPPORT
1906  if (UNLIKELY(ompt_enabled.enabled)) {
1907  parent->ompt_task_info.frame.enter_frame = ompt_data_none;
1908  }
1909 #endif
1910  return TASK_CURRENT_NOT_QUEUED;
1911 }
1912 
1913 // __kmp_omp_task: Schedule a non-thread-switchable task for execution
1914 //
1915 // gtid: Global Thread ID of encountering thread
1916 // new_task:non-thread-switchable task thunk allocated by __kmp_omp_task_alloc()
1917 // serialize_immediate: if TRUE then if the task is executed immediately its
1918 // execution will be serialized
1919 // Returns:
1920 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to
1921 // be resumed later.
1922 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be
1923 // resumed later.
1924 kmp_int32 __kmp_omp_task(kmp_int32 gtid, kmp_task_t *new_task,
1925  bool serialize_immediate) {
1926  kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
1927 
1928  /* Should we execute the new task or queue it? For now, let's just always try
1929  to queue it. If the queue fills up, then we'll execute it. */
1930  if (new_taskdata->td_flags.proxy == TASK_PROXY ||
1931  __kmp_push_task(gtid, new_task) == TASK_NOT_PUSHED) // if cannot defer
1932  { // Execute this task immediately
1933  kmp_taskdata_t *current_task = __kmp_threads[gtid]->th.th_current_task;
1934  if (serialize_immediate)
1935  new_taskdata->td_flags.task_serial = 1;
1936  __kmp_invoke_task(gtid, new_task, current_task);
1937  } else if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME &&
1938  __kmp_wpolicy_passive) {
1939  kmp_info_t *this_thr = __kmp_threads[gtid];
1940  kmp_team_t *team = this_thr->th.th_team;
1941  kmp_int32 nthreads = this_thr->th.th_team_nproc;
1942  for (int i = 0; i < nthreads; ++i) {
1943  kmp_info_t *thread = team->t.t_threads[i];
1944  if (thread == this_thr)
1945  continue;
1946  if (thread->th.th_sleep_loc != NULL) {
1947  __kmp_null_resume_wrapper(thread);
1948  break; // awake one thread at a time
1949  }
1950  }
1951  }
1952  return TASK_CURRENT_NOT_QUEUED;
1953 }
1954 
1955 // __kmpc_omp_task: Wrapper around __kmp_omp_task to schedule a
1956 // non-thread-switchable task from the parent thread only!
1957 //
1958 // loc_ref: location of original task pragma (ignored)
1959 // gtid: Global Thread ID of encountering thread
1960 // new_task: non-thread-switchable task thunk allocated by
1961 // __kmp_omp_task_alloc()
1962 // Returns:
1963 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to
1964 // be resumed later.
1965 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be
1966 // resumed later.
1967 kmp_int32 __kmpc_omp_task(ident_t *loc_ref, kmp_int32 gtid,
1968  kmp_task_t *new_task) {
1969  kmp_int32 res;
1970  KMP_SET_THREAD_STATE_BLOCK(EXPLICIT_TASK);
1971 
1972 #if KMP_DEBUG || OMPT_SUPPORT
1973  kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
1974 #endif
1975  KA_TRACE(10, ("__kmpc_omp_task(enter): T#%d loc=%p task=%p\n", gtid, loc_ref,
1976  new_taskdata));
1977  __kmp_assert_valid_gtid(gtid);
1978 
1979 #if OMPT_SUPPORT
1980  kmp_taskdata_t *parent = NULL;
1981  if (UNLIKELY(ompt_enabled.enabled)) {
1982  if (!new_taskdata->td_flags.started) {
1983  OMPT_STORE_RETURN_ADDRESS(gtid);
1984  parent = new_taskdata->td_parent;
1985  if (!parent->ompt_task_info.frame.enter_frame.ptr) {
1986  parent->ompt_task_info.frame.enter_frame.ptr =
1987  OMPT_GET_FRAME_ADDRESS(0);
1988  }
1989  if (ompt_enabled.ompt_callback_task_create) {
1990  ompt_callbacks.ompt_callback(ompt_callback_task_create)(
1991  &(parent->ompt_task_info.task_data),
1992  &(parent->ompt_task_info.frame),
1993  &(new_taskdata->ompt_task_info.task_data),
1994  ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(new_taskdata), 0,
1995  OMPT_LOAD_RETURN_ADDRESS(gtid));
1996  }
1997  } else {
1998  // We are scheduling the continuation of an UNTIED task.
1999  // Scheduling back to the parent task.
2000  __ompt_task_finish(new_task,
2001  new_taskdata->ompt_task_info.scheduling_parent,
2002  ompt_task_switch);
2003  new_taskdata->ompt_task_info.frame.exit_frame = ompt_data_none;
2004  }
2005  }
2006 #endif
2007 
2008  res = __kmp_omp_task(gtid, new_task, true);
2009 
2010  KA_TRACE(10, ("__kmpc_omp_task(exit): T#%d returning "
2011  "TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n",
2012  gtid, loc_ref, new_taskdata));
2013 #if OMPT_SUPPORT
2014  if (UNLIKELY(ompt_enabled.enabled && parent != NULL)) {
2015  parent->ompt_task_info.frame.enter_frame = ompt_data_none;
2016  }
2017 #endif
2018  return res;
2019 }
2020 
2021 // __kmp_omp_taskloop_task: Wrapper around __kmp_omp_task to schedule
2022 // a taskloop task with the correct OMPT return address
2023 //
2024 // loc_ref: location of original task pragma (ignored)
2025 // gtid: Global Thread ID of encountering thread
2026 // new_task: non-thread-switchable task thunk allocated by
2027 // __kmp_omp_task_alloc()
2028 // codeptr_ra: return address for OMPT callback
2029 // Returns:
2030 // TASK_CURRENT_NOT_QUEUED (0) if did not suspend and queue current task to
2031 // be resumed later.
2032 // TASK_CURRENT_QUEUED (1) if suspended and queued the current task to be
2033 // resumed later.
2034 kmp_int32 __kmp_omp_taskloop_task(ident_t *loc_ref, kmp_int32 gtid,
2035  kmp_task_t *new_task, void *codeptr_ra) {
2036  kmp_int32 res;
2037  KMP_SET_THREAD_STATE_BLOCK(EXPLICIT_TASK);
2038 
2039 #if KMP_DEBUG || OMPT_SUPPORT
2040  kmp_taskdata_t *new_taskdata = KMP_TASK_TO_TASKDATA(new_task);
2041 #endif
2042  KA_TRACE(10, ("__kmpc_omp_task(enter): T#%d loc=%p task=%p\n", gtid, loc_ref,
2043  new_taskdata));
2044 
2045 #if OMPT_SUPPORT
2046  kmp_taskdata_t *parent = NULL;
2047  if (UNLIKELY(ompt_enabled.enabled && !new_taskdata->td_flags.started)) {
2048  parent = new_taskdata->td_parent;
2049  if (!parent->ompt_task_info.frame.enter_frame.ptr)
2050  parent->ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
2051  if (ompt_enabled.ompt_callback_task_create) {
2052  ompt_callbacks.ompt_callback(ompt_callback_task_create)(
2053  &(parent->ompt_task_info.task_data), &(parent->ompt_task_info.frame),
2054  &(new_taskdata->ompt_task_info.task_data),
2055  ompt_task_explicit | TASK_TYPE_DETAILS_FORMAT(new_taskdata), 0,
2056  codeptr_ra);
2057  }
2058  }
2059 #endif
2060 
2061  res = __kmp_omp_task(gtid, new_task, true);
2062 
2063  KA_TRACE(10, ("__kmpc_omp_task(exit): T#%d returning "
2064  "TASK_CURRENT_NOT_QUEUED: loc=%p task=%p\n",
2065  gtid, loc_ref, new_taskdata));
2066 #if OMPT_SUPPORT
2067  if (UNLIKELY(ompt_enabled.enabled && parent != NULL)) {
2068  parent->ompt_task_info.frame.enter_frame = ompt_data_none;
2069  }
2070 #endif
2071  return res;
2072 }
2073 
2074 template <bool ompt>
2075 static kmp_int32 __kmpc_omp_taskwait_template(ident_t *loc_ref, kmp_int32 gtid,
2076  void *frame_address,
2077  void *return_address) {
2078  kmp_taskdata_t *taskdata = nullptr;
2079  kmp_info_t *thread;
2080  int thread_finished = FALSE;
2081  KMP_SET_THREAD_STATE_BLOCK(TASKWAIT);
2082 
2083  KA_TRACE(10, ("__kmpc_omp_taskwait(enter): T#%d loc=%p\n", gtid, loc_ref));
2084  KMP_DEBUG_ASSERT(gtid >= 0);
2085 
2086  if (__kmp_tasking_mode != tskm_immediate_exec) {
2087  thread = __kmp_threads[gtid];
2088  taskdata = thread->th.th_current_task;
2089 
2090 #if OMPT_SUPPORT && OMPT_OPTIONAL
2091  ompt_data_t *my_task_data;
2092  ompt_data_t *my_parallel_data;
2093 
2094  if (ompt) {
2095  my_task_data = &(taskdata->ompt_task_info.task_data);
2096  my_parallel_data = OMPT_CUR_TEAM_DATA(thread);
2097 
2098  taskdata->ompt_task_info.frame.enter_frame.ptr = frame_address;
2099 
2100  if (ompt_enabled.ompt_callback_sync_region) {
2101  ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
2102  ompt_sync_region_taskwait, ompt_scope_begin, my_parallel_data,
2103  my_task_data, return_address);
2104  }
2105 
2106  if (ompt_enabled.ompt_callback_sync_region_wait) {
2107  ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
2108  ompt_sync_region_taskwait, ompt_scope_begin, my_parallel_data,
2109  my_task_data, return_address);
2110  }
2111  }
2112 #endif // OMPT_SUPPORT && OMPT_OPTIONAL
2113 
2114 // Debugger: The taskwait is active. Store location and thread encountered the
2115 // taskwait.
2116 #if USE_ITT_BUILD
2117 // Note: These values are used by ITT events as well.
2118 #endif /* USE_ITT_BUILD */
2119  taskdata->td_taskwait_counter += 1;
2120  taskdata->td_taskwait_ident = loc_ref;
2121  taskdata->td_taskwait_thread = gtid + 1;
2122 
2123 #if USE_ITT_BUILD
2124  void *itt_sync_obj = NULL;
2125 #if USE_ITT_NOTIFY
2126  KMP_ITT_TASKWAIT_STARTING(itt_sync_obj);
2127 #endif /* USE_ITT_NOTIFY */
2128 #endif /* USE_ITT_BUILD */
2129 
2130  bool must_wait =
2131  !taskdata->td_flags.team_serial && !taskdata->td_flags.final;
2132 
2133  must_wait = must_wait || (thread->th.th_task_team != NULL &&
2134  thread->th.th_task_team->tt.tt_found_proxy_tasks);
2135  // If hidden helper thread is encountered, we must enable wait here.
2136  must_wait =
2137  must_wait ||
2138  (__kmp_enable_hidden_helper && thread->th.th_task_team != NULL &&
2139  thread->th.th_task_team->tt.tt_hidden_helper_task_encountered);
2140 
2141  if (must_wait) {
2142  kmp_flag_32<false, false> flag(
2143  RCAST(std::atomic<kmp_uint32> *,
2144  &(taskdata->td_incomplete_child_tasks)),
2145  0U);
2146  while (KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks) != 0) {
2147  flag.execute_tasks(thread, gtid, FALSE,
2148  &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj),
2149  __kmp_task_stealing_constraint);
2150  }
2151  }
2152 #if USE_ITT_BUILD
2153  KMP_ITT_TASKWAIT_FINISHED(itt_sync_obj);
2154  KMP_FSYNC_ACQUIRED(taskdata); // acquire self - sync with children
2155 #endif /* USE_ITT_BUILD */
2156 
2157  // Debugger: The taskwait is completed. Location remains, but thread is
2158  // negated.
2159  taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread;
2160 
2161 #if OMPT_SUPPORT && OMPT_OPTIONAL
2162  if (ompt) {
2163  if (ompt_enabled.ompt_callback_sync_region_wait) {
2164  ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
2165  ompt_sync_region_taskwait, ompt_scope_end, my_parallel_data,
2166  my_task_data, return_address);
2167  }
2168  if (ompt_enabled.ompt_callback_sync_region) {
2169  ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
2170  ompt_sync_region_taskwait, ompt_scope_end, my_parallel_data,
2171  my_task_data, return_address);
2172  }
2173  taskdata->ompt_task_info.frame.enter_frame = ompt_data_none;
2174  }
2175 #endif // OMPT_SUPPORT && OMPT_OPTIONAL
2176 
2177  }
2178 
2179  KA_TRACE(10, ("__kmpc_omp_taskwait(exit): T#%d task %p finished waiting, "
2180  "returning TASK_CURRENT_NOT_QUEUED\n",
2181  gtid, taskdata));
2182 
2183  return TASK_CURRENT_NOT_QUEUED;
2184 }
2185 
2186 #if OMPT_SUPPORT && OMPT_OPTIONAL
2187 OMPT_NOINLINE
2188 static kmp_int32 __kmpc_omp_taskwait_ompt(ident_t *loc_ref, kmp_int32 gtid,
2189  void *frame_address,
2190  void *return_address) {
2191  return __kmpc_omp_taskwait_template<true>(loc_ref, gtid, frame_address,
2192  return_address);
2193 }
2194 #endif // OMPT_SUPPORT && OMPT_OPTIONAL
2195 
2196 // __kmpc_omp_taskwait: Wait until all tasks generated by the current task are
2197 // complete
2198 kmp_int32 __kmpc_omp_taskwait(ident_t *loc_ref, kmp_int32 gtid) {
2199 #if OMPT_SUPPORT && OMPT_OPTIONAL
2200  if (UNLIKELY(ompt_enabled.enabled)) {
2201  OMPT_STORE_RETURN_ADDRESS(gtid);
2202  return __kmpc_omp_taskwait_ompt(loc_ref, gtid, OMPT_GET_FRAME_ADDRESS(0),
2203  OMPT_LOAD_RETURN_ADDRESS(gtid));
2204  }
2205 #endif
2206  return __kmpc_omp_taskwait_template<false>(loc_ref, gtid, NULL, NULL);
2207 }
2208 
2209 // __kmpc_omp_taskyield: switch to a different task
2210 kmp_int32 __kmpc_omp_taskyield(ident_t *loc_ref, kmp_int32 gtid, int end_part) {
2211  kmp_taskdata_t *taskdata = NULL;
2212  kmp_info_t *thread;
2213  int thread_finished = FALSE;
2214 
2215  KMP_COUNT_BLOCK(OMP_TASKYIELD);
2216  KMP_SET_THREAD_STATE_BLOCK(TASKYIELD);
2217 
2218  KA_TRACE(10, ("__kmpc_omp_taskyield(enter): T#%d loc=%p end_part = %d\n",
2219  gtid, loc_ref, end_part));
2220  __kmp_assert_valid_gtid(gtid);
2221 
2222  if (__kmp_tasking_mode != tskm_immediate_exec && __kmp_init_parallel) {
2223  thread = __kmp_threads[gtid];
2224  taskdata = thread->th.th_current_task;
2225 // Should we model this as a task wait or not?
2226 // Debugger: The taskwait is active. Store location and thread encountered the
2227 // taskwait.
2228 #if USE_ITT_BUILD
2229 // Note: These values are used by ITT events as well.
2230 #endif /* USE_ITT_BUILD */
2231  taskdata->td_taskwait_counter += 1;
2232  taskdata->td_taskwait_ident = loc_ref;
2233  taskdata->td_taskwait_thread = gtid + 1;
2234 
2235 #if USE_ITT_BUILD
2236  void *itt_sync_obj = NULL;
2237 #if USE_ITT_NOTIFY
2238  KMP_ITT_TASKWAIT_STARTING(itt_sync_obj);
2239 #endif /* USE_ITT_NOTIFY */
2240 #endif /* USE_ITT_BUILD */
2241  if (!taskdata->td_flags.team_serial) {
2242  kmp_task_team_t *task_team = thread->th.th_task_team;
2243  if (task_team != NULL) {
2244  if (KMP_TASKING_ENABLED(task_team)) {
2245 #if OMPT_SUPPORT
2246  if (UNLIKELY(ompt_enabled.enabled))
2247  thread->th.ompt_thread_info.ompt_task_yielded = 1;
2248 #endif
2249  __kmp_execute_tasks_32(
2250  thread, gtid, (kmp_flag_32<> *)NULL, FALSE,
2251  &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj),
2252  __kmp_task_stealing_constraint);
2253 #if OMPT_SUPPORT
2254  if (UNLIKELY(ompt_enabled.enabled))
2255  thread->th.ompt_thread_info.ompt_task_yielded = 0;
2256 #endif
2257  }
2258  }
2259  }
2260 #if USE_ITT_BUILD
2261  KMP_ITT_TASKWAIT_FINISHED(itt_sync_obj);
2262 #endif /* USE_ITT_BUILD */
2263 
2264  // Debugger: The taskwait is completed. Location remains, but thread is
2265  // negated.
2266  taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread;
2267  }
2268 
2269  KA_TRACE(10, ("__kmpc_omp_taskyield(exit): T#%d task %p resuming, "
2270  "returning TASK_CURRENT_NOT_QUEUED\n",
2271  gtid, taskdata));
2272 
2273  return TASK_CURRENT_NOT_QUEUED;
2274 }
2275 
2276 // Task Reduction implementation
2277 //
2278 // Note: initial implementation didn't take into account the possibility
2279 // to specify omp_orig for initializer of the UDR (user defined reduction).
2280 // Corrected implementation takes into account the omp_orig object.
2281 // Compiler is free to use old implementation if omp_orig is not specified.
2282 
2291 typedef struct kmp_taskred_flags {
2293  unsigned lazy_priv : 1;
2294  unsigned reserved31 : 31;
2296 
2300 typedef struct kmp_task_red_input {
2301  void *reduce_shar;
2302  size_t reduce_size;
2303  // three compiler-generated routines (init, fini are optional):
2304  void *reduce_init;
2305  void *reduce_fini;
2306  void *reduce_comb;
2309 
2313 typedef struct kmp_taskred_data {
2314  void *reduce_shar;
2315  size_t reduce_size;
2317  void *reduce_priv;
2318  void *reduce_pend;
2319  // three compiler-generated routines (init, fini are optional):
2320  void *reduce_comb;
2321  void *reduce_init;
2322  void *reduce_fini;
2323  void *reduce_orig;
2325 
2331 typedef struct kmp_taskred_input {
2332  void *reduce_shar;
2333  void *reduce_orig;
2334  size_t reduce_size;
2335  // three compiler-generated routines (init, fini are optional):
2336  void *reduce_init;
2337  void *reduce_fini;
2338  void *reduce_comb;
2345 template <typename T> void __kmp_assign_orig(kmp_taskred_data_t &item, T &src);
2346 template <>
2347 void __kmp_assign_orig<kmp_task_red_input_t>(kmp_taskred_data_t &item,
2348  kmp_task_red_input_t &src) {
2349  item.reduce_orig = NULL;
2350 }
2351 template <>
2352 void __kmp_assign_orig<kmp_taskred_input_t>(kmp_taskred_data_t &item,
2353  kmp_taskred_input_t &src) {
2354  if (src.reduce_orig != NULL) {
2355  item.reduce_orig = src.reduce_orig;
2356  } else {
2357  item.reduce_orig = src.reduce_shar;
2358  } // non-NULL reduce_orig means new interface used
2359 }
2360 
2361 template <typename T> void __kmp_call_init(kmp_taskred_data_t &item, size_t j);
2362 template <>
2363 void __kmp_call_init<kmp_task_red_input_t>(kmp_taskred_data_t &item,
2364  size_t offset) {
2365  ((void (*)(void *))item.reduce_init)((char *)(item.reduce_priv) + offset);
2366 }
2367 template <>
2368 void __kmp_call_init<kmp_taskred_input_t>(kmp_taskred_data_t &item,
2369  size_t offset) {
2370  ((void (*)(void *, void *))item.reduce_init)(
2371  (char *)(item.reduce_priv) + offset, item.reduce_orig);
2372 }
2373 
2374 template <typename T>
2375 void *__kmp_task_reduction_init(int gtid, int num, T *data) {
2376  __kmp_assert_valid_gtid(gtid);
2377  kmp_info_t *thread = __kmp_threads[gtid];
2378  kmp_taskgroup_t *tg = thread->th.th_current_task->td_taskgroup;
2379  kmp_uint32 nth = thread->th.th_team_nproc;
2380  kmp_taskred_data_t *arr;
2381 
2382  // check input data just in case
2383  KMP_ASSERT(tg != NULL);
2384  KMP_ASSERT(data != NULL);
2385  KMP_ASSERT(num > 0);
2386  if (nth == 1) {
2387  KA_TRACE(10, ("__kmpc_task_reduction_init: T#%d, tg %p, exiting nth=1\n",
2388  gtid, tg));
2389  return (void *)tg;
2390  }
2391  KA_TRACE(10, ("__kmpc_task_reduction_init: T#%d, taskgroup %p, #items %d\n",
2392  gtid, tg, num));
2393  arr = (kmp_taskred_data_t *)__kmp_thread_malloc(
2394  thread, num * sizeof(kmp_taskred_data_t));
2395  for (int i = 0; i < num; ++i) {
2396  size_t size = data[i].reduce_size - 1;
2397  // round the size up to cache line per thread-specific item
2398  size += CACHE_LINE - size % CACHE_LINE;
2399  KMP_ASSERT(data[i].reduce_comb != NULL); // combiner is mandatory
2400  arr[i].reduce_shar = data[i].reduce_shar;
2401  arr[i].reduce_size = size;
2402  arr[i].flags = data[i].flags;
2403  arr[i].reduce_comb = data[i].reduce_comb;
2404  arr[i].reduce_init = data[i].reduce_init;
2405  arr[i].reduce_fini = data[i].reduce_fini;
2406  __kmp_assign_orig<T>(arr[i], data[i]);
2407  if (!arr[i].flags.lazy_priv) {
2408  // allocate cache-line aligned block and fill it with zeros
2409  arr[i].reduce_priv = __kmp_allocate(nth * size);
2410  arr[i].reduce_pend = (char *)(arr[i].reduce_priv) + nth * size;
2411  if (arr[i].reduce_init != NULL) {
2412  // initialize all thread-specific items
2413  for (size_t j = 0; j < nth; ++j) {
2414  __kmp_call_init<T>(arr[i], j * size);
2415  }
2416  }
2417  } else {
2418  // only allocate space for pointers now,
2419  // objects will be lazily allocated/initialized if/when requested
2420  // note that __kmp_allocate zeroes the allocated memory
2421  arr[i].reduce_priv = __kmp_allocate(nth * sizeof(void *));
2422  }
2423  }
2424  tg->reduce_data = (void *)arr;
2425  tg->reduce_num_data = num;
2426  return (void *)tg;
2427 }
2428 
2443 void *__kmpc_task_reduction_init(int gtid, int num, void *data) {
2444  return __kmp_task_reduction_init(gtid, num, (kmp_task_red_input_t *)data);
2445 }
2446 
2459 void *__kmpc_taskred_init(int gtid, int num, void *data) {
2460  return __kmp_task_reduction_init(gtid, num, (kmp_taskred_input_t *)data);
2461 }
2462 
2463 // Copy task reduction data (except for shared pointers).
2464 template <typename T>
2465 void __kmp_task_reduction_init_copy(kmp_info_t *thr, int num, T *data,
2466  kmp_taskgroup_t *tg, void *reduce_data) {
2467  kmp_taskred_data_t *arr;
2468  KA_TRACE(20, ("__kmp_task_reduction_init_copy: Th %p, init taskgroup %p,"
2469  " from data %p\n",
2470  thr, tg, reduce_data));
2471  arr = (kmp_taskred_data_t *)__kmp_thread_malloc(
2472  thr, num * sizeof(kmp_taskred_data_t));
2473  // threads will share private copies, thunk routines, sizes, flags, etc.:
2474  KMP_MEMCPY(arr, reduce_data, num * sizeof(kmp_taskred_data_t));
2475  for (int i = 0; i < num; ++i) {
2476  arr[i].reduce_shar = data[i].reduce_shar; // init unique shared pointers
2477  }
2478  tg->reduce_data = (void *)arr;
2479  tg->reduce_num_data = num;
2480 }
2481 
2491 void *__kmpc_task_reduction_get_th_data(int gtid, void *tskgrp, void *data) {
2492  __kmp_assert_valid_gtid(gtid);
2493  kmp_info_t *thread = __kmp_threads[gtid];
2494  kmp_int32 nth = thread->th.th_team_nproc;
2495  if (nth == 1)
2496  return data; // nothing to do
2497 
2498  kmp_taskgroup_t *tg = (kmp_taskgroup_t *)tskgrp;
2499  if (tg == NULL)
2500  tg = thread->th.th_current_task->td_taskgroup;
2501  KMP_ASSERT(tg != NULL);
2502  kmp_taskred_data_t *arr = (kmp_taskred_data_t *)(tg->reduce_data);
2503  kmp_int32 num = tg->reduce_num_data;
2504  kmp_int32 tid = thread->th.th_info.ds.ds_tid;
2505 
2506  KMP_ASSERT(data != NULL);
2507  while (tg != NULL) {
2508  for (int i = 0; i < num; ++i) {
2509  if (!arr[i].flags.lazy_priv) {
2510  if (data == arr[i].reduce_shar ||
2511  (data >= arr[i].reduce_priv && data < arr[i].reduce_pend))
2512  return (char *)(arr[i].reduce_priv) + tid * arr[i].reduce_size;
2513  } else {
2514  // check shared location first
2515  void **p_priv = (void **)(arr[i].reduce_priv);
2516  if (data == arr[i].reduce_shar)
2517  goto found;
2518  // check if we get some thread specific location as parameter
2519  for (int j = 0; j < nth; ++j)
2520  if (data == p_priv[j])
2521  goto found;
2522  continue; // not found, continue search
2523  found:
2524  if (p_priv[tid] == NULL) {
2525  // allocate thread specific object lazily
2526  p_priv[tid] = __kmp_allocate(arr[i].reduce_size);
2527  if (arr[i].reduce_init != NULL) {
2528  if (arr[i].reduce_orig != NULL) { // new interface
2529  ((void (*)(void *, void *))arr[i].reduce_init)(
2530  p_priv[tid], arr[i].reduce_orig);
2531  } else { // old interface (single parameter)
2532  ((void (*)(void *))arr[i].reduce_init)(p_priv[tid]);
2533  }
2534  }
2535  }
2536  return p_priv[tid];
2537  }
2538  }
2539  tg = tg->parent;
2540  arr = (kmp_taskred_data_t *)(tg->reduce_data);
2541  num = tg->reduce_num_data;
2542  }
2543  KMP_ASSERT2(0, "Unknown task reduction item");
2544  return NULL; // ERROR, this line never executed
2545 }
2546 
2547 // Finalize task reduction.
2548 // Called from __kmpc_end_taskgroup()
2549 static void __kmp_task_reduction_fini(kmp_info_t *th, kmp_taskgroup_t *tg) {
2550  kmp_int32 nth = th->th.th_team_nproc;
2551  KMP_DEBUG_ASSERT(nth > 1); // should not be called if nth == 1
2552  kmp_taskred_data_t *arr = (kmp_taskred_data_t *)tg->reduce_data;
2553  kmp_int32 num = tg->reduce_num_data;
2554  for (int i = 0; i < num; ++i) {
2555  void *sh_data = arr[i].reduce_shar;
2556  void (*f_fini)(void *) = (void (*)(void *))(arr[i].reduce_fini);
2557  void (*f_comb)(void *, void *) =
2558  (void (*)(void *, void *))(arr[i].reduce_comb);
2559  if (!arr[i].flags.lazy_priv) {
2560  void *pr_data = arr[i].reduce_priv;
2561  size_t size = arr[i].reduce_size;
2562  for (int j = 0; j < nth; ++j) {
2563  void *priv_data = (char *)pr_data + j * size;
2564  f_comb(sh_data, priv_data); // combine results
2565  if (f_fini)
2566  f_fini(priv_data); // finalize if needed
2567  }
2568  } else {
2569  void **pr_data = (void **)(arr[i].reduce_priv);
2570  for (int j = 0; j < nth; ++j) {
2571  if (pr_data[j] != NULL) {
2572  f_comb(sh_data, pr_data[j]); // combine results
2573  if (f_fini)
2574  f_fini(pr_data[j]); // finalize if needed
2575  __kmp_free(pr_data[j]);
2576  }
2577  }
2578  }
2579  __kmp_free(arr[i].reduce_priv);
2580  }
2581  __kmp_thread_free(th, arr);
2582  tg->reduce_data = NULL;
2583  tg->reduce_num_data = 0;
2584 }
2585 
2586 // Cleanup task reduction data for parallel or worksharing,
2587 // do not touch task private data other threads still working with.
2588 // Called from __kmpc_end_taskgroup()
2589 static void __kmp_task_reduction_clean(kmp_info_t *th, kmp_taskgroup_t *tg) {
2590  __kmp_thread_free(th, tg->reduce_data);
2591  tg->reduce_data = NULL;
2592  tg->reduce_num_data = 0;
2593 }
2594 
2595 template <typename T>
2596 void *__kmp_task_reduction_modifier_init(ident_t *loc, int gtid, int is_ws,
2597  int num, T *data) {
2598  __kmp_assert_valid_gtid(gtid);
2599  kmp_info_t *thr = __kmp_threads[gtid];
2600  kmp_int32 nth = thr->th.th_team_nproc;
2601  __kmpc_taskgroup(loc, gtid); // form new taskgroup first
2602  if (nth == 1) {
2603  KA_TRACE(10,
2604  ("__kmpc_reduction_modifier_init: T#%d, tg %p, exiting nth=1\n",
2605  gtid, thr->th.th_current_task->td_taskgroup));
2606  return (void *)thr->th.th_current_task->td_taskgroup;
2607  }
2608  kmp_team_t *team = thr->th.th_team;
2609  void *reduce_data;
2610  kmp_taskgroup_t *tg;
2611  reduce_data = KMP_ATOMIC_LD_RLX(&team->t.t_tg_reduce_data[is_ws]);
2612  if (reduce_data == NULL &&
2613  __kmp_atomic_compare_store(&team->t.t_tg_reduce_data[is_ws], reduce_data,
2614  (void *)1)) {
2615  // single thread enters this block to initialize common reduction data
2616  KMP_DEBUG_ASSERT(reduce_data == NULL);
2617  // first initialize own data, then make a copy other threads can use
2618  tg = (kmp_taskgroup_t *)__kmp_task_reduction_init<T>(gtid, num, data);
2619  reduce_data = __kmp_thread_malloc(thr, num * sizeof(kmp_taskred_data_t));
2620  KMP_MEMCPY(reduce_data, tg->reduce_data, num * sizeof(kmp_taskred_data_t));
2621  // fini counters should be 0 at this point
2622  KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&team->t.t_tg_fini_counter[0]) == 0);
2623  KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&team->t.t_tg_fini_counter[1]) == 0);
2624  KMP_ATOMIC_ST_REL(&team->t.t_tg_reduce_data[is_ws], reduce_data);
2625  } else {
2626  while (
2627  (reduce_data = KMP_ATOMIC_LD_ACQ(&team->t.t_tg_reduce_data[is_ws])) ==
2628  (void *)1) { // wait for task reduction initialization
2629  KMP_CPU_PAUSE();
2630  }
2631  KMP_DEBUG_ASSERT(reduce_data > (void *)1); // should be valid pointer here
2632  tg = thr->th.th_current_task->td_taskgroup;
2633  __kmp_task_reduction_init_copy<T>(thr, num, data, tg, reduce_data);
2634  }
2635  return tg;
2636 }
2637 
2654 void *__kmpc_task_reduction_modifier_init(ident_t *loc, int gtid, int is_ws,
2655  int num, void *data) {
2656  return __kmp_task_reduction_modifier_init(loc, gtid, is_ws, num,
2657  (kmp_task_red_input_t *)data);
2658 }
2659 
2674 void *__kmpc_taskred_modifier_init(ident_t *loc, int gtid, int is_ws, int num,
2675  void *data) {
2676  return __kmp_task_reduction_modifier_init(loc, gtid, is_ws, num,
2677  (kmp_taskred_input_t *)data);
2678 }
2679 
2688 void __kmpc_task_reduction_modifier_fini(ident_t *loc, int gtid, int is_ws) {
2689  __kmpc_end_taskgroup(loc, gtid);
2690 }
2691 
2692 // __kmpc_taskgroup: Start a new taskgroup
2693 void __kmpc_taskgroup(ident_t *loc, int gtid) {
2694  __kmp_assert_valid_gtid(gtid);
2695  kmp_info_t *thread = __kmp_threads[gtid];
2696  kmp_taskdata_t *taskdata = thread->th.th_current_task;
2697  kmp_taskgroup_t *tg_new =
2698  (kmp_taskgroup_t *)__kmp_thread_malloc(thread, sizeof(kmp_taskgroup_t));
2699  KA_TRACE(10, ("__kmpc_taskgroup: T#%d loc=%p group=%p\n", gtid, loc, tg_new));
2700  KMP_ATOMIC_ST_RLX(&tg_new->count, 0);
2701  KMP_ATOMIC_ST_RLX(&tg_new->cancel_request, cancel_noreq);
2702  tg_new->parent = taskdata->td_taskgroup;
2703  tg_new->reduce_data = NULL;
2704  tg_new->reduce_num_data = 0;
2705  tg_new->gomp_data = NULL;
2706  taskdata->td_taskgroup = tg_new;
2707 
2708 #if OMPT_SUPPORT && OMPT_OPTIONAL
2709  if (UNLIKELY(ompt_enabled.ompt_callback_sync_region)) {
2710  void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2711  if (!codeptr)
2712  codeptr = OMPT_GET_RETURN_ADDRESS(0);
2713  kmp_team_t *team = thread->th.th_team;
2714  ompt_data_t my_task_data = taskdata->ompt_task_info.task_data;
2715  // FIXME: I think this is wrong for lwt!
2716  ompt_data_t my_parallel_data = team->t.ompt_team_info.parallel_data;
2717 
2718  ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
2719  ompt_sync_region_taskgroup, ompt_scope_begin, &(my_parallel_data),
2720  &(my_task_data), codeptr);
2721  }
2722 #endif
2723 }
2724 
2725 // __kmpc_end_taskgroup: Wait until all tasks generated by the current task
2726 // and its descendants are complete
2727 void __kmpc_end_taskgroup(ident_t *loc, int gtid) {
2728  __kmp_assert_valid_gtid(gtid);
2729  kmp_info_t *thread = __kmp_threads[gtid];
2730  kmp_taskdata_t *taskdata = thread->th.th_current_task;
2731  kmp_taskgroup_t *taskgroup = taskdata->td_taskgroup;
2732  int thread_finished = FALSE;
2733 
2734 #if OMPT_SUPPORT && OMPT_OPTIONAL
2735  kmp_team_t *team;
2736  ompt_data_t my_task_data;
2737  ompt_data_t my_parallel_data;
2738  void *codeptr = nullptr;
2739  if (UNLIKELY(ompt_enabled.enabled)) {
2740  team = thread->th.th_team;
2741  my_task_data = taskdata->ompt_task_info.task_data;
2742  // FIXME: I think this is wrong for lwt!
2743  my_parallel_data = team->t.ompt_team_info.parallel_data;
2744  codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2745  if (!codeptr)
2746  codeptr = OMPT_GET_RETURN_ADDRESS(0);
2747  }
2748 #endif
2749 
2750  KA_TRACE(10, ("__kmpc_end_taskgroup(enter): T#%d loc=%p\n", gtid, loc));
2751  KMP_DEBUG_ASSERT(taskgroup != NULL);
2752  KMP_SET_THREAD_STATE_BLOCK(TASKGROUP);
2753 
2754  if (__kmp_tasking_mode != tskm_immediate_exec) {
2755  // mark task as waiting not on a barrier
2756  taskdata->td_taskwait_counter += 1;
2757  taskdata->td_taskwait_ident = loc;
2758  taskdata->td_taskwait_thread = gtid + 1;
2759 #if USE_ITT_BUILD
2760  // For ITT the taskgroup wait is similar to taskwait until we need to
2761  // distinguish them
2762  void *itt_sync_obj = NULL;
2763 #if USE_ITT_NOTIFY
2764  KMP_ITT_TASKWAIT_STARTING(itt_sync_obj);
2765 #endif /* USE_ITT_NOTIFY */
2766 #endif /* USE_ITT_BUILD */
2767 
2768 #if OMPT_SUPPORT && OMPT_OPTIONAL
2769  if (UNLIKELY(ompt_enabled.ompt_callback_sync_region_wait)) {
2770  ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
2771  ompt_sync_region_taskgroup, ompt_scope_begin, &(my_parallel_data),
2772  &(my_task_data), codeptr);
2773  }
2774 #endif
2775 
2776  if (!taskdata->td_flags.team_serial ||
2777  (thread->th.th_task_team != NULL &&
2778  (thread->th.th_task_team->tt.tt_found_proxy_tasks ||
2779  thread->th.th_task_team->tt.tt_hidden_helper_task_encountered))) {
2780  kmp_flag_32<false, false> flag(
2781  RCAST(std::atomic<kmp_uint32> *, &(taskgroup->count)), 0U);
2782  while (KMP_ATOMIC_LD_ACQ(&taskgroup->count) != 0) {
2783  flag.execute_tasks(thread, gtid, FALSE,
2784  &thread_finished USE_ITT_BUILD_ARG(itt_sync_obj),
2785  __kmp_task_stealing_constraint);
2786  }
2787  }
2788  taskdata->td_taskwait_thread = -taskdata->td_taskwait_thread; // end waiting
2789 
2790 #if OMPT_SUPPORT && OMPT_OPTIONAL
2791  if (UNLIKELY(ompt_enabled.ompt_callback_sync_region_wait)) {
2792  ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
2793  ompt_sync_region_taskgroup, ompt_scope_end, &(my_parallel_data),
2794  &(my_task_data), codeptr);
2795  }
2796 #endif
2797 
2798 #if USE_ITT_BUILD
2799  KMP_ITT_TASKWAIT_FINISHED(itt_sync_obj);
2800  KMP_FSYNC_ACQUIRED(taskdata); // acquire self - sync with descendants
2801 #endif /* USE_ITT_BUILD */
2802  }
2803  KMP_DEBUG_ASSERT(taskgroup->count == 0);
2804 
2805  if (taskgroup->reduce_data != NULL &&
2806  !taskgroup->gomp_data) { // need to reduce?
2807  int cnt;
2808  void *reduce_data;
2809  kmp_team_t *t = thread->th.th_team;
2810  kmp_taskred_data_t *arr = (kmp_taskred_data_t *)taskgroup->reduce_data;
2811  // check if <priv> data of the first reduction variable shared for the team
2812  void *priv0 = arr[0].reduce_priv;
2813  if ((reduce_data = KMP_ATOMIC_LD_ACQ(&t->t.t_tg_reduce_data[0])) != NULL &&
2814  ((kmp_taskred_data_t *)reduce_data)[0].reduce_priv == priv0) {
2815  // finishing task reduction on parallel
2816  cnt = KMP_ATOMIC_INC(&t->t.t_tg_fini_counter[0]);
2817  if (cnt == thread->th.th_team_nproc - 1) {
2818  // we are the last thread passing __kmpc_reduction_modifier_fini()
2819  // finalize task reduction:
2820  __kmp_task_reduction_fini(thread, taskgroup);
2821  // cleanup fields in the team structure:
2822  // TODO: is relaxed store enough here (whole barrier should follow)?
2823  __kmp_thread_free(thread, reduce_data);
2824  KMP_ATOMIC_ST_REL(&t->t.t_tg_reduce_data[0], NULL);
2825  KMP_ATOMIC_ST_REL(&t->t.t_tg_fini_counter[0], 0);
2826  } else {
2827  // we are not the last thread passing __kmpc_reduction_modifier_fini(),
2828  // so do not finalize reduction, just clean own copy of the data
2829  __kmp_task_reduction_clean(thread, taskgroup);
2830  }
2831  } else if ((reduce_data = KMP_ATOMIC_LD_ACQ(&t->t.t_tg_reduce_data[1])) !=
2832  NULL &&
2833  ((kmp_taskred_data_t *)reduce_data)[0].reduce_priv == priv0) {
2834  // finishing task reduction on worksharing
2835  cnt = KMP_ATOMIC_INC(&t->t.t_tg_fini_counter[1]);
2836  if (cnt == thread->th.th_team_nproc - 1) {
2837  // we are the last thread passing __kmpc_reduction_modifier_fini()
2838  __kmp_task_reduction_fini(thread, taskgroup);
2839  // cleanup fields in team structure:
2840  // TODO: is relaxed store enough here (whole barrier should follow)?
2841  __kmp_thread_free(thread, reduce_data);
2842  KMP_ATOMIC_ST_REL(&t->t.t_tg_reduce_data[1], NULL);
2843  KMP_ATOMIC_ST_REL(&t->t.t_tg_fini_counter[1], 0);
2844  } else {
2845  // we are not the last thread passing __kmpc_reduction_modifier_fini(),
2846  // so do not finalize reduction, just clean own copy of the data
2847  __kmp_task_reduction_clean(thread, taskgroup);
2848  }
2849  } else {
2850  // finishing task reduction on taskgroup
2851  __kmp_task_reduction_fini(thread, taskgroup);
2852  }
2853  }
2854  // Restore parent taskgroup for the current task
2855  taskdata->td_taskgroup = taskgroup->parent;
2856  __kmp_thread_free(thread, taskgroup);
2857 
2858  KA_TRACE(10, ("__kmpc_end_taskgroup(exit): T#%d task %p finished waiting\n",
2859  gtid, taskdata));
2860 
2861 #if OMPT_SUPPORT && OMPT_OPTIONAL
2862  if (UNLIKELY(ompt_enabled.ompt_callback_sync_region)) {
2863  ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
2864  ompt_sync_region_taskgroup, ompt_scope_end, &(my_parallel_data),
2865  &(my_task_data), codeptr);
2866  }
2867 #endif
2868 }
2869 
2870 static kmp_task_t *__kmp_get_priority_task(kmp_int32 gtid,
2871  kmp_task_team_t *task_team,
2872  kmp_int32 is_constrained) {
2873  kmp_task_t *task = NULL;
2874  kmp_taskdata_t *taskdata;
2875  kmp_taskdata_t *current;
2876  kmp_thread_data_t *thread_data;
2877  int ntasks = task_team->tt.tt_num_task_pri;
2878  if (ntasks == 0) {
2879  KA_TRACE(
2880  20, ("__kmp_get_priority_task(exit #1): T#%d No tasks to get\n", gtid));
2881  return NULL;
2882  }
2883  do {
2884  // decrement num_tasks to "reserve" one task to get for execution
2885  if (__kmp_atomic_compare_store(&task_team->tt.tt_num_task_pri, ntasks,
2886  ntasks - 1))
2887  break;
2888  } while (ntasks > 0);
2889  if (ntasks == 0) {
2890  KA_TRACE(20, ("__kmp_get_priority_task(exit #2): T#%d No tasks to get\n",
2891  __kmp_get_gtid()));
2892  return NULL;
2893  }
2894  // We got a "ticket" to get a "reserved" priority task
2895  int deque_ntasks;
2896  kmp_task_pri_t *list = task_team->tt.tt_task_pri_list;
2897  do {
2898  KMP_ASSERT(list != NULL);
2899  thread_data = &list->td;
2900  __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
2901  deque_ntasks = thread_data->td.td_deque_ntasks;
2902  if (deque_ntasks == 0) {
2903  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
2904  KA_TRACE(20, ("__kmp_get_priority_task: T#%d No tasks to get from %p\n",
2905  __kmp_get_gtid(), thread_data));
2906  list = list->next;
2907  }
2908  } while (deque_ntasks == 0);
2909  KMP_DEBUG_ASSERT(deque_ntasks);
2910  int target = thread_data->td.td_deque_head;
2911  current = __kmp_threads[gtid]->th.th_current_task;
2912  taskdata = thread_data->td.td_deque[target];
2913  if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) {
2914  // Bump head pointer and Wrap.
2915  thread_data->td.td_deque_head =
2916  (target + 1) & TASK_DEQUE_MASK(thread_data->td);
2917  } else {
2918  if (!task_team->tt.tt_untied_task_encountered) {
2919  // The TSC does not allow to steal victim task
2920  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
2921  KA_TRACE(20, ("__kmp_get_priority_task(exit #3): T#%d could not get task "
2922  "from %p: task_team=%p ntasks=%d head=%u tail=%u\n",
2923  gtid, thread_data, task_team, deque_ntasks, target,
2924  thread_data->td.td_deque_tail));
2925  task_team->tt.tt_num_task_pri++; // atomic inc, restore value
2926  return NULL;
2927  }
2928  int i;
2929  // walk through the deque trying to steal any task
2930  taskdata = NULL;
2931  for (i = 1; i < deque_ntasks; ++i) {
2932  target = (target + 1) & TASK_DEQUE_MASK(thread_data->td);
2933  taskdata = thread_data->td.td_deque[target];
2934  if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) {
2935  break; // found task to execute
2936  } else {
2937  taskdata = NULL;
2938  }
2939  }
2940  if (taskdata == NULL) {
2941  // No appropriate candidate found to execute
2942  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
2943  KA_TRACE(
2944  10, ("__kmp_get_priority_task(exit #4): T#%d could not get task from "
2945  "%p: task_team=%p ntasks=%d head=%u tail=%u\n",
2946  gtid, thread_data, task_team, deque_ntasks,
2947  thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
2948  task_team->tt.tt_num_task_pri++; // atomic inc, restore value
2949  return NULL;
2950  }
2951  int prev = target;
2952  for (i = i + 1; i < deque_ntasks; ++i) {
2953  // shift remaining tasks in the deque left by 1
2954  target = (target + 1) & TASK_DEQUE_MASK(thread_data->td);
2955  thread_data->td.td_deque[prev] = thread_data->td.td_deque[target];
2956  prev = target;
2957  }
2958  KMP_DEBUG_ASSERT(
2959  thread_data->td.td_deque_tail ==
2960  (kmp_uint32)((target + 1) & TASK_DEQUE_MASK(thread_data->td)));
2961  thread_data->td.td_deque_tail = target; // tail -= 1 (wrapped))
2962  }
2963  thread_data->td.td_deque_ntasks = deque_ntasks - 1;
2964  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
2965  task = KMP_TASKDATA_TO_TASK(taskdata);
2966  return task;
2967 }
2968 
2969 // __kmp_remove_my_task: remove a task from my own deque
2970 static kmp_task_t *__kmp_remove_my_task(kmp_info_t *thread, kmp_int32 gtid,
2971  kmp_task_team_t *task_team,
2972  kmp_int32 is_constrained) {
2973  kmp_task_t *task;
2974  kmp_taskdata_t *taskdata;
2975  kmp_thread_data_t *thread_data;
2976  kmp_uint32 tail;
2977 
2978  KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
2979  KMP_DEBUG_ASSERT(task_team->tt.tt_threads_data !=
2980  NULL); // Caller should check this condition
2981 
2982  thread_data = &task_team->tt.tt_threads_data[__kmp_tid_from_gtid(gtid)];
2983 
2984  KA_TRACE(10, ("__kmp_remove_my_task(enter): T#%d ntasks=%d head=%u tail=%u\n",
2985  gtid, thread_data->td.td_deque_ntasks,
2986  thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
2987 
2988  if (TCR_4(thread_data->td.td_deque_ntasks) == 0) {
2989  KA_TRACE(10,
2990  ("__kmp_remove_my_task(exit #1): T#%d No tasks to remove: "
2991  "ntasks=%d head=%u tail=%u\n",
2992  gtid, thread_data->td.td_deque_ntasks,
2993  thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
2994  return NULL;
2995  }
2996 
2997  __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
2998 
2999  if (TCR_4(thread_data->td.td_deque_ntasks) == 0) {
3000  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
3001  KA_TRACE(10,
3002  ("__kmp_remove_my_task(exit #2): T#%d No tasks to remove: "
3003  "ntasks=%d head=%u tail=%u\n",
3004  gtid, thread_data->td.td_deque_ntasks,
3005  thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
3006  return NULL;
3007  }
3008 
3009  tail = (thread_data->td.td_deque_tail - 1) &
3010  TASK_DEQUE_MASK(thread_data->td); // Wrap index.
3011  taskdata = thread_data->td.td_deque[tail];
3012 
3013  if (!__kmp_task_is_allowed(gtid, is_constrained, taskdata,
3014  thread->th.th_current_task)) {
3015  // The TSC does not allow to steal victim task
3016  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
3017  KA_TRACE(10,
3018  ("__kmp_remove_my_task(exit #3): T#%d TSC blocks tail task: "
3019  "ntasks=%d head=%u tail=%u\n",
3020  gtid, thread_data->td.td_deque_ntasks,
3021  thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
3022  return NULL;
3023  }
3024 
3025  thread_data->td.td_deque_tail = tail;
3026  TCW_4(thread_data->td.td_deque_ntasks, thread_data->td.td_deque_ntasks - 1);
3027 
3028  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
3029 
3030  KA_TRACE(10, ("__kmp_remove_my_task(exit #4): T#%d task %p removed: "
3031  "ntasks=%d head=%u tail=%u\n",
3032  gtid, taskdata, thread_data->td.td_deque_ntasks,
3033  thread_data->td.td_deque_head, thread_data->td.td_deque_tail));
3034 
3035  task = KMP_TASKDATA_TO_TASK(taskdata);
3036  return task;
3037 }
3038 
3039 // __kmp_steal_task: remove a task from another thread's deque
3040 // Assume that calling thread has already checked existence of
3041 // task_team thread_data before calling this routine.
3042 static kmp_task_t *__kmp_steal_task(kmp_info_t *victim_thr, kmp_int32 gtid,
3043  kmp_task_team_t *task_team,
3044  std::atomic<kmp_int32> *unfinished_threads,
3045  int *thread_finished,
3046  kmp_int32 is_constrained) {
3047  kmp_task_t *task;
3048  kmp_taskdata_t *taskdata;
3049  kmp_taskdata_t *current;
3050  kmp_thread_data_t *victim_td, *threads_data;
3051  kmp_int32 target;
3052  kmp_int32 victim_tid;
3053 
3054  KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
3055 
3056  threads_data = task_team->tt.tt_threads_data;
3057  KMP_DEBUG_ASSERT(threads_data != NULL); // Caller should check this condition
3058 
3059  victim_tid = victim_thr->th.th_info.ds.ds_tid;
3060  victim_td = &threads_data[victim_tid];
3061 
3062  KA_TRACE(10, ("__kmp_steal_task(enter): T#%d try to steal from T#%d: "
3063  "task_team=%p ntasks=%d head=%u tail=%u\n",
3064  gtid, __kmp_gtid_from_thread(victim_thr), task_team,
3065  victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head,
3066  victim_td->td.td_deque_tail));
3067 
3068  if (TCR_4(victim_td->td.td_deque_ntasks) == 0) {
3069  KA_TRACE(10, ("__kmp_steal_task(exit #1): T#%d could not steal from T#%d: "
3070  "task_team=%p ntasks=%d head=%u tail=%u\n",
3071  gtid, __kmp_gtid_from_thread(victim_thr), task_team,
3072  victim_td->td.td_deque_ntasks, victim_td->td.td_deque_head,
3073  victim_td->td.td_deque_tail));
3074  return NULL;
3075  }
3076 
3077  __kmp_acquire_bootstrap_lock(&victim_td->td.td_deque_lock);
3078 
3079  int ntasks = TCR_4(victim_td->td.td_deque_ntasks);
3080  // Check again after we acquire the lock
3081  if (ntasks == 0) {
3082  __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock);
3083  KA_TRACE(10, ("__kmp_steal_task(exit #2): T#%d could not steal from T#%d: "
3084  "task_team=%p ntasks=%d head=%u tail=%u\n",
3085  gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks,
3086  victim_td->td.td_deque_head, victim_td->td.td_deque_tail));
3087  return NULL;
3088  }
3089 
3090  KMP_DEBUG_ASSERT(victim_td->td.td_deque != NULL);
3091  current = __kmp_threads[gtid]->th.th_current_task;
3092  taskdata = victim_td->td.td_deque[victim_td->td.td_deque_head];
3093  if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) {
3094  // Bump head pointer and Wrap.
3095  victim_td->td.td_deque_head =
3096  (victim_td->td.td_deque_head + 1) & TASK_DEQUE_MASK(victim_td->td);
3097  } else {
3098  if (!task_team->tt.tt_untied_task_encountered) {
3099  // The TSC does not allow to steal victim task
3100  __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock);
3101  KA_TRACE(10, ("__kmp_steal_task(exit #3): T#%d could not steal from "
3102  "T#%d: task_team=%p ntasks=%d head=%u tail=%u\n",
3103  gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks,
3104  victim_td->td.td_deque_head, victim_td->td.td_deque_tail));
3105  return NULL;
3106  }
3107  int i;
3108  // walk through victim's deque trying to steal any task
3109  target = victim_td->td.td_deque_head;
3110  taskdata = NULL;
3111  for (i = 1; i < ntasks; ++i) {
3112  target = (target + 1) & TASK_DEQUE_MASK(victim_td->td);
3113  taskdata = victim_td->td.td_deque[target];
3114  if (__kmp_task_is_allowed(gtid, is_constrained, taskdata, current)) {
3115  break; // found victim task
3116  } else {
3117  taskdata = NULL;
3118  }
3119  }
3120  if (taskdata == NULL) {
3121  // No appropriate candidate to steal found
3122  __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock);
3123  KA_TRACE(10, ("__kmp_steal_task(exit #4): T#%d could not steal from "
3124  "T#%d: task_team=%p ntasks=%d head=%u tail=%u\n",
3125  gtid, __kmp_gtid_from_thread(victim_thr), task_team, ntasks,
3126  victim_td->td.td_deque_head, victim_td->td.td_deque_tail));
3127  return NULL;
3128  }
3129  int prev = target;
3130  for (i = i + 1; i < ntasks; ++i) {
3131  // shift remaining tasks in the deque left by 1
3132  target = (target + 1) & TASK_DEQUE_MASK(victim_td->td);
3133  victim_td->td.td_deque[prev] = victim_td->td.td_deque[target];
3134  prev = target;
3135  }
3136  KMP_DEBUG_ASSERT(
3137  victim_td->td.td_deque_tail ==
3138  (kmp_uint32)((target + 1) & TASK_DEQUE_MASK(victim_td->td)));
3139  victim_td->td.td_deque_tail = target; // tail -= 1 (wrapped))
3140  }
3141  if (*thread_finished) {
3142  // We need to un-mark this victim as a finished victim. This must be done
3143  // before releasing the lock, or else other threads (starting with the
3144  // primary thread victim) might be prematurely released from the barrier!!!
3145 #if KMP_DEBUG
3146  kmp_int32 count =
3147 #endif
3148  KMP_ATOMIC_INC(unfinished_threads);
3149  KA_TRACE(
3150  20,
3151  ("__kmp_steal_task: T#%d inc unfinished_threads to %d: task_team=%p\n",
3152  gtid, count + 1, task_team));
3153  *thread_finished = FALSE;
3154  }
3155  TCW_4(victim_td->td.td_deque_ntasks, ntasks - 1);
3156 
3157  __kmp_release_bootstrap_lock(&victim_td->td.td_deque_lock);
3158 
3159  KMP_COUNT_BLOCK(TASK_stolen);
3160  KA_TRACE(10,
3161  ("__kmp_steal_task(exit #5): T#%d stole task %p from T#%d: "
3162  "task_team=%p ntasks=%d head=%u tail=%u\n",
3163  gtid, taskdata, __kmp_gtid_from_thread(victim_thr), task_team,
3164  ntasks, victim_td->td.td_deque_head, victim_td->td.td_deque_tail));
3165 
3166  task = KMP_TASKDATA_TO_TASK(taskdata);
3167  return task;
3168 }
3169 
3170 // __kmp_execute_tasks_template: Choose and execute tasks until either the
3171 // condition is statisfied (return true) or there are none left (return false).
3172 //
3173 // final_spin is TRUE if this is the spin at the release barrier.
3174 // thread_finished indicates whether the thread is finished executing all
3175 // the tasks it has on its deque, and is at the release barrier.
3176 // spinner is the location on which to spin.
3177 // spinner == NULL means only execute a single task and return.
3178 // checker is the value to check to terminate the spin.
3179 template <class C>
3180 static inline int __kmp_execute_tasks_template(
3181  kmp_info_t *thread, kmp_int32 gtid, C *flag, int final_spin,
3182  int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),
3183  kmp_int32 is_constrained) {
3184  kmp_task_team_t *task_team = thread->th.th_task_team;
3185  kmp_thread_data_t *threads_data;
3186  kmp_task_t *task;
3187  kmp_info_t *other_thread;
3188  kmp_taskdata_t *current_task = thread->th.th_current_task;
3189  std::atomic<kmp_int32> *unfinished_threads;
3190  kmp_int32 nthreads, victim_tid = -2, use_own_tasks = 1, new_victim = 0,
3191  tid = thread->th.th_info.ds.ds_tid;
3192 
3193  KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
3194  KMP_DEBUG_ASSERT(thread == __kmp_threads[gtid]);
3195 
3196  if (task_team == NULL || current_task == NULL)
3197  return FALSE;
3198 
3199  KA_TRACE(15, ("__kmp_execute_tasks_template(enter): T#%d final_spin=%d "
3200  "*thread_finished=%d\n",
3201  gtid, final_spin, *thread_finished));
3202 
3203  thread->th.th_reap_state = KMP_NOT_SAFE_TO_REAP;
3204  threads_data = (kmp_thread_data_t *)TCR_PTR(task_team->tt.tt_threads_data);
3205 
3206  KMP_DEBUG_ASSERT(threads_data != NULL);
3207 
3208  nthreads = task_team->tt.tt_nproc;
3209  unfinished_threads = &(task_team->tt.tt_unfinished_threads);
3210  KMP_DEBUG_ASSERT(nthreads > 1 || task_team->tt.tt_found_proxy_tasks ||
3211  task_team->tt.tt_hidden_helper_task_encountered);
3212  KMP_DEBUG_ASSERT(*unfinished_threads >= 0);
3213 
3214  while (1) { // Outer loop keeps trying to find tasks in case of single thread
3215  // getting tasks from target constructs
3216  while (1) { // Inner loop to find a task and execute it
3217  task = NULL;
3218  if (task_team->tt.tt_num_task_pri) { // get priority task first
3219  task = __kmp_get_priority_task(gtid, task_team, is_constrained);
3220  }
3221  if (task == NULL && use_own_tasks) { // check own queue next
3222  task = __kmp_remove_my_task(thread, gtid, task_team, is_constrained);
3223  }
3224  if ((task == NULL) && (nthreads > 1)) { // Steal a task finally
3225  int asleep = 1;
3226  use_own_tasks = 0;
3227  // Try to steal from the last place I stole from successfully.
3228  if (victim_tid == -2) { // haven't stolen anything yet
3229  victim_tid = threads_data[tid].td.td_deque_last_stolen;
3230  if (victim_tid !=
3231  -1) // if we have a last stolen from victim, get the thread
3232  other_thread = threads_data[victim_tid].td.td_thr;
3233  }
3234  if (victim_tid != -1) { // found last victim
3235  asleep = 0;
3236  } else if (!new_victim) { // no recent steals and we haven't already
3237  // used a new victim; select a random thread
3238  do { // Find a different thread to steal work from.
3239  // Pick a random thread. Initial plan was to cycle through all the
3240  // threads, and only return if we tried to steal from every thread,
3241  // and failed. Arch says that's not such a great idea.
3242  victim_tid = __kmp_get_random(thread) % (nthreads - 1);
3243  if (victim_tid >= tid) {
3244  ++victim_tid; // Adjusts random distribution to exclude self
3245  }
3246  // Found a potential victim
3247  other_thread = threads_data[victim_tid].td.td_thr;
3248  // There is a slight chance that __kmp_enable_tasking() did not wake
3249  // up all threads waiting at the barrier. If victim is sleeping,
3250  // then wake it up. Since we were going to pay the cache miss
3251  // penalty for referencing another thread's kmp_info_t struct
3252  // anyway,
3253  // the check shouldn't cost too much performance at this point. In
3254  // extra barrier mode, tasks do not sleep at the separate tasking
3255  // barrier, so this isn't a problem.
3256  asleep = 0;
3257  if ((__kmp_tasking_mode == tskm_task_teams) &&
3258  (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) &&
3259  (TCR_PTR(CCAST(void *, other_thread->th.th_sleep_loc)) !=
3260  NULL)) {
3261  asleep = 1;
3262  __kmp_null_resume_wrapper(other_thread);
3263  // A sleeping thread should not have any tasks on it's queue.
3264  // There is a slight possibility that it resumes, steals a task
3265  // from another thread, which spawns more tasks, all in the time
3266  // that it takes this thread to check => don't write an assertion
3267  // that the victim's queue is empty. Try stealing from a
3268  // different thread.
3269  }
3270  } while (asleep);
3271  }
3272 
3273  if (!asleep) {
3274  // We have a victim to try to steal from
3275  task = __kmp_steal_task(other_thread, gtid, task_team,
3276  unfinished_threads, thread_finished,
3277  is_constrained);
3278  }
3279  if (task != NULL) { // set last stolen to victim
3280  if (threads_data[tid].td.td_deque_last_stolen != victim_tid) {
3281  threads_data[tid].td.td_deque_last_stolen = victim_tid;
3282  // The pre-refactored code did not try more than 1 successful new
3283  // vicitm, unless the last one generated more local tasks;
3284  // new_victim keeps track of this
3285  new_victim = 1;
3286  }
3287  } else { // No tasks found; unset last_stolen
3288  KMP_CHECK_UPDATE(threads_data[tid].td.td_deque_last_stolen, -1);
3289  victim_tid = -2; // no successful victim found
3290  }
3291  }
3292 
3293  if (task == NULL)
3294  break; // break out of tasking loop
3295 
3296 // Found a task; execute it
3297 #if USE_ITT_BUILD && USE_ITT_NOTIFY
3298  if (__itt_sync_create_ptr || KMP_ITT_DEBUG) {
3299  if (itt_sync_obj == NULL) { // we are at fork barrier where we could not
3300  // get the object reliably
3301  itt_sync_obj = __kmp_itt_barrier_object(gtid, bs_forkjoin_barrier);
3302  }
3303  __kmp_itt_task_starting(itt_sync_obj);
3304  }
3305 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY */
3306  __kmp_invoke_task(gtid, task, current_task);
3307 #if USE_ITT_BUILD
3308  if (itt_sync_obj != NULL)
3309  __kmp_itt_task_finished(itt_sync_obj);
3310 #endif /* USE_ITT_BUILD */
3311  // If this thread is only partway through the barrier and the condition is
3312  // met, then return now, so that the barrier gather/release pattern can
3313  // proceed. If this thread is in the last spin loop in the barrier,
3314  // waiting to be released, we know that the termination condition will not
3315  // be satisfied, so don't waste any cycles checking it.
3316  if (flag == NULL || (!final_spin && flag->done_check())) {
3317  KA_TRACE(
3318  15,
3319  ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n",
3320  gtid));
3321  return TRUE;
3322  }
3323  if (thread->th.th_task_team == NULL) {
3324  break;
3325  }
3326  KMP_YIELD(__kmp_library == library_throughput); // Yield before next task
3327  // If execution of a stolen task results in more tasks being placed on our
3328  // run queue, reset use_own_tasks
3329  if (!use_own_tasks && TCR_4(threads_data[tid].td.td_deque_ntasks) != 0) {
3330  KA_TRACE(20, ("__kmp_execute_tasks_template: T#%d stolen task spawned "
3331  "other tasks, restart\n",
3332  gtid));
3333  use_own_tasks = 1;
3334  new_victim = 0;
3335  }
3336  }
3337 
3338  // The task source has been exhausted. If in final spin loop of barrier,
3339  // check if termination condition is satisfied. The work queue may be empty
3340  // but there might be proxy tasks still executing.
3341  if (final_spin &&
3342  KMP_ATOMIC_LD_ACQ(&current_task->td_incomplete_child_tasks) == 0) {
3343  // First, decrement the #unfinished threads, if that has not already been
3344  // done. This decrement might be to the spin location, and result in the
3345  // termination condition being satisfied.
3346  if (!*thread_finished) {
3347 #if KMP_DEBUG
3348  kmp_int32 count = -1 +
3349 #endif
3350  KMP_ATOMIC_DEC(unfinished_threads);
3351  KA_TRACE(20, ("__kmp_execute_tasks_template: T#%d dec "
3352  "unfinished_threads to %d task_team=%p\n",
3353  gtid, count, task_team));
3354  *thread_finished = TRUE;
3355  }
3356 
3357  // It is now unsafe to reference thread->th.th_team !!!
3358  // Decrementing task_team->tt.tt_unfinished_threads can allow the primary
3359  // thread to pass through the barrier, where it might reset each thread's
3360  // th.th_team field for the next parallel region. If we can steal more
3361  // work, we know that this has not happened yet.
3362  if (flag != NULL && flag->done_check()) {
3363  KA_TRACE(
3364  15,
3365  ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n",
3366  gtid));
3367  return TRUE;
3368  }
3369  }
3370 
3371  // If this thread's task team is NULL, primary thread has recognized that
3372  // there are no more tasks; bail out
3373  if (thread->th.th_task_team == NULL) {
3374  KA_TRACE(15,
3375  ("__kmp_execute_tasks_template: T#%d no more tasks\n", gtid));
3376  return FALSE;
3377  }
3378 
3379  // Check the flag again to see if it has already done in case to be trapped
3380  // into infinite loop when a if0 task depends on a hidden helper task
3381  // outside any parallel region. Detached tasks are not impacted in this case
3382  // because the only thread executing this function has to execute the proxy
3383  // task so it is in another code path that has the same check.
3384  if (flag == NULL || (!final_spin && flag->done_check())) {
3385  KA_TRACE(15,
3386  ("__kmp_execute_tasks_template: T#%d spin condition satisfied\n",
3387  gtid));
3388  return TRUE;
3389  }
3390 
3391  // We could be getting tasks from target constructs; if this is the only
3392  // thread, keep trying to execute tasks from own queue
3393  if (nthreads == 1 &&
3394  KMP_ATOMIC_LD_ACQ(&current_task->td_incomplete_child_tasks))
3395  use_own_tasks = 1;
3396  else {
3397  KA_TRACE(15,
3398  ("__kmp_execute_tasks_template: T#%d can't find work\n", gtid));
3399  return FALSE;
3400  }
3401  }
3402 }
3403 
3404 template <bool C, bool S>
3405 int __kmp_execute_tasks_32(
3406  kmp_info_t *thread, kmp_int32 gtid, kmp_flag_32<C, S> *flag, int final_spin,
3407  int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),
3408  kmp_int32 is_constrained) {
3409  return __kmp_execute_tasks_template(
3410  thread, gtid, flag, final_spin,
3411  thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
3412 }
3413 
3414 template <bool C, bool S>
3415 int __kmp_execute_tasks_64(
3416  kmp_info_t *thread, kmp_int32 gtid, kmp_flag_64<C, S> *flag, int final_spin,
3417  int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),
3418  kmp_int32 is_constrained) {
3419  return __kmp_execute_tasks_template(
3420  thread, gtid, flag, final_spin,
3421  thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
3422 }
3423 
3424 template <bool C, bool S>
3425 int __kmp_atomic_execute_tasks_64(
3426  kmp_info_t *thread, kmp_int32 gtid, kmp_atomic_flag_64<C, S> *flag,
3427  int final_spin, int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),
3428  kmp_int32 is_constrained) {
3429  return __kmp_execute_tasks_template(
3430  thread, gtid, flag, final_spin,
3431  thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
3432 }
3433 
3434 int __kmp_execute_tasks_oncore(
3435  kmp_info_t *thread, kmp_int32 gtid, kmp_flag_oncore *flag, int final_spin,
3436  int *thread_finished USE_ITT_BUILD_ARG(void *itt_sync_obj),
3437  kmp_int32 is_constrained) {
3438  return __kmp_execute_tasks_template(
3439  thread, gtid, flag, final_spin,
3440  thread_finished USE_ITT_BUILD_ARG(itt_sync_obj), is_constrained);
3441 }
3442 
3443 template int
3444 __kmp_execute_tasks_32<false, false>(kmp_info_t *, kmp_int32,
3445  kmp_flag_32<false, false> *, int,
3446  int *USE_ITT_BUILD_ARG(void *), kmp_int32);
3447 
3448 template int __kmp_execute_tasks_64<false, true>(kmp_info_t *, kmp_int32,
3449  kmp_flag_64<false, true> *,
3450  int,
3451  int *USE_ITT_BUILD_ARG(void *),
3452  kmp_int32);
3453 
3454 template int __kmp_execute_tasks_64<true, false>(kmp_info_t *, kmp_int32,
3455  kmp_flag_64<true, false> *,
3456  int,
3457  int *USE_ITT_BUILD_ARG(void *),
3458  kmp_int32);
3459 
3460 template int __kmp_atomic_execute_tasks_64<false, true>(
3461  kmp_info_t *, kmp_int32, kmp_atomic_flag_64<false, true> *, int,
3462  int *USE_ITT_BUILD_ARG(void *), kmp_int32);
3463 
3464 template int __kmp_atomic_execute_tasks_64<true, false>(
3465  kmp_info_t *, kmp_int32, kmp_atomic_flag_64<true, false> *, int,
3466  int *USE_ITT_BUILD_ARG(void *), kmp_int32);
3467 
3468 // __kmp_enable_tasking: Allocate task team and resume threads sleeping at the
3469 // next barrier so they can assist in executing enqueued tasks.
3470 // First thread in allocates the task team atomically.
3471 static void __kmp_enable_tasking(kmp_task_team_t *task_team,
3472  kmp_info_t *this_thr) {
3473  kmp_thread_data_t *threads_data;
3474  int nthreads, i, is_init_thread;
3475 
3476  KA_TRACE(10, ("__kmp_enable_tasking(enter): T#%d\n",
3477  __kmp_gtid_from_thread(this_thr)));
3478 
3479  KMP_DEBUG_ASSERT(task_team != NULL);
3480  KMP_DEBUG_ASSERT(this_thr->th.th_team != NULL);
3481 
3482  nthreads = task_team->tt.tt_nproc;
3483  KMP_DEBUG_ASSERT(nthreads > 0);
3484  KMP_DEBUG_ASSERT(nthreads == this_thr->th.th_team->t.t_nproc);
3485 
3486  // Allocate or increase the size of threads_data if necessary
3487  is_init_thread = __kmp_realloc_task_threads_data(this_thr, task_team);
3488 
3489  if (!is_init_thread) {
3490  // Some other thread already set up the array.
3491  KA_TRACE(
3492  20,
3493  ("__kmp_enable_tasking(exit): T#%d: threads array already set up.\n",
3494  __kmp_gtid_from_thread(this_thr)));
3495  return;
3496  }
3497  threads_data = (kmp_thread_data_t *)TCR_PTR(task_team->tt.tt_threads_data);
3498  KMP_DEBUG_ASSERT(threads_data != NULL);
3499 
3500  if (__kmp_tasking_mode == tskm_task_teams &&
3501  (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME)) {
3502  // Release any threads sleeping at the barrier, so that they can steal
3503  // tasks and execute them. In extra barrier mode, tasks do not sleep
3504  // at the separate tasking barrier, so this isn't a problem.
3505  for (i = 0; i < nthreads; i++) {
3506  void *sleep_loc;
3507  kmp_info_t *thread = threads_data[i].td.td_thr;
3508 
3509  if (i == this_thr->th.th_info.ds.ds_tid) {
3510  continue;
3511  }
3512  // Since we haven't locked the thread's suspend mutex lock at this
3513  // point, there is a small window where a thread might be putting
3514  // itself to sleep, but hasn't set the th_sleep_loc field yet.
3515  // To work around this, __kmp_execute_tasks_template() periodically checks
3516  // see if other threads are sleeping (using the same random mechanism that
3517  // is used for task stealing) and awakens them if they are.
3518  if ((sleep_loc = TCR_PTR(CCAST(void *, thread->th.th_sleep_loc))) !=
3519  NULL) {
3520  KF_TRACE(50, ("__kmp_enable_tasking: T#%d waking up thread T#%d\n",
3521  __kmp_gtid_from_thread(this_thr),
3522  __kmp_gtid_from_thread(thread)));
3523  __kmp_null_resume_wrapper(thread);
3524  } else {
3525  KF_TRACE(50, ("__kmp_enable_tasking: T#%d don't wake up thread T#%d\n",
3526  __kmp_gtid_from_thread(this_thr),
3527  __kmp_gtid_from_thread(thread)));
3528  }
3529  }
3530  }
3531 
3532  KA_TRACE(10, ("__kmp_enable_tasking(exit): T#%d\n",
3533  __kmp_gtid_from_thread(this_thr)));
3534 }
3535 
3536 /* // TODO: Check the comment consistency
3537  * Utility routines for "task teams". A task team (kmp_task_t) is kind of
3538  * like a shadow of the kmp_team_t data struct, with a different lifetime.
3539  * After a child * thread checks into a barrier and calls __kmp_release() from
3540  * the particular variant of __kmp_<barrier_kind>_barrier_gather(), it can no
3541  * longer assume that the kmp_team_t structure is intact (at any moment, the
3542  * primary thread may exit the barrier code and free the team data structure,
3543  * and return the threads to the thread pool).
3544  *
3545  * This does not work with the tasking code, as the thread is still
3546  * expected to participate in the execution of any tasks that may have been
3547  * spawned my a member of the team, and the thread still needs access to all
3548  * to each thread in the team, so that it can steal work from it.
3549  *
3550  * Enter the existence of the kmp_task_team_t struct. It employs a reference
3551  * counting mechanism, and is allocated by the primary thread before calling
3552  * __kmp_<barrier_kind>_release, and then is release by the last thread to
3553  * exit __kmp_<barrier_kind>_release at the next barrier. I.e. the lifetimes
3554  * of the kmp_task_team_t structs for consecutive barriers can overlap
3555  * (and will, unless the primary thread is the last thread to exit the barrier
3556  * release phase, which is not typical). The existence of such a struct is
3557  * useful outside the context of tasking.
3558  *
3559  * We currently use the existence of the threads array as an indicator that
3560  * tasks were spawned since the last barrier. If the structure is to be
3561  * useful outside the context of tasking, then this will have to change, but
3562  * not setting the field minimizes the performance impact of tasking on
3563  * barriers, when no explicit tasks were spawned (pushed, actually).
3564  */
3565 
3566 static kmp_task_team_t *__kmp_free_task_teams =
3567  NULL; // Free list for task_team data structures
3568 // Lock for task team data structures
3569 kmp_bootstrap_lock_t __kmp_task_team_lock =
3570  KMP_BOOTSTRAP_LOCK_INITIALIZER(__kmp_task_team_lock);
3571 
3572 // __kmp_alloc_task_deque:
3573 // Allocates a task deque for a particular thread, and initialize the necessary
3574 // data structures relating to the deque. This only happens once per thread
3575 // per task team since task teams are recycled. No lock is needed during
3576 // allocation since each thread allocates its own deque.
3577 static void __kmp_alloc_task_deque(kmp_info_t *thread,
3578  kmp_thread_data_t *thread_data) {
3579  __kmp_init_bootstrap_lock(&thread_data->td.td_deque_lock);
3580  KMP_DEBUG_ASSERT(thread_data->td.td_deque == NULL);
3581 
3582  // Initialize last stolen task field to "none"
3583  thread_data->td.td_deque_last_stolen = -1;
3584 
3585  KMP_DEBUG_ASSERT(TCR_4(thread_data->td.td_deque_ntasks) == 0);
3586  KMP_DEBUG_ASSERT(thread_data->td.td_deque_head == 0);
3587  KMP_DEBUG_ASSERT(thread_data->td.td_deque_tail == 0);
3588 
3589  KE_TRACE(
3590  10,
3591  ("__kmp_alloc_task_deque: T#%d allocating deque[%d] for thread_data %p\n",
3592  __kmp_gtid_from_thread(thread), INITIAL_TASK_DEQUE_SIZE, thread_data));
3593  // Allocate space for task deque, and zero the deque
3594  // Cannot use __kmp_thread_calloc() because threads not around for
3595  // kmp_reap_task_team( ).
3596  thread_data->td.td_deque = (kmp_taskdata_t **)__kmp_allocate(
3597  INITIAL_TASK_DEQUE_SIZE * sizeof(kmp_taskdata_t *));
3598  thread_data->td.td_deque_size = INITIAL_TASK_DEQUE_SIZE;
3599 }
3600 
3601 // __kmp_free_task_deque:
3602 // Deallocates a task deque for a particular thread. Happens at library
3603 // deallocation so don't need to reset all thread data fields.
3604 static void __kmp_free_task_deque(kmp_thread_data_t *thread_data) {
3605  if (thread_data->td.td_deque != NULL) {
3606  __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
3607  TCW_4(thread_data->td.td_deque_ntasks, 0);
3608  __kmp_free(thread_data->td.td_deque);
3609  thread_data->td.td_deque = NULL;
3610  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
3611  }
3612 
3613 #ifdef BUILD_TIED_TASK_STACK
3614  // GEH: Figure out what to do here for td_susp_tied_tasks
3615  if (thread_data->td.td_susp_tied_tasks.ts_entries != TASK_STACK_EMPTY) {
3616  __kmp_free_task_stack(__kmp_thread_from_gtid(gtid), thread_data);
3617  }
3618 #endif // BUILD_TIED_TASK_STACK
3619 }
3620 
3621 // __kmp_realloc_task_threads_data:
3622 // Allocates a threads_data array for a task team, either by allocating an
3623 // initial array or enlarging an existing array. Only the first thread to get
3624 // the lock allocs or enlarges the array and re-initializes the array elements.
3625 // That thread returns "TRUE", the rest return "FALSE".
3626 // Assumes that the new array size is given by task_team -> tt.tt_nproc.
3627 // The current size is given by task_team -> tt.tt_max_threads.
3628 static int __kmp_realloc_task_threads_data(kmp_info_t *thread,
3629  kmp_task_team_t *task_team) {
3630  kmp_thread_data_t **threads_data_p;
3631  kmp_int32 nthreads, maxthreads;
3632  int is_init_thread = FALSE;
3633 
3634  if (TCR_4(task_team->tt.tt_found_tasks)) {
3635  // Already reallocated and initialized.
3636  return FALSE;
3637  }
3638 
3639  threads_data_p = &task_team->tt.tt_threads_data;
3640  nthreads = task_team->tt.tt_nproc;
3641  maxthreads = task_team->tt.tt_max_threads;
3642 
3643  // All threads must lock when they encounter the first task of the implicit
3644  // task region to make sure threads_data fields are (re)initialized before
3645  // used.
3646  __kmp_acquire_bootstrap_lock(&task_team->tt.tt_threads_lock);
3647 
3648  if (!TCR_4(task_team->tt.tt_found_tasks)) {
3649  // first thread to enable tasking
3650  kmp_team_t *team = thread->th.th_team;
3651  int i;
3652 
3653  is_init_thread = TRUE;
3654  if (maxthreads < nthreads) {
3655 
3656  if (*threads_data_p != NULL) {
3657  kmp_thread_data_t *old_data = *threads_data_p;
3658  kmp_thread_data_t *new_data = NULL;
3659 
3660  KE_TRACE(
3661  10,
3662  ("__kmp_realloc_task_threads_data: T#%d reallocating "
3663  "threads data for task_team %p, new_size = %d, old_size = %d\n",
3664  __kmp_gtid_from_thread(thread), task_team, nthreads, maxthreads));
3665  // Reallocate threads_data to have more elements than current array
3666  // Cannot use __kmp_thread_realloc() because threads not around for
3667  // kmp_reap_task_team( ). Note all new array entries are initialized
3668  // to zero by __kmp_allocate().
3669  new_data = (kmp_thread_data_t *)__kmp_allocate(
3670  nthreads * sizeof(kmp_thread_data_t));
3671  // copy old data to new data
3672  KMP_MEMCPY_S((void *)new_data, nthreads * sizeof(kmp_thread_data_t),
3673  (void *)old_data, maxthreads * sizeof(kmp_thread_data_t));
3674 
3675 #ifdef BUILD_TIED_TASK_STACK
3676  // GEH: Figure out if this is the right thing to do
3677  for (i = maxthreads; i < nthreads; i++) {
3678  kmp_thread_data_t *thread_data = &(*threads_data_p)[i];
3679  __kmp_init_task_stack(__kmp_gtid_from_thread(thread), thread_data);
3680  }
3681 #endif // BUILD_TIED_TASK_STACK
3682  // Install the new data and free the old data
3683  (*threads_data_p) = new_data;
3684  __kmp_free(old_data);
3685  } else {
3686  KE_TRACE(10, ("__kmp_realloc_task_threads_data: T#%d allocating "
3687  "threads data for task_team %p, size = %d\n",
3688  __kmp_gtid_from_thread(thread), task_team, nthreads));
3689  // Make the initial allocate for threads_data array, and zero entries
3690  // Cannot use __kmp_thread_calloc() because threads not around for
3691  // kmp_reap_task_team( ).
3692  *threads_data_p = (kmp_thread_data_t *)__kmp_allocate(
3693  nthreads * sizeof(kmp_thread_data_t));
3694 #ifdef BUILD_TIED_TASK_STACK
3695  // GEH: Figure out if this is the right thing to do
3696  for (i = 0; i < nthreads; i++) {
3697  kmp_thread_data_t *thread_data = &(*threads_data_p)[i];
3698  __kmp_init_task_stack(__kmp_gtid_from_thread(thread), thread_data);
3699  }
3700 #endif // BUILD_TIED_TASK_STACK
3701  }
3702  task_team->tt.tt_max_threads = nthreads;
3703  } else {
3704  // If array has (more than) enough elements, go ahead and use it
3705  KMP_DEBUG_ASSERT(*threads_data_p != NULL);
3706  }
3707 
3708  // initialize threads_data pointers back to thread_info structures
3709  for (i = 0; i < nthreads; i++) {
3710  kmp_thread_data_t *thread_data = &(*threads_data_p)[i];
3711  thread_data->td.td_thr = team->t.t_threads[i];
3712 
3713  if (thread_data->td.td_deque_last_stolen >= nthreads) {
3714  // The last stolen field survives across teams / barrier, and the number
3715  // of threads may have changed. It's possible (likely?) that a new
3716  // parallel region will exhibit the same behavior as previous region.
3717  thread_data->td.td_deque_last_stolen = -1;
3718  }
3719  }
3720 
3721  KMP_MB();
3722  TCW_SYNC_4(task_team->tt.tt_found_tasks, TRUE);
3723  }
3724 
3725  __kmp_release_bootstrap_lock(&task_team->tt.tt_threads_lock);
3726  return is_init_thread;
3727 }
3728 
3729 // __kmp_free_task_threads_data:
3730 // Deallocates a threads_data array for a task team, including any attached
3731 // tasking deques. Only occurs at library shutdown.
3732 static void __kmp_free_task_threads_data(kmp_task_team_t *task_team) {
3733  __kmp_acquire_bootstrap_lock(&task_team->tt.tt_threads_lock);
3734  if (task_team->tt.tt_threads_data != NULL) {
3735  int i;
3736  for (i = 0; i < task_team->tt.tt_max_threads; i++) {
3737  __kmp_free_task_deque(&task_team->tt.tt_threads_data[i]);
3738  }
3739  __kmp_free(task_team->tt.tt_threads_data);
3740  task_team->tt.tt_threads_data = NULL;
3741  }
3742  __kmp_release_bootstrap_lock(&task_team->tt.tt_threads_lock);
3743 }
3744 
3745 // __kmp_free_task_pri_list:
3746 // Deallocates tasking deques used for priority tasks.
3747 // Only occurs at library shutdown.
3748 static void __kmp_free_task_pri_list(kmp_task_team_t *task_team) {
3749  __kmp_acquire_bootstrap_lock(&task_team->tt.tt_task_pri_lock);
3750  if (task_team->tt.tt_task_pri_list != NULL) {
3751  kmp_task_pri_t *list = task_team->tt.tt_task_pri_list;
3752  while (list != NULL) {
3753  kmp_task_pri_t *next = list->next;
3754  __kmp_free_task_deque(&list->td);
3755  __kmp_free(list);
3756  list = next;
3757  }
3758  task_team->tt.tt_task_pri_list = NULL;
3759  }
3760  __kmp_release_bootstrap_lock(&task_team->tt.tt_task_pri_lock);
3761 }
3762 
3763 // __kmp_allocate_task_team:
3764 // Allocates a task team associated with a specific team, taking it from
3765 // the global task team free list if possible. Also initializes data
3766 // structures.
3767 static kmp_task_team_t *__kmp_allocate_task_team(kmp_info_t *thread,
3768  kmp_team_t *team) {
3769  kmp_task_team_t *task_team = NULL;
3770  int nthreads;
3771 
3772  KA_TRACE(20, ("__kmp_allocate_task_team: T#%d entering; team = %p\n",
3773  (thread ? __kmp_gtid_from_thread(thread) : -1), team));
3774 
3775  if (TCR_PTR(__kmp_free_task_teams) != NULL) {
3776  // Take a task team from the task team pool
3777  __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock);
3778  if (__kmp_free_task_teams != NULL) {
3779  task_team = __kmp_free_task_teams;
3780  TCW_PTR(__kmp_free_task_teams, task_team->tt.tt_next);
3781  task_team->tt.tt_next = NULL;
3782  }
3783  __kmp_release_bootstrap_lock(&__kmp_task_team_lock);
3784  }
3785 
3786  if (task_team == NULL) {
3787  KE_TRACE(10, ("__kmp_allocate_task_team: T#%d allocating "
3788  "task team for team %p\n",
3789  __kmp_gtid_from_thread(thread), team));
3790  // Allocate a new task team if one is not available. Cannot use
3791  // __kmp_thread_malloc because threads not around for kmp_reap_task_team.
3792  task_team = (kmp_task_team_t *)__kmp_allocate(sizeof(kmp_task_team_t));
3793  __kmp_init_bootstrap_lock(&task_team->tt.tt_threads_lock);
3794  __kmp_init_bootstrap_lock(&task_team->tt.tt_task_pri_lock);
3795 #if USE_ITT_BUILD && USE_ITT_NOTIFY && KMP_DEBUG
3796  // suppress race conditions detection on synchronization flags in debug mode
3797  // this helps to analyze library internals eliminating false positives
3798  __itt_suppress_mark_range(
3799  __itt_suppress_range, __itt_suppress_threading_errors,
3800  &task_team->tt.tt_found_tasks, sizeof(task_team->tt.tt_found_tasks));
3801  __itt_suppress_mark_range(__itt_suppress_range,
3802  __itt_suppress_threading_errors,
3803  CCAST(kmp_uint32 *, &task_team->tt.tt_active),
3804  sizeof(task_team->tt.tt_active));
3805 #endif /* USE_ITT_BUILD && USE_ITT_NOTIFY && KMP_DEBUG */
3806  // Note: __kmp_allocate zeroes returned memory, othewise we would need:
3807  // task_team->tt.tt_threads_data = NULL;
3808  // task_team->tt.tt_max_threads = 0;
3809  // task_team->tt.tt_next = NULL;
3810  }
3811 
3812  TCW_4(task_team->tt.tt_found_tasks, FALSE);
3813  TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE);
3814  TCW_4(task_team->tt.tt_hidden_helper_task_encountered, FALSE);
3815  task_team->tt.tt_nproc = nthreads = team->t.t_nproc;
3816 
3817  KMP_ATOMIC_ST_REL(&task_team->tt.tt_unfinished_threads, nthreads);
3818  TCW_4(task_team->tt.tt_hidden_helper_task_encountered, FALSE);
3819  TCW_4(task_team->tt.tt_active, TRUE);
3820 
3821  KA_TRACE(20, ("__kmp_allocate_task_team: T#%d exiting; task_team = %p "
3822  "unfinished_threads init'd to %d\n",
3823  (thread ? __kmp_gtid_from_thread(thread) : -1), task_team,
3824  KMP_ATOMIC_LD_RLX(&task_team->tt.tt_unfinished_threads)));
3825  return task_team;
3826 }
3827 
3828 // __kmp_free_task_team:
3829 // Frees the task team associated with a specific thread, and adds it
3830 // to the global task team free list.
3831 void __kmp_free_task_team(kmp_info_t *thread, kmp_task_team_t *task_team) {
3832  KA_TRACE(20, ("__kmp_free_task_team: T#%d task_team = %p\n",
3833  thread ? __kmp_gtid_from_thread(thread) : -1, task_team));
3834 
3835  // Put task team back on free list
3836  __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock);
3837 
3838  KMP_DEBUG_ASSERT(task_team->tt.tt_next == NULL);
3839  task_team->tt.tt_next = __kmp_free_task_teams;
3840  TCW_PTR(__kmp_free_task_teams, task_team);
3841 
3842  __kmp_release_bootstrap_lock(&__kmp_task_team_lock);
3843 }
3844 
3845 // __kmp_reap_task_teams:
3846 // Free all the task teams on the task team free list.
3847 // Should only be done during library shutdown.
3848 // Cannot do anything that needs a thread structure or gtid since they are
3849 // already gone.
3850 void __kmp_reap_task_teams(void) {
3851  kmp_task_team_t *task_team;
3852 
3853  if (TCR_PTR(__kmp_free_task_teams) != NULL) {
3854  // Free all task_teams on the free list
3855  __kmp_acquire_bootstrap_lock(&__kmp_task_team_lock);
3856  while ((task_team = __kmp_free_task_teams) != NULL) {
3857  __kmp_free_task_teams = task_team->tt.tt_next;
3858  task_team->tt.tt_next = NULL;
3859 
3860  // Free threads_data if necessary
3861  if (task_team->tt.tt_threads_data != NULL) {
3862  __kmp_free_task_threads_data(task_team);
3863  }
3864  if (task_team->tt.tt_task_pri_list != NULL) {
3865  __kmp_free_task_pri_list(task_team);
3866  }
3867  __kmp_free(task_team);
3868  }
3869  __kmp_release_bootstrap_lock(&__kmp_task_team_lock);
3870  }
3871 }
3872 
3873 // __kmp_wait_to_unref_task_teams:
3874 // Some threads could still be in the fork barrier release code, possibly
3875 // trying to steal tasks. Wait for each thread to unreference its task team.
3876 void __kmp_wait_to_unref_task_teams(void) {
3877  kmp_info_t *thread;
3878  kmp_uint32 spins;
3879  kmp_uint64 time;
3880  int done;
3881 
3882  KMP_INIT_YIELD(spins);
3883  KMP_INIT_BACKOFF(time);
3884 
3885  for (;;) {
3886  done = TRUE;
3887 
3888  // TODO: GEH - this may be is wrong because some sync would be necessary
3889  // in case threads are added to the pool during the traversal. Need to
3890  // verify that lock for thread pool is held when calling this routine.
3891  for (thread = CCAST(kmp_info_t *, __kmp_thread_pool); thread != NULL;
3892  thread = thread->th.th_next_pool) {
3893 #if KMP_OS_WINDOWS
3894  DWORD exit_val;
3895 #endif
3896  if (TCR_PTR(thread->th.th_task_team) == NULL) {
3897  KA_TRACE(10, ("__kmp_wait_to_unref_task_team: T#%d task_team == NULL\n",
3898  __kmp_gtid_from_thread(thread)));
3899  continue;
3900  }
3901 #if KMP_OS_WINDOWS
3902  // TODO: GEH - add this check for Linux* OS / OS X* as well?
3903  if (!__kmp_is_thread_alive(thread, &exit_val)) {
3904  thread->th.th_task_team = NULL;
3905  continue;
3906  }
3907 #endif
3908 
3909  done = FALSE; // Because th_task_team pointer is not NULL for this thread
3910 
3911  KA_TRACE(10, ("__kmp_wait_to_unref_task_team: Waiting for T#%d to "
3912  "unreference task_team\n",
3913  __kmp_gtid_from_thread(thread)));
3914 
3915  if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
3916  void *sleep_loc;
3917  // If the thread is sleeping, awaken it.
3918  if ((sleep_loc = TCR_PTR(CCAST(void *, thread->th.th_sleep_loc))) !=
3919  NULL) {
3920  KA_TRACE(
3921  10,
3922  ("__kmp_wait_to_unref_task_team: T#%d waking up thread T#%d\n",
3923  __kmp_gtid_from_thread(thread), __kmp_gtid_from_thread(thread)));
3924  __kmp_null_resume_wrapper(thread);
3925  }
3926  }
3927  }
3928  if (done) {
3929  break;
3930  }
3931 
3932  // If oversubscribed or have waited a bit, yield.
3933  KMP_YIELD_OVERSUB_ELSE_SPIN(spins, time);
3934  }
3935 }
3936 
3937 // __kmp_task_team_setup: Create a task_team for the current team, but use
3938 // an already created, unused one if it already exists.
3939 void __kmp_task_team_setup(kmp_info_t *this_thr, kmp_team_t *team, int always) {
3940  KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
3941 
3942  // If this task_team hasn't been created yet, allocate it. It will be used in
3943  // the region after the next.
3944  // If it exists, it is the current task team and shouldn't be touched yet as
3945  // it may still be in use.
3946  if (team->t.t_task_team[this_thr->th.th_task_state] == NULL &&
3947  (always || team->t.t_nproc > 1)) {
3948  team->t.t_task_team[this_thr->th.th_task_state] =
3949  __kmp_allocate_task_team(this_thr, team);
3950  KA_TRACE(20, ("__kmp_task_team_setup: Primary T#%d created new task_team %p"
3951  " for team %d at parity=%d\n",
3952  __kmp_gtid_from_thread(this_thr),
3953  team->t.t_task_team[this_thr->th.th_task_state], team->t.t_id,
3954  this_thr->th.th_task_state));
3955  }
3956 
3957  // After threads exit the release, they will call sync, and then point to this
3958  // other task_team; make sure it is allocated and properly initialized. As
3959  // threads spin in the barrier release phase, they will continue to use the
3960  // previous task_team struct(above), until they receive the signal to stop
3961  // checking for tasks (they can't safely reference the kmp_team_t struct,
3962  // which could be reallocated by the primary thread). No task teams are formed
3963  // for serialized teams.
3964  if (team->t.t_nproc > 1) {
3965  int other_team = 1 - this_thr->th.th_task_state;
3966  KMP_DEBUG_ASSERT(other_team >= 0 && other_team < 2);
3967  if (team->t.t_task_team[other_team] == NULL) { // setup other team as well
3968  team->t.t_task_team[other_team] =
3969  __kmp_allocate_task_team(this_thr, team);
3970  KA_TRACE(20, ("__kmp_task_team_setup: Primary T#%d created second new "
3971  "task_team %p for team %d at parity=%d\n",
3972  __kmp_gtid_from_thread(this_thr),
3973  team->t.t_task_team[other_team], team->t.t_id, other_team));
3974  } else { // Leave the old task team struct in place for the upcoming region;
3975  // adjust as needed
3976  kmp_task_team_t *task_team = team->t.t_task_team[other_team];
3977  if (!task_team->tt.tt_active ||
3978  team->t.t_nproc != task_team->tt.tt_nproc) {
3979  TCW_4(task_team->tt.tt_nproc, team->t.t_nproc);
3980  TCW_4(task_team->tt.tt_found_tasks, FALSE);
3981  TCW_4(task_team->tt.tt_found_proxy_tasks, FALSE);
3982  TCW_4(task_team->tt.tt_hidden_helper_task_encountered, FALSE);
3983  KMP_ATOMIC_ST_REL(&task_team->tt.tt_unfinished_threads,
3984  team->t.t_nproc);
3985  TCW_4(task_team->tt.tt_active, TRUE);
3986  }
3987  // if team size has changed, the first thread to enable tasking will
3988  // realloc threads_data if necessary
3989  KA_TRACE(20, ("__kmp_task_team_setup: Primary T#%d reset next task_team "
3990  "%p for team %d at parity=%d\n",
3991  __kmp_gtid_from_thread(this_thr),
3992  team->t.t_task_team[other_team], team->t.t_id, other_team));
3993  }
3994  }
3995 
3996  // For regular thread, task enabling should be called when the task is going
3997  // to be pushed to a dequeue. However, for the hidden helper thread, we need
3998  // it ahead of time so that some operations can be performed without race
3999  // condition.
4000  if (this_thr == __kmp_hidden_helper_main_thread) {
4001  for (int i = 0; i < 2; ++i) {
4002  kmp_task_team_t *task_team = team->t.t_task_team[i];
4003  if (KMP_TASKING_ENABLED(task_team)) {
4004  continue;
4005  }
4006  __kmp_enable_tasking(task_team, this_thr);
4007  for (int j = 0; j < task_team->tt.tt_nproc; ++j) {
4008  kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[j];
4009  if (thread_data->td.td_deque == NULL) {
4010  __kmp_alloc_task_deque(__kmp_hidden_helper_threads[j], thread_data);
4011  }
4012  }
4013  }
4014  }
4015 }
4016 
4017 // __kmp_task_team_sync: Propagation of task team data from team to threads
4018 // which happens just after the release phase of a team barrier. This may be
4019 // called by any thread, but only for teams with # threads > 1.
4020 void __kmp_task_team_sync(kmp_info_t *this_thr, kmp_team_t *team) {
4021  KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
4022 
4023  // Toggle the th_task_state field, to switch which task_team this thread
4024  // refers to
4025  this_thr->th.th_task_state = (kmp_uint8)(1 - this_thr->th.th_task_state);
4026 
4027  // It is now safe to propagate the task team pointer from the team struct to
4028  // the current thread.
4029  TCW_PTR(this_thr->th.th_task_team,
4030  team->t.t_task_team[this_thr->th.th_task_state]);
4031  KA_TRACE(20,
4032  ("__kmp_task_team_sync: Thread T#%d task team switched to task_team "
4033  "%p from Team #%d (parity=%d)\n",
4034  __kmp_gtid_from_thread(this_thr), this_thr->th.th_task_team,
4035  team->t.t_id, this_thr->th.th_task_state));
4036 }
4037 
4038 // __kmp_task_team_wait: Primary thread waits for outstanding tasks after the
4039 // barrier gather phase. Only called by primary thread if #threads in team > 1
4040 // or if proxy tasks were created.
4041 //
4042 // wait is a flag that defaults to 1 (see kmp.h), but waiting can be turned off
4043 // by passing in 0 optionally as the last argument. When wait is zero, primary
4044 // thread does not wait for unfinished_threads to reach 0.
4045 void __kmp_task_team_wait(
4046  kmp_info_t *this_thr,
4047  kmp_team_t *team USE_ITT_BUILD_ARG(void *itt_sync_obj), int wait) {
4048  kmp_task_team_t *task_team = team->t.t_task_team[this_thr->th.th_task_state];
4049 
4050  KMP_DEBUG_ASSERT(__kmp_tasking_mode != tskm_immediate_exec);
4051  KMP_DEBUG_ASSERT(task_team == this_thr->th.th_task_team);
4052 
4053  if ((task_team != NULL) && KMP_TASKING_ENABLED(task_team)) {
4054  if (wait) {
4055  KA_TRACE(20, ("__kmp_task_team_wait: Primary T#%d waiting for all tasks "
4056  "(for unfinished_threads to reach 0) on task_team = %p\n",
4057  __kmp_gtid_from_thread(this_thr), task_team));
4058  // Worker threads may have dropped through to release phase, but could
4059  // still be executing tasks. Wait here for tasks to complete. To avoid
4060  // memory contention, only primary thread checks termination condition.
4061  kmp_flag_32<false, false> flag(
4062  RCAST(std::atomic<kmp_uint32> *,
4063  &task_team->tt.tt_unfinished_threads),
4064  0U);
4065  flag.wait(this_thr, TRUE USE_ITT_BUILD_ARG(itt_sync_obj));
4066  }
4067  // Deactivate the old task team, so that the worker threads will stop
4068  // referencing it while spinning.
4069  KA_TRACE(
4070  20,
4071  ("__kmp_task_team_wait: Primary T#%d deactivating task_team %p: "
4072  "setting active to false, setting local and team's pointer to NULL\n",
4073  __kmp_gtid_from_thread(this_thr), task_team));
4074  KMP_DEBUG_ASSERT(task_team->tt.tt_nproc > 1 ||
4075  task_team->tt.tt_found_proxy_tasks == TRUE ||
4076  task_team->tt.tt_hidden_helper_task_encountered == TRUE);
4077  TCW_SYNC_4(task_team->tt.tt_found_proxy_tasks, FALSE);
4078  TCW_SYNC_4(task_team->tt.tt_hidden_helper_task_encountered, FALSE);
4079  KMP_CHECK_UPDATE(task_team->tt.tt_untied_task_encountered, 0);
4080  TCW_SYNC_4(task_team->tt.tt_active, FALSE);
4081  KMP_MB();
4082 
4083  TCW_PTR(this_thr->th.th_task_team, NULL);
4084  }
4085 }
4086 
4087 // __kmp_tasking_barrier:
4088 // This routine is called only when __kmp_tasking_mode == tskm_extra_barrier.
4089 // Internal function to execute all tasks prior to a regular barrier or a join
4090 // barrier. It is a full barrier itself, which unfortunately turns regular
4091 // barriers into double barriers and join barriers into 1 1/2 barriers.
4092 void __kmp_tasking_barrier(kmp_team_t *team, kmp_info_t *thread, int gtid) {
4093  std::atomic<kmp_uint32> *spin = RCAST(
4094  std::atomic<kmp_uint32> *,
4095  &team->t.t_task_team[thread->th.th_task_state]->tt.tt_unfinished_threads);
4096  int flag = FALSE;
4097  KMP_DEBUG_ASSERT(__kmp_tasking_mode == tskm_extra_barrier);
4098 
4099 #if USE_ITT_BUILD
4100  KMP_FSYNC_SPIN_INIT(spin, NULL);
4101 #endif /* USE_ITT_BUILD */
4102  kmp_flag_32<false, false> spin_flag(spin, 0U);
4103  while (!spin_flag.execute_tasks(thread, gtid, TRUE,
4104  &flag USE_ITT_BUILD_ARG(NULL), 0)) {
4105 #if USE_ITT_BUILD
4106  // TODO: What about itt_sync_obj??
4107  KMP_FSYNC_SPIN_PREPARE(RCAST(void *, spin));
4108 #endif /* USE_ITT_BUILD */
4109 
4110  if (TCR_4(__kmp_global.g.g_done)) {
4111  if (__kmp_global.g.g_abort)
4112  __kmp_abort_thread();
4113  break;
4114  }
4115  KMP_YIELD(TRUE);
4116  }
4117 #if USE_ITT_BUILD
4118  KMP_FSYNC_SPIN_ACQUIRED(RCAST(void *, spin));
4119 #endif /* USE_ITT_BUILD */
4120 }
4121 
4122 // __kmp_give_task puts a task into a given thread queue if:
4123 // - the queue for that thread was created
4124 // - there's space in that queue
4125 // Because of this, __kmp_push_task needs to check if there's space after
4126 // getting the lock
4127 static bool __kmp_give_task(kmp_info_t *thread, kmp_int32 tid, kmp_task_t *task,
4128  kmp_int32 pass) {
4129  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
4130  kmp_task_team_t *task_team = taskdata->td_task_team;
4131 
4132  KA_TRACE(20, ("__kmp_give_task: trying to give task %p to thread %d.\n",
4133  taskdata, tid));
4134 
4135  // If task_team is NULL something went really bad...
4136  KMP_DEBUG_ASSERT(task_team != NULL);
4137 
4138  bool result = false;
4139  kmp_thread_data_t *thread_data = &task_team->tt.tt_threads_data[tid];
4140 
4141  if (thread_data->td.td_deque == NULL) {
4142  // There's no queue in this thread, go find another one
4143  // We're guaranteed that at least one thread has a queue
4144  KA_TRACE(30,
4145  ("__kmp_give_task: thread %d has no queue while giving task %p.\n",
4146  tid, taskdata));
4147  return result;
4148  }
4149 
4150  if (TCR_4(thread_data->td.td_deque_ntasks) >=
4151  TASK_DEQUE_SIZE(thread_data->td)) {
4152  KA_TRACE(
4153  30,
4154  ("__kmp_give_task: queue is full while giving task %p to thread %d.\n",
4155  taskdata, tid));
4156 
4157  // if this deque is bigger than the pass ratio give a chance to another
4158  // thread
4159  if (TASK_DEQUE_SIZE(thread_data->td) / INITIAL_TASK_DEQUE_SIZE >= pass)
4160  return result;
4161 
4162  __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
4163  if (TCR_4(thread_data->td.td_deque_ntasks) >=
4164  TASK_DEQUE_SIZE(thread_data->td)) {
4165  // expand deque to push the task which is not allowed to execute
4166  __kmp_realloc_task_deque(thread, thread_data);
4167  }
4168 
4169  } else {
4170 
4171  __kmp_acquire_bootstrap_lock(&thread_data->td.td_deque_lock);
4172 
4173  if (TCR_4(thread_data->td.td_deque_ntasks) >=
4174  TASK_DEQUE_SIZE(thread_data->td)) {
4175  KA_TRACE(30, ("__kmp_give_task: queue is full while giving task %p to "
4176  "thread %d.\n",
4177  taskdata, tid));
4178 
4179  // if this deque is bigger than the pass ratio give a chance to another
4180  // thread
4181  if (TASK_DEQUE_SIZE(thread_data->td) / INITIAL_TASK_DEQUE_SIZE >= pass)
4182  goto release_and_exit;
4183 
4184  __kmp_realloc_task_deque(thread, thread_data);
4185  }
4186  }
4187 
4188  // lock is held here, and there is space in the deque
4189 
4190  thread_data->td.td_deque[thread_data->td.td_deque_tail] = taskdata;
4191  // Wrap index.
4192  thread_data->td.td_deque_tail =
4193  (thread_data->td.td_deque_tail + 1) & TASK_DEQUE_MASK(thread_data->td);
4194  TCW_4(thread_data->td.td_deque_ntasks,
4195  TCR_4(thread_data->td.td_deque_ntasks) + 1);
4196 
4197  result = true;
4198  KA_TRACE(30, ("__kmp_give_task: successfully gave task %p to thread %d.\n",
4199  taskdata, tid));
4200 
4201 release_and_exit:
4202  __kmp_release_bootstrap_lock(&thread_data->td.td_deque_lock);
4203 
4204  return result;
4205 }
4206 
4207 #define PROXY_TASK_FLAG 0x40000000
4208 /* The finish of the proxy tasks is divided in two pieces:
4209  - the top half is the one that can be done from a thread outside the team
4210  - the bottom half must be run from a thread within the team
4211 
4212  In order to run the bottom half the task gets queued back into one of the
4213  threads of the team. Once the td_incomplete_child_task counter of the parent
4214  is decremented the threads can leave the barriers. So, the bottom half needs
4215  to be queued before the counter is decremented. The top half is therefore
4216  divided in two parts:
4217  - things that can be run before queuing the bottom half
4218  - things that must be run after queuing the bottom half
4219 
4220  This creates a second race as the bottom half can free the task before the
4221  second top half is executed. To avoid this we use the
4222  td_incomplete_child_task of the proxy task to synchronize the top and bottom
4223  half. */
4224 static void __kmp_first_top_half_finish_proxy(kmp_taskdata_t *taskdata) {
4225  KMP_DEBUG_ASSERT(taskdata->td_flags.tasktype == TASK_EXPLICIT);
4226  KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY);
4227  KMP_DEBUG_ASSERT(taskdata->td_flags.complete == 0);
4228  KMP_DEBUG_ASSERT(taskdata->td_flags.freed == 0);
4229 
4230  taskdata->td_flags.complete = 1; // mark the task as completed
4231 
4232  if (taskdata->td_taskgroup)
4233  KMP_ATOMIC_DEC(&taskdata->td_taskgroup->count);
4234 
4235  // Create an imaginary children for this task so the bottom half cannot
4236  // release the task before we have completed the second top half
4237  KMP_ATOMIC_OR(&taskdata->td_incomplete_child_tasks, PROXY_TASK_FLAG);
4238 }
4239 
4240 static void __kmp_second_top_half_finish_proxy(kmp_taskdata_t *taskdata) {
4241 #if KMP_DEBUG
4242  kmp_int32 children = 0;
4243  // Predecrement simulated by "- 1" calculation
4244  children = -1 +
4245 #endif
4246  KMP_ATOMIC_DEC(&taskdata->td_parent->td_incomplete_child_tasks);
4247  KMP_DEBUG_ASSERT(children >= 0);
4248 
4249  // Remove the imaginary children
4250  KMP_ATOMIC_AND(&taskdata->td_incomplete_child_tasks, ~PROXY_TASK_FLAG);
4251 }
4252 
4253 static void __kmp_bottom_half_finish_proxy(kmp_int32 gtid, kmp_task_t *ptask) {
4254  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask);
4255  kmp_info_t *thread = __kmp_threads[gtid];
4256 
4257  KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY);
4258  KMP_DEBUG_ASSERT(taskdata->td_flags.complete ==
4259  1); // top half must run before bottom half
4260 
4261  // We need to wait to make sure the top half is finished
4262  // Spinning here should be ok as this should happen quickly
4263  while ((KMP_ATOMIC_LD_ACQ(&taskdata->td_incomplete_child_tasks) &
4264  PROXY_TASK_FLAG) > 0)
4265  ;
4266 
4267  __kmp_release_deps(gtid, taskdata);
4268  __kmp_free_task_and_ancestors(gtid, taskdata, thread);
4269 }
4270 
4279 void __kmpc_proxy_task_completed(kmp_int32 gtid, kmp_task_t *ptask) {
4280  KMP_DEBUG_ASSERT(ptask != NULL);
4281  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask);
4282  KA_TRACE(
4283  10, ("__kmp_proxy_task_completed(enter): T#%d proxy task %p completing\n",
4284  gtid, taskdata));
4285  __kmp_assert_valid_gtid(gtid);
4286  KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY);
4287 
4288  __kmp_first_top_half_finish_proxy(taskdata);
4289  __kmp_second_top_half_finish_proxy(taskdata);
4290  __kmp_bottom_half_finish_proxy(gtid, ptask);
4291 
4292  KA_TRACE(10,
4293  ("__kmp_proxy_task_completed(exit): T#%d proxy task %p completing\n",
4294  gtid, taskdata));
4295 }
4296 
4297 void __kmpc_give_task(kmp_task_t *ptask, kmp_int32 start = 0) {
4298  KMP_DEBUG_ASSERT(ptask != NULL);
4299  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask);
4300 
4301  // Enqueue task to complete bottom half completion from a thread within the
4302  // corresponding team
4303  kmp_team_t *team = taskdata->td_team;
4304  kmp_int32 nthreads = team->t.t_nproc;
4305  kmp_info_t *thread;
4306 
4307  // This should be similar to start_k = __kmp_get_random( thread ) % nthreads
4308  // but we cannot use __kmp_get_random here
4309  kmp_int32 start_k = start % nthreads;
4310  kmp_int32 pass = 1;
4311  kmp_int32 k = start_k;
4312 
4313  do {
4314  // For now we're just linearly trying to find a thread
4315  thread = team->t.t_threads[k];
4316  k = (k + 1) % nthreads;
4317 
4318  // we did a full pass through all the threads
4319  if (k == start_k)
4320  pass = pass << 1;
4321 
4322  } while (!__kmp_give_task(thread, k, ptask, pass));
4323 
4324  if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME && __kmp_wpolicy_passive) {
4325  // awake at least one thread to execute given task
4326  for (int i = 0; i < nthreads; ++i) {
4327  thread = team->t.t_threads[i];
4328  if (thread->th.th_sleep_loc != NULL) {
4329  __kmp_null_resume_wrapper(thread);
4330  break;
4331  }
4332  }
4333  }
4334 }
4335 
4343 void __kmpc_proxy_task_completed_ooo(kmp_task_t *ptask) {
4344  KMP_DEBUG_ASSERT(ptask != NULL);
4345  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask);
4346 
4347  KA_TRACE(
4348  10,
4349  ("__kmp_proxy_task_completed_ooo(enter): proxy task completing ooo %p\n",
4350  taskdata));
4351 
4352  KMP_DEBUG_ASSERT(taskdata->td_flags.proxy == TASK_PROXY);
4353 
4354  __kmp_first_top_half_finish_proxy(taskdata);
4355 
4356  __kmpc_give_task(ptask);
4357 
4358  __kmp_second_top_half_finish_proxy(taskdata);
4359 
4360  KA_TRACE(
4361  10,
4362  ("__kmp_proxy_task_completed_ooo(exit): proxy task completing ooo %p\n",
4363  taskdata));
4364 }
4365 
4366 kmp_event_t *__kmpc_task_allow_completion_event(ident_t *loc_ref, int gtid,
4367  kmp_task_t *task) {
4368  kmp_taskdata_t *td = KMP_TASK_TO_TASKDATA(task);
4369  if (td->td_allow_completion_event.type == KMP_EVENT_UNINITIALIZED) {
4370  td->td_allow_completion_event.type = KMP_EVENT_ALLOW_COMPLETION;
4371  td->td_allow_completion_event.ed.task = task;
4372  __kmp_init_tas_lock(&td->td_allow_completion_event.lock);
4373  }
4374  return &td->td_allow_completion_event;
4375 }
4376 
4377 void __kmp_fulfill_event(kmp_event_t *event) {
4378  if (event->type == KMP_EVENT_ALLOW_COMPLETION) {
4379  kmp_task_t *ptask = event->ed.task;
4380  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(ptask);
4381  bool detached = false;
4382  int gtid = __kmp_get_gtid();
4383 
4384  // The associated task might have completed or could be completing at this
4385  // point.
4386  // We need to take the lock to avoid races
4387  __kmp_acquire_tas_lock(&event->lock, gtid);
4388  if (taskdata->td_flags.proxy == TASK_PROXY) {
4389  detached = true;
4390  } else {
4391 #if OMPT_SUPPORT
4392  // The OMPT event must occur under mutual exclusion,
4393  // otherwise the tool might access ptask after free
4394  if (UNLIKELY(ompt_enabled.enabled))
4395  __ompt_task_finish(ptask, NULL, ompt_task_early_fulfill);
4396 #endif
4397  }
4398  event->type = KMP_EVENT_UNINITIALIZED;
4399  __kmp_release_tas_lock(&event->lock, gtid);
4400 
4401  if (detached) {
4402 #if OMPT_SUPPORT
4403  // We free ptask afterwards and know the task is finished,
4404  // so locking is not necessary
4405  if (UNLIKELY(ompt_enabled.enabled))
4406  __ompt_task_finish(ptask, NULL, ompt_task_late_fulfill);
4407 #endif
4408  // If the task detached complete the proxy task
4409  if (gtid >= 0) {
4410  kmp_team_t *team = taskdata->td_team;
4411  kmp_info_t *thread = __kmp_get_thread();
4412  if (thread->th.th_team == team) {
4413  __kmpc_proxy_task_completed(gtid, ptask);
4414  return;
4415  }
4416  }
4417 
4418  // fallback
4420  }
4421  }
4422 }
4423 
4424 // __kmp_task_dup_alloc: Allocate the taskdata and make a copy of source task
4425 // for taskloop
4426 //
4427 // thread: allocating thread
4428 // task_src: pointer to source task to be duplicated
4429 // returns: a pointer to the allocated kmp_task_t structure (task).
4430 kmp_task_t *__kmp_task_dup_alloc(kmp_info_t *thread, kmp_task_t *task_src) {
4431  kmp_task_t *task;
4432  kmp_taskdata_t *taskdata;
4433  kmp_taskdata_t *taskdata_src = KMP_TASK_TO_TASKDATA(task_src);
4434  kmp_taskdata_t *parent_task = taskdata_src->td_parent; // same parent task
4435  size_t shareds_offset;
4436  size_t task_size;
4437 
4438  KA_TRACE(10, ("__kmp_task_dup_alloc(enter): Th %p, source task %p\n", thread,
4439  task_src));
4440  KMP_DEBUG_ASSERT(taskdata_src->td_flags.proxy ==
4441  TASK_FULL); // it should not be proxy task
4442  KMP_DEBUG_ASSERT(taskdata_src->td_flags.tasktype == TASK_EXPLICIT);
4443  task_size = taskdata_src->td_size_alloc;
4444 
4445  // Allocate a kmp_taskdata_t block and a kmp_task_t block.
4446  KA_TRACE(30, ("__kmp_task_dup_alloc: Th %p, malloc size %ld\n", thread,
4447  task_size));
4448 #if USE_FAST_MEMORY
4449  taskdata = (kmp_taskdata_t *)__kmp_fast_allocate(thread, task_size);
4450 #else
4451  taskdata = (kmp_taskdata_t *)__kmp_thread_malloc(thread, task_size);
4452 #endif /* USE_FAST_MEMORY */
4453  KMP_MEMCPY(taskdata, taskdata_src, task_size);
4454 
4455  task = KMP_TASKDATA_TO_TASK(taskdata);
4456 
4457  // Initialize new task (only specific fields not affected by memcpy)
4458  taskdata->td_task_id = KMP_GEN_TASK_ID();
4459  if (task->shareds != NULL) { // need setup shareds pointer
4460  shareds_offset = (char *)task_src->shareds - (char *)taskdata_src;
4461  task->shareds = &((char *)taskdata)[shareds_offset];
4462  KMP_DEBUG_ASSERT((((kmp_uintptr_t)task->shareds) & (sizeof(void *) - 1)) ==
4463  0);
4464  }
4465  taskdata->td_alloc_thread = thread;
4466  taskdata->td_parent = parent_task;
4467  // task inherits the taskgroup from the parent task
4468  taskdata->td_taskgroup = parent_task->td_taskgroup;
4469  // tied task needs to initialize the td_last_tied at creation,
4470  // untied one does this when it is scheduled for execution
4471  if (taskdata->td_flags.tiedness == TASK_TIED)
4472  taskdata->td_last_tied = taskdata;
4473 
4474  // Only need to keep track of child task counts if team parallel and tasking
4475  // not serialized
4476  if (!(taskdata->td_flags.team_serial || taskdata->td_flags.tasking_ser)) {
4477  KMP_ATOMIC_INC(&parent_task->td_incomplete_child_tasks);
4478  if (parent_task->td_taskgroup)
4479  KMP_ATOMIC_INC(&parent_task->td_taskgroup->count);
4480  // Only need to keep track of allocated child tasks for explicit tasks since
4481  // implicit not deallocated
4482  if (taskdata->td_parent->td_flags.tasktype == TASK_EXPLICIT)
4483  KMP_ATOMIC_INC(&taskdata->td_parent->td_allocated_child_tasks);
4484  }
4485 
4486  KA_TRACE(20,
4487  ("__kmp_task_dup_alloc(exit): Th %p, created task %p, parent=%p\n",
4488  thread, taskdata, taskdata->td_parent));
4489 #if OMPT_SUPPORT
4490  if (UNLIKELY(ompt_enabled.enabled))
4491  __ompt_task_init(taskdata, thread->th.th_info.ds.ds_gtid);
4492 #endif
4493  return task;
4494 }
4495 
4496 // Routine optionally generated by the compiler for setting the lastprivate flag
4497 // and calling needed constructors for private/firstprivate objects
4498 // (used to form taskloop tasks from pattern task)
4499 // Parameters: dest task, src task, lastprivate flag.
4500 typedef void (*p_task_dup_t)(kmp_task_t *, kmp_task_t *, kmp_int32);
4501 
4502 KMP_BUILD_ASSERT(sizeof(long) == 4 || sizeof(long) == 8);
4503 
4504 // class to encapsulate manipulating loop bounds in a taskloop task.
4505 // this abstracts away the Intel vs GOMP taskloop interface for setting/getting
4506 // the loop bound variables.
4507 class kmp_taskloop_bounds_t {
4508  kmp_task_t *task;
4509  const kmp_taskdata_t *taskdata;
4510  size_t lower_offset;
4511  size_t upper_offset;
4512 
4513 public:
4514  kmp_taskloop_bounds_t(kmp_task_t *_task, kmp_uint64 *lb, kmp_uint64 *ub)
4515  : task(_task), taskdata(KMP_TASK_TO_TASKDATA(task)),
4516  lower_offset((char *)lb - (char *)task),
4517  upper_offset((char *)ub - (char *)task) {
4518  KMP_DEBUG_ASSERT((char *)lb > (char *)_task);
4519  KMP_DEBUG_ASSERT((char *)ub > (char *)_task);
4520  }
4521  kmp_taskloop_bounds_t(kmp_task_t *_task, const kmp_taskloop_bounds_t &bounds)
4522  : task(_task), taskdata(KMP_TASK_TO_TASKDATA(_task)),
4523  lower_offset(bounds.lower_offset), upper_offset(bounds.upper_offset) {}
4524  size_t get_lower_offset() const { return lower_offset; }
4525  size_t get_upper_offset() const { return upper_offset; }
4526  kmp_uint64 get_lb() const {
4527  kmp_int64 retval;
4528 #if defined(KMP_GOMP_COMPAT)
4529  // Intel task just returns the lower bound normally
4530  if (!taskdata->td_flags.native) {
4531  retval = *(kmp_int64 *)((char *)task + lower_offset);
4532  } else {
4533  // GOMP task has to take into account the sizeof(long)
4534  if (taskdata->td_size_loop_bounds == 4) {
4535  kmp_int32 *lb = RCAST(kmp_int32 *, task->shareds);
4536  retval = (kmp_int64)*lb;
4537  } else {
4538  kmp_int64 *lb = RCAST(kmp_int64 *, task->shareds);
4539  retval = (kmp_int64)*lb;
4540  }
4541  }
4542 #else
4543  (void)taskdata;
4544  retval = *(kmp_int64 *)((char *)task + lower_offset);
4545 #endif // defined(KMP_GOMP_COMPAT)
4546  return retval;
4547  }
4548  kmp_uint64 get_ub() const {
4549  kmp_int64 retval;
4550 #if defined(KMP_GOMP_COMPAT)
4551  // Intel task just returns the upper bound normally
4552  if (!taskdata->td_flags.native) {
4553  retval = *(kmp_int64 *)((char *)task + upper_offset);
4554  } else {
4555  // GOMP task has to take into account the sizeof(long)
4556  if (taskdata->td_size_loop_bounds == 4) {
4557  kmp_int32 *ub = RCAST(kmp_int32 *, task->shareds) + 1;
4558  retval = (kmp_int64)*ub;
4559  } else {
4560  kmp_int64 *ub = RCAST(kmp_int64 *, task->shareds) + 1;
4561  retval = (kmp_int64)*ub;
4562  }
4563  }
4564 #else
4565  retval = *(kmp_int64 *)((char *)task + upper_offset);
4566 #endif // defined(KMP_GOMP_COMPAT)
4567  return retval;
4568  }
4569  void set_lb(kmp_uint64 lb) {
4570 #if defined(KMP_GOMP_COMPAT)
4571  // Intel task just sets the lower bound normally
4572  if (!taskdata->td_flags.native) {
4573  *(kmp_uint64 *)((char *)task + lower_offset) = lb;
4574  } else {
4575  // GOMP task has to take into account the sizeof(long)
4576  if (taskdata->td_size_loop_bounds == 4) {
4577  kmp_uint32 *lower = RCAST(kmp_uint32 *, task->shareds);
4578  *lower = (kmp_uint32)lb;
4579  } else {
4580  kmp_uint64 *lower = RCAST(kmp_uint64 *, task->shareds);
4581  *lower = (kmp_uint64)lb;
4582  }
4583  }
4584 #else
4585  *(kmp_uint64 *)((char *)task + lower_offset) = lb;
4586 #endif // defined(KMP_GOMP_COMPAT)
4587  }
4588  void set_ub(kmp_uint64 ub) {
4589 #if defined(KMP_GOMP_COMPAT)
4590  // Intel task just sets the upper bound normally
4591  if (!taskdata->td_flags.native) {
4592  *(kmp_uint64 *)((char *)task + upper_offset) = ub;
4593  } else {
4594  // GOMP task has to take into account the sizeof(long)
4595  if (taskdata->td_size_loop_bounds == 4) {
4596  kmp_uint32 *upper = RCAST(kmp_uint32 *, task->shareds) + 1;
4597  *upper = (kmp_uint32)ub;
4598  } else {
4599  kmp_uint64 *upper = RCAST(kmp_uint64 *, task->shareds) + 1;
4600  *upper = (kmp_uint64)ub;
4601  }
4602  }
4603 #else
4604  *(kmp_uint64 *)((char *)task + upper_offset) = ub;
4605 #endif // defined(KMP_GOMP_COMPAT)
4606  }
4607 };
4608 
4609 // __kmp_taskloop_linear: Start tasks of the taskloop linearly
4610 //
4611 // loc Source location information
4612 // gtid Global thread ID
4613 // task Pattern task, exposes the loop iteration range
4614 // lb Pointer to loop lower bound in task structure
4615 // ub Pointer to loop upper bound in task structure
4616 // st Loop stride
4617 // ub_glob Global upper bound (used for lastprivate check)
4618 // num_tasks Number of tasks to execute
4619 // grainsize Number of loop iterations per task
4620 // extras Number of chunks with grainsize+1 iterations
4621 // last_chunk Reduction of grainsize for last task
4622 // tc Iterations count
4623 // task_dup Tasks duplication routine
4624 // codeptr_ra Return address for OMPT events
4625 void __kmp_taskloop_linear(ident_t *loc, int gtid, kmp_task_t *task,
4626  kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
4627  kmp_uint64 ub_glob, kmp_uint64 num_tasks,
4628  kmp_uint64 grainsize, kmp_uint64 extras,
4629  kmp_int64 last_chunk, kmp_uint64 tc,
4630 #if OMPT_SUPPORT
4631  void *codeptr_ra,
4632 #endif
4633  void *task_dup) {
4634  KMP_COUNT_BLOCK(OMP_TASKLOOP);
4635  KMP_TIME_PARTITIONED_BLOCK(OMP_taskloop_scheduling);
4636  p_task_dup_t ptask_dup = (p_task_dup_t)task_dup;
4637  // compiler provides global bounds here
4638  kmp_taskloop_bounds_t task_bounds(task, lb, ub);
4639  kmp_uint64 lower = task_bounds.get_lb();
4640  kmp_uint64 upper = task_bounds.get_ub();
4641  kmp_uint64 i;
4642  kmp_info_t *thread = __kmp_threads[gtid];
4643  kmp_taskdata_t *current_task = thread->th.th_current_task;
4644  kmp_task_t *next_task;
4645  kmp_int32 lastpriv = 0;
4646 
4647  KMP_DEBUG_ASSERT(tc == num_tasks * grainsize +
4648  (last_chunk < 0 ? last_chunk : extras));
4649  KMP_DEBUG_ASSERT(num_tasks > extras);
4650  KMP_DEBUG_ASSERT(num_tasks > 0);
4651  KA_TRACE(20, ("__kmp_taskloop_linear: T#%d: %lld tasks, grainsize %lld, "
4652  "extras %lld, last_chunk %lld, i=%lld,%lld(%d)%lld, dup %p\n",
4653  gtid, num_tasks, grainsize, extras, last_chunk, lower, upper,
4654  ub_glob, st, task_dup));
4655 
4656  // Launch num_tasks tasks, assign grainsize iterations each task
4657  for (i = 0; i < num_tasks; ++i) {
4658  kmp_uint64 chunk_minus_1;
4659  if (extras == 0) {
4660  chunk_minus_1 = grainsize - 1;
4661  } else {
4662  chunk_minus_1 = grainsize;
4663  --extras; // first extras iterations get bigger chunk (grainsize+1)
4664  }
4665  upper = lower + st * chunk_minus_1;
4666  if (upper > *ub) {
4667  upper = *ub;
4668  }
4669  if (i == num_tasks - 1) {
4670  // schedule the last task, set lastprivate flag if needed
4671  if (st == 1) { // most common case
4672  KMP_DEBUG_ASSERT(upper == *ub);
4673  if (upper == ub_glob)
4674  lastpriv = 1;
4675  } else if (st > 0) { // positive loop stride
4676  KMP_DEBUG_ASSERT((kmp_uint64)st > *ub - upper);
4677  if ((kmp_uint64)st > ub_glob - upper)
4678  lastpriv = 1;
4679  } else { // negative loop stride
4680  KMP_DEBUG_ASSERT(upper + st < *ub);
4681  if (upper - ub_glob < (kmp_uint64)(-st))
4682  lastpriv = 1;
4683  }
4684  }
4685  next_task = __kmp_task_dup_alloc(thread, task); // allocate new task
4686  kmp_taskdata_t *next_taskdata = KMP_TASK_TO_TASKDATA(next_task);
4687  kmp_taskloop_bounds_t next_task_bounds =
4688  kmp_taskloop_bounds_t(next_task, task_bounds);
4689 
4690  // adjust task-specific bounds
4691  next_task_bounds.set_lb(lower);
4692  if (next_taskdata->td_flags.native) {
4693  next_task_bounds.set_ub(upper + (st > 0 ? 1 : -1));
4694  } else {
4695  next_task_bounds.set_ub(upper);
4696  }
4697  if (ptask_dup != NULL) // set lastprivate flag, construct firstprivates,
4698  // etc.
4699  ptask_dup(next_task, task, lastpriv);
4700  KA_TRACE(40,
4701  ("__kmp_taskloop_linear: T#%d; task #%llu: task %p: lower %lld, "
4702  "upper %lld stride %lld, (offsets %p %p)\n",
4703  gtid, i, next_task, lower, upper, st,
4704  next_task_bounds.get_lower_offset(),
4705  next_task_bounds.get_upper_offset()));
4706 #if OMPT_SUPPORT
4707  __kmp_omp_taskloop_task(NULL, gtid, next_task,
4708  codeptr_ra); // schedule new task
4709 #if OMPT_OPTIONAL
4710  if (ompt_enabled.ompt_callback_dispatch) {
4711  OMPT_GET_DISPATCH_CHUNK(next_taskdata->ompt_task_info.dispatch_chunk,
4712  lower, upper, st);
4713  }
4714 #endif // OMPT_OPTIONAL
4715 #else
4716  __kmp_omp_task(gtid, next_task, true); // schedule new task
4717 #endif
4718  lower = upper + st; // adjust lower bound for the next iteration
4719  }
4720  // free the pattern task and exit
4721  __kmp_task_start(gtid, task, current_task); // make internal bookkeeping
4722  // do not execute the pattern task, just do internal bookkeeping
4723  __kmp_task_finish<false>(gtid, task, current_task);
4724 }
4725 
4726 // Structure to keep taskloop parameters for auxiliary task
4727 // kept in the shareds of the task structure.
4728 typedef struct __taskloop_params {
4729  kmp_task_t *task;
4730  kmp_uint64 *lb;
4731  kmp_uint64 *ub;
4732  void *task_dup;
4733  kmp_int64 st;
4734  kmp_uint64 ub_glob;
4735  kmp_uint64 num_tasks;
4736  kmp_uint64 grainsize;
4737  kmp_uint64 extras;
4738  kmp_int64 last_chunk;
4739  kmp_uint64 tc;
4740  kmp_uint64 num_t_min;
4741 #if OMPT_SUPPORT
4742  void *codeptr_ra;
4743 #endif
4744 } __taskloop_params_t;
4745 
4746 void __kmp_taskloop_recur(ident_t *, int, kmp_task_t *, kmp_uint64 *,
4747  kmp_uint64 *, kmp_int64, kmp_uint64, kmp_uint64,
4748  kmp_uint64, kmp_uint64, kmp_int64, kmp_uint64,
4749  kmp_uint64,
4750 #if OMPT_SUPPORT
4751  void *,
4752 #endif
4753  void *);
4754 
4755 // Execute part of the taskloop submitted as a task.
4756 int __kmp_taskloop_task(int gtid, void *ptask) {
4757  __taskloop_params_t *p =
4758  (__taskloop_params_t *)((kmp_task_t *)ptask)->shareds;
4759  kmp_task_t *task = p->task;
4760  kmp_uint64 *lb = p->lb;
4761  kmp_uint64 *ub = p->ub;
4762  void *task_dup = p->task_dup;
4763  // p_task_dup_t ptask_dup = (p_task_dup_t)task_dup;
4764  kmp_int64 st = p->st;
4765  kmp_uint64 ub_glob = p->ub_glob;
4766  kmp_uint64 num_tasks = p->num_tasks;
4767  kmp_uint64 grainsize = p->grainsize;
4768  kmp_uint64 extras = p->extras;
4769  kmp_int64 last_chunk = p->last_chunk;
4770  kmp_uint64 tc = p->tc;
4771  kmp_uint64 num_t_min = p->num_t_min;
4772 #if OMPT_SUPPORT
4773  void *codeptr_ra = p->codeptr_ra;
4774 #endif
4775 #if KMP_DEBUG
4776  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
4777  KMP_DEBUG_ASSERT(task != NULL);
4778  KA_TRACE(20,
4779  ("__kmp_taskloop_task: T#%d, task %p: %lld tasks, grainsize"
4780  " %lld, extras %lld, last_chunk %lld, i=%lld,%lld(%d), dup %p\n",
4781  gtid, taskdata, num_tasks, grainsize, extras, last_chunk, *lb, *ub,
4782  st, task_dup));
4783 #endif
4784  KMP_DEBUG_ASSERT(num_tasks * 2 + 1 > num_t_min);
4785  if (num_tasks > num_t_min)
4786  __kmp_taskloop_recur(NULL, gtid, task, lb, ub, st, ub_glob, num_tasks,
4787  grainsize, extras, last_chunk, tc, num_t_min,
4788 #if OMPT_SUPPORT
4789  codeptr_ra,
4790 #endif
4791  task_dup);
4792  else
4793  __kmp_taskloop_linear(NULL, gtid, task, lb, ub, st, ub_glob, num_tasks,
4794  grainsize, extras, last_chunk, tc,
4795 #if OMPT_SUPPORT
4796  codeptr_ra,
4797 #endif
4798  task_dup);
4799 
4800  KA_TRACE(40, ("__kmp_taskloop_task(exit): T#%d\n", gtid));
4801  return 0;
4802 }
4803 
4804 // Schedule part of the taskloop as a task,
4805 // execute the rest of the taskloop.
4806 //
4807 // loc Source location information
4808 // gtid Global thread ID
4809 // task Pattern task, exposes the loop iteration range
4810 // lb Pointer to loop lower bound in task structure
4811 // ub Pointer to loop upper bound in task structure
4812 // st Loop stride
4813 // ub_glob Global upper bound (used for lastprivate check)
4814 // num_tasks Number of tasks to execute
4815 // grainsize Number of loop iterations per task
4816 // extras Number of chunks with grainsize+1 iterations
4817 // last_chunk Reduction of grainsize for last task
4818 // tc Iterations count
4819 // num_t_min Threshold to launch tasks recursively
4820 // task_dup Tasks duplication routine
4821 // codeptr_ra Return address for OMPT events
4822 void __kmp_taskloop_recur(ident_t *loc, int gtid, kmp_task_t *task,
4823  kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
4824  kmp_uint64 ub_glob, kmp_uint64 num_tasks,
4825  kmp_uint64 grainsize, kmp_uint64 extras,
4826  kmp_int64 last_chunk, kmp_uint64 tc,
4827  kmp_uint64 num_t_min,
4828 #if OMPT_SUPPORT
4829  void *codeptr_ra,
4830 #endif
4831  void *task_dup) {
4832  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
4833  KMP_DEBUG_ASSERT(task != NULL);
4834  KMP_DEBUG_ASSERT(num_tasks > num_t_min);
4835  KA_TRACE(20,
4836  ("__kmp_taskloop_recur: T#%d, task %p: %lld tasks, grainsize"
4837  " %lld, extras %lld, last_chunk %lld, i=%lld,%lld(%d), dup %p\n",
4838  gtid, taskdata, num_tasks, grainsize, extras, last_chunk, *lb, *ub,
4839  st, task_dup));
4840  p_task_dup_t ptask_dup = (p_task_dup_t)task_dup;
4841  kmp_uint64 lower = *lb;
4842  kmp_info_t *thread = __kmp_threads[gtid];
4843  // kmp_taskdata_t *current_task = thread->th.th_current_task;
4844  kmp_task_t *next_task;
4845  size_t lower_offset =
4846  (char *)lb - (char *)task; // remember offset of lb in the task structure
4847  size_t upper_offset =
4848  (char *)ub - (char *)task; // remember offset of ub in the task structure
4849 
4850  KMP_DEBUG_ASSERT(tc == num_tasks * grainsize +
4851  (last_chunk < 0 ? last_chunk : extras));
4852  KMP_DEBUG_ASSERT(num_tasks > extras);
4853  KMP_DEBUG_ASSERT(num_tasks > 0);
4854 
4855  // split the loop in two halves
4856  kmp_uint64 lb1, ub0, tc0, tc1, ext0, ext1;
4857  kmp_int64 last_chunk0 = 0, last_chunk1 = 0;
4858  kmp_uint64 gr_size0 = grainsize;
4859  kmp_uint64 n_tsk0 = num_tasks >> 1; // num_tasks/2 to execute
4860  kmp_uint64 n_tsk1 = num_tasks - n_tsk0; // to schedule as a task
4861  if (last_chunk < 0) {
4862  ext0 = ext1 = 0;
4863  last_chunk1 = last_chunk;
4864  tc0 = grainsize * n_tsk0;
4865  tc1 = tc - tc0;
4866  } else if (n_tsk0 <= extras) {
4867  gr_size0++; // integrate extras into grainsize
4868  ext0 = 0; // no extra iters in 1st half
4869  ext1 = extras - n_tsk0; // remaining extras
4870  tc0 = gr_size0 * n_tsk0;
4871  tc1 = tc - tc0;
4872  } else { // n_tsk0 > extras
4873  ext1 = 0; // no extra iters in 2nd half
4874  ext0 = extras;
4875  tc1 = grainsize * n_tsk1;
4876  tc0 = tc - tc1;
4877  }
4878  ub0 = lower + st * (tc0 - 1);
4879  lb1 = ub0 + st;
4880 
4881  // create pattern task for 2nd half of the loop
4882  next_task = __kmp_task_dup_alloc(thread, task); // duplicate the task
4883  // adjust lower bound (upper bound is not changed) for the 2nd half
4884  *(kmp_uint64 *)((char *)next_task + lower_offset) = lb1;
4885  if (ptask_dup != NULL) // construct firstprivates, etc.
4886  ptask_dup(next_task, task, 0);
4887  *ub = ub0; // adjust upper bound for the 1st half
4888 
4889  // create auxiliary task for 2nd half of the loop
4890  // make sure new task has same parent task as the pattern task
4891  kmp_taskdata_t *current_task = thread->th.th_current_task;
4892  thread->th.th_current_task = taskdata->td_parent;
4893  kmp_task_t *new_task =
4894  __kmpc_omp_task_alloc(loc, gtid, 1, 3 * sizeof(void *),
4895  sizeof(__taskloop_params_t), &__kmp_taskloop_task);
4896  // restore current task
4897  thread->th.th_current_task = current_task;
4898  __taskloop_params_t *p = (__taskloop_params_t *)new_task->shareds;
4899  p->task = next_task;
4900  p->lb = (kmp_uint64 *)((char *)next_task + lower_offset);
4901  p->ub = (kmp_uint64 *)((char *)next_task + upper_offset);
4902  p->task_dup = task_dup;
4903  p->st = st;
4904  p->ub_glob = ub_glob;
4905  p->num_tasks = n_tsk1;
4906  p->grainsize = grainsize;
4907  p->extras = ext1;
4908  p->last_chunk = last_chunk1;
4909  p->tc = tc1;
4910  p->num_t_min = num_t_min;
4911 #if OMPT_SUPPORT
4912  p->codeptr_ra = codeptr_ra;
4913 #endif
4914 
4915 #if OMPT_SUPPORT
4916  // schedule new task with correct return address for OMPT events
4917  __kmp_omp_taskloop_task(NULL, gtid, new_task, codeptr_ra);
4918 #else
4919  __kmp_omp_task(gtid, new_task, true); // schedule new task
4920 #endif
4921 
4922  // execute the 1st half of current subrange
4923  if (n_tsk0 > num_t_min)
4924  __kmp_taskloop_recur(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0, gr_size0,
4925  ext0, last_chunk0, tc0, num_t_min,
4926 #if OMPT_SUPPORT
4927  codeptr_ra,
4928 #endif
4929  task_dup);
4930  else
4931  __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, n_tsk0,
4932  gr_size0, ext0, last_chunk0, tc0,
4933 #if OMPT_SUPPORT
4934  codeptr_ra,
4935 #endif
4936  task_dup);
4937 
4938  KA_TRACE(40, ("__kmp_taskloop_recur(exit): T#%d\n", gtid));
4939 }
4940 
4941 static void __kmp_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
4942  kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
4943  int nogroup, int sched, kmp_uint64 grainsize,
4944  int modifier, void *task_dup) {
4945  kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task);
4946  KMP_DEBUG_ASSERT(task != NULL);
4947  if (nogroup == 0) {
4948 #if OMPT_SUPPORT && OMPT_OPTIONAL
4949  OMPT_STORE_RETURN_ADDRESS(gtid);
4950 #endif
4951  __kmpc_taskgroup(loc, gtid);
4952  }
4953 
4954  // =========================================================================
4955  // calculate loop parameters
4956  kmp_taskloop_bounds_t task_bounds(task, lb, ub);
4957  kmp_uint64 tc;
4958  // compiler provides global bounds here
4959  kmp_uint64 lower = task_bounds.get_lb();
4960  kmp_uint64 upper = task_bounds.get_ub();
4961  kmp_uint64 ub_glob = upper; // global upper used to calc lastprivate flag
4962  kmp_uint64 num_tasks = 0, extras = 0;
4963  kmp_int64 last_chunk =
4964  0; // reduce grainsize of last task by last_chunk in strict mode
4965  kmp_uint64 num_tasks_min = __kmp_taskloop_min_tasks;
4966  kmp_info_t *thread = __kmp_threads[gtid];
4967  kmp_taskdata_t *current_task = thread->th.th_current_task;
4968 
4969  KA_TRACE(20, ("__kmp_taskloop: T#%d, task %p, lb %lld, ub %lld, st %lld, "
4970  "grain %llu(%d, %d), dup %p\n",
4971  gtid, taskdata, lower, upper, st, grainsize, sched, modifier,
4972  task_dup));
4973 
4974  // compute trip count
4975  if (st == 1) { // most common case
4976  tc = upper - lower + 1;
4977  } else if (st < 0) {
4978  tc = (lower - upper) / (-st) + 1;
4979  } else { // st > 0
4980  tc = (upper - lower) / st + 1;
4981  }
4982  if (tc == 0) {
4983  KA_TRACE(20, ("__kmp_taskloop(exit): T#%d zero-trip loop\n", gtid));
4984  // free the pattern task and exit
4985  __kmp_task_start(gtid, task, current_task);
4986  // do not execute anything for zero-trip loop
4987  __kmp_task_finish<false>(gtid, task, current_task);
4988  return;
4989  }
4990 
4991 #if OMPT_SUPPORT && OMPT_OPTIONAL
4992  ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
4993  ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
4994  if (ompt_enabled.ompt_callback_work) {
4995  ompt_callbacks.ompt_callback(ompt_callback_work)(
4996  ompt_work_taskloop, ompt_scope_begin, &(team_info->parallel_data),
4997  &(task_info->task_data), tc, OMPT_GET_RETURN_ADDRESS(0));
4998  }
4999 #endif
5000 
5001  if (num_tasks_min == 0)
5002  // TODO: can we choose better default heuristic?
5003  num_tasks_min =
5004  KMP_MIN(thread->th.th_team_nproc * 10, INITIAL_TASK_DEQUE_SIZE);
5005 
5006  // compute num_tasks/grainsize based on the input provided
5007  switch (sched) {
5008  case 0: // no schedule clause specified, we can choose the default
5009  // let's try to schedule (team_size*10) tasks
5010  grainsize = thread->th.th_team_nproc * 10;
5011  KMP_FALLTHROUGH();
5012  case 2: // num_tasks provided
5013  if (grainsize > tc) {
5014  num_tasks = tc; // too big num_tasks requested, adjust values
5015  grainsize = 1;
5016  extras = 0;
5017  } else {
5018  num_tasks = grainsize;
5019  grainsize = tc / num_tasks;
5020  extras = tc % num_tasks;
5021  }
5022  break;
5023  case 1: // grainsize provided
5024  if (grainsize > tc) {
5025  num_tasks = 1;
5026  grainsize = tc; // too big grainsize requested, adjust values
5027  extras = 0;
5028  } else {
5029  if (modifier) {
5030  num_tasks = (tc + grainsize - 1) / grainsize;
5031  last_chunk = tc - (num_tasks * grainsize);
5032  extras = 0;
5033  } else {
5034  num_tasks = tc / grainsize;
5035  // adjust grainsize for balanced distribution of iterations
5036  grainsize = tc / num_tasks;
5037  extras = tc % num_tasks;
5038  }
5039  }
5040  break;
5041  default:
5042  KMP_ASSERT2(0, "unknown scheduling of taskloop");
5043  }
5044 
5045  KMP_DEBUG_ASSERT(tc == num_tasks * grainsize +
5046  (last_chunk < 0 ? last_chunk : extras));
5047  KMP_DEBUG_ASSERT(num_tasks > extras);
5048  KMP_DEBUG_ASSERT(num_tasks > 0);
5049  // =========================================================================
5050 
5051  // check if clause value first
5052  // Also require GOMP_taskloop to reduce to linear (taskdata->td_flags.native)
5053  if (if_val == 0) { // if(0) specified, mark task as serial
5054  taskdata->td_flags.task_serial = 1;
5055  taskdata->td_flags.tiedness = TASK_TIED; // AC: serial task cannot be untied
5056  // always start serial tasks linearly
5057  __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
5058  grainsize, extras, last_chunk, tc,
5059 #if OMPT_SUPPORT
5060  OMPT_GET_RETURN_ADDRESS(0),
5061 #endif
5062  task_dup);
5063  // !taskdata->td_flags.native => currently force linear spawning of tasks
5064  // for GOMP_taskloop
5065  } else if (num_tasks > num_tasks_min && !taskdata->td_flags.native) {
5066  KA_TRACE(20, ("__kmp_taskloop: T#%d, go recursive: tc %llu, #tasks %llu"
5067  "(%lld), grain %llu, extras %llu, last_chunk %lld\n",
5068  gtid, tc, num_tasks, num_tasks_min, grainsize, extras,
5069  last_chunk));
5070  __kmp_taskloop_recur(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
5071  grainsize, extras, last_chunk, tc, num_tasks_min,
5072 #if OMPT_SUPPORT
5073  OMPT_GET_RETURN_ADDRESS(0),
5074 #endif
5075  task_dup);
5076  } else {
5077  KA_TRACE(20, ("__kmp_taskloop: T#%d, go linear: tc %llu, #tasks %llu"
5078  "(%lld), grain %llu, extras %llu, last_chunk %lld\n",
5079  gtid, tc, num_tasks, num_tasks_min, grainsize, extras,
5080  last_chunk));
5081  __kmp_taskloop_linear(loc, gtid, task, lb, ub, st, ub_glob, num_tasks,
5082  grainsize, extras, last_chunk, tc,
5083 #if OMPT_SUPPORT
5084  OMPT_GET_RETURN_ADDRESS(0),
5085 #endif
5086  task_dup);
5087  }
5088 
5089 #if OMPT_SUPPORT && OMPT_OPTIONAL
5090  if (ompt_enabled.ompt_callback_work) {
5091  ompt_callbacks.ompt_callback(ompt_callback_work)(
5092  ompt_work_taskloop, ompt_scope_end, &(team_info->parallel_data),
5093  &(task_info->task_data), tc, OMPT_GET_RETURN_ADDRESS(0));
5094  }
5095 #endif
5096 
5097  if (nogroup == 0) {
5098 #if OMPT_SUPPORT && OMPT_OPTIONAL
5099  OMPT_STORE_RETURN_ADDRESS(gtid);
5100 #endif
5101  __kmpc_end_taskgroup(loc, gtid);
5102  }
5103  KA_TRACE(20, ("__kmp_taskloop(exit): T#%d\n", gtid));
5104 }
5105 
5122 void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
5123  kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup,
5124  int sched, kmp_uint64 grainsize, void *task_dup) {
5125  __kmp_assert_valid_gtid(gtid);
5126  KA_TRACE(20, ("__kmpc_taskloop(enter): T#%d\n", gtid));
5127  __kmp_taskloop(loc, gtid, task, if_val, lb, ub, st, nogroup, sched, grainsize,
5128  0, task_dup);
5129  KA_TRACE(20, ("__kmpc_taskloop(exit): T#%d\n", gtid));
5130 }
5131 
5149 void __kmpc_taskloop_5(ident_t *loc, int gtid, kmp_task_t *task, int if_val,
5150  kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st,
5151  int nogroup, int sched, kmp_uint64 grainsize,
5152  int modifier, void *task_dup) {
5153  __kmp_assert_valid_gtid(gtid);
5154  KA_TRACE(20, ("__kmpc_taskloop_5(enter): T#%d\n", gtid));
5155  __kmp_taskloop(loc, gtid, task, if_val, lb, ub, st, nogroup, sched, grainsize,
5156  modifier, task_dup);
5157  KA_TRACE(20, ("__kmpc_taskloop_5(exit): T#%d\n", gtid));
5158 }
5159 
5169  if (gtid == KMP_GTID_DNE)
5170  return NULL;
5171 
5172  kmp_info_t *thread = __kmp_thread_from_gtid(gtid);
5173  kmp_taskdata_t *taskdata = thread->th.th_current_task;
5174 
5175  if (!taskdata)
5176  return NULL;
5177 
5178  return &taskdata->td_target_data.async_handle;
5179 }
5180 
5189 bool __kmpc_omp_has_task_team(kmp_int32 gtid) {
5190  if (gtid == KMP_GTID_DNE)
5191  return FALSE;
5192 
5193  kmp_info_t *thread = __kmp_thread_from_gtid(gtid);
5194  kmp_taskdata_t *taskdata = thread->th.th_current_task;
5195 
5196  if (!taskdata)
5197  return FALSE;
5198 
5199  return taskdata->td_task_team != NULL;
5200 }
struct kmp_taskred_data kmp_taskred_data_t
struct kmp_task_red_input kmp_task_red_input_t
struct kmp_taskred_flags kmp_taskred_flags_t
struct kmp_taskred_input kmp_taskred_input_t
#define KMP_COUNT_BLOCK(name)
Increments specified counter (name).
Definition: kmp_stats.h:911
void __kmpc_taskloop(ident_t *loc, int gtid, kmp_task_t *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int sched, kmp_uint64 grainsize, void *task_dup)
void * __kmpc_taskred_modifier_init(ident_t *loc, int gtid, int is_ws, int num, void *data)
void * __kmpc_taskred_init(int gtid, int num, void *data)
void * __kmpc_task_reduction_init(int gtid, int num, void *data)
bool __kmpc_omp_has_task_team(kmp_int32 gtid)
void __kmpc_proxy_task_completed_ooo(kmp_task_t *ptask)
void __kmpc_task_reduction_modifier_fini(ident_t *loc, int gtid, int is_ws)
kmp_int32 __kmpc_omp_reg_task_with_affinity(ident_t *loc_ref, kmp_int32 gtid, kmp_task_t *new_task, kmp_int32 naffins, kmp_task_affinity_info_t *affin_list)
void * __kmpc_task_reduction_modifier_init(ident_t *loc, int gtid, int is_ws, int num, void *data)
void __kmpc_taskloop_5(ident_t *loc, int gtid, kmp_task_t *task, int if_val, kmp_uint64 *lb, kmp_uint64 *ub, kmp_int64 st, int nogroup, int sched, kmp_uint64 grainsize, int modifier, void *task_dup)
void __kmpc_proxy_task_completed(kmp_int32 gtid, kmp_task_t *ptask)
void * __kmpc_task_reduction_get_th_data(int gtid, void *tskgrp, void *data)
void ** __kmpc_omp_get_target_async_handle_ptr(kmp_int32 gtid)
Definition: kmp.h:234
kmp_taskred_flags_t flags
kmp_taskred_flags_t flags
kmp_taskred_flags_t flags