LLVM OpenMP* Runtime Library
kmp_csupport.cpp
1/*
2 * kmp_csupport.cpp -- kfront linkage support for OpenMP.
3 */
4
5//===----------------------------------------------------------------------===//
6//
7// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8// See https://llvm.org/LICENSE.txt for license information.
9// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10//
11//===----------------------------------------------------------------------===//
12
13#define __KMP_IMP
14#include "omp.h" /* extern "C" declarations of user-visible routines */
15#include "kmp.h"
16#include "kmp_error.h"
17#include "kmp_i18n.h"
18#include "kmp_itt.h"
19#include "kmp_lock.h"
20#include "kmp_stats.h"
21#include "kmp_utils.h"
22#include "ompt-specific.h"
23
24#define MAX_MESSAGE 512
25
26// flags will be used in future, e.g. to implement openmp_strict library
27// restrictions
28
37void __kmpc_begin(ident_t *loc, kmp_int32 flags) {
38 // By default __kmpc_begin() is no-op.
39 char *env;
40 if ((env = getenv("KMP_INITIAL_THREAD_BIND")) != NULL &&
41 __kmp_str_match_true(env)) {
42 __kmp_middle_initialize();
43 __kmp_assign_root_init_mask();
44 KC_TRACE(10, ("__kmpc_begin: middle initialization called\n"));
45 } else if (__kmp_ignore_mppbeg() == FALSE) {
46 // By default __kmp_ignore_mppbeg() returns TRUE.
47 __kmp_internal_begin();
48 KC_TRACE(10, ("__kmpc_begin: called\n"));
49 }
50}
51
60void __kmpc_end(ident_t *loc) {
61 // By default, __kmp_ignore_mppend() returns TRUE which makes __kmpc_end()
62 // call no-op. However, this can be overridden with KMP_IGNORE_MPPEND
63 // environment variable. If KMP_IGNORE_MPPEND is 0, __kmp_ignore_mppend()
64 // returns FALSE and __kmpc_end() will unregister this root (it can cause
65 // library shut down).
66 if (__kmp_ignore_mppend() == FALSE) {
67 KC_TRACE(10, ("__kmpc_end: called\n"));
68 KA_TRACE(30, ("__kmpc_end\n"));
69
70 __kmp_internal_end_thread(-1);
71 }
72#if KMP_OS_WINDOWS && OMPT_SUPPORT
73 // Normal exit process on Windows does not allow worker threads of the final
74 // parallel region to finish reporting their events, so shutting down the
75 // library here fixes the issue at least for the cases where __kmpc_end() is
76 // placed properly.
77 if (ompt_enabled.enabled)
78 __kmp_internal_end_library(__kmp_gtid_get_specific());
79#endif
80}
81
100kmp_int32 __kmpc_global_thread_num(ident_t *loc) {
101 kmp_int32 gtid = __kmp_entry_gtid();
102
103 KC_TRACE(10, ("__kmpc_global_thread_num: T#%d\n", gtid));
104
105 return gtid;
106}
107
122kmp_int32 __kmpc_global_num_threads(ident_t *loc) {
123 KC_TRACE(10,
124 ("__kmpc_global_num_threads: num_threads = %d\n", __kmp_all_nth));
125
126 return TCR_4(__kmp_all_nth);
127}
128
135kmp_int32 __kmpc_bound_thread_num(ident_t *loc) {
136 KC_TRACE(10, ("__kmpc_bound_thread_num: called\n"));
137 return __kmp_tid_from_gtid(__kmp_entry_gtid());
138}
139
145kmp_int32 __kmpc_bound_num_threads(ident_t *loc) {
146 KC_TRACE(10, ("__kmpc_bound_num_threads: called\n"));
147
148 return __kmp_entry_thread()->th.th_team->t.t_nproc;
149}
150
157kmp_int32 __kmpc_ok_to_fork(ident_t *loc) {
158#ifndef KMP_DEBUG
159
160 return TRUE;
161
162#else
163
164 const char *semi2;
165 const char *semi3;
166 int line_no;
167
168 if (__kmp_par_range == 0) {
169 return TRUE;
170 }
171 semi2 = loc->psource;
172 if (semi2 == NULL) {
173 return TRUE;
174 }
175 semi2 = strchr(semi2, ';');
176 if (semi2 == NULL) {
177 return TRUE;
178 }
179 semi2 = strchr(semi2 + 1, ';');
180 if (semi2 == NULL) {
181 return TRUE;
182 }
183 if (__kmp_par_range_filename[0]) {
184 const char *name = semi2 - 1;
185 while ((name > loc->psource) && (*name != '/') && (*name != ';')) {
186 name--;
187 }
188 if ((*name == '/') || (*name == ';')) {
189 name++;
190 }
191 if (strncmp(__kmp_par_range_filename, name, semi2 - name)) {
192 return __kmp_par_range < 0;
193 }
194 }
195 semi3 = strchr(semi2 + 1, ';');
196 if (__kmp_par_range_routine[0]) {
197 if ((semi3 != NULL) && (semi3 > semi2) &&
198 (strncmp(__kmp_par_range_routine, semi2 + 1, semi3 - semi2 - 1))) {
199 return __kmp_par_range < 0;
200 }
201 }
202 if (KMP_SSCANF(semi3 + 1, "%d", &line_no) == 1) {
203 if ((line_no >= __kmp_par_range_lb) && (line_no <= __kmp_par_range_ub)) {
204 return __kmp_par_range > 0;
205 }
206 return __kmp_par_range < 0;
207 }
208 return TRUE;
209
210#endif /* KMP_DEBUG */
211}
212
219kmp_int32 __kmpc_in_parallel(ident_t *loc) {
220 return __kmp_entry_thread()->th.th_root->r.r_active;
221}
222
232void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
233 kmp_int32 num_threads) {
234 KA_TRACE(20, ("__kmpc_push_num_threads: enter T#%d num_threads=%d\n",
235 global_tid, num_threads));
236 __kmp_assert_valid_gtid(global_tid);
237 __kmp_push_num_threads(loc, global_tid, num_threads);
238}
239
240void __kmpc_push_num_threads_strict(ident_t *loc, kmp_int32 global_tid,
241 kmp_int32 num_threads, int severity,
242 const char *message) {
243 __kmp_push_num_threads(loc, global_tid, num_threads);
244 __kmp_set_strict_num_threads(loc, global_tid, severity, message);
245}
246
260void __kmpc_push_num_threads_list(ident_t *loc, kmp_int32 global_tid,
261 kmp_uint32 list_length,
262 kmp_int32 *num_threads_list) {
263 KA_TRACE(20, ("__kmpc_push_num_threads_list: enter T#%d num_threads_list=",
264 global_tid));
265 KA_TRACE(20, ("%d", num_threads_list[0]));
266#ifdef KMP_DEBUG
267 for (kmp_uint32 i = 1; i < list_length; ++i)
268 KA_TRACE(20, (", %d", num_threads_list[i]));
269#endif
270 KA_TRACE(20, ("/n"));
271
272 __kmp_assert_valid_gtid(global_tid);
273 __kmp_push_num_threads_list(loc, global_tid, list_length, num_threads_list);
274}
275
276void __kmpc_push_num_threads_list_strict(ident_t *loc, kmp_int32 global_tid,
277 kmp_uint32 list_length,
278 kmp_int32 *num_threads_list,
279 int severity, const char *message) {
280 __kmp_push_num_threads_list(loc, global_tid, list_length, num_threads_list);
281 __kmp_set_strict_num_threads(loc, global_tid, severity, message);
282}
283
284void __kmpc_pop_num_threads(ident_t *loc, kmp_int32 global_tid) {
285 KA_TRACE(20, ("__kmpc_pop_num_threads: enter\n"));
286 /* the num_threads are automatically popped */
287}
288
289void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
290 kmp_int32 proc_bind) {
291 KA_TRACE(20, ("__kmpc_push_proc_bind: enter T#%d proc_bind=%d\n", global_tid,
292 proc_bind));
293 __kmp_assert_valid_gtid(global_tid);
294 __kmp_push_proc_bind(loc, global_tid, (kmp_proc_bind_t)proc_bind);
295}
296
307void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, ...) {
308 int gtid = __kmp_entry_gtid();
309
310#if (KMP_STATS_ENABLED)
311 // If we were in a serial region, then stop the serial timer, record
312 // the event, and start parallel region timer
313 stats_state_e previous_state = KMP_GET_THREAD_STATE();
314 if (previous_state == stats_state_e::SERIAL_REGION) {
315 KMP_EXCHANGE_PARTITIONED_TIMER(OMP_parallel_overhead);
316 } else {
317 KMP_PUSH_PARTITIONED_TIMER(OMP_parallel_overhead);
318 }
319 int inParallel = __kmpc_in_parallel(loc);
320 if (inParallel) {
321 KMP_COUNT_BLOCK(OMP_NESTED_PARALLEL);
322 } else {
323 KMP_COUNT_BLOCK(OMP_PARALLEL);
324 }
325#endif
326
327 // maybe to save thr_state is enough here
328 {
329 va_list ap;
330 va_start(ap, microtask);
331
332#if OMPT_SUPPORT
333 ompt_frame_t *ompt_frame;
334 if (ompt_enabled.enabled) {
335 kmp_info_t *master_th = __kmp_threads[gtid];
336 ompt_frame = &master_th->th.th_current_task->ompt_task_info.frame;
337 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
338 }
339 OMPT_STORE_RETURN_ADDRESS(gtid);
340#endif
341
342#if INCLUDE_SSC_MARKS
343 SSC_MARK_FORKING();
344#endif
345 __kmp_fork_call(loc, gtid, fork_context_intel, argc,
346 VOLATILE_CAST(microtask_t) microtask, // "wrapped" task
347 VOLATILE_CAST(launch_t) __kmp_invoke_task_func,
348 kmp_va_addr_of(ap));
349#if INCLUDE_SSC_MARKS
350 SSC_MARK_JOINING();
351#endif
352 __kmp_join_call(loc, gtid
353#if OMPT_SUPPORT
354 ,
355 fork_context_intel
356#endif
357 );
358
359 va_end(ap);
360
361#if OMPT_SUPPORT
362 if (ompt_enabled.enabled) {
363 ompt_frame->enter_frame = ompt_data_none;
364 }
365#endif
366 }
367
368#if KMP_STATS_ENABLED
369 if (previous_state == stats_state_e::SERIAL_REGION) {
370 KMP_EXCHANGE_PARTITIONED_TIMER(OMP_serial);
371 KMP_SET_THREAD_STATE(previous_state);
372 } else {
373 KMP_POP_PARTITIONED_TIMER();
374 }
375#endif // KMP_STATS_ENABLED
376}
377
388void __kmpc_fork_call_if(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,
389 kmp_int32 cond, void *args) {
390 int gtid = __kmp_entry_gtid();
391 if (cond) {
392 if (args)
393 __kmpc_fork_call(loc, argc, microtask, args);
394 else
395 __kmpc_fork_call(loc, argc, microtask);
396 } else {
397 __kmpc_serialized_parallel(loc, gtid);
398
399#if OMPT_SUPPORT
400 void *exit_frame_ptr;
401#endif
402
403 if (args)
404 __kmp_invoke_microtask(VOLATILE_CAST(microtask_t) microtask, gtid,
405 /*npr=*/0,
406 /*argc=*/1, &args
407#if OMPT_SUPPORT
408 ,
409 &exit_frame_ptr
410#endif
411 );
412 else
413 __kmp_invoke_microtask(VOLATILE_CAST(microtask_t) microtask, gtid,
414 /*npr=*/0,
415 /*argc=*/0,
416 /*args=*/nullptr
417#if OMPT_SUPPORT
418 ,
419 &exit_frame_ptr
420#endif
421 );
422
423 __kmpc_end_serialized_parallel(loc, gtid);
424 }
425}
426
438void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid,
439 kmp_int32 num_teams, kmp_int32 num_threads) {
440 KA_TRACE(20,
441 ("__kmpc_push_num_teams: enter T#%d num_teams=%d num_threads=%d\n",
442 global_tid, num_teams, num_threads));
443 __kmp_assert_valid_gtid(global_tid);
444 __kmp_push_num_teams(loc, global_tid, num_teams, num_threads);
445}
446
457void __kmpc_set_thread_limit(ident_t *loc, kmp_int32 global_tid,
458 kmp_int32 thread_limit) {
459 __kmp_assert_valid_gtid(global_tid);
460 kmp_info_t *thread = __kmp_threads[global_tid];
461 if (thread_limit > 0)
462 thread->th.th_current_task->td_icvs.task_thread_limit = thread_limit;
463}
464
481void __kmpc_push_num_teams_51(ident_t *loc, kmp_int32 global_tid,
482 kmp_int32 num_teams_lb, kmp_int32 num_teams_ub,
483 kmp_int32 num_threads) {
484 KA_TRACE(20, ("__kmpc_push_num_teams_51: enter T#%d num_teams_lb=%d"
485 " num_teams_ub=%d num_threads=%d\n",
486 global_tid, num_teams_lb, num_teams_ub, num_threads));
487 __kmp_assert_valid_gtid(global_tid);
488 __kmp_push_num_teams_51(loc, global_tid, num_teams_lb, num_teams_ub,
489 num_threads);
490}
491
502void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,
503 ...) {
504 int gtid = __kmp_entry_gtid();
505 kmp_info_t *this_thr = __kmp_threads[gtid];
506 va_list ap;
507 va_start(ap, microtask);
508
509#if KMP_STATS_ENABLED
510 KMP_COUNT_BLOCK(OMP_TEAMS);
511 stats_state_e previous_state = KMP_GET_THREAD_STATE();
512 if (previous_state == stats_state_e::SERIAL_REGION) {
513 KMP_EXCHANGE_PARTITIONED_TIMER(OMP_teams_overhead);
514 } else {
515 KMP_PUSH_PARTITIONED_TIMER(OMP_teams_overhead);
516 }
517#endif
518
519 // remember teams entry point and nesting level
520 this_thr->th.th_teams_microtask = microtask;
521 this_thr->th.th_teams_level =
522 this_thr->th.th_team->t.t_level; // AC: can be >0 on host
523
524#if OMPT_SUPPORT
525 kmp_team_t *parent_team = this_thr->th.th_team;
526 int tid = __kmp_tid_from_gtid(gtid);
527 if (ompt_enabled.enabled) {
528 parent_team->t.t_implicit_task_taskdata[tid]
529 .ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
530 }
531 OMPT_STORE_RETURN_ADDRESS(gtid);
532#endif
533
534 // check if __kmpc_push_num_teams called, set default number of teams
535 // otherwise
536 if (this_thr->th.th_teams_size.nteams == 0) {
537 __kmp_push_num_teams(loc, gtid, 0, 0);
538 }
539 KMP_DEBUG_ASSERT(this_thr->th.th_set_nproc >= 1);
540 KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nteams >= 1);
541 KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nth >= 1);
542
543 __kmp_fork_call(
544 loc, gtid, fork_context_intel, argc,
545 VOLATILE_CAST(microtask_t) __kmp_teams_master, // "wrapped" task
546 VOLATILE_CAST(launch_t) __kmp_invoke_teams_master, kmp_va_addr_of(ap));
547 __kmp_join_call(loc, gtid
548#if OMPT_SUPPORT
549 ,
550 fork_context_intel
551#endif
552 );
553
554 // Pop current CG root off list
555 KMP_DEBUG_ASSERT(this_thr->th.th_cg_roots);
556 kmp_cg_root_t *tmp = this_thr->th.th_cg_roots;
557 this_thr->th.th_cg_roots = tmp->up;
558 KA_TRACE(100, ("__kmpc_fork_teams: Thread %p popping node %p and moving up"
559 " to node %p. cg_nthreads was %d\n",
560 this_thr, tmp, this_thr->th.th_cg_roots, tmp->cg_nthreads));
561 KMP_DEBUG_ASSERT(tmp->cg_nthreads);
562 int i = tmp->cg_nthreads--;
563 if (i == 1) { // check is we are the last thread in CG (not always the case)
564 __kmp_free(tmp);
565 }
566 // Restore current task's thread_limit from CG root
567 KMP_DEBUG_ASSERT(this_thr->th.th_cg_roots);
568 this_thr->th.th_current_task->td_icvs.thread_limit =
569 this_thr->th.th_cg_roots->cg_thread_limit;
570
571 this_thr->th.th_teams_microtask = NULL;
572 this_thr->th.th_teams_level = 0;
573 memset(&this_thr->th.th_teams_size, 0, sizeof(kmp_teams_size_t));
574 va_end(ap);
575#if KMP_STATS_ENABLED
576 if (previous_state == stats_state_e::SERIAL_REGION) {
577 KMP_EXCHANGE_PARTITIONED_TIMER(OMP_serial);
578 KMP_SET_THREAD_STATE(previous_state);
579 } else {
580 KMP_POP_PARTITIONED_TIMER();
581 }
582#endif // KMP_STATS_ENABLED
583}
584
585// I don't think this function should ever have been exported.
586// The __kmpc_ prefix was misapplied. I'm fairly certain that no generated
587// openmp code ever called it, but it's been exported from the RTL for so
588// long that I'm afraid to remove the definition.
589int __kmpc_invoke_task_func(int gtid) { return __kmp_invoke_task_func(gtid); }
590
603void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
604 // The implementation is now in kmp_runtime.cpp so that it can share static
605 // functions with kmp_fork_call since the tasks to be done are similar in
606 // each case.
607 __kmp_assert_valid_gtid(global_tid);
608#if OMPT_SUPPORT
609 OMPT_STORE_RETURN_ADDRESS(global_tid);
610#endif
611 __kmp_serialized_parallel(loc, global_tid);
612}
613
621void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
622 kmp_internal_control_t *top;
623 kmp_info_t *this_thr;
624 kmp_team_t *serial_team;
625
626 KC_TRACE(10,
627 ("__kmpc_end_serialized_parallel: called by T#%d\n", global_tid));
628
629 /* skip all this code for autopar serialized loops since it results in
630 unacceptable overhead */
631 if (loc != NULL && (loc->flags & KMP_IDENT_AUTOPAR))
632 return;
633
634 // Not autopar code
635 __kmp_assert_valid_gtid(global_tid);
636 if (!TCR_4(__kmp_init_parallel))
637 __kmp_parallel_initialize();
638
639 __kmp_resume_if_soft_paused();
640
641 this_thr = __kmp_threads[global_tid];
642 serial_team = this_thr->th.th_serial_team;
643
644 kmp_task_team_t *task_team = this_thr->th.th_task_team;
645 // we need to wait for the proxy tasks before finishing the thread
646 if (task_team != NULL && (task_team->tt.tt_found_proxy_tasks ||
647 task_team->tt.tt_hidden_helper_task_encountered))
648 __kmp_task_team_wait(this_thr, serial_team USE_ITT_BUILD_ARG(NULL));
649
650 KMP_MB();
651 KMP_DEBUG_ASSERT(serial_team);
652 KMP_ASSERT(serial_team->t.t_serialized);
653 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team);
654 KMP_DEBUG_ASSERT(serial_team != this_thr->th.th_root->r.r_root_team);
655 KMP_DEBUG_ASSERT(serial_team->t.t_threads);
656 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
657
658#if OMPT_SUPPORT
659 if (ompt_enabled.enabled &&
660 this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
661 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame = ompt_data_none;
662 if (ompt_enabled.ompt_callback_implicit_task) {
663 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
664 ompt_scope_end, NULL, OMPT_CUR_TASK_DATA(this_thr), 1,
665 OMPT_CUR_TASK_INFO(this_thr)->thread_num, ompt_task_implicit);
666 }
667
668 // reset clear the task id only after unlinking the task
669 ompt_data_t *parent_task_data;
670 __ompt_get_task_info_internal(1, NULL, &parent_task_data, NULL, NULL, NULL);
671
672 if (ompt_enabled.ompt_callback_parallel_end) {
673 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
674 &(serial_team->t.ompt_team_info.parallel_data), parent_task_data,
675 ompt_parallel_invoker_program | ompt_parallel_team,
676 OMPT_LOAD_RETURN_ADDRESS(global_tid));
677 }
678 __ompt_lw_taskteam_unlink(this_thr);
679 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
680 }
681#endif
682
683 /* If necessary, pop the internal control stack values and replace the team
684 * values */
685 top = serial_team->t.t_control_stack_top;
686 if (top && top->serial_nesting_level == serial_team->t.t_serialized) {
687 copy_icvs(&serial_team->t.t_threads[0]->th.th_current_task->td_icvs, top);
688 serial_team->t.t_control_stack_top = top->next;
689 __kmp_free(top);
690 }
691
692 /* pop dispatch buffers stack */
693 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch->th_disp_buffer);
694 {
695 dispatch_private_info_t *disp_buffer =
696 serial_team->t.t_dispatch->th_disp_buffer;
697 serial_team->t.t_dispatch->th_disp_buffer =
698 serial_team->t.t_dispatch->th_disp_buffer->next;
699 __kmp_free(disp_buffer);
700 }
701
702 /* pop the task team stack */
703 if (serial_team->t.t_serialized > 1) {
704 __kmp_pop_task_team_node(this_thr, serial_team);
705 }
706
707 this_thr->th.th_def_allocator = serial_team->t.t_def_allocator; // restore
708
709 --serial_team->t.t_serialized;
710 if (serial_team->t.t_serialized == 0) {
711
712 /* return to the parallel section */
713
714#if KMP_ARCH_X86 || KMP_ARCH_X86_64
715 if (__kmp_inherit_fp_control && serial_team->t.t_fp_control_saved) {
716 __kmp_clear_x87_fpu_status_word();
717 __kmp_load_x87_fpu_control_word(&serial_team->t.t_x87_fpu_control_word);
718 __kmp_load_mxcsr(&serial_team->t.t_mxcsr);
719 }
720#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
721
722 __kmp_pop_current_task_from_thread(this_thr);
723#if OMPD_SUPPORT
724 if (ompd_state & OMPD_ENABLE_BP)
725 ompd_bp_parallel_end();
726#endif
727
728 this_thr->th.th_team = serial_team->t.t_parent;
729 this_thr->th.th_info.ds.ds_tid = serial_team->t.t_master_tid;
730
731 /* restore values cached in the thread */
732 this_thr->th.th_team_nproc = serial_team->t.t_parent->t.t_nproc; /* JPH */
733 this_thr->th.th_team_master =
734 serial_team->t.t_parent->t.t_threads[0]; /* JPH */
735 this_thr->th.th_team_serialized = this_thr->th.th_team->t.t_serialized;
736
737 /* TODO the below shouldn't need to be adjusted for serialized teams */
738 this_thr->th.th_dispatch =
739 &this_thr->th.th_team->t.t_dispatch[serial_team->t.t_master_tid];
740
741 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 0);
742 this_thr->th.th_current_task->td_flags.executing = 1;
743
744 if (__kmp_tasking_mode != tskm_immediate_exec) {
745 // Restore task state from serial team structure
746 KMP_DEBUG_ASSERT(serial_team->t.t_primary_task_state == 0 ||
747 serial_team->t.t_primary_task_state == 1);
748 this_thr->th.th_task_state =
749 (kmp_uint8)serial_team->t.t_primary_task_state;
750 // Copy the task team from the new child / old parent team to the thread.
751 this_thr->th.th_task_team =
752 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state];
753 KA_TRACE(20,
754 ("__kmpc_end_serialized_parallel: T#%d restoring task_team %p / "
755 "team %p\n",
756 global_tid, this_thr->th.th_task_team, this_thr->th.th_team));
757 }
758#if KMP_AFFINITY_SUPPORTED
759 if (this_thr->th.th_team->t.t_level == 0 && __kmp_affinity.flags.reset) {
760 __kmp_reset_root_init_mask(global_tid);
761 }
762#endif
763 } else {
764 if (__kmp_tasking_mode != tskm_immediate_exec) {
765 KA_TRACE(20, ("__kmpc_end_serialized_parallel: T#%d decreasing nesting "
766 "depth of serial team %p to %d\n",
767 global_tid, serial_team, serial_team->t.t_serialized));
768 }
769 }
770
771 serial_team->t.t_level--;
772 if (__kmp_env_consistency_check)
773 __kmp_pop_parallel(global_tid, NULL);
774#if OMPT_SUPPORT
775 if (ompt_enabled.enabled)
776 this_thr->th.ompt_thread_info.state =
777 ((this_thr->th.th_team_serialized) ? ompt_state_work_serial
778 : ompt_state_work_parallel);
779#endif
780}
781
790void __kmpc_flush(ident_t *loc) {
791 KC_TRACE(10, ("__kmpc_flush: called\n"));
792
793 /* need explicit __mf() here since use volatile instead in library */
794 KMP_MFENCE(); /* Flush all pending memory write invalidates. */
795
796#if OMPT_SUPPORT && OMPT_OPTIONAL
797 if (ompt_enabled.ompt_callback_flush) {
798 ompt_callbacks.ompt_callback(ompt_callback_flush)(
799 __ompt_get_thread_data_internal(), OMPT_GET_RETURN_ADDRESS(0));
800 }
801#endif
802}
803
811void __kmpc_flush_explicit(ident_t *loc, kmp_int32 order, kmp_int32 scope) {
812 KC_TRACE(10, ("__kmpc_flush_explicit: called\n"));
813
814 // `scope` is not used on the initial device: a host fence orders memory for
815 // all threads whatever the scope.
816 switch (order) {
817 case kmp_flush_order_relaxed:
818 std::atomic_thread_fence(std::memory_order_relaxed);
819 break;
820 case kmp_flush_order_consume:
821 [[fallthrough]];
822 case kmp_flush_order_acquire:
823 std::atomic_thread_fence(std::memory_order_acquire);
824 break;
825 case kmp_flush_order_release:
826 std::atomic_thread_fence(std::memory_order_release);
827 break;
828 case kmp_flush_order_acq_rel:
829 std::atomic_thread_fence(std::memory_order_acq_rel);
830 break;
831 case kmp_flush_order_seq_cst:
832 [[fallthrough]];
833 default:
834 // An order this runtime does not know gets the strongest fence.
835 std::atomic_thread_fence(std::memory_order_seq_cst);
836 break;
837 }
838
839#if OMPT_SUPPORT && OMPT_OPTIONAL
840 if (ompt_enabled.ompt_callback_flush) {
841 ompt_callbacks.ompt_callback(ompt_callback_flush)(
842 __ompt_get_thread_data_internal(), OMPT_GET_RETURN_ADDRESS(0));
843 }
844#endif
845}
846
847/* -------------------------------------------------------------------------- */
855void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid) {
856 KMP_COUNT_BLOCK(OMP_BARRIER);
857 KC_TRACE(10, ("__kmpc_barrier: called T#%d\n", global_tid));
858 __kmp_assert_valid_gtid(global_tid);
859
860 if (!TCR_4(__kmp_init_parallel))
861 __kmp_parallel_initialize();
862
863 __kmp_resume_if_soft_paused();
864
865 if (__kmp_env_consistency_check) {
866 if (loc == 0) {
867 KMP_WARNING(ConstructIdentInvalid); // ??? What does it mean for the user?
868 }
869 __kmp_check_barrier(global_tid, ct_barrier, loc);
870 }
871
872#if OMPT_SUPPORT
873 ompt_frame_t *ompt_frame;
874 if (ompt_enabled.enabled) {
875 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
876 if (ompt_frame->enter_frame.ptr == NULL)
877 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
878 }
879 OMPT_STORE_RETURN_ADDRESS(global_tid);
880#endif
881 __kmp_threads[global_tid]->th.th_ident = loc;
882 // TODO: explicit barrier_wait_id:
883 // this function is called when 'barrier' directive is present or
884 // implicit barrier at the end of a worksharing construct.
885 // 1) better to add a per-thread barrier counter to a thread data structure
886 // 2) set to 0 when a new team is created
887 // 4) no sync is required
888
889 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
890#if OMPT_SUPPORT && OMPT_OPTIONAL
891 if (ompt_enabled.enabled) {
892 ompt_frame->enter_frame = ompt_data_none;
893 }
894#endif
895}
896
897/* The BARRIER for a MASTER section is always explicit */
904kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid) {
905 int status = 0;
906
907 KC_TRACE(10, ("__kmpc_master: called T#%d\n", global_tid));
908 __kmp_assert_valid_gtid(global_tid);
909
910 if (!TCR_4(__kmp_init_parallel))
911 __kmp_parallel_initialize();
912
913 __kmp_resume_if_soft_paused();
914
915 if (KMP_MASTER_GTID(global_tid)) {
916 KMP_COUNT_BLOCK(OMP_MASTER);
917 KMP_PUSH_PARTITIONED_TIMER(OMP_master);
918 status = 1;
919 }
920
921#if OMPT_SUPPORT && OMPT_OPTIONAL
922 if (status) {
923 if (ompt_enabled.ompt_callback_masked) {
924 kmp_info_t *this_thr = __kmp_threads[global_tid];
925 kmp_team_t *team = this_thr->th.th_team;
926
927 int tid = __kmp_tid_from_gtid(global_tid);
928 ompt_callbacks.ompt_callback(ompt_callback_masked)(
929 ompt_scope_begin, &(team->t.ompt_team_info.parallel_data),
930 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
931 OMPT_GET_RETURN_ADDRESS(0));
932 }
933 }
934#endif
935
936 if (__kmp_env_consistency_check) {
937#if KMP_USE_DYNAMIC_LOCK
938 if (status)
939 __kmp_push_sync(global_tid, ct_master, loc, NULL, 0);
940 else
941 __kmp_check_sync(global_tid, ct_master, loc, NULL, 0);
942#else
943 if (status)
944 __kmp_push_sync(global_tid, ct_master, loc, NULL);
945 else
946 __kmp_check_sync(global_tid, ct_master, loc, NULL);
947#endif
948 }
949
950 return status;
951}
952
961void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid) {
962 KC_TRACE(10, ("__kmpc_end_master: called T#%d\n", global_tid));
963 __kmp_assert_valid_gtid(global_tid);
964 KMP_DEBUG_ASSERT(KMP_MASTER_GTID(global_tid));
965 KMP_POP_PARTITIONED_TIMER();
966
967#if OMPT_SUPPORT && OMPT_OPTIONAL
968 kmp_info_t *this_thr = __kmp_threads[global_tid];
969 kmp_team_t *team = this_thr->th.th_team;
970 if (ompt_enabled.ompt_callback_masked) {
971 int tid = __kmp_tid_from_gtid(global_tid);
972 ompt_callbacks.ompt_callback(ompt_callback_masked)(
973 ompt_scope_end, &(team->t.ompt_team_info.parallel_data),
974 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
975 OMPT_GET_RETURN_ADDRESS(0));
976 }
977#endif
978
979 if (__kmp_env_consistency_check) {
980 if (KMP_MASTER_GTID(global_tid))
981 __kmp_pop_sync(global_tid, ct_master, loc);
982 }
983}
984
993kmp_int32 __kmpc_masked(ident_t *loc, kmp_int32 global_tid, kmp_int32 filter) {
994 int status = 0;
995 int tid;
996 KC_TRACE(10, ("__kmpc_masked: called T#%d\n", global_tid));
997 __kmp_assert_valid_gtid(global_tid);
998
999 if (!TCR_4(__kmp_init_parallel))
1000 __kmp_parallel_initialize();
1001
1002 __kmp_resume_if_soft_paused();
1003
1004 tid = __kmp_tid_from_gtid(global_tid);
1005 if (tid == filter) {
1006 KMP_COUNT_BLOCK(OMP_MASKED);
1007 KMP_PUSH_PARTITIONED_TIMER(OMP_masked);
1008 status = 1;
1009 }
1010
1011#if OMPT_SUPPORT && OMPT_OPTIONAL
1012 if (status) {
1013 if (ompt_enabled.ompt_callback_masked) {
1014 kmp_info_t *this_thr = __kmp_threads[global_tid];
1015 kmp_team_t *team = this_thr->th.th_team;
1016 ompt_callbacks.ompt_callback(ompt_callback_masked)(
1017 ompt_scope_begin, &(team->t.ompt_team_info.parallel_data),
1018 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1019 OMPT_GET_RETURN_ADDRESS(0));
1020 }
1021 }
1022#endif
1023
1024 if (__kmp_env_consistency_check) {
1025#if KMP_USE_DYNAMIC_LOCK
1026 if (status)
1027 __kmp_push_sync(global_tid, ct_masked, loc, NULL, 0);
1028 else
1029 __kmp_check_sync(global_tid, ct_masked, loc, NULL, 0);
1030#else
1031 if (status)
1032 __kmp_push_sync(global_tid, ct_masked, loc, NULL);
1033 else
1034 __kmp_check_sync(global_tid, ct_masked, loc, NULL);
1035#endif
1036 }
1037
1038 return status;
1039}
1040
1049void __kmpc_end_masked(ident_t *loc, kmp_int32 global_tid) {
1050 KC_TRACE(10, ("__kmpc_end_masked: called T#%d\n", global_tid));
1051 __kmp_assert_valid_gtid(global_tid);
1052 KMP_POP_PARTITIONED_TIMER();
1053
1054#if OMPT_SUPPORT && OMPT_OPTIONAL
1055 kmp_info_t *this_thr = __kmp_threads[global_tid];
1056 kmp_team_t *team = this_thr->th.th_team;
1057 if (ompt_enabled.ompt_callback_masked) {
1058 int tid = __kmp_tid_from_gtid(global_tid);
1059 ompt_callbacks.ompt_callback(ompt_callback_masked)(
1060 ompt_scope_end, &(team->t.ompt_team_info.parallel_data),
1061 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1062 OMPT_GET_RETURN_ADDRESS(0));
1063 }
1064#endif
1065
1066 if (__kmp_env_consistency_check) {
1067 __kmp_pop_sync(global_tid, ct_masked, loc);
1068 }
1069}
1070
1078void __kmpc_ordered(ident_t *loc, kmp_int32 gtid) {
1079 int cid = 0;
1080 kmp_info_t *th;
1081 KMP_DEBUG_ASSERT(__kmp_init_serial);
1082
1083 KC_TRACE(10, ("__kmpc_ordered: called T#%d\n", gtid));
1084 __kmp_assert_valid_gtid(gtid);
1085
1086 if (!TCR_4(__kmp_init_parallel))
1087 __kmp_parallel_initialize();
1088
1089 __kmp_resume_if_soft_paused();
1090
1091#if USE_ITT_BUILD
1092 __kmp_itt_ordered_prep(gtid);
1093// TODO: ordered_wait_id
1094#endif /* USE_ITT_BUILD */
1095
1096 th = __kmp_threads[gtid];
1097
1098#if OMPT_SUPPORT && OMPT_OPTIONAL
1099 kmp_team_t *team;
1100 ompt_wait_id_t lck;
1101 void *codeptr_ra;
1102 OMPT_STORE_RETURN_ADDRESS(gtid);
1103 if (ompt_enabled.enabled) {
1104 team = __kmp_team_from_gtid(gtid);
1105 lck = (ompt_wait_id_t)(uintptr_t)&team->t.t_ordered.dt.t_value;
1106 /* OMPT state update */
1107 th->th.ompt_thread_info.wait_id = lck;
1108 th->th.ompt_thread_info.state = ompt_state_wait_ordered;
1109
1110 /* OMPT event callback */
1111 codeptr_ra = OMPT_LOAD_RETURN_ADDRESS(gtid);
1112 if (ompt_enabled.ompt_callback_mutex_acquire) {
1113 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1114 ompt_mutex_ordered, omp_lock_hint_none, kmp_mutex_impl_spin, lck,
1115 codeptr_ra);
1116 }
1117 }
1118#endif
1119
1120 if (th->th.th_dispatch->th_deo_fcn != 0)
1121 (*th->th.th_dispatch->th_deo_fcn)(&gtid, &cid, loc);
1122 else
1123 __kmp_parallel_deo(&gtid, &cid, loc);
1124
1125#if OMPT_SUPPORT && OMPT_OPTIONAL
1126 if (ompt_enabled.enabled) {
1127 /* OMPT state update */
1128 th->th.ompt_thread_info.state = ompt_state_work_parallel;
1129 th->th.ompt_thread_info.wait_id = 0;
1130
1131 /* OMPT event callback */
1132 if (ompt_enabled.ompt_callback_mutex_acquired) {
1133 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
1134 ompt_mutex_ordered, (ompt_wait_id_t)(uintptr_t)lck, codeptr_ra);
1135 }
1136 }
1137#endif
1138
1139#if USE_ITT_BUILD
1140 __kmp_itt_ordered_start(gtid);
1141#endif /* USE_ITT_BUILD */
1142}
1143
1151void __kmpc_end_ordered(ident_t *loc, kmp_int32 gtid) {
1152 int cid = 0;
1153 kmp_info_t *th;
1154
1155 KC_TRACE(10, ("__kmpc_end_ordered: called T#%d\n", gtid));
1156 __kmp_assert_valid_gtid(gtid);
1157
1158#if USE_ITT_BUILD
1159 __kmp_itt_ordered_end(gtid);
1160// TODO: ordered_wait_id
1161#endif /* USE_ITT_BUILD */
1162
1163 th = __kmp_threads[gtid];
1164
1165 if (th->th.th_dispatch->th_dxo_fcn != 0)
1166 (*th->th.th_dispatch->th_dxo_fcn)(&gtid, &cid, loc);
1167 else
1168 __kmp_parallel_dxo(&gtid, &cid, loc);
1169
1170#if OMPT_SUPPORT && OMPT_OPTIONAL
1171 OMPT_STORE_RETURN_ADDRESS(gtid);
1172 if (ompt_enabled.ompt_callback_mutex_released) {
1173 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
1174 ompt_mutex_ordered,
1175 (ompt_wait_id_t)(uintptr_t)&__kmp_team_from_gtid(gtid)
1176 ->t.t_ordered.dt.t_value,
1177 OMPT_LOAD_RETURN_ADDRESS(gtid));
1178 }
1179#endif
1180}
1181
1182#if KMP_USE_DYNAMIC_LOCK
1183
1184static __forceinline void
1185__kmp_init_indirect_csptr(kmp_critical_name *crit, ident_t const *loc,
1186 kmp_int32 gtid, kmp_indirect_locktag_t tag) {
1187 // Pointer to the allocated indirect lock is written to crit, while indexing
1188 // is ignored.
1189 void *idx;
1190 kmp_indirect_lock_t **lck;
1191 lck = (kmp_indirect_lock_t **)crit;
1192 kmp_indirect_lock_t *ilk = __kmp_allocate_indirect_lock(&idx, gtid, tag);
1193 KMP_I_LOCK_FUNC(ilk, init)(ilk->lock);
1194 KMP_SET_I_LOCK_LOCATION(ilk, loc);
1195 KMP_SET_I_LOCK_FLAGS(ilk, kmp_lf_critical_section);
1196 KA_TRACE(20,
1197 ("__kmp_init_indirect_csptr: initialized indirect lock #%d\n", tag));
1198#if USE_ITT_BUILD
1199 __kmp_itt_critical_creating(ilk->lock, loc);
1200#endif
1201 int status = KMP_COMPARE_AND_STORE_PTR(lck, nullptr, ilk);
1202 if (status == 0) {
1203#if USE_ITT_BUILD
1204 __kmp_itt_critical_destroyed(ilk->lock);
1205#endif
1206 // We don't really need to destroy the unclaimed lock here since it will be
1207 // cleaned up at program exit.
1208 // KMP_D_LOCK_FUNC(&idx, destroy)((kmp_dyna_lock_t *)&idx);
1209 }
1210 KMP_DEBUG_ASSERT(*lck != NULL);
1211}
1212
1213// Fast-path acquire tas lock
1214#define KMP_ACQUIRE_TAS_LOCK(lock, gtid) \
1215 { \
1216 kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \
1217 kmp_int32 tas_free = KMP_LOCK_FREE(tas); \
1218 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas); \
1219 if (KMP_ATOMIC_LD_RLX(&l->lk.poll) != tas_free || \
1220 !__kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy)) { \
1221 kmp_uint32 spins; \
1222 KMP_FSYNC_PREPARE(l); \
1223 KMP_INIT_YIELD(spins); \
1224 kmp_backoff_t backoff = __kmp_spin_backoff_params; \
1225 do { \
1226 if (TCR_4(__kmp_nth) > \
1227 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \
1228 KMP_YIELD(TRUE); \
1229 } else { \
1230 KMP_YIELD_SPIN(spins); \
1231 } \
1232 __kmp_spin_backoff(&backoff); \
1233 } while ( \
1234 KMP_ATOMIC_LD_RLX(&l->lk.poll) != tas_free || \
1235 !__kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy)); \
1236 } \
1237 KMP_FSYNC_ACQUIRED(l); \
1238 }
1239
1240// Fast-path test tas lock
1241#define KMP_TEST_TAS_LOCK(lock, gtid, rc) \
1242 { \
1243 kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \
1244 kmp_int32 tas_free = KMP_LOCK_FREE(tas); \
1245 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas); \
1246 rc = KMP_ATOMIC_LD_RLX(&l->lk.poll) == tas_free && \
1247 __kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy); \
1248 }
1249
1250// Fast-path release tas lock
1251#define KMP_RELEASE_TAS_LOCK(lock, gtid) \
1252 { KMP_ATOMIC_ST_REL(&((kmp_tas_lock_t *)lock)->lk.poll, KMP_LOCK_FREE(tas)); }
1253
1254#if KMP_USE_FUTEX
1255
1256#include <sys/syscall.h>
1257#include <unistd.h>
1258#ifndef FUTEX_WAIT
1259#define FUTEX_WAIT 0
1260#endif
1261#ifndef FUTEX_WAKE
1262#define FUTEX_WAKE 1
1263#endif
1264
1265// Fast-path acquire futex lock
1266#define KMP_ACQUIRE_FUTEX_LOCK(lock, gtid) \
1267 { \
1268 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
1269 kmp_int32 gtid_code = (gtid + 1) << 1; \
1270 KMP_MB(); \
1271 KMP_FSYNC_PREPARE(ftx); \
1272 kmp_int32 poll_val; \
1273 while ((poll_val = KMP_COMPARE_AND_STORE_RET32( \
1274 &(ftx->lk.poll), KMP_LOCK_FREE(futex), \
1275 KMP_LOCK_BUSY(gtid_code, futex))) != KMP_LOCK_FREE(futex)) { \
1276 kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1; \
1277 if (!cond) { \
1278 if (!KMP_COMPARE_AND_STORE_RET32(&(ftx->lk.poll), poll_val, \
1279 poll_val | \
1280 KMP_LOCK_BUSY(1, futex))) { \
1281 continue; \
1282 } \
1283 poll_val |= KMP_LOCK_BUSY(1, futex); \
1284 } \
1285 kmp_int32 rc; \
1286 if ((rc = syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAIT, poll_val, \
1287 NULL, NULL, 0)) != 0) { \
1288 continue; \
1289 } \
1290 gtid_code |= 1; \
1291 } \
1292 KMP_FSYNC_ACQUIRED(ftx); \
1293 }
1294
1295// Fast-path test futex lock
1296#define KMP_TEST_FUTEX_LOCK(lock, gtid, rc) \
1297 { \
1298 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
1299 if (KMP_COMPARE_AND_STORE_ACQ32(&(ftx->lk.poll), KMP_LOCK_FREE(futex), \
1300 KMP_LOCK_BUSY(gtid + 1 << 1, futex))) { \
1301 KMP_FSYNC_ACQUIRED(ftx); \
1302 rc = TRUE; \
1303 } else { \
1304 rc = FALSE; \
1305 } \
1306 }
1307
1308// Fast-path release futex lock
1309#define KMP_RELEASE_FUTEX_LOCK(lock, gtid) \
1310 { \
1311 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
1312 KMP_MB(); \
1313 KMP_FSYNC_RELEASING(ftx); \
1314 kmp_int32 poll_val = \
1315 KMP_XCHG_FIXED32(&(ftx->lk.poll), KMP_LOCK_FREE(futex)); \
1316 if (KMP_LOCK_STRIP(poll_val) & 1) { \
1317 syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAKE, \
1318 KMP_LOCK_BUSY(1, futex), NULL, NULL, 0); \
1319 } \
1320 KMP_MB(); \
1321 KMP_YIELD_OVERSUB(); \
1322 }
1323
1324#endif // KMP_USE_FUTEX
1325
1326#else // KMP_USE_DYNAMIC_LOCK
1327
1328static kmp_user_lock_p __kmp_get_critical_section_ptr(kmp_critical_name *crit,
1329 ident_t const *loc,
1330 kmp_int32 gtid) {
1331 kmp_user_lock_p *lck_pp = (kmp_user_lock_p *)crit;
1332
1333 // Because of the double-check, the following load doesn't need to be volatile
1334 kmp_user_lock_p lck = (kmp_user_lock_p)TCR_PTR(*lck_pp);
1335
1336 if (lck == NULL) {
1337 void *idx;
1338
1339 // Allocate & initialize the lock.
1340 // Remember alloc'ed locks in table in order to free them in __kmp_cleanup()
1341 lck = __kmp_user_lock_allocate(&idx, gtid, kmp_lf_critical_section);
1342 __kmp_init_user_lock_with_checks(lck);
1343 __kmp_set_user_lock_location(lck, loc);
1344#if USE_ITT_BUILD
1345 __kmp_itt_critical_creating(lck);
1346// __kmp_itt_critical_creating() should be called *before* the first usage
1347// of underlying lock. It is the only place where we can guarantee it. There
1348// are chances the lock will destroyed with no usage, but it is not a
1349// problem, because this is not real event seen by user but rather setting
1350// name for object (lock). See more details in kmp_itt.h.
1351#endif /* USE_ITT_BUILD */
1352
1353 // Use a cmpxchg instruction to slam the start of the critical section with
1354 // the lock pointer. If another thread beat us to it, deallocate the lock,
1355 // and use the lock that the other thread allocated.
1356 int status = KMP_COMPARE_AND_STORE_PTR(lck_pp, 0, lck);
1357
1358 if (status == 0) {
1359// Deallocate the lock and reload the value.
1360#if USE_ITT_BUILD
1361 __kmp_itt_critical_destroyed(lck);
1362// Let ITT know the lock is destroyed and the same memory location may be reused
1363// for another purpose.
1364#endif /* USE_ITT_BUILD */
1365 __kmp_destroy_user_lock_with_checks(lck);
1366 __kmp_user_lock_free(&idx, gtid, lck);
1367 lck = (kmp_user_lock_p)TCR_PTR(*lck_pp);
1368 KMP_DEBUG_ASSERT(lck != NULL);
1369 }
1370 }
1371 return lck;
1372}
1373
1374#endif // KMP_USE_DYNAMIC_LOCK
1375
1386void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
1387 kmp_critical_name *crit) {
1388#if KMP_USE_DYNAMIC_LOCK
1389#if OMPT_SUPPORT && OMPT_OPTIONAL
1390 OMPT_STORE_RETURN_ADDRESS(global_tid);
1391#endif // OMPT_SUPPORT
1392 __kmpc_critical_with_hint(loc, global_tid, crit, omp_lock_hint_none);
1393#else
1394 KMP_COUNT_BLOCK(OMP_CRITICAL);
1395#if OMPT_SUPPORT && OMPT_OPTIONAL
1396 ompt_state_t prev_state = ompt_state_undefined;
1397 ompt_thread_info_t ti;
1398#endif
1399 kmp_user_lock_p lck;
1400
1401 KC_TRACE(10, ("__kmpc_critical: called T#%d\n", global_tid));
1402 __kmp_assert_valid_gtid(global_tid);
1403
1404 // TODO: add THR_OVHD_STATE
1405
1406 KMP_PUSH_PARTITIONED_TIMER(OMP_critical_wait);
1407 KMP_CHECK_USER_LOCK_INIT();
1408
1409 if ((__kmp_user_lock_kind == lk_tas) &&
1410 (sizeof(lck->tas.lk.poll) <= OMP_CRITICAL_SIZE)) {
1411 lck = (kmp_user_lock_p)crit;
1412 }
1413#if KMP_USE_FUTEX
1414 else if ((__kmp_user_lock_kind == lk_futex) &&
1415 (sizeof(lck->futex.lk.poll) <= OMP_CRITICAL_SIZE)) {
1416 lck = (kmp_user_lock_p)crit;
1417 }
1418#endif
1419 else { // ticket, queuing or drdpa
1420 lck = __kmp_get_critical_section_ptr(crit, loc, global_tid);
1421 }
1422
1423 if (__kmp_env_consistency_check)
1424 __kmp_push_sync(global_tid, ct_critical, loc, lck);
1425
1426 // since the critical directive binds to all threads, not just the current
1427 // team we have to check this even if we are in a serialized team.
1428 // also, even if we are the uber thread, we still have to conduct the lock,
1429 // as we have to contend with sibling threads.
1430
1431#if USE_ITT_BUILD
1432 __kmp_itt_critical_acquiring(lck);
1433#endif /* USE_ITT_BUILD */
1434#if OMPT_SUPPORT && OMPT_OPTIONAL
1435 OMPT_STORE_RETURN_ADDRESS(gtid);
1436 void *codeptr_ra = NULL;
1437 if (ompt_enabled.enabled) {
1438 ti = __kmp_threads[global_tid]->th.ompt_thread_info;
1439 /* OMPT state update */
1440 prev_state = ti.state;
1441 ti.wait_id = (ompt_wait_id_t)(uintptr_t)lck;
1442 ti.state = ompt_state_wait_critical;
1443
1444 /* OMPT event callback */
1445 codeptr_ra = OMPT_LOAD_RETURN_ADDRESS(gtid);
1446 if (ompt_enabled.ompt_callback_mutex_acquire) {
1447 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1448 ompt_mutex_critical, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
1449 (ompt_wait_id_t)(uintptr_t)lck, codeptr_ra);
1450 }
1451 }
1452#endif
1453 // Value of 'crit' should be good for using as a critical_id of the critical
1454 // section directive.
1455 __kmp_acquire_user_lock_with_checks(lck, global_tid);
1456
1457#if USE_ITT_BUILD
1458 __kmp_itt_critical_acquired(lck);
1459#endif /* USE_ITT_BUILD */
1460#if OMPT_SUPPORT && OMPT_OPTIONAL
1461 if (ompt_enabled.enabled) {
1462 /* OMPT state update */
1463 ti.state = prev_state;
1464 ti.wait_id = 0;
1465
1466 /* OMPT event callback */
1467 if (ompt_enabled.ompt_callback_mutex_acquired) {
1468 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
1469 ompt_mutex_critical, (ompt_wait_id_t)(uintptr_t)lck, codeptr_ra);
1470 }
1471 }
1472#endif
1473 KMP_POP_PARTITIONED_TIMER();
1474
1475 KMP_PUSH_PARTITIONED_TIMER(OMP_critical);
1476 KA_TRACE(15, ("__kmpc_critical: done T#%d\n", global_tid));
1477#endif // KMP_USE_DYNAMIC_LOCK
1478}
1479
1480#if KMP_USE_DYNAMIC_LOCK
1481
1482// Converts the given hint to an internal lock implementation
1483static __forceinline kmp_dyna_lockseq_t __kmp_map_hint_to_lock(uintptr_t hint) {
1484#if KMP_USE_TSX
1485#define KMP_TSX_LOCK(seq) lockseq_##seq
1486#else
1487#define KMP_TSX_LOCK(seq) __kmp_user_lock_seq
1488#endif
1489
1490#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1491#define KMP_CPUINFO_RTM (__kmp_cpuinfo.flags.rtm)
1492#else
1493#define KMP_CPUINFO_RTM 0
1494#endif
1495
1496 // Hints that do not require further logic
1497 if (hint & kmp_lock_hint_hle)
1498 return KMP_TSX_LOCK(hle);
1499 if (hint & kmp_lock_hint_rtm)
1500 return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(rtm_queuing) : __kmp_user_lock_seq;
1501 if (hint & kmp_lock_hint_adaptive)
1502 return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(adaptive) : __kmp_user_lock_seq;
1503
1504 // Rule out conflicting hints first by returning the default lock
1505 if ((hint & omp_lock_hint_contended) && (hint & omp_lock_hint_uncontended))
1506 return __kmp_user_lock_seq;
1507 if ((hint & omp_lock_hint_speculative) &&
1508 (hint & omp_lock_hint_nonspeculative))
1509 return __kmp_user_lock_seq;
1510
1511 // Do not even consider speculation when it appears to be contended
1512 if (hint & omp_lock_hint_contended)
1513 return lockseq_queuing;
1514
1515 // Uncontended lock without speculation
1516 if ((hint & omp_lock_hint_uncontended) && !(hint & omp_lock_hint_speculative))
1517 return lockseq_tas;
1518
1519 // Use RTM lock for speculation
1520 if (hint & omp_lock_hint_speculative)
1521 return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(rtm_spin) : __kmp_user_lock_seq;
1522
1523 return __kmp_user_lock_seq;
1524}
1525
1526#if OMPT_SUPPORT && OMPT_OPTIONAL
1527#if KMP_USE_DYNAMIC_LOCK
1528static kmp_mutex_impl_t
1529__ompt_get_mutex_impl_type(void *user_lock, kmp_indirect_lock_t *ilock = 0) {
1530 if (user_lock) {
1531 switch (KMP_EXTRACT_D_TAG(user_lock)) {
1532 case 0:
1533 break;
1534#if KMP_USE_FUTEX
1535 case locktag_futex:
1536 return kmp_mutex_impl_queuing;
1537#endif
1538 case locktag_tas:
1539 return kmp_mutex_impl_spin;
1540#if KMP_USE_TSX
1541 case locktag_hle:
1542 case locktag_rtm_spin:
1543 return kmp_mutex_impl_speculative;
1544#endif
1545 default:
1546 return kmp_mutex_impl_none;
1547 }
1548 ilock = KMP_LOOKUP_I_LOCK(user_lock);
1549 }
1550 KMP_ASSERT(ilock);
1551 switch (ilock->type) {
1552#if KMP_USE_TSX
1553 case locktag_adaptive:
1554 case locktag_rtm_queuing:
1555 return kmp_mutex_impl_speculative;
1556#endif
1557 case locktag_nested_tas:
1558 return kmp_mutex_impl_spin;
1559#if KMP_USE_FUTEX
1560 case locktag_nested_futex:
1561#endif
1562 case locktag_ticket:
1563 case locktag_queuing:
1564 case locktag_drdpa:
1565 case locktag_nested_ticket:
1566 case locktag_nested_queuing:
1567 case locktag_nested_drdpa:
1568 return kmp_mutex_impl_queuing;
1569 default:
1570 return kmp_mutex_impl_none;
1571 }
1572}
1573#else
1574// For locks without dynamic binding
1575static kmp_mutex_impl_t __ompt_get_mutex_impl_type() {
1576 switch (__kmp_user_lock_kind) {
1577 case lk_tas:
1578 return kmp_mutex_impl_spin;
1579#if KMP_USE_FUTEX
1580 case lk_futex:
1581#endif
1582 case lk_ticket:
1583 case lk_queuing:
1584 case lk_drdpa:
1585 return kmp_mutex_impl_queuing;
1586#if KMP_USE_TSX
1587 case lk_hle:
1588 case lk_rtm_queuing:
1589 case lk_rtm_spin:
1590 case lk_adaptive:
1591 return kmp_mutex_impl_speculative;
1592#endif
1593 default:
1594 return kmp_mutex_impl_none;
1595 }
1596}
1597#endif // KMP_USE_DYNAMIC_LOCK
1598#endif // OMPT_SUPPORT && OMPT_OPTIONAL
1599
1613void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid,
1614 kmp_critical_name *crit, uint32_t hint) {
1615 KMP_COUNT_BLOCK(OMP_CRITICAL);
1616 kmp_user_lock_p lck;
1617#if OMPT_SUPPORT && OMPT_OPTIONAL
1618 ompt_state_t prev_state = ompt_state_undefined;
1619 ompt_thread_info_t ti;
1620 // This is the case, if called from __kmpc_critical:
1621 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid);
1622 if (!codeptr)
1623 codeptr = OMPT_GET_RETURN_ADDRESS(0);
1624#endif
1625
1626 KC_TRACE(10, ("__kmpc_critical: called T#%d\n", global_tid));
1627 __kmp_assert_valid_gtid(global_tid);
1628
1629 kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit;
1630 // Check if it is initialized.
1631 KMP_PUSH_PARTITIONED_TIMER(OMP_critical_wait);
1632 kmp_dyna_lockseq_t lockseq = __kmp_map_hint_to_lock(hint);
1633 if (*lk == 0) {
1634 if (KMP_IS_D_LOCK(lockseq)) {
1635 (void)KMP_COMPARE_AND_STORE_ACQ32(
1636 (volatile kmp_int32 *)&((kmp_base_tas_lock_t *)crit)->poll, 0,
1637 KMP_GET_D_TAG(lockseq));
1638 } else {
1639 __kmp_init_indirect_csptr(crit, loc, global_tid, KMP_GET_I_TAG(lockseq));
1640 }
1641 }
1642 // Branch for accessing the actual lock object and set operation. This
1643 // branching is inevitable since this lock initialization does not follow the
1644 // normal dispatch path (lock table is not used).
1645 if (KMP_EXTRACT_D_TAG(lk) != 0) {
1646 lck = (kmp_user_lock_p)lk;
1647 if (__kmp_env_consistency_check) {
1648 __kmp_push_sync(global_tid, ct_critical, loc, lck,
1649 __kmp_map_hint_to_lock(hint));
1650 }
1651#if USE_ITT_BUILD
1652 __kmp_itt_critical_acquiring(lck);
1653#endif
1654#if OMPT_SUPPORT && OMPT_OPTIONAL
1655 if (ompt_enabled.enabled) {
1656 ti = __kmp_threads[global_tid]->th.ompt_thread_info;
1657 /* OMPT state update */
1658 prev_state = ti.state;
1659 ti.wait_id = (ompt_wait_id_t)(uintptr_t)lck;
1660 ti.state = ompt_state_wait_critical;
1661
1662 /* OMPT event callback */
1663 if (ompt_enabled.ompt_callback_mutex_acquire) {
1664 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1665 ompt_mutex_critical, (unsigned int)hint,
1666 __ompt_get_mutex_impl_type(crit), (ompt_wait_id_t)(uintptr_t)lck,
1667 codeptr);
1668 }
1669 }
1670#endif
1671#if KMP_USE_INLINED_TAS
1672 if (lockseq == lockseq_tas && !__kmp_env_consistency_check) {
1673 KMP_ACQUIRE_TAS_LOCK(lck, global_tid);
1674 } else
1675#elif KMP_USE_INLINED_FUTEX
1676 if (lockseq == lockseq_futex && !__kmp_env_consistency_check) {
1677 KMP_ACQUIRE_FUTEX_LOCK(lck, global_tid);
1678 } else
1679#endif
1680 {
1681 KMP_D_LOCK_FUNC(lk, set)(lk, global_tid);
1682 }
1683 } else {
1684 kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk);
1685 lck = ilk->lock;
1686 if (__kmp_env_consistency_check) {
1687 __kmp_push_sync(global_tid, ct_critical, loc, lck,
1688 __kmp_map_hint_to_lock(hint));
1689 }
1690#if USE_ITT_BUILD
1691 __kmp_itt_critical_acquiring(lck);
1692#endif
1693#if OMPT_SUPPORT && OMPT_OPTIONAL
1694 if (ompt_enabled.enabled) {
1695 ti = __kmp_threads[global_tid]->th.ompt_thread_info;
1696 /* OMPT state update */
1697 prev_state = ti.state;
1698 ti.wait_id = (ompt_wait_id_t)(uintptr_t)lck;
1699 ti.state = ompt_state_wait_critical;
1700
1701 /* OMPT event callback */
1702 if (ompt_enabled.ompt_callback_mutex_acquire) {
1703 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1704 ompt_mutex_critical, (unsigned int)hint,
1705 __ompt_get_mutex_impl_type(0, ilk), (ompt_wait_id_t)(uintptr_t)lck,
1706 codeptr);
1707 }
1708 }
1709#endif
1710 KMP_I_LOCK_FUNC(ilk, set)(lck, global_tid);
1711 }
1712 KMP_POP_PARTITIONED_TIMER();
1713
1714#if USE_ITT_BUILD
1715 __kmp_itt_critical_acquired(lck);
1716#endif /* USE_ITT_BUILD */
1717#if OMPT_SUPPORT && OMPT_OPTIONAL
1718 if (ompt_enabled.enabled) {
1719 /* OMPT state update */
1720 ti.state = prev_state;
1721 ti.wait_id = 0;
1722
1723 /* OMPT event callback */
1724 if (ompt_enabled.ompt_callback_mutex_acquired) {
1725 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
1726 ompt_mutex_critical, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
1727 }
1728 }
1729#endif
1730
1731 KMP_PUSH_PARTITIONED_TIMER(OMP_critical);
1732 KA_TRACE(15, ("__kmpc_critical: done T#%d\n", global_tid));
1733} // __kmpc_critical_with_hint
1734
1735#endif // KMP_USE_DYNAMIC_LOCK
1736
1746void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
1747 kmp_critical_name *crit) {
1748 kmp_user_lock_p lck;
1749
1750 KC_TRACE(10, ("__kmpc_end_critical: called T#%d\n", global_tid));
1751
1752#if KMP_USE_DYNAMIC_LOCK
1753 int locktag = KMP_EXTRACT_D_TAG(crit);
1754 if (locktag) {
1755 lck = (kmp_user_lock_p)crit;
1756 KMP_ASSERT(lck != NULL);
1757 if (__kmp_env_consistency_check) {
1758 __kmp_pop_sync(global_tid, ct_critical, loc);
1759 }
1760#if USE_ITT_BUILD
1761 __kmp_itt_critical_releasing(lck);
1762#endif
1763#if KMP_USE_INLINED_TAS
1764 if (locktag == locktag_tas && !__kmp_env_consistency_check) {
1765 KMP_RELEASE_TAS_LOCK(lck, global_tid);
1766 } else
1767#elif KMP_USE_INLINED_FUTEX
1768 if (locktag == locktag_futex && !__kmp_env_consistency_check) {
1769 KMP_RELEASE_FUTEX_LOCK(lck, global_tid);
1770 } else
1771#endif
1772 {
1773 KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
1774 }
1775 } else {
1776 kmp_indirect_lock_t *ilk =
1777 (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
1778 KMP_ASSERT(ilk != NULL);
1779 lck = ilk->lock;
1780 if (__kmp_env_consistency_check) {
1781 __kmp_pop_sync(global_tid, ct_critical, loc);
1782 }
1783#if USE_ITT_BUILD
1784 __kmp_itt_critical_releasing(lck);
1785#endif
1786 KMP_I_LOCK_FUNC(ilk, unset)(lck, global_tid);
1787 }
1788
1789#else // KMP_USE_DYNAMIC_LOCK
1790
1791 if ((__kmp_user_lock_kind == lk_tas) &&
1792 (sizeof(lck->tas.lk.poll) <= OMP_CRITICAL_SIZE)) {
1793 lck = (kmp_user_lock_p)crit;
1794 }
1795#if KMP_USE_FUTEX
1796 else if ((__kmp_user_lock_kind == lk_futex) &&
1797 (sizeof(lck->futex.lk.poll) <= OMP_CRITICAL_SIZE)) {
1798 lck = (kmp_user_lock_p)crit;
1799 }
1800#endif
1801 else { // ticket, queuing or drdpa
1802 lck = (kmp_user_lock_p)TCR_PTR(*((kmp_user_lock_p *)crit));
1803 }
1804
1805 KMP_ASSERT(lck != NULL);
1806
1807 if (__kmp_env_consistency_check)
1808 __kmp_pop_sync(global_tid, ct_critical, loc);
1809
1810#if USE_ITT_BUILD
1811 __kmp_itt_critical_releasing(lck);
1812#endif /* USE_ITT_BUILD */
1813 // Value of 'crit' should be good for using as a critical_id of the critical
1814 // section directive.
1815 __kmp_release_user_lock_with_checks(lck, global_tid);
1816
1817#endif // KMP_USE_DYNAMIC_LOCK
1818
1819#if OMPT_SUPPORT && OMPT_OPTIONAL
1820 /* OMPT release event triggers after lock is released; place here to trigger
1821 * for all #if branches */
1822 OMPT_STORE_RETURN_ADDRESS(global_tid);
1823 if (ompt_enabled.ompt_callback_mutex_released) {
1824 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
1825 ompt_mutex_critical, (ompt_wait_id_t)(uintptr_t)lck,
1826 OMPT_LOAD_RETURN_ADDRESS(global_tid));
1827 }
1828#endif
1829
1830 KMP_POP_PARTITIONED_TIMER();
1831 KA_TRACE(15, ("__kmpc_end_critical: done T#%d\n", global_tid));
1832}
1833
1843kmp_int32 __kmpc_barrier_master(ident_t *loc, kmp_int32 global_tid) {
1844 int status;
1845 KC_TRACE(10, ("__kmpc_barrier_master: called T#%d\n", global_tid));
1846 __kmp_assert_valid_gtid(global_tid);
1847
1848 if (!TCR_4(__kmp_init_parallel))
1849 __kmp_parallel_initialize();
1850
1851 __kmp_resume_if_soft_paused();
1852
1853 if (__kmp_env_consistency_check)
1854 __kmp_check_barrier(global_tid, ct_barrier, loc);
1855
1856#if OMPT_SUPPORT
1857 ompt_frame_t *ompt_frame;
1858 if (ompt_enabled.enabled) {
1859 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
1860 if (ompt_frame->enter_frame.ptr == NULL)
1861 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1862 }
1863 OMPT_STORE_RETURN_ADDRESS(global_tid);
1864#endif
1865#if USE_ITT_NOTIFY
1866 __kmp_threads[global_tid]->th.th_ident = loc;
1867#endif
1868 status = __kmp_barrier(bs_plain_barrier, global_tid, TRUE, 0, NULL, NULL);
1869#if OMPT_SUPPORT && OMPT_OPTIONAL
1870 if (ompt_enabled.enabled) {
1871 ompt_frame->enter_frame = ompt_data_none;
1872 }
1873#endif
1874
1875 return (status != 0) ? 0 : 1;
1876}
1877
1887void __kmpc_end_barrier_master(ident_t *loc, kmp_int32 global_tid) {
1888 KC_TRACE(10, ("__kmpc_end_barrier_master: called T#%d\n", global_tid));
1889 __kmp_assert_valid_gtid(global_tid);
1890 __kmp_end_split_barrier(bs_plain_barrier, global_tid);
1891}
1892
1903kmp_int32 __kmpc_barrier_master_nowait(ident_t *loc, kmp_int32 global_tid) {
1904 kmp_int32 ret;
1905 KC_TRACE(10, ("__kmpc_barrier_master_nowait: called T#%d\n", global_tid));
1906 __kmp_assert_valid_gtid(global_tid);
1907
1908 if (!TCR_4(__kmp_init_parallel))
1909 __kmp_parallel_initialize();
1910
1911 __kmp_resume_if_soft_paused();
1912
1913 if (__kmp_env_consistency_check) {
1914 if (loc == 0) {
1915 KMP_WARNING(ConstructIdentInvalid); // ??? What does it mean for the user?
1916 }
1917 __kmp_check_barrier(global_tid, ct_barrier, loc);
1918 }
1919
1920#if OMPT_SUPPORT
1921 ompt_frame_t *ompt_frame;
1922 if (ompt_enabled.enabled) {
1923 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
1924 if (ompt_frame->enter_frame.ptr == NULL)
1925 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1926 }
1927 OMPT_STORE_RETURN_ADDRESS(global_tid);
1928#endif
1929#if USE_ITT_NOTIFY
1930 __kmp_threads[global_tid]->th.th_ident = loc;
1931#endif
1932 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
1933#if OMPT_SUPPORT && OMPT_OPTIONAL
1934 if (ompt_enabled.enabled) {
1935 ompt_frame->enter_frame = ompt_data_none;
1936 }
1937#endif
1938
1939 ret = __kmpc_master(loc, global_tid);
1940
1941 if (__kmp_env_consistency_check) {
1942 /* there's no __kmpc_end_master called; so the (stats) */
1943 /* actions of __kmpc_end_master are done here */
1944 if (ret) {
1945 /* only one thread should do the pop since only */
1946 /* one did the push (see __kmpc_master()) */
1947 __kmp_pop_sync(global_tid, ct_master, loc);
1948 }
1949 }
1950
1951 return (ret);
1952}
1953
1954/* The BARRIER for a SINGLE process section is always explicit */
1966kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid) {
1967 __kmp_assert_valid_gtid(global_tid);
1968 kmp_int32 rc = __kmp_enter_single(global_tid, loc, TRUE);
1969
1970 if (rc) {
1971 // We are going to execute the single statement, so we should count it.
1972 KMP_COUNT_BLOCK(OMP_SINGLE);
1973 KMP_PUSH_PARTITIONED_TIMER(OMP_single);
1974 }
1975
1976#if OMPT_SUPPORT && OMPT_OPTIONAL
1977 kmp_info_t *this_thr = __kmp_threads[global_tid];
1978 kmp_team_t *team = this_thr->th.th_team;
1979 int tid = __kmp_tid_from_gtid(global_tid);
1980
1981 if (ompt_enabled.enabled) {
1982 if (rc) {
1983 if (ompt_enabled.ompt_callback_work) {
1984 ompt_callbacks.ompt_callback(ompt_callback_work)(
1985 ompt_work_single_executor, ompt_scope_begin,
1986 &(team->t.ompt_team_info.parallel_data),
1987 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1988 1, OMPT_GET_RETURN_ADDRESS(0));
1989 }
1990 } else {
1991 if (ompt_enabled.ompt_callback_work) {
1992 ompt_callbacks.ompt_callback(ompt_callback_work)(
1993 ompt_work_single_other, ompt_scope_begin,
1994 &(team->t.ompt_team_info.parallel_data),
1995 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1996 1, OMPT_GET_RETURN_ADDRESS(0));
1997 ompt_callbacks.ompt_callback(ompt_callback_work)(
1998 ompt_work_single_other, ompt_scope_end,
1999 &(team->t.ompt_team_info.parallel_data),
2000 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
2001 1, OMPT_GET_RETURN_ADDRESS(0));
2002 }
2003 }
2004 }
2005#endif
2006
2007 return rc;
2008}
2009
2019void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid) {
2020 __kmp_assert_valid_gtid(global_tid);
2021 __kmp_exit_single(global_tid);
2022 KMP_POP_PARTITIONED_TIMER();
2023
2024#if OMPT_SUPPORT && OMPT_OPTIONAL
2025 kmp_info_t *this_thr = __kmp_threads[global_tid];
2026 kmp_team_t *team = this_thr->th.th_team;
2027 int tid = __kmp_tid_from_gtid(global_tid);
2028
2029 if (ompt_enabled.ompt_callback_work) {
2030 ompt_callbacks.ompt_callback(ompt_callback_work)(
2031 ompt_work_single_executor, ompt_scope_end,
2032 &(team->t.ompt_team_info.parallel_data),
2033 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 1,
2034 OMPT_GET_RETURN_ADDRESS(0));
2035 }
2036#endif
2037}
2038
2046void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid) {
2047 KMP_POP_PARTITIONED_TIMER();
2048 KE_TRACE(10, ("__kmpc_for_static_fini called T#%d\n", global_tid));
2049
2050#if OMPT_SUPPORT && OMPT_OPTIONAL
2051 if (ompt_enabled.ompt_callback_work) {
2052 ompt_work_t ompt_work_type = ompt_work_loop_static;
2053 ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
2054 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2055 // Determine workshare type
2056 if (loc != NULL) {
2057 if ((loc->flags & KMP_IDENT_WORK_LOOP) != 0) {
2058 ompt_work_type = ompt_work_loop_static;
2059 } else if ((loc->flags & KMP_IDENT_WORK_SECTIONS) != 0) {
2060 ompt_work_type = ompt_work_sections;
2061 } else if ((loc->flags & KMP_IDENT_WORK_DISTRIBUTE) != 0) {
2062 ompt_work_type = ompt_work_distribute;
2063 } else {
2064 // use default set above.
2065 // a warning about this case is provided in __kmpc_for_static_init
2066 }
2067 KMP_DEBUG_ASSERT(ompt_work_type);
2068 }
2069 ompt_callbacks.ompt_callback(ompt_callback_work)(
2070 ompt_work_type, ompt_scope_end, &(team_info->parallel_data),
2071 &(task_info->task_data), 0, OMPT_GET_RETURN_ADDRESS(0));
2072 }
2073#endif
2074 if (__kmp_env_consistency_check)
2075 __kmp_pop_workshare(global_tid, ct_pdo, loc);
2076}
2077
2078// User routines which take C-style arguments (call by value)
2079// different from the Fortran equivalent routines
2080
2081void ompc_set_num_threads(int arg) {
2082 // !!!!! TODO: check the per-task binding
2083 __kmp_set_num_threads(arg, __kmp_entry_gtid());
2084}
2085
2086void kmpc_set_num_threads_8(kmp_int64 arg) {
2087 if (arg > (kmp_int64)INT_MAX)
2088 arg = INT_MAX;
2089 else if (arg < (kmp_int64)INT_MIN)
2090 arg = INT_MIN;
2091 __kmp_set_num_threads((int)arg, __kmp_entry_gtid());
2092}
2093
2094void ompc_set_dynamic(int flag) {
2095 kmp_info_t *thread;
2096
2097 /* For the thread-private implementation of the internal controls */
2098 thread = __kmp_entry_thread();
2099
2100 __kmp_save_internal_controls(thread);
2101
2102 set__dynamic(thread, flag ? true : false);
2103}
2104
2105void ompc_set_nested(int flag) {
2106 kmp_info_t *thread;
2107
2108 /* For the thread-private internal controls implementation */
2109 thread = __kmp_entry_thread();
2110
2111 __kmp_save_internal_controls(thread);
2112
2113 set__max_active_levels(thread, flag ? __kmp_dflt_max_active_levels : 1);
2114}
2115
2116void ompc_set_max_active_levels(int max_active_levels) {
2117 /* TO DO */
2118 /* we want per-task implementation of this internal control */
2119
2120 /* For the per-thread internal controls implementation */
2121 __kmp_set_max_active_levels(__kmp_entry_gtid(), max_active_levels);
2122}
2123
2124void ompc_set_schedule(omp_sched_t kind, int modifier) {
2125 // !!!!! TODO: check the per-task binding
2126 __kmp_set_schedule(__kmp_entry_gtid(), (kmp_sched_t)kind, modifier);
2127}
2128
2129int ompc_get_ancestor_thread_num(int level) {
2130 return __kmp_get_ancestor_thread_num(__kmp_entry_gtid(), level);
2131}
2132
2133int ompc_get_team_size(int level) {
2134 return __kmp_get_team_size(__kmp_entry_gtid(), level);
2135}
2136
2137/* OpenMP 5.0 Affinity Format API */
2138void KMP_EXPAND_NAME(ompc_set_affinity_format)(char const *format) {
2139 if (!__kmp_init_serial) {
2140 __kmp_serial_initialize();
2141 }
2142 __kmp_strncpy_truncate(__kmp_affinity_format, KMP_AFFINITY_FORMAT_SIZE,
2143 format, KMP_STRLEN(format) + 1);
2144}
2145
2146size_t KMP_EXPAND_NAME(ompc_get_affinity_format)(char *buffer, size_t size) {
2147 size_t format_size;
2148 if (!__kmp_init_serial) {
2149 __kmp_serial_initialize();
2150 }
2151 format_size = KMP_STRLEN(__kmp_affinity_format);
2152 if (buffer && size) {
2153 __kmp_strncpy_truncate(buffer, size, __kmp_affinity_format,
2154 format_size + 1);
2155 }
2156 return format_size;
2157}
2158
2159void KMP_EXPAND_NAME(ompc_display_affinity)(char const *format) {
2160 int gtid;
2161 if (!TCR_4(__kmp_init_middle)) {
2162 __kmp_middle_initialize();
2163 }
2164 __kmp_assign_root_init_mask();
2165 gtid = __kmp_get_gtid();
2166#if KMP_AFFINITY_SUPPORTED
2167 if (__kmp_threads[gtid]->th.th_team->t.t_level == 0 &&
2168 __kmp_affinity.flags.reset) {
2169 __kmp_reset_root_init_mask(gtid);
2170 }
2171#endif
2172 __kmp_aux_display_affinity(gtid, format);
2173}
2174
2175size_t KMP_EXPAND_NAME(ompc_capture_affinity)(char *buffer, size_t buf_size,
2176 char const *format) {
2177 int gtid;
2178 size_t num_required;
2179 kmp_str_buf_t capture_buf;
2180 if (!TCR_4(__kmp_init_middle)) {
2181 __kmp_middle_initialize();
2182 }
2183 __kmp_assign_root_init_mask();
2184 gtid = __kmp_get_gtid();
2185#if KMP_AFFINITY_SUPPORTED
2186 if (__kmp_threads[gtid]->th.th_team->t.t_level == 0 &&
2187 __kmp_affinity.flags.reset) {
2188 __kmp_reset_root_init_mask(gtid);
2189 }
2190#endif
2191 __kmp_str_buf_init(&capture_buf);
2192 num_required = __kmp_aux_capture_affinity(gtid, format, &capture_buf);
2193 if (buffer && buf_size) {
2194 __kmp_strncpy_truncate(buffer, buf_size, capture_buf.str,
2195 capture_buf.used + 1);
2196 }
2197 __kmp_str_buf_free(&capture_buf);
2198 return num_required;
2199}
2200
2201void kmpc_set_stacksize(int arg) {
2202 // __kmp_aux_set_stacksize initializes the library if needed
2203 __kmp_aux_set_stacksize(arg);
2204}
2205
2206void kmpc_set_stacksize_s(size_t arg) {
2207 // __kmp_aux_set_stacksize initializes the library if needed
2208 __kmp_aux_set_stacksize(arg);
2209}
2210
2211void kmpc_set_blocktime(int arg) {
2212 int gtid, tid, bt = arg;
2213 kmp_info_t *thread;
2214
2215 gtid = __kmp_entry_gtid();
2216 tid = __kmp_tid_from_gtid(gtid);
2217 thread = __kmp_thread_from_gtid(gtid);
2218
2219 __kmp_aux_convert_blocktime(&bt);
2220 __kmp_aux_set_blocktime(bt, thread, tid);
2221}
2222
2223void kmpc_set_library(int arg) {
2224 // __kmp_user_set_library initializes the library if needed
2225 __kmp_user_set_library((enum library_type)arg);
2226}
2227
2228void kmpc_set_defaults(char const *str) {
2229 // __kmp_aux_set_defaults initializes the library if needed
2230 __kmp_aux_set_defaults(str, KMP_STRLEN(str));
2231}
2232
2233void kmpc_set_disp_num_buffers(int arg) {
2234 // ignore after initialization because some teams have already
2235 // allocated dispatch buffers
2236 if (__kmp_init_serial == FALSE && arg >= KMP_MIN_DISP_NUM_BUFF &&
2237 arg <= KMP_MAX_DISP_NUM_BUFF) {
2238 __kmp_dispatch_num_buffers = arg;
2239 }
2240}
2241
2242int kmpc_set_affinity_mask_proc(int proc, void **mask) {
2243#if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
2244 return -1;
2245#else
2246 if (!TCR_4(__kmp_init_middle)) {
2247 __kmp_middle_initialize();
2248 }
2249 __kmp_assign_root_init_mask();
2250 return __kmp_aux_set_affinity_mask_proc(proc, mask);
2251#endif
2252}
2253
2254int kmpc_unset_affinity_mask_proc(int proc, void **mask) {
2255#if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
2256 return -1;
2257#else
2258 if (!TCR_4(__kmp_init_middle)) {
2259 __kmp_middle_initialize();
2260 }
2261 __kmp_assign_root_init_mask();
2262 return __kmp_aux_unset_affinity_mask_proc(proc, mask);
2263#endif
2264}
2265
2266int kmpc_get_affinity_mask_proc(int proc, void **mask) {
2267#if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
2268 return -1;
2269#else
2270 if (!TCR_4(__kmp_init_middle)) {
2271 __kmp_middle_initialize();
2272 }
2273 __kmp_assign_root_init_mask();
2274 return __kmp_aux_get_affinity_mask_proc(proc, mask);
2275#endif
2276}
2277
2278/* -------------------------------------------------------------------------- */
2323void __kmpc_copyprivate(ident_t *loc, kmp_int32 gtid, size_t cpy_size,
2324 void *cpy_data, void (*cpy_func)(void *, void *),
2325 kmp_int32 didit) {
2326 void **data_ptr;
2327 KC_TRACE(10, ("__kmpc_copyprivate: called T#%d\n", gtid));
2328 __kmp_assert_valid_gtid(gtid);
2329
2330 KMP_MB();
2331
2332 data_ptr = &__kmp_team_from_gtid(gtid)->t.t_copypriv_data;
2333
2334 if (__kmp_env_consistency_check) {
2335 if (loc == 0) {
2336 KMP_WARNING(ConstructIdentInvalid);
2337 }
2338 }
2339
2340 // ToDo: Optimize the following two barriers into some kind of split barrier
2341
2342 if (didit)
2343 *data_ptr = cpy_data;
2344
2345#if OMPT_SUPPORT
2346 ompt_frame_t *ompt_frame;
2347 if (ompt_enabled.enabled) {
2348 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
2349 if (ompt_frame->enter_frame.ptr == NULL)
2350 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
2351 }
2352 OMPT_STORE_RETURN_ADDRESS(gtid);
2353#endif
2354/* This barrier is not a barrier region boundary */
2355#if USE_ITT_NOTIFY
2356 __kmp_threads[gtid]->th.th_ident = loc;
2357#endif
2358 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
2359
2360 if (!didit)
2361 (*cpy_func)(cpy_data, *data_ptr);
2362
2363 // Consider next barrier a user-visible barrier for barrier region boundaries
2364 // Nesting checks are already handled by the single construct checks
2365 {
2366#if OMPT_SUPPORT
2367 OMPT_STORE_RETURN_ADDRESS(gtid);
2368#endif
2369#if USE_ITT_NOTIFY
2370 __kmp_threads[gtid]->th.th_ident = loc; // TODO: check if it is needed (e.g.
2371// tasks can overwrite the location)
2372#endif
2373 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
2374#if OMPT_SUPPORT && OMPT_OPTIONAL
2375 if (ompt_enabled.enabled) {
2376 ompt_frame->enter_frame = ompt_data_none;
2377 }
2378#endif
2379 }
2380}
2381
2382/* --------------------------------------------------------------------------*/
2399void *__kmpc_copyprivate_light(ident_t *loc, kmp_int32 gtid, void *cpy_data) {
2400 void **data_ptr;
2401
2402 KC_TRACE(10, ("__kmpc_copyprivate_light: called T#%d\n", gtid));
2403
2404 KMP_MB();
2405
2406 data_ptr = &__kmp_team_from_gtid(gtid)->t.t_copypriv_data;
2407
2408 if (__kmp_env_consistency_check) {
2409 if (loc == 0) {
2410 KMP_WARNING(ConstructIdentInvalid);
2411 }
2412 }
2413
2414 // ToDo: Optimize the following barrier
2415
2416 if (cpy_data)
2417 *data_ptr = cpy_data;
2418
2419#if OMPT_SUPPORT
2420 ompt_frame_t *ompt_frame;
2421 if (ompt_enabled.enabled) {
2422 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
2423 if (ompt_frame->enter_frame.ptr == NULL)
2424 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
2425 OMPT_STORE_RETURN_ADDRESS(gtid);
2426 }
2427#endif
2428/* This barrier is not a barrier region boundary */
2429#if USE_ITT_NOTIFY
2430 __kmp_threads[gtid]->th.th_ident = loc;
2431#endif
2432 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
2433
2434 return *data_ptr;
2435}
2436
2437/* -------------------------------------------------------------------------- */
2438
2439#define INIT_LOCK __kmp_init_user_lock_with_checks
2440#define INIT_NESTED_LOCK __kmp_init_nested_user_lock_with_checks
2441#define ACQUIRE_LOCK __kmp_acquire_user_lock_with_checks
2442#define ACQUIRE_LOCK_TIMED __kmp_acquire_user_lock_with_checks_timed
2443#define ACQUIRE_NESTED_LOCK __kmp_acquire_nested_user_lock_with_checks
2444#define ACQUIRE_NESTED_LOCK_TIMED \
2445 __kmp_acquire_nested_user_lock_with_checks_timed
2446#define RELEASE_LOCK __kmp_release_user_lock_with_checks
2447#define RELEASE_NESTED_LOCK __kmp_release_nested_user_lock_with_checks
2448#define TEST_LOCK __kmp_test_user_lock_with_checks
2449#define TEST_NESTED_LOCK __kmp_test_nested_user_lock_with_checks
2450#define DESTROY_LOCK __kmp_destroy_user_lock_with_checks
2451#define DESTROY_NESTED_LOCK __kmp_destroy_nested_user_lock_with_checks
2452
2453// TODO: Make check abort messages use location info & pass it into
2454// with_checks routines
2455
2456#if KMP_USE_DYNAMIC_LOCK
2457
2458// internal lock initializer
2459static __forceinline void __kmp_init_lock_with_hint(ident_t *loc, void **lock,
2460 kmp_dyna_lockseq_t seq) {
2461 if (KMP_IS_D_LOCK(seq)) {
2462 KMP_INIT_D_LOCK(lock, seq);
2463#if USE_ITT_BUILD
2464 __kmp_itt_lock_creating((kmp_user_lock_p)lock, NULL);
2465#endif
2466 } else {
2467 KMP_INIT_I_LOCK(lock, seq);
2468#if USE_ITT_BUILD
2469 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
2470 __kmp_itt_lock_creating(ilk->lock, loc);
2471#endif
2472 }
2473}
2474
2475// internal nest lock initializer
2476static __forceinline void
2477__kmp_init_nest_lock_with_hint(ident_t *loc, void **lock,
2478 kmp_dyna_lockseq_t seq) {
2479#if KMP_USE_TSX
2480 // Don't have nested lock implementation for speculative locks
2481 if (seq == lockseq_hle || seq == lockseq_rtm_queuing ||
2482 seq == lockseq_rtm_spin || seq == lockseq_adaptive)
2483 seq = __kmp_user_lock_seq;
2484#endif
2485 switch (seq) {
2486 case lockseq_tas:
2487 seq = lockseq_nested_tas;
2488 break;
2489#if KMP_USE_FUTEX
2490 case lockseq_futex:
2491 seq = lockseq_nested_futex;
2492 break;
2493#endif
2494 case lockseq_ticket:
2495 seq = lockseq_nested_ticket;
2496 break;
2497 case lockseq_queuing:
2498 seq = lockseq_nested_queuing;
2499 break;
2500 case lockseq_drdpa:
2501 seq = lockseq_nested_drdpa;
2502 break;
2503 default:
2504 seq = lockseq_nested_queuing;
2505 }
2506 KMP_INIT_I_LOCK(lock, seq);
2507#if USE_ITT_BUILD
2508 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
2509 __kmp_itt_lock_creating(ilk->lock, loc);
2510#endif
2511}
2512
2513/* initialize the lock with a hint */
2514void __kmpc_init_lock_with_hint(ident_t *loc, kmp_int32 gtid, void **user_lock,
2515 uintptr_t hint) {
2516 KMP_DEBUG_ASSERT(__kmp_init_serial);
2517 if (__kmp_env_consistency_check && user_lock == NULL) {
2518 KMP_FATAL(LockIsUninitialized, "omp_init_lock_with_hint");
2519 }
2520
2521 __kmp_init_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint));
2522
2523#if OMPT_SUPPORT && OMPT_OPTIONAL
2524 // This is the case, if called from omp_init_lock_with_hint:
2525 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2526 if (!codeptr)
2527 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2528 if (ompt_enabled.ompt_callback_lock_init) {
2529 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2530 ompt_mutex_lock, (omp_lock_hint_t)hint,
2531 __ompt_get_mutex_impl_type(user_lock),
2532 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2533 }
2534#endif
2535}
2536
2537/* initialize the lock with a hint */
2538void __kmpc_init_nest_lock_with_hint(ident_t *loc, kmp_int32 gtid,
2539 void **user_lock, uintptr_t hint) {
2540 KMP_DEBUG_ASSERT(__kmp_init_serial);
2541 if (__kmp_env_consistency_check && user_lock == NULL) {
2542 KMP_FATAL(LockIsUninitialized, "omp_init_nest_lock_with_hint");
2543 }
2544
2545 __kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint));
2546
2547#if OMPT_SUPPORT && OMPT_OPTIONAL
2548 // This is the case, if called from omp_init_lock_with_hint:
2549 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2550 if (!codeptr)
2551 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2552 if (ompt_enabled.ompt_callback_lock_init) {
2553 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2554 ompt_mutex_nest_lock, (omp_lock_hint_t)hint,
2555 __ompt_get_mutex_impl_type(user_lock),
2556 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2557 }
2558#endif
2559}
2560
2561#endif // KMP_USE_DYNAMIC_LOCK
2562
2563/* initialize the lock */
2564void __kmpc_init_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) {
2565#if KMP_USE_DYNAMIC_LOCK
2566
2567 KMP_DEBUG_ASSERT(__kmp_init_serial);
2568 if (__kmp_env_consistency_check && user_lock == NULL) {
2569 KMP_FATAL(LockIsUninitialized, "omp_init_lock");
2570 }
2571 __kmp_init_lock_with_hint(loc, user_lock, __kmp_user_lock_seq);
2572
2573#if OMPT_SUPPORT && OMPT_OPTIONAL
2574 // This is the case, if called from omp_init_lock_with_hint:
2575 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2576 if (!codeptr)
2577 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2578 if (ompt_enabled.ompt_callback_lock_init) {
2579 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2580 ompt_mutex_lock, omp_lock_hint_none,
2581 __ompt_get_mutex_impl_type(user_lock),
2582 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2583 }
2584#endif
2585
2586#else // KMP_USE_DYNAMIC_LOCK
2587
2588 static char const *const func = "omp_init_lock";
2589 kmp_user_lock_p lck;
2590 KMP_DEBUG_ASSERT(__kmp_init_serial);
2591
2592 if (__kmp_env_consistency_check) {
2593 if (user_lock == NULL) {
2594 KMP_FATAL(LockIsUninitialized, func);
2595 }
2596 }
2597
2598 KMP_CHECK_USER_LOCK_INIT();
2599
2600 if ((__kmp_user_lock_kind == lk_tas) &&
2601 (sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2602 lck = (kmp_user_lock_p)user_lock;
2603 }
2604#if KMP_USE_FUTEX
2605 else if ((__kmp_user_lock_kind == lk_futex) &&
2606 (sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2607 lck = (kmp_user_lock_p)user_lock;
2608 }
2609#endif
2610 else {
2611 lck = __kmp_user_lock_allocate(user_lock, gtid, 0);
2612 }
2613 INIT_LOCK(lck);
2614 __kmp_set_user_lock_location(lck, loc);
2615
2616#if OMPT_SUPPORT && OMPT_OPTIONAL
2617 // This is the case, if called from omp_init_lock_with_hint:
2618 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2619 if (!codeptr)
2620 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2621 if (ompt_enabled.ompt_callback_lock_init) {
2622 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2623 ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2624 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2625 }
2626#endif
2627
2628#if USE_ITT_BUILD
2629 __kmp_itt_lock_creating(lck);
2630#endif /* USE_ITT_BUILD */
2631
2632#endif // KMP_USE_DYNAMIC_LOCK
2633} // __kmpc_init_lock
2634
2635/* initialize the lock */
2636void __kmpc_init_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) {
2637#if KMP_USE_DYNAMIC_LOCK
2638
2639 KMP_DEBUG_ASSERT(__kmp_init_serial);
2640 if (__kmp_env_consistency_check && user_lock == NULL) {
2641 KMP_FATAL(LockIsUninitialized, "omp_init_nest_lock");
2642 }
2643 __kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_user_lock_seq);
2644
2645#if OMPT_SUPPORT && OMPT_OPTIONAL
2646 // This is the case, if called from omp_init_lock_with_hint:
2647 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2648 if (!codeptr)
2649 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2650 if (ompt_enabled.ompt_callback_lock_init) {
2651 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2652 ompt_mutex_nest_lock, omp_lock_hint_none,
2653 __ompt_get_mutex_impl_type(user_lock),
2654 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2655 }
2656#endif
2657
2658#else // KMP_USE_DYNAMIC_LOCK
2659
2660 static char const *const func = "omp_init_nest_lock";
2661 kmp_user_lock_p lck;
2662 KMP_DEBUG_ASSERT(__kmp_init_serial);
2663
2664 if (__kmp_env_consistency_check) {
2665 if (user_lock == NULL) {
2666 KMP_FATAL(LockIsUninitialized, func);
2667 }
2668 }
2669
2670 KMP_CHECK_USER_LOCK_INIT();
2671
2672 if ((__kmp_user_lock_kind == lk_tas) &&
2673 (sizeof(lck->tas.lk.poll) + sizeof(lck->tas.lk.depth_locked) <=
2674 OMP_NEST_LOCK_T_SIZE)) {
2675 lck = (kmp_user_lock_p)user_lock;
2676 }
2677#if KMP_USE_FUTEX
2678 else if ((__kmp_user_lock_kind == lk_futex) &&
2679 (sizeof(lck->futex.lk.poll) + sizeof(lck->futex.lk.depth_locked) <=
2680 OMP_NEST_LOCK_T_SIZE)) {
2681 lck = (kmp_user_lock_p)user_lock;
2682 }
2683#endif
2684 else {
2685 lck = __kmp_user_lock_allocate(user_lock, gtid, 0);
2686 }
2687
2688 INIT_NESTED_LOCK(lck);
2689 __kmp_set_user_lock_location(lck, loc);
2690
2691#if OMPT_SUPPORT && OMPT_OPTIONAL
2692 // This is the case, if called from omp_init_lock_with_hint:
2693 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2694 if (!codeptr)
2695 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2696 if (ompt_enabled.ompt_callback_lock_init) {
2697 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2698 ompt_mutex_nest_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2699 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2700 }
2701#endif
2702
2703#if USE_ITT_BUILD
2704 __kmp_itt_lock_creating(lck);
2705#endif /* USE_ITT_BUILD */
2706
2707#endif // KMP_USE_DYNAMIC_LOCK
2708} // __kmpc_init_nest_lock
2709
2710void __kmpc_destroy_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) {
2711#if KMP_USE_DYNAMIC_LOCK
2712
2713#if USE_ITT_BUILD
2714 kmp_user_lock_p lck;
2715 if (KMP_EXTRACT_D_TAG(user_lock) == 0) {
2716 lck = ((kmp_indirect_lock_t *)KMP_LOOKUP_I_LOCK(user_lock))->lock;
2717 } else {
2718 lck = (kmp_user_lock_p)user_lock;
2719 }
2720 __kmp_itt_lock_destroyed(lck);
2721#endif
2722#if OMPT_SUPPORT && OMPT_OPTIONAL
2723 // This is the case, if called from omp_init_lock_with_hint:
2724 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2725 if (!codeptr)
2726 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2727 if (ompt_enabled.ompt_callback_lock_destroy) {
2728 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2729 ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2730 }
2731#endif
2732 KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
2733#else
2734 kmp_user_lock_p lck;
2735
2736 if ((__kmp_user_lock_kind == lk_tas) &&
2737 (sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2738 lck = (kmp_user_lock_p)user_lock;
2739 }
2740#if KMP_USE_FUTEX
2741 else if ((__kmp_user_lock_kind == lk_futex) &&
2742 (sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2743 lck = (kmp_user_lock_p)user_lock;
2744 }
2745#endif
2746 else {
2747 lck = __kmp_lookup_user_lock(user_lock, "omp_destroy_lock");
2748 }
2749
2750#if OMPT_SUPPORT && OMPT_OPTIONAL
2751 // This is the case, if called from omp_init_lock_with_hint:
2752 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2753 if (!codeptr)
2754 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2755 if (ompt_enabled.ompt_callback_lock_destroy) {
2756 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2757 ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2758 }
2759#endif
2760
2761#if USE_ITT_BUILD
2762 __kmp_itt_lock_destroyed(lck);
2763#endif /* USE_ITT_BUILD */
2764 DESTROY_LOCK(lck);
2765
2766 if ((__kmp_user_lock_kind == lk_tas) &&
2767 (sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2768 ;
2769 }
2770#if KMP_USE_FUTEX
2771 else if ((__kmp_user_lock_kind == lk_futex) &&
2772 (sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2773 ;
2774 }
2775#endif
2776 else {
2777 __kmp_user_lock_free(user_lock, gtid, lck);
2778 }
2779#endif // KMP_USE_DYNAMIC_LOCK
2780} // __kmpc_destroy_lock
2781
2782/* destroy the lock */
2783void __kmpc_destroy_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) {
2784#if KMP_USE_DYNAMIC_LOCK
2785
2786#if USE_ITT_BUILD
2787 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(user_lock);
2788 __kmp_itt_lock_destroyed(ilk->lock);
2789#endif
2790#if OMPT_SUPPORT && OMPT_OPTIONAL
2791 // This is the case, if called from omp_init_lock_with_hint:
2792 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2793 if (!codeptr)
2794 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2795 if (ompt_enabled.ompt_callback_lock_destroy) {
2796 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2797 ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2798 }
2799#endif
2800 KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
2801
2802#else // KMP_USE_DYNAMIC_LOCK
2803
2804 kmp_user_lock_p lck;
2805
2806 if ((__kmp_user_lock_kind == lk_tas) &&
2807 (sizeof(lck->tas.lk.poll) + sizeof(lck->tas.lk.depth_locked) <=
2808 OMP_NEST_LOCK_T_SIZE)) {
2809 lck = (kmp_user_lock_p)user_lock;
2810 }
2811#if KMP_USE_FUTEX
2812 else if ((__kmp_user_lock_kind == lk_futex) &&
2813 (sizeof(lck->futex.lk.poll) + sizeof(lck->futex.lk.depth_locked) <=
2814 OMP_NEST_LOCK_T_SIZE)) {
2815 lck = (kmp_user_lock_p)user_lock;
2816 }
2817#endif
2818 else {
2819 lck = __kmp_lookup_user_lock(user_lock, "omp_destroy_nest_lock");
2820 }
2821
2822#if OMPT_SUPPORT && OMPT_OPTIONAL
2823 // This is the case, if called from omp_init_lock_with_hint:
2824 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2825 if (!codeptr)
2826 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2827 if (ompt_enabled.ompt_callback_lock_destroy) {
2828 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2829 ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2830 }
2831#endif
2832
2833#if USE_ITT_BUILD
2834 __kmp_itt_lock_destroyed(lck);
2835#endif /* USE_ITT_BUILD */
2836
2837 DESTROY_NESTED_LOCK(lck);
2838
2839 if ((__kmp_user_lock_kind == lk_tas) &&
2840 (sizeof(lck->tas.lk.poll) + sizeof(lck->tas.lk.depth_locked) <=
2841 OMP_NEST_LOCK_T_SIZE)) {
2842 ;
2843 }
2844#if KMP_USE_FUTEX
2845 else if ((__kmp_user_lock_kind == lk_futex) &&
2846 (sizeof(lck->futex.lk.poll) + sizeof(lck->futex.lk.depth_locked) <=
2847 OMP_NEST_LOCK_T_SIZE)) {
2848 ;
2849 }
2850#endif
2851 else {
2852 __kmp_user_lock_free(user_lock, gtid, lck);
2853 }
2854#endif // KMP_USE_DYNAMIC_LOCK
2855} // __kmpc_destroy_nest_lock
2856
2857void __kmpc_set_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) {
2858 KMP_COUNT_BLOCK(OMP_set_lock);
2859#if KMP_USE_DYNAMIC_LOCK
2860 int tag = KMP_EXTRACT_D_TAG(user_lock);
2861#if USE_ITT_BUILD
2862 __kmp_itt_lock_acquiring(
2863 (kmp_user_lock_p)
2864 user_lock); // itt function will get to the right lock object.
2865#endif
2866#if OMPT_SUPPORT && OMPT_OPTIONAL
2867 // This is the case, if called from omp_init_lock_with_hint:
2868 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2869 if (!codeptr)
2870 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2871 if (ompt_enabled.ompt_callback_mutex_acquire) {
2872 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2873 ompt_mutex_lock, omp_lock_hint_none,
2874 __ompt_get_mutex_impl_type(user_lock),
2875 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2876 }
2877#endif
2878#if KMP_USE_INLINED_TAS
2879 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2880 KMP_ACQUIRE_TAS_LOCK(user_lock, gtid);
2881 } else
2882#elif KMP_USE_INLINED_FUTEX
2883 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2884 KMP_ACQUIRE_FUTEX_LOCK(user_lock, gtid);
2885 } else
2886#endif
2887 {
2888 __kmp_direct_set[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2889 }
2890#if USE_ITT_BUILD
2891 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2892#endif
2893#if OMPT_SUPPORT && OMPT_OPTIONAL
2894 if (ompt_enabled.ompt_callback_mutex_acquired) {
2895 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2896 ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2897 }
2898#endif
2899
2900#else // KMP_USE_DYNAMIC_LOCK
2901
2902 kmp_user_lock_p lck;
2903
2904 if ((__kmp_user_lock_kind == lk_tas) &&
2905 (sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2906 lck = (kmp_user_lock_p)user_lock;
2907 }
2908#if KMP_USE_FUTEX
2909 else if ((__kmp_user_lock_kind == lk_futex) &&
2910 (sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2911 lck = (kmp_user_lock_p)user_lock;
2912 }
2913#endif
2914 else {
2915 lck = __kmp_lookup_user_lock(user_lock, "omp_set_lock");
2916 }
2917
2918#if USE_ITT_BUILD
2919 __kmp_itt_lock_acquiring(lck);
2920#endif /* USE_ITT_BUILD */
2921#if OMPT_SUPPORT && OMPT_OPTIONAL
2922 // This is the case, if called from omp_init_lock_with_hint:
2923 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2924 if (!codeptr)
2925 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2926 if (ompt_enabled.ompt_callback_mutex_acquire) {
2927 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2928 ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2929 (ompt_wait_id_t)(uintptr_t)lck, codeptr);
2930 }
2931#endif
2932
2933 ACQUIRE_LOCK(lck, gtid);
2934
2935#if USE_ITT_BUILD
2936 __kmp_itt_lock_acquired(lck);
2937#endif /* USE_ITT_BUILD */
2938
2939#if OMPT_SUPPORT && OMPT_OPTIONAL
2940 if (ompt_enabled.ompt_callback_mutex_acquired) {
2941 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2942 ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
2943 }
2944#endif
2945
2946#endif // KMP_USE_DYNAMIC_LOCK
2947}
2948
2949void __kmpc_set_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) {
2950#if KMP_USE_DYNAMIC_LOCK
2951
2952#if USE_ITT_BUILD
2953 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2954#endif
2955#if OMPT_SUPPORT && OMPT_OPTIONAL
2956 // This is the case, if called from omp_init_lock_with_hint:
2957 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2958 if (!codeptr)
2959 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2960 if (ompt_enabled.enabled) {
2961 if (ompt_enabled.ompt_callback_mutex_acquire) {
2962 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2963 ompt_mutex_nest_lock, omp_lock_hint_none,
2964 __ompt_get_mutex_impl_type(user_lock),
2965 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2966 }
2967 }
2968#endif
2969 int acquire_status =
2970 KMP_D_LOCK_FUNC(user_lock, set)((kmp_dyna_lock_t *)user_lock, gtid);
2971 (void)acquire_status;
2972#if USE_ITT_BUILD
2973 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2974#endif
2975
2976#if OMPT_SUPPORT && OMPT_OPTIONAL
2977 if (ompt_enabled.enabled) {
2978 if (acquire_status == KMP_LOCK_ACQUIRED_FIRST) {
2979 if (ompt_enabled.ompt_callback_mutex_acquired) {
2980 // lock_first
2981 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2982 ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)user_lock,
2983 codeptr);
2984 }
2985 } else {
2986 if (ompt_enabled.ompt_callback_nest_lock) {
2987 // lock_next
2988 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2989 ompt_scope_begin, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2990 }
2991 }
2992 }
2993#endif
2994
2995#else // KMP_USE_DYNAMIC_LOCK
2996 int acquire_status;
2997 kmp_user_lock_p lck;
2998
2999 if ((__kmp_user_lock_kind == lk_tas) &&
3000 (sizeof(lck->tas.lk.poll) + sizeof(lck->tas.lk.depth_locked) <=
3001 OMP_NEST_LOCK_T_SIZE)) {
3002 lck = (kmp_user_lock_p)user_lock;
3003 }
3004#if KMP_USE_FUTEX
3005 else if ((__kmp_user_lock_kind == lk_futex) &&
3006 (sizeof(lck->futex.lk.poll) + sizeof(lck->futex.lk.depth_locked) <=
3007 OMP_NEST_LOCK_T_SIZE)) {
3008 lck = (kmp_user_lock_p)user_lock;
3009 }
3010#endif
3011 else {
3012 lck = __kmp_lookup_user_lock(user_lock, "omp_set_nest_lock");
3013 }
3014
3015#if USE_ITT_BUILD
3016 __kmp_itt_lock_acquiring(lck);
3017#endif /* USE_ITT_BUILD */
3018#if OMPT_SUPPORT && OMPT_OPTIONAL
3019 // This is the case, if called from omp_init_lock_with_hint:
3020 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3021 if (!codeptr)
3022 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3023 if (ompt_enabled.enabled) {
3024 if (ompt_enabled.ompt_callback_mutex_acquire) {
3025 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
3026 ompt_mutex_nest_lock, omp_lock_hint_none,
3027 __ompt_get_mutex_impl_type(), (ompt_wait_id_t)(uintptr_t)lck,
3028 codeptr);
3029 }
3030 }
3031#endif
3032
3033 ACQUIRE_NESTED_LOCK(lck, gtid, &acquire_status);
3034
3035#if USE_ITT_BUILD
3036 __kmp_itt_lock_acquired(lck);
3037#endif /* USE_ITT_BUILD */
3038
3039#if OMPT_SUPPORT && OMPT_OPTIONAL
3040 if (ompt_enabled.enabled) {
3041 if (acquire_status == KMP_LOCK_ACQUIRED_FIRST) {
3042 if (ompt_enabled.ompt_callback_mutex_acquired) {
3043 // lock_first
3044 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
3045 ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3046 }
3047 } else {
3048 if (ompt_enabled.ompt_callback_nest_lock) {
3049 // lock_next
3050 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
3051 ompt_scope_begin, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3052 }
3053 }
3054 }
3055#endif
3056
3057#endif // KMP_USE_DYNAMIC_LOCK
3058}
3059
3060void __kmpc_unset_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) {
3061#if KMP_USE_DYNAMIC_LOCK
3062
3063 int tag = KMP_EXTRACT_D_TAG(user_lock);
3064#if USE_ITT_BUILD
3065 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
3066#endif
3067#if KMP_USE_INLINED_TAS
3068 if (tag == locktag_tas && !__kmp_env_consistency_check) {
3069 KMP_RELEASE_TAS_LOCK(user_lock, gtid);
3070 } else
3071#elif KMP_USE_INLINED_FUTEX
3072 if (tag == locktag_futex && !__kmp_env_consistency_check) {
3073 KMP_RELEASE_FUTEX_LOCK(user_lock, gtid);
3074 } else
3075#endif
3076 {
3077 __kmp_direct_unset[tag]((kmp_dyna_lock_t *)user_lock, gtid);
3078 }
3079
3080#if OMPT_SUPPORT && OMPT_OPTIONAL
3081 // This is the case, if called from omp_init_lock_with_hint:
3082 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3083 if (!codeptr)
3084 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3085 if (ompt_enabled.ompt_callback_mutex_released) {
3086 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
3087 ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
3088 }
3089#endif
3090
3091#else // KMP_USE_DYNAMIC_LOCK
3092
3093 kmp_user_lock_p lck;
3094
3095 /* Can't use serial interval since not block structured */
3096 /* release the lock */
3097
3098 if ((__kmp_user_lock_kind == lk_tas) &&
3099 (sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
3100#if KMP_OS_LINUX && \
3101 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
3102// "fast" path implemented to fix customer performance issue
3103#if USE_ITT_BUILD
3104 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
3105#endif /* USE_ITT_BUILD */
3106 TCW_4(((kmp_user_lock_p)user_lock)->tas.lk.poll, 0);
3107 KMP_MB();
3108
3109#if OMPT_SUPPORT && OMPT_OPTIONAL
3110 // This is the case, if called from omp_init_lock_with_hint:
3111 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3112 if (!codeptr)
3113 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3114 if (ompt_enabled.ompt_callback_mutex_released) {
3115 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
3116 ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3117 }
3118#endif
3119
3120 return;
3121#else
3122 lck = (kmp_user_lock_p)user_lock;
3123#endif
3124 }
3125#if KMP_USE_FUTEX
3126 else if ((__kmp_user_lock_kind == lk_futex) &&
3127 (sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
3128 lck = (kmp_user_lock_p)user_lock;
3129 }
3130#endif
3131 else {
3132 lck = __kmp_lookup_user_lock(user_lock, "omp_unset_lock");
3133 }
3134
3135#if USE_ITT_BUILD
3136 __kmp_itt_lock_releasing(lck);
3137#endif /* USE_ITT_BUILD */
3138
3139 RELEASE_LOCK(lck, gtid);
3140
3141#if OMPT_SUPPORT && OMPT_OPTIONAL
3142 // This is the case, if called from omp_init_lock_with_hint:
3143 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3144 if (!codeptr)
3145 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3146 if (ompt_enabled.ompt_callback_mutex_released) {
3147 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
3148 ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3149 }
3150#endif
3151
3152#endif // KMP_USE_DYNAMIC_LOCK
3153}
3154
3155/* release the lock */
3156void __kmpc_unset_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) {
3157#if KMP_USE_DYNAMIC_LOCK
3158
3159#if USE_ITT_BUILD
3160 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
3161#endif
3162 int release_status =
3163 KMP_D_LOCK_FUNC(user_lock, unset)((kmp_dyna_lock_t *)user_lock, gtid);
3164 (void)release_status;
3165
3166#if OMPT_SUPPORT && OMPT_OPTIONAL
3167 // This is the case, if called from omp_init_lock_with_hint:
3168 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3169 if (!codeptr)
3170 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3171 if (ompt_enabled.enabled) {
3172 if (release_status == KMP_LOCK_RELEASED) {
3173 if (ompt_enabled.ompt_callback_mutex_released) {
3174 // release_lock_last
3175 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
3176 ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)user_lock,
3177 codeptr);
3178 }
3179 } else if (ompt_enabled.ompt_callback_nest_lock) {
3180 // release_lock_prev
3181 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
3182 ompt_scope_end, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
3183 }
3184 }
3185#endif
3186
3187#else // KMP_USE_DYNAMIC_LOCK
3188
3189 kmp_user_lock_p lck;
3190
3191 /* Can't use serial interval since not block structured */
3192
3193 if ((__kmp_user_lock_kind == lk_tas) &&
3194 (sizeof(lck->tas.lk.poll) + sizeof(lck->tas.lk.depth_locked) <=
3195 OMP_NEST_LOCK_T_SIZE)) {
3196#if KMP_OS_LINUX && \
3197 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
3198 // "fast" path implemented to fix customer performance issue
3199 kmp_tas_lock_t *tl = (kmp_tas_lock_t *)user_lock;
3200#if USE_ITT_BUILD
3201 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
3202#endif /* USE_ITT_BUILD */
3203
3204#if OMPT_SUPPORT && OMPT_OPTIONAL
3205 int release_status = KMP_LOCK_STILL_HELD;
3206#endif
3207
3208 if (--(tl->lk.depth_locked) == 0) {
3209 TCW_4(tl->lk.poll, 0);
3210#if OMPT_SUPPORT && OMPT_OPTIONAL
3211 release_status = KMP_LOCK_RELEASED;
3212#endif
3213 }
3214 KMP_MB();
3215
3216#if OMPT_SUPPORT && OMPT_OPTIONAL
3217 // This is the case, if called from omp_init_lock_with_hint:
3218 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3219 if (!codeptr)
3220 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3221 if (ompt_enabled.enabled) {
3222 if (release_status == KMP_LOCK_RELEASED) {
3223 if (ompt_enabled.ompt_callback_mutex_released) {
3224 // release_lock_last
3225 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
3226 ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3227 }
3228 } else if (ompt_enabled.ompt_callback_nest_lock) {
3229 // release_lock_previous
3230 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
3231 ompt_mutex_scope_end, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3232 }
3233 }
3234#endif
3235
3236 return;
3237#else
3238 lck = (kmp_user_lock_p)user_lock;
3239#endif
3240 }
3241#if KMP_USE_FUTEX
3242 else if ((__kmp_user_lock_kind == lk_futex) &&
3243 (sizeof(lck->futex.lk.poll) + sizeof(lck->futex.lk.depth_locked) <=
3244 OMP_NEST_LOCK_T_SIZE)) {
3245 lck = (kmp_user_lock_p)user_lock;
3246 }
3247#endif
3248 else {
3249 lck = __kmp_lookup_user_lock(user_lock, "omp_unset_nest_lock");
3250 }
3251
3252#if USE_ITT_BUILD
3253 __kmp_itt_lock_releasing(lck);
3254#endif /* USE_ITT_BUILD */
3255
3256 int release_status;
3257 release_status = RELEASE_NESTED_LOCK(lck, gtid);
3258#if OMPT_SUPPORT && OMPT_OPTIONAL
3259 // This is the case, if called from omp_init_lock_with_hint:
3260 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3261 if (!codeptr)
3262 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3263 if (ompt_enabled.enabled) {
3264 if (release_status == KMP_LOCK_RELEASED) {
3265 if (ompt_enabled.ompt_callback_mutex_released) {
3266 // release_lock_last
3267 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
3268 ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3269 }
3270 } else if (ompt_enabled.ompt_callback_nest_lock) {
3271 // release_lock_previous
3272 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
3273 ompt_mutex_scope_end, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3274 }
3275 }
3276#endif
3277
3278#endif // KMP_USE_DYNAMIC_LOCK
3279}
3280
3281/* try to acquire the lock */
3282int __kmpc_test_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) {
3283 KMP_COUNT_BLOCK(OMP_test_lock);
3284
3285#if KMP_USE_DYNAMIC_LOCK
3286 int rc;
3287 int tag = KMP_EXTRACT_D_TAG(user_lock);
3288#if USE_ITT_BUILD
3289 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
3290#endif
3291#if OMPT_SUPPORT && OMPT_OPTIONAL
3292 // This is the case, if called from omp_init_lock_with_hint:
3293 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3294 if (!codeptr)
3295 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3296 if (ompt_enabled.ompt_callback_mutex_acquire) {
3297 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
3298 ompt_mutex_test_lock, omp_lock_hint_none,
3299 __ompt_get_mutex_impl_type(user_lock),
3300 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
3301 }
3302#endif
3303#if KMP_USE_INLINED_TAS
3304 if (tag == locktag_tas && !__kmp_env_consistency_check) {
3305 KMP_TEST_TAS_LOCK(user_lock, gtid, rc);
3306 } else
3307#elif KMP_USE_INLINED_FUTEX
3308 if (tag == locktag_futex && !__kmp_env_consistency_check) {
3309 KMP_TEST_FUTEX_LOCK(user_lock, gtid, rc);
3310 } else
3311#endif
3312 {
3313 rc = __kmp_direct_test[tag]((kmp_dyna_lock_t *)user_lock, gtid);
3314 }
3315 if (rc) {
3316#if USE_ITT_BUILD
3317 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
3318#endif
3319#if OMPT_SUPPORT && OMPT_OPTIONAL
3320 if (ompt_enabled.ompt_callback_mutex_acquired) {
3321 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
3322 ompt_mutex_test_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
3323 }
3324#endif
3325 return FTN_TRUE;
3326 } else {
3327#if USE_ITT_BUILD
3328 __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
3329#endif
3330 return FTN_FALSE;
3331 }
3332
3333#else // KMP_USE_DYNAMIC_LOCK
3334
3335 kmp_user_lock_p lck;
3336 int rc;
3337
3338 if ((__kmp_user_lock_kind == lk_tas) &&
3339 (sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
3340 lck = (kmp_user_lock_p)user_lock;
3341 }
3342#if KMP_USE_FUTEX
3343 else if ((__kmp_user_lock_kind == lk_futex) &&
3344 (sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
3345 lck = (kmp_user_lock_p)user_lock;
3346 }
3347#endif
3348 else {
3349 lck = __kmp_lookup_user_lock(user_lock, "omp_test_lock");
3350 }
3351
3352#if USE_ITT_BUILD
3353 __kmp_itt_lock_acquiring(lck);
3354#endif /* USE_ITT_BUILD */
3355#if OMPT_SUPPORT && OMPT_OPTIONAL
3356 // This is the case, if called from omp_init_lock_with_hint:
3357 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3358 if (!codeptr)
3359 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3360 if (ompt_enabled.ompt_callback_mutex_acquire) {
3361 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
3362 ompt_mutex_test_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
3363 (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3364 }
3365#endif
3366
3367 rc = TEST_LOCK(lck, gtid);
3368#if USE_ITT_BUILD
3369 if (rc) {
3370 __kmp_itt_lock_acquired(lck);
3371 } else {
3372 __kmp_itt_lock_cancelled(lck);
3373 }
3374#endif /* USE_ITT_BUILD */
3375#if OMPT_SUPPORT && OMPT_OPTIONAL
3376 if (rc && ompt_enabled.ompt_callback_mutex_acquired) {
3377 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
3378 ompt_mutex_test_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3379 }
3380#endif
3381
3382 return (rc ? FTN_TRUE : FTN_FALSE);
3383
3384 /* Can't use serial interval since not block structured */
3385
3386#endif // KMP_USE_DYNAMIC_LOCK
3387}
3388
3389/* try to acquire the lock */
3390int __kmpc_test_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) {
3391#if KMP_USE_DYNAMIC_LOCK
3392 int rc;
3393#if USE_ITT_BUILD
3394 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
3395#endif
3396#if OMPT_SUPPORT && OMPT_OPTIONAL
3397 // This is the case, if called from omp_init_lock_with_hint:
3398 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3399 if (!codeptr)
3400 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3401 if (ompt_enabled.ompt_callback_mutex_acquire) {
3402 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
3403 ompt_mutex_test_nest_lock, omp_lock_hint_none,
3404 __ompt_get_mutex_impl_type(user_lock),
3405 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
3406 }
3407#endif
3408 rc = KMP_D_LOCK_FUNC(user_lock, test)((kmp_dyna_lock_t *)user_lock, gtid);
3409#if USE_ITT_BUILD
3410 if (rc) {
3411 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
3412 } else {
3413 __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
3414 }
3415#endif
3416#if OMPT_SUPPORT && OMPT_OPTIONAL
3417 if (ompt_enabled.enabled && rc) {
3418 if (rc == 1) {
3419 if (ompt_enabled.ompt_callback_mutex_acquired) {
3420 // lock_first
3421 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
3422 ompt_mutex_test_nest_lock, (ompt_wait_id_t)(uintptr_t)user_lock,
3423 codeptr);
3424 }
3425 } else {
3426 if (ompt_enabled.ompt_callback_nest_lock) {
3427 // lock_next
3428 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
3429 ompt_scope_begin, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
3430 }
3431 }
3432 }
3433#endif
3434 return rc;
3435
3436#else // KMP_USE_DYNAMIC_LOCK
3437
3438 kmp_user_lock_p lck;
3439 int rc;
3440
3441 if ((__kmp_user_lock_kind == lk_tas) &&
3442 (sizeof(lck->tas.lk.poll) + sizeof(lck->tas.lk.depth_locked) <=
3443 OMP_NEST_LOCK_T_SIZE)) {
3444 lck = (kmp_user_lock_p)user_lock;
3445 }
3446#if KMP_USE_FUTEX
3447 else if ((__kmp_user_lock_kind == lk_futex) &&
3448 (sizeof(lck->futex.lk.poll) + sizeof(lck->futex.lk.depth_locked) <=
3449 OMP_NEST_LOCK_T_SIZE)) {
3450 lck = (kmp_user_lock_p)user_lock;
3451 }
3452#endif
3453 else {
3454 lck = __kmp_lookup_user_lock(user_lock, "omp_test_nest_lock");
3455 }
3456
3457#if USE_ITT_BUILD
3458 __kmp_itt_lock_acquiring(lck);
3459#endif /* USE_ITT_BUILD */
3460
3461#if OMPT_SUPPORT && OMPT_OPTIONAL
3462 // This is the case, if called from omp_init_lock_with_hint:
3463 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3464 if (!codeptr)
3465 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3466 if (ompt_enabled.enabled) &&
3467 ompt_enabled.ompt_callback_mutex_acquire) {
3468 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
3469 ompt_mutex_test_nest_lock, omp_lock_hint_none,
3470 __ompt_get_mutex_impl_type(), (ompt_wait_id_t)(uintptr_t)lck,
3471 codeptr);
3472 }
3473#endif
3474
3475 rc = TEST_NESTED_LOCK(lck, gtid);
3476#if USE_ITT_BUILD
3477 if (rc) {
3478 __kmp_itt_lock_acquired(lck);
3479 } else {
3480 __kmp_itt_lock_cancelled(lck);
3481 }
3482#endif /* USE_ITT_BUILD */
3483#if OMPT_SUPPORT && OMPT_OPTIONAL
3484 if (ompt_enabled.enabled && rc) {
3485 if (rc == 1) {
3486 if (ompt_enabled.ompt_callback_mutex_acquired) {
3487 // lock_first
3488 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
3489 ompt_mutex_test_nest_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3490 }
3491 } else {
3492 if (ompt_enabled.ompt_callback_nest_lock) {
3493 // lock_next
3494 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
3495 ompt_mutex_scope_begin, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3496 }
3497 }
3498 }
3499#endif
3500 return rc;
3501
3502 /* Can't use serial interval since not block structured */
3503
3504#endif // KMP_USE_DYNAMIC_LOCK
3505}
3506
3507// Interface to fast scalable reduce methods routines
3508
3509// keep the selected method in a thread local structure for cross-function
3510// usage: will be used in __kmpc_end_reduce* functions;
3511// another solution: to re-determine the method one more time in
3512// __kmpc_end_reduce* functions (new prototype required then)
3513// AT: which solution is better?
3514#define __KMP_SET_REDUCTION_METHOD(gtid, rmethod) \
3515 ((__kmp_threads[(gtid)]->th.th_local.packed_reduction_method) = (rmethod))
3516
3517#define __KMP_GET_REDUCTION_METHOD(gtid) \
3518 (__kmp_threads[(gtid)]->th.th_local.packed_reduction_method)
3519
3520// description of the packed_reduction_method variable: look at the macros in
3521// kmp.h
3522
3523// used in a critical section reduce block
3524static __forceinline void
3525__kmp_enter_critical_section_reduce_block(ident_t *loc, kmp_int32 global_tid,
3526 kmp_critical_name *crit) {
3527
3528 // this lock was visible to a customer and to the threading profile tool as a
3529 // serial overhead span (although it's used for an internal purpose only)
3530 // why was it visible in previous implementation?
3531 // should we keep it visible in new reduce block?
3532 kmp_user_lock_p lck;
3533
3534#if KMP_USE_DYNAMIC_LOCK
3535
3536 kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit;
3537 // Check if it is initialized.
3538 if (*lk == 0) {
3539 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
3540 (void)KMP_COMPARE_AND_STORE_ACQ32((volatile kmp_int32 *)crit, 0,
3541 KMP_GET_D_TAG(__kmp_user_lock_seq));
3542 } else {
3543 __kmp_init_indirect_csptr(crit, loc, global_tid,
3544 KMP_GET_I_TAG(__kmp_user_lock_seq));
3545 }
3546 }
3547 // Branch for accessing the actual lock object and set operation. This
3548 // branching is inevitable since this lock initialization does not follow the
3549 // normal dispatch path (lock table is not used).
3550 if (KMP_EXTRACT_D_TAG(lk) != 0) {
3551 lck = (kmp_user_lock_p)lk;
3552 KMP_DEBUG_ASSERT(lck != NULL);
3553 if (__kmp_env_consistency_check) {
3554 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
3555 }
3556 KMP_D_LOCK_FUNC(lk, set)(lk, global_tid);
3557 } else {
3558 kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk);
3559 lck = ilk->lock;
3560 KMP_DEBUG_ASSERT(lck != NULL);
3561 if (__kmp_env_consistency_check) {
3562 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
3563 }
3564 KMP_I_LOCK_FUNC(ilk, set)(lck, global_tid);
3565 }
3566
3567#else // KMP_USE_DYNAMIC_LOCK
3568
3569 // We know that the fast reduction code is only emitted by Intel compilers
3570 // with 32 byte critical sections. If there isn't enough space, then we
3571 // have to use a pointer.
3572 if (__kmp_base_user_lock_size <= INTEL_CRITICAL_SIZE) {
3573 lck = (kmp_user_lock_p)crit;
3574 } else {
3575 lck = __kmp_get_critical_section_ptr(crit, loc, global_tid);
3576 }
3577 KMP_DEBUG_ASSERT(lck != NULL);
3578
3579 if (__kmp_env_consistency_check)
3580 __kmp_push_sync(global_tid, ct_critical, loc, lck);
3581
3582 __kmp_acquire_user_lock_with_checks(lck, global_tid);
3583
3584#endif // KMP_USE_DYNAMIC_LOCK
3585}
3586
3587// used in a critical section reduce block
3588static __forceinline void
3589__kmp_end_critical_section_reduce_block(ident_t *loc, kmp_int32 global_tid,
3590 kmp_critical_name *crit) {
3591
3592 kmp_user_lock_p lck;
3593
3594#if KMP_USE_DYNAMIC_LOCK
3595
3596 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
3597 lck = (kmp_user_lock_p)crit;
3598 if (__kmp_env_consistency_check)
3599 __kmp_pop_sync(global_tid, ct_critical, loc);
3600 KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
3601 } else {
3602 kmp_indirect_lock_t *ilk =
3603 (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
3604 if (__kmp_env_consistency_check)
3605 __kmp_pop_sync(global_tid, ct_critical, loc);
3606 KMP_I_LOCK_FUNC(ilk, unset)(ilk->lock, global_tid);
3607 }
3608
3609#else // KMP_USE_DYNAMIC_LOCK
3610
3611 // We know that the fast reduction code is only emitted by Intel compilers
3612 // with 32 byte critical sections. If there isn't enough space, then we have
3613 // to use a pointer.
3614 if (__kmp_base_user_lock_size > 32) {
3615 lck = *((kmp_user_lock_p *)crit);
3616 KMP_ASSERT(lck != NULL);
3617 } else {
3618 lck = (kmp_user_lock_p)crit;
3619 }
3620
3621 if (__kmp_env_consistency_check)
3622 __kmp_pop_sync(global_tid, ct_critical, loc);
3623
3624 __kmp_release_user_lock_with_checks(lck, global_tid);
3625
3626#endif // KMP_USE_DYNAMIC_LOCK
3627} // __kmp_end_critical_section_reduce_block
3628
3629static __forceinline int
3630__kmp_swap_teams_for_teams_reduction(kmp_info_t *th, kmp_team_t **team_p,
3631 int *task_state) {
3632 kmp_team_t *team;
3633
3634 // Check if we are inside the teams construct?
3635 if (th->th.th_teams_microtask) {
3636 *team_p = team = th->th.th_team;
3637 if (team->t.t_level == th->th.th_teams_level) {
3638 // This is reduction at teams construct.
3639 KMP_DEBUG_ASSERT(!th->th.th_info.ds.ds_tid); // AC: check that tid == 0
3640 // Let's swap teams temporarily for the reduction.
3641 th->th.th_info.ds.ds_tid = team->t.t_master_tid;
3642 th->th.th_team = team->t.t_parent;
3643 th->th.th_team_nproc = th->th.th_team->t.t_nproc;
3644 th->th.th_task_team = th->th.th_team->t.t_task_team[0];
3645 *task_state = th->th.th_task_state;
3646 th->th.th_task_state = 0;
3647
3648 return 1;
3649 }
3650 }
3651 return 0;
3652}
3653
3654static __forceinline void
3655__kmp_restore_swapped_teams(kmp_info_t *th, kmp_team_t *team, int task_state) {
3656 // Restore thread structure swapped in __kmp_swap_teams_for_teams_reduction.
3657 th->th.th_info.ds.ds_tid = 0;
3658 th->th.th_team = team;
3659 th->th.th_team_nproc = team->t.t_nproc;
3660 th->th.th_task_team = team->t.t_task_team[task_state];
3661 __kmp_type_convert(task_state, &(th->th.th_task_state));
3662}
3663
3664/* 2.a.i. Reduce Block without a terminating barrier */
3680kmp_int32
3681__kmpc_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars,
3682 size_t reduce_size, void *reduce_data,
3683 void (*reduce_func)(void *lhs_data, void *rhs_data),
3684 kmp_critical_name *lck) {
3685
3686 KMP_COUNT_BLOCK(REDUCE_nowait);
3687 int retval = 0;
3688 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3689 kmp_info_t *th;
3690 kmp_team_t *team;
3691 int teams_swapped = 0, task_state;
3692 KA_TRACE(10, ("__kmpc_reduce_nowait() enter: called T#%d\n", global_tid));
3693 __kmp_assert_valid_gtid(global_tid);
3694
3695 // why do we need this initialization here at all?
3696 // Reduction clause can not be used as a stand-alone directive.
3697
3698 // do not call __kmp_serial_initialize(), it will be called by
3699 // __kmp_parallel_initialize() if needed
3700 // possible detection of false-positive race by the threadchecker ???
3701 if (!TCR_4(__kmp_init_parallel))
3702 __kmp_parallel_initialize();
3703
3704 __kmp_resume_if_soft_paused();
3705
3706// check correctness of reduce block nesting
3707#if KMP_USE_DYNAMIC_LOCK
3708 if (__kmp_env_consistency_check)
3709 __kmp_push_sync(global_tid, ct_reduce, loc, NULL, 0);
3710#else
3711 if (__kmp_env_consistency_check)
3712 __kmp_push_sync(global_tid, ct_reduce, loc, NULL);
3713#endif
3714
3715 th = __kmp_thread_from_gtid(global_tid);
3716 teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state);
3717
3718 // packed_reduction_method value will be reused by __kmp_end_reduce* function,
3719 // the value should be kept in a variable
3720 // the variable should be either a construct-specific or thread-specific
3721 // property, not a team specific property
3722 // (a thread can reach the next reduce block on the next construct, reduce
3723 // method may differ on the next construct)
3724 // an ident_t "loc" parameter could be used as a construct-specific property
3725 // (what if loc == 0?)
3726 // (if both construct-specific and team-specific variables were shared,
3727 // then unness extra syncs should be needed)
3728 // a thread-specific variable is better regarding two issues above (next
3729 // construct and extra syncs)
3730 // a thread-specific "th_local.reduction_method" variable is used currently
3731 // each thread executes 'determine' and 'set' lines (no need to execute by one
3732 // thread, to avoid unness extra syncs)
3733
3734 packed_reduction_method = __kmp_determine_reduction_method(
3735 loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck);
3736 __KMP_SET_REDUCTION_METHOD(global_tid, packed_reduction_method);
3737
3738 OMPT_REDUCTION_DECL(th, global_tid);
3739 if (packed_reduction_method == critical_reduce_block) {
3740
3741 OMPT_REDUCTION_BEGIN;
3742
3743 __kmp_enter_critical_section_reduce_block(loc, global_tid, lck);
3744 retval = 1;
3745
3746 } else if (packed_reduction_method == empty_reduce_block) {
3747
3748 OMPT_REDUCTION_BEGIN;
3749
3750 // usage: if team size == 1, no synchronization is required ( Intel
3751 // platforms only )
3752 retval = 1;
3753
3754 } else if (packed_reduction_method == atomic_reduce_block) {
3755
3756 retval = 2;
3757
3758 // all threads should do this pop here (because __kmpc_end_reduce_nowait()
3759 // won't be called by the code gen)
3760 // (it's not quite good, because the checking block has been closed by
3761 // this 'pop',
3762 // but atomic operation has not been executed yet, will be executed
3763 // slightly later, literally on next instruction)
3764 if (__kmp_env_consistency_check)
3765 __kmp_pop_sync(global_tid, ct_reduce, loc);
3766
3767 } else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3768 tree_reduce_block)) {
3769
3770// AT: performance issue: a real barrier here
3771// AT: (if primary thread is slow, other threads are blocked here waiting for
3772// the primary thread to come and release them)
3773// AT: (it's not what a customer might expect specifying NOWAIT clause)
3774// AT: (specifying NOWAIT won't result in improvement of performance, it'll
3775// be confusing to a customer)
3776// AT: another implementation of *barrier_gather*nowait() (or some other design)
3777// might go faster and be more in line with sense of NOWAIT
3778// AT: TO DO: do epcc test and compare times
3779
3780// this barrier should be invisible to a customer and to the threading profile
3781// tool (it's neither a terminating barrier nor customer's code, it's
3782// used for an internal purpose)
3783#if OMPT_SUPPORT
3784 // JP: can this barrier potentially leed to task scheduling?
3785 // JP: as long as there is a barrier in the implementation, OMPT should and
3786 // will provide the barrier events
3787 // so we set-up the necessary frame/return addresses.
3788 ompt_frame_t *ompt_frame;
3789 if (ompt_enabled.enabled) {
3790 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3791 if (ompt_frame->enter_frame.ptr == NULL)
3792 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
3793 }
3794 OMPT_STORE_RETURN_ADDRESS(global_tid);
3795#endif
3796#if USE_ITT_NOTIFY
3797 __kmp_threads[global_tid]->th.th_ident = loc;
3798#endif
3799 retval =
3800 __kmp_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
3801 global_tid, FALSE, reduce_size, reduce_data, reduce_func);
3802 retval = (retval != 0) ? (0) : (1);
3803#if OMPT_SUPPORT && OMPT_OPTIONAL
3804 if (ompt_enabled.enabled) {
3805 ompt_frame->enter_frame = ompt_data_none;
3806 }
3807#endif
3808
3809 // all other workers except primary thread should do this pop here
3810 // ( none of other workers will get to __kmpc_end_reduce_nowait() )
3811 if (__kmp_env_consistency_check) {
3812 if (retval == 0) {
3813 __kmp_pop_sync(global_tid, ct_reduce, loc);
3814 }
3815 }
3816
3817 } else {
3818
3819 // should never reach this block
3820 KMP_ASSERT(0); // "unexpected method"
3821 }
3822 if (teams_swapped) {
3823 __kmp_restore_swapped_teams(th, team, task_state);
3824 }
3825 KA_TRACE(
3826 10,
3827 ("__kmpc_reduce_nowait() exit: called T#%d: method %08x, returns %08x\n",
3828 global_tid, packed_reduction_method, retval));
3829
3830 return retval;
3831}
3832
3841void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
3842 kmp_critical_name *lck) {
3843
3844 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3845
3846 KA_TRACE(10, ("__kmpc_end_reduce_nowait() enter: called T#%d\n", global_tid));
3847 __kmp_assert_valid_gtid(global_tid);
3848
3849 packed_reduction_method = __KMP_GET_REDUCTION_METHOD(global_tid);
3850
3851 OMPT_REDUCTION_DECL(__kmp_thread_from_gtid(global_tid), global_tid);
3852
3853 if (packed_reduction_method == critical_reduce_block) {
3854
3855 __kmp_end_critical_section_reduce_block(loc, global_tid, lck);
3856 OMPT_REDUCTION_END;
3857
3858 } else if (packed_reduction_method == empty_reduce_block) {
3859
3860 // usage: if team size == 1, no synchronization is required ( on Intel
3861 // platforms only )
3862
3863 OMPT_REDUCTION_END;
3864
3865 } else if (packed_reduction_method == atomic_reduce_block) {
3866
3867 // neither primary thread nor other workers should get here
3868 // (code gen does not generate this call in case 2: atomic reduce block)
3869 // actually it's better to remove this elseif at all;
3870 // after removal this value will checked by the 'else' and will assert
3871
3872 } else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3873 tree_reduce_block)) {
3874
3875 // only primary thread gets here
3876 // OMPT: tree reduction is annotated in the barrier code
3877
3878 } else {
3879
3880 // should never reach this block
3881 KMP_ASSERT(0); // "unexpected method"
3882 }
3883
3884 if (__kmp_env_consistency_check)
3885 __kmp_pop_sync(global_tid, ct_reduce, loc);
3886
3887 KA_TRACE(10, ("__kmpc_end_reduce_nowait() exit: called T#%d: method %08x\n",
3888 global_tid, packed_reduction_method));
3889
3890 return;
3891}
3892
3893/* 2.a.ii. Reduce Block with a terminating barrier */
3894
3910kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars,
3911 size_t reduce_size, void *reduce_data,
3912 void (*reduce_func)(void *lhs_data, void *rhs_data),
3913 kmp_critical_name *lck) {
3914 KMP_COUNT_BLOCK(REDUCE_wait);
3915 int retval = 0;
3916 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3917 kmp_info_t *th;
3918 kmp_team_t *team;
3919 int teams_swapped = 0, task_state;
3920
3921 KA_TRACE(10, ("__kmpc_reduce() enter: called T#%d\n", global_tid));
3922 __kmp_assert_valid_gtid(global_tid);
3923
3924 // why do we need this initialization here at all?
3925 // Reduction clause can not be a stand-alone directive.
3926
3927 // do not call __kmp_serial_initialize(), it will be called by
3928 // __kmp_parallel_initialize() if needed
3929 // possible detection of false-positive race by the threadchecker ???
3930 if (!TCR_4(__kmp_init_parallel))
3931 __kmp_parallel_initialize();
3932
3933 __kmp_resume_if_soft_paused();
3934
3935// check correctness of reduce block nesting
3936#if KMP_USE_DYNAMIC_LOCK
3937 if (__kmp_env_consistency_check)
3938 __kmp_push_sync(global_tid, ct_reduce, loc, NULL, 0);
3939#else
3940 if (__kmp_env_consistency_check)
3941 __kmp_push_sync(global_tid, ct_reduce, loc, NULL);
3942#endif
3943
3944 th = __kmp_thread_from_gtid(global_tid);
3945 teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state);
3946
3947 packed_reduction_method = __kmp_determine_reduction_method(
3948 loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck);
3949 __KMP_SET_REDUCTION_METHOD(global_tid, packed_reduction_method);
3950
3951 OMPT_REDUCTION_DECL(th, global_tid);
3952
3953 if (packed_reduction_method == critical_reduce_block) {
3954
3955 OMPT_REDUCTION_BEGIN;
3956 __kmp_enter_critical_section_reduce_block(loc, global_tid, lck);
3957 retval = 1;
3958
3959 } else if (packed_reduction_method == empty_reduce_block) {
3960
3961 OMPT_REDUCTION_BEGIN;
3962 // usage: if team size == 1, no synchronization is required ( Intel
3963 // platforms only )
3964 retval = 1;
3965
3966 } else if (packed_reduction_method == atomic_reduce_block) {
3967
3968 retval = 2;
3969
3970 } else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3971 tree_reduce_block)) {
3972
3973// case tree_reduce_block:
3974// this barrier should be visible to a customer and to the threading profile
3975// tool (it's a terminating barrier on constructs if NOWAIT not specified)
3976#if OMPT_SUPPORT
3977 ompt_frame_t *ompt_frame;
3978 if (ompt_enabled.enabled) {
3979 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3980 if (ompt_frame->enter_frame.ptr == NULL)
3981 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
3982 }
3983 OMPT_STORE_RETURN_ADDRESS(global_tid);
3984#endif
3985#if USE_ITT_NOTIFY
3986 __kmp_threads[global_tid]->th.th_ident =
3987 loc; // needed for correct notification of frames
3988#endif
3989 retval =
3990 __kmp_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
3991 global_tid, TRUE, reduce_size, reduce_data, reduce_func);
3992 retval = (retval != 0) ? (0) : (1);
3993#if OMPT_SUPPORT && OMPT_OPTIONAL
3994 if (ompt_enabled.enabled) {
3995 ompt_frame->enter_frame = ompt_data_none;
3996 }
3997#endif
3998
3999 // all other workers except primary thread should do this pop here
4000 // (none of other workers except primary will enter __kmpc_end_reduce())
4001 if (__kmp_env_consistency_check) {
4002 if (retval == 0) { // 0: all other workers; 1: primary thread
4003 __kmp_pop_sync(global_tid, ct_reduce, loc);
4004 }
4005 }
4006
4007 } else {
4008
4009 // should never reach this block
4010 KMP_ASSERT(0); // "unexpected method"
4011 }
4012 if (teams_swapped) {
4013 __kmp_restore_swapped_teams(th, team, task_state);
4014 }
4015
4016 KA_TRACE(10,
4017 ("__kmpc_reduce() exit: called T#%d: method %08x, returns %08x\n",
4018 global_tid, packed_reduction_method, retval));
4019 return retval;
4020}
4021
4032void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
4033 kmp_critical_name *lck) {
4034
4035 PACKED_REDUCTION_METHOD_T packed_reduction_method;
4036 kmp_info_t *th;
4037 kmp_team_t *team;
4038 int teams_swapped = 0, task_state;
4039
4040 KA_TRACE(10, ("__kmpc_end_reduce() enter: called T#%d\n", global_tid));
4041 __kmp_assert_valid_gtid(global_tid);
4042
4043 th = __kmp_thread_from_gtid(global_tid);
4044 teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state);
4045
4046 packed_reduction_method = __KMP_GET_REDUCTION_METHOD(global_tid);
4047
4048 // this barrier should be visible to a customer and to the threading profile
4049 // tool (it's a terminating barrier on constructs if NOWAIT not specified)
4050 OMPT_REDUCTION_DECL(th, global_tid);
4051
4052 if (packed_reduction_method == critical_reduce_block) {
4053 __kmp_end_critical_section_reduce_block(loc, global_tid, lck);
4054
4055 OMPT_REDUCTION_END;
4056
4057// TODO: implicit barrier: should be exposed
4058#if OMPT_SUPPORT
4059 ompt_frame_t *ompt_frame;
4060 if (ompt_enabled.enabled) {
4061 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
4062 if (ompt_frame->enter_frame.ptr == NULL)
4063 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
4064 }
4065 OMPT_STORE_RETURN_ADDRESS(global_tid);
4066#endif
4067#if USE_ITT_NOTIFY
4068 __kmp_threads[global_tid]->th.th_ident = loc;
4069#endif
4070 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
4071#if OMPT_SUPPORT && OMPT_OPTIONAL
4072 if (ompt_enabled.enabled) {
4073 ompt_frame->enter_frame = ompt_data_none;
4074 }
4075#endif
4076
4077 } else if (packed_reduction_method == empty_reduce_block) {
4078
4079 OMPT_REDUCTION_END;
4080
4081// usage: if team size==1, no synchronization is required (Intel platforms only)
4082
4083// TODO: implicit barrier: should be exposed
4084#if OMPT_SUPPORT
4085 ompt_frame_t *ompt_frame;
4086 if (ompt_enabled.enabled) {
4087 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
4088 if (ompt_frame->enter_frame.ptr == NULL)
4089 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
4090 }
4091 OMPT_STORE_RETURN_ADDRESS(global_tid);
4092#endif
4093#if USE_ITT_NOTIFY
4094 __kmp_threads[global_tid]->th.th_ident = loc;
4095#endif
4096 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
4097#if OMPT_SUPPORT && OMPT_OPTIONAL
4098 if (ompt_enabled.enabled) {
4099 ompt_frame->enter_frame = ompt_data_none;
4100 }
4101#endif
4102
4103 } else if (packed_reduction_method == atomic_reduce_block) {
4104
4105#if OMPT_SUPPORT
4106 ompt_frame_t *ompt_frame;
4107 if (ompt_enabled.enabled) {
4108 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
4109 if (ompt_frame->enter_frame.ptr == NULL)
4110 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
4111 }
4112 OMPT_STORE_RETURN_ADDRESS(global_tid);
4113#endif
4114// TODO: implicit barrier: should be exposed
4115#if USE_ITT_NOTIFY
4116 __kmp_threads[global_tid]->th.th_ident = loc;
4117#endif
4118 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
4119#if OMPT_SUPPORT && OMPT_OPTIONAL
4120 if (ompt_enabled.enabled) {
4121 ompt_frame->enter_frame = ompt_data_none;
4122 }
4123#endif
4124
4125 } else if (TEST_REDUCTION_METHOD(packed_reduction_method,
4126 tree_reduce_block)) {
4127
4128 // only primary thread executes here (primary releases all other workers)
4129 __kmp_end_split_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
4130 global_tid);
4131
4132 } else {
4133
4134 // should never reach this block
4135 KMP_ASSERT(0); // "unexpected method"
4136 }
4137 if (teams_swapped) {
4138 __kmp_restore_swapped_teams(th, team, task_state);
4139 }
4140
4141 if (__kmp_env_consistency_check)
4142 __kmp_pop_sync(global_tid, ct_reduce, loc);
4143
4144 KA_TRACE(10, ("__kmpc_end_reduce() exit: called T#%d: method %08x\n",
4145 global_tid, packed_reduction_method));
4146
4147 return;
4148}
4149
4150#undef __KMP_GET_REDUCTION_METHOD
4151#undef __KMP_SET_REDUCTION_METHOD
4152
4153/* end of interface to fast scalable reduce routines */
4154
4155kmp_uint64 __kmpc_get_taskid() {
4156
4157 kmp_int32 gtid;
4158 kmp_info_t *thread;
4159
4160 gtid = __kmp_get_gtid();
4161 if (gtid < 0) {
4162 return 0;
4163 }
4164 thread = __kmp_thread_from_gtid(gtid);
4165 return thread->th.th_current_task->td_task_id;
4166
4167} // __kmpc_get_taskid
4168
4169kmp_uint64 __kmpc_get_parent_taskid() {
4170
4171 kmp_int32 gtid;
4172 kmp_info_t *thread;
4173 kmp_taskdata_t *parent_task;
4174
4175 gtid = __kmp_get_gtid();
4176 if (gtid < 0) {
4177 return 0;
4178 }
4179 thread = __kmp_thread_from_gtid(gtid);
4180 parent_task = thread->th.th_current_task->td_parent;
4181 return (parent_task == NULL ? 0 : parent_task->td_task_id);
4182
4183} // __kmpc_get_parent_taskid
4184
4196void __kmpc_doacross_init(ident_t *loc, int gtid, int num_dims,
4197 const struct kmp_dim *dims) {
4198 __kmp_assert_valid_gtid(gtid);
4199 int j, idx;
4200 kmp_int64 last, trace_count;
4201 kmp_info_t *th = __kmp_threads[gtid];
4202 kmp_team_t *team = th->th.th_team;
4203 kmp_uint32 *flags;
4204 kmp_disp_t *pr_buf = th->th.th_dispatch;
4205 dispatch_shared_info_t *sh_buf;
4206
4207 KA_TRACE(
4208 20,
4209 ("__kmpc_doacross_init() enter: called T#%d, num dims %d, active %d\n",
4210 gtid, num_dims, !team->t.t_serialized));
4211 KMP_DEBUG_ASSERT(dims != NULL);
4212 KMP_DEBUG_ASSERT(num_dims > 0);
4213
4214 if (team->t.t_serialized) {
4215 KA_TRACE(20, ("__kmpc_doacross_init() exit: serialized team\n"));
4216 return; // no dependencies if team is serialized
4217 }
4218 KMP_DEBUG_ASSERT(team->t.t_nproc > 1);
4219 idx = pr_buf->th_doacross_buf_idx++; // Increment index of shared buffer for
4220 // the next loop
4221 sh_buf = &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers];
4222
4223 // Save bounds info into allocated private buffer
4224 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info == NULL);
4225 pr_buf->th_doacross_info = (kmp_int64 *)__kmp_thread_malloc(
4226 th, sizeof(kmp_int64) * (4 * num_dims + 1));
4227 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
4228 pr_buf->th_doacross_info[0] =
4229 (kmp_int64)num_dims; // first element is number of dimensions
4230 // Save also address of num_done in order to access it later without knowing
4231 // the buffer index
4232 pr_buf->th_doacross_info[1] = (kmp_int64)&sh_buf->doacross_num_done;
4233 pr_buf->th_doacross_info[2] = dims[0].lo;
4234 pr_buf->th_doacross_info[3] = dims[0].up;
4235 pr_buf->th_doacross_info[4] = dims[0].st;
4236 last = 5;
4237 for (j = 1; j < num_dims; ++j) {
4238 kmp_int64
4239 range_length; // To keep ranges of all dimensions but the first dims[0]
4240 if (dims[j].st == 1) { // most common case
4241 // AC: should we care of ranges bigger than LLONG_MAX? (not for now)
4242 range_length = dims[j].up - dims[j].lo + 1;
4243 } else {
4244 if (dims[j].st > 0) {
4245 KMP_DEBUG_ASSERT(dims[j].up > dims[j].lo);
4246 range_length = (kmp_uint64)(dims[j].up - dims[j].lo) / dims[j].st + 1;
4247 } else { // negative increment
4248 KMP_DEBUG_ASSERT(dims[j].lo > dims[j].up);
4249 range_length =
4250 (kmp_uint64)(dims[j].lo - dims[j].up) / (-dims[j].st) + 1;
4251 }
4252 }
4253 pr_buf->th_doacross_info[last++] = range_length;
4254 pr_buf->th_doacross_info[last++] = dims[j].lo;
4255 pr_buf->th_doacross_info[last++] = dims[j].up;
4256 pr_buf->th_doacross_info[last++] = dims[j].st;
4257 }
4258
4259 // Compute total trip count.
4260 // Start with range of dims[0] which we don't need to keep in the buffer.
4261 if (dims[0].st == 1) { // most common case
4262 trace_count = dims[0].up - dims[0].lo + 1;
4263 } else if (dims[0].st > 0) {
4264 KMP_DEBUG_ASSERT(dims[0].up > dims[0].lo);
4265 trace_count = (kmp_uint64)(dims[0].up - dims[0].lo) / dims[0].st + 1;
4266 } else { // negative increment
4267 KMP_DEBUG_ASSERT(dims[0].lo > dims[0].up);
4268 trace_count = (kmp_uint64)(dims[0].lo - dims[0].up) / (-dims[0].st) + 1;
4269 }
4270 for (j = 1; j < num_dims; ++j) {
4271 trace_count *= pr_buf->th_doacross_info[4 * j + 1]; // use kept ranges
4272 }
4273 KMP_DEBUG_ASSERT(trace_count > 0);
4274
4275 // Check if shared buffer is not occupied by other loop (idx -
4276 // __kmp_dispatch_num_buffers)
4277 if (idx != sh_buf->doacross_buf_idx) {
4278 // Shared buffer is occupied, wait for it to be free
4279 __kmp_wait_4((volatile kmp_uint32 *)&sh_buf->doacross_buf_idx, idx,
4280 __kmp_eq_4, NULL);
4281 }
4282#if KMP_32_BIT_ARCH
4283 // Check if we are the first thread. After the CAS the first thread gets 0,
4284 // others get 1 if initialization is in progress, allocated pointer otherwise.
4285 // Treat pointer as volatile integer (value 0 or 1) until memory is allocated.
4286 flags = (kmp_uint32 *)KMP_COMPARE_AND_STORE_RET32(
4287 (volatile kmp_int32 *)&sh_buf->doacross_flags, NULL, 1);
4288#else
4289 flags = (kmp_uint32 *)KMP_COMPARE_AND_STORE_RET64(
4290 (volatile kmp_int64 *)&sh_buf->doacross_flags, NULL, 1LL);
4291#endif
4292 if (flags == NULL) {
4293 // we are the first thread, allocate the array of flags
4294 size_t size =
4295 (size_t)trace_count / 8 + 8; // in bytes, use single bit per iteration
4296 flags = (kmp_uint32 *)__kmp_thread_calloc(th, size, 1);
4297 KMP_MB();
4298 sh_buf->doacross_flags = flags;
4299 } else if (flags == (kmp_uint32 *)1) {
4300#if KMP_32_BIT_ARCH
4301 // initialization is still in progress, need to wait
4302 while (*(volatile kmp_int32 *)&sh_buf->doacross_flags == 1)
4303#else
4304 while (*(volatile kmp_int64 *)&sh_buf->doacross_flags == 1LL)
4305#endif
4306 KMP_YIELD(TRUE);
4307 KMP_MB();
4308 } else {
4309 KMP_MB();
4310 }
4311 KMP_DEBUG_ASSERT(sh_buf->doacross_flags > (kmp_uint32 *)1); // check ptr value
4312 pr_buf->th_doacross_flags =
4313 sh_buf->doacross_flags; // save private copy in order to not
4314 // touch shared buffer on each iteration
4315 KA_TRACE(20, ("__kmpc_doacross_init() exit: T#%d\n", gtid));
4316}
4317
4318void __kmpc_doacross_wait(ident_t *loc, int gtid, const kmp_int64 *vec) {
4319 __kmp_assert_valid_gtid(gtid);
4320 kmp_int64 shft;
4321 size_t num_dims, i;
4322 kmp_uint32 flag;
4323 kmp_int64 iter_number; // iteration number of "collapsed" loop nest
4324 kmp_info_t *th = __kmp_threads[gtid];
4325 kmp_team_t *team = th->th.th_team;
4326 kmp_disp_t *pr_buf;
4327 kmp_int64 lo, up, st;
4328
4329 KA_TRACE(20, ("__kmpc_doacross_wait() enter: called T#%d\n", gtid));
4330 if (team->t.t_serialized) {
4331 KA_TRACE(20, ("__kmpc_doacross_wait() exit: serialized team\n"));
4332 return; // no dependencies if team is serialized
4333 }
4334
4335 // calculate sequential iteration number and check out-of-bounds condition
4336 pr_buf = th->th.th_dispatch;
4337 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
4338 num_dims = (size_t)pr_buf->th_doacross_info[0];
4339 lo = pr_buf->th_doacross_info[2];
4340 up = pr_buf->th_doacross_info[3];
4341 st = pr_buf->th_doacross_info[4];
4342#if OMPT_SUPPORT && OMPT_OPTIONAL
4343 SimpleVLA<ompt_dependence_t> deps(num_dims);
4344#endif
4345 if (st == 1) { // most common case
4346 if (vec[0] < lo || vec[0] > up) {
4347 KA_TRACE(20, ("__kmpc_doacross_wait() exit: T#%d iter %lld is out of "
4348 "bounds [%lld,%lld]\n",
4349 gtid, vec[0], lo, up));
4350 return;
4351 }
4352 iter_number = vec[0] - lo;
4353 } else if (st > 0) {
4354 if (vec[0] < lo || vec[0] > up) {
4355 KA_TRACE(20, ("__kmpc_doacross_wait() exit: T#%d iter %lld is out of "
4356 "bounds [%lld,%lld]\n",
4357 gtid, vec[0], lo, up));
4358 return;
4359 }
4360 iter_number = (kmp_uint64)(vec[0] - lo) / st;
4361 } else { // negative increment
4362 if (vec[0] > lo || vec[0] < up) {
4363 KA_TRACE(20, ("__kmpc_doacross_wait() exit: T#%d iter %lld is out of "
4364 "bounds [%lld,%lld]\n",
4365 gtid, vec[0], lo, up));
4366 return;
4367 }
4368 iter_number = (kmp_uint64)(lo - vec[0]) / (-st);
4369 }
4370#if OMPT_SUPPORT && OMPT_OPTIONAL
4371 deps[0].variable.value = iter_number;
4372 deps[0].dependence_type = ompt_dependence_type_sink;
4373#endif
4374 for (i = 1; i < num_dims; ++i) {
4375 kmp_int64 iter, ln;
4376 size_t j = i * 4;
4377 ln = pr_buf->th_doacross_info[j + 1];
4378 lo = pr_buf->th_doacross_info[j + 2];
4379 up = pr_buf->th_doacross_info[j + 3];
4380 st = pr_buf->th_doacross_info[j + 4];
4381 if (st == 1) {
4382 if (vec[i] < lo || vec[i] > up) {
4383 KA_TRACE(20, ("__kmpc_doacross_wait() exit: T#%d iter %lld is out of "
4384 "bounds [%lld,%lld]\n",
4385 gtid, vec[i], lo, up));
4386 return;
4387 }
4388 iter = vec[i] - lo;
4389 } else if (st > 0) {
4390 if (vec[i] < lo || vec[i] > up) {
4391 KA_TRACE(20, ("__kmpc_doacross_wait() exit: T#%d iter %lld is out of "
4392 "bounds [%lld,%lld]\n",
4393 gtid, vec[i], lo, up));
4394 return;
4395 }
4396 iter = (kmp_uint64)(vec[i] - lo) / st;
4397 } else { // st < 0
4398 if (vec[i] > lo || vec[i] < up) {
4399 KA_TRACE(20, ("__kmpc_doacross_wait() exit: T#%d iter %lld is out of "
4400 "bounds [%lld,%lld]\n",
4401 gtid, vec[i], lo, up));
4402 return;
4403 }
4404 iter = (kmp_uint64)(lo - vec[i]) / (-st);
4405 }
4406 iter_number = iter + ln * iter_number;
4407#if OMPT_SUPPORT && OMPT_OPTIONAL
4408 deps[i].variable.value = iter;
4409 deps[i].dependence_type = ompt_dependence_type_sink;
4410#endif
4411 }
4412 shft = iter_number % 32; // use 32-bit granularity
4413 iter_number >>= 5; // divided by 32
4414 flag = 1 << shft;
4415 while ((flag & pr_buf->th_doacross_flags[iter_number]) == 0) {
4416 KMP_YIELD(TRUE);
4417 }
4418 KMP_MB();
4419#if OMPT_SUPPORT && OMPT_OPTIONAL
4420 if (ompt_enabled.ompt_callback_dependences) {
4421 ompt_callbacks.ompt_callback(ompt_callback_dependences)(
4422 &(OMPT_CUR_TASK_INFO(th)->task_data), deps, (kmp_uint32)num_dims);
4423 }
4424#endif
4425 KA_TRACE(20,
4426 ("__kmpc_doacross_wait() exit: T#%d wait for iter %lld completed\n",
4427 gtid, (iter_number << 5) + shft));
4428}
4429
4430void __kmpc_doacross_post(ident_t *loc, int gtid, const kmp_int64 *vec) {
4431 __kmp_assert_valid_gtid(gtid);
4432 kmp_int64 shft;
4433 size_t num_dims, i;
4434 kmp_uint32 flag;
4435 kmp_int64 iter_number; // iteration number of "collapsed" loop nest
4436 kmp_info_t *th = __kmp_threads[gtid];
4437 kmp_team_t *team = th->th.th_team;
4438 kmp_disp_t *pr_buf;
4439 kmp_int64 lo, st;
4440
4441 KA_TRACE(20, ("__kmpc_doacross_post() enter: called T#%d\n", gtid));
4442 if (team->t.t_serialized) {
4443 KA_TRACE(20, ("__kmpc_doacross_post() exit: serialized team\n"));
4444 return; // no dependencies if team is serialized
4445 }
4446
4447 // calculate sequential iteration number (same as in "wait" but no
4448 // out-of-bounds checks)
4449 pr_buf = th->th.th_dispatch;
4450 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
4451 num_dims = (size_t)pr_buf->th_doacross_info[0];
4452 lo = pr_buf->th_doacross_info[2];
4453 st = pr_buf->th_doacross_info[4];
4454#if OMPT_SUPPORT && OMPT_OPTIONAL
4455 SimpleVLA<ompt_dependence_t> deps(num_dims);
4456#endif
4457 if (st == 1) { // most common case
4458 iter_number = vec[0] - lo;
4459 } else if (st > 0) {
4460 iter_number = (kmp_uint64)(vec[0] - lo) / st;
4461 } else { // negative increment
4462 iter_number = (kmp_uint64)(lo - vec[0]) / (-st);
4463 }
4464#if OMPT_SUPPORT && OMPT_OPTIONAL
4465 deps[0].variable.value = iter_number;
4466 deps[0].dependence_type = ompt_dependence_type_source;
4467#endif
4468 for (i = 1; i < num_dims; ++i) {
4469 kmp_int64 iter, ln;
4470 size_t j = i * 4;
4471 ln = pr_buf->th_doacross_info[j + 1];
4472 lo = pr_buf->th_doacross_info[j + 2];
4473 st = pr_buf->th_doacross_info[j + 4];
4474 if (st == 1) {
4475 iter = vec[i] - lo;
4476 } else if (st > 0) {
4477 iter = (kmp_uint64)(vec[i] - lo) / st;
4478 } else { // st < 0
4479 iter = (kmp_uint64)(lo - vec[i]) / (-st);
4480 }
4481 iter_number = iter + ln * iter_number;
4482#if OMPT_SUPPORT && OMPT_OPTIONAL
4483 deps[i].variable.value = iter;
4484 deps[i].dependence_type = ompt_dependence_type_source;
4485#endif
4486 }
4487#if OMPT_SUPPORT && OMPT_OPTIONAL
4488 if (ompt_enabled.ompt_callback_dependences) {
4489 ompt_callbacks.ompt_callback(ompt_callback_dependences)(
4490 &(OMPT_CUR_TASK_INFO(th)->task_data), deps, (kmp_uint32)num_dims);
4491 }
4492#endif
4493 shft = iter_number % 32; // use 32-bit granularity
4494 iter_number >>= 5; // divided by 32
4495 flag = 1 << shft;
4496 KMP_MB();
4497 if ((flag & pr_buf->th_doacross_flags[iter_number]) == 0)
4498 KMP_TEST_THEN_OR32(&pr_buf->th_doacross_flags[iter_number], flag);
4499 KA_TRACE(20, ("__kmpc_doacross_post() exit: T#%d iter %lld posted\n", gtid,
4500 (iter_number << 5) + shft));
4501}
4502
4503void __kmpc_doacross_fini(ident_t *loc, int gtid) {
4504 __kmp_assert_valid_gtid(gtid);
4505 kmp_int32 num_done;
4506 kmp_info_t *th = __kmp_threads[gtid];
4507 kmp_team_t *team = th->th.th_team;
4508 kmp_disp_t *pr_buf = th->th.th_dispatch;
4509
4510 KA_TRACE(20, ("__kmpc_doacross_fini() enter: called T#%d\n", gtid));
4511 if (team->t.t_serialized) {
4512 KA_TRACE(20, ("__kmpc_doacross_fini() exit: serialized team %p\n", team));
4513 return; // nothing to do
4514 }
4515 num_done =
4516 KMP_TEST_THEN_INC32((kmp_uintptr_t)(pr_buf->th_doacross_info[1])) + 1;
4517 if (num_done == th->th.th_team_nproc) {
4518 // we are the last thread, need to free shared resources
4519 int idx = pr_buf->th_doacross_buf_idx - 1;
4520 dispatch_shared_info_t *sh_buf =
4521 &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers];
4522 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info[1] ==
4523 (kmp_int64)&sh_buf->doacross_num_done);
4524 KMP_DEBUG_ASSERT(num_done == sh_buf->doacross_num_done);
4525 KMP_DEBUG_ASSERT(idx == sh_buf->doacross_buf_idx);
4526 __kmp_thread_free(th, CCAST(kmp_uint32 *, sh_buf->doacross_flags));
4527 sh_buf->doacross_flags = NULL;
4528 sh_buf->doacross_num_done = 0;
4529 sh_buf->doacross_buf_idx +=
4530 __kmp_dispatch_num_buffers; // free buffer for future re-use
4531 }
4532 // free private resources (need to keep buffer index forever)
4533 pr_buf->th_doacross_flags = NULL;
4534 __kmp_thread_free(th, (void *)pr_buf->th_doacross_info);
4535 pr_buf->th_doacross_info = NULL;
4536 KA_TRACE(20, ("__kmpc_doacross_fini() exit: T#%d\n", gtid));
4537}
4538
4539/* OpenMP 5.1 Memory Management routines */
4540void *omp_alloc(size_t size, omp_allocator_handle_t allocator) {
4541 return __kmp_alloc(__kmp_entry_gtid(), 0, size, allocator);
4542}
4543
4544void *omp_aligned_alloc(size_t align, size_t size,
4545 omp_allocator_handle_t allocator) {
4546 return __kmp_alloc(__kmp_entry_gtid(), align, size, allocator);
4547}
4548
4549void *omp_calloc(size_t nmemb, size_t size, omp_allocator_handle_t allocator) {
4550 return __kmp_calloc(__kmp_entry_gtid(), 0, nmemb, size, allocator);
4551}
4552
4553void *omp_aligned_calloc(size_t align, size_t nmemb, size_t size,
4554 omp_allocator_handle_t allocator) {
4555 return __kmp_calloc(__kmp_entry_gtid(), align, nmemb, size, allocator);
4556}
4557
4558void *omp_realloc(void *ptr, size_t size, omp_allocator_handle_t allocator,
4559 omp_allocator_handle_t free_allocator) {
4560 return __kmp_realloc(__kmp_entry_gtid(), ptr, size, allocator,
4561 free_allocator);
4562}
4563
4564void omp_free(void *ptr, omp_allocator_handle_t allocator) {
4565 ___kmpc_free(__kmp_entry_gtid(), ptr, allocator);
4566}
4567/* end of OpenMP 5.1 Memory Management routines */
4568
4569void *omp_get_dyn_gprivate_ptr(size_t offset, omp_access_t access_group) {
4570 return NULL;
4571}
4572
4573void *omp_get_dyn_gprivate_nofb_ptr(size_t offset, omp_access_t access_group) {
4574 return NULL;
4575}
4576
4577size_t omp_get_dyn_gprivate_size(omp_access_t access_group) { return 0; }
4578
4579omp_memspace_handle_t omp_get_dyn_gprivate_memspace(omp_access_t access_group) {
4580 return omp_null_mem_space;
4581}
4582
4583int __kmpc_get_target_offload(void) {
4584 if (!__kmp_init_serial) {
4585 __kmp_serial_initialize();
4586 }
4587 return __kmp_target_offload;
4588}
4589
4590int __kmpc_pause_resource(kmp_pause_status_t level) {
4591 if (!__kmp_init_serial) {
4592 return 1; // Can't pause if runtime is not initialized
4593 }
4594 return __kmp_pause_resource(level);
4595}
4596
4597void __kmpc_error(ident_t *loc, int severity, const char *message) {
4598 if (!__kmp_init_serial)
4599 __kmp_serial_initialize();
4600
4601 KMP_ASSERT(severity == severity_warning || severity == severity_fatal);
4602
4603#if OMPT_SUPPORT
4604 if (ompt_enabled.enabled && ompt_enabled.ompt_callback_error) {
4605 ompt_callbacks.ompt_callback(ompt_callback_error)(
4606 (ompt_severity_t)severity, message, KMP_STRLEN(message),
4607 OMPT_GET_RETURN_ADDRESS(0));
4608 }
4609#endif // OMPT_SUPPORT
4610
4611 char *src_loc;
4612 if (loc && loc->psource) {
4613 kmp_str_loc_t str_loc = __kmp_str_loc_init(loc->psource, false);
4614 src_loc =
4615 __kmp_str_format("%s:%d:%d", str_loc.file, str_loc.line, str_loc.col);
4616 __kmp_str_loc_free(&str_loc);
4617 } else {
4618 src_loc = __kmp_str_format("unknown");
4619 }
4620
4621 if (severity == severity_warning)
4622 KMP_WARNING(UserDirectedWarning, src_loc, message);
4623 else
4624 KMP_FATAL(UserDirectedError, src_loc, message);
4625
4626 __kmp_str_free(&src_loc);
4627}
4628
4629// Mark begin of scope directive.
4630void __kmpc_scope(ident_t *loc, kmp_int32 gtid, void *reserved) {
4631// reserved is for extension of scope directive and not used.
4632#if OMPT_SUPPORT && OMPT_OPTIONAL
4633 if (ompt_enabled.enabled && ompt_enabled.ompt_callback_work) {
4634 kmp_team_t *team = __kmp_threads[gtid]->th.th_team;
4635 int tid = __kmp_tid_from_gtid(gtid);
4636 ompt_callbacks.ompt_callback(ompt_callback_work)(
4637 ompt_work_scope, ompt_scope_begin,
4638 &(team->t.ompt_team_info.parallel_data),
4639 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 1,
4640 OMPT_GET_RETURN_ADDRESS(0));
4641 }
4642#endif // OMPT_SUPPORT && OMPT_OPTIONAL
4643}
4644
4645// Mark end of scope directive
4646void __kmpc_end_scope(ident_t *loc, kmp_int32 gtid, void *reserved) {
4647// reserved is for extension of scope directive and not used.
4648#if OMPT_SUPPORT && OMPT_OPTIONAL
4649 if (ompt_enabled.enabled && ompt_enabled.ompt_callback_work) {
4650 kmp_team_t *team = __kmp_threads[gtid]->th.th_team;
4651 int tid = __kmp_tid_from_gtid(gtid);
4652 ompt_callbacks.ompt_callback(ompt_callback_work)(
4653 ompt_work_scope, ompt_scope_end,
4654 &(team->t.ompt_team_info.parallel_data),
4655 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 1,
4656 OMPT_GET_RETURN_ADDRESS(0));
4657 }
4658#endif // OMPT_SUPPORT && OMPT_OPTIONAL
4659}
4660
4661#ifdef KMP_USE_VERSION_SYMBOLS
4662// For GOMP compatibility there are two versions of each omp_* API.
4663// One is the plain C symbol and one is the Fortran symbol with an appended
4664// underscore. When we implement a specific ompc_* version of an omp_*
4665// function, we want the plain GOMP versioned symbol to alias the ompc_* version
4666// instead of the Fortran versions in kmp_ftn_entry.h
4667extern "C" {
4668// Have to undef these from omp.h so they aren't translated into
4669// their ompc counterparts in the KMP_VERSION_OMPC_SYMBOL macros below
4670#ifdef omp_set_affinity_format
4671#undef omp_set_affinity_format
4672#endif
4673#ifdef omp_get_affinity_format
4674#undef omp_get_affinity_format
4675#endif
4676#ifdef omp_display_affinity
4677#undef omp_display_affinity
4678#endif
4679#ifdef omp_capture_affinity
4680#undef omp_capture_affinity
4681#endif
4682KMP_VERSION_OMPC_SYMBOL(ompc_set_affinity_format, omp_set_affinity_format, 50,
4683 "OMP_5.0");
4684KMP_VERSION_OMPC_SYMBOL(ompc_get_affinity_format, omp_get_affinity_format, 50,
4685 "OMP_5.0");
4686KMP_VERSION_OMPC_SYMBOL(ompc_display_affinity, omp_display_affinity, 50,
4687 "OMP_5.0");
4688KMP_VERSION_OMPC_SYMBOL(ompc_capture_affinity, omp_capture_affinity, 50,
4689 "OMP_5.0");
4690} // extern "C"
4691#endif
KMP_IDENT_WORK_LOOP
@ KMP_IDENT_WORK_LOOP
Definition: kmp.h:197
KMP_IDENT_WORK_SECTIONS
@ KMP_IDENT_WORK_SECTIONS
Definition: kmp.h:199
KMP_IDENT_AUTOPAR
@ KMP_IDENT_AUTOPAR
Definition: kmp.h:182
KMP_IDENT_WORK_DISTRIBUTE
@ KMP_IDENT_WORK_DISTRIBUTE
Definition: kmp.h:201
__kmpc_ok_to_fork
kmp_int32 __kmpc_ok_to_fork(ident_t *loc)
Definition: kmp_csupport.cpp:157
__kmpc_fork_teams
void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)
Definition: kmp_csupport.cpp:502
__kmpc_fork_call_if
void __kmpc_fork_call_if(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, kmp_int32 cond, void *args)
Definition: kmp_csupport.cpp:388
__kmpc_push_num_threads
void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)
Definition: kmp_csupport.cpp:232
__kmpc_set_thread_limit
void __kmpc_set_thread_limit(ident_t *loc, kmp_int32 global_tid, kmp_int32 thread_limit)
Definition: kmp_csupport.cpp:457
__kmpc_serialized_parallel
void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
Definition: kmp_csupport.cpp:603
__kmpc_push_num_threads_list
void __kmpc_push_num_threads_list(ident_t *loc, kmp_int32 global_tid, kmp_uint32 list_length, kmp_int32 *num_threads_list)
Definition: kmp_csupport.cpp:260
__kmpc_push_num_teams
void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams, kmp_int32 num_threads)
Definition: kmp_csupport.cpp:438
__kmpc_fork_call
void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)
Definition: kmp_csupport.cpp:307
__kmpc_end_serialized_parallel
void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
Definition: kmp_csupport.cpp:621
kmpc_micro
void(* kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
Definition: kmp.h:1764
__kmpc_push_num_teams_51
void __kmpc_push_num_teams_51(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams_lb, kmp_int32 num_teams_ub, kmp_int32 num_threads)
Definition: kmp_csupport.cpp:481
__kmpc_begin
void __kmpc_begin(ident_t *loc, kmp_int32 flags)
Definition: kmp_csupport.cpp:37
__kmpc_end
void __kmpc_end(ident_t *loc)
Definition: kmp_csupport.cpp:60
KMP_COUNT_BLOCK
#define KMP_COUNT_BLOCK(name)
Increments specified counter (name).
Definition: kmp_stats.h:911
stats_state_e
stats_state_e
the states which a thread can be in
Definition: kmp_stats.h:63
__kmpc_end_reduce
void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
Definition: kmp_csupport.cpp:4032
__kmpc_end_barrier_master
void __kmpc_end_barrier_master(ident_t *loc, kmp_int32 global_tid)
Definition: kmp_csupport.cpp:1887
__kmpc_barrier_master_nowait
kmp_int32 __kmpc_barrier_master_nowait(ident_t *loc, kmp_int32 global_tid)
Definition: kmp_csupport.cpp:1903
__kmpc_end_reduce_nowait
void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
Definition: kmp_csupport.cpp:3841
__kmpc_reduce
kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
Definition: kmp_csupport.cpp:3910
__kmpc_flush_explicit
void __kmpc_flush_explicit(ident_t *loc, kmp_int32 order, kmp_int32 scope)
Definition: kmp_csupport.cpp:811
__kmpc_barrier
void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid)
Definition: kmp_csupport.cpp:855
__kmpc_flush
void __kmpc_flush(ident_t *loc)
Definition: kmp_csupport.cpp:790
__kmpc_barrier_master
kmp_int32 __kmpc_barrier_master(ident_t *loc, kmp_int32 global_tid)
Definition: kmp_csupport.cpp:1843
__kmpc_reduce_nowait
kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
Definition: kmp_csupport.cpp:3681
__kmpc_copyprivate
void __kmpc_copyprivate(ident_t *loc, kmp_int32 gtid, size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *), kmp_int32 didit)
Definition: kmp_csupport.cpp:2323
__kmpc_copyprivate_light
void * __kmpc_copyprivate_light(ident_t *loc, kmp_int32 gtid, void *cpy_data)
Definition: kmp_csupport.cpp:2399
__kmpc_global_num_threads
kmp_int32 __kmpc_global_num_threads(ident_t *loc)
Definition: kmp_csupport.cpp:122
__kmpc_global_thread_num
kmp_int32 __kmpc_global_thread_num(ident_t *loc)
Definition: kmp_csupport.cpp:100
__kmpc_in_parallel
kmp_int32 __kmpc_in_parallel(ident_t *loc)
Definition: kmp_csupport.cpp:219
__kmpc_bound_thread_num
kmp_int32 __kmpc_bound_thread_num(ident_t *loc)
Definition: kmp_csupport.cpp:135
__kmpc_bound_num_threads
kmp_int32 __kmpc_bound_num_threads(ident_t *loc)
Definition: kmp_csupport.cpp:145
__kmpc_end_ordered
void __kmpc_end_ordered(ident_t *loc, kmp_int32 gtid)
Definition: kmp_csupport.cpp:1151
__kmpc_end_critical
void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
Definition: kmp_csupport.cpp:1746
__kmpc_for_static_fini
void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid)
Definition: kmp_csupport.cpp:2046
__kmpc_end_masked
void __kmpc_end_masked(ident_t *loc, kmp_int32 global_tid)
Definition: kmp_csupport.cpp:1049
__kmpc_master
kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid)
Definition: kmp_csupport.cpp:904
__kmpc_critical_with_hint
void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit, uint32_t hint)
Definition: kmp_csupport.cpp:1613
__kmpc_single
kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid)
Definition: kmp_csupport.cpp:1966
__kmpc_doacross_init
void __kmpc_doacross_init(ident_t *loc, int gtid, int num_dims, const struct kmp_dim *dims)
Definition: kmp_csupport.cpp:4196
__kmpc_end_master
void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid)
Definition: kmp_csupport.cpp:961
__kmpc_end_single
void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid)
Definition: kmp_csupport.cpp:2019
__kmpc_ordered
void __kmpc_ordered(ident_t *loc, kmp_int32 gtid)
Definition: kmp_csupport.cpp:1078
__kmpc_masked
kmp_int32 __kmpc_masked(ident_t *loc, kmp_int32 global_tid, kmp_int32 filter)
Definition: kmp_csupport.cpp:993
__kmpc_critical
void __kmpc_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
Definition: kmp_csupport.cpp:1386
ident
Definition: kmp.h:217
ident::psource
char const * psource
Definition: kmp.h:227
ident::flags
kmp_int32 flags
Definition: kmp.h:219
Generated by doxygen 1.9.4