LLVM OpenMP* Runtime Library
Loading...
Searching...
No Matches
kmp_csupport.cpp
1/*
2 * kmp_csupport.cpp -- kfront linkage support for OpenMP.
3 */
4
5//===----------------------------------------------------------------------===//
6//
7// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8// See https://llvm.org/LICENSE.txt for license information.
9// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10//
11//===----------------------------------------------------------------------===//
12
13#define __KMP_IMP
14#include "omp.h" /* extern "C" declarations of user-visible routines */
15#include "kmp.h"
16#include "kmp_error.h"
17#include "kmp_i18n.h"
18#include "kmp_itt.h"
19#include "kmp_lock.h"
20#include "kmp_stats.h"
21#include "kmp_utils.h"
22#include "ompt-specific.h"
23
24#define MAX_MESSAGE 512
25
26// flags will be used in future, e.g. to implement openmp_strict library
27// restrictions
28
37void __kmpc_begin(ident_t *loc, kmp_int32 flags) {
38 // By default __kmpc_begin() is no-op.
39 char *env;
40 if ((env = getenv("KMP_INITIAL_THREAD_BIND")) != NULL &&
41 __kmp_str_match_true(env)) {
42 __kmp_middle_initialize();
43 __kmp_assign_root_init_mask();
44 KC_TRACE(10, ("__kmpc_begin: middle initialization called\n"));
45 } else if (__kmp_ignore_mppbeg() == FALSE) {
46 // By default __kmp_ignore_mppbeg() returns TRUE.
47 __kmp_internal_begin();
48 KC_TRACE(10, ("__kmpc_begin: called\n"));
49 }
50}
51
60void __kmpc_end(ident_t *loc) {
61 // By default, __kmp_ignore_mppend() returns TRUE which makes __kmpc_end()
62 // call no-op. However, this can be overridden with KMP_IGNORE_MPPEND
63 // environment variable. If KMP_IGNORE_MPPEND is 0, __kmp_ignore_mppend()
64 // returns FALSE and __kmpc_end() will unregister this root (it can cause
65 // library shut down).
66 if (__kmp_ignore_mppend() == FALSE) {
67 KC_TRACE(10, ("__kmpc_end: called\n"));
68 KA_TRACE(30, ("__kmpc_end\n"));
69
70 __kmp_internal_end_thread(-1);
71 }
72#if KMP_OS_WINDOWS && OMPT_SUPPORT
73 // Normal exit process on Windows does not allow worker threads of the final
74 // parallel region to finish reporting their events, so shutting down the
75 // library here fixes the issue at least for the cases where __kmpc_end() is
76 // placed properly.
77 if (ompt_enabled.enabled)
78 __kmp_internal_end_library(__kmp_gtid_get_specific());
79#endif
80}
81
100kmp_int32 __kmpc_global_thread_num(ident_t *loc) {
101 kmp_int32 gtid = __kmp_entry_gtid();
102
103 KC_TRACE(10, ("__kmpc_global_thread_num: T#%d\n", gtid));
104
105 return gtid;
106}
107
122kmp_int32 __kmpc_global_num_threads(ident_t *loc) {
123 KC_TRACE(10,
124 ("__kmpc_global_num_threads: num_threads = %d\n", __kmp_all_nth));
125
126 return TCR_4(__kmp_all_nth);
127}
128
135kmp_int32 __kmpc_bound_thread_num(ident_t *loc) {
136 KC_TRACE(10, ("__kmpc_bound_thread_num: called\n"));
137 return __kmp_tid_from_gtid(__kmp_entry_gtid());
138}
139
145kmp_int32 __kmpc_bound_num_threads(ident_t *loc) {
146 KC_TRACE(10, ("__kmpc_bound_num_threads: called\n"));
147
148 return __kmp_entry_thread()->th.th_team->t.t_nproc;
149}
150
157kmp_int32 __kmpc_ok_to_fork(ident_t *loc) {
158#ifndef KMP_DEBUG
159
160 return TRUE;
161
162#else
163
164 const char *semi2;
165 const char *semi3;
166 int line_no;
167
168 if (__kmp_par_range == 0) {
169 return TRUE;
170 }
171 semi2 = loc->psource;
172 if (semi2 == NULL) {
173 return TRUE;
174 }
175 semi2 = strchr(semi2, ';');
176 if (semi2 == NULL) {
177 return TRUE;
178 }
179 semi2 = strchr(semi2 + 1, ';');
180 if (semi2 == NULL) {
181 return TRUE;
182 }
183 if (__kmp_par_range_filename[0]) {
184 const char *name = semi2 - 1;
185 while ((name > loc->psource) && (*name != '/') && (*name != ';')) {
186 name--;
187 }
188 if ((*name == '/') || (*name == ';')) {
189 name++;
190 }
191 if (strncmp(__kmp_par_range_filename, name, semi2 - name)) {
192 return __kmp_par_range < 0;
193 }
194 }
195 semi3 = strchr(semi2 + 1, ';');
196 if (__kmp_par_range_routine[0]) {
197 if ((semi3 != NULL) && (semi3 > semi2) &&
198 (strncmp(__kmp_par_range_routine, semi2 + 1, semi3 - semi2 - 1))) {
199 return __kmp_par_range < 0;
200 }
201 }
202 if (KMP_SSCANF(semi3 + 1, "%d", &line_no) == 1) {
203 if ((line_no >= __kmp_par_range_lb) && (line_no <= __kmp_par_range_ub)) {
204 return __kmp_par_range > 0;
205 }
206 return __kmp_par_range < 0;
207 }
208 return TRUE;
209
210#endif /* KMP_DEBUG */
211}
212
219kmp_int32 __kmpc_in_parallel(ident_t *loc) {
220 return __kmp_entry_thread()->th.th_root->r.r_active;
221}
222
232void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid,
233 kmp_int32 num_threads) {
234 KA_TRACE(20, ("__kmpc_push_num_threads: enter T#%d num_threads=%d\n",
235 global_tid, num_threads));
236 __kmp_assert_valid_gtid(global_tid);
237 __kmp_push_num_threads(loc, global_tid, num_threads);
238}
239
240void __kmpc_push_num_threads_strict(ident_t *loc, kmp_int32 global_tid,
241 kmp_int32 num_threads, int severity,
242 const char *message) {
243 __kmp_push_num_threads(loc, global_tid, num_threads);
244 __kmp_set_strict_num_threads(loc, global_tid, severity, message);
245}
246
260void __kmpc_push_num_threads_list(ident_t *loc, kmp_int32 global_tid,
261 kmp_uint32 list_length,
262 kmp_int32 *num_threads_list) {
263 KA_TRACE(20, ("__kmpc_push_num_threads_list: enter T#%d num_threads_list=",
264 global_tid));
265 KA_TRACE(20, ("%d", num_threads_list[0]));
266#ifdef KMP_DEBUG
267 for (kmp_uint32 i = 1; i < list_length; ++i)
268 KA_TRACE(20, (", %d", num_threads_list[i]));
269#endif
270 KA_TRACE(20, ("/n"));
271
272 __kmp_assert_valid_gtid(global_tid);
273 __kmp_push_num_threads_list(loc, global_tid, list_length, num_threads_list);
274}
275
276void __kmpc_push_num_threads_list_strict(ident_t *loc, kmp_int32 global_tid,
277 kmp_uint32 list_length,
278 kmp_int32 *num_threads_list,
279 int severity, const char *message) {
280 __kmp_push_num_threads_list(loc, global_tid, list_length, num_threads_list);
281 __kmp_set_strict_num_threads(loc, global_tid, severity, message);
282}
283
284void __kmpc_pop_num_threads(ident_t *loc, kmp_int32 global_tid) {
285 KA_TRACE(20, ("__kmpc_pop_num_threads: enter\n"));
286 /* the num_threads are automatically popped */
287}
288
289void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid,
290 kmp_int32 proc_bind) {
291 KA_TRACE(20, ("__kmpc_push_proc_bind: enter T#%d proc_bind=%d\n", global_tid,
292 proc_bind));
293 __kmp_assert_valid_gtid(global_tid);
294 __kmp_push_proc_bind(loc, global_tid, (kmp_proc_bind_t)proc_bind);
295}
296
307void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, ...) {
308 int gtid = __kmp_entry_gtid();
309
310#if (KMP_STATS_ENABLED)
311 // If we were in a serial region, then stop the serial timer, record
312 // the event, and start parallel region timer
313 stats_state_e previous_state = KMP_GET_THREAD_STATE();
314 if (previous_state == stats_state_e::SERIAL_REGION) {
315 KMP_EXCHANGE_PARTITIONED_TIMER(OMP_parallel_overhead);
316 } else {
317 KMP_PUSH_PARTITIONED_TIMER(OMP_parallel_overhead);
318 }
319 int inParallel = __kmpc_in_parallel(loc);
320 if (inParallel) {
321 KMP_COUNT_BLOCK(OMP_NESTED_PARALLEL);
322 } else {
323 KMP_COUNT_BLOCK(OMP_PARALLEL);
324 }
325#endif
326
327 // maybe to save thr_state is enough here
328 {
329 va_list ap;
330 va_start(ap, microtask);
331
332#if OMPT_SUPPORT
333 ompt_frame_t *ompt_frame;
334 if (ompt_enabled.enabled) {
335 kmp_info_t *master_th = __kmp_threads[gtid];
336 ompt_frame = &master_th->th.th_current_task->ompt_task_info.frame;
337 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
338 }
339 OMPT_STORE_RETURN_ADDRESS(gtid);
340#endif
341
342#if INCLUDE_SSC_MARKS
343 SSC_MARK_FORKING();
344#endif
345 __kmp_fork_call(loc, gtid, fork_context_intel, argc,
346 VOLATILE_CAST(microtask_t) microtask, // "wrapped" task
347 VOLATILE_CAST(launch_t) __kmp_invoke_task_func,
348 kmp_va_addr_of(ap));
349#if INCLUDE_SSC_MARKS
350 SSC_MARK_JOINING();
351#endif
352 __kmp_join_call(loc, gtid
353#if OMPT_SUPPORT
354 ,
355 fork_context_intel
356#endif
357 );
358
359 va_end(ap);
360
361#if OMPT_SUPPORT
362 if (ompt_enabled.enabled) {
363 ompt_frame->enter_frame = ompt_data_none;
364 }
365#endif
366 }
367
368#if KMP_STATS_ENABLED
369 if (previous_state == stats_state_e::SERIAL_REGION) {
370 KMP_EXCHANGE_PARTITIONED_TIMER(OMP_serial);
371 KMP_SET_THREAD_STATE(previous_state);
372 } else {
373 KMP_POP_PARTITIONED_TIMER();
374 }
375#endif // KMP_STATS_ENABLED
376}
377
388void __kmpc_fork_call_if(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,
389 kmp_int32 cond, void *args) {
390 int gtid = __kmp_entry_gtid();
391 if (cond) {
392 if (args)
393 __kmpc_fork_call(loc, argc, microtask, args);
394 else
395 __kmpc_fork_call(loc, argc, microtask);
396 } else {
397 __kmpc_serialized_parallel(loc, gtid);
398
399#if OMPT_SUPPORT
400 void *exit_frame_ptr;
401#endif
402
403 if (args)
404 __kmp_invoke_microtask(VOLATILE_CAST(microtask_t) microtask, gtid,
405 /*npr=*/0,
406 /*argc=*/1, &args
407#if OMPT_SUPPORT
408 ,
409 &exit_frame_ptr
410#endif
411 );
412 else
413 __kmp_invoke_microtask(VOLATILE_CAST(microtask_t) microtask, gtid,
414 /*npr=*/0,
415 /*argc=*/0,
416 /*args=*/nullptr
417#if OMPT_SUPPORT
418 ,
419 &exit_frame_ptr
420#endif
421 );
422
423 __kmpc_end_serialized_parallel(loc, gtid);
424 }
425}
426
438void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid,
439 kmp_int32 num_teams, kmp_int32 num_threads) {
440 KA_TRACE(20,
441 ("__kmpc_push_num_teams: enter T#%d num_teams=%d num_threads=%d\n",
442 global_tid, num_teams, num_threads));
443 __kmp_assert_valid_gtid(global_tid);
444 __kmp_push_num_teams(loc, global_tid, num_teams, num_threads);
445}
446
457void __kmpc_set_thread_limit(ident_t *loc, kmp_int32 global_tid,
458 kmp_int32 thread_limit) {
459 __kmp_assert_valid_gtid(global_tid);
460 kmp_info_t *thread = __kmp_threads[global_tid];
461 if (thread_limit > 0)
462 thread->th.th_current_task->td_icvs.task_thread_limit = thread_limit;
463}
464
481void __kmpc_push_num_teams_51(ident_t *loc, kmp_int32 global_tid,
482 kmp_int32 num_teams_lb, kmp_int32 num_teams_ub,
483 kmp_int32 num_threads) {
484 KA_TRACE(20, ("__kmpc_push_num_teams_51: enter T#%d num_teams_lb=%d"
485 " num_teams_ub=%d num_threads=%d\n",
486 global_tid, num_teams_lb, num_teams_ub, num_threads));
487 __kmp_assert_valid_gtid(global_tid);
488 __kmp_push_num_teams_51(loc, global_tid, num_teams_lb, num_teams_ub,
489 num_threads);
490}
491
502void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,
503 ...) {
504 int gtid = __kmp_entry_gtid();
505 kmp_info_t *this_thr = __kmp_threads[gtid];
506 va_list ap;
507 va_start(ap, microtask);
508
509#if KMP_STATS_ENABLED
510 KMP_COUNT_BLOCK(OMP_TEAMS);
511 stats_state_e previous_state = KMP_GET_THREAD_STATE();
512 if (previous_state == stats_state_e::SERIAL_REGION) {
513 KMP_EXCHANGE_PARTITIONED_TIMER(OMP_teams_overhead);
514 } else {
515 KMP_PUSH_PARTITIONED_TIMER(OMP_teams_overhead);
516 }
517#endif
518
519 // remember teams entry point and nesting level
520 this_thr->th.th_teams_microtask = microtask;
521 this_thr->th.th_teams_level =
522 this_thr->th.th_team->t.t_level; // AC: can be >0 on host
523
524#if OMPT_SUPPORT
525 kmp_team_t *parent_team = this_thr->th.th_team;
526 int tid = __kmp_tid_from_gtid(gtid);
527 if (ompt_enabled.enabled) {
528 parent_team->t.t_implicit_task_taskdata[tid]
529 .ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
530 }
531 OMPT_STORE_RETURN_ADDRESS(gtid);
532#endif
533
534 // check if __kmpc_push_num_teams called, set default number of teams
535 // otherwise
536 if (this_thr->th.th_teams_size.nteams == 0) {
537 __kmp_push_num_teams(loc, gtid, 0, 0);
538 }
539 KMP_DEBUG_ASSERT(this_thr->th.th_set_nproc >= 1);
540 KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nteams >= 1);
541 KMP_DEBUG_ASSERT(this_thr->th.th_teams_size.nth >= 1);
542
543 __kmp_fork_call(
544 loc, gtid, fork_context_intel, argc,
545 VOLATILE_CAST(microtask_t) __kmp_teams_master, // "wrapped" task
546 VOLATILE_CAST(launch_t) __kmp_invoke_teams_master, kmp_va_addr_of(ap));
547 __kmp_join_call(loc, gtid
548#if OMPT_SUPPORT
549 ,
550 fork_context_intel
551#endif
552 );
553
554 // Pop current CG root off list
555 KMP_DEBUG_ASSERT(this_thr->th.th_cg_roots);
556 kmp_cg_root_t *tmp = this_thr->th.th_cg_roots;
557 this_thr->th.th_cg_roots = tmp->up;
558 KA_TRACE(100, ("__kmpc_fork_teams: Thread %p popping node %p and moving up"
559 " to node %p. cg_nthreads was %d\n",
560 this_thr, tmp, this_thr->th.th_cg_roots, tmp->cg_nthreads));
561 KMP_DEBUG_ASSERT(tmp->cg_nthreads);
562 int i = tmp->cg_nthreads--;
563 if (i == 1) { // check is we are the last thread in CG (not always the case)
564 __kmp_free(tmp);
565 }
566 // Restore current task's thread_limit from CG root
567 KMP_DEBUG_ASSERT(this_thr->th.th_cg_roots);
568 this_thr->th.th_current_task->td_icvs.thread_limit =
569 this_thr->th.th_cg_roots->cg_thread_limit;
570
571 this_thr->th.th_teams_microtask = NULL;
572 this_thr->th.th_teams_level = 0;
573 memset(&this_thr->th.th_teams_size, 0, sizeof(kmp_teams_size_t));
574 va_end(ap);
575#if KMP_STATS_ENABLED
576 if (previous_state == stats_state_e::SERIAL_REGION) {
577 KMP_EXCHANGE_PARTITIONED_TIMER(OMP_serial);
578 KMP_SET_THREAD_STATE(previous_state);
579 } else {
580 KMP_POP_PARTITIONED_TIMER();
581 }
582#endif // KMP_STATS_ENABLED
583}
584
585// I don't think this function should ever have been exported.
586// The __kmpc_ prefix was misapplied. I'm fairly certain that no generated
587// openmp code ever called it, but it's been exported from the RTL for so
588// long that I'm afraid to remove the definition.
589int __kmpc_invoke_task_func(int gtid) { return __kmp_invoke_task_func(gtid); }
590
603void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
604 // The implementation is now in kmp_runtime.cpp so that it can share static
605 // functions with kmp_fork_call since the tasks to be done are similar in
606 // each case.
607 __kmp_assert_valid_gtid(global_tid);
608#if OMPT_SUPPORT
609 OMPT_STORE_RETURN_ADDRESS(global_tid);
610#endif
611 __kmp_serialized_parallel(loc, global_tid);
612}
613
621void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid) {
622 kmp_internal_control_t *top;
623 kmp_info_t *this_thr;
624 kmp_team_t *serial_team;
625
626 KC_TRACE(10,
627 ("__kmpc_end_serialized_parallel: called by T#%d\n", global_tid));
628
629 /* skip all this code for autopar serialized loops since it results in
630 unacceptable overhead */
631 if (loc != NULL && (loc->flags & KMP_IDENT_AUTOPAR))
632 return;
633
634 // Not autopar code
635 __kmp_assert_valid_gtid(global_tid);
636 if (!TCR_4(__kmp_init_parallel))
637 __kmp_parallel_initialize();
638
639 __kmp_resume_if_soft_paused();
640
641 this_thr = __kmp_threads[global_tid];
642 serial_team = this_thr->th.th_serial_team;
643
644 kmp_task_team_t *task_team = this_thr->th.th_task_team;
645 // we need to wait for the proxy tasks before finishing the thread
646 if (task_team != NULL && (task_team->tt.tt_found_proxy_tasks ||
647 task_team->tt.tt_hidden_helper_task_encountered))
648 __kmp_task_team_wait(this_thr, serial_team USE_ITT_BUILD_ARG(NULL));
649
650 KMP_MB();
651 KMP_DEBUG_ASSERT(serial_team);
652 KMP_ASSERT(serial_team->t.t_serialized);
653 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team);
654 KMP_DEBUG_ASSERT(serial_team != this_thr->th.th_root->r.r_root_team);
655 KMP_DEBUG_ASSERT(serial_team->t.t_threads);
656 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
657
658#if OMPT_SUPPORT
659 if (ompt_enabled.enabled &&
660 this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
661 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame = ompt_data_none;
662 if (ompt_enabled.ompt_callback_implicit_task) {
663 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
664 ompt_scope_end, NULL, OMPT_CUR_TASK_DATA(this_thr), 1,
665 OMPT_CUR_TASK_INFO(this_thr)->thread_num, ompt_task_implicit);
666 }
667
668 // reset clear the task id only after unlinking the task
669 ompt_data_t *parent_task_data;
670 __ompt_get_task_info_internal(1, NULL, &parent_task_data, NULL, NULL, NULL);
671
672 if (ompt_enabled.ompt_callback_parallel_end) {
673 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
674 &(serial_team->t.ompt_team_info.parallel_data), parent_task_data,
675 ompt_parallel_invoker_program | ompt_parallel_team,
676 OMPT_LOAD_RETURN_ADDRESS(global_tid));
677 }
678 __ompt_lw_taskteam_unlink(this_thr);
679 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
680 }
681#endif
682
683 /* If necessary, pop the internal control stack values and replace the team
684 * values */
685 top = serial_team->t.t_control_stack_top;
686 if (top && top->serial_nesting_level == serial_team->t.t_serialized) {
687 copy_icvs(&serial_team->t.t_threads[0]->th.th_current_task->td_icvs, top);
688 serial_team->t.t_control_stack_top = top->next;
689 __kmp_free(top);
690 }
691
692 /* pop dispatch buffers stack */
693 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch->th_disp_buffer);
694 {
695 dispatch_private_info_t *disp_buffer =
696 serial_team->t.t_dispatch->th_disp_buffer;
697 serial_team->t.t_dispatch->th_disp_buffer =
698 serial_team->t.t_dispatch->th_disp_buffer->next;
699 __kmp_free(disp_buffer);
700 }
701
702 /* pop the task team stack */
703 if (serial_team->t.t_serialized > 1) {
704 __kmp_pop_task_team_node(this_thr, serial_team);
705 }
706
707 this_thr->th.th_def_allocator = serial_team->t.t_def_allocator; // restore
708
709 --serial_team->t.t_serialized;
710 if (serial_team->t.t_serialized == 0) {
711
712 /* return to the parallel section */
713
714#if KMP_ARCH_X86 || KMP_ARCH_X86_64
715 if (__kmp_inherit_fp_control && serial_team->t.t_fp_control_saved) {
716 __kmp_clear_x87_fpu_status_word();
717 __kmp_load_x87_fpu_control_word(&serial_team->t.t_x87_fpu_control_word);
718 __kmp_load_mxcsr(&serial_team->t.t_mxcsr);
719 }
720#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
721
722 __kmp_pop_current_task_from_thread(this_thr);
723#if OMPD_SUPPORT
724 if (ompd_state & OMPD_ENABLE_BP)
725 ompd_bp_parallel_end();
726#endif
727
728 this_thr->th.th_team = serial_team->t.t_parent;
729 this_thr->th.th_info.ds.ds_tid = serial_team->t.t_master_tid;
730
731 /* restore values cached in the thread */
732 this_thr->th.th_team_nproc = serial_team->t.t_parent->t.t_nproc; /* JPH */
733 this_thr->th.th_team_master =
734 serial_team->t.t_parent->t.t_threads[0]; /* JPH */
735 this_thr->th.th_team_serialized = this_thr->th.th_team->t.t_serialized;
736
737 /* TODO the below shouldn't need to be adjusted for serialized teams */
738 this_thr->th.th_dispatch =
739 &this_thr->th.th_team->t.t_dispatch[serial_team->t.t_master_tid];
740
741 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 0);
742 this_thr->th.th_current_task->td_flags.executing = 1;
743
744 if (__kmp_tasking_mode != tskm_immediate_exec) {
745 // Restore task state from serial team structure
746 KMP_DEBUG_ASSERT(serial_team->t.t_primary_task_state == 0 ||
747 serial_team->t.t_primary_task_state == 1);
748 this_thr->th.th_task_state =
749 (kmp_uint8)serial_team->t.t_primary_task_state;
750 // Copy the task team from the new child / old parent team to the thread.
751 this_thr->th.th_task_team =
752 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state];
753 KA_TRACE(20,
754 ("__kmpc_end_serialized_parallel: T#%d restoring task_team %p / "
755 "team %p\n",
756 global_tid, this_thr->th.th_task_team, this_thr->th.th_team));
757 }
758#if KMP_AFFINITY_SUPPORTED
759 if (this_thr->th.th_team->t.t_level == 0 && __kmp_affinity.flags.reset) {
760 __kmp_reset_root_init_mask(global_tid);
761 }
762#endif
763 } else {
764 if (__kmp_tasking_mode != tskm_immediate_exec) {
765 KA_TRACE(20, ("__kmpc_end_serialized_parallel: T#%d decreasing nesting "
766 "depth of serial team %p to %d\n",
767 global_tid, serial_team, serial_team->t.t_serialized));
768 }
769 }
770
771 serial_team->t.t_level--;
772 if (__kmp_env_consistency_check)
773 __kmp_pop_parallel(global_tid, NULL);
774#if OMPT_SUPPORT
775 if (ompt_enabled.enabled)
776 this_thr->th.ompt_thread_info.state =
777 ((this_thr->th.th_team_serialized) ? ompt_state_work_serial
778 : ompt_state_work_parallel);
779#endif
780}
781
790void __kmpc_flush(ident_t *loc) {
791 KC_TRACE(10, ("__kmpc_flush: called\n"));
792
793 /* need explicit __mf() here since use volatile instead in library */
794 KMP_MFENCE(); /* Flush all pending memory write invalidates. */
795
796#if OMPT_SUPPORT && OMPT_OPTIONAL
797 if (ompt_enabled.ompt_callback_flush) {
798 ompt_callbacks.ompt_callback(ompt_callback_flush)(
799 __ompt_get_thread_data_internal(), OMPT_GET_RETURN_ADDRESS(0));
800 }
801#endif
802}
803
804/* -------------------------------------------------------------------------- */
812void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid) {
813 KMP_COUNT_BLOCK(OMP_BARRIER);
814 KC_TRACE(10, ("__kmpc_barrier: called T#%d\n", global_tid));
815 __kmp_assert_valid_gtid(global_tid);
816
817 if (!TCR_4(__kmp_init_parallel))
818 __kmp_parallel_initialize();
819
820 __kmp_resume_if_soft_paused();
821
822 if (__kmp_env_consistency_check) {
823 if (loc == 0) {
824 KMP_WARNING(ConstructIdentInvalid); // ??? What does it mean for the user?
825 }
826 __kmp_check_barrier(global_tid, ct_barrier, loc);
827 }
828
829#if OMPT_SUPPORT
830 ompt_frame_t *ompt_frame;
831 if (ompt_enabled.enabled) {
832 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
833 if (ompt_frame->enter_frame.ptr == NULL)
834 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
835 }
836 OMPT_STORE_RETURN_ADDRESS(global_tid);
837#endif
838 __kmp_threads[global_tid]->th.th_ident = loc;
839 // TODO: explicit barrier_wait_id:
840 // this function is called when 'barrier' directive is present or
841 // implicit barrier at the end of a worksharing construct.
842 // 1) better to add a per-thread barrier counter to a thread data structure
843 // 2) set to 0 when a new team is created
844 // 4) no sync is required
845
846 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
847#if OMPT_SUPPORT && OMPT_OPTIONAL
848 if (ompt_enabled.enabled) {
849 ompt_frame->enter_frame = ompt_data_none;
850 }
851#endif
852}
853
854/* The BARRIER for a MASTER section is always explicit */
861kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid) {
862 int status = 0;
863
864 KC_TRACE(10, ("__kmpc_master: called T#%d\n", global_tid));
865 __kmp_assert_valid_gtid(global_tid);
866
867 if (!TCR_4(__kmp_init_parallel))
868 __kmp_parallel_initialize();
869
870 __kmp_resume_if_soft_paused();
871
872 if (KMP_MASTER_GTID(global_tid)) {
873 KMP_COUNT_BLOCK(OMP_MASTER);
874 KMP_PUSH_PARTITIONED_TIMER(OMP_master);
875 status = 1;
876 }
877
878#if OMPT_SUPPORT && OMPT_OPTIONAL
879 if (status) {
880 if (ompt_enabled.ompt_callback_masked) {
881 kmp_info_t *this_thr = __kmp_threads[global_tid];
882 kmp_team_t *team = this_thr->th.th_team;
883
884 int tid = __kmp_tid_from_gtid(global_tid);
885 ompt_callbacks.ompt_callback(ompt_callback_masked)(
886 ompt_scope_begin, &(team->t.ompt_team_info.parallel_data),
887 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
888 OMPT_GET_RETURN_ADDRESS(0));
889 }
890 }
891#endif
892
893 if (__kmp_env_consistency_check) {
894#if KMP_USE_DYNAMIC_LOCK
895 if (status)
896 __kmp_push_sync(global_tid, ct_master, loc, NULL, 0);
897 else
898 __kmp_check_sync(global_tid, ct_master, loc, NULL, 0);
899#else
900 if (status)
901 __kmp_push_sync(global_tid, ct_master, loc, NULL);
902 else
903 __kmp_check_sync(global_tid, ct_master, loc, NULL);
904#endif
905 }
906
907 return status;
908}
909
918void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid) {
919 KC_TRACE(10, ("__kmpc_end_master: called T#%d\n", global_tid));
920 __kmp_assert_valid_gtid(global_tid);
921 KMP_DEBUG_ASSERT(KMP_MASTER_GTID(global_tid));
922 KMP_POP_PARTITIONED_TIMER();
923
924#if OMPT_SUPPORT && OMPT_OPTIONAL
925 kmp_info_t *this_thr = __kmp_threads[global_tid];
926 kmp_team_t *team = this_thr->th.th_team;
927 if (ompt_enabled.ompt_callback_masked) {
928 int tid = __kmp_tid_from_gtid(global_tid);
929 ompt_callbacks.ompt_callback(ompt_callback_masked)(
930 ompt_scope_end, &(team->t.ompt_team_info.parallel_data),
931 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
932 OMPT_GET_RETURN_ADDRESS(0));
933 }
934#endif
935
936 if (__kmp_env_consistency_check) {
937 if (KMP_MASTER_GTID(global_tid))
938 __kmp_pop_sync(global_tid, ct_master, loc);
939 }
940}
941
950kmp_int32 __kmpc_masked(ident_t *loc, kmp_int32 global_tid, kmp_int32 filter) {
951 int status = 0;
952 int tid;
953 KC_TRACE(10, ("__kmpc_masked: called T#%d\n", global_tid));
954 __kmp_assert_valid_gtid(global_tid);
955
956 if (!TCR_4(__kmp_init_parallel))
957 __kmp_parallel_initialize();
958
959 __kmp_resume_if_soft_paused();
960
961 tid = __kmp_tid_from_gtid(global_tid);
962 if (tid == filter) {
963 KMP_COUNT_BLOCK(OMP_MASKED);
964 KMP_PUSH_PARTITIONED_TIMER(OMP_masked);
965 status = 1;
966 }
967
968#if OMPT_SUPPORT && OMPT_OPTIONAL
969 if (status) {
970 if (ompt_enabled.ompt_callback_masked) {
971 kmp_info_t *this_thr = __kmp_threads[global_tid];
972 kmp_team_t *team = this_thr->th.th_team;
973 ompt_callbacks.ompt_callback(ompt_callback_masked)(
974 ompt_scope_begin, &(team->t.ompt_team_info.parallel_data),
975 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
976 OMPT_GET_RETURN_ADDRESS(0));
977 }
978 }
979#endif
980
981 if (__kmp_env_consistency_check) {
982#if KMP_USE_DYNAMIC_LOCK
983 if (status)
984 __kmp_push_sync(global_tid, ct_masked, loc, NULL, 0);
985 else
986 __kmp_check_sync(global_tid, ct_masked, loc, NULL, 0);
987#else
988 if (status)
989 __kmp_push_sync(global_tid, ct_masked, loc, NULL);
990 else
991 __kmp_check_sync(global_tid, ct_masked, loc, NULL);
992#endif
993 }
994
995 return status;
996}
997
1006void __kmpc_end_masked(ident_t *loc, kmp_int32 global_tid) {
1007 KC_TRACE(10, ("__kmpc_end_masked: called T#%d\n", global_tid));
1008 __kmp_assert_valid_gtid(global_tid);
1009 KMP_POP_PARTITIONED_TIMER();
1010
1011#if OMPT_SUPPORT && OMPT_OPTIONAL
1012 kmp_info_t *this_thr = __kmp_threads[global_tid];
1013 kmp_team_t *team = this_thr->th.th_team;
1014 if (ompt_enabled.ompt_callback_masked) {
1015 int tid = __kmp_tid_from_gtid(global_tid);
1016 ompt_callbacks.ompt_callback(ompt_callback_masked)(
1017 ompt_scope_end, &(team->t.ompt_team_info.parallel_data),
1018 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1019 OMPT_GET_RETURN_ADDRESS(0));
1020 }
1021#endif
1022
1023 if (__kmp_env_consistency_check) {
1024 __kmp_pop_sync(global_tid, ct_masked, loc);
1025 }
1026}
1027
1035void __kmpc_ordered(ident_t *loc, kmp_int32 gtid) {
1036 int cid = 0;
1037 kmp_info_t *th;
1038 KMP_DEBUG_ASSERT(__kmp_init_serial);
1039
1040 KC_TRACE(10, ("__kmpc_ordered: called T#%d\n", gtid));
1041 __kmp_assert_valid_gtid(gtid);
1042
1043 if (!TCR_4(__kmp_init_parallel))
1044 __kmp_parallel_initialize();
1045
1046 __kmp_resume_if_soft_paused();
1047
1048#if USE_ITT_BUILD
1049 __kmp_itt_ordered_prep(gtid);
1050// TODO: ordered_wait_id
1051#endif /* USE_ITT_BUILD */
1052
1053 th = __kmp_threads[gtid];
1054
1055#if OMPT_SUPPORT && OMPT_OPTIONAL
1056 kmp_team_t *team;
1057 ompt_wait_id_t lck;
1058 void *codeptr_ra;
1059 OMPT_STORE_RETURN_ADDRESS(gtid);
1060 if (ompt_enabled.enabled) {
1061 team = __kmp_team_from_gtid(gtid);
1062 lck = (ompt_wait_id_t)(uintptr_t)&team->t.t_ordered.dt.t_value;
1063 /* OMPT state update */
1064 th->th.ompt_thread_info.wait_id = lck;
1065 th->th.ompt_thread_info.state = ompt_state_wait_ordered;
1066
1067 /* OMPT event callback */
1068 codeptr_ra = OMPT_LOAD_RETURN_ADDRESS(gtid);
1069 if (ompt_enabled.ompt_callback_mutex_acquire) {
1070 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1071 ompt_mutex_ordered, omp_lock_hint_none, kmp_mutex_impl_spin, lck,
1072 codeptr_ra);
1073 }
1074 }
1075#endif
1076
1077 if (th->th.th_dispatch->th_deo_fcn != 0)
1078 (*th->th.th_dispatch->th_deo_fcn)(&gtid, &cid, loc);
1079 else
1080 __kmp_parallel_deo(&gtid, &cid, loc);
1081
1082#if OMPT_SUPPORT && OMPT_OPTIONAL
1083 if (ompt_enabled.enabled) {
1084 /* OMPT state update */
1085 th->th.ompt_thread_info.state = ompt_state_work_parallel;
1086 th->th.ompt_thread_info.wait_id = 0;
1087
1088 /* OMPT event callback */
1089 if (ompt_enabled.ompt_callback_mutex_acquired) {
1090 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
1091 ompt_mutex_ordered, (ompt_wait_id_t)(uintptr_t)lck, codeptr_ra);
1092 }
1093 }
1094#endif
1095
1096#if USE_ITT_BUILD
1097 __kmp_itt_ordered_start(gtid);
1098#endif /* USE_ITT_BUILD */
1099}
1100
1108void __kmpc_end_ordered(ident_t *loc, kmp_int32 gtid) {
1109 int cid = 0;
1110 kmp_info_t *th;
1111
1112 KC_TRACE(10, ("__kmpc_end_ordered: called T#%d\n", gtid));
1113 __kmp_assert_valid_gtid(gtid);
1114
1115#if USE_ITT_BUILD
1116 __kmp_itt_ordered_end(gtid);
1117// TODO: ordered_wait_id
1118#endif /* USE_ITT_BUILD */
1119
1120 th = __kmp_threads[gtid];
1121
1122 if (th->th.th_dispatch->th_dxo_fcn != 0)
1123 (*th->th.th_dispatch->th_dxo_fcn)(&gtid, &cid, loc);
1124 else
1125 __kmp_parallel_dxo(&gtid, &cid, loc);
1126
1127#if OMPT_SUPPORT && OMPT_OPTIONAL
1128 OMPT_STORE_RETURN_ADDRESS(gtid);
1129 if (ompt_enabled.ompt_callback_mutex_released) {
1130 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
1131 ompt_mutex_ordered,
1132 (ompt_wait_id_t)(uintptr_t)&__kmp_team_from_gtid(gtid)
1133 ->t.t_ordered.dt.t_value,
1134 OMPT_LOAD_RETURN_ADDRESS(gtid));
1135 }
1136#endif
1137}
1138
1139#if KMP_USE_DYNAMIC_LOCK
1140
1141static __forceinline void
1142__kmp_init_indirect_csptr(kmp_critical_name *crit, ident_t const *loc,
1143 kmp_int32 gtid, kmp_indirect_locktag_t tag) {
1144 // Pointer to the allocated indirect lock is written to crit, while indexing
1145 // is ignored.
1146 void *idx;
1147 kmp_indirect_lock_t **lck;
1148 lck = (kmp_indirect_lock_t **)crit;
1149 kmp_indirect_lock_t *ilk = __kmp_allocate_indirect_lock(&idx, gtid, tag);
1150 KMP_I_LOCK_FUNC(ilk, init)(ilk->lock);
1151 KMP_SET_I_LOCK_LOCATION(ilk, loc);
1152 KMP_SET_I_LOCK_FLAGS(ilk, kmp_lf_critical_section);
1153 KA_TRACE(20,
1154 ("__kmp_init_indirect_csptr: initialized indirect lock #%d\n", tag));
1155#if USE_ITT_BUILD
1156 __kmp_itt_critical_creating(ilk->lock, loc);
1157#endif
1158 int status = KMP_COMPARE_AND_STORE_PTR(lck, nullptr, ilk);
1159 if (status == 0) {
1160#if USE_ITT_BUILD
1161 __kmp_itt_critical_destroyed(ilk->lock);
1162#endif
1163 // We don't really need to destroy the unclaimed lock here since it will be
1164 // cleaned up at program exit.
1165 // KMP_D_LOCK_FUNC(&idx, destroy)((kmp_dyna_lock_t *)&idx);
1166 }
1167 KMP_DEBUG_ASSERT(*lck != NULL);
1168}
1169
1170// Fast-path acquire tas lock
1171#define KMP_ACQUIRE_TAS_LOCK(lock, gtid) \
1172 { \
1173 kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \
1174 kmp_int32 tas_free = KMP_LOCK_FREE(tas); \
1175 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas); \
1176 if (KMP_ATOMIC_LD_RLX(&l->lk.poll) != tas_free || \
1177 !__kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy)) { \
1178 kmp_uint32 spins; \
1179 KMP_FSYNC_PREPARE(l); \
1180 KMP_INIT_YIELD(spins); \
1181 kmp_backoff_t backoff = __kmp_spin_backoff_params; \
1182 do { \
1183 if (TCR_4(__kmp_nth) > \
1184 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) { \
1185 KMP_YIELD(TRUE); \
1186 } else { \
1187 KMP_YIELD_SPIN(spins); \
1188 } \
1189 __kmp_spin_backoff(&backoff); \
1190 } while ( \
1191 KMP_ATOMIC_LD_RLX(&l->lk.poll) != tas_free || \
1192 !__kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy)); \
1193 } \
1194 KMP_FSYNC_ACQUIRED(l); \
1195 }
1196
1197// Fast-path test tas lock
1198#define KMP_TEST_TAS_LOCK(lock, gtid, rc) \
1199 { \
1200 kmp_tas_lock_t *l = (kmp_tas_lock_t *)lock; \
1201 kmp_int32 tas_free = KMP_LOCK_FREE(tas); \
1202 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas); \
1203 rc = KMP_ATOMIC_LD_RLX(&l->lk.poll) == tas_free && \
1204 __kmp_atomic_compare_store_acq(&l->lk.poll, tas_free, tas_busy); \
1205 }
1206
1207// Fast-path release tas lock
1208#define KMP_RELEASE_TAS_LOCK(lock, gtid) \
1209 { KMP_ATOMIC_ST_REL(&((kmp_tas_lock_t *)lock)->lk.poll, KMP_LOCK_FREE(tas)); }
1210
1211#if KMP_USE_FUTEX
1212
1213#include <sys/syscall.h>
1214#include <unistd.h>
1215#ifndef FUTEX_WAIT
1216#define FUTEX_WAIT 0
1217#endif
1218#ifndef FUTEX_WAKE
1219#define FUTEX_WAKE 1
1220#endif
1221
1222// Fast-path acquire futex lock
1223#define KMP_ACQUIRE_FUTEX_LOCK(lock, gtid) \
1224 { \
1225 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
1226 kmp_int32 gtid_code = (gtid + 1) << 1; \
1227 KMP_MB(); \
1228 KMP_FSYNC_PREPARE(ftx); \
1229 kmp_int32 poll_val; \
1230 while ((poll_val = KMP_COMPARE_AND_STORE_RET32( \
1231 &(ftx->lk.poll), KMP_LOCK_FREE(futex), \
1232 KMP_LOCK_BUSY(gtid_code, futex))) != KMP_LOCK_FREE(futex)) { \
1233 kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1; \
1234 if (!cond) { \
1235 if (!KMP_COMPARE_AND_STORE_RET32(&(ftx->lk.poll), poll_val, \
1236 poll_val | \
1237 KMP_LOCK_BUSY(1, futex))) { \
1238 continue; \
1239 } \
1240 poll_val |= KMP_LOCK_BUSY(1, futex); \
1241 } \
1242 kmp_int32 rc; \
1243 if ((rc = syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAIT, poll_val, \
1244 NULL, NULL, 0)) != 0) { \
1245 continue; \
1246 } \
1247 gtid_code |= 1; \
1248 } \
1249 KMP_FSYNC_ACQUIRED(ftx); \
1250 }
1251
1252// Fast-path test futex lock
1253#define KMP_TEST_FUTEX_LOCK(lock, gtid, rc) \
1254 { \
1255 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
1256 if (KMP_COMPARE_AND_STORE_ACQ32(&(ftx->lk.poll), KMP_LOCK_FREE(futex), \
1257 KMP_LOCK_BUSY(gtid + 1 << 1, futex))) { \
1258 KMP_FSYNC_ACQUIRED(ftx); \
1259 rc = TRUE; \
1260 } else { \
1261 rc = FALSE; \
1262 } \
1263 }
1264
1265// Fast-path release futex lock
1266#define KMP_RELEASE_FUTEX_LOCK(lock, gtid) \
1267 { \
1268 kmp_futex_lock_t *ftx = (kmp_futex_lock_t *)lock; \
1269 KMP_MB(); \
1270 KMP_FSYNC_RELEASING(ftx); \
1271 kmp_int32 poll_val = \
1272 KMP_XCHG_FIXED32(&(ftx->lk.poll), KMP_LOCK_FREE(futex)); \
1273 if (KMP_LOCK_STRIP(poll_val) & 1) { \
1274 syscall(__NR_futex, &(ftx->lk.poll), FUTEX_WAKE, \
1275 KMP_LOCK_BUSY(1, futex), NULL, NULL, 0); \
1276 } \
1277 KMP_MB(); \
1278 KMP_YIELD_OVERSUB(); \
1279 }
1280
1281#endif // KMP_USE_FUTEX
1282
1283#else // KMP_USE_DYNAMIC_LOCK
1284
1285static kmp_user_lock_p __kmp_get_critical_section_ptr(kmp_critical_name *crit,
1286 ident_t const *loc,
1287 kmp_int32 gtid) {
1288 kmp_user_lock_p *lck_pp = (kmp_user_lock_p *)crit;
1289
1290 // Because of the double-check, the following load doesn't need to be volatile
1291 kmp_user_lock_p lck = (kmp_user_lock_p)TCR_PTR(*lck_pp);
1292
1293 if (lck == NULL) {
1294 void *idx;
1295
1296 // Allocate & initialize the lock.
1297 // Remember alloc'ed locks in table in order to free them in __kmp_cleanup()
1298 lck = __kmp_user_lock_allocate(&idx, gtid, kmp_lf_critical_section);
1299 __kmp_init_user_lock_with_checks(lck);
1300 __kmp_set_user_lock_location(lck, loc);
1301#if USE_ITT_BUILD
1302 __kmp_itt_critical_creating(lck);
1303// __kmp_itt_critical_creating() should be called *before* the first usage
1304// of underlying lock. It is the only place where we can guarantee it. There
1305// are chances the lock will destroyed with no usage, but it is not a
1306// problem, because this is not real event seen by user but rather setting
1307// name for object (lock). See more details in kmp_itt.h.
1308#endif /* USE_ITT_BUILD */
1309
1310 // Use a cmpxchg instruction to slam the start of the critical section with
1311 // the lock pointer. If another thread beat us to it, deallocate the lock,
1312 // and use the lock that the other thread allocated.
1313 int status = KMP_COMPARE_AND_STORE_PTR(lck_pp, 0, lck);
1314
1315 if (status == 0) {
1316// Deallocate the lock and reload the value.
1317#if USE_ITT_BUILD
1318 __kmp_itt_critical_destroyed(lck);
1319// Let ITT know the lock is destroyed and the same memory location may be reused
1320// for another purpose.
1321#endif /* USE_ITT_BUILD */
1322 __kmp_destroy_user_lock_with_checks(lck);
1323 __kmp_user_lock_free(&idx, gtid, lck);
1324 lck = (kmp_user_lock_p)TCR_PTR(*lck_pp);
1325 KMP_DEBUG_ASSERT(lck != NULL);
1326 }
1327 }
1328 return lck;
1329}
1330
1331#endif // KMP_USE_DYNAMIC_LOCK
1332
1343void __kmpc_critical(ident_t *loc, kmp_int32 global_tid,
1344 kmp_critical_name *crit) {
1345#if KMP_USE_DYNAMIC_LOCK
1346#if OMPT_SUPPORT && OMPT_OPTIONAL
1347 OMPT_STORE_RETURN_ADDRESS(global_tid);
1348#endif // OMPT_SUPPORT
1349 __kmpc_critical_with_hint(loc, global_tid, crit, omp_lock_hint_none);
1350#else
1351 KMP_COUNT_BLOCK(OMP_CRITICAL);
1352#if OMPT_SUPPORT && OMPT_OPTIONAL
1353 ompt_state_t prev_state = ompt_state_undefined;
1354 ompt_thread_info_t ti;
1355#endif
1356 kmp_user_lock_p lck;
1357
1358 KC_TRACE(10, ("__kmpc_critical: called T#%d\n", global_tid));
1359 __kmp_assert_valid_gtid(global_tid);
1360
1361 // TODO: add THR_OVHD_STATE
1362
1363 KMP_PUSH_PARTITIONED_TIMER(OMP_critical_wait);
1364 KMP_CHECK_USER_LOCK_INIT();
1365
1366 if ((__kmp_user_lock_kind == lk_tas) &&
1367 (sizeof(lck->tas.lk.poll) <= OMP_CRITICAL_SIZE)) {
1368 lck = (kmp_user_lock_p)crit;
1369 }
1370#if KMP_USE_FUTEX
1371 else if ((__kmp_user_lock_kind == lk_futex) &&
1372 (sizeof(lck->futex.lk.poll) <= OMP_CRITICAL_SIZE)) {
1373 lck = (kmp_user_lock_p)crit;
1374 }
1375#endif
1376 else { // ticket, queuing or drdpa
1377 lck = __kmp_get_critical_section_ptr(crit, loc, global_tid);
1378 }
1379
1380 if (__kmp_env_consistency_check)
1381 __kmp_push_sync(global_tid, ct_critical, loc, lck);
1382
1383 // since the critical directive binds to all threads, not just the current
1384 // team we have to check this even if we are in a serialized team.
1385 // also, even if we are the uber thread, we still have to conduct the lock,
1386 // as we have to contend with sibling threads.
1387
1388#if USE_ITT_BUILD
1389 __kmp_itt_critical_acquiring(lck);
1390#endif /* USE_ITT_BUILD */
1391#if OMPT_SUPPORT && OMPT_OPTIONAL
1392 OMPT_STORE_RETURN_ADDRESS(gtid);
1393 void *codeptr_ra = NULL;
1394 if (ompt_enabled.enabled) {
1395 ti = __kmp_threads[global_tid]->th.ompt_thread_info;
1396 /* OMPT state update */
1397 prev_state = ti.state;
1398 ti.wait_id = (ompt_wait_id_t)(uintptr_t)lck;
1399 ti.state = ompt_state_wait_critical;
1400
1401 /* OMPT event callback */
1402 codeptr_ra = OMPT_LOAD_RETURN_ADDRESS(gtid);
1403 if (ompt_enabled.ompt_callback_mutex_acquire) {
1404 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1405 ompt_mutex_critical, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
1406 (ompt_wait_id_t)(uintptr_t)lck, codeptr_ra);
1407 }
1408 }
1409#endif
1410 // Value of 'crit' should be good for using as a critical_id of the critical
1411 // section directive.
1412 __kmp_acquire_user_lock_with_checks(lck, global_tid);
1413
1414#if USE_ITT_BUILD
1415 __kmp_itt_critical_acquired(lck);
1416#endif /* USE_ITT_BUILD */
1417#if OMPT_SUPPORT && OMPT_OPTIONAL
1418 if (ompt_enabled.enabled) {
1419 /* OMPT state update */
1420 ti.state = prev_state;
1421 ti.wait_id = 0;
1422
1423 /* OMPT event callback */
1424 if (ompt_enabled.ompt_callback_mutex_acquired) {
1425 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
1426 ompt_mutex_critical, (ompt_wait_id_t)(uintptr_t)lck, codeptr_ra);
1427 }
1428 }
1429#endif
1430 KMP_POP_PARTITIONED_TIMER();
1431
1432 KMP_PUSH_PARTITIONED_TIMER(OMP_critical);
1433 KA_TRACE(15, ("__kmpc_critical: done T#%d\n", global_tid));
1434#endif // KMP_USE_DYNAMIC_LOCK
1435}
1436
1437#if KMP_USE_DYNAMIC_LOCK
1438
1439// Converts the given hint to an internal lock implementation
1440static __forceinline kmp_dyna_lockseq_t __kmp_map_hint_to_lock(uintptr_t hint) {
1441#if KMP_USE_TSX
1442#define KMP_TSX_LOCK(seq) lockseq_##seq
1443#else
1444#define KMP_TSX_LOCK(seq) __kmp_user_lock_seq
1445#endif
1446
1447#if KMP_ARCH_X86 || KMP_ARCH_X86_64
1448#define KMP_CPUINFO_RTM (__kmp_cpuinfo.flags.rtm)
1449#else
1450#define KMP_CPUINFO_RTM 0
1451#endif
1452
1453 // Hints that do not require further logic
1454 if (hint & kmp_lock_hint_hle)
1455 return KMP_TSX_LOCK(hle);
1456 if (hint & kmp_lock_hint_rtm)
1457 return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(rtm_queuing) : __kmp_user_lock_seq;
1458 if (hint & kmp_lock_hint_adaptive)
1459 return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(adaptive) : __kmp_user_lock_seq;
1460
1461 // Rule out conflicting hints first by returning the default lock
1462 if ((hint & omp_lock_hint_contended) && (hint & omp_lock_hint_uncontended))
1463 return __kmp_user_lock_seq;
1464 if ((hint & omp_lock_hint_speculative) &&
1465 (hint & omp_lock_hint_nonspeculative))
1466 return __kmp_user_lock_seq;
1467
1468 // Do not even consider speculation when it appears to be contended
1469 if (hint & omp_lock_hint_contended)
1470 return lockseq_queuing;
1471
1472 // Uncontended lock without speculation
1473 if ((hint & omp_lock_hint_uncontended) && !(hint & omp_lock_hint_speculative))
1474 return lockseq_tas;
1475
1476 // Use RTM lock for speculation
1477 if (hint & omp_lock_hint_speculative)
1478 return KMP_CPUINFO_RTM ? KMP_TSX_LOCK(rtm_spin) : __kmp_user_lock_seq;
1479
1480 return __kmp_user_lock_seq;
1481}
1482
1483#if OMPT_SUPPORT && OMPT_OPTIONAL
1484#if KMP_USE_DYNAMIC_LOCK
1485static kmp_mutex_impl_t
1486__ompt_get_mutex_impl_type(void *user_lock, kmp_indirect_lock_t *ilock = 0) {
1487 if (user_lock) {
1488 switch (KMP_EXTRACT_D_TAG(user_lock)) {
1489 case 0:
1490 break;
1491#if KMP_USE_FUTEX
1492 case locktag_futex:
1493 return kmp_mutex_impl_queuing;
1494#endif
1495 case locktag_tas:
1496 return kmp_mutex_impl_spin;
1497#if KMP_USE_TSX
1498 case locktag_hle:
1499 case locktag_rtm_spin:
1500 return kmp_mutex_impl_speculative;
1501#endif
1502 default:
1503 return kmp_mutex_impl_none;
1504 }
1505 ilock = KMP_LOOKUP_I_LOCK(user_lock);
1506 }
1507 KMP_ASSERT(ilock);
1508 switch (ilock->type) {
1509#if KMP_USE_TSX
1510 case locktag_adaptive:
1511 case locktag_rtm_queuing:
1512 return kmp_mutex_impl_speculative;
1513#endif
1514 case locktag_nested_tas:
1515 return kmp_mutex_impl_spin;
1516#if KMP_USE_FUTEX
1517 case locktag_nested_futex:
1518#endif
1519 case locktag_ticket:
1520 case locktag_queuing:
1521 case locktag_drdpa:
1522 case locktag_nested_ticket:
1523 case locktag_nested_queuing:
1524 case locktag_nested_drdpa:
1525 return kmp_mutex_impl_queuing;
1526 default:
1527 return kmp_mutex_impl_none;
1528 }
1529}
1530#else
1531// For locks without dynamic binding
1532static kmp_mutex_impl_t __ompt_get_mutex_impl_type() {
1533 switch (__kmp_user_lock_kind) {
1534 case lk_tas:
1535 return kmp_mutex_impl_spin;
1536#if KMP_USE_FUTEX
1537 case lk_futex:
1538#endif
1539 case lk_ticket:
1540 case lk_queuing:
1541 case lk_drdpa:
1542 return kmp_mutex_impl_queuing;
1543#if KMP_USE_TSX
1544 case lk_hle:
1545 case lk_rtm_queuing:
1546 case lk_rtm_spin:
1547 case lk_adaptive:
1548 return kmp_mutex_impl_speculative;
1549#endif
1550 default:
1551 return kmp_mutex_impl_none;
1552 }
1553}
1554#endif // KMP_USE_DYNAMIC_LOCK
1555#endif // OMPT_SUPPORT && OMPT_OPTIONAL
1556
1570void __kmpc_critical_with_hint(ident_t *loc, kmp_int32 global_tid,
1571 kmp_critical_name *crit, uint32_t hint) {
1572 KMP_COUNT_BLOCK(OMP_CRITICAL);
1573 kmp_user_lock_p lck;
1574#if OMPT_SUPPORT && OMPT_OPTIONAL
1575 ompt_state_t prev_state = ompt_state_undefined;
1576 ompt_thread_info_t ti;
1577 // This is the case, if called from __kmpc_critical:
1578 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid);
1579 if (!codeptr)
1580 codeptr = OMPT_GET_RETURN_ADDRESS(0);
1581#endif
1582
1583 KC_TRACE(10, ("__kmpc_critical: called T#%d\n", global_tid));
1584 __kmp_assert_valid_gtid(global_tid);
1585
1586 kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit;
1587 // Check if it is initialized.
1588 KMP_PUSH_PARTITIONED_TIMER(OMP_critical_wait);
1589 kmp_dyna_lockseq_t lockseq = __kmp_map_hint_to_lock(hint);
1590 if (*lk == 0) {
1591 if (KMP_IS_D_LOCK(lockseq)) {
1592 (void)KMP_COMPARE_AND_STORE_ACQ32(
1593 (volatile kmp_int32 *)&((kmp_base_tas_lock_t *)crit)->poll, 0,
1594 KMP_GET_D_TAG(lockseq));
1595 } else {
1596 __kmp_init_indirect_csptr(crit, loc, global_tid, KMP_GET_I_TAG(lockseq));
1597 }
1598 }
1599 // Branch for accessing the actual lock object and set operation. This
1600 // branching is inevitable since this lock initialization does not follow the
1601 // normal dispatch path (lock table is not used).
1602 if (KMP_EXTRACT_D_TAG(lk) != 0) {
1603 lck = (kmp_user_lock_p)lk;
1604 if (__kmp_env_consistency_check) {
1605 __kmp_push_sync(global_tid, ct_critical, loc, lck,
1606 __kmp_map_hint_to_lock(hint));
1607 }
1608#if USE_ITT_BUILD
1609 __kmp_itt_critical_acquiring(lck);
1610#endif
1611#if OMPT_SUPPORT && OMPT_OPTIONAL
1612 if (ompt_enabled.enabled) {
1613 ti = __kmp_threads[global_tid]->th.ompt_thread_info;
1614 /* OMPT state update */
1615 prev_state = ti.state;
1616 ti.wait_id = (ompt_wait_id_t)(uintptr_t)lck;
1617 ti.state = ompt_state_wait_critical;
1618
1619 /* OMPT event callback */
1620 if (ompt_enabled.ompt_callback_mutex_acquire) {
1621 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1622 ompt_mutex_critical, (unsigned int)hint,
1623 __ompt_get_mutex_impl_type(crit), (ompt_wait_id_t)(uintptr_t)lck,
1624 codeptr);
1625 }
1626 }
1627#endif
1628#if KMP_USE_INLINED_TAS
1629 if (lockseq == lockseq_tas && !__kmp_env_consistency_check) {
1630 KMP_ACQUIRE_TAS_LOCK(lck, global_tid);
1631 } else
1632#elif KMP_USE_INLINED_FUTEX
1633 if (lockseq == lockseq_futex && !__kmp_env_consistency_check) {
1634 KMP_ACQUIRE_FUTEX_LOCK(lck, global_tid);
1635 } else
1636#endif
1637 {
1638 KMP_D_LOCK_FUNC(lk, set)(lk, global_tid);
1639 }
1640 } else {
1641 kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk);
1642 lck = ilk->lock;
1643 if (__kmp_env_consistency_check) {
1644 __kmp_push_sync(global_tid, ct_critical, loc, lck,
1645 __kmp_map_hint_to_lock(hint));
1646 }
1647#if USE_ITT_BUILD
1648 __kmp_itt_critical_acquiring(lck);
1649#endif
1650#if OMPT_SUPPORT && OMPT_OPTIONAL
1651 if (ompt_enabled.enabled) {
1652 ti = __kmp_threads[global_tid]->th.ompt_thread_info;
1653 /* OMPT state update */
1654 prev_state = ti.state;
1655 ti.wait_id = (ompt_wait_id_t)(uintptr_t)lck;
1656 ti.state = ompt_state_wait_critical;
1657
1658 /* OMPT event callback */
1659 if (ompt_enabled.ompt_callback_mutex_acquire) {
1660 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
1661 ompt_mutex_critical, (unsigned int)hint,
1662 __ompt_get_mutex_impl_type(0, ilk), (ompt_wait_id_t)(uintptr_t)lck,
1663 codeptr);
1664 }
1665 }
1666#endif
1667 KMP_I_LOCK_FUNC(ilk, set)(lck, global_tid);
1668 }
1669 KMP_POP_PARTITIONED_TIMER();
1670
1671#if USE_ITT_BUILD
1672 __kmp_itt_critical_acquired(lck);
1673#endif /* USE_ITT_BUILD */
1674#if OMPT_SUPPORT && OMPT_OPTIONAL
1675 if (ompt_enabled.enabled) {
1676 /* OMPT state update */
1677 ti.state = prev_state;
1678 ti.wait_id = 0;
1679
1680 /* OMPT event callback */
1681 if (ompt_enabled.ompt_callback_mutex_acquired) {
1682 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
1683 ompt_mutex_critical, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
1684 }
1685 }
1686#endif
1687
1688 KMP_PUSH_PARTITIONED_TIMER(OMP_critical);
1689 KA_TRACE(15, ("__kmpc_critical: done T#%d\n", global_tid));
1690} // __kmpc_critical_with_hint
1691
1692#endif // KMP_USE_DYNAMIC_LOCK
1693
1703void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid,
1704 kmp_critical_name *crit) {
1705 kmp_user_lock_p lck;
1706
1707 KC_TRACE(10, ("__kmpc_end_critical: called T#%d\n", global_tid));
1708
1709#if KMP_USE_DYNAMIC_LOCK
1710 int locktag = KMP_EXTRACT_D_TAG(crit);
1711 if (locktag) {
1712 lck = (kmp_user_lock_p)crit;
1713 KMP_ASSERT(lck != NULL);
1714 if (__kmp_env_consistency_check) {
1715 __kmp_pop_sync(global_tid, ct_critical, loc);
1716 }
1717#if USE_ITT_BUILD
1718 __kmp_itt_critical_releasing(lck);
1719#endif
1720#if KMP_USE_INLINED_TAS
1721 if (locktag == locktag_tas && !__kmp_env_consistency_check) {
1722 KMP_RELEASE_TAS_LOCK(lck, global_tid);
1723 } else
1724#elif KMP_USE_INLINED_FUTEX
1725 if (locktag == locktag_futex && !__kmp_env_consistency_check) {
1726 KMP_RELEASE_FUTEX_LOCK(lck, global_tid);
1727 } else
1728#endif
1729 {
1730 KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
1731 }
1732 } else {
1733 kmp_indirect_lock_t *ilk =
1734 (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
1735 KMP_ASSERT(ilk != NULL);
1736 lck = ilk->lock;
1737 if (__kmp_env_consistency_check) {
1738 __kmp_pop_sync(global_tid, ct_critical, loc);
1739 }
1740#if USE_ITT_BUILD
1741 __kmp_itt_critical_releasing(lck);
1742#endif
1743 KMP_I_LOCK_FUNC(ilk, unset)(lck, global_tid);
1744 }
1745
1746#else // KMP_USE_DYNAMIC_LOCK
1747
1748 if ((__kmp_user_lock_kind == lk_tas) &&
1749 (sizeof(lck->tas.lk.poll) <= OMP_CRITICAL_SIZE)) {
1750 lck = (kmp_user_lock_p)crit;
1751 }
1752#if KMP_USE_FUTEX
1753 else if ((__kmp_user_lock_kind == lk_futex) &&
1754 (sizeof(lck->futex.lk.poll) <= OMP_CRITICAL_SIZE)) {
1755 lck = (kmp_user_lock_p)crit;
1756 }
1757#endif
1758 else { // ticket, queuing or drdpa
1759 lck = (kmp_user_lock_p)TCR_PTR(*((kmp_user_lock_p *)crit));
1760 }
1761
1762 KMP_ASSERT(lck != NULL);
1763
1764 if (__kmp_env_consistency_check)
1765 __kmp_pop_sync(global_tid, ct_critical, loc);
1766
1767#if USE_ITT_BUILD
1768 __kmp_itt_critical_releasing(lck);
1769#endif /* USE_ITT_BUILD */
1770 // Value of 'crit' should be good for using as a critical_id of the critical
1771 // section directive.
1772 __kmp_release_user_lock_with_checks(lck, global_tid);
1773
1774#endif // KMP_USE_DYNAMIC_LOCK
1775
1776#if OMPT_SUPPORT && OMPT_OPTIONAL
1777 /* OMPT release event triggers after lock is released; place here to trigger
1778 * for all #if branches */
1779 OMPT_STORE_RETURN_ADDRESS(global_tid);
1780 if (ompt_enabled.ompt_callback_mutex_released) {
1781 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
1782 ompt_mutex_critical, (ompt_wait_id_t)(uintptr_t)lck,
1783 OMPT_LOAD_RETURN_ADDRESS(global_tid));
1784 }
1785#endif
1786
1787 KMP_POP_PARTITIONED_TIMER();
1788 KA_TRACE(15, ("__kmpc_end_critical: done T#%d\n", global_tid));
1789}
1790
1800kmp_int32 __kmpc_barrier_master(ident_t *loc, kmp_int32 global_tid) {
1801 int status;
1802 KC_TRACE(10, ("__kmpc_barrier_master: called T#%d\n", global_tid));
1803 __kmp_assert_valid_gtid(global_tid);
1804
1805 if (!TCR_4(__kmp_init_parallel))
1806 __kmp_parallel_initialize();
1807
1808 __kmp_resume_if_soft_paused();
1809
1810 if (__kmp_env_consistency_check)
1811 __kmp_check_barrier(global_tid, ct_barrier, loc);
1812
1813#if OMPT_SUPPORT
1814 ompt_frame_t *ompt_frame;
1815 if (ompt_enabled.enabled) {
1816 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
1817 if (ompt_frame->enter_frame.ptr == NULL)
1818 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1819 }
1820 OMPT_STORE_RETURN_ADDRESS(global_tid);
1821#endif
1822#if USE_ITT_NOTIFY
1823 __kmp_threads[global_tid]->th.th_ident = loc;
1824#endif
1825 status = __kmp_barrier(bs_plain_barrier, global_tid, TRUE, 0, NULL, NULL);
1826#if OMPT_SUPPORT && OMPT_OPTIONAL
1827 if (ompt_enabled.enabled) {
1828 ompt_frame->enter_frame = ompt_data_none;
1829 }
1830#endif
1831
1832 return (status != 0) ? 0 : 1;
1833}
1834
1844void __kmpc_end_barrier_master(ident_t *loc, kmp_int32 global_tid) {
1845 KC_TRACE(10, ("__kmpc_end_barrier_master: called T#%d\n", global_tid));
1846 __kmp_assert_valid_gtid(global_tid);
1847 __kmp_end_split_barrier(bs_plain_barrier, global_tid);
1848}
1849
1860kmp_int32 __kmpc_barrier_master_nowait(ident_t *loc, kmp_int32 global_tid) {
1861 kmp_int32 ret;
1862 KC_TRACE(10, ("__kmpc_barrier_master_nowait: called T#%d\n", global_tid));
1863 __kmp_assert_valid_gtid(global_tid);
1864
1865 if (!TCR_4(__kmp_init_parallel))
1866 __kmp_parallel_initialize();
1867
1868 __kmp_resume_if_soft_paused();
1869
1870 if (__kmp_env_consistency_check) {
1871 if (loc == 0) {
1872 KMP_WARNING(ConstructIdentInvalid); // ??? What does it mean for the user?
1873 }
1874 __kmp_check_barrier(global_tid, ct_barrier, loc);
1875 }
1876
1877#if OMPT_SUPPORT
1878 ompt_frame_t *ompt_frame;
1879 if (ompt_enabled.enabled) {
1880 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
1881 if (ompt_frame->enter_frame.ptr == NULL)
1882 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1883 }
1884 OMPT_STORE_RETURN_ADDRESS(global_tid);
1885#endif
1886#if USE_ITT_NOTIFY
1887 __kmp_threads[global_tid]->th.th_ident = loc;
1888#endif
1889 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
1890#if OMPT_SUPPORT && OMPT_OPTIONAL
1891 if (ompt_enabled.enabled) {
1892 ompt_frame->enter_frame = ompt_data_none;
1893 }
1894#endif
1895
1896 ret = __kmpc_master(loc, global_tid);
1897
1898 if (__kmp_env_consistency_check) {
1899 /* there's no __kmpc_end_master called; so the (stats) */
1900 /* actions of __kmpc_end_master are done here */
1901 if (ret) {
1902 /* only one thread should do the pop since only */
1903 /* one did the push (see __kmpc_master()) */
1904 __kmp_pop_sync(global_tid, ct_master, loc);
1905 }
1906 }
1907
1908 return (ret);
1909}
1910
1911/* The BARRIER for a SINGLE process section is always explicit */
1923kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid) {
1924 __kmp_assert_valid_gtid(global_tid);
1925 kmp_int32 rc = __kmp_enter_single(global_tid, loc, TRUE);
1926
1927 if (rc) {
1928 // We are going to execute the single statement, so we should count it.
1929 KMP_COUNT_BLOCK(OMP_SINGLE);
1930 KMP_PUSH_PARTITIONED_TIMER(OMP_single);
1931 }
1932
1933#if OMPT_SUPPORT && OMPT_OPTIONAL
1934 kmp_info_t *this_thr = __kmp_threads[global_tid];
1935 kmp_team_t *team = this_thr->th.th_team;
1936 int tid = __kmp_tid_from_gtid(global_tid);
1937
1938 if (ompt_enabled.enabled) {
1939 if (rc) {
1940 if (ompt_enabled.ompt_callback_work) {
1941 ompt_callbacks.ompt_callback(ompt_callback_work)(
1942 ompt_work_single_executor, ompt_scope_begin,
1943 &(team->t.ompt_team_info.parallel_data),
1944 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1945 1, OMPT_GET_RETURN_ADDRESS(0));
1946 }
1947 } else {
1948 if (ompt_enabled.ompt_callback_work) {
1949 ompt_callbacks.ompt_callback(ompt_callback_work)(
1950 ompt_work_single_other, ompt_scope_begin,
1951 &(team->t.ompt_team_info.parallel_data),
1952 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1953 1, OMPT_GET_RETURN_ADDRESS(0));
1954 ompt_callbacks.ompt_callback(ompt_callback_work)(
1955 ompt_work_single_other, ompt_scope_end,
1956 &(team->t.ompt_team_info.parallel_data),
1957 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data),
1958 1, OMPT_GET_RETURN_ADDRESS(0));
1959 }
1960 }
1961 }
1962#endif
1963
1964 return rc;
1965}
1966
1976void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid) {
1977 __kmp_assert_valid_gtid(global_tid);
1978 __kmp_exit_single(global_tid);
1979 KMP_POP_PARTITIONED_TIMER();
1980
1981#if OMPT_SUPPORT && OMPT_OPTIONAL
1982 kmp_info_t *this_thr = __kmp_threads[global_tid];
1983 kmp_team_t *team = this_thr->th.th_team;
1984 int tid = __kmp_tid_from_gtid(global_tid);
1985
1986 if (ompt_enabled.ompt_callback_work) {
1987 ompt_callbacks.ompt_callback(ompt_callback_work)(
1988 ompt_work_single_executor, ompt_scope_end,
1989 &(team->t.ompt_team_info.parallel_data),
1990 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 1,
1991 OMPT_GET_RETURN_ADDRESS(0));
1992 }
1993#endif
1994}
1995
2003void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid) {
2004 KMP_POP_PARTITIONED_TIMER();
2005 KE_TRACE(10, ("__kmpc_for_static_fini called T#%d\n", global_tid));
2006
2007#if OMPT_SUPPORT && OMPT_OPTIONAL
2008 if (ompt_enabled.ompt_callback_work) {
2009 ompt_work_t ompt_work_type = ompt_work_loop_static;
2010 ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
2011 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2012 // Determine workshare type
2013 if (loc != NULL) {
2014 if ((loc->flags & KMP_IDENT_WORK_LOOP) != 0) {
2015 ompt_work_type = ompt_work_loop_static;
2016 } else if ((loc->flags & KMP_IDENT_WORK_SECTIONS) != 0) {
2017 ompt_work_type = ompt_work_sections;
2018 } else if ((loc->flags & KMP_IDENT_WORK_DISTRIBUTE) != 0) {
2019 ompt_work_type = ompt_work_distribute;
2020 } else {
2021 // use default set above.
2022 // a warning about this case is provided in __kmpc_for_static_init
2023 }
2024 KMP_DEBUG_ASSERT(ompt_work_type);
2025 }
2026 ompt_callbacks.ompt_callback(ompt_callback_work)(
2027 ompt_work_type, ompt_scope_end, &(team_info->parallel_data),
2028 &(task_info->task_data), 0, OMPT_GET_RETURN_ADDRESS(0));
2029 }
2030#endif
2031 if (__kmp_env_consistency_check)
2032 __kmp_pop_workshare(global_tid, ct_pdo, loc);
2033}
2034
2035// User routines which take C-style arguments (call by value)
2036// different from the Fortran equivalent routines
2037
2038void ompc_set_num_threads(int arg) {
2039 // !!!!! TODO: check the per-task binding
2040 __kmp_set_num_threads(arg, __kmp_entry_gtid());
2041}
2042
2043void kmpc_set_num_threads_8(kmp_int64 arg) {
2044 if (arg > (kmp_int64)INT_MAX)
2045 arg = INT_MAX;
2046 else if (arg < (kmp_int64)INT_MIN)
2047 arg = INT_MIN;
2048 __kmp_set_num_threads((int)arg, __kmp_entry_gtid());
2049}
2050
2051void ompc_set_dynamic(int flag) {
2052 kmp_info_t *thread;
2053
2054 /* For the thread-private implementation of the internal controls */
2055 thread = __kmp_entry_thread();
2056
2057 __kmp_save_internal_controls(thread);
2058
2059 set__dynamic(thread, flag ? true : false);
2060}
2061
2062void ompc_set_nested(int flag) {
2063 kmp_info_t *thread;
2064
2065 /* For the thread-private internal controls implementation */
2066 thread = __kmp_entry_thread();
2067
2068 __kmp_save_internal_controls(thread);
2069
2070 set__max_active_levels(thread, flag ? __kmp_dflt_max_active_levels : 1);
2071}
2072
2073void ompc_set_max_active_levels(int max_active_levels) {
2074 /* TO DO */
2075 /* we want per-task implementation of this internal control */
2076
2077 /* For the per-thread internal controls implementation */
2078 __kmp_set_max_active_levels(__kmp_entry_gtid(), max_active_levels);
2079}
2080
2081void ompc_set_schedule(omp_sched_t kind, int modifier) {
2082 // !!!!! TODO: check the per-task binding
2083 __kmp_set_schedule(__kmp_entry_gtid(), (kmp_sched_t)kind, modifier);
2084}
2085
2086int ompc_get_ancestor_thread_num(int level) {
2087 return __kmp_get_ancestor_thread_num(__kmp_entry_gtid(), level);
2088}
2089
2090int ompc_get_team_size(int level) {
2091 return __kmp_get_team_size(__kmp_entry_gtid(), level);
2092}
2093
2094/* OpenMP 5.0 Affinity Format API */
2095void KMP_EXPAND_NAME(ompc_set_affinity_format)(char const *format) {
2096 if (!__kmp_init_serial) {
2097 __kmp_serial_initialize();
2098 }
2099 __kmp_strncpy_truncate(__kmp_affinity_format, KMP_AFFINITY_FORMAT_SIZE,
2100 format, KMP_STRLEN(format) + 1);
2101}
2102
2103size_t KMP_EXPAND_NAME(ompc_get_affinity_format)(char *buffer, size_t size) {
2104 size_t format_size;
2105 if (!__kmp_init_serial) {
2106 __kmp_serial_initialize();
2107 }
2108 format_size = KMP_STRLEN(__kmp_affinity_format);
2109 if (buffer && size) {
2110 __kmp_strncpy_truncate(buffer, size, __kmp_affinity_format,
2111 format_size + 1);
2112 }
2113 return format_size;
2114}
2115
2116void KMP_EXPAND_NAME(ompc_display_affinity)(char const *format) {
2117 int gtid;
2118 if (!TCR_4(__kmp_init_middle)) {
2119 __kmp_middle_initialize();
2120 }
2121 __kmp_assign_root_init_mask();
2122 gtid = __kmp_get_gtid();
2123#if KMP_AFFINITY_SUPPORTED
2124 if (__kmp_threads[gtid]->th.th_team->t.t_level == 0 &&
2125 __kmp_affinity.flags.reset) {
2126 __kmp_reset_root_init_mask(gtid);
2127 }
2128#endif
2129 __kmp_aux_display_affinity(gtid, format);
2130}
2131
2132size_t KMP_EXPAND_NAME(ompc_capture_affinity)(char *buffer, size_t buf_size,
2133 char const *format) {
2134 int gtid;
2135 size_t num_required;
2136 kmp_str_buf_t capture_buf;
2137 if (!TCR_4(__kmp_init_middle)) {
2138 __kmp_middle_initialize();
2139 }
2140 __kmp_assign_root_init_mask();
2141 gtid = __kmp_get_gtid();
2142#if KMP_AFFINITY_SUPPORTED
2143 if (__kmp_threads[gtid]->th.th_team->t.t_level == 0 &&
2144 __kmp_affinity.flags.reset) {
2145 __kmp_reset_root_init_mask(gtid);
2146 }
2147#endif
2148 __kmp_str_buf_init(&capture_buf);
2149 num_required = __kmp_aux_capture_affinity(gtid, format, &capture_buf);
2150 if (buffer && buf_size) {
2151 __kmp_strncpy_truncate(buffer, buf_size, capture_buf.str,
2152 capture_buf.used + 1);
2153 }
2154 __kmp_str_buf_free(&capture_buf);
2155 return num_required;
2156}
2157
2158void kmpc_set_stacksize(int arg) {
2159 // __kmp_aux_set_stacksize initializes the library if needed
2160 __kmp_aux_set_stacksize(arg);
2161}
2162
2163void kmpc_set_stacksize_s(size_t arg) {
2164 // __kmp_aux_set_stacksize initializes the library if needed
2165 __kmp_aux_set_stacksize(arg);
2166}
2167
2168void kmpc_set_blocktime(int arg) {
2169 int gtid, tid, bt = arg;
2170 kmp_info_t *thread;
2171
2172 gtid = __kmp_entry_gtid();
2173 tid = __kmp_tid_from_gtid(gtid);
2174 thread = __kmp_thread_from_gtid(gtid);
2175
2176 __kmp_aux_convert_blocktime(&bt);
2177 __kmp_aux_set_blocktime(bt, thread, tid);
2178}
2179
2180void kmpc_set_library(int arg) {
2181 // __kmp_user_set_library initializes the library if needed
2182 __kmp_user_set_library((enum library_type)arg);
2183}
2184
2185void kmpc_set_defaults(char const *str) {
2186 // __kmp_aux_set_defaults initializes the library if needed
2187 __kmp_aux_set_defaults(str, KMP_STRLEN(str));
2188}
2189
2190void kmpc_set_disp_num_buffers(int arg) {
2191 // ignore after initialization because some teams have already
2192 // allocated dispatch buffers
2193 if (__kmp_init_serial == FALSE && arg >= KMP_MIN_DISP_NUM_BUFF &&
2194 arg <= KMP_MAX_DISP_NUM_BUFF) {
2195 __kmp_dispatch_num_buffers = arg;
2196 }
2197}
2198
2199int kmpc_set_affinity_mask_proc(int proc, void **mask) {
2200#if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
2201 return -1;
2202#else
2203 if (!TCR_4(__kmp_init_middle)) {
2204 __kmp_middle_initialize();
2205 }
2206 __kmp_assign_root_init_mask();
2207 return __kmp_aux_set_affinity_mask_proc(proc, mask);
2208#endif
2209}
2210
2211int kmpc_unset_affinity_mask_proc(int proc, void **mask) {
2212#if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
2213 return -1;
2214#else
2215 if (!TCR_4(__kmp_init_middle)) {
2216 __kmp_middle_initialize();
2217 }
2218 __kmp_assign_root_init_mask();
2219 return __kmp_aux_unset_affinity_mask_proc(proc, mask);
2220#endif
2221}
2222
2223int kmpc_get_affinity_mask_proc(int proc, void **mask) {
2224#if defined(KMP_STUB) || !KMP_AFFINITY_SUPPORTED
2225 return -1;
2226#else
2227 if (!TCR_4(__kmp_init_middle)) {
2228 __kmp_middle_initialize();
2229 }
2230 __kmp_assign_root_init_mask();
2231 return __kmp_aux_get_affinity_mask_proc(proc, mask);
2232#endif
2233}
2234
2235/* -------------------------------------------------------------------------- */
2280void __kmpc_copyprivate(ident_t *loc, kmp_int32 gtid, size_t cpy_size,
2281 void *cpy_data, void (*cpy_func)(void *, void *),
2282 kmp_int32 didit) {
2283 void **data_ptr;
2284 KC_TRACE(10, ("__kmpc_copyprivate: called T#%d\n", gtid));
2285 __kmp_assert_valid_gtid(gtid);
2286
2287 KMP_MB();
2288
2289 data_ptr = &__kmp_team_from_gtid(gtid)->t.t_copypriv_data;
2290
2291 if (__kmp_env_consistency_check) {
2292 if (loc == 0) {
2293 KMP_WARNING(ConstructIdentInvalid);
2294 }
2295 }
2296
2297 // ToDo: Optimize the following two barriers into some kind of split barrier
2298
2299 if (didit)
2300 *data_ptr = cpy_data;
2301
2302#if OMPT_SUPPORT
2303 ompt_frame_t *ompt_frame;
2304 if (ompt_enabled.enabled) {
2305 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
2306 if (ompt_frame->enter_frame.ptr == NULL)
2307 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
2308 }
2309 OMPT_STORE_RETURN_ADDRESS(gtid);
2310#endif
2311/* This barrier is not a barrier region boundary */
2312#if USE_ITT_NOTIFY
2313 __kmp_threads[gtid]->th.th_ident = loc;
2314#endif
2315 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
2316
2317 if (!didit)
2318 (*cpy_func)(cpy_data, *data_ptr);
2319
2320 // Consider next barrier a user-visible barrier for barrier region boundaries
2321 // Nesting checks are already handled by the single construct checks
2322 {
2323#if OMPT_SUPPORT
2324 OMPT_STORE_RETURN_ADDRESS(gtid);
2325#endif
2326#if USE_ITT_NOTIFY
2327 __kmp_threads[gtid]->th.th_ident = loc; // TODO: check if it is needed (e.g.
2328// tasks can overwrite the location)
2329#endif
2330 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
2331#if OMPT_SUPPORT && OMPT_OPTIONAL
2332 if (ompt_enabled.enabled) {
2333 ompt_frame->enter_frame = ompt_data_none;
2334 }
2335#endif
2336 }
2337}
2338
2339/* --------------------------------------------------------------------------*/
2356void *__kmpc_copyprivate_light(ident_t *loc, kmp_int32 gtid, void *cpy_data) {
2357 void **data_ptr;
2358
2359 KC_TRACE(10, ("__kmpc_copyprivate_light: called T#%d\n", gtid));
2360
2361 KMP_MB();
2362
2363 data_ptr = &__kmp_team_from_gtid(gtid)->t.t_copypriv_data;
2364
2365 if (__kmp_env_consistency_check) {
2366 if (loc == 0) {
2367 KMP_WARNING(ConstructIdentInvalid);
2368 }
2369 }
2370
2371 // ToDo: Optimize the following barrier
2372
2373 if (cpy_data)
2374 *data_ptr = cpy_data;
2375
2376#if OMPT_SUPPORT
2377 ompt_frame_t *ompt_frame;
2378 if (ompt_enabled.enabled) {
2379 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
2380 if (ompt_frame->enter_frame.ptr == NULL)
2381 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
2382 OMPT_STORE_RETURN_ADDRESS(gtid);
2383 }
2384#endif
2385/* This barrier is not a barrier region boundary */
2386#if USE_ITT_NOTIFY
2387 __kmp_threads[gtid]->th.th_ident = loc;
2388#endif
2389 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL);
2390
2391 return *data_ptr;
2392}
2393
2394/* -------------------------------------------------------------------------- */
2395
2396#define INIT_LOCK __kmp_init_user_lock_with_checks
2397#define INIT_NESTED_LOCK __kmp_init_nested_user_lock_with_checks
2398#define ACQUIRE_LOCK __kmp_acquire_user_lock_with_checks
2399#define ACQUIRE_LOCK_TIMED __kmp_acquire_user_lock_with_checks_timed
2400#define ACQUIRE_NESTED_LOCK __kmp_acquire_nested_user_lock_with_checks
2401#define ACQUIRE_NESTED_LOCK_TIMED \
2402 __kmp_acquire_nested_user_lock_with_checks_timed
2403#define RELEASE_LOCK __kmp_release_user_lock_with_checks
2404#define RELEASE_NESTED_LOCK __kmp_release_nested_user_lock_with_checks
2405#define TEST_LOCK __kmp_test_user_lock_with_checks
2406#define TEST_NESTED_LOCK __kmp_test_nested_user_lock_with_checks
2407#define DESTROY_LOCK __kmp_destroy_user_lock_with_checks
2408#define DESTROY_NESTED_LOCK __kmp_destroy_nested_user_lock_with_checks
2409
2410// TODO: Make check abort messages use location info & pass it into
2411// with_checks routines
2412
2413#if KMP_USE_DYNAMIC_LOCK
2414
2415// internal lock initializer
2416static __forceinline void __kmp_init_lock_with_hint(ident_t *loc, void **lock,
2417 kmp_dyna_lockseq_t seq) {
2418 if (KMP_IS_D_LOCK(seq)) {
2419 KMP_INIT_D_LOCK(lock, seq);
2420#if USE_ITT_BUILD
2421 __kmp_itt_lock_creating((kmp_user_lock_p)lock, NULL);
2422#endif
2423 } else {
2424 KMP_INIT_I_LOCK(lock, seq);
2425#if USE_ITT_BUILD
2426 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
2427 __kmp_itt_lock_creating(ilk->lock, loc);
2428#endif
2429 }
2430}
2431
2432// internal nest lock initializer
2433static __forceinline void
2434__kmp_init_nest_lock_with_hint(ident_t *loc, void **lock,
2435 kmp_dyna_lockseq_t seq) {
2436#if KMP_USE_TSX
2437 // Don't have nested lock implementation for speculative locks
2438 if (seq == lockseq_hle || seq == lockseq_rtm_queuing ||
2439 seq == lockseq_rtm_spin || seq == lockseq_adaptive)
2440 seq = __kmp_user_lock_seq;
2441#endif
2442 switch (seq) {
2443 case lockseq_tas:
2444 seq = lockseq_nested_tas;
2445 break;
2446#if KMP_USE_FUTEX
2447 case lockseq_futex:
2448 seq = lockseq_nested_futex;
2449 break;
2450#endif
2451 case lockseq_ticket:
2452 seq = lockseq_nested_ticket;
2453 break;
2454 case lockseq_queuing:
2455 seq = lockseq_nested_queuing;
2456 break;
2457 case lockseq_drdpa:
2458 seq = lockseq_nested_drdpa;
2459 break;
2460 default:
2461 seq = lockseq_nested_queuing;
2462 }
2463 KMP_INIT_I_LOCK(lock, seq);
2464#if USE_ITT_BUILD
2465 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
2466 __kmp_itt_lock_creating(ilk->lock, loc);
2467#endif
2468}
2469
2470/* initialize the lock with a hint */
2471void __kmpc_init_lock_with_hint(ident_t *loc, kmp_int32 gtid, void **user_lock,
2472 uintptr_t hint) {
2473 KMP_DEBUG_ASSERT(__kmp_init_serial);
2474 if (__kmp_env_consistency_check && user_lock == NULL) {
2475 KMP_FATAL(LockIsUninitialized, "omp_init_lock_with_hint");
2476 }
2477
2478 __kmp_init_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint));
2479
2480#if OMPT_SUPPORT && OMPT_OPTIONAL
2481 // This is the case, if called from omp_init_lock_with_hint:
2482 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2483 if (!codeptr)
2484 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2485 if (ompt_enabled.ompt_callback_lock_init) {
2486 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2487 ompt_mutex_lock, (omp_lock_hint_t)hint,
2488 __ompt_get_mutex_impl_type(user_lock),
2489 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2490 }
2491#endif
2492}
2493
2494/* initialize the lock with a hint */
2495void __kmpc_init_nest_lock_with_hint(ident_t *loc, kmp_int32 gtid,
2496 void **user_lock, uintptr_t hint) {
2497 KMP_DEBUG_ASSERT(__kmp_init_serial);
2498 if (__kmp_env_consistency_check && user_lock == NULL) {
2499 KMP_FATAL(LockIsUninitialized, "omp_init_nest_lock_with_hint");
2500 }
2501
2502 __kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_map_hint_to_lock(hint));
2503
2504#if OMPT_SUPPORT && OMPT_OPTIONAL
2505 // This is the case, if called from omp_init_lock_with_hint:
2506 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2507 if (!codeptr)
2508 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2509 if (ompt_enabled.ompt_callback_lock_init) {
2510 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2511 ompt_mutex_nest_lock, (omp_lock_hint_t)hint,
2512 __ompt_get_mutex_impl_type(user_lock),
2513 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2514 }
2515#endif
2516}
2517
2518#endif // KMP_USE_DYNAMIC_LOCK
2519
2520/* initialize the lock */
2521void __kmpc_init_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) {
2522#if KMP_USE_DYNAMIC_LOCK
2523
2524 KMP_DEBUG_ASSERT(__kmp_init_serial);
2525 if (__kmp_env_consistency_check && user_lock == NULL) {
2526 KMP_FATAL(LockIsUninitialized, "omp_init_lock");
2527 }
2528 __kmp_init_lock_with_hint(loc, user_lock, __kmp_user_lock_seq);
2529
2530#if OMPT_SUPPORT && OMPT_OPTIONAL
2531 // This is the case, if called from omp_init_lock_with_hint:
2532 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2533 if (!codeptr)
2534 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2535 if (ompt_enabled.ompt_callback_lock_init) {
2536 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2537 ompt_mutex_lock, omp_lock_hint_none,
2538 __ompt_get_mutex_impl_type(user_lock),
2539 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2540 }
2541#endif
2542
2543#else // KMP_USE_DYNAMIC_LOCK
2544
2545 static char const *const func = "omp_init_lock";
2546 kmp_user_lock_p lck;
2547 KMP_DEBUG_ASSERT(__kmp_init_serial);
2548
2549 if (__kmp_env_consistency_check) {
2550 if (user_lock == NULL) {
2551 KMP_FATAL(LockIsUninitialized, func);
2552 }
2553 }
2554
2555 KMP_CHECK_USER_LOCK_INIT();
2556
2557 if ((__kmp_user_lock_kind == lk_tas) &&
2558 (sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2559 lck = (kmp_user_lock_p)user_lock;
2560 }
2561#if KMP_USE_FUTEX
2562 else if ((__kmp_user_lock_kind == lk_futex) &&
2563 (sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2564 lck = (kmp_user_lock_p)user_lock;
2565 }
2566#endif
2567 else {
2568 lck = __kmp_user_lock_allocate(user_lock, gtid, 0);
2569 }
2570 INIT_LOCK(lck);
2571 __kmp_set_user_lock_location(lck, loc);
2572
2573#if OMPT_SUPPORT && OMPT_OPTIONAL
2574 // This is the case, if called from omp_init_lock_with_hint:
2575 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2576 if (!codeptr)
2577 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2578 if (ompt_enabled.ompt_callback_lock_init) {
2579 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2580 ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2581 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2582 }
2583#endif
2584
2585#if USE_ITT_BUILD
2586 __kmp_itt_lock_creating(lck);
2587#endif /* USE_ITT_BUILD */
2588
2589#endif // KMP_USE_DYNAMIC_LOCK
2590} // __kmpc_init_lock
2591
2592/* initialize the lock */
2593void __kmpc_init_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) {
2594#if KMP_USE_DYNAMIC_LOCK
2595
2596 KMP_DEBUG_ASSERT(__kmp_init_serial);
2597 if (__kmp_env_consistency_check && user_lock == NULL) {
2598 KMP_FATAL(LockIsUninitialized, "omp_init_nest_lock");
2599 }
2600 __kmp_init_nest_lock_with_hint(loc, user_lock, __kmp_user_lock_seq);
2601
2602#if OMPT_SUPPORT && OMPT_OPTIONAL
2603 // This is the case, if called from omp_init_lock_with_hint:
2604 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2605 if (!codeptr)
2606 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2607 if (ompt_enabled.ompt_callback_lock_init) {
2608 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2609 ompt_mutex_nest_lock, omp_lock_hint_none,
2610 __ompt_get_mutex_impl_type(user_lock),
2611 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2612 }
2613#endif
2614
2615#else // KMP_USE_DYNAMIC_LOCK
2616
2617 static char const *const func = "omp_init_nest_lock";
2618 kmp_user_lock_p lck;
2619 KMP_DEBUG_ASSERT(__kmp_init_serial);
2620
2621 if (__kmp_env_consistency_check) {
2622 if (user_lock == NULL) {
2623 KMP_FATAL(LockIsUninitialized, func);
2624 }
2625 }
2626
2627 KMP_CHECK_USER_LOCK_INIT();
2628
2629 if ((__kmp_user_lock_kind == lk_tas) &&
2630 (sizeof(lck->tas.lk.poll) + sizeof(lck->tas.lk.depth_locked) <=
2631 OMP_NEST_LOCK_T_SIZE)) {
2632 lck = (kmp_user_lock_p)user_lock;
2633 }
2634#if KMP_USE_FUTEX
2635 else if ((__kmp_user_lock_kind == lk_futex) &&
2636 (sizeof(lck->futex.lk.poll) + sizeof(lck->futex.lk.depth_locked) <=
2637 OMP_NEST_LOCK_T_SIZE)) {
2638 lck = (kmp_user_lock_p)user_lock;
2639 }
2640#endif
2641 else {
2642 lck = __kmp_user_lock_allocate(user_lock, gtid, 0);
2643 }
2644
2645 INIT_NESTED_LOCK(lck);
2646 __kmp_set_user_lock_location(lck, loc);
2647
2648#if OMPT_SUPPORT && OMPT_OPTIONAL
2649 // This is the case, if called from omp_init_lock_with_hint:
2650 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2651 if (!codeptr)
2652 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2653 if (ompt_enabled.ompt_callback_lock_init) {
2654 ompt_callbacks.ompt_callback(ompt_callback_lock_init)(
2655 ompt_mutex_nest_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2656 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2657 }
2658#endif
2659
2660#if USE_ITT_BUILD
2661 __kmp_itt_lock_creating(lck);
2662#endif /* USE_ITT_BUILD */
2663
2664#endif // KMP_USE_DYNAMIC_LOCK
2665} // __kmpc_init_nest_lock
2666
2667void __kmpc_destroy_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) {
2668#if KMP_USE_DYNAMIC_LOCK
2669
2670#if USE_ITT_BUILD
2671 kmp_user_lock_p lck;
2672 if (KMP_EXTRACT_D_TAG(user_lock) == 0) {
2673 lck = ((kmp_indirect_lock_t *)KMP_LOOKUP_I_LOCK(user_lock))->lock;
2674 } else {
2675 lck = (kmp_user_lock_p)user_lock;
2676 }
2677 __kmp_itt_lock_destroyed(lck);
2678#endif
2679#if OMPT_SUPPORT && OMPT_OPTIONAL
2680 // This is the case, if called from omp_init_lock_with_hint:
2681 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2682 if (!codeptr)
2683 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2684 if (ompt_enabled.ompt_callback_lock_destroy) {
2685 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2686 ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2687 }
2688#endif
2689 KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
2690#else
2691 kmp_user_lock_p lck;
2692
2693 if ((__kmp_user_lock_kind == lk_tas) &&
2694 (sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2695 lck = (kmp_user_lock_p)user_lock;
2696 }
2697#if KMP_USE_FUTEX
2698 else if ((__kmp_user_lock_kind == lk_futex) &&
2699 (sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2700 lck = (kmp_user_lock_p)user_lock;
2701 }
2702#endif
2703 else {
2704 lck = __kmp_lookup_user_lock(user_lock, "omp_destroy_lock");
2705 }
2706
2707#if OMPT_SUPPORT && OMPT_OPTIONAL
2708 // This is the case, if called from omp_init_lock_with_hint:
2709 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2710 if (!codeptr)
2711 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2712 if (ompt_enabled.ompt_callback_lock_destroy) {
2713 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2714 ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2715 }
2716#endif
2717
2718#if USE_ITT_BUILD
2719 __kmp_itt_lock_destroyed(lck);
2720#endif /* USE_ITT_BUILD */
2721 DESTROY_LOCK(lck);
2722
2723 if ((__kmp_user_lock_kind == lk_tas) &&
2724 (sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2725 ;
2726 }
2727#if KMP_USE_FUTEX
2728 else if ((__kmp_user_lock_kind == lk_futex) &&
2729 (sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2730 ;
2731 }
2732#endif
2733 else {
2734 __kmp_user_lock_free(user_lock, gtid, lck);
2735 }
2736#endif // KMP_USE_DYNAMIC_LOCK
2737} // __kmpc_destroy_lock
2738
2739/* destroy the lock */
2740void __kmpc_destroy_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) {
2741#if KMP_USE_DYNAMIC_LOCK
2742
2743#if USE_ITT_BUILD
2744 kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(user_lock);
2745 __kmp_itt_lock_destroyed(ilk->lock);
2746#endif
2747#if OMPT_SUPPORT && OMPT_OPTIONAL
2748 // This is the case, if called from omp_init_lock_with_hint:
2749 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2750 if (!codeptr)
2751 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2752 if (ompt_enabled.ompt_callback_lock_destroy) {
2753 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2754 ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2755 }
2756#endif
2757 KMP_D_LOCK_FUNC(user_lock, destroy)((kmp_dyna_lock_t *)user_lock);
2758
2759#else // KMP_USE_DYNAMIC_LOCK
2760
2761 kmp_user_lock_p lck;
2762
2763 if ((__kmp_user_lock_kind == lk_tas) &&
2764 (sizeof(lck->tas.lk.poll) + sizeof(lck->tas.lk.depth_locked) <=
2765 OMP_NEST_LOCK_T_SIZE)) {
2766 lck = (kmp_user_lock_p)user_lock;
2767 }
2768#if KMP_USE_FUTEX
2769 else if ((__kmp_user_lock_kind == lk_futex) &&
2770 (sizeof(lck->futex.lk.poll) + sizeof(lck->futex.lk.depth_locked) <=
2771 OMP_NEST_LOCK_T_SIZE)) {
2772 lck = (kmp_user_lock_p)user_lock;
2773 }
2774#endif
2775 else {
2776 lck = __kmp_lookup_user_lock(user_lock, "omp_destroy_nest_lock");
2777 }
2778
2779#if OMPT_SUPPORT && OMPT_OPTIONAL
2780 // This is the case, if called from omp_init_lock_with_hint:
2781 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2782 if (!codeptr)
2783 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2784 if (ompt_enabled.ompt_callback_lock_destroy) {
2785 ompt_callbacks.ompt_callback(ompt_callback_lock_destroy)(
2786 ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2787 }
2788#endif
2789
2790#if USE_ITT_BUILD
2791 __kmp_itt_lock_destroyed(lck);
2792#endif /* USE_ITT_BUILD */
2793
2794 DESTROY_NESTED_LOCK(lck);
2795
2796 if ((__kmp_user_lock_kind == lk_tas) &&
2797 (sizeof(lck->tas.lk.poll) + sizeof(lck->tas.lk.depth_locked) <=
2798 OMP_NEST_LOCK_T_SIZE)) {
2799 ;
2800 }
2801#if KMP_USE_FUTEX
2802 else if ((__kmp_user_lock_kind == lk_futex) &&
2803 (sizeof(lck->futex.lk.poll) + sizeof(lck->futex.lk.depth_locked) <=
2804 OMP_NEST_LOCK_T_SIZE)) {
2805 ;
2806 }
2807#endif
2808 else {
2809 __kmp_user_lock_free(user_lock, gtid, lck);
2810 }
2811#endif // KMP_USE_DYNAMIC_LOCK
2812} // __kmpc_destroy_nest_lock
2813
2814void __kmpc_set_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) {
2815 KMP_COUNT_BLOCK(OMP_set_lock);
2816#if KMP_USE_DYNAMIC_LOCK
2817 int tag = KMP_EXTRACT_D_TAG(user_lock);
2818#if USE_ITT_BUILD
2819 __kmp_itt_lock_acquiring(
2820 (kmp_user_lock_p)
2821 user_lock); // itt function will get to the right lock object.
2822#endif
2823#if OMPT_SUPPORT && OMPT_OPTIONAL
2824 // This is the case, if called from omp_init_lock_with_hint:
2825 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2826 if (!codeptr)
2827 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2828 if (ompt_enabled.ompt_callback_mutex_acquire) {
2829 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2830 ompt_mutex_lock, omp_lock_hint_none,
2831 __ompt_get_mutex_impl_type(user_lock),
2832 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2833 }
2834#endif
2835#if KMP_USE_INLINED_TAS
2836 if (tag == locktag_tas && !__kmp_env_consistency_check) {
2837 KMP_ACQUIRE_TAS_LOCK(user_lock, gtid);
2838 } else
2839#elif KMP_USE_INLINED_FUTEX
2840 if (tag == locktag_futex && !__kmp_env_consistency_check) {
2841 KMP_ACQUIRE_FUTEX_LOCK(user_lock, gtid);
2842 } else
2843#endif
2844 {
2845 __kmp_direct_set[tag]((kmp_dyna_lock_t *)user_lock, gtid);
2846 }
2847#if USE_ITT_BUILD
2848 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2849#endif
2850#if OMPT_SUPPORT && OMPT_OPTIONAL
2851 if (ompt_enabled.ompt_callback_mutex_acquired) {
2852 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2853 ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2854 }
2855#endif
2856
2857#else // KMP_USE_DYNAMIC_LOCK
2858
2859 kmp_user_lock_p lck;
2860
2861 if ((__kmp_user_lock_kind == lk_tas) &&
2862 (sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
2863 lck = (kmp_user_lock_p)user_lock;
2864 }
2865#if KMP_USE_FUTEX
2866 else if ((__kmp_user_lock_kind == lk_futex) &&
2867 (sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
2868 lck = (kmp_user_lock_p)user_lock;
2869 }
2870#endif
2871 else {
2872 lck = __kmp_lookup_user_lock(user_lock, "omp_set_lock");
2873 }
2874
2875#if USE_ITT_BUILD
2876 __kmp_itt_lock_acquiring(lck);
2877#endif /* USE_ITT_BUILD */
2878#if OMPT_SUPPORT && OMPT_OPTIONAL
2879 // This is the case, if called from omp_init_lock_with_hint:
2880 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2881 if (!codeptr)
2882 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2883 if (ompt_enabled.ompt_callback_mutex_acquire) {
2884 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2885 ompt_mutex_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
2886 (ompt_wait_id_t)(uintptr_t)lck, codeptr);
2887 }
2888#endif
2889
2890 ACQUIRE_LOCK(lck, gtid);
2891
2892#if USE_ITT_BUILD
2893 __kmp_itt_lock_acquired(lck);
2894#endif /* USE_ITT_BUILD */
2895
2896#if OMPT_SUPPORT && OMPT_OPTIONAL
2897 if (ompt_enabled.ompt_callback_mutex_acquired) {
2898 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2899 ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
2900 }
2901#endif
2902
2903#endif // KMP_USE_DYNAMIC_LOCK
2904}
2905
2906void __kmpc_set_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) {
2907#if KMP_USE_DYNAMIC_LOCK
2908
2909#if USE_ITT_BUILD
2910 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
2911#endif
2912#if OMPT_SUPPORT && OMPT_OPTIONAL
2913 // This is the case, if called from omp_init_lock_with_hint:
2914 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2915 if (!codeptr)
2916 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2917 if (ompt_enabled.enabled) {
2918 if (ompt_enabled.ompt_callback_mutex_acquire) {
2919 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2920 ompt_mutex_nest_lock, omp_lock_hint_none,
2921 __ompt_get_mutex_impl_type(user_lock),
2922 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2923 }
2924 }
2925#endif
2926 int acquire_status =
2927 KMP_D_LOCK_FUNC(user_lock, set)((kmp_dyna_lock_t *)user_lock, gtid);
2928 (void)acquire_status;
2929#if USE_ITT_BUILD
2930 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
2931#endif
2932
2933#if OMPT_SUPPORT && OMPT_OPTIONAL
2934 if (ompt_enabled.enabled) {
2935 if (acquire_status == KMP_LOCK_ACQUIRED_FIRST) {
2936 if (ompt_enabled.ompt_callback_mutex_acquired) {
2937 // lock_first
2938 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
2939 ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)user_lock,
2940 codeptr);
2941 }
2942 } else {
2943 if (ompt_enabled.ompt_callback_nest_lock) {
2944 // lock_next
2945 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
2946 ompt_scope_begin, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
2947 }
2948 }
2949 }
2950#endif
2951
2952#else // KMP_USE_DYNAMIC_LOCK
2953 int acquire_status;
2954 kmp_user_lock_p lck;
2955
2956 if ((__kmp_user_lock_kind == lk_tas) &&
2957 (sizeof(lck->tas.lk.poll) + sizeof(lck->tas.lk.depth_locked) <=
2958 OMP_NEST_LOCK_T_SIZE)) {
2959 lck = (kmp_user_lock_p)user_lock;
2960 }
2961#if KMP_USE_FUTEX
2962 else if ((__kmp_user_lock_kind == lk_futex) &&
2963 (sizeof(lck->futex.lk.poll) + sizeof(lck->futex.lk.depth_locked) <=
2964 OMP_NEST_LOCK_T_SIZE)) {
2965 lck = (kmp_user_lock_p)user_lock;
2966 }
2967#endif
2968 else {
2969 lck = __kmp_lookup_user_lock(user_lock, "omp_set_nest_lock");
2970 }
2971
2972#if USE_ITT_BUILD
2973 __kmp_itt_lock_acquiring(lck);
2974#endif /* USE_ITT_BUILD */
2975#if OMPT_SUPPORT && OMPT_OPTIONAL
2976 // This is the case, if called from omp_init_lock_with_hint:
2977 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
2978 if (!codeptr)
2979 codeptr = OMPT_GET_RETURN_ADDRESS(0);
2980 if (ompt_enabled.enabled) {
2981 if (ompt_enabled.ompt_callback_mutex_acquire) {
2982 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
2983 ompt_mutex_nest_lock, omp_lock_hint_none,
2984 __ompt_get_mutex_impl_type(), (ompt_wait_id_t)(uintptr_t)lck,
2985 codeptr);
2986 }
2987 }
2988#endif
2989
2990 ACQUIRE_NESTED_LOCK(lck, gtid, &acquire_status);
2991
2992#if USE_ITT_BUILD
2993 __kmp_itt_lock_acquired(lck);
2994#endif /* USE_ITT_BUILD */
2995
2996#if OMPT_SUPPORT && OMPT_OPTIONAL
2997 if (ompt_enabled.enabled) {
2998 if (acquire_status == KMP_LOCK_ACQUIRED_FIRST) {
2999 if (ompt_enabled.ompt_callback_mutex_acquired) {
3000 // lock_first
3001 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
3002 ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3003 }
3004 } else {
3005 if (ompt_enabled.ompt_callback_nest_lock) {
3006 // lock_next
3007 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
3008 ompt_scope_begin, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3009 }
3010 }
3011 }
3012#endif
3013
3014#endif // KMP_USE_DYNAMIC_LOCK
3015}
3016
3017void __kmpc_unset_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) {
3018#if KMP_USE_DYNAMIC_LOCK
3019
3020 int tag = KMP_EXTRACT_D_TAG(user_lock);
3021#if USE_ITT_BUILD
3022 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
3023#endif
3024#if KMP_USE_INLINED_TAS
3025 if (tag == locktag_tas && !__kmp_env_consistency_check) {
3026 KMP_RELEASE_TAS_LOCK(user_lock, gtid);
3027 } else
3028#elif KMP_USE_INLINED_FUTEX
3029 if (tag == locktag_futex && !__kmp_env_consistency_check) {
3030 KMP_RELEASE_FUTEX_LOCK(user_lock, gtid);
3031 } else
3032#endif
3033 {
3034 __kmp_direct_unset[tag]((kmp_dyna_lock_t *)user_lock, gtid);
3035 }
3036
3037#if OMPT_SUPPORT && OMPT_OPTIONAL
3038 // This is the case, if called from omp_init_lock_with_hint:
3039 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3040 if (!codeptr)
3041 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3042 if (ompt_enabled.ompt_callback_mutex_released) {
3043 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
3044 ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
3045 }
3046#endif
3047
3048#else // KMP_USE_DYNAMIC_LOCK
3049
3050 kmp_user_lock_p lck;
3051
3052 /* Can't use serial interval since not block structured */
3053 /* release the lock */
3054
3055 if ((__kmp_user_lock_kind == lk_tas) &&
3056 (sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
3057#if KMP_OS_LINUX && \
3058 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
3059// "fast" path implemented to fix customer performance issue
3060#if USE_ITT_BUILD
3061 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
3062#endif /* USE_ITT_BUILD */
3063 TCW_4(((kmp_user_lock_p)user_lock)->tas.lk.poll, 0);
3064 KMP_MB();
3065
3066#if OMPT_SUPPORT && OMPT_OPTIONAL
3067 // This is the case, if called from omp_init_lock_with_hint:
3068 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3069 if (!codeptr)
3070 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3071 if (ompt_enabled.ompt_callback_mutex_released) {
3072 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
3073 ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3074 }
3075#endif
3076
3077 return;
3078#else
3079 lck = (kmp_user_lock_p)user_lock;
3080#endif
3081 }
3082#if KMP_USE_FUTEX
3083 else if ((__kmp_user_lock_kind == lk_futex) &&
3084 (sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
3085 lck = (kmp_user_lock_p)user_lock;
3086 }
3087#endif
3088 else {
3089 lck = __kmp_lookup_user_lock(user_lock, "omp_unset_lock");
3090 }
3091
3092#if USE_ITT_BUILD
3093 __kmp_itt_lock_releasing(lck);
3094#endif /* USE_ITT_BUILD */
3095
3096 RELEASE_LOCK(lck, gtid);
3097
3098#if OMPT_SUPPORT && OMPT_OPTIONAL
3099 // This is the case, if called from omp_init_lock_with_hint:
3100 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3101 if (!codeptr)
3102 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3103 if (ompt_enabled.ompt_callback_mutex_released) {
3104 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
3105 ompt_mutex_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3106 }
3107#endif
3108
3109#endif // KMP_USE_DYNAMIC_LOCK
3110}
3111
3112/* release the lock */
3113void __kmpc_unset_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) {
3114#if KMP_USE_DYNAMIC_LOCK
3115
3116#if USE_ITT_BUILD
3117 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
3118#endif
3119 int release_status =
3120 KMP_D_LOCK_FUNC(user_lock, unset)((kmp_dyna_lock_t *)user_lock, gtid);
3121 (void)release_status;
3122
3123#if OMPT_SUPPORT && OMPT_OPTIONAL
3124 // This is the case, if called from omp_init_lock_with_hint:
3125 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3126 if (!codeptr)
3127 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3128 if (ompt_enabled.enabled) {
3129 if (release_status == KMP_LOCK_RELEASED) {
3130 if (ompt_enabled.ompt_callback_mutex_released) {
3131 // release_lock_last
3132 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
3133 ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)user_lock,
3134 codeptr);
3135 }
3136 } else if (ompt_enabled.ompt_callback_nest_lock) {
3137 // release_lock_prev
3138 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
3139 ompt_scope_end, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
3140 }
3141 }
3142#endif
3143
3144#else // KMP_USE_DYNAMIC_LOCK
3145
3146 kmp_user_lock_p lck;
3147
3148 /* Can't use serial interval since not block structured */
3149
3150 if ((__kmp_user_lock_kind == lk_tas) &&
3151 (sizeof(lck->tas.lk.poll) + sizeof(lck->tas.lk.depth_locked) <=
3152 OMP_NEST_LOCK_T_SIZE)) {
3153#if KMP_OS_LINUX && \
3154 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
3155 // "fast" path implemented to fix customer performance issue
3156 kmp_tas_lock_t *tl = (kmp_tas_lock_t *)user_lock;
3157#if USE_ITT_BUILD
3158 __kmp_itt_lock_releasing((kmp_user_lock_p)user_lock);
3159#endif /* USE_ITT_BUILD */
3160
3161#if OMPT_SUPPORT && OMPT_OPTIONAL
3162 int release_status = KMP_LOCK_STILL_HELD;
3163#endif
3164
3165 if (--(tl->lk.depth_locked) == 0) {
3166 TCW_4(tl->lk.poll, 0);
3167#if OMPT_SUPPORT && OMPT_OPTIONAL
3168 release_status = KMP_LOCK_RELEASED;
3169#endif
3170 }
3171 KMP_MB();
3172
3173#if OMPT_SUPPORT && OMPT_OPTIONAL
3174 // This is the case, if called from omp_init_lock_with_hint:
3175 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3176 if (!codeptr)
3177 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3178 if (ompt_enabled.enabled) {
3179 if (release_status == KMP_LOCK_RELEASED) {
3180 if (ompt_enabled.ompt_callback_mutex_released) {
3181 // release_lock_last
3182 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
3183 ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3184 }
3185 } else if (ompt_enabled.ompt_callback_nest_lock) {
3186 // release_lock_previous
3187 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
3188 ompt_mutex_scope_end, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3189 }
3190 }
3191#endif
3192
3193 return;
3194#else
3195 lck = (kmp_user_lock_p)user_lock;
3196#endif
3197 }
3198#if KMP_USE_FUTEX
3199 else if ((__kmp_user_lock_kind == lk_futex) &&
3200 (sizeof(lck->futex.lk.poll) + sizeof(lck->futex.lk.depth_locked) <=
3201 OMP_NEST_LOCK_T_SIZE)) {
3202 lck = (kmp_user_lock_p)user_lock;
3203 }
3204#endif
3205 else {
3206 lck = __kmp_lookup_user_lock(user_lock, "omp_unset_nest_lock");
3207 }
3208
3209#if USE_ITT_BUILD
3210 __kmp_itt_lock_releasing(lck);
3211#endif /* USE_ITT_BUILD */
3212
3213 int release_status;
3214 release_status = RELEASE_NESTED_LOCK(lck, gtid);
3215#if OMPT_SUPPORT && OMPT_OPTIONAL
3216 // This is the case, if called from omp_init_lock_with_hint:
3217 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3218 if (!codeptr)
3219 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3220 if (ompt_enabled.enabled) {
3221 if (release_status == KMP_LOCK_RELEASED) {
3222 if (ompt_enabled.ompt_callback_mutex_released) {
3223 // release_lock_last
3224 ompt_callbacks.ompt_callback(ompt_callback_mutex_released)(
3225 ompt_mutex_nest_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3226 }
3227 } else if (ompt_enabled.ompt_callback_nest_lock) {
3228 // release_lock_previous
3229 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
3230 ompt_mutex_scope_end, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3231 }
3232 }
3233#endif
3234
3235#endif // KMP_USE_DYNAMIC_LOCK
3236}
3237
3238/* try to acquire the lock */
3239int __kmpc_test_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) {
3240 KMP_COUNT_BLOCK(OMP_test_lock);
3241
3242#if KMP_USE_DYNAMIC_LOCK
3243 int rc;
3244 int tag = KMP_EXTRACT_D_TAG(user_lock);
3245#if USE_ITT_BUILD
3246 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
3247#endif
3248#if OMPT_SUPPORT && OMPT_OPTIONAL
3249 // This is the case, if called from omp_init_lock_with_hint:
3250 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3251 if (!codeptr)
3252 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3253 if (ompt_enabled.ompt_callback_mutex_acquire) {
3254 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
3255 ompt_mutex_test_lock, omp_lock_hint_none,
3256 __ompt_get_mutex_impl_type(user_lock),
3257 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
3258 }
3259#endif
3260#if KMP_USE_INLINED_TAS
3261 if (tag == locktag_tas && !__kmp_env_consistency_check) {
3262 KMP_TEST_TAS_LOCK(user_lock, gtid, rc);
3263 } else
3264#elif KMP_USE_INLINED_FUTEX
3265 if (tag == locktag_futex && !__kmp_env_consistency_check) {
3266 KMP_TEST_FUTEX_LOCK(user_lock, gtid, rc);
3267 } else
3268#endif
3269 {
3270 rc = __kmp_direct_test[tag]((kmp_dyna_lock_t *)user_lock, gtid);
3271 }
3272 if (rc) {
3273#if USE_ITT_BUILD
3274 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
3275#endif
3276#if OMPT_SUPPORT && OMPT_OPTIONAL
3277 if (ompt_enabled.ompt_callback_mutex_acquired) {
3278 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
3279 ompt_mutex_test_lock, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
3280 }
3281#endif
3282 return FTN_TRUE;
3283 } else {
3284#if USE_ITT_BUILD
3285 __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
3286#endif
3287 return FTN_FALSE;
3288 }
3289
3290#else // KMP_USE_DYNAMIC_LOCK
3291
3292 kmp_user_lock_p lck;
3293 int rc;
3294
3295 if ((__kmp_user_lock_kind == lk_tas) &&
3296 (sizeof(lck->tas.lk.poll) <= OMP_LOCK_T_SIZE)) {
3297 lck = (kmp_user_lock_p)user_lock;
3298 }
3299#if KMP_USE_FUTEX
3300 else if ((__kmp_user_lock_kind == lk_futex) &&
3301 (sizeof(lck->futex.lk.poll) <= OMP_LOCK_T_SIZE)) {
3302 lck = (kmp_user_lock_p)user_lock;
3303 }
3304#endif
3305 else {
3306 lck = __kmp_lookup_user_lock(user_lock, "omp_test_lock");
3307 }
3308
3309#if USE_ITT_BUILD
3310 __kmp_itt_lock_acquiring(lck);
3311#endif /* USE_ITT_BUILD */
3312#if OMPT_SUPPORT && OMPT_OPTIONAL
3313 // This is the case, if called from omp_init_lock_with_hint:
3314 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3315 if (!codeptr)
3316 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3317 if (ompt_enabled.ompt_callback_mutex_acquire) {
3318 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
3319 ompt_mutex_test_lock, omp_lock_hint_none, __ompt_get_mutex_impl_type(),
3320 (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3321 }
3322#endif
3323
3324 rc = TEST_LOCK(lck, gtid);
3325#if USE_ITT_BUILD
3326 if (rc) {
3327 __kmp_itt_lock_acquired(lck);
3328 } else {
3329 __kmp_itt_lock_cancelled(lck);
3330 }
3331#endif /* USE_ITT_BUILD */
3332#if OMPT_SUPPORT && OMPT_OPTIONAL
3333 if (rc && ompt_enabled.ompt_callback_mutex_acquired) {
3334 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
3335 ompt_mutex_test_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3336 }
3337#endif
3338
3339 return (rc ? FTN_TRUE : FTN_FALSE);
3340
3341 /* Can't use serial interval since not block structured */
3342
3343#endif // KMP_USE_DYNAMIC_LOCK
3344}
3345
3346/* try to acquire the lock */
3347int __kmpc_test_nest_lock(ident_t *loc, kmp_int32 gtid, void **user_lock) {
3348#if KMP_USE_DYNAMIC_LOCK
3349 int rc;
3350#if USE_ITT_BUILD
3351 __kmp_itt_lock_acquiring((kmp_user_lock_p)user_lock);
3352#endif
3353#if OMPT_SUPPORT && OMPT_OPTIONAL
3354 // This is the case, if called from omp_init_lock_with_hint:
3355 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3356 if (!codeptr)
3357 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3358 if (ompt_enabled.ompt_callback_mutex_acquire) {
3359 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
3360 ompt_mutex_test_nest_lock, omp_lock_hint_none,
3361 __ompt_get_mutex_impl_type(user_lock),
3362 (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
3363 }
3364#endif
3365 rc = KMP_D_LOCK_FUNC(user_lock, test)((kmp_dyna_lock_t *)user_lock, gtid);
3366#if USE_ITT_BUILD
3367 if (rc) {
3368 __kmp_itt_lock_acquired((kmp_user_lock_p)user_lock);
3369 } else {
3370 __kmp_itt_lock_cancelled((kmp_user_lock_p)user_lock);
3371 }
3372#endif
3373#if OMPT_SUPPORT && OMPT_OPTIONAL
3374 if (ompt_enabled.enabled && rc) {
3375 if (rc == 1) {
3376 if (ompt_enabled.ompt_callback_mutex_acquired) {
3377 // lock_first
3378 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
3379 ompt_mutex_test_nest_lock, (ompt_wait_id_t)(uintptr_t)user_lock,
3380 codeptr);
3381 }
3382 } else {
3383 if (ompt_enabled.ompt_callback_nest_lock) {
3384 // lock_next
3385 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
3386 ompt_scope_begin, (ompt_wait_id_t)(uintptr_t)user_lock, codeptr);
3387 }
3388 }
3389 }
3390#endif
3391 return rc;
3392
3393#else // KMP_USE_DYNAMIC_LOCK
3394
3395 kmp_user_lock_p lck;
3396 int rc;
3397
3398 if ((__kmp_user_lock_kind == lk_tas) &&
3399 (sizeof(lck->tas.lk.poll) + sizeof(lck->tas.lk.depth_locked) <=
3400 OMP_NEST_LOCK_T_SIZE)) {
3401 lck = (kmp_user_lock_p)user_lock;
3402 }
3403#if KMP_USE_FUTEX
3404 else if ((__kmp_user_lock_kind == lk_futex) &&
3405 (sizeof(lck->futex.lk.poll) + sizeof(lck->futex.lk.depth_locked) <=
3406 OMP_NEST_LOCK_T_SIZE)) {
3407 lck = (kmp_user_lock_p)user_lock;
3408 }
3409#endif
3410 else {
3411 lck = __kmp_lookup_user_lock(user_lock, "omp_test_nest_lock");
3412 }
3413
3414#if USE_ITT_BUILD
3415 __kmp_itt_lock_acquiring(lck);
3416#endif /* USE_ITT_BUILD */
3417
3418#if OMPT_SUPPORT && OMPT_OPTIONAL
3419 // This is the case, if called from omp_init_lock_with_hint:
3420 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(gtid);
3421 if (!codeptr)
3422 codeptr = OMPT_GET_RETURN_ADDRESS(0);
3423 if (ompt_enabled.enabled) &&
3424 ompt_enabled.ompt_callback_mutex_acquire) {
3425 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquire)(
3426 ompt_mutex_test_nest_lock, omp_lock_hint_none,
3427 __ompt_get_mutex_impl_type(), (ompt_wait_id_t)(uintptr_t)lck,
3428 codeptr);
3429 }
3430#endif
3431
3432 rc = TEST_NESTED_LOCK(lck, gtid);
3433#if USE_ITT_BUILD
3434 if (rc) {
3435 __kmp_itt_lock_acquired(lck);
3436 } else {
3437 __kmp_itt_lock_cancelled(lck);
3438 }
3439#endif /* USE_ITT_BUILD */
3440#if OMPT_SUPPORT && OMPT_OPTIONAL
3441 if (ompt_enabled.enabled && rc) {
3442 if (rc == 1) {
3443 if (ompt_enabled.ompt_callback_mutex_acquired) {
3444 // lock_first
3445 ompt_callbacks.ompt_callback(ompt_callback_mutex_acquired)(
3446 ompt_mutex_test_nest_lock, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3447 }
3448 } else {
3449 if (ompt_enabled.ompt_callback_nest_lock) {
3450 // lock_next
3451 ompt_callbacks.ompt_callback(ompt_callback_nest_lock)(
3452 ompt_mutex_scope_begin, (ompt_wait_id_t)(uintptr_t)lck, codeptr);
3453 }
3454 }
3455 }
3456#endif
3457 return rc;
3458
3459 /* Can't use serial interval since not block structured */
3460
3461#endif // KMP_USE_DYNAMIC_LOCK
3462}
3463
3464// Interface to fast scalable reduce methods routines
3465
3466// keep the selected method in a thread local structure for cross-function
3467// usage: will be used in __kmpc_end_reduce* functions;
3468// another solution: to re-determine the method one more time in
3469// __kmpc_end_reduce* functions (new prototype required then)
3470// AT: which solution is better?
3471#define __KMP_SET_REDUCTION_METHOD(gtid, rmethod) \
3472 ((__kmp_threads[(gtid)]->th.th_local.packed_reduction_method) = (rmethod))
3473
3474#define __KMP_GET_REDUCTION_METHOD(gtid) \
3475 (__kmp_threads[(gtid)]->th.th_local.packed_reduction_method)
3476
3477// description of the packed_reduction_method variable: look at the macros in
3478// kmp.h
3479
3480// used in a critical section reduce block
3481static __forceinline void
3482__kmp_enter_critical_section_reduce_block(ident_t *loc, kmp_int32 global_tid,
3483 kmp_critical_name *crit) {
3484
3485 // this lock was visible to a customer and to the threading profile tool as a
3486 // serial overhead span (although it's used for an internal purpose only)
3487 // why was it visible in previous implementation?
3488 // should we keep it visible in new reduce block?
3489 kmp_user_lock_p lck;
3490
3491#if KMP_USE_DYNAMIC_LOCK
3492
3493 kmp_dyna_lock_t *lk = (kmp_dyna_lock_t *)crit;
3494 // Check if it is initialized.
3495 if (*lk == 0) {
3496 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
3497 (void)KMP_COMPARE_AND_STORE_ACQ32((volatile kmp_int32 *)crit, 0,
3498 KMP_GET_D_TAG(__kmp_user_lock_seq));
3499 } else {
3500 __kmp_init_indirect_csptr(crit, loc, global_tid,
3501 KMP_GET_I_TAG(__kmp_user_lock_seq));
3502 }
3503 }
3504 // Branch for accessing the actual lock object and set operation. This
3505 // branching is inevitable since this lock initialization does not follow the
3506 // normal dispatch path (lock table is not used).
3507 if (KMP_EXTRACT_D_TAG(lk) != 0) {
3508 lck = (kmp_user_lock_p)lk;
3509 KMP_DEBUG_ASSERT(lck != NULL);
3510 if (__kmp_env_consistency_check) {
3511 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
3512 }
3513 KMP_D_LOCK_FUNC(lk, set)(lk, global_tid);
3514 } else {
3515 kmp_indirect_lock_t *ilk = *((kmp_indirect_lock_t **)lk);
3516 lck = ilk->lock;
3517 KMP_DEBUG_ASSERT(lck != NULL);
3518 if (__kmp_env_consistency_check) {
3519 __kmp_push_sync(global_tid, ct_critical, loc, lck, __kmp_user_lock_seq);
3520 }
3521 KMP_I_LOCK_FUNC(ilk, set)(lck, global_tid);
3522 }
3523
3524#else // KMP_USE_DYNAMIC_LOCK
3525
3526 // We know that the fast reduction code is only emitted by Intel compilers
3527 // with 32 byte critical sections. If there isn't enough space, then we
3528 // have to use a pointer.
3529 if (__kmp_base_user_lock_size <= INTEL_CRITICAL_SIZE) {
3530 lck = (kmp_user_lock_p)crit;
3531 } else {
3532 lck = __kmp_get_critical_section_ptr(crit, loc, global_tid);
3533 }
3534 KMP_DEBUG_ASSERT(lck != NULL);
3535
3536 if (__kmp_env_consistency_check)
3537 __kmp_push_sync(global_tid, ct_critical, loc, lck);
3538
3539 __kmp_acquire_user_lock_with_checks(lck, global_tid);
3540
3541#endif // KMP_USE_DYNAMIC_LOCK
3542}
3543
3544// used in a critical section reduce block
3545static __forceinline void
3546__kmp_end_critical_section_reduce_block(ident_t *loc, kmp_int32 global_tid,
3547 kmp_critical_name *crit) {
3548
3549 kmp_user_lock_p lck;
3550
3551#if KMP_USE_DYNAMIC_LOCK
3552
3553 if (KMP_IS_D_LOCK(__kmp_user_lock_seq)) {
3554 lck = (kmp_user_lock_p)crit;
3555 if (__kmp_env_consistency_check)
3556 __kmp_pop_sync(global_tid, ct_critical, loc);
3557 KMP_D_LOCK_FUNC(lck, unset)((kmp_dyna_lock_t *)lck, global_tid);
3558 } else {
3559 kmp_indirect_lock_t *ilk =
3560 (kmp_indirect_lock_t *)TCR_PTR(*((kmp_indirect_lock_t **)crit));
3561 if (__kmp_env_consistency_check)
3562 __kmp_pop_sync(global_tid, ct_critical, loc);
3563 KMP_I_LOCK_FUNC(ilk, unset)(ilk->lock, global_tid);
3564 }
3565
3566#else // KMP_USE_DYNAMIC_LOCK
3567
3568 // We know that the fast reduction code is only emitted by Intel compilers
3569 // with 32 byte critical sections. If there isn't enough space, then we have
3570 // to use a pointer.
3571 if (__kmp_base_user_lock_size > 32) {
3572 lck = *((kmp_user_lock_p *)crit);
3573 KMP_ASSERT(lck != NULL);
3574 } else {
3575 lck = (kmp_user_lock_p)crit;
3576 }
3577
3578 if (__kmp_env_consistency_check)
3579 __kmp_pop_sync(global_tid, ct_critical, loc);
3580
3581 __kmp_release_user_lock_with_checks(lck, global_tid);
3582
3583#endif // KMP_USE_DYNAMIC_LOCK
3584} // __kmp_end_critical_section_reduce_block
3585
3586static __forceinline int
3587__kmp_swap_teams_for_teams_reduction(kmp_info_t *th, kmp_team_t **team_p,
3588 int *task_state) {
3589 kmp_team_t *team;
3590
3591 // Check if we are inside the teams construct?
3592 if (th->th.th_teams_microtask) {
3593 *team_p = team = th->th.th_team;
3594 if (team->t.t_level == th->th.th_teams_level) {
3595 // This is reduction at teams construct.
3596 KMP_DEBUG_ASSERT(!th->th.th_info.ds.ds_tid); // AC: check that tid == 0
3597 // Let's swap teams temporarily for the reduction.
3598 th->th.th_info.ds.ds_tid = team->t.t_master_tid;
3599 th->th.th_team = team->t.t_parent;
3600 th->th.th_team_nproc = th->th.th_team->t.t_nproc;
3601 th->th.th_task_team = th->th.th_team->t.t_task_team[0];
3602 *task_state = th->th.th_task_state;
3603 th->th.th_task_state = 0;
3604
3605 return 1;
3606 }
3607 }
3608 return 0;
3609}
3610
3611static __forceinline void
3612__kmp_restore_swapped_teams(kmp_info_t *th, kmp_team_t *team, int task_state) {
3613 // Restore thread structure swapped in __kmp_swap_teams_for_teams_reduction.
3614 th->th.th_info.ds.ds_tid = 0;
3615 th->th.th_team = team;
3616 th->th.th_team_nproc = team->t.t_nproc;
3617 th->th.th_task_team = team->t.t_task_team[task_state];
3618 __kmp_type_convert(task_state, &(th->th.th_task_state));
3619}
3620
3621/* 2.a.i. Reduce Block without a terminating barrier */
3637kmp_int32
3638__kmpc_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars,
3639 size_t reduce_size, void *reduce_data,
3640 void (*reduce_func)(void *lhs_data, void *rhs_data),
3641 kmp_critical_name *lck) {
3642
3643 KMP_COUNT_BLOCK(REDUCE_nowait);
3644 int retval = 0;
3645 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3646 kmp_info_t *th;
3647 kmp_team_t *team;
3648 int teams_swapped = 0, task_state;
3649 KA_TRACE(10, ("__kmpc_reduce_nowait() enter: called T#%d\n", global_tid));
3650 __kmp_assert_valid_gtid(global_tid);
3651
3652 // why do we need this initialization here at all?
3653 // Reduction clause can not be used as a stand-alone directive.
3654
3655 // do not call __kmp_serial_initialize(), it will be called by
3656 // __kmp_parallel_initialize() if needed
3657 // possible detection of false-positive race by the threadchecker ???
3658 if (!TCR_4(__kmp_init_parallel))
3659 __kmp_parallel_initialize();
3660
3661 __kmp_resume_if_soft_paused();
3662
3663// check correctness of reduce block nesting
3664#if KMP_USE_DYNAMIC_LOCK
3665 if (__kmp_env_consistency_check)
3666 __kmp_push_sync(global_tid, ct_reduce, loc, NULL, 0);
3667#else
3668 if (__kmp_env_consistency_check)
3669 __kmp_push_sync(global_tid, ct_reduce, loc, NULL);
3670#endif
3671
3672 th = __kmp_thread_from_gtid(global_tid);
3673 teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state);
3674
3675 // packed_reduction_method value will be reused by __kmp_end_reduce* function,
3676 // the value should be kept in a variable
3677 // the variable should be either a construct-specific or thread-specific
3678 // property, not a team specific property
3679 // (a thread can reach the next reduce block on the next construct, reduce
3680 // method may differ on the next construct)
3681 // an ident_t "loc" parameter could be used as a construct-specific property
3682 // (what if loc == 0?)
3683 // (if both construct-specific and team-specific variables were shared,
3684 // then unness extra syncs should be needed)
3685 // a thread-specific variable is better regarding two issues above (next
3686 // construct and extra syncs)
3687 // a thread-specific "th_local.reduction_method" variable is used currently
3688 // each thread executes 'determine' and 'set' lines (no need to execute by one
3689 // thread, to avoid unness extra syncs)
3690
3691 packed_reduction_method = __kmp_determine_reduction_method(
3692 loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck);
3693 __KMP_SET_REDUCTION_METHOD(global_tid, packed_reduction_method);
3694
3695 OMPT_REDUCTION_DECL(th, global_tid);
3696 if (packed_reduction_method == critical_reduce_block) {
3697
3698 OMPT_REDUCTION_BEGIN;
3699
3700 __kmp_enter_critical_section_reduce_block(loc, global_tid, lck);
3701 retval = 1;
3702
3703 } else if (packed_reduction_method == empty_reduce_block) {
3704
3705 OMPT_REDUCTION_BEGIN;
3706
3707 // usage: if team size == 1, no synchronization is required ( Intel
3708 // platforms only )
3709 retval = 1;
3710
3711 } else if (packed_reduction_method == atomic_reduce_block) {
3712
3713 retval = 2;
3714
3715 // all threads should do this pop here (because __kmpc_end_reduce_nowait()
3716 // won't be called by the code gen)
3717 // (it's not quite good, because the checking block has been closed by
3718 // this 'pop',
3719 // but atomic operation has not been executed yet, will be executed
3720 // slightly later, literally on next instruction)
3721 if (__kmp_env_consistency_check)
3722 __kmp_pop_sync(global_tid, ct_reduce, loc);
3723
3724 } else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3725 tree_reduce_block)) {
3726
3727// AT: performance issue: a real barrier here
3728// AT: (if primary thread is slow, other threads are blocked here waiting for
3729// the primary thread to come and release them)
3730// AT: (it's not what a customer might expect specifying NOWAIT clause)
3731// AT: (specifying NOWAIT won't result in improvement of performance, it'll
3732// be confusing to a customer)
3733// AT: another implementation of *barrier_gather*nowait() (or some other design)
3734// might go faster and be more in line with sense of NOWAIT
3735// AT: TO DO: do epcc test and compare times
3736
3737// this barrier should be invisible to a customer and to the threading profile
3738// tool (it's neither a terminating barrier nor customer's code, it's
3739// used for an internal purpose)
3740#if OMPT_SUPPORT
3741 // JP: can this barrier potentially leed to task scheduling?
3742 // JP: as long as there is a barrier in the implementation, OMPT should and
3743 // will provide the barrier events
3744 // so we set-up the necessary frame/return addresses.
3745 ompt_frame_t *ompt_frame;
3746 if (ompt_enabled.enabled) {
3747 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3748 if (ompt_frame->enter_frame.ptr == NULL)
3749 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
3750 }
3751 OMPT_STORE_RETURN_ADDRESS(global_tid);
3752#endif
3753#if USE_ITT_NOTIFY
3754 __kmp_threads[global_tid]->th.th_ident = loc;
3755#endif
3756 retval =
3757 __kmp_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
3758 global_tid, FALSE, reduce_size, reduce_data, reduce_func);
3759 retval = (retval != 0) ? (0) : (1);
3760#if OMPT_SUPPORT && OMPT_OPTIONAL
3761 if (ompt_enabled.enabled) {
3762 ompt_frame->enter_frame = ompt_data_none;
3763 }
3764#endif
3765
3766 // all other workers except primary thread should do this pop here
3767 // ( none of other workers will get to __kmpc_end_reduce_nowait() )
3768 if (__kmp_env_consistency_check) {
3769 if (retval == 0) {
3770 __kmp_pop_sync(global_tid, ct_reduce, loc);
3771 }
3772 }
3773
3774 } else {
3775
3776 // should never reach this block
3777 KMP_ASSERT(0); // "unexpected method"
3778 }
3779 if (teams_swapped) {
3780 __kmp_restore_swapped_teams(th, team, task_state);
3781 }
3782 KA_TRACE(
3783 10,
3784 ("__kmpc_reduce_nowait() exit: called T#%d: method %08x, returns %08x\n",
3785 global_tid, packed_reduction_method, retval));
3786
3787 return retval;
3788}
3789
3798void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid,
3799 kmp_critical_name *lck) {
3800
3801 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3802
3803 KA_TRACE(10, ("__kmpc_end_reduce_nowait() enter: called T#%d\n", global_tid));
3804 __kmp_assert_valid_gtid(global_tid);
3805
3806 packed_reduction_method = __KMP_GET_REDUCTION_METHOD(global_tid);
3807
3808 OMPT_REDUCTION_DECL(__kmp_thread_from_gtid(global_tid), global_tid);
3809
3810 if (packed_reduction_method == critical_reduce_block) {
3811
3812 __kmp_end_critical_section_reduce_block(loc, global_tid, lck);
3813 OMPT_REDUCTION_END;
3814
3815 } else if (packed_reduction_method == empty_reduce_block) {
3816
3817 // usage: if team size == 1, no synchronization is required ( on Intel
3818 // platforms only )
3819
3820 OMPT_REDUCTION_END;
3821
3822 } else if (packed_reduction_method == atomic_reduce_block) {
3823
3824 // neither primary thread nor other workers should get here
3825 // (code gen does not generate this call in case 2: atomic reduce block)
3826 // actually it's better to remove this elseif at all;
3827 // after removal this value will checked by the 'else' and will assert
3828
3829 } else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3830 tree_reduce_block)) {
3831
3832 // only primary thread gets here
3833 // OMPT: tree reduction is annotated in the barrier code
3834
3835 } else {
3836
3837 // should never reach this block
3838 KMP_ASSERT(0); // "unexpected method"
3839 }
3840
3841 if (__kmp_env_consistency_check)
3842 __kmp_pop_sync(global_tid, ct_reduce, loc);
3843
3844 KA_TRACE(10, ("__kmpc_end_reduce_nowait() exit: called T#%d: method %08x\n",
3845 global_tid, packed_reduction_method));
3846
3847 return;
3848}
3849
3850/* 2.a.ii. Reduce Block with a terminating barrier */
3851
3867kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars,
3868 size_t reduce_size, void *reduce_data,
3869 void (*reduce_func)(void *lhs_data, void *rhs_data),
3870 kmp_critical_name *lck) {
3871 KMP_COUNT_BLOCK(REDUCE_wait);
3872 int retval = 0;
3873 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3874 kmp_info_t *th;
3875 kmp_team_t *team;
3876 int teams_swapped = 0, task_state;
3877
3878 KA_TRACE(10, ("__kmpc_reduce() enter: called T#%d\n", global_tid));
3879 __kmp_assert_valid_gtid(global_tid);
3880
3881 // why do we need this initialization here at all?
3882 // Reduction clause can not be a stand-alone directive.
3883
3884 // do not call __kmp_serial_initialize(), it will be called by
3885 // __kmp_parallel_initialize() if needed
3886 // possible detection of false-positive race by the threadchecker ???
3887 if (!TCR_4(__kmp_init_parallel))
3888 __kmp_parallel_initialize();
3889
3890 __kmp_resume_if_soft_paused();
3891
3892// check correctness of reduce block nesting
3893#if KMP_USE_DYNAMIC_LOCK
3894 if (__kmp_env_consistency_check)
3895 __kmp_push_sync(global_tid, ct_reduce, loc, NULL, 0);
3896#else
3897 if (__kmp_env_consistency_check)
3898 __kmp_push_sync(global_tid, ct_reduce, loc, NULL);
3899#endif
3900
3901 th = __kmp_thread_from_gtid(global_tid);
3902 teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state);
3903
3904 packed_reduction_method = __kmp_determine_reduction_method(
3905 loc, global_tid, num_vars, reduce_size, reduce_data, reduce_func, lck);
3906 __KMP_SET_REDUCTION_METHOD(global_tid, packed_reduction_method);
3907
3908 OMPT_REDUCTION_DECL(th, global_tid);
3909
3910 if (packed_reduction_method == critical_reduce_block) {
3911
3912 OMPT_REDUCTION_BEGIN;
3913 __kmp_enter_critical_section_reduce_block(loc, global_tid, lck);
3914 retval = 1;
3915
3916 } else if (packed_reduction_method == empty_reduce_block) {
3917
3918 OMPT_REDUCTION_BEGIN;
3919 // usage: if team size == 1, no synchronization is required ( Intel
3920 // platforms only )
3921 retval = 1;
3922
3923 } else if (packed_reduction_method == atomic_reduce_block) {
3924
3925 retval = 2;
3926
3927 } else if (TEST_REDUCTION_METHOD(packed_reduction_method,
3928 tree_reduce_block)) {
3929
3930// case tree_reduce_block:
3931// this barrier should be visible to a customer and to the threading profile
3932// tool (it's a terminating barrier on constructs if NOWAIT not specified)
3933#if OMPT_SUPPORT
3934 ompt_frame_t *ompt_frame;
3935 if (ompt_enabled.enabled) {
3936 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
3937 if (ompt_frame->enter_frame.ptr == NULL)
3938 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
3939 }
3940 OMPT_STORE_RETURN_ADDRESS(global_tid);
3941#endif
3942#if USE_ITT_NOTIFY
3943 __kmp_threads[global_tid]->th.th_ident =
3944 loc; // needed for correct notification of frames
3945#endif
3946 retval =
3947 __kmp_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
3948 global_tid, TRUE, reduce_size, reduce_data, reduce_func);
3949 retval = (retval != 0) ? (0) : (1);
3950#if OMPT_SUPPORT && OMPT_OPTIONAL
3951 if (ompt_enabled.enabled) {
3952 ompt_frame->enter_frame = ompt_data_none;
3953 }
3954#endif
3955
3956 // all other workers except primary thread should do this pop here
3957 // (none of other workers except primary will enter __kmpc_end_reduce())
3958 if (__kmp_env_consistency_check) {
3959 if (retval == 0) { // 0: all other workers; 1: primary thread
3960 __kmp_pop_sync(global_tid, ct_reduce, loc);
3961 }
3962 }
3963
3964 } else {
3965
3966 // should never reach this block
3967 KMP_ASSERT(0); // "unexpected method"
3968 }
3969 if (teams_swapped) {
3970 __kmp_restore_swapped_teams(th, team, task_state);
3971 }
3972
3973 KA_TRACE(10,
3974 ("__kmpc_reduce() exit: called T#%d: method %08x, returns %08x\n",
3975 global_tid, packed_reduction_method, retval));
3976 return retval;
3977}
3978
3989void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid,
3990 kmp_critical_name *lck) {
3991
3992 PACKED_REDUCTION_METHOD_T packed_reduction_method;
3993 kmp_info_t *th;
3994 kmp_team_t *team;
3995 int teams_swapped = 0, task_state;
3996
3997 KA_TRACE(10, ("__kmpc_end_reduce() enter: called T#%d\n", global_tid));
3998 __kmp_assert_valid_gtid(global_tid);
3999
4000 th = __kmp_thread_from_gtid(global_tid);
4001 teams_swapped = __kmp_swap_teams_for_teams_reduction(th, &team, &task_state);
4002
4003 packed_reduction_method = __KMP_GET_REDUCTION_METHOD(global_tid);
4004
4005 // this barrier should be visible to a customer and to the threading profile
4006 // tool (it's a terminating barrier on constructs if NOWAIT not specified)
4007 OMPT_REDUCTION_DECL(th, global_tid);
4008
4009 if (packed_reduction_method == critical_reduce_block) {
4010 __kmp_end_critical_section_reduce_block(loc, global_tid, lck);
4011
4012 OMPT_REDUCTION_END;
4013
4014// TODO: implicit barrier: should be exposed
4015#if OMPT_SUPPORT
4016 ompt_frame_t *ompt_frame;
4017 if (ompt_enabled.enabled) {
4018 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
4019 if (ompt_frame->enter_frame.ptr == NULL)
4020 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
4021 }
4022 OMPT_STORE_RETURN_ADDRESS(global_tid);
4023#endif
4024#if USE_ITT_NOTIFY
4025 __kmp_threads[global_tid]->th.th_ident = loc;
4026#endif
4027 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
4028#if OMPT_SUPPORT && OMPT_OPTIONAL
4029 if (ompt_enabled.enabled) {
4030 ompt_frame->enter_frame = ompt_data_none;
4031 }
4032#endif
4033
4034 } else if (packed_reduction_method == empty_reduce_block) {
4035
4036 OMPT_REDUCTION_END;
4037
4038// usage: if team size==1, no synchronization is required (Intel platforms only)
4039
4040// TODO: implicit barrier: should be exposed
4041#if OMPT_SUPPORT
4042 ompt_frame_t *ompt_frame;
4043 if (ompt_enabled.enabled) {
4044 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
4045 if (ompt_frame->enter_frame.ptr == NULL)
4046 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
4047 }
4048 OMPT_STORE_RETURN_ADDRESS(global_tid);
4049#endif
4050#if USE_ITT_NOTIFY
4051 __kmp_threads[global_tid]->th.th_ident = loc;
4052#endif
4053 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
4054#if OMPT_SUPPORT && OMPT_OPTIONAL
4055 if (ompt_enabled.enabled) {
4056 ompt_frame->enter_frame = ompt_data_none;
4057 }
4058#endif
4059
4060 } else if (packed_reduction_method == atomic_reduce_block) {
4061
4062#if OMPT_SUPPORT
4063 ompt_frame_t *ompt_frame;
4064 if (ompt_enabled.enabled) {
4065 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL);
4066 if (ompt_frame->enter_frame.ptr == NULL)
4067 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
4068 }
4069 OMPT_STORE_RETURN_ADDRESS(global_tid);
4070#endif
4071// TODO: implicit barrier: should be exposed
4072#if USE_ITT_NOTIFY
4073 __kmp_threads[global_tid]->th.th_ident = loc;
4074#endif
4075 __kmp_barrier(bs_plain_barrier, global_tid, FALSE, 0, NULL, NULL);
4076#if OMPT_SUPPORT && OMPT_OPTIONAL
4077 if (ompt_enabled.enabled) {
4078 ompt_frame->enter_frame = ompt_data_none;
4079 }
4080#endif
4081
4082 } else if (TEST_REDUCTION_METHOD(packed_reduction_method,
4083 tree_reduce_block)) {
4084
4085 // only primary thread executes here (primary releases all other workers)
4086 __kmp_end_split_barrier(UNPACK_REDUCTION_BARRIER(packed_reduction_method),
4087 global_tid);
4088
4089 } else {
4090
4091 // should never reach this block
4092 KMP_ASSERT(0); // "unexpected method"
4093 }
4094 if (teams_swapped) {
4095 __kmp_restore_swapped_teams(th, team, task_state);
4096 }
4097
4098 if (__kmp_env_consistency_check)
4099 __kmp_pop_sync(global_tid, ct_reduce, loc);
4100
4101 KA_TRACE(10, ("__kmpc_end_reduce() exit: called T#%d: method %08x\n",
4102 global_tid, packed_reduction_method));
4103
4104 return;
4105}
4106
4107#undef __KMP_GET_REDUCTION_METHOD
4108#undef __KMP_SET_REDUCTION_METHOD
4109
4110/* end of interface to fast scalable reduce routines */
4111
4112kmp_uint64 __kmpc_get_taskid() {
4113
4114 kmp_int32 gtid;
4115 kmp_info_t *thread;
4116
4117 gtid = __kmp_get_gtid();
4118 if (gtid < 0) {
4119 return 0;
4120 }
4121 thread = __kmp_thread_from_gtid(gtid);
4122 return thread->th.th_current_task->td_task_id;
4123
4124} // __kmpc_get_taskid
4125
4126kmp_uint64 __kmpc_get_parent_taskid() {
4127
4128 kmp_int32 gtid;
4129 kmp_info_t *thread;
4130 kmp_taskdata_t *parent_task;
4131
4132 gtid = __kmp_get_gtid();
4133 if (gtid < 0) {
4134 return 0;
4135 }
4136 thread = __kmp_thread_from_gtid(gtid);
4137 parent_task = thread->th.th_current_task->td_parent;
4138 return (parent_task == NULL ? 0 : parent_task->td_task_id);
4139
4140} // __kmpc_get_parent_taskid
4141
4153void __kmpc_doacross_init(ident_t *loc, int gtid, int num_dims,
4154 const struct kmp_dim *dims) {
4155 __kmp_assert_valid_gtid(gtid);
4156 int j, idx;
4157 kmp_int64 last, trace_count;
4158 kmp_info_t *th = __kmp_threads[gtid];
4159 kmp_team_t *team = th->th.th_team;
4160 kmp_uint32 *flags;
4161 kmp_disp_t *pr_buf = th->th.th_dispatch;
4162 dispatch_shared_info_t *sh_buf;
4163
4164 KA_TRACE(
4165 20,
4166 ("__kmpc_doacross_init() enter: called T#%d, num dims %d, active %d\n",
4167 gtid, num_dims, !team->t.t_serialized));
4168 KMP_DEBUG_ASSERT(dims != NULL);
4169 KMP_DEBUG_ASSERT(num_dims > 0);
4170
4171 if (team->t.t_serialized) {
4172 KA_TRACE(20, ("__kmpc_doacross_init() exit: serialized team\n"));
4173 return; // no dependencies if team is serialized
4174 }
4175 KMP_DEBUG_ASSERT(team->t.t_nproc > 1);
4176 idx = pr_buf->th_doacross_buf_idx++; // Increment index of shared buffer for
4177 // the next loop
4178 sh_buf = &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers];
4179
4180 // Save bounds info into allocated private buffer
4181 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info == NULL);
4182 pr_buf->th_doacross_info = (kmp_int64 *)__kmp_thread_malloc(
4183 th, sizeof(kmp_int64) * (4 * num_dims + 1));
4184 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
4185 pr_buf->th_doacross_info[0] =
4186 (kmp_int64)num_dims; // first element is number of dimensions
4187 // Save also address of num_done in order to access it later without knowing
4188 // the buffer index
4189 pr_buf->th_doacross_info[1] = (kmp_int64)&sh_buf->doacross_num_done;
4190 pr_buf->th_doacross_info[2] = dims[0].lo;
4191 pr_buf->th_doacross_info[3] = dims[0].up;
4192 pr_buf->th_doacross_info[4] = dims[0].st;
4193 last = 5;
4194 for (j = 1; j < num_dims; ++j) {
4195 kmp_int64
4196 range_length; // To keep ranges of all dimensions but the first dims[0]
4197 if (dims[j].st == 1) { // most common case
4198 // AC: should we care of ranges bigger than LLONG_MAX? (not for now)
4199 range_length = dims[j].up - dims[j].lo + 1;
4200 } else {
4201 if (dims[j].st > 0) {
4202 KMP_DEBUG_ASSERT(dims[j].up > dims[j].lo);
4203 range_length = (kmp_uint64)(dims[j].up - dims[j].lo) / dims[j].st + 1;
4204 } else { // negative increment
4205 KMP_DEBUG_ASSERT(dims[j].lo > dims[j].up);
4206 range_length =
4207 (kmp_uint64)(dims[j].lo - dims[j].up) / (-dims[j].st) + 1;
4208 }
4209 }
4210 pr_buf->th_doacross_info[last++] = range_length;
4211 pr_buf->th_doacross_info[last++] = dims[j].lo;
4212 pr_buf->th_doacross_info[last++] = dims[j].up;
4213 pr_buf->th_doacross_info[last++] = dims[j].st;
4214 }
4215
4216 // Compute total trip count.
4217 // Start with range of dims[0] which we don't need to keep in the buffer.
4218 if (dims[0].st == 1) { // most common case
4219 trace_count = dims[0].up - dims[0].lo + 1;
4220 } else if (dims[0].st > 0) {
4221 KMP_DEBUG_ASSERT(dims[0].up > dims[0].lo);
4222 trace_count = (kmp_uint64)(dims[0].up - dims[0].lo) / dims[0].st + 1;
4223 } else { // negative increment
4224 KMP_DEBUG_ASSERT(dims[0].lo > dims[0].up);
4225 trace_count = (kmp_uint64)(dims[0].lo - dims[0].up) / (-dims[0].st) + 1;
4226 }
4227 for (j = 1; j < num_dims; ++j) {
4228 trace_count *= pr_buf->th_doacross_info[4 * j + 1]; // use kept ranges
4229 }
4230 KMP_DEBUG_ASSERT(trace_count > 0);
4231
4232 // Check if shared buffer is not occupied by other loop (idx -
4233 // __kmp_dispatch_num_buffers)
4234 if (idx != sh_buf->doacross_buf_idx) {
4235 // Shared buffer is occupied, wait for it to be free
4236 __kmp_wait_4((volatile kmp_uint32 *)&sh_buf->doacross_buf_idx, idx,
4237 __kmp_eq_4, NULL);
4238 }
4239#if KMP_32_BIT_ARCH
4240 // Check if we are the first thread. After the CAS the first thread gets 0,
4241 // others get 1 if initialization is in progress, allocated pointer otherwise.
4242 // Treat pointer as volatile integer (value 0 or 1) until memory is allocated.
4243 flags = (kmp_uint32 *)KMP_COMPARE_AND_STORE_RET32(
4244 (volatile kmp_int32 *)&sh_buf->doacross_flags, NULL, 1);
4245#else
4246 flags = (kmp_uint32 *)KMP_COMPARE_AND_STORE_RET64(
4247 (volatile kmp_int64 *)&sh_buf->doacross_flags, NULL, 1LL);
4248#endif
4249 if (flags == NULL) {
4250 // we are the first thread, allocate the array of flags
4251 size_t size =
4252 (size_t)trace_count / 8 + 8; // in bytes, use single bit per iteration
4253 flags = (kmp_uint32 *)__kmp_thread_calloc(th, size, 1);
4254 KMP_MB();
4255 sh_buf->doacross_flags = flags;
4256 } else if (flags == (kmp_uint32 *)1) {
4257#if KMP_32_BIT_ARCH
4258 // initialization is still in progress, need to wait
4259 while (*(volatile kmp_int32 *)&sh_buf->doacross_flags == 1)
4260#else
4261 while (*(volatile kmp_int64 *)&sh_buf->doacross_flags == 1LL)
4262#endif
4263 KMP_YIELD(TRUE);
4264 KMP_MB();
4265 } else {
4266 KMP_MB();
4267 }
4268 KMP_DEBUG_ASSERT(sh_buf->doacross_flags > (kmp_uint32 *)1); // check ptr value
4269 pr_buf->th_doacross_flags =
4270 sh_buf->doacross_flags; // save private copy in order to not
4271 // touch shared buffer on each iteration
4272 KA_TRACE(20, ("__kmpc_doacross_init() exit: T#%d\n", gtid));
4273}
4274
4275void __kmpc_doacross_wait(ident_t *loc, int gtid, const kmp_int64 *vec) {
4276 __kmp_assert_valid_gtid(gtid);
4277 kmp_int64 shft;
4278 size_t num_dims, i;
4279 kmp_uint32 flag;
4280 kmp_int64 iter_number; // iteration number of "collapsed" loop nest
4281 kmp_info_t *th = __kmp_threads[gtid];
4282 kmp_team_t *team = th->th.th_team;
4283 kmp_disp_t *pr_buf;
4284 kmp_int64 lo, up, st;
4285
4286 KA_TRACE(20, ("__kmpc_doacross_wait() enter: called T#%d\n", gtid));
4287 if (team->t.t_serialized) {
4288 KA_TRACE(20, ("__kmpc_doacross_wait() exit: serialized team\n"));
4289 return; // no dependencies if team is serialized
4290 }
4291
4292 // calculate sequential iteration number and check out-of-bounds condition
4293 pr_buf = th->th.th_dispatch;
4294 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
4295 num_dims = (size_t)pr_buf->th_doacross_info[0];
4296 lo = pr_buf->th_doacross_info[2];
4297 up = pr_buf->th_doacross_info[3];
4298 st = pr_buf->th_doacross_info[4];
4299#if OMPT_SUPPORT && OMPT_OPTIONAL
4300 SimpleVLA<ompt_dependence_t> deps(num_dims);
4301#endif
4302 if (st == 1) { // most common case
4303 if (vec[0] < lo || vec[0] > up) {
4304 KA_TRACE(20, ("__kmpc_doacross_wait() exit: T#%d iter %lld is out of "
4305 "bounds [%lld,%lld]\n",
4306 gtid, vec[0], lo, up));
4307 return;
4308 }
4309 iter_number = vec[0] - lo;
4310 } else if (st > 0) {
4311 if (vec[0] < lo || vec[0] > up) {
4312 KA_TRACE(20, ("__kmpc_doacross_wait() exit: T#%d iter %lld is out of "
4313 "bounds [%lld,%lld]\n",
4314 gtid, vec[0], lo, up));
4315 return;
4316 }
4317 iter_number = (kmp_uint64)(vec[0] - lo) / st;
4318 } else { // negative increment
4319 if (vec[0] > lo || vec[0] < up) {
4320 KA_TRACE(20, ("__kmpc_doacross_wait() exit: T#%d iter %lld is out of "
4321 "bounds [%lld,%lld]\n",
4322 gtid, vec[0], lo, up));
4323 return;
4324 }
4325 iter_number = (kmp_uint64)(lo - vec[0]) / (-st);
4326 }
4327#if OMPT_SUPPORT && OMPT_OPTIONAL
4328 deps[0].variable.value = iter_number;
4329 deps[0].dependence_type = ompt_dependence_type_sink;
4330#endif
4331 for (i = 1; i < num_dims; ++i) {
4332 kmp_int64 iter, ln;
4333 size_t j = i * 4;
4334 ln = pr_buf->th_doacross_info[j + 1];
4335 lo = pr_buf->th_doacross_info[j + 2];
4336 up = pr_buf->th_doacross_info[j + 3];
4337 st = pr_buf->th_doacross_info[j + 4];
4338 if (st == 1) {
4339 if (vec[i] < lo || vec[i] > up) {
4340 KA_TRACE(20, ("__kmpc_doacross_wait() exit: T#%d iter %lld is out of "
4341 "bounds [%lld,%lld]\n",
4342 gtid, vec[i], lo, up));
4343 return;
4344 }
4345 iter = vec[i] - lo;
4346 } else if (st > 0) {
4347 if (vec[i] < lo || vec[i] > up) {
4348 KA_TRACE(20, ("__kmpc_doacross_wait() exit: T#%d iter %lld is out of "
4349 "bounds [%lld,%lld]\n",
4350 gtid, vec[i], lo, up));
4351 return;
4352 }
4353 iter = (kmp_uint64)(vec[i] - lo) / st;
4354 } else { // st < 0
4355 if (vec[i] > lo || vec[i] < up) {
4356 KA_TRACE(20, ("__kmpc_doacross_wait() exit: T#%d iter %lld is out of "
4357 "bounds [%lld,%lld]\n",
4358 gtid, vec[i], lo, up));
4359 return;
4360 }
4361 iter = (kmp_uint64)(lo - vec[i]) / (-st);
4362 }
4363 iter_number = iter + ln * iter_number;
4364#if OMPT_SUPPORT && OMPT_OPTIONAL
4365 deps[i].variable.value = iter;
4366 deps[i].dependence_type = ompt_dependence_type_sink;
4367#endif
4368 }
4369 shft = iter_number % 32; // use 32-bit granularity
4370 iter_number >>= 5; // divided by 32
4371 flag = 1 << shft;
4372 while ((flag & pr_buf->th_doacross_flags[iter_number]) == 0) {
4373 KMP_YIELD(TRUE);
4374 }
4375 KMP_MB();
4376#if OMPT_SUPPORT && OMPT_OPTIONAL
4377 if (ompt_enabled.ompt_callback_dependences) {
4378 ompt_callbacks.ompt_callback(ompt_callback_dependences)(
4379 &(OMPT_CUR_TASK_INFO(th)->task_data), deps, (kmp_uint32)num_dims);
4380 }
4381#endif
4382 KA_TRACE(20,
4383 ("__kmpc_doacross_wait() exit: T#%d wait for iter %lld completed\n",
4384 gtid, (iter_number << 5) + shft));
4385}
4386
4387void __kmpc_doacross_post(ident_t *loc, int gtid, const kmp_int64 *vec) {
4388 __kmp_assert_valid_gtid(gtid);
4389 kmp_int64 shft;
4390 size_t num_dims, i;
4391 kmp_uint32 flag;
4392 kmp_int64 iter_number; // iteration number of "collapsed" loop nest
4393 kmp_info_t *th = __kmp_threads[gtid];
4394 kmp_team_t *team = th->th.th_team;
4395 kmp_disp_t *pr_buf;
4396 kmp_int64 lo, st;
4397
4398 KA_TRACE(20, ("__kmpc_doacross_post() enter: called T#%d\n", gtid));
4399 if (team->t.t_serialized) {
4400 KA_TRACE(20, ("__kmpc_doacross_post() exit: serialized team\n"));
4401 return; // no dependencies if team is serialized
4402 }
4403
4404 // calculate sequential iteration number (same as in "wait" but no
4405 // out-of-bounds checks)
4406 pr_buf = th->th.th_dispatch;
4407 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info != NULL);
4408 num_dims = (size_t)pr_buf->th_doacross_info[0];
4409 lo = pr_buf->th_doacross_info[2];
4410 st = pr_buf->th_doacross_info[4];
4411#if OMPT_SUPPORT && OMPT_OPTIONAL
4412 SimpleVLA<ompt_dependence_t> deps(num_dims);
4413#endif
4414 if (st == 1) { // most common case
4415 iter_number = vec[0] - lo;
4416 } else if (st > 0) {
4417 iter_number = (kmp_uint64)(vec[0] - lo) / st;
4418 } else { // negative increment
4419 iter_number = (kmp_uint64)(lo - vec[0]) / (-st);
4420 }
4421#if OMPT_SUPPORT && OMPT_OPTIONAL
4422 deps[0].variable.value = iter_number;
4423 deps[0].dependence_type = ompt_dependence_type_source;
4424#endif
4425 for (i = 1; i < num_dims; ++i) {
4426 kmp_int64 iter, ln;
4427 size_t j = i * 4;
4428 ln = pr_buf->th_doacross_info[j + 1];
4429 lo = pr_buf->th_doacross_info[j + 2];
4430 st = pr_buf->th_doacross_info[j + 4];
4431 if (st == 1) {
4432 iter = vec[i] - lo;
4433 } else if (st > 0) {
4434 iter = (kmp_uint64)(vec[i] - lo) / st;
4435 } else { // st < 0
4436 iter = (kmp_uint64)(lo - vec[i]) / (-st);
4437 }
4438 iter_number = iter + ln * iter_number;
4439#if OMPT_SUPPORT && OMPT_OPTIONAL
4440 deps[i].variable.value = iter;
4441 deps[i].dependence_type = ompt_dependence_type_source;
4442#endif
4443 }
4444#if OMPT_SUPPORT && OMPT_OPTIONAL
4445 if (ompt_enabled.ompt_callback_dependences) {
4446 ompt_callbacks.ompt_callback(ompt_callback_dependences)(
4447 &(OMPT_CUR_TASK_INFO(th)->task_data), deps, (kmp_uint32)num_dims);
4448 }
4449#endif
4450 shft = iter_number % 32; // use 32-bit granularity
4451 iter_number >>= 5; // divided by 32
4452 flag = 1 << shft;
4453 KMP_MB();
4454 if ((flag & pr_buf->th_doacross_flags[iter_number]) == 0)
4455 KMP_TEST_THEN_OR32(&pr_buf->th_doacross_flags[iter_number], flag);
4456 KA_TRACE(20, ("__kmpc_doacross_post() exit: T#%d iter %lld posted\n", gtid,
4457 (iter_number << 5) + shft));
4458}
4459
4460void __kmpc_doacross_fini(ident_t *loc, int gtid) {
4461 __kmp_assert_valid_gtid(gtid);
4462 kmp_int32 num_done;
4463 kmp_info_t *th = __kmp_threads[gtid];
4464 kmp_team_t *team = th->th.th_team;
4465 kmp_disp_t *pr_buf = th->th.th_dispatch;
4466
4467 KA_TRACE(20, ("__kmpc_doacross_fini() enter: called T#%d\n", gtid));
4468 if (team->t.t_serialized) {
4469 KA_TRACE(20, ("__kmpc_doacross_fini() exit: serialized team %p\n", team));
4470 return; // nothing to do
4471 }
4472 num_done =
4473 KMP_TEST_THEN_INC32((kmp_uintptr_t)(pr_buf->th_doacross_info[1])) + 1;
4474 if (num_done == th->th.th_team_nproc) {
4475 // we are the last thread, need to free shared resources
4476 int idx = pr_buf->th_doacross_buf_idx - 1;
4477 dispatch_shared_info_t *sh_buf =
4478 &team->t.t_disp_buffer[idx % __kmp_dispatch_num_buffers];
4479 KMP_DEBUG_ASSERT(pr_buf->th_doacross_info[1] ==
4480 (kmp_int64)&sh_buf->doacross_num_done);
4481 KMP_DEBUG_ASSERT(num_done == sh_buf->doacross_num_done);
4482 KMP_DEBUG_ASSERT(idx == sh_buf->doacross_buf_idx);
4483 __kmp_thread_free(th, CCAST(kmp_uint32 *, sh_buf->doacross_flags));
4484 sh_buf->doacross_flags = NULL;
4485 sh_buf->doacross_num_done = 0;
4486 sh_buf->doacross_buf_idx +=
4487 __kmp_dispatch_num_buffers; // free buffer for future re-use
4488 }
4489 // free private resources (need to keep buffer index forever)
4490 pr_buf->th_doacross_flags = NULL;
4491 __kmp_thread_free(th, (void *)pr_buf->th_doacross_info);
4492 pr_buf->th_doacross_info = NULL;
4493 KA_TRACE(20, ("__kmpc_doacross_fini() exit: T#%d\n", gtid));
4494}
4495
4496/* OpenMP 5.1 Memory Management routines */
4497void *omp_alloc(size_t size, omp_allocator_handle_t allocator) {
4498 return __kmp_alloc(__kmp_entry_gtid(), 0, size, allocator);
4499}
4500
4501void *omp_aligned_alloc(size_t align, size_t size,
4502 omp_allocator_handle_t allocator) {
4503 return __kmp_alloc(__kmp_entry_gtid(), align, size, allocator);
4504}
4505
4506void *omp_calloc(size_t nmemb, size_t size, omp_allocator_handle_t allocator) {
4507 return __kmp_calloc(__kmp_entry_gtid(), 0, nmemb, size, allocator);
4508}
4509
4510void *omp_aligned_calloc(size_t align, size_t nmemb, size_t size,
4511 omp_allocator_handle_t allocator) {
4512 return __kmp_calloc(__kmp_entry_gtid(), align, nmemb, size, allocator);
4513}
4514
4515void *omp_realloc(void *ptr, size_t size, omp_allocator_handle_t allocator,
4516 omp_allocator_handle_t free_allocator) {
4517 return __kmp_realloc(__kmp_entry_gtid(), ptr, size, allocator,
4518 free_allocator);
4519}
4520
4521void omp_free(void *ptr, omp_allocator_handle_t allocator) {
4522 ___kmpc_free(__kmp_entry_gtid(), ptr, allocator);
4523}
4524/* end of OpenMP 5.1 Memory Management routines */
4525
4526void *omp_get_dyn_gprivate_ptr(size_t offset, omp_access_t access_group) {
4527 return NULL;
4528}
4529
4530void *omp_get_dyn_gprivate_nofb_ptr(size_t offset, omp_access_t access_group) {
4531 return NULL;
4532}
4533
4534size_t omp_get_dyn_gprivate_size(omp_access_t access_group) { return 0; }
4535
4536omp_memspace_handle_t omp_get_dyn_gprivate_memspace(omp_access_t access_group) {
4537 return omp_null_mem_space;
4538}
4539
4540int __kmpc_get_target_offload(void) {
4541 if (!__kmp_init_serial) {
4542 __kmp_serial_initialize();
4543 }
4544 return __kmp_target_offload;
4545}
4546
4547int __kmpc_pause_resource(kmp_pause_status_t level) {
4548 if (!__kmp_init_serial) {
4549 return 1; // Can't pause if runtime is not initialized
4550 }
4551 return __kmp_pause_resource(level);
4552}
4553
4554void __kmpc_error(ident_t *loc, int severity, const char *message) {
4555 if (!__kmp_init_serial)
4556 __kmp_serial_initialize();
4557
4558 KMP_ASSERT(severity == severity_warning || severity == severity_fatal);
4559
4560#if OMPT_SUPPORT
4561 if (ompt_enabled.enabled && ompt_enabled.ompt_callback_error) {
4562 ompt_callbacks.ompt_callback(ompt_callback_error)(
4563 (ompt_severity_t)severity, message, KMP_STRLEN(message),
4564 OMPT_GET_RETURN_ADDRESS(0));
4565 }
4566#endif // OMPT_SUPPORT
4567
4568 char *src_loc;
4569 if (loc && loc->psource) {
4570 kmp_str_loc_t str_loc = __kmp_str_loc_init(loc->psource, false);
4571 src_loc =
4572 __kmp_str_format("%s:%d:%d", str_loc.file, str_loc.line, str_loc.col);
4573 __kmp_str_loc_free(&str_loc);
4574 } else {
4575 src_loc = __kmp_str_format("unknown");
4576 }
4577
4578 if (severity == severity_warning)
4579 KMP_WARNING(UserDirectedWarning, src_loc, message);
4580 else
4581 KMP_FATAL(UserDirectedError, src_loc, message);
4582
4583 __kmp_str_free(&src_loc);
4584}
4585
4586// Mark begin of scope directive.
4587void __kmpc_scope(ident_t *loc, kmp_int32 gtid, void *reserved) {
4588// reserved is for extension of scope directive and not used.
4589#if OMPT_SUPPORT && OMPT_OPTIONAL
4590 if (ompt_enabled.enabled && ompt_enabled.ompt_callback_work) {
4591 kmp_team_t *team = __kmp_threads[gtid]->th.th_team;
4592 int tid = __kmp_tid_from_gtid(gtid);
4593 ompt_callbacks.ompt_callback(ompt_callback_work)(
4594 ompt_work_scope, ompt_scope_begin,
4595 &(team->t.ompt_team_info.parallel_data),
4596 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 1,
4597 OMPT_GET_RETURN_ADDRESS(0));
4598 }
4599#endif // OMPT_SUPPORT && OMPT_OPTIONAL
4600}
4601
4602// Mark end of scope directive
4603void __kmpc_end_scope(ident_t *loc, kmp_int32 gtid, void *reserved) {
4604// reserved is for extension of scope directive and not used.
4605#if OMPT_SUPPORT && OMPT_OPTIONAL
4606 if (ompt_enabled.enabled && ompt_enabled.ompt_callback_work) {
4607 kmp_team_t *team = __kmp_threads[gtid]->th.th_team;
4608 int tid = __kmp_tid_from_gtid(gtid);
4609 ompt_callbacks.ompt_callback(ompt_callback_work)(
4610 ompt_work_scope, ompt_scope_end,
4611 &(team->t.ompt_team_info.parallel_data),
4612 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 1,
4613 OMPT_GET_RETURN_ADDRESS(0));
4614 }
4615#endif // OMPT_SUPPORT && OMPT_OPTIONAL
4616}
4617
4618#ifdef KMP_USE_VERSION_SYMBOLS
4619// For GOMP compatibility there are two versions of each omp_* API.
4620// One is the plain C symbol and one is the Fortran symbol with an appended
4621// underscore. When we implement a specific ompc_* version of an omp_*
4622// function, we want the plain GOMP versioned symbol to alias the ompc_* version
4623// instead of the Fortran versions in kmp_ftn_entry.h
4624extern "C" {
4625// Have to undef these from omp.h so they aren't translated into
4626// their ompc counterparts in the KMP_VERSION_OMPC_SYMBOL macros below
4627#ifdef omp_set_affinity_format
4628#undef omp_set_affinity_format
4629#endif
4630#ifdef omp_get_affinity_format
4631#undef omp_get_affinity_format
4632#endif
4633#ifdef omp_display_affinity
4634#undef omp_display_affinity
4635#endif
4636#ifdef omp_capture_affinity
4637#undef omp_capture_affinity
4638#endif
4639KMP_VERSION_OMPC_SYMBOL(ompc_set_affinity_format, omp_set_affinity_format, 50,
4640 "OMP_5.0");
4641KMP_VERSION_OMPC_SYMBOL(ompc_get_affinity_format, omp_get_affinity_format, 50,
4642 "OMP_5.0");
4643KMP_VERSION_OMPC_SYMBOL(ompc_display_affinity, omp_display_affinity, 50,
4644 "OMP_5.0");
4645KMP_VERSION_OMPC_SYMBOL(ompc_capture_affinity, omp_capture_affinity, 50,
4646 "OMP_5.0");
4647} // extern "C"
4648#endif
KMP_IDENT_WORK_LOOP
@ KMP_IDENT_WORK_LOOP
Definition kmp.h:197
KMP_IDENT_WORK_SECTIONS
@ KMP_IDENT_WORK_SECTIONS
Definition kmp.h:199
KMP_IDENT_AUTOPAR
@ KMP_IDENT_AUTOPAR
Definition kmp.h:182
KMP_IDENT_WORK_DISTRIBUTE
@ KMP_IDENT_WORK_DISTRIBUTE
Definition kmp.h:201
__kmpc_ok_to_fork
kmp_int32 __kmpc_ok_to_fork(ident_t *loc)
Definition kmp_csupport.cpp:157
__kmpc_fork_teams
void __kmpc_fork_teams(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)
Definition kmp_csupport.cpp:502
__kmpc_fork_call_if
void __kmpc_fork_call_if(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, kmp_int32 cond, void *args)
Definition kmp_csupport.cpp:388
__kmpc_push_num_threads
void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)
Definition kmp_csupport.cpp:232
__kmpc_set_thread_limit
void __kmpc_set_thread_limit(ident_t *loc, kmp_int32 global_tid, kmp_int32 thread_limit)
Definition kmp_csupport.cpp:457
__kmpc_serialized_parallel
void __kmpc_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
Definition kmp_csupport.cpp:603
__kmpc_push_num_threads_list
void __kmpc_push_num_threads_list(ident_t *loc, kmp_int32 global_tid, kmp_uint32 list_length, kmp_int32 *num_threads_list)
Definition kmp_csupport.cpp:260
__kmpc_push_num_teams
void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams, kmp_int32 num_threads)
Definition kmp_csupport.cpp:438
__kmpc_fork_call
void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask,...)
Definition kmp_csupport.cpp:307
__kmpc_end_serialized_parallel
void __kmpc_end_serialized_parallel(ident_t *loc, kmp_int32 global_tid)
Definition kmp_csupport.cpp:621
kmpc_micro
void(* kmpc_micro)(kmp_int32 *global_tid, kmp_int32 *bound_tid,...)
Definition kmp.h:1764
__kmpc_push_num_teams_51
void __kmpc_push_num_teams_51(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams_lb, kmp_int32 num_teams_ub, kmp_int32 num_threads)
Definition kmp_csupport.cpp:481
__kmpc_begin
void __kmpc_begin(ident_t *loc, kmp_int32 flags)
Definition kmp_csupport.cpp:37
__kmpc_end
void __kmpc_end(ident_t *loc)
Definition kmp_csupport.cpp:60
KMP_COUNT_BLOCK
#define KMP_COUNT_BLOCK(name)
Increments specified counter (name).
Definition kmp_stats.h:911
stats_state_e
stats_state_e
the states which a thread can be in
Definition kmp_stats.h:63
__kmpc_end_reduce
void __kmpc_end_reduce(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
Definition kmp_csupport.cpp:3989
__kmpc_end_barrier_master
void __kmpc_end_barrier_master(ident_t *loc, kmp_int32 global_tid)
Definition kmp_csupport.cpp:1844
__kmpc_barrier_master_nowait
kmp_int32 __kmpc_barrier_master_nowait(ident_t *loc, kmp_int32 global_tid)
Definition kmp_csupport.cpp:1860
__kmpc_end_reduce_nowait
void __kmpc_end_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *lck)
Definition kmp_csupport.cpp:3798
__kmpc_reduce
kmp_int32 __kmpc_reduce(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
Definition kmp_csupport.cpp:3867
__kmpc_barrier
void __kmpc_barrier(ident_t *loc, kmp_int32 global_tid)
Definition kmp_csupport.cpp:812
__kmpc_flush
void __kmpc_flush(ident_t *loc)
Definition kmp_csupport.cpp:790
__kmpc_barrier_master
kmp_int32 __kmpc_barrier_master(ident_t *loc, kmp_int32 global_tid)
Definition kmp_csupport.cpp:1800
__kmpc_reduce_nowait
kmp_int32 __kmpc_reduce_nowait(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars, size_t reduce_size, void *reduce_data, void(*reduce_func)(void *lhs_data, void *rhs_data), kmp_critical_name *lck)
Definition kmp_csupport.cpp:3638
__kmpc_copyprivate
void __kmpc_copyprivate(ident_t *loc, kmp_int32 gtid, size_t cpy_size, void *cpy_data, void(*cpy_func)(void *, void *), kmp_int32 didit)
Definition kmp_csupport.cpp:2280
__kmpc_copyprivate_light
void * __kmpc_copyprivate_light(ident_t *loc, kmp_int32 gtid, void *cpy_data)
Definition kmp_csupport.cpp:2356
__kmpc_global_num_threads
kmp_int32 __kmpc_global_num_threads(ident_t *loc)
Definition kmp_csupport.cpp:122
__kmpc_global_thread_num
kmp_int32 __kmpc_global_thread_num(ident_t *loc)
Definition kmp_csupport.cpp:100
__kmpc_in_parallel
kmp_int32 __kmpc_in_parallel(ident_t *loc)
Definition kmp_csupport.cpp:219
__kmpc_bound_thread_num
kmp_int32 __kmpc_bound_thread_num(ident_t *loc)
Definition kmp_csupport.cpp:135
__kmpc_bound_num_threads
kmp_int32 __kmpc_bound_num_threads(ident_t *loc)
Definition kmp_csupport.cpp:145
__kmpc_end_ordered
void __kmpc_end_ordered(ident_t *loc, kmp_int32 gtid)
Definition kmp_csupport.cpp:1108
__kmpc_end_critical
void __kmpc_end_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
Definition kmp_csupport.cpp:1703
__kmpc_for_static_fini
void __kmpc_for_static_fini(ident_t *loc, kmp_int32 global_tid)
Definition kmp_csupport.cpp:2003
__kmpc_end_masked
void __kmpc_end_masked(ident_t *loc, kmp_int32 global_tid)
Definition kmp_csupport.cpp:1006
__kmpc_master
kmp_int32 __kmpc_master(ident_t *loc, kmp_int32 global_tid)
Definition kmp_csupport.cpp:861
__kmpc_single
kmp_int32 __kmpc_single(ident_t *loc, kmp_int32 global_tid)
Definition kmp_csupport.cpp:1923
__kmpc_doacross_init
void __kmpc_doacross_init(ident_t *loc, int gtid, int num_dims, const struct kmp_dim *dims)
Definition kmp_csupport.cpp:4153
__kmpc_end_master
void __kmpc_end_master(ident_t *loc, kmp_int32 global_tid)
Definition kmp_csupport.cpp:918
__kmpc_end_single
void __kmpc_end_single(ident_t *loc, kmp_int32 global_tid)
Definition kmp_csupport.cpp:1976
__kmpc_ordered
void __kmpc_ordered(ident_t *loc, kmp_int32 gtid)
Definition kmp_csupport.cpp:1035
__kmpc_masked
kmp_int32 __kmpc_masked(ident_t *loc, kmp_int32 global_tid, kmp_int32 filter)
Definition kmp_csupport.cpp:950
__kmpc_critical
void __kmpc_critical(ident_t *loc, kmp_int32 global_tid, kmp_critical_name *crit)
Definition kmp_csupport.cpp:1343
ident
Definition kmp.h:217
ident::psource
char const * psource
Definition kmp.h:227
ident::flags
kmp_int32 flags
Definition kmp.h:219
Generated by doxygen 1.9.8