14 #include "kmp_affinity.h"
19 #include "kmp_stats.h"
21 #include "kmp_wait_release.h"
22 #include "kmp_wrapper_getpid.h"
24 #if !KMP_OS_DRAGONFLY && !KMP_OS_FREEBSD && !KMP_OS_NETBSD && !KMP_OS_OPENBSD
29 #include <semaphore.h>
31 #include <sys/resource.h>
34 #include <libperfstat.h>
36 #include <sys/syscall.h>
39 #include <sys/times.h>
43 #include <sys/sysinfo.h>
58 #include <mach/mach.h>
59 #include <sys/sysctl.h>
60 #elif KMP_OS_DRAGONFLY || KMP_OS_FREEBSD
61 #include <sys/types.h>
62 #include <sys/sysctl.h>
64 #include <pthread_np.h>
68 #elif KMP_OS_NETBSD || KMP_OS_OPENBSD
69 #include <sys/types.h>
70 #include <sys/sysctl.h>
75 #include <pthread_np.h>
80 #include <sys/loadavg.h>
87 struct kmp_sys_timer {
88 struct timespec start;
91 #ifndef TIMEVAL_TO_TIMESPEC
93 #define TIMEVAL_TO_TIMESPEC(tv, ts) \
95 (ts)->tv_sec = (tv)->tv_sec; \
96 (ts)->tv_nsec = (tv)->tv_usec * 1000; \
101 #define TS2NS(timespec) \
102 (((timespec).tv_sec * (long int)1e9) + (timespec).tv_nsec)
104 static struct kmp_sys_timer __kmp_sys_timer_data;
106 #if KMP_HANDLE_SIGNALS
107 typedef void (*sig_func_t)(int);
108 STATIC_EFI2_WORKAROUND
struct sigaction __kmp_sighldrs[NSIG];
109 static sigset_t __kmp_sigset;
112 static int __kmp_init_runtime = FALSE;
114 static int __kmp_fork_count = 0;
116 static pthread_condattr_t __kmp_suspend_cond_attr;
117 static pthread_mutexattr_t __kmp_suspend_mutex_attr;
119 static kmp_cond_align_t __kmp_wait_cv;
120 static kmp_mutex_align_t __kmp_wait_mx;
122 kmp_uint64 __kmp_ticks_per_msec = 1000000;
123 kmp_uint64 __kmp_ticks_per_usec = 1000;
126 static void __kmp_print_cond(
char *buffer, kmp_cond_align_t *cond) {
127 KMP_SNPRINTF(buffer, 128,
"(cond (lock (%ld, %d)), (descr (%p)))",
128 cond->c_cond.__c_lock.__status, cond->c_cond.__c_lock.__spinlock,
129 cond->c_cond.__c_waiting);
133 #if ((KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \
135 KMP_AFFINITY_SUPPORTED)
139 void __kmp_affinity_bind_thread(
int which) {
140 KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
141 "Illegal set affinity operation when not capable");
143 kmp_affin_mask_t *mask;
144 KMP_CPU_ALLOC_ON_STACK(mask);
146 KMP_CPU_SET(which, mask);
147 __kmp_set_system_affinity(mask, TRUE);
148 KMP_CPU_FREE_FROM_STACK(mask);
152 void __kmp_affinity_determine_capable(
const char *env_var) {
155 size_t mask_size = __kmp_xproc / CHAR_BIT;
157 if (__kmp_xproc % CHAR_BIT)
161 if (mask_size %
sizeof(__kmp_affin_mask_size))
162 mask_size +=
sizeof(__kmp_affin_mask_size) -
163 mask_size %
sizeof(__kmp_affin_mask_size);
164 KMP_AFFINITY_ENABLE(mask_size);
166 (
"__kmp_affinity_determine_capable: "
167 "AIX OS affinity interface bindprocessor functional (mask size = "
168 "%" KMP_SIZE_T_SPEC
").\n",
169 __kmp_affin_mask_size));
177 void __kmp_affinity_determine_capable(
const char *env_var) {
181 #define KMP_CPU_SET_SIZE_LIMIT (1024 * 1024)
182 #define KMP_CPU_SET_TRY_SIZE CACHE_LINE
183 #elif KMP_OS_FREEBSD || KMP_OS_DRAGONFLY
184 #define KMP_CPU_SET_SIZE_LIMIT (sizeof(cpuset_t))
186 #define KMP_CPU_SET_SIZE_LIMIT (256)
189 int verbose = __kmp_affinity.flags.verbose;
190 int warnings = __kmp_affinity.flags.warnings;
191 enum affinity_type type = __kmp_affinity.type;
196 buf = (
unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT);
200 gCode = syscall(__NR_sched_getaffinity, 0, KMP_CPU_SET_TRY_SIZE, buf);
201 KA_TRACE(30, (
"__kmp_affinity_determine_capable: "
202 "initial getaffinity call returned %ld errno = %d\n",
205 if (gCode < 0 && errno != EINVAL) {
208 (warnings && (type != affinity_none) && (type != affinity_default) &&
209 (type != affinity_disabled))) {
211 kmp_msg_t err_code = KMP_ERR(error);
212 __kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var),
213 err_code, __kmp_msg_null);
214 if (__kmp_generate_warnings == kmp_warnings_off) {
215 __kmp_str_free(&err_code.str);
218 KMP_AFFINITY_DISABLE();
219 KMP_INTERNAL_FREE(buf);
221 }
else if (gCode > 0) {
223 KMP_AFFINITY_ENABLE(gCode);
224 KA_TRACE(10, (
"__kmp_affinity_determine_capable: "
225 "affinity supported (mask size %d)\n",
226 (
int)__kmp_affin_mask_size));
227 KMP_INTERNAL_FREE(buf);
233 KA_TRACE(30, (
"__kmp_affinity_determine_capable: "
234 "searching for proper set size\n"));
236 for (size = 1; size <= KMP_CPU_SET_SIZE_LIMIT; size *= 2) {
237 gCode = syscall(__NR_sched_getaffinity, 0, size, buf);
238 KA_TRACE(30, (
"__kmp_affinity_determine_capable: "
239 "getaffinity for mask size %ld returned %ld errno = %d\n",
240 size, gCode, errno));
243 if (errno == ENOSYS) {
245 KA_TRACE(30, (
"__kmp_affinity_determine_capable: "
246 "inconsistent OS call behavior: errno == ENOSYS for mask "
250 (warnings && (type != affinity_none) &&
251 (type != affinity_default) && (type != affinity_disabled))) {
253 kmp_msg_t err_code = KMP_ERR(error);
254 __kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var),
255 err_code, __kmp_msg_null);
256 if (__kmp_generate_warnings == kmp_warnings_off) {
257 __kmp_str_free(&err_code.str);
260 KMP_AFFINITY_DISABLE();
261 KMP_INTERNAL_FREE(buf);
267 KMP_AFFINITY_ENABLE(gCode);
268 KA_TRACE(10, (
"__kmp_affinity_determine_capable: "
269 "affinity supported (mask size %d)\n",
270 (
int)__kmp_affin_mask_size));
271 KMP_INTERNAL_FREE(buf);
274 #elif KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY
277 buf = (
unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT);
278 gCode = pthread_getaffinity_np(pthread_self(), KMP_CPU_SET_SIZE_LIMIT,
279 reinterpret_cast<cpuset_t *
>(buf));
280 KA_TRACE(30, (
"__kmp_affinity_determine_capable: "
281 "initial getaffinity call returned %d errno = %d\n",
284 KMP_AFFINITY_ENABLE(KMP_CPU_SET_SIZE_LIMIT);
285 KA_TRACE(10, (
"__kmp_affinity_determine_capable: "
286 "affinity supported (mask size %d)\n",
287 (
int)__kmp_affin_mask_size));
288 KMP_INTERNAL_FREE(buf);
292 KMP_INTERNAL_FREE(buf);
295 KMP_AFFINITY_DISABLE();
296 KA_TRACE(10, (
"__kmp_affinity_determine_capable: "
297 "cannot determine mask size - affinity not supported\n"));
298 if (verbose || (warnings && (type != affinity_none) &&
299 (type != affinity_default) && (type != affinity_disabled))) {
300 KMP_WARNING(AffCantGetMaskSize, env_var);
309 int __kmp_futex_determine_capable() {
311 long rc = syscall(__NR_futex, &loc, FUTEX_WAKE, 1, NULL, NULL, 0);
312 int retval = (rc == 0) || (errno != ENOSYS);
315 (
"__kmp_futex_determine_capable: rc = %d errno = %d\n", rc, errno));
316 KA_TRACE(10, (
"__kmp_futex_determine_capable: futex syscall%s supported\n",
317 retval ?
"" :
" not"));
324 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_WASM) && (!KMP_ASM_INTRINS)
328 kmp_int8 __kmp_test_then_or8(
volatile kmp_int8 *p, kmp_int8 d) {
329 kmp_int8 old_value, new_value;
331 old_value = TCR_1(*p);
332 new_value = old_value | d;
334 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
336 old_value = TCR_1(*p);
337 new_value = old_value | d;
342 kmp_int8 __kmp_test_then_and8(
volatile kmp_int8 *p, kmp_int8 d) {
343 kmp_int8 old_value, new_value;
345 old_value = TCR_1(*p);
346 new_value = old_value & d;
348 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
350 old_value = TCR_1(*p);
351 new_value = old_value & d;
356 kmp_uint32 __kmp_test_then_or32(
volatile kmp_uint32 *p, kmp_uint32 d) {
357 kmp_uint32 old_value, new_value;
359 old_value = TCR_4(*p);
360 new_value = old_value | d;
362 while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) {
364 old_value = TCR_4(*p);
365 new_value = old_value | d;
370 kmp_uint32 __kmp_test_then_and32(
volatile kmp_uint32 *p, kmp_uint32 d) {
371 kmp_uint32 old_value, new_value;
373 old_value = TCR_4(*p);
374 new_value = old_value & d;
376 while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) {
378 old_value = TCR_4(*p);
379 new_value = old_value & d;
384 #if KMP_ARCH_X86 || KMP_ARCH_WASM
385 kmp_int8 __kmp_test_then_add8(
volatile kmp_int8 *p, kmp_int8 d) {
386 kmp_int8 old_value, new_value;
388 old_value = TCR_1(*p);
389 new_value = old_value + d;
391 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
393 old_value = TCR_1(*p);
394 new_value = old_value + d;
399 kmp_int64 __kmp_test_then_add64(
volatile kmp_int64 *p, kmp_int64 d) {
400 kmp_int64 old_value, new_value;
402 old_value = TCR_8(*p);
403 new_value = old_value + d;
405 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
407 old_value = TCR_8(*p);
408 new_value = old_value + d;
414 kmp_uint64 __kmp_test_then_or64(
volatile kmp_uint64 *p, kmp_uint64 d) {
415 kmp_uint64 old_value, new_value;
417 old_value = TCR_8(*p);
418 new_value = old_value | d;
419 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
421 old_value = TCR_8(*p);
422 new_value = old_value | d;
427 kmp_uint64 __kmp_test_then_and64(
volatile kmp_uint64 *p, kmp_uint64 d) {
428 kmp_uint64 old_value, new_value;
430 old_value = TCR_8(*p);
431 new_value = old_value & d;
432 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
434 old_value = TCR_8(*p);
435 new_value = old_value & d;
442 void __kmp_terminate_thread(
int gtid) {
444 kmp_info_t *th = __kmp_threads[gtid];
449 #ifdef KMP_CANCEL_THREADS
450 KA_TRACE(10, (
"__kmp_terminate_thread: kill (%d)\n", gtid));
451 status = pthread_cancel(th->th.th_info.ds.ds_thread);
452 if (status != 0 && status != ESRCH) {
453 __kmp_fatal(KMP_MSG(CantTerminateWorkerThread), KMP_ERR(status),
464 static kmp_int32 __kmp_set_stack_info(
int gtid, kmp_info_t *th) {
466 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
467 KMP_OS_HAIKU || KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX
475 if (!KMP_UBER_GTID(gtid)) {
479 if ((status = thr_stksegment(&s)) < 0) {
480 KMP_CHECK_SYSFAIL(
"thr_stksegment", status);
485 KA_TRACE(60, (
"__kmp_set_stack_info: T#%d thr_stksegment returned size:"
486 " %lu, low addr: %p\n",
491 status = pthread_attr_init(&attr);
492 KMP_CHECK_SYSFAIL(
"pthread_attr_init", status);
493 #if KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD
494 status = pthread_attr_get_np(pthread_self(), &attr);
495 KMP_CHECK_SYSFAIL(
"pthread_attr_get_np", status);
497 status = pthread_getattr_np(pthread_self(), &attr);
498 KMP_CHECK_SYSFAIL(
"pthread_getattr_np", status);
500 status = pthread_attr_getstack(&attr, &addr, &size);
501 KMP_CHECK_SYSFAIL(
"pthread_attr_getstack", status);
503 (
"__kmp_set_stack_info: T#%d pthread_attr_getstack returned size:"
504 " %lu, low addr: %p\n",
506 status = pthread_attr_destroy(&attr);
507 KMP_CHECK_SYSFAIL(
"pthread_attr_destroy", status);
511 if (size != 0 && addr != 0) {
513 TCW_PTR(th->th.th_info.ds.ds_stackbase, (((
char *)addr) + size));
514 TCW_PTR(th->th.th_info.ds.ds_stacksize, size);
515 TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
521 TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
522 TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
523 TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
527 static void *__kmp_launch_worker(
void *thr) {
528 int status, old_type, old_state;
529 #ifdef KMP_BLOCK_SIGNALS
530 sigset_t new_set, old_set;
533 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
534 KMP_OS_OPENBSD || KMP_OS_HAIKU || KMP_OS_HURD || KMP_OS_SOLARIS || \
536 void *
volatile padding = 0;
540 gtid = ((kmp_info_t *)thr)->th.th_info.ds.ds_gtid;
541 __kmp_gtid_set_specific(gtid);
542 #ifdef KMP_TDATA_GTID
545 #if KMP_STATS_ENABLED
547 __kmp_stats_thread_ptr = ((kmp_info_t *)thr)->th.th_stats;
548 __kmp_stats_thread_ptr->startLife();
549 KMP_SET_THREAD_STATE(IDLE);
554 __kmp_itt_thread_name(gtid);
557 #if KMP_AFFINITY_SUPPORTED
558 __kmp_affinity_bind_init_mask(gtid);
561 #ifdef KMP_CANCEL_THREADS
562 status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type);
563 KMP_CHECK_SYSFAIL(
"pthread_setcanceltype", status);
565 status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state);
566 KMP_CHECK_SYSFAIL(
"pthread_setcancelstate", status);
569 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
571 __kmp_clear_x87_fpu_status_word();
572 __kmp_load_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
573 __kmp_load_mxcsr(&__kmp_init_mxcsr);
576 #ifdef KMP_BLOCK_SIGNALS
577 status = sigfillset(&new_set);
578 KMP_CHECK_SYSFAIL_ERRNO(
"sigfillset", status);
579 status = pthread_sigmask(SIG_BLOCK, &new_set, &old_set);
580 KMP_CHECK_SYSFAIL(
"pthread_sigmask", status);
583 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
584 KMP_OS_OPENBSD || KMP_OS_HAIKU || KMP_OS_HURD || KMP_OS_SOLARIS || \
586 if (__kmp_stkoffset > 0 && gtid > 0) {
587 padding = KMP_ALLOCA(gtid * __kmp_stkoffset);
593 __kmp_set_stack_info(gtid, (kmp_info_t *)thr);
595 __kmp_check_stack_overlap((kmp_info_t *)thr);
597 exit_val = __kmp_launch_thread((kmp_info_t *)thr);
599 #ifdef KMP_BLOCK_SIGNALS
600 status = pthread_sigmask(SIG_SETMASK, &old_set, NULL);
601 KMP_CHECK_SYSFAIL(
"pthread_sigmask", status);
610 static void *__kmp_launch_monitor(
void *thr) {
611 int status, old_type, old_state;
612 #ifdef KMP_BLOCK_SIGNALS
615 struct timespec interval;
619 KA_TRACE(10, (
"__kmp_launch_monitor: #1 launched\n"));
622 __kmp_gtid_set_specific(KMP_GTID_MONITOR);
623 #ifdef KMP_TDATA_GTID
624 __kmp_gtid = KMP_GTID_MONITOR;
631 __kmp_itt_thread_ignore();
634 __kmp_set_stack_info(((kmp_info_t *)thr)->th.th_info.ds.ds_gtid,
637 __kmp_check_stack_overlap((kmp_info_t *)thr);
639 #ifdef KMP_CANCEL_THREADS
640 status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type);
641 KMP_CHECK_SYSFAIL(
"pthread_setcanceltype", status);
643 status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state);
644 KMP_CHECK_SYSFAIL(
"pthread_setcancelstate", status);
647 #if KMP_REAL_TIME_FIX
652 int sched = sched_getscheduler(0);
653 if (sched == SCHED_FIFO || sched == SCHED_RR) {
656 struct sched_param param;
657 int max_priority = sched_get_priority_max(sched);
659 KMP_WARNING(RealTimeSchedNotSupported);
660 sched_getparam(0, ¶m);
661 if (param.sched_priority < max_priority) {
662 param.sched_priority += 1;
663 rc = sched_setscheduler(0, sched, ¶m);
666 kmp_msg_t err_code = KMP_ERR(error);
667 __kmp_msg(kmp_ms_warning, KMP_MSG(CantChangeMonitorPriority),
668 err_code, KMP_MSG(MonitorWillStarve), __kmp_msg_null);
669 if (__kmp_generate_warnings == kmp_warnings_off) {
670 __kmp_str_free(&err_code.str);
677 __kmp_msg(kmp_ms_warning, KMP_MSG(RunningAtMaxPriority),
678 KMP_MSG(MonitorWillStarve), KMP_HNT(RunningAtMaxPriority),
683 TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
689 if (__kmp_monitor_wakeups == 1) {
691 interval.tv_nsec = 0;
694 interval.tv_nsec = (KMP_NSEC_PER_SEC / __kmp_monitor_wakeups);
697 KA_TRACE(10, (
"__kmp_launch_monitor: #2 monitor\n"));
699 while (!TCR_4(__kmp_global.g.g_done)) {
705 KA_TRACE(15, (
"__kmp_launch_monitor: update\n"));
707 status = gettimeofday(&tval, NULL);
708 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status);
709 TIMEVAL_TO_TIMESPEC(&tval, &now);
711 now.tv_sec += interval.tv_sec;
712 now.tv_nsec += interval.tv_nsec;
714 if (now.tv_nsec >= KMP_NSEC_PER_SEC) {
716 now.tv_nsec -= KMP_NSEC_PER_SEC;
719 status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex);
720 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status);
722 if (!TCR_4(__kmp_global.g.g_done)) {
723 status = pthread_cond_timedwait(&__kmp_wait_cv.c_cond,
724 &__kmp_wait_mx.m_mutex, &now);
726 if (status != ETIMEDOUT && status != EINTR) {
727 KMP_SYSFAIL(
"pthread_cond_timedwait", status);
731 status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex);
732 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status);
734 TCW_4(__kmp_global.g.g_time.dt.t_value,
735 TCR_4(__kmp_global.g.g_time.dt.t_value) + 1);
740 KA_TRACE(10, (
"__kmp_launch_monitor: #3 cleanup\n"));
742 #ifdef KMP_BLOCK_SIGNALS
743 status = sigfillset(&new_set);
744 KMP_CHECK_SYSFAIL_ERRNO(
"sigfillset", status);
745 status = pthread_sigmask(SIG_UNBLOCK, &new_set, NULL);
746 KMP_CHECK_SYSFAIL(
"pthread_sigmask", status);
749 KA_TRACE(10, (
"__kmp_launch_monitor: #4 finished\n"));
751 if (__kmp_global.g.g_abort != 0) {
757 KA_TRACE(10, (
"__kmp_launch_monitor: #5 terminate sig=%d\n",
758 __kmp_global.g.g_abort));
763 for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
764 __kmp_terminate_thread(gtid);
768 KA_TRACE(10, (
"__kmp_launch_monitor: #6 raise sig=%d\n",
769 __kmp_global.g.g_abort));
771 if (__kmp_global.g.g_abort > 0)
772 raise(__kmp_global.g.g_abort);
775 KA_TRACE(10, (
"__kmp_launch_monitor: #7 exit\n"));
781 void __kmp_create_worker(
int gtid, kmp_info_t *th,
size_t stack_size) {
783 pthread_attr_t thread_attr;
786 th->th.th_info.ds.ds_gtid = gtid;
788 #if KMP_STATS_ENABLED
790 __kmp_acquire_tas_lock(&__kmp_stats_lock, gtid);
796 if (!KMP_UBER_GTID(gtid)) {
797 th->th.th_stats = __kmp_stats_list->push_back(gtid);
801 th->th.th_stats = __kmp_stats_thread_ptr;
803 __kmp_release_tas_lock(&__kmp_stats_lock, gtid);
807 if (KMP_UBER_GTID(gtid)) {
808 KA_TRACE(10, (
"__kmp_create_worker: uber thread (%d)\n", gtid));
809 th->th.th_info.ds.ds_thread = pthread_self();
810 __kmp_set_stack_info(gtid, th);
811 __kmp_check_stack_overlap(th);
815 KA_TRACE(10, (
"__kmp_create_worker: try to create thread (%d)\n", gtid));
819 #ifdef KMP_THREAD_ATTR
820 status = pthread_attr_init(&thread_attr);
822 __kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null);
824 status = pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE);
826 __kmp_fatal(KMP_MSG(CantSetWorkerState), KMP_ERR(status), __kmp_msg_null);
836 stack_size += gtid * __kmp_stkoffset * 2;
838 KA_TRACE(10, (
"__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
839 "__kmp_stksize = %lu bytes, final stacksize = %lu bytes\n",
840 gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size));
842 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
843 status = pthread_attr_setstacksize(&thread_attr, stack_size);
844 #ifdef KMP_BACKUP_STKSIZE
846 if (!__kmp_env_stksize) {
847 stack_size = KMP_BACKUP_STKSIZE + gtid * __kmp_stkoffset;
848 __kmp_stksize = KMP_BACKUP_STKSIZE;
849 KA_TRACE(10, (
"__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
850 "__kmp_stksize = %lu bytes, (backup) final stacksize = %lu "
852 gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size));
853 status = pthread_attr_setstacksize(&thread_attr, stack_size);
858 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
859 KMP_HNT(ChangeWorkerStackSize), __kmp_msg_null);
866 pthread_create(&handle, &thread_attr, __kmp_launch_worker, (
void *)th);
867 if (status != 0 || !handle) {
868 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
869 if (status == EINVAL) {
870 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
871 KMP_HNT(IncreaseWorkerStackSize), __kmp_msg_null);
873 if (status == ENOMEM) {
874 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
875 KMP_HNT(DecreaseWorkerStackSize), __kmp_msg_null);
878 if (status == EAGAIN) {
879 __kmp_fatal(KMP_MSG(NoResourcesForWorkerThread), KMP_ERR(status),
880 KMP_HNT(Decrease_NUM_THREADS), __kmp_msg_null);
882 KMP_SYSFAIL(
"pthread_create", status);
886 if (!KMP_UBER_GTID(gtid)) {
887 #if defined(LIBOMP_HAVE_PTHREAD_SET_NAME_NP)
888 pthread_set_name_np(handle,
"openmp_worker");
889 #elif defined(LIBOMP_HAVE_PTHREAD_SETNAME_NP) && !KMP_OS_DARWIN
891 pthread_setname_np(handle,
"%s",
const_cast<char *
>(
"openmp_worker"));
893 pthread_setname_np(handle,
"openmp_worker");
898 th->th.th_info.ds.ds_thread = handle;
900 #ifdef KMP_THREAD_ATTR
901 status = pthread_attr_destroy(&thread_attr);
903 kmp_msg_t err_code = KMP_ERR(status);
904 __kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code,
906 if (__kmp_generate_warnings == kmp_warnings_off) {
907 __kmp_str_free(&err_code.str);
914 KA_TRACE(10, (
"__kmp_create_worker: done creating thread (%d)\n", gtid));
919 void __kmp_create_monitor(kmp_info_t *th) {
921 pthread_attr_t thread_attr;
924 int auto_adj_size = FALSE;
926 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
928 KA_TRACE(10, (
"__kmp_create_monitor: skipping monitor thread because of "
930 th->th.th_info.ds.ds_tid = 0;
931 th->th.th_info.ds.ds_gtid = 0;
934 KA_TRACE(10, (
"__kmp_create_monitor: try to create monitor\n"));
938 th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
939 th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
940 #if KMP_REAL_TIME_FIX
941 TCW_4(__kmp_global.g.g_time.dt.t_value,
944 TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
947 #ifdef KMP_THREAD_ATTR
948 if (__kmp_monitor_stksize == 0) {
949 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
950 auto_adj_size = TRUE;
952 status = pthread_attr_init(&thread_attr);
954 __kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null);
956 status = pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE);
958 __kmp_fatal(KMP_MSG(CantSetMonitorState), KMP_ERR(status), __kmp_msg_null);
961 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
962 status = pthread_attr_getstacksize(&thread_attr, &size);
963 KMP_CHECK_SYSFAIL(
"pthread_attr_getstacksize", status);
965 size = __kmp_sys_min_stksize;
969 if (__kmp_monitor_stksize == 0) {
970 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
972 if (__kmp_monitor_stksize < __kmp_sys_min_stksize) {
973 __kmp_monitor_stksize = __kmp_sys_min_stksize;
976 KA_TRACE(10, (
"__kmp_create_monitor: default stacksize = %lu bytes,"
977 "requested stacksize = %lu bytes\n",
978 size, __kmp_monitor_stksize));
983 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
984 KA_TRACE(10, (
"__kmp_create_monitor: setting stacksize = %lu bytes,",
985 __kmp_monitor_stksize));
986 status = pthread_attr_setstacksize(&thread_attr, __kmp_monitor_stksize);
989 __kmp_monitor_stksize *= 2;
992 kmp_msg_t err_code = KMP_ERR(status);
993 __kmp_msg(kmp_ms_warning,
994 KMP_MSG(CantSetMonitorStackSize, (
long int)__kmp_monitor_stksize),
995 err_code, KMP_HNT(ChangeMonitorStackSize), __kmp_msg_null);
996 if (__kmp_generate_warnings == kmp_warnings_off) {
997 __kmp_str_free(&err_code.str);
1003 pthread_create(&handle, &thread_attr, __kmp_launch_monitor, (
void *)th);
1006 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1007 if (status == EINVAL) {
1008 if (auto_adj_size && (__kmp_monitor_stksize < (
size_t)0x40000000)) {
1009 __kmp_monitor_stksize *= 2;
1012 __kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize),
1013 KMP_ERR(status), KMP_HNT(IncreaseMonitorStackSize),
1016 if (status == ENOMEM) {
1017 __kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize),
1018 KMP_ERR(status), KMP_HNT(DecreaseMonitorStackSize),
1022 if (status == EAGAIN) {
1023 __kmp_fatal(KMP_MSG(NoResourcesForMonitorThread), KMP_ERR(status),
1024 KMP_HNT(DecreaseNumberOfThreadsInUse), __kmp_msg_null);
1026 KMP_SYSFAIL(
"pthread_create", status);
1029 th->th.th_info.ds.ds_thread = handle;
1031 #if KMP_REAL_TIME_FIX
1033 KMP_DEBUG_ASSERT(
sizeof(kmp_uint32) ==
1034 sizeof(__kmp_global.g.g_time.dt.t_value));
1035 __kmp_wait_4((kmp_uint32
volatile *)&__kmp_global.g.g_time.dt.t_value, -1,
1036 &__kmp_neq_4, NULL);
1039 #ifdef KMP_THREAD_ATTR
1040 status = pthread_attr_destroy(&thread_attr);
1042 kmp_msg_t err_code = KMP_ERR(status);
1043 __kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code,
1045 if (__kmp_generate_warnings == kmp_warnings_off) {
1046 __kmp_str_free(&err_code.str);
1053 KA_TRACE(10, (
"__kmp_create_monitor: monitor created %#.8lx\n",
1054 th->th.th_info.ds.ds_thread));
1059 void __kmp_exit_thread(
int exit_status) {
1063 pthread_exit((
void *)(intptr_t)exit_status);
1068 void __kmp_resume_monitor();
1070 extern "C" void __kmp_reap_monitor(kmp_info_t *th) {
1074 KA_TRACE(10, (
"__kmp_reap_monitor: try to reap monitor thread with handle"
1076 th->th.th_info.ds.ds_thread));
1081 KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid);
1082 if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) {
1083 KA_TRACE(10, (
"__kmp_reap_monitor: monitor did not start, returning\n"));
1093 status = pthread_kill(th->th.th_info.ds.ds_thread, 0);
1094 if (status != ESRCH) {
1095 __kmp_resume_monitor();
1097 KA_TRACE(10, (
"__kmp_reap_monitor: try to join with monitor\n"));
1098 status = pthread_join(th->th.th_info.ds.ds_thread, &exit_val);
1099 if (exit_val != th) {
1100 __kmp_fatal(KMP_MSG(ReapMonitorError), KMP_ERR(status), __kmp_msg_null);
1103 th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1104 th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1106 KA_TRACE(10, (
"__kmp_reap_monitor: done reaping monitor thread with handle"
1108 th->th.th_info.ds.ds_thread));
1115 extern "C" void __kmp_reap_monitor(kmp_info_t *th) { (void)th; }
1118 void __kmp_reap_worker(kmp_info_t *th) {
1125 10, (
"__kmp_reap_worker: try to reap T#%d\n", th->th.th_info.ds.ds_gtid));
1127 status = pthread_join(th->th.th_info.ds.ds_thread, &exit_val);
1131 __kmp_fatal(KMP_MSG(ReapWorkerError), KMP_ERR(status), __kmp_msg_null);
1133 if (exit_val != th) {
1134 KA_TRACE(10, (
"__kmp_reap_worker: worker T#%d did not reap properly, "
1136 th->th.th_info.ds.ds_gtid, exit_val));
1142 KA_TRACE(10, (
"__kmp_reap_worker: done reaping T#%d\n",
1143 th->th.th_info.ds.ds_gtid));
1148 #if KMP_HANDLE_SIGNALS
1150 static void __kmp_null_handler(
int signo) {
1154 static void __kmp_team_handler(
int signo) {
1155 if (__kmp_global.g.g_abort == 0) {
1158 __kmp_debug_printf(
"__kmp_team_handler: caught signal = %d\n", signo);
1173 if (__kmp_debug_buf) {
1174 __kmp_dump_debug_buffer();
1176 __kmp_unregister_library();
1178 TCW_4(__kmp_global.g.g_abort, signo);
1180 TCW_4(__kmp_global.g.g_done, TRUE);
1185 __kmp_debug_printf(
"__kmp_team_handler: unknown signal type");
1192 static void __kmp_sigaction(
int signum,
const struct sigaction *act,
1193 struct sigaction *oldact) {
1194 int rc = sigaction(signum, act, oldact);
1195 KMP_CHECK_SYSFAIL_ERRNO(
"sigaction", rc);
1198 static void __kmp_install_one_handler(
int sig, sig_func_t handler_func,
1199 int parallel_init) {
1202 (
"__kmp_install_one_handler( %d, ..., %d )\n", sig, parallel_init));
1203 if (parallel_init) {
1204 struct sigaction new_action;
1205 struct sigaction old_action;
1206 new_action.sa_handler = handler_func;
1207 new_action.sa_flags = 0;
1208 sigfillset(&new_action.sa_mask);
1209 __kmp_sigaction(sig, &new_action, &old_action);
1210 if (old_action.sa_handler == __kmp_sighldrs[sig].sa_handler) {
1211 sigaddset(&__kmp_sigset, sig);
1214 __kmp_sigaction(sig, &old_action, NULL);
1218 __kmp_sigaction(sig, NULL, &__kmp_sighldrs[sig]);
1223 static void __kmp_remove_one_handler(
int sig) {
1224 KB_TRACE(60, (
"__kmp_remove_one_handler( %d )\n", sig));
1225 if (sigismember(&__kmp_sigset, sig)) {
1226 struct sigaction old;
1228 __kmp_sigaction(sig, &__kmp_sighldrs[sig], &old);
1229 if ((old.sa_handler != __kmp_team_handler) &&
1230 (old.sa_handler != __kmp_null_handler)) {
1232 KB_TRACE(10, (
"__kmp_remove_one_handler: oops, not our handler, "
1233 "restoring: sig=%d\n",
1235 __kmp_sigaction(sig, &old, NULL);
1237 sigdelset(&__kmp_sigset, sig);
1242 void __kmp_install_signals(
int parallel_init) {
1243 KB_TRACE(10, (
"__kmp_install_signals( %d )\n", parallel_init));
1244 if (__kmp_handle_signals || !parallel_init) {
1247 sigemptyset(&__kmp_sigset);
1248 __kmp_install_one_handler(SIGHUP, __kmp_team_handler, parallel_init);
1249 __kmp_install_one_handler(SIGINT, __kmp_team_handler, parallel_init);
1250 __kmp_install_one_handler(SIGQUIT, __kmp_team_handler, parallel_init);
1251 __kmp_install_one_handler(SIGILL, __kmp_team_handler, parallel_init);
1252 __kmp_install_one_handler(SIGABRT, __kmp_team_handler, parallel_init);
1253 __kmp_install_one_handler(SIGFPE, __kmp_team_handler, parallel_init);
1254 __kmp_install_one_handler(SIGBUS, __kmp_team_handler, parallel_init);
1255 __kmp_install_one_handler(SIGSEGV, __kmp_team_handler, parallel_init);
1257 __kmp_install_one_handler(SIGSYS, __kmp_team_handler, parallel_init);
1259 __kmp_install_one_handler(SIGTERM, __kmp_team_handler, parallel_init);
1261 __kmp_install_one_handler(SIGPIPE, __kmp_team_handler, parallel_init);
1266 void __kmp_remove_signals(
void) {
1268 KB_TRACE(10, (
"__kmp_remove_signals()\n"));
1269 for (sig = 1; sig < NSIG; ++sig) {
1270 __kmp_remove_one_handler(sig);
1276 void __kmp_enable(
int new_state) {
1277 #ifdef KMP_CANCEL_THREADS
1278 int status, old_state;
1279 status = pthread_setcancelstate(new_state, &old_state);
1280 KMP_CHECK_SYSFAIL(
"pthread_setcancelstate", status);
1281 KMP_DEBUG_ASSERT(old_state == PTHREAD_CANCEL_DISABLE);
1285 void __kmp_disable(
int *old_state) {
1286 #ifdef KMP_CANCEL_THREADS
1288 status = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, old_state);
1289 KMP_CHECK_SYSFAIL(
"pthread_setcancelstate", status);
1293 static void __kmp_atfork_prepare(
void) {
1294 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
1295 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
1298 static void __kmp_atfork_parent(
void) {
1299 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1300 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
1306 static void __kmp_atfork_child(
void) {
1307 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1308 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
1315 #if KMP_AFFINITY_SUPPORTED
1316 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \
1320 kmp_set_thread_affinity_mask_initial();
1325 if (__kmp_nested_proc_bind.bind_types != NULL) {
1326 __kmp_nested_proc_bind.bind_types[0] = proc_bind_false;
1328 for (kmp_affinity_t *affinity : __kmp_affinities)
1329 *affinity = KMP_AFFINITY_INIT(affinity->env_var);
1330 __kmp_affin_fullMask =
nullptr;
1331 __kmp_affin_origMask =
nullptr;
1332 __kmp_topology =
nullptr;
1336 __kmp_init_monitor = 0;
1338 __kmp_init_parallel = FALSE;
1339 __kmp_init_middle = FALSE;
1340 __kmp_init_serial = FALSE;
1341 TCW_4(__kmp_init_gtid, FALSE);
1342 __kmp_init_common = FALSE;
1344 TCW_4(__kmp_init_user_locks, FALSE);
1345 #if !KMP_USE_DYNAMIC_LOCK
1346 __kmp_user_lock_table.used = 1;
1347 __kmp_user_lock_table.allocated = 0;
1348 __kmp_user_lock_table.table = NULL;
1349 __kmp_lock_blocks = NULL;
1353 TCW_4(__kmp_nth, 0);
1355 __kmp_thread_pool = NULL;
1356 __kmp_thread_pool_insert_pt = NULL;
1357 __kmp_team_pool = NULL;
1361 KA_TRACE(10, (
"__kmp_atfork_child: checking cache address list %p\n",
1362 __kmp_threadpriv_cache_list));
1364 while (__kmp_threadpriv_cache_list != NULL) {
1366 if (*__kmp_threadpriv_cache_list->addr != NULL) {
1367 KC_TRACE(50, (
"__kmp_atfork_child: zeroing cache at address %p\n",
1368 &(*__kmp_threadpriv_cache_list->addr)));
1370 *__kmp_threadpriv_cache_list->addr = NULL;
1372 __kmp_threadpriv_cache_list = __kmp_threadpriv_cache_list->next;
1375 __kmp_init_runtime = FALSE;
1378 __kmp_init_bootstrap_lock(&__kmp_initz_lock);
1379 __kmp_init_bootstrap_lock(&__kmp_stdio_lock);
1380 __kmp_init_bootstrap_lock(&__kmp_console_lock);
1381 __kmp_init_bootstrap_lock(&__kmp_task_team_lock);
1391 __kmp_need_register_serial = FALSE;
1392 __kmp_serial_initialize();
1406 void __kmp_register_atfork(
void) {
1407 if (__kmp_need_register_atfork) {
1409 int status = pthread_atfork(__kmp_atfork_prepare, __kmp_atfork_parent,
1410 __kmp_atfork_child);
1411 KMP_CHECK_SYSFAIL(
"pthread_atfork", status);
1413 __kmp_need_register_atfork = FALSE;
1417 void __kmp_suspend_initialize(
void) {
1419 status = pthread_mutexattr_init(&__kmp_suspend_mutex_attr);
1420 KMP_CHECK_SYSFAIL(
"pthread_mutexattr_init", status);
1421 status = pthread_condattr_init(&__kmp_suspend_cond_attr);
1422 KMP_CHECK_SYSFAIL(
"pthread_condattr_init", status);
1425 void __kmp_suspend_initialize_thread(kmp_info_t *th) {
1426 int old_value = KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init_count);
1427 int new_value = __kmp_fork_count + 1;
1429 if (old_value == new_value)
1432 if (old_value == -1 || !__kmp_atomic_compare_store(
1433 &th->th.th_suspend_init_count, old_value, -1)) {
1434 while (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) != new_value) {
1440 status = pthread_cond_init(&th->th.th_suspend_cv.c_cond,
1441 &__kmp_suspend_cond_attr);
1442 KMP_CHECK_SYSFAIL(
"pthread_cond_init", status);
1443 status = pthread_mutex_init(&th->th.th_suspend_mx.m_mutex,
1444 &__kmp_suspend_mutex_attr);
1445 KMP_CHECK_SYSFAIL(
"pthread_mutex_init", status);
1446 KMP_ATOMIC_ST_REL(&th->th.th_suspend_init_count, new_value);
1450 void __kmp_suspend_uninitialize_thread(kmp_info_t *th) {
1451 if (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) > __kmp_fork_count) {
1456 status = pthread_cond_destroy(&th->th.th_suspend_cv.c_cond);
1457 if (status != 0 && status != EBUSY) {
1458 KMP_SYSFAIL(
"pthread_cond_destroy", status);
1460 status = pthread_mutex_destroy(&th->th.th_suspend_mx.m_mutex);
1461 if (status != 0 && status != EBUSY) {
1462 KMP_SYSFAIL(
"pthread_mutex_destroy", status);
1464 --th->th.th_suspend_init_count;
1465 KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init_count) ==
1471 int __kmp_try_suspend_mx(kmp_info_t *th) {
1472 return (pthread_mutex_trylock(&th->th.th_suspend_mx.m_mutex) == 0);
1475 void __kmp_lock_suspend_mx(kmp_info_t *th) {
1476 int status = pthread_mutex_lock(&th->th.th_suspend_mx.m_mutex);
1477 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status);
1480 void __kmp_unlock_suspend_mx(kmp_info_t *th) {
1481 int status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex);
1482 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status);
1488 static inline void __kmp_suspend_template(
int th_gtid, C *flag) {
1489 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_suspend);
1490 kmp_info_t *th = __kmp_threads[th_gtid];
1492 typename C::flag_t old_spin;
1494 KF_TRACE(30, (
"__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid,
1497 __kmp_suspend_initialize_thread(th);
1499 __kmp_lock_suspend_mx(th);
1501 KF_TRACE(10, (
"__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n",
1502 th_gtid, flag->get()));
1506 old_spin = flag->set_sleeping();
1507 TCW_PTR(th->th.th_sleep_loc, (
void *)flag);
1508 th->th.th_sleep_loc_type = flag->get_type();
1509 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
1510 __kmp_pause_status != kmp_soft_paused) {
1511 flag->unset_sleeping();
1512 TCW_PTR(th->th.th_sleep_loc, NULL);
1513 th->th.th_sleep_loc_type = flag_unset;
1514 __kmp_unlock_suspend_mx(th);
1517 KF_TRACE(5, (
"__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%x,"
1519 th_gtid, flag->get(), flag->load(), old_spin));
1521 if (flag->done_check_val(old_spin) || flag->done_check()) {
1522 flag->unset_sleeping();
1523 TCW_PTR(th->th.th_sleep_loc, NULL);
1524 th->th.th_sleep_loc_type = flag_unset;
1525 KF_TRACE(5, (
"__kmp_suspend_template: T#%d false alarm, reset sleep bit "
1527 th_gtid, flag->get()));
1532 int deactivated = FALSE;
1534 while (flag->is_sleeping()) {
1535 #ifdef DEBUG_SUSPEND
1537 __kmp_suspend_count++;
1538 __kmp_print_cond(buffer, &th->th.th_suspend_cv);
1539 __kmp_printf(
"__kmp_suspend_template: suspending T#%d: %s\n", th_gtid,
1545 th->th.th_active = FALSE;
1546 if (th->th.th_active_in_pool) {
1547 th->th.th_active_in_pool = FALSE;
1548 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
1549 KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
1554 KMP_DEBUG_ASSERT(th->th.th_sleep_loc);
1555 KMP_DEBUG_ASSERT(flag->get_type() == th->th.th_sleep_loc_type);
1557 #if USE_SUSPEND_TIMEOUT
1558 struct timespec now;
1559 struct timeval tval;
1562 status = gettimeofday(&tval, NULL);
1563 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status);
1564 TIMEVAL_TO_TIMESPEC(&tval, &now);
1566 msecs = (4 * __kmp_dflt_blocktime) + 200;
1567 now.tv_sec += msecs / 1000;
1568 now.tv_nsec += (msecs % 1000) * 1000;
1570 KF_TRACE(15, (
"__kmp_suspend_template: T#%d about to perform "
1571 "pthread_cond_timedwait\n",
1573 status = pthread_cond_timedwait(&th->th.th_suspend_cv.c_cond,
1574 &th->th.th_suspend_mx.m_mutex, &now);
1576 KF_TRACE(15, (
"__kmp_suspend_template: T#%d about to perform"
1577 " pthread_cond_wait\n",
1579 status = pthread_cond_wait(&th->th.th_suspend_cv.c_cond,
1580 &th->th.th_suspend_mx.m_mutex);
1583 if ((status != 0) && (status != EINTR) && (status != ETIMEDOUT)) {
1584 KMP_SYSFAIL(
"pthread_cond_wait", status);
1587 KMP_DEBUG_ASSERT(flag->get_type() == flag->get_ptr_type());
1589 if (!flag->is_sleeping() &&
1590 ((status == EINTR) || (status == ETIMEDOUT))) {
1594 flag->unset_sleeping();
1595 TCW_PTR(th->th.th_sleep_loc, NULL);
1596 th->th.th_sleep_loc_type = flag_unset;
1599 if (status == ETIMEDOUT) {
1600 if (flag->is_sleeping()) {
1602 (
"__kmp_suspend_template: T#%d timeout wakeup\n", th_gtid));
1604 KF_TRACE(2, (
"__kmp_suspend_template: T#%d timeout wakeup, sleep bit "
1607 TCW_PTR(th->th.th_sleep_loc, NULL);
1608 th->th.th_sleep_loc_type = flag_unset;
1610 }
else if (flag->is_sleeping()) {
1612 (
"__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid));
1619 th->th.th_active = TRUE;
1620 if (TCR_4(th->th.th_in_pool)) {
1621 KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
1622 th->th.th_active_in_pool = TRUE;
1628 TCW_PTR(th->th.th_sleep_loc, NULL);
1629 th->th.th_sleep_loc_type = flag_unset;
1631 KMP_DEBUG_ASSERT(!flag->is_sleeping());
1632 KMP_DEBUG_ASSERT(!th->th.th_sleep_loc);
1633 #ifdef DEBUG_SUSPEND
1636 __kmp_print_cond(buffer, &th->th.th_suspend_cv);
1637 __kmp_printf(
"__kmp_suspend_template: T#%d has awakened: %s\n", th_gtid,
1642 __kmp_unlock_suspend_mx(th);
1643 KF_TRACE(30, (
"__kmp_suspend_template: T#%d exit\n", th_gtid));
1646 template <
bool C,
bool S>
1647 void __kmp_suspend_32(
int th_gtid, kmp_flag_32<C, S> *flag) {
1648 __kmp_suspend_template(th_gtid, flag);
1650 template <
bool C,
bool S>
1651 void __kmp_suspend_64(
int th_gtid, kmp_flag_64<C, S> *flag) {
1652 __kmp_suspend_template(th_gtid, flag);
1654 template <
bool C,
bool S>
1655 void __kmp_atomic_suspend_64(
int th_gtid, kmp_atomic_flag_64<C, S> *flag) {
1656 __kmp_suspend_template(th_gtid, flag);
1658 void __kmp_suspend_oncore(
int th_gtid, kmp_flag_oncore *flag) {
1659 __kmp_suspend_template(th_gtid, flag);
1662 template void __kmp_suspend_32<false, false>(
int, kmp_flag_32<false, false> *);
1663 template void __kmp_suspend_64<false, true>(
int, kmp_flag_64<false, true> *);
1664 template void __kmp_suspend_64<true, false>(
int, kmp_flag_64<true, false> *);
1666 __kmp_atomic_suspend_64<false, true>(
int, kmp_atomic_flag_64<false, true> *);
1668 __kmp_atomic_suspend_64<true, false>(
int, kmp_atomic_flag_64<true, false> *);
1674 static inline void __kmp_resume_template(
int target_gtid, C *flag) {
1675 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
1676 kmp_info_t *th = __kmp_threads[target_gtid];
1680 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1683 KF_TRACE(30, (
"__kmp_resume_template: T#%d wants to wakeup T#%d enter\n",
1684 gtid, target_gtid));
1685 KMP_DEBUG_ASSERT(gtid != target_gtid);
1687 __kmp_suspend_initialize_thread(th);
1689 __kmp_lock_suspend_mx(th);
1691 if (!flag || flag != th->th.th_sleep_loc) {
1694 flag = (C *)CCAST(
void *, th->th.th_sleep_loc);
1700 KF_TRACE(5, (
"__kmp_resume_template: T#%d exiting, thread T#%d already "
1701 "awake: flag(%p)\n",
1702 gtid, target_gtid, (
void *)NULL));
1703 __kmp_unlock_suspend_mx(th);
1705 }
else if (flag->get_type() != th->th.th_sleep_loc_type) {
1710 (
"__kmp_resume_template: T#%d retrying, thread T#%d Mismatch flag(%p), "
1711 "spin(%p) type=%d ptr_type=%d\n",
1712 gtid, target_gtid, flag, flag->get(), flag->get_type(),
1713 th->th.th_sleep_loc_type));
1714 __kmp_unlock_suspend_mx(th);
1715 __kmp_null_resume_wrapper(th);
1719 if (!flag->is_sleeping()) {
1720 KF_TRACE(5, (
"__kmp_resume_template: T#%d exiting, thread T#%d already "
1721 "awake: flag(%p): %u\n",
1722 gtid, target_gtid, flag->get(), (
unsigned int)flag->load()));
1723 __kmp_unlock_suspend_mx(th);
1727 KMP_DEBUG_ASSERT(flag);
1728 flag->unset_sleeping();
1729 TCW_PTR(th->th.th_sleep_loc, NULL);
1730 th->th.th_sleep_loc_type = flag_unset;
1732 KF_TRACE(5, (
"__kmp_resume_template: T#%d about to wakeup T#%d, reset "
1733 "sleep bit for flag's loc(%p): %u\n",
1734 gtid, target_gtid, flag->get(), (
unsigned int)flag->load()));
1736 #ifdef DEBUG_SUSPEND
1739 __kmp_print_cond(buffer, &th->th.th_suspend_cv);
1740 __kmp_printf(
"__kmp_resume_template: T#%d resuming T#%d: %s\n", gtid,
1741 target_gtid, buffer);
1744 status = pthread_cond_signal(&th->th.th_suspend_cv.c_cond);
1745 KMP_CHECK_SYSFAIL(
"pthread_cond_signal", status);
1746 __kmp_unlock_suspend_mx(th);
1747 KF_TRACE(30, (
"__kmp_resume_template: T#%d exiting after signaling wake up"
1749 gtid, target_gtid));
1752 template <
bool C,
bool S>
1753 void __kmp_resume_32(
int target_gtid, kmp_flag_32<C, S> *flag) {
1754 __kmp_resume_template(target_gtid, flag);
1756 template <
bool C,
bool S>
1757 void __kmp_resume_64(
int target_gtid, kmp_flag_64<C, S> *flag) {
1758 __kmp_resume_template(target_gtid, flag);
1760 template <
bool C,
bool S>
1761 void __kmp_atomic_resume_64(
int target_gtid, kmp_atomic_flag_64<C, S> *flag) {
1762 __kmp_resume_template(target_gtid, flag);
1764 void __kmp_resume_oncore(
int target_gtid, kmp_flag_oncore *flag) {
1765 __kmp_resume_template(target_gtid, flag);
1768 template void __kmp_resume_32<false, true>(
int, kmp_flag_32<false, true> *);
1769 template void __kmp_resume_32<false, false>(
int, kmp_flag_32<false, false> *);
1770 template void __kmp_resume_64<false, true>(
int, kmp_flag_64<false, true> *);
1772 __kmp_atomic_resume_64<false, true>(
int, kmp_atomic_flag_64<false, true> *);
1775 void __kmp_resume_monitor() {
1776 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
1779 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1780 KF_TRACE(30, (
"__kmp_resume_monitor: T#%d wants to wakeup T#%d enter\n", gtid,
1782 KMP_DEBUG_ASSERT(gtid != KMP_GTID_MONITOR);
1784 status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex);
1785 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status);
1786 #ifdef DEBUG_SUSPEND
1789 __kmp_print_cond(buffer, &__kmp_wait_cv.c_cond);
1790 __kmp_printf(
"__kmp_resume_monitor: T#%d resuming T#%d: %s\n", gtid,
1791 KMP_GTID_MONITOR, buffer);
1794 status = pthread_cond_signal(&__kmp_wait_cv.c_cond);
1795 KMP_CHECK_SYSFAIL(
"pthread_cond_signal", status);
1796 status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex);
1797 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status);
1798 KF_TRACE(30, (
"__kmp_resume_monitor: T#%d exiting after signaling wake up"
1800 gtid, KMP_GTID_MONITOR));
1804 void __kmp_yield() { sched_yield(); }
1806 void __kmp_gtid_set_specific(
int gtid) {
1807 if (__kmp_init_gtid) {
1809 status = pthread_setspecific(__kmp_gtid_threadprivate_key,
1810 (
void *)(intptr_t)(gtid + 1));
1811 KMP_CHECK_SYSFAIL(
"pthread_setspecific", status);
1813 KA_TRACE(50, (
"__kmp_gtid_set_specific: runtime shutdown, returning\n"));
1817 int __kmp_gtid_get_specific() {
1819 if (!__kmp_init_gtid) {
1820 KA_TRACE(50, (
"__kmp_gtid_get_specific: runtime shutdown, returning "
1821 "KMP_GTID_SHUTDOWN\n"));
1822 return KMP_GTID_SHUTDOWN;
1824 gtid = (int)(
size_t)pthread_getspecific(__kmp_gtid_threadprivate_key);
1826 gtid = KMP_GTID_DNE;
1830 KA_TRACE(50, (
"__kmp_gtid_get_specific: key:%d gtid:%d\n",
1831 __kmp_gtid_threadprivate_key, gtid));
1835 double __kmp_read_cpu_time(
void) {
1841 return (
double)(buffer.tms_utime + buffer.tms_cutime) /
1842 (
double)CLOCKS_PER_SEC;
1845 int __kmp_read_system_info(
struct kmp_sys_info *info) {
1847 struct rusage r_usage;
1849 memset(info, 0,
sizeof(*info));
1851 status = getrusage(RUSAGE_SELF, &r_usage);
1852 KMP_CHECK_SYSFAIL_ERRNO(
"getrusage", status);
1856 info->maxrss = r_usage.ru_maxrss;
1858 info->minflt = r_usage.ru_minflt;
1860 info->majflt = r_usage.ru_majflt;
1862 info->nswap = r_usage.ru_nswap;
1864 info->inblock = r_usage.ru_inblock;
1866 info->oublock = r_usage.ru_oublock;
1868 info->nvcsw = r_usage.ru_nvcsw;
1870 info->nivcsw = r_usage.ru_nivcsw;
1873 return (status != 0);
1876 void __kmp_read_system_time(
double *delta) {
1878 struct timeval tval;
1879 struct timespec stop;
1882 status = gettimeofday(&tval, NULL);
1883 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status);
1884 TIMEVAL_TO_TIMESPEC(&tval, &stop);
1885 t_ns = (double)(TS2NS(stop) - TS2NS(__kmp_sys_timer_data.start));
1886 *delta = (t_ns * 1e-9);
1889 void __kmp_clear_system_time(
void) {
1890 struct timeval tval;
1892 status = gettimeofday(&tval, NULL);
1893 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status);
1894 TIMEVAL_TO_TIMESPEC(&tval, &__kmp_sys_timer_data.start);
1897 static int __kmp_get_xproc(
void) {
1903 __kmp_type_convert(sysconf(_SC_NPROCESSORS_CONF), &(r));
1905 #elif KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_OPENBSD || \
1906 KMP_OS_HAIKU || KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_WASI || KMP_OS_AIX
1908 __kmp_type_convert(sysconf(_SC_NPROCESSORS_ONLN), &(r));
1912 size_t len =
sizeof(r);
1913 sysctlbyname(
"hw.logicalcpu", &r, &len, NULL, 0);
1917 #error "Unknown or unsupported OS."
1921 return r > 0 ? r : 2;
1925 int __kmp_read_from_file(
char const *path,
char const *format, ...) {
1929 va_start(args, format);
1930 FILE *f = fopen(path,
"rb");
1935 result = vfscanf(f, format, args);
1942 void __kmp_runtime_initialize(
void) {
1944 pthread_mutexattr_t mutex_attr;
1945 pthread_condattr_t cond_attr;
1947 if (__kmp_init_runtime) {
1951 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1952 if (!__kmp_cpuinfo.initialized) {
1953 __kmp_query_cpuid(&__kmp_cpuinfo);
1957 __kmp_xproc = __kmp_get_xproc();
1959 #if !KMP_32_BIT_ARCH
1963 status = getrlimit(RLIMIT_STACK, &rlim);
1965 __kmp_stksize = rlim.rlim_cur;
1966 __kmp_check_stksize(&__kmp_stksize);
1970 if (sysconf(_SC_THREADS)) {
1973 __kmp_type_convert(sysconf(_SC_THREAD_THREADS_MAX), &(__kmp_sys_max_nth));
1975 if (__kmp_sys_max_nth == -1) {
1978 __kmp_sys_max_nth = KMP_MAX_NTH;
1981 if (__kmp_sys_max_nth == -1) {
1983 __kmp_sys_max_nth = INT_MAX;
1984 }
else if (__kmp_sys_max_nth <= 1) {
1986 __kmp_sys_max_nth = KMP_MAX_NTH;
1991 __kmp_sys_min_stksize = sysconf(_SC_THREAD_STACK_MIN);
1992 if (__kmp_sys_min_stksize <= 1) {
1993 __kmp_sys_min_stksize = KMP_MIN_STKSIZE;
1998 __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
2000 status = pthread_key_create(&__kmp_gtid_threadprivate_key,
2001 __kmp_internal_end_dest);
2002 KMP_CHECK_SYSFAIL(
"pthread_key_create", status);
2003 status = pthread_mutexattr_init(&mutex_attr);
2004 KMP_CHECK_SYSFAIL(
"pthread_mutexattr_init", status);
2005 status = pthread_mutex_init(&__kmp_wait_mx.m_mutex, &mutex_attr);
2006 KMP_CHECK_SYSFAIL(
"pthread_mutex_init", status);
2007 status = pthread_mutexattr_destroy(&mutex_attr);
2008 KMP_CHECK_SYSFAIL(
"pthread_mutexattr_destroy", status);
2009 status = pthread_condattr_init(&cond_attr);
2010 KMP_CHECK_SYSFAIL(
"pthread_condattr_init", status);
2011 status = pthread_cond_init(&__kmp_wait_cv.c_cond, &cond_attr);
2012 KMP_CHECK_SYSFAIL(
"pthread_cond_init", status);
2013 status = pthread_condattr_destroy(&cond_attr);
2014 KMP_CHECK_SYSFAIL(
"pthread_condattr_destroy", status);
2016 __kmp_itt_initialize();
2019 __kmp_init_runtime = TRUE;
2022 void __kmp_runtime_destroy(
void) {
2025 if (!__kmp_init_runtime) {
2030 __kmp_itt_destroy();
2033 status = pthread_key_delete(__kmp_gtid_threadprivate_key);
2034 KMP_CHECK_SYSFAIL(
"pthread_key_delete", status);
2036 status = pthread_mutex_destroy(&__kmp_wait_mx.m_mutex);
2037 if (status != 0 && status != EBUSY) {
2038 KMP_SYSFAIL(
"pthread_mutex_destroy", status);
2040 status = pthread_cond_destroy(&__kmp_wait_cv.c_cond);
2041 if (status != 0 && status != EBUSY) {
2042 KMP_SYSFAIL(
"pthread_cond_destroy", status);
2044 #if KMP_AFFINITY_SUPPORTED
2045 __kmp_affinity_uninitialize();
2048 __kmp_init_runtime = FALSE;
2053 void __kmp_thread_sleep(
int millis) { sleep((millis + 500) / 1000); }
2056 void __kmp_elapsed(
double *t) {
2058 #ifdef FIX_SGI_CLOCK
2061 status = clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts);
2062 KMP_CHECK_SYSFAIL_ERRNO(
"clock_gettime", status);
2064 (double)ts.tv_nsec * (1.0 / (
double)KMP_NSEC_PER_SEC) + (
double)ts.tv_sec;
2068 status = gettimeofday(&tv, NULL);
2069 KMP_CHECK_SYSFAIL_ERRNO(
"gettimeofday", status);
2071 (double)tv.tv_usec * (1.0 / (
double)KMP_USEC_PER_SEC) + (
double)tv.tv_sec;
2076 void __kmp_elapsed_tick(
double *t) { *t = 1 / (double)CLOCKS_PER_SEC; }
2079 kmp_uint64 __kmp_now_nsec() {
2081 gettimeofday(&t, NULL);
2082 kmp_uint64 nsec = (kmp_uint64)KMP_NSEC_PER_SEC * (kmp_uint64)t.tv_sec +
2083 (kmp_uint64)1000 * (kmp_uint64)t.tv_usec;
2087 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
2089 void __kmp_initialize_system_tick() {
2090 kmp_uint64 now, nsec2, diff;
2091 kmp_uint64 delay = 1000000;
2092 kmp_uint64 nsec = __kmp_now_nsec();
2093 kmp_uint64 goal = __kmp_hardware_timestamp() + delay;
2094 while ((now = __kmp_hardware_timestamp()) < goal)
2096 nsec2 = __kmp_now_nsec();
2097 diff = nsec2 - nsec;
2099 double tpus = 1000.0 * (double)(delay + (now - goal)) / (
double)diff;
2101 __kmp_ticks_per_msec = (kmp_uint64)(tpus * 1000.0);
2102 __kmp_ticks_per_usec = (kmp_uint64)tpus;
2111 int __kmp_is_address_mapped(
void *addr) {
2116 #if KMP_OS_LINUX || KMP_OS_HURD
2121 char *name = __kmp_str_format(
"/proc/%d/maps", getpid());
2124 file = fopen(name,
"r");
2125 KMP_ASSERT(file != NULL);
2129 void *beginning = NULL;
2130 void *ending = NULL;
2133 rc = fscanf(file,
"%p-%p %4s %*[^\n]\n", &beginning, &ending, perms);
2137 KMP_ASSERT(rc == 3 &&
2138 KMP_STRLEN(perms) == 4);
2141 if ((addr >= beginning) && (addr < ending)) {
2143 if (strcmp(perms,
"rw") == 0) {
2153 KMP_INTERNAL_FREE(name);
2154 #elif KMP_OS_FREEBSD
2157 int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, getpid()};
2158 rc = sysctl(mib, 4, NULL, &lstsz, NULL, 0);
2163 lstsz = lstsz * 4 / 3;
2164 buf =
reinterpret_cast<char *
>(KMP_INTERNAL_MALLOC(lstsz));
2165 rc = sysctl(mib, 4, buf, &lstsz, NULL, 0);
2167 KMP_INTERNAL_FREE(buf);
2172 char *up = buf + lstsz;
2175 struct kinfo_vmentry *cur =
reinterpret_cast<struct kinfo_vmentry *
>(lw);
2176 size_t cursz = cur->kve_structsize;
2179 void *start =
reinterpret_cast<void *
>(cur->kve_start);
2180 void *end =
reinterpret_cast<void *
>(cur->kve_end);
2182 if ((addr >= start) && (addr < end)) {
2183 if ((cur->kve_protection & KVME_PROT_READ) != 0 &&
2184 (cur->kve_protection & KVME_PROT_WRITE) != 0) {
2191 KMP_INTERNAL_FREE(buf);
2192 #elif KMP_OS_DRAGONFLY
2193 char err[_POSIX2_LINE_MAX];
2197 vm_map_entry entry, *c;
2203 fd = kvm_openfiles(
nullptr,
nullptr,
nullptr, O_RDONLY, err);
2208 proc = kvm_getprocs(fd, KERN_PROC_PID, getpid(), &num);
2210 if (kvm_read(fd,
static_cast<uintptr_t
>(proc->kp_paddr), &p,
sizeof(p)) !=
2212 kvm_read(fd,
reinterpret_cast<uintptr_t
>(p.p_vmspace), &sp,
sizeof(sp)) !=
2220 uaddr =
reinterpret_cast<uintptr_t
>(addr);
2221 for (c = kvm_vm_map_entry_first(fd, cur, &entry); c;
2222 c = kvm_vm_map_entry_next(fd, c, &entry)) {
2223 if ((uaddr >= entry.ba.start) && (uaddr <= entry.ba.end)) {
2224 if ((entry.protection & VM_PROT_READ) != 0 &&
2225 (entry.protection & VM_PROT_WRITE) != 0) {
2233 #elif KMP_OS_SOLARIS
2234 prxmap_t *cur, *map;
2239 pid_t pid = getpid();
2240 char *name = __kmp_str_format(
"/proc/%d/xmap", pid);
2241 fd = open(name, O_RDONLY);
2243 KMP_INTERNAL_FREE(name);
2247 size_t sz = (1 << 20);
2248 buf = KMP_INTERNAL_MALLOC(sz);
2250 while (sz > 0 && (rd = pread(fd, buf, sz, 0)) == sz) {
2253 newbuf = KMP_INTERNAL_REALLOC(buf, sz);
2257 map =
reinterpret_cast<prxmap_t *
>(buf);
2258 uaddr =
reinterpret_cast<uintptr_t
>(addr);
2260 for (cur = map; rd > 0; cur++, rd = -
sizeof(*map)) {
2261 if (uaddr >= cur->pr_vaddr && uaddr < cur->pr_vaddr) {
2262 if ((cur->pr_mflags & MA_READ) != 0 && (cur->pr_mflags & MA_WRITE) != 0) {
2269 KMP_INTERNAL_FREE(map);
2271 KMP_INTERNAL_FREE(name);
2279 rc = vm_read_overwrite(
2281 (vm_address_t)(addr),
2283 (vm_address_t)(&buffer),
2296 mib[2] = VM_PROC_MAP;
2298 mib[4] =
sizeof(
struct kinfo_vmentry);
2301 rc = sysctl(mib, __arraycount(mib), NULL, &size, NULL, 0);
2305 size = size * 4 / 3;
2306 struct kinfo_vmentry *kiv = (
struct kinfo_vmentry *)KMP_INTERNAL_MALLOC(size);
2309 rc = sysctl(mib, __arraycount(mib), kiv, &size, NULL, 0);
2313 for (
size_t i = 0; i < size; i++) {
2314 if (kiv[i].kve_start >= (uint64_t)addr &&
2315 kiv[i].kve_end <= (uint64_t)addr) {
2320 KMP_INTERNAL_FREE(kiv);
2321 #elif KMP_OS_OPENBSD
2325 mib[1] = KERN_PROC_VMMAP;
2330 rc = sysctl(mib, 3, NULL, &size, NULL, 0);
2335 struct kinfo_vmentry kiv = {.kve_start = 0};
2337 while ((rc = sysctl(mib, 3, &kiv, &size, NULL, 0)) == 0) {
2339 if (kiv.kve_end == end)
2342 if (kiv.kve_start >= (uint64_t)addr && kiv.kve_end <= (uint64_t)addr) {
2349 found = (int)addr < (__builtin_wasm_memory_size(0) * PAGESIZE);
2352 uint32_t loadQueryBufSize = 4096u;
2356 loadQueryBuf = (
char *)KMP_INTERNAL_MALLOC(loadQueryBufSize);
2357 if (loadQueryBuf == NULL) {
2361 rc = loadquery(L_GETXINFO | L_IGNOREUNLOAD, loadQueryBuf, loadQueryBufSize);
2363 KMP_INTERNAL_FREE(loadQueryBuf);
2364 if (errno != ENOMEM) {
2368 loadQueryBufSize <<= 1;
2375 struct ld_xinfo *curLdInfo = (
struct ld_xinfo *)loadQueryBuf;
2379 uintptr_t curDataStart = (uintptr_t)curLdInfo->ldinfo_dataorg;
2380 uintptr_t curDataEnd = curDataStart + curLdInfo->ldinfo_datasize;
2383 if (curDataStart <= (uintptr_t)addr && (uintptr_t)addr < curDataEnd) {
2387 if (curLdInfo->ldinfo_next == 0u) {
2391 curLdInfo = (
struct ld_xinfo *)((
char *)curLdInfo + curLdInfo->ldinfo_next);
2393 KMP_INTERNAL_FREE(loadQueryBuf);
2400 #error "Unknown or unsupported OS"
2408 #ifdef USE_LOAD_BALANCE
2410 #if KMP_OS_DARWIN || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
2411 KMP_OS_OPENBSD || KMP_OS_SOLARIS
2418 int __kmp_get_load_balance(
int max) {
2422 int res = getloadavg(averages, 3);
2427 if (__kmp_load_balance_interval < 180 && (res >= 1)) {
2428 ret_avg = (int)averages[0];
2429 }
else if ((__kmp_load_balance_interval >= 180 &&
2430 __kmp_load_balance_interval < 600) &&
2432 ret_avg = (int)averages[1];
2433 }
else if ((__kmp_load_balance_interval >= 600) && (res == 3)) {
2434 ret_avg = (int)averages[2];
2446 int __kmp_get_load_balance(
int max) {
2448 static int glb_running_threads = 0;
2450 static double glb_call_time = 0;
2451 int running_threads = 0;
2453 double call_time = 0.0;
2455 __kmp_elapsed(&call_time);
2457 if (glb_call_time &&
2458 (call_time - glb_call_time < __kmp_load_balance_interval))
2459 return glb_running_threads;
2461 glb_call_time = call_time;
2468 int logical_cpus = perfstat_cpu(NULL, NULL,
sizeof(perfstat_cpu_t), 0);
2469 if (logical_cpus <= 0) {
2474 perfstat_cpu_t *cpu_stat = (perfstat_cpu_t *)KMP_INTERNAL_MALLOC(
2475 logical_cpus *
sizeof(perfstat_cpu_t));
2476 if (cpu_stat == NULL) {
2483 perfstat_id_t first_cpu_name;
2484 strcpy(first_cpu_name.name, FIRST_CPU);
2487 int rc = perfstat_cpu(&first_cpu_name, cpu_stat,
sizeof(perfstat_cpu_t),
2489 KMP_DEBUG_ASSERT(rc == logical_cpus);
2491 KMP_INTERNAL_FREE(cpu_stat);
2495 for (
int i = 0; i < logical_cpus; ++i) {
2496 running_threads += cpu_stat[i].runque;
2497 if (running_threads >= max)
2504 KMP_DEBUG_ASSERT(running_threads > 0);
2505 if (running_threads <= 0)
2506 running_threads = 1;
2508 KMP_INTERNAL_FREE(cpu_stat);
2510 glb_running_threads = running_threads;
2512 return running_threads;
2521 int __kmp_get_load_balance(
int max) {
2522 static int permanent_error = 0;
2523 static int glb_running_threads = 0;
2525 static double glb_call_time = 0;
2527 int running_threads = 0;
2529 DIR *proc_dir = NULL;
2530 struct dirent *proc_entry = NULL;
2532 kmp_str_buf_t task_path;
2533 DIR *task_dir = NULL;
2534 struct dirent *task_entry = NULL;
2535 int task_path_fixed_len;
2537 kmp_str_buf_t stat_path;
2539 int stat_path_fixed_len;
2542 int total_processes = 0;
2545 double call_time = 0.0;
2547 __kmp_str_buf_init(&task_path);
2548 __kmp_str_buf_init(&stat_path);
2550 __kmp_elapsed(&call_time);
2552 if (glb_call_time &&
2553 (call_time - glb_call_time < __kmp_load_balance_interval)) {
2554 running_threads = glb_running_threads;
2558 glb_call_time = call_time;
2561 if (permanent_error) {
2562 running_threads = -1;
2571 proc_dir = opendir(
"/proc");
2572 if (proc_dir == NULL) {
2575 running_threads = -1;
2576 permanent_error = 1;
2581 __kmp_str_buf_cat(&task_path,
"/proc/", 6);
2582 task_path_fixed_len = task_path.used;
2584 proc_entry = readdir(proc_dir);
2585 while (proc_entry != NULL) {
2588 if (proc_entry->d_type == DT_DIR && isdigit(proc_entry->d_name[0])) {
2598 KMP_DEBUG_ASSERT(total_processes != 1 ||
2599 strcmp(proc_entry->d_name,
"1") == 0);
2602 task_path.used = task_path_fixed_len;
2603 __kmp_str_buf_cat(&task_path, proc_entry->d_name,
2604 KMP_STRLEN(proc_entry->d_name));
2605 __kmp_str_buf_cat(&task_path,
"/task", 5);
2607 task_dir = opendir(task_path.str);
2608 if (task_dir == NULL) {
2617 if (strcmp(proc_entry->d_name,
"1") == 0) {
2618 running_threads = -1;
2619 permanent_error = 1;
2624 __kmp_str_buf_clear(&stat_path);
2625 __kmp_str_buf_cat(&stat_path, task_path.str, task_path.used);
2626 __kmp_str_buf_cat(&stat_path,
"/", 1);
2627 stat_path_fixed_len = stat_path.used;
2629 task_entry = readdir(task_dir);
2630 while (task_entry != NULL) {
2632 if (proc_entry->d_type == DT_DIR && isdigit(task_entry->d_name[0])) {
2639 stat_path_fixed_len;
2640 __kmp_str_buf_cat(&stat_path, task_entry->d_name,
2641 KMP_STRLEN(task_entry->d_name));
2642 __kmp_str_buf_cat(&stat_path,
"/stat", 5);
2646 stat_file = open(stat_path.str, O_RDONLY);
2647 if (stat_file == -1) {
2677 len = read(stat_file, buffer,
sizeof(buffer) - 1);
2684 char *close_parent = strstr(buffer,
") ");
2685 if (close_parent != NULL) {
2686 char state = *(close_parent + 2);
2689 if (running_threads >= max) {
2699 task_entry = readdir(task_dir);
2705 proc_entry = readdir(proc_dir);
2711 KMP_DEBUG_ASSERT(running_threads > 0);
2712 if (running_threads <= 0) {
2713 running_threads = 1;
2717 if (proc_dir != NULL) {
2720 __kmp_str_buf_free(&task_path);
2721 if (task_dir != NULL) {
2724 __kmp_str_buf_free(&stat_path);
2725 if (stat_file != -1) {
2729 glb_running_threads = running_threads;
2731 return running_threads;
2739 #if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || \
2740 ((KMP_OS_LINUX || KMP_OS_DARWIN) && KMP_ARCH_AARCH64) || \
2741 KMP_ARCH_PPC64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 || \
2742 KMP_ARCH_ARM || KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_PPC_XCOFF || \
2743 KMP_ARCH_AARCH64_32)
2749 typedef void (*microtask_t0)(
int *,
int *);
2750 typedef void (*microtask_t1)(
int *,
int *,
void *);
2751 typedef void (*microtask_t2)(
int *,
int *,
void *,
void *);
2752 typedef void (*microtask_t3)(
int *,
int *,
void *,
void *,
void *);
2753 typedef void (*microtask_t4)(
int *,
int *,
void *,
void *,
void *,
void *);
2754 typedef void (*microtask_t5)(
int *,
int *,
void *,
void *,
void *,
void *,
2756 typedef void (*microtask_t6)(
int *,
int *,
void *,
void *,
void *,
void *,
2758 typedef void (*microtask_t7)(
int *,
int *,
void *,
void *,
void *,
void *,
2759 void *,
void *,
void *);
2760 typedef void (*microtask_t8)(
int *,
int *,
void *,
void *,
void *,
void *,
2761 void *,
void *,
void *,
void *);
2762 typedef void (*microtask_t9)(
int *,
int *,
void *,
void *,
void *,
void *,
2763 void *,
void *,
void *,
void *,
void *);
2764 typedef void (*microtask_t10)(
int *,
int *,
void *,
void *,
void *,
void *,
2765 void *,
void *,
void *,
void *,
void *,
void *);
2766 typedef void (*microtask_t11)(
int *,
int *,
void *,
void *,
void *,
void *,
2767 void *,
void *,
void *,
void *,
void *,
void *,
2769 typedef void (*microtask_t12)(
int *,
int *,
void *,
void *,
void *,
void *,
2770 void *,
void *,
void *,
void *,
void *,
void *,
2772 typedef void (*microtask_t13)(
int *,
int *,
void *,
void *,
void *,
void *,
2773 void *,
void *,
void *,
void *,
void *,
void *,
2774 void *,
void *,
void *);
2775 typedef void (*microtask_t14)(
int *,
int *,
void *,
void *,
void *,
void *,
2776 void *,
void *,
void *,
void *,
void *,
void *,
2777 void *,
void *,
void *,
void *);
2778 typedef void (*microtask_t15)(
int *,
int *,
void *,
void *,
void *,
void *,
2779 void *,
void *,
void *,
void *,
void *,
void *,
2780 void *,
void *,
void *,
void *,
void *);
2784 int __kmp_invoke_microtask(microtask_t pkfn,
int gtid,
int tid,
int argc,
2788 void **exit_frame_ptr
2792 *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
2797 fprintf(stderr,
"Too many args to microtask: %d!\n", argc);
2801 (*(microtask_t0)pkfn)(>id, &tid);
2804 (*(microtask_t1)pkfn)(>id, &tid, p_argv[0]);
2807 (*(microtask_t2)pkfn)(>id, &tid, p_argv[0], p_argv[1]);
2810 (*(microtask_t3)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2]);
2813 (*(microtask_t4)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2817 (*(microtask_t5)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2818 p_argv[3], p_argv[4]);
2821 (*(microtask_t6)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2822 p_argv[3], p_argv[4], p_argv[5]);
2825 (*(microtask_t7)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2826 p_argv[3], p_argv[4], p_argv[5], p_argv[6]);
2829 (*(microtask_t8)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2830 p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2834 (*(microtask_t9)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2835 p_argv[3], p_argv[4], p_argv[5], p_argv[6], p_argv[7],
2839 (*(microtask_t10)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2840 p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2841 p_argv[7], p_argv[8], p_argv[9]);
2844 (*(microtask_t11)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2845 p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2846 p_argv[7], p_argv[8], p_argv[9], p_argv[10]);
2849 (*(microtask_t12)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2850 p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2851 p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2855 (*(microtask_t13)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2856 p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2857 p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2858 p_argv[11], p_argv[12]);
2861 (*(microtask_t14)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2862 p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2863 p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2864 p_argv[11], p_argv[12], p_argv[13]);
2867 (*(microtask_t15)pkfn)(>id, &tid, p_argv[0], p_argv[1], p_argv[2],
2868 p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2869 p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2870 p_argv[11], p_argv[12], p_argv[13], p_argv[14]);
2883 pthread_cond_t hidden_helper_threads_initz_cond_var;
2884 pthread_mutex_t hidden_helper_threads_initz_lock;
2885 volatile int hidden_helper_initz_signaled = FALSE;
2888 pthread_cond_t hidden_helper_threads_deinitz_cond_var;
2889 pthread_mutex_t hidden_helper_threads_deinitz_lock;
2890 volatile int hidden_helper_deinitz_signaled = FALSE;
2893 pthread_cond_t hidden_helper_main_thread_cond_var;
2894 pthread_mutex_t hidden_helper_main_thread_lock;
2895 volatile int hidden_helper_main_thread_signaled = FALSE;
2900 sem_t hidden_helper_task_sem;
2903 void __kmp_hidden_helper_worker_thread_wait() {
2904 int status = sem_wait(&hidden_helper_task_sem);
2905 KMP_CHECK_SYSFAIL(
"sem_wait", status);
2908 void __kmp_do_initialize_hidden_helper_threads() {
2911 pthread_cond_init(&hidden_helper_threads_initz_cond_var,
nullptr);
2912 KMP_CHECK_SYSFAIL(
"pthread_cond_init", status);
2914 status = pthread_cond_init(&hidden_helper_threads_deinitz_cond_var,
nullptr);
2915 KMP_CHECK_SYSFAIL(
"pthread_cond_init", status);
2917 status = pthread_cond_init(&hidden_helper_main_thread_cond_var,
nullptr);
2918 KMP_CHECK_SYSFAIL(
"pthread_cond_init", status);
2920 status = pthread_mutex_init(&hidden_helper_threads_initz_lock,
nullptr);
2921 KMP_CHECK_SYSFAIL(
"pthread_mutex_init", status);
2923 status = pthread_mutex_init(&hidden_helper_threads_deinitz_lock,
nullptr);
2924 KMP_CHECK_SYSFAIL(
"pthread_mutex_init", status);
2926 status = pthread_mutex_init(&hidden_helper_main_thread_lock,
nullptr);
2927 KMP_CHECK_SYSFAIL(
"pthread_mutex_init", status);
2930 status = sem_init(&hidden_helper_task_sem, 0, 0);
2931 KMP_CHECK_SYSFAIL(
"sem_init", status);
2935 status = pthread_create(
2937 [](
void *) ->
void * {
2938 __kmp_hidden_helper_threads_initz_routine();
2942 KMP_CHECK_SYSFAIL(
"pthread_create", status);
2945 void __kmp_hidden_helper_threads_initz_wait() {
2948 int status = pthread_mutex_lock(&hidden_helper_threads_initz_lock);
2949 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status);
2951 if (!TCR_4(hidden_helper_initz_signaled)) {
2952 status = pthread_cond_wait(&hidden_helper_threads_initz_cond_var,
2953 &hidden_helper_threads_initz_lock);
2954 KMP_CHECK_SYSFAIL(
"pthread_cond_wait", status);
2957 status = pthread_mutex_unlock(&hidden_helper_threads_initz_lock);
2958 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status);
2961 void __kmp_hidden_helper_initz_release() {
2963 int status = pthread_mutex_lock(&hidden_helper_threads_initz_lock);
2964 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status);
2966 status = pthread_cond_signal(&hidden_helper_threads_initz_cond_var);
2967 KMP_CHECK_SYSFAIL(
"pthread_cond_wait", status);
2969 TCW_SYNC_4(hidden_helper_initz_signaled, TRUE);
2971 status = pthread_mutex_unlock(&hidden_helper_threads_initz_lock);
2972 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status);
2975 void __kmp_hidden_helper_main_thread_wait() {
2978 int status = pthread_mutex_lock(&hidden_helper_main_thread_lock);
2979 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status);
2981 if (!TCR_4(hidden_helper_main_thread_signaled)) {
2982 status = pthread_cond_wait(&hidden_helper_main_thread_cond_var,
2983 &hidden_helper_main_thread_lock);
2984 KMP_CHECK_SYSFAIL(
"pthread_cond_wait", status);
2987 status = pthread_mutex_unlock(&hidden_helper_main_thread_lock);
2988 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status);
2991 void __kmp_hidden_helper_main_thread_release() {
2994 int status = pthread_mutex_lock(&hidden_helper_main_thread_lock);
2995 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status);
2997 status = pthread_cond_signal(&hidden_helper_main_thread_cond_var);
2998 KMP_CHECK_SYSFAIL(
"pthread_cond_signal", status);
3001 TCW_SYNC_4(hidden_helper_main_thread_signaled, TRUE);
3003 status = pthread_mutex_unlock(&hidden_helper_main_thread_lock);
3004 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status);
3007 void __kmp_hidden_helper_worker_thread_signal() {
3008 int status = sem_post(&hidden_helper_task_sem);
3009 KMP_CHECK_SYSFAIL(
"sem_post", status);
3012 void __kmp_hidden_helper_threads_deinitz_wait() {
3015 int status = pthread_mutex_lock(&hidden_helper_threads_deinitz_lock);
3016 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status);
3018 if (!TCR_4(hidden_helper_deinitz_signaled)) {
3019 status = pthread_cond_wait(&hidden_helper_threads_deinitz_cond_var,
3020 &hidden_helper_threads_deinitz_lock);
3021 KMP_CHECK_SYSFAIL(
"pthread_cond_wait", status);
3024 status = pthread_mutex_unlock(&hidden_helper_threads_deinitz_lock);
3025 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status);
3028 void __kmp_hidden_helper_threads_deinitz_release() {
3029 int status = pthread_mutex_lock(&hidden_helper_threads_deinitz_lock);
3030 KMP_CHECK_SYSFAIL(
"pthread_mutex_lock", status);
3032 status = pthread_cond_signal(&hidden_helper_threads_deinitz_cond_var);
3033 KMP_CHECK_SYSFAIL(
"pthread_cond_wait", status);
3035 TCW_SYNC_4(hidden_helper_deinitz_signaled, TRUE);
3037 status = pthread_mutex_unlock(&hidden_helper_threads_deinitz_lock);
3038 KMP_CHECK_SYSFAIL(
"pthread_mutex_unlock", status);
3041 void __kmp_hidden_helper_worker_thread_wait() {
3042 KMP_ASSERT(0 &&
"Hidden helper task is not supported on this OS");
3045 void __kmp_do_initialize_hidden_helper_threads() {
3046 KMP_ASSERT(0 &&
"Hidden helper task is not supported on this OS");
3049 void __kmp_hidden_helper_threads_initz_wait() {
3050 KMP_ASSERT(0 &&
"Hidden helper task is not supported on this OS");
3053 void __kmp_hidden_helper_initz_release() {
3054 KMP_ASSERT(0 &&
"Hidden helper task is not supported on this OS");
3057 void __kmp_hidden_helper_main_thread_wait() {
3058 KMP_ASSERT(0 &&
"Hidden helper task is not supported on this OS");
3061 void __kmp_hidden_helper_main_thread_release() {
3062 KMP_ASSERT(0 &&
"Hidden helper task is not supported on this OS");
3065 void __kmp_hidden_helper_worker_thread_signal() {
3066 KMP_ASSERT(0 &&
"Hidden helper task is not supported on this OS");
3069 void __kmp_hidden_helper_threads_deinitz_wait() {
3070 KMP_ASSERT(0 &&
"Hidden helper task is not supported on this OS");
3073 void __kmp_hidden_helper_threads_deinitz_release() {
3074 KMP_ASSERT(0 &&
"Hidden helper task is not supported on this OS");
3078 bool __kmp_detect_shm() {
3079 DIR *dir = opendir(
"/dev/shm");
3083 }
else if (ENOENT == errno) {
3090 bool __kmp_detect_tmp() {
3091 DIR *dir = opendir(
"/tmp");
3095 }
else if (ENOENT == errno) {
#define KMP_INIT_PARTITIONED_TIMERS(name)
Initializes the partitioned timers to begin with name.