LLVM OpenMP* Runtime Library
Loading...
Searching...
No Matches
z_Linux_util.cpp
1/*
2 * z_Linux_util.cpp -- platform specific routines.
3 */
4
5//===----------------------------------------------------------------------===//
6//
7// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8// See https://llvm.org/LICENSE.txt for license information.
9// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10//
11//===----------------------------------------------------------------------===//
12
13#include "kmp.h"
14#include "kmp_affinity.h"
15#include "kmp_i18n.h"
16#include "kmp_io.h"
17#include "kmp_itt.h"
18#include "kmp_lock.h"
19#include "kmp_stats.h"
20#include "kmp_str.h"
21#include "kmp_wait_release.h"
22#include "kmp_wrapper_getpid.h"
23
24#if !KMP_OS_DRAGONFLY && !KMP_OS_FREEBSD && !KMP_OS_NETBSD && !KMP_OS_OPENBSD
25#include <alloca.h>
26#endif
27#include <math.h> // HUGE_VAL.
28#if KMP_OS_LINUX
29#include <semaphore.h>
30#endif // KMP_OS_LINUX
31#include <sys/resource.h>
32#if KMP_OS_AIX
33#include <sys/ldr.h>
34#include <libperfstat.h>
35#elif !KMP_OS_HAIKU
36#include <sys/syscall.h>
37#endif
38#include <sys/time.h>
39#include <sys/times.h>
40#include <unistd.h>
41
42#if KMP_OS_LINUX
43#include <sys/sysinfo.h>
44#if KMP_USE_FUTEX
45// We should really include <futex.h>, but that causes compatibility problems on
46// different Linux* OS distributions that either require that you include (or
47// break when you try to include) <pci/types.h>. Since all we need is the two
48// macros below (which are part of the kernel ABI, so can't change) we just
49// define the constants here and don't include <futex.h>
50#ifndef FUTEX_WAIT
51#define FUTEX_WAIT 0
52#endif
53#ifndef FUTEX_WAKE
54#define FUTEX_WAKE 1
55#endif
56#endif
57#elif KMP_OS_DARWIN
58#include <mach/mach.h>
59#include <sys/sysctl.h>
60#elif KMP_OS_DRAGONFLY || KMP_OS_FREEBSD
61#include <sys/types.h>
62#include <sys/sysctl.h>
63#include <sys/user.h>
64#include <pthread_np.h>
65#if KMP_OS_DRAGONFLY
66#include <kvm.h>
67#endif
68#elif KMP_OS_NETBSD || KMP_OS_OPENBSD
69#include <sys/types.h>
70#include <sys/sysctl.h>
71#if KMP_OS_NETBSD
72#include <sched.h>
73#endif
74#if KMP_OS_OPENBSD
75#include <pthread_np.h>
76#endif
77#elif KMP_OS_SOLARIS
78#include <libproc.h>
79#include <procfs.h>
80#include <thread.h>
81#include <sys/loadavg.h>
82#endif
83
84#include <ctype.h>
85#include <dirent.h>
86#include <fcntl.h>
87
88struct kmp_sys_timer {
89 struct timespec start;
90};
91
92#ifndef TIMEVAL_TO_TIMESPEC
93// Convert timeval to timespec.
94#define TIMEVAL_TO_TIMESPEC(tv, ts) \
95 do { \
96 (ts)->tv_sec = (tv)->tv_sec; \
97 (ts)->tv_nsec = (tv)->tv_usec * 1000; \
98 } while (0)
99#endif
100
101// Convert timespec to nanoseconds.
102#define TS2NS(timespec) \
103 (((timespec).tv_sec * (long int)1e9) + (timespec).tv_nsec)
104
105static struct kmp_sys_timer __kmp_sys_timer_data;
106
107#if KMP_HANDLE_SIGNALS
108typedef void (*sig_func_t)(int);
109STATIC_EFI2_WORKAROUND struct sigaction __kmp_sighldrs[NSIG];
110static sigset_t __kmp_sigset;
111#endif
112
113static int __kmp_init_runtime = FALSE;
114
115static int __kmp_fork_count = 0;
116
117static pthread_condattr_t __kmp_suspend_cond_attr;
118static pthread_mutexattr_t __kmp_suspend_mutex_attr;
119
120static kmp_cond_align_t __kmp_wait_cv;
121static kmp_mutex_align_t __kmp_wait_mx;
122
123kmp_uint64 __kmp_ticks_per_msec = 1000000;
124kmp_uint64 __kmp_ticks_per_usec = 1000;
125
126#ifdef DEBUG_SUSPEND
127static void __kmp_print_cond(char *buffer, kmp_cond_align_t *cond) {
128 KMP_SNPRINTF(buffer, 128, "(cond (lock (%ld, %d)), (descr (%p)))",
129 cond->c_cond.__c_lock.__status, cond->c_cond.__c_lock.__spinlock,
130 cond->c_cond.__c_waiting);
131}
132#endif
133
134#if ((KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \
135 KMP_OS_AIX) && \
136 KMP_AFFINITY_SUPPORTED)
137
138/* Affinity support */
139
140void __kmp_affinity_bind_thread(int which) {
141 KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
142 "Illegal set affinity operation when not capable");
143
144 kmp_affin_mask_t *mask;
145 KMP_CPU_ALLOC_ON_STACK(mask);
146 KMP_CPU_ZERO(mask);
147 KMP_CPU_SET(which, mask);
148 __kmp_set_system_affinity(mask, TRUE);
149 KMP_CPU_FREE_FROM_STACK(mask);
150}
151
152#if KMP_OS_AIX
153void __kmp_affinity_determine_capable(const char *env_var) {
154 // All versions of AIX support bindprocessor().
155
156 size_t mask_size = __kmp_xproc / CHAR_BIT;
157 // Round up to byte boundary.
158 if (__kmp_xproc % CHAR_BIT)
159 ++mask_size;
160
161 // Round up to the mask_size_type boundary.
162 if (mask_size % sizeof(__kmp_affin_mask_size))
163 mask_size += sizeof(__kmp_affin_mask_size) -
164 mask_size % sizeof(__kmp_affin_mask_size);
165 KMP_AFFINITY_ENABLE(mask_size);
166 KA_TRACE(10,
167 ("__kmp_affinity_determine_capable: "
168 "AIX OS affinity interface bindprocessor functional (mask size = "
169 "%" KMP_SIZE_T_SPEC ").\n",
170 __kmp_affin_mask_size));
171}
172
173#else // !KMP_OS_AIX
174
175/* Determine if we can access affinity functionality on this version of
176 * Linux* OS by checking __NR_sched_{get,set}affinity system calls, and set
177 * __kmp_affin_mask_size to the appropriate value (0 means not capable). */
178void __kmp_affinity_determine_capable(const char *env_var) {
179 // Check and see if the OS supports thread affinity.
180
181#if KMP_OS_LINUX
182#define KMP_CPU_SET_SIZE_LIMIT (1024 * 1024)
183#define KMP_CPU_SET_TRY_SIZE CACHE_LINE
184#elif KMP_OS_FREEBSD || KMP_OS_DRAGONFLY
185#define KMP_CPU_SET_SIZE_LIMIT (sizeof(cpuset_t))
186#elif KMP_OS_NETBSD
187#define KMP_CPU_SET_SIZE_LIMIT (256)
188#endif
189
190 int verbose = __kmp_affinity.flags.verbose;
191 int warnings = __kmp_affinity.flags.warnings;
192 enum affinity_type type = __kmp_affinity.type;
193
194#if KMP_OS_LINUX
195 long gCode;
196 unsigned char *buf;
197 buf = (unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT);
198
199 // If the syscall returns a suggestion for the size,
200 // then we don't have to search for an appropriate size.
201 gCode = syscall(__NR_sched_getaffinity, 0, KMP_CPU_SET_TRY_SIZE, buf);
202 KA_TRACE(30, ("__kmp_affinity_determine_capable: "
203 "initial getaffinity call returned %ld errno = %d\n",
204 gCode, errno));
205
206 if (gCode < 0 && errno != EINVAL) {
207 // System call not supported
208 if (verbose ||
209 (warnings && (type != affinity_none) && (type != affinity_default) &&
210 (type != affinity_disabled))) {
211 int error = errno;
212 kmp_msg_t err_code = KMP_ERR(error);
213 __kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var),
214 err_code, __kmp_msg_null);
215 if (__kmp_generate_warnings == kmp_warnings_off) {
216 __kmp_str_free(&err_code.str);
217 }
218 }
219 KMP_AFFINITY_DISABLE();
220 KMP_INTERNAL_FREE(buf);
221 return;
222 } else if (gCode > 0) {
223 // The optimal situation: the OS returns the size of the buffer it expects.
224 KMP_AFFINITY_ENABLE(gCode);
225 KA_TRACE(10, ("__kmp_affinity_determine_capable: "
226 "affinity supported (mask size %d)\n",
227 (int)__kmp_affin_mask_size));
228 KMP_INTERNAL_FREE(buf);
229 return;
230 }
231
232 // Call the getaffinity system call repeatedly with increasing set sizes
233 // until we succeed, or reach an upper bound on the search.
234 KA_TRACE(30, ("__kmp_affinity_determine_capable: "
235 "searching for proper set size\n"));
236 int size;
237 for (size = 1; size <= KMP_CPU_SET_SIZE_LIMIT; size *= 2) {
238 gCode = syscall(__NR_sched_getaffinity, 0, size, buf);
239 KA_TRACE(30, ("__kmp_affinity_determine_capable: "
240 "getaffinity for mask size %ld returned %ld errno = %d\n",
241 size, gCode, errno));
242
243 if (gCode < 0) {
244 if (errno == ENOSYS) {
245 // We shouldn't get here
246 KA_TRACE(30, ("__kmp_affinity_determine_capable: "
247 "inconsistent OS call behavior: errno == ENOSYS for mask "
248 "size %d\n",
249 size));
250 if (verbose ||
251 (warnings && (type != affinity_none) &&
252 (type != affinity_default) && (type != affinity_disabled))) {
253 int error = errno;
254 kmp_msg_t err_code = KMP_ERR(error);
255 __kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var),
256 err_code, __kmp_msg_null);
257 if (__kmp_generate_warnings == kmp_warnings_off) {
258 __kmp_str_free(&err_code.str);
259 }
260 }
261 KMP_AFFINITY_DISABLE();
262 KMP_INTERNAL_FREE(buf);
263 return;
264 }
265 continue;
266 }
267
268 KMP_AFFINITY_ENABLE(gCode);
269 KA_TRACE(10, ("__kmp_affinity_determine_capable: "
270 "affinity supported (mask size %d)\n",
271 (int)__kmp_affin_mask_size));
272 KMP_INTERNAL_FREE(buf);
273 return;
274 }
275#elif KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY
276 long gCode;
277 unsigned char *buf;
278 buf = (unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT);
279 gCode = pthread_getaffinity_np(pthread_self(), KMP_CPU_SET_SIZE_LIMIT,
280 reinterpret_cast<cpuset_t *>(buf));
281 KA_TRACE(30, ("__kmp_affinity_determine_capable: "
282 "initial getaffinity call returned %d errno = %d\n",
283 gCode, errno));
284 if (gCode == 0) {
285 KMP_AFFINITY_ENABLE(KMP_CPU_SET_SIZE_LIMIT);
286 KA_TRACE(10, ("__kmp_affinity_determine_capable: "
287 "affinity supported (mask size %d)\n",
288 (int)__kmp_affin_mask_size));
289 KMP_INTERNAL_FREE(buf);
290 return;
291 }
292#endif
293 KMP_INTERNAL_FREE(buf);
294
295 // Affinity is not supported
296 KMP_AFFINITY_DISABLE();
297 KA_TRACE(10, ("__kmp_affinity_determine_capable: "
298 "cannot determine mask size - affinity not supported\n"));
299 if (verbose || (warnings && (type != affinity_none) &&
300 (type != affinity_default) && (type != affinity_disabled))) {
301 KMP_WARNING(AffCantGetMaskSize, env_var);
302 }
303}
304#endif // KMP_OS_AIX
305#endif // (KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
306 KMP_OS_DRAGONFLY || KMP_OS_AIX) && KMP_AFFINITY_SUPPORTED
307
308#if KMP_USE_FUTEX
309
310int __kmp_futex_determine_capable() {
311 int loc = 0;
312 long rc = syscall(__NR_futex, &loc, FUTEX_WAKE, 1, NULL, NULL, 0);
313 int retval = (rc == 0) || (errno != ENOSYS);
314
315 KA_TRACE(10,
316 ("__kmp_futex_determine_capable: rc = %d errno = %d\n", rc, errno));
317 KA_TRACE(10, ("__kmp_futex_determine_capable: futex syscall%s supported\n",
318 retval ? "" : " not"));
319
320 return retval;
321}
322
323#endif // KMP_USE_FUTEX
324
325#if (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_WASM) && (!KMP_ASM_INTRINS)
326/* Only 32-bit "add-exchange" instruction on IA-32 architecture causes us to
327 use compare_and_store for these routines */
328
329kmp_int8 __kmp_test_then_or8(volatile kmp_int8 *p, kmp_int8 d) {
330 kmp_int8 old_value, new_value;
331
332 old_value = TCR_1(*p);
333 new_value = old_value | d;
334
335 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
336 KMP_CPU_PAUSE();
337 old_value = TCR_1(*p);
338 new_value = old_value | d;
339 }
340 return old_value;
341}
342
343kmp_int8 __kmp_test_then_and8(volatile kmp_int8 *p, kmp_int8 d) {
344 kmp_int8 old_value, new_value;
345
346 old_value = TCR_1(*p);
347 new_value = old_value & d;
348
349 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
350 KMP_CPU_PAUSE();
351 old_value = TCR_1(*p);
352 new_value = old_value & d;
353 }
354 return old_value;
355}
356
357kmp_uint32 __kmp_test_then_or32(volatile kmp_uint32 *p, kmp_uint32 d) {
358 kmp_uint32 old_value, new_value;
359
360 old_value = TCR_4(*p);
361 new_value = old_value | d;
362
363 while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) {
364 KMP_CPU_PAUSE();
365 old_value = TCR_4(*p);
366 new_value = old_value | d;
367 }
368 return old_value;
369}
370
371kmp_uint32 __kmp_test_then_and32(volatile kmp_uint32 *p, kmp_uint32 d) {
372 kmp_uint32 old_value, new_value;
373
374 old_value = TCR_4(*p);
375 new_value = old_value & d;
376
377 while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) {
378 KMP_CPU_PAUSE();
379 old_value = TCR_4(*p);
380 new_value = old_value & d;
381 }
382 return old_value;
383}
384
385#if KMP_ARCH_X86 || KMP_ARCH_WASM
386kmp_int8 __kmp_test_then_add8(volatile kmp_int8 *p, kmp_int8 d) {
387 kmp_int8 old_value, new_value;
388
389 old_value = TCR_1(*p);
390 new_value = old_value + d;
391
392 while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
393 KMP_CPU_PAUSE();
394 old_value = TCR_1(*p);
395 new_value = old_value + d;
396 }
397 return old_value;
398}
399
400kmp_int64 __kmp_test_then_add64(volatile kmp_int64 *p, kmp_int64 d) {
401 kmp_int64 old_value, new_value;
402
403 old_value = TCR_8(*p);
404 new_value = old_value + d;
405
406 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
407 KMP_CPU_PAUSE();
408 old_value = TCR_8(*p);
409 new_value = old_value + d;
410 }
411 return old_value;
412}
413#endif /* KMP_ARCH_X86 */
414
415kmp_uint64 __kmp_test_then_or64(volatile kmp_uint64 *p, kmp_uint64 d) {
416 kmp_uint64 old_value, new_value;
417
418 old_value = TCR_8(*p);
419 new_value = old_value | d;
420 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
421 KMP_CPU_PAUSE();
422 old_value = TCR_8(*p);
423 new_value = old_value | d;
424 }
425 return old_value;
426}
427
428kmp_uint64 __kmp_test_then_and64(volatile kmp_uint64 *p, kmp_uint64 d) {
429 kmp_uint64 old_value, new_value;
430
431 old_value = TCR_8(*p);
432 new_value = old_value & d;
433 while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
434 KMP_CPU_PAUSE();
435 old_value = TCR_8(*p);
436 new_value = old_value & d;
437 }
438 return old_value;
439}
440
441#endif /* (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS) */
442
443void __kmp_terminate_thread(int gtid) {
444 int status;
445 kmp_info_t *th = __kmp_threads[gtid];
446
447 if (!th)
448 return;
449
450#ifdef KMP_CANCEL_THREADS
451 KA_TRACE(10, ("__kmp_terminate_thread: kill (%d)\n", gtid));
452 status = pthread_cancel(th->th.th_info.ds.ds_thread);
453 if (status != 0 && status != ESRCH) {
454 __kmp_fatal(KMP_MSG(CantTerminateWorkerThread), KMP_ERR(status),
455 __kmp_msg_null);
456 }
457#endif
458 KMP_YIELD(TRUE);
459} //
460
461/* Set thread stack info.
462 If values are unreasonable, assume call failed and use incremental stack
463 refinement method instead. Returns TRUE if the stack parameters could be
464 determined exactly, FALSE if incremental refinement is necessary. */
465static kmp_int32 __kmp_set_stack_info(int gtid, kmp_info_t *th) {
466 int stack_data;
467#if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
468 KMP_OS_HAIKU || KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX
469 int status;
470 size_t size = 0;
471 void *addr = 0;
472
473 /* Always do incremental stack refinement for ubermaster threads since the
474 initial thread stack range can be reduced by sibling thread creation so
475 pthread_attr_getstack may cause thread gtid aliasing */
476 if (!KMP_UBER_GTID(gtid)) {
477
478#if KMP_OS_SOLARIS
479 stack_t s;
480 if ((status = thr_stksegment(&s)) < 0) {
481 KMP_CHECK_SYSFAIL("thr_stksegment", status);
482 }
483
484 addr = s.ss_sp;
485 size = s.ss_size;
486 KA_TRACE(60, ("__kmp_set_stack_info: T#%d thr_stksegment returned size:"
487 " %lu, low addr: %p\n",
488 gtid, size, addr));
489#else
490 pthread_attr_t attr;
491 /* Fetch the real thread attributes */
492 status = pthread_attr_init(&attr);
493 KMP_CHECK_SYSFAIL("pthread_attr_init", status);
494#if KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD
495 status = pthread_attr_get_np(pthread_self(), &attr);
496 KMP_CHECK_SYSFAIL("pthread_attr_get_np", status);
497#else
498 status = pthread_getattr_np(pthread_self(), &attr);
499 KMP_CHECK_SYSFAIL("pthread_getattr_np", status);
500#endif
501 status = pthread_attr_getstack(&attr, &addr, &size);
502 KMP_CHECK_SYSFAIL("pthread_attr_getstack", status);
503 KA_TRACE(60,
504 ("__kmp_set_stack_info: T#%d pthread_attr_getstack returned size:"
505 " %lu, low addr: %p\n",
506 gtid, size, addr));
507 status = pthread_attr_destroy(&attr);
508 KMP_CHECK_SYSFAIL("pthread_attr_destroy", status);
509#endif
510 }
511
512 if (size != 0 && addr != 0) { // was stack parameter determination successful?
513 /* Store the correct base and size */
514 TCW_PTR(th->th.th_info.ds.ds_stackbase, (((char *)addr) + size));
515 TCW_PTR(th->th.th_info.ds.ds_stacksize, size);
516 TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
517 return TRUE;
518 }
519#endif /* KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD \
520 || KMP_OS_HAIKU || KMP_OS_HURD || KMP_OS_SOLARIS */
521 /* Use incremental refinement starting from initial conservative estimate */
522 TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
523 TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
524 TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
525 return FALSE;
526}
527
528static void *__kmp_launch_worker(void *thr) {
529 int status, old_type, old_state;
530#ifdef KMP_BLOCK_SIGNALS
531 sigset_t new_set, old_set;
532#endif /* KMP_BLOCK_SIGNALS */
533 void *exit_val;
534#if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
535 KMP_OS_OPENBSD || KMP_OS_HAIKU || KMP_OS_HURD || KMP_OS_SOLARIS || \
536 KMP_OS_AIX
537 void *volatile padding = 0;
538#endif
539 int gtid;
540
541 gtid = ((kmp_info_t *)thr)->th.th_info.ds.ds_gtid;
542 __kmp_gtid_set_specific(gtid);
543#ifdef KMP_TDATA_GTID
544 __kmp_gtid = gtid;
545#endif
546#if KMP_STATS_ENABLED
547 // set thread local index to point to thread-specific stats
548 __kmp_stats_thread_ptr = ((kmp_info_t *)thr)->th.th_stats;
549 __kmp_stats_thread_ptr->startLife();
550 KMP_SET_THREAD_STATE(IDLE);
552#endif
553
554#if USE_ITT_BUILD
555 __kmp_itt_thread_name(gtid);
556#endif /* USE_ITT_BUILD */
557
558#if KMP_AFFINITY_SUPPORTED
559 __kmp_affinity_bind_init_mask(gtid);
560#endif
561
562#ifdef KMP_CANCEL_THREADS
563 status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type);
564 KMP_CHECK_SYSFAIL("pthread_setcanceltype", status);
565 // josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads?
566 status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state);
567 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
568#endif
569
570#if KMP_ARCH_X86 || KMP_ARCH_X86_64
571 // Set FP control regs to be a copy of the parallel initialization thread's.
572 __kmp_clear_x87_fpu_status_word();
573 __kmp_load_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
574 __kmp_load_mxcsr(&__kmp_init_mxcsr);
575#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
576
577#ifdef KMP_BLOCK_SIGNALS
578 status = sigfillset(&new_set);
579 KMP_CHECK_SYSFAIL_ERRNO("sigfillset", status);
580 status = pthread_sigmask(SIG_BLOCK, &new_set, &old_set);
581 KMP_CHECK_SYSFAIL("pthread_sigmask", status);
582#endif /* KMP_BLOCK_SIGNALS */
583
584#if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
585 KMP_OS_OPENBSD || KMP_OS_HAIKU || KMP_OS_HURD || KMP_OS_SOLARIS || \
586 KMP_OS_AIX
587 if (__kmp_stkoffset > 0 && gtid > 0) {
588 padding = KMP_ALLOCA(gtid * __kmp_stkoffset);
589 (void)padding;
590 }
591#endif
592
593 KMP_MB();
594 __kmp_set_stack_info(gtid, (kmp_info_t *)thr);
595
596 __kmp_check_stack_overlap((kmp_info_t *)thr);
597
598 exit_val = __kmp_launch_thread((kmp_info_t *)thr);
599
600#ifdef KMP_BLOCK_SIGNALS
601 status = pthread_sigmask(SIG_SETMASK, &old_set, NULL);
602 KMP_CHECK_SYSFAIL("pthread_sigmask", status);
603#endif /* KMP_BLOCK_SIGNALS */
604
605 return exit_val;
606}
607
608#if KMP_USE_MONITOR
609/* The monitor thread controls all of the threads in the complex */
610
611static void *__kmp_launch_monitor(void *thr) {
612 int status, old_type, old_state;
613#ifdef KMP_BLOCK_SIGNALS
614 sigset_t new_set;
615#endif /* KMP_BLOCK_SIGNALS */
616 struct timespec interval;
617
618 KMP_MB(); /* Flush all pending memory write invalidates. */
619
620 KA_TRACE(10, ("__kmp_launch_monitor: #1 launched\n"));
621
622 /* register us as the monitor thread */
623 __kmp_gtid_set_specific(KMP_GTID_MONITOR);
624#ifdef KMP_TDATA_GTID
625 __kmp_gtid = KMP_GTID_MONITOR;
626#endif
627
628 KMP_MB();
629
630#if USE_ITT_BUILD
631 // Instruct Intel(R) Threading Tools to ignore monitor thread.
632 __kmp_itt_thread_ignore();
633#endif /* USE_ITT_BUILD */
634
635 __kmp_set_stack_info(((kmp_info_t *)thr)->th.th_info.ds.ds_gtid,
636 (kmp_info_t *)thr);
637
638 __kmp_check_stack_overlap((kmp_info_t *)thr);
639
640#ifdef KMP_CANCEL_THREADS
641 status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type);
642 KMP_CHECK_SYSFAIL("pthread_setcanceltype", status);
643 // josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads?
644 status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state);
645 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
646#endif
647
648#if KMP_REAL_TIME_FIX
649 // This is a potential fix which allows application with real-time scheduling
650 // policy work. However, decision about the fix is not made yet, so it is
651 // disabled by default.
652 { // Are program started with real-time scheduling policy?
653 int sched = sched_getscheduler(0);
654 if (sched == SCHED_FIFO || sched == SCHED_RR) {
655 // Yes, we are a part of real-time application. Try to increase the
656 // priority of the monitor.
657 struct sched_param param;
658 int max_priority = sched_get_priority_max(sched);
659 int rc;
660 KMP_WARNING(RealTimeSchedNotSupported);
661 sched_getparam(0, &param);
662 if (param.sched_priority < max_priority) {
663 param.sched_priority += 1;
664 rc = sched_setscheduler(0, sched, &param);
665 if (rc != 0) {
666 int error = errno;
667 kmp_msg_t err_code = KMP_ERR(error);
668 __kmp_msg(kmp_ms_warning, KMP_MSG(CantChangeMonitorPriority),
669 err_code, KMP_MSG(MonitorWillStarve), __kmp_msg_null);
670 if (__kmp_generate_warnings == kmp_warnings_off) {
671 __kmp_str_free(&err_code.str);
672 }
673 }
674 } else {
675 // We cannot abort here, because number of CPUs may be enough for all
676 // the threads, including the monitor thread, so application could
677 // potentially work...
678 __kmp_msg(kmp_ms_warning, KMP_MSG(RunningAtMaxPriority),
679 KMP_MSG(MonitorWillStarve), KMP_HNT(RunningAtMaxPriority),
680 __kmp_msg_null);
681 }
682 }
683 // AC: free thread that waits for monitor started
684 TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
685 }
686#endif // KMP_REAL_TIME_FIX
687
688 KMP_MB(); /* Flush all pending memory write invalidates. */
689
690 if (__kmp_monitor_wakeups == 1) {
691 interval.tv_sec = 1;
692 interval.tv_nsec = 0;
693 } else {
694 interval.tv_sec = 0;
695 interval.tv_nsec = (KMP_NSEC_PER_SEC / __kmp_monitor_wakeups);
696 }
697
698 KA_TRACE(10, ("__kmp_launch_monitor: #2 monitor\n"));
699
700 while (!TCR_4(__kmp_global.g.g_done)) {
701 struct timespec now;
702 struct timeval tval;
703
704 /* This thread monitors the state of the system */
705
706 KA_TRACE(15, ("__kmp_launch_monitor: update\n"));
707
708 status = gettimeofday(&tval, NULL);
709 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
710 TIMEVAL_TO_TIMESPEC(&tval, &now);
711
712 now.tv_sec += interval.tv_sec;
713 now.tv_nsec += interval.tv_nsec;
714
715 if (now.tv_nsec >= KMP_NSEC_PER_SEC) {
716 now.tv_sec += 1;
717 now.tv_nsec -= KMP_NSEC_PER_SEC;
718 }
719
720 status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex);
721 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
722 // AC: the monitor should not fall asleep if g_done has been set
723 if (!TCR_4(__kmp_global.g.g_done)) { // check once more under mutex
724 status = pthread_cond_timedwait(&__kmp_wait_cv.c_cond,
725 &__kmp_wait_mx.m_mutex, &now);
726 if (status != 0) {
727 if (status != ETIMEDOUT && status != EINTR) {
728 KMP_SYSFAIL("pthread_cond_timedwait", status);
729 }
730 }
731 }
732 status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex);
733 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
734
735 TCW_4(__kmp_global.g.g_time.dt.t_value,
736 TCR_4(__kmp_global.g.g_time.dt.t_value) + 1);
737
738 KMP_MB(); /* Flush all pending memory write invalidates. */
739 }
740
741 KA_TRACE(10, ("__kmp_launch_monitor: #3 cleanup\n"));
742
743#ifdef KMP_BLOCK_SIGNALS
744 status = sigfillset(&new_set);
745 KMP_CHECK_SYSFAIL_ERRNO("sigfillset", status);
746 status = pthread_sigmask(SIG_UNBLOCK, &new_set, NULL);
747 KMP_CHECK_SYSFAIL("pthread_sigmask", status);
748#endif /* KMP_BLOCK_SIGNALS */
749
750 KA_TRACE(10, ("__kmp_launch_monitor: #4 finished\n"));
751
752 if (__kmp_global.g.g_abort != 0) {
753 /* now we need to terminate the worker threads */
754 /* the value of t_abort is the signal we caught */
755
756 int gtid;
757
758 KA_TRACE(10, ("__kmp_launch_monitor: #5 terminate sig=%d\n",
759 __kmp_global.g.g_abort));
760
761 /* terminate the OpenMP worker threads */
762 /* TODO this is not valid for sibling threads!!
763 * the uber master might not be 0 anymore.. */
764 for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
765 __kmp_terminate_thread(gtid);
766
767 __kmp_cleanup();
768
769 KA_TRACE(10, ("__kmp_launch_monitor: #6 raise sig=%d\n",
770 __kmp_global.g.g_abort));
771
772 if (__kmp_global.g.g_abort > 0)
773 raise(__kmp_global.g.g_abort);
774 }
775
776 KA_TRACE(10, ("__kmp_launch_monitor: #7 exit\n"));
777
778 return thr;
779}
780#endif // KMP_USE_MONITOR
781
782void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size) {
783 pthread_t handle;
784 pthread_attr_t thread_attr;
785 int status;
786
787 th->th.th_info.ds.ds_gtid = gtid;
788
789#if KMP_STATS_ENABLED
790 // sets up worker thread stats
791 __kmp_acquire_tas_lock(&__kmp_stats_lock, gtid);
792
793 // th->th.th_stats is used to transfer thread-specific stats-pointer to
794 // __kmp_launch_worker. So when thread is created (goes into
795 // __kmp_launch_worker) it will set its thread local pointer to
796 // th->th.th_stats
797 if (!KMP_UBER_GTID(gtid)) {
798 th->th.th_stats = __kmp_stats_list->push_back(gtid);
799 } else {
800 // For root threads, __kmp_stats_thread_ptr is set in __kmp_register_root(),
801 // so set the th->th.th_stats field to it.
802 th->th.th_stats = __kmp_stats_thread_ptr;
803 }
804 __kmp_release_tas_lock(&__kmp_stats_lock, gtid);
805
806#endif // KMP_STATS_ENABLED
807
808 if (KMP_UBER_GTID(gtid)) {
809 KA_TRACE(10, ("__kmp_create_worker: uber thread (%d)\n", gtid));
810 th->th.th_info.ds.ds_thread = pthread_self();
811 __kmp_set_stack_info(gtid, th);
812 __kmp_check_stack_overlap(th);
813 return;
814 }
815
816 KA_TRACE(10, ("__kmp_create_worker: try to create thread (%d)\n", gtid));
817
818 KMP_MB(); /* Flush all pending memory write invalidates. */
819
820#ifdef KMP_THREAD_ATTR
821 status = pthread_attr_init(&thread_attr);
822 if (status != 0) {
823 __kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null);
824 }
825 status = pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE);
826 if (status != 0) {
827 __kmp_fatal(KMP_MSG(CantSetWorkerState), KMP_ERR(status), __kmp_msg_null);
828 }
829
830 /* Set stack size for this thread now.
831 The multiple of 2 is there because on some machines, requesting an unusual
832 stacksize causes the thread to have an offset before the dummy alloca()
833 takes place to create the offset. Since we want the user to have a
834 sufficient stacksize AND support a stack offset, we alloca() twice the
835 offset so that the upcoming alloca() does not eliminate any premade offset,
836 and also gives the user the stack space they requested for all threads */
837 stack_size += gtid * __kmp_stkoffset * 2;
838
839 KA_TRACE(10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
840 "__kmp_stksize = %lu bytes, final stacksize = %lu bytes\n",
841 gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size));
842
843#ifdef _POSIX_THREAD_ATTR_STACKSIZE
844 status = pthread_attr_setstacksize(&thread_attr, stack_size);
845#ifdef KMP_BACKUP_STKSIZE
846 if (status != 0) {
847 if (!__kmp_env_stksize) {
848 stack_size = KMP_BACKUP_STKSIZE + gtid * __kmp_stkoffset;
849 __kmp_stksize = KMP_BACKUP_STKSIZE;
850 KA_TRACE(10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
851 "__kmp_stksize = %lu bytes, (backup) final stacksize = %lu "
852 "bytes\n",
853 gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size));
854 status = pthread_attr_setstacksize(&thread_attr, stack_size);
855 }
856 }
857#endif /* KMP_BACKUP_STKSIZE */
858 if (status != 0) {
859 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
860 KMP_HNT(ChangeWorkerStackSize), __kmp_msg_null);
861 }
862#endif /* _POSIX_THREAD_ATTR_STACKSIZE */
863
864#endif /* KMP_THREAD_ATTR */
865
866 status =
867 pthread_create(&handle, &thread_attr, __kmp_launch_worker, (void *)th);
868 if (status != 0 || !handle) { // ??? Why do we check handle??
869#ifdef _POSIX_THREAD_ATTR_STACKSIZE
870 if (status == EINVAL) {
871 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
872 KMP_HNT(IncreaseWorkerStackSize), __kmp_msg_null);
873 }
874 if (status == ENOMEM) {
875 __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
876 KMP_HNT(DecreaseWorkerStackSize), __kmp_msg_null);
877 }
878#endif /* _POSIX_THREAD_ATTR_STACKSIZE */
879 if (status == EAGAIN) {
880 __kmp_fatal(KMP_MSG(NoResourcesForWorkerThread), KMP_ERR(status),
881 KMP_HNT(Decrease_NUM_THREADS), __kmp_msg_null);
882 }
883 KMP_SYSFAIL("pthread_create", status);
884 }
885
886 // Rename worker threads for improved debuggability
887 if (!KMP_UBER_GTID(gtid)) {
888#if defined(LIBOMP_HAVE_PTHREAD_SET_NAME_NP)
889 pthread_set_name_np(handle, "openmp_worker");
890#elif defined(LIBOMP_HAVE_PTHREAD_SETNAME_NP) && !KMP_OS_DARWIN
891#if KMP_OS_NETBSD
892 pthread_setname_np(handle, "%s", const_cast<char *>("openmp_worker"));
893#else
894 pthread_setname_np(handle, "openmp_worker");
895#endif
896#endif
897 }
898
899 th->th.th_info.ds.ds_thread = handle;
900
901#ifdef KMP_THREAD_ATTR
902 status = pthread_attr_destroy(&thread_attr);
903 if (status) {
904 kmp_msg_t err_code = KMP_ERR(status);
905 __kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code,
906 __kmp_msg_null);
907 if (__kmp_generate_warnings == kmp_warnings_off) {
908 __kmp_str_free(&err_code.str);
909 }
910 }
911#endif /* KMP_THREAD_ATTR */
912
913 KMP_MB(); /* Flush all pending memory write invalidates. */
914
915 KA_TRACE(10, ("__kmp_create_worker: done creating thread (%d)\n", gtid));
916
917} // __kmp_create_worker
918
919#if KMP_USE_MONITOR
920void __kmp_create_monitor(kmp_info_t *th) {
921 pthread_t handle;
922 pthread_attr_t thread_attr;
923 size_t size;
924 int status;
925 int auto_adj_size = FALSE;
926
927 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
928 // We don't need monitor thread in case of MAX_BLOCKTIME
929 KA_TRACE(10, ("__kmp_create_monitor: skipping monitor thread because of "
930 "MAX blocktime\n"));
931 th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op
932 th->th.th_info.ds.ds_gtid = 0;
933 return;
934 }
935 KA_TRACE(10, ("__kmp_create_monitor: try to create monitor\n"));
936
937 KMP_MB(); /* Flush all pending memory write invalidates. */
938
939 th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
940 th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
941#if KMP_REAL_TIME_FIX
942 TCW_4(__kmp_global.g.g_time.dt.t_value,
943 -1); // Will use it for synchronization a bit later.
944#else
945 TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
946#endif // KMP_REAL_TIME_FIX
947
948#ifdef KMP_THREAD_ATTR
949 if (__kmp_monitor_stksize == 0) {
950 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
951 auto_adj_size = TRUE;
952 }
953 status = pthread_attr_init(&thread_attr);
954 if (status != 0) {
955 __kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null);
956 }
957 status = pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE);
958 if (status != 0) {
959 __kmp_fatal(KMP_MSG(CantSetMonitorState), KMP_ERR(status), __kmp_msg_null);
960 }
961
962#ifdef _POSIX_THREAD_ATTR_STACKSIZE
963 status = pthread_attr_getstacksize(&thread_attr, &size);
964 KMP_CHECK_SYSFAIL("pthread_attr_getstacksize", status);
965#else
966 size = __kmp_sys_min_stksize;
967#endif /* _POSIX_THREAD_ATTR_STACKSIZE */
968#endif /* KMP_THREAD_ATTR */
969
970 if (__kmp_monitor_stksize == 0) {
971 __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
972 }
973 if (__kmp_monitor_stksize < __kmp_sys_min_stksize) {
974 __kmp_monitor_stksize = __kmp_sys_min_stksize;
975 }
976
977 KA_TRACE(10, ("__kmp_create_monitor: default stacksize = %lu bytes,"
978 "requested stacksize = %lu bytes\n",
979 size, __kmp_monitor_stksize));
980
981retry:
982
983/* Set stack size for this thread now. */
984#ifdef _POSIX_THREAD_ATTR_STACKSIZE
985 KA_TRACE(10, ("__kmp_create_monitor: setting stacksize = %lu bytes,",
986 __kmp_monitor_stksize));
987 status = pthread_attr_setstacksize(&thread_attr, __kmp_monitor_stksize);
988 if (status != 0) {
989 if (auto_adj_size) {
990 __kmp_monitor_stksize *= 2;
991 goto retry;
992 }
993 kmp_msg_t err_code = KMP_ERR(status);
994 __kmp_msg(kmp_ms_warning, // should this be fatal? BB
995 KMP_MSG(CantSetMonitorStackSize, (long int)__kmp_monitor_stksize),
996 err_code, KMP_HNT(ChangeMonitorStackSize), __kmp_msg_null);
997 if (__kmp_generate_warnings == kmp_warnings_off) {
998 __kmp_str_free(&err_code.str);
999 }
1000 }
1001#endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1002
1003 status =
1004 pthread_create(&handle, &thread_attr, __kmp_launch_monitor, (void *)th);
1005
1006 if (status != 0) {
1007#ifdef _POSIX_THREAD_ATTR_STACKSIZE
1008 if (status == EINVAL) {
1009 if (auto_adj_size && (__kmp_monitor_stksize < (size_t)0x40000000)) {
1010 __kmp_monitor_stksize *= 2;
1011 goto retry;
1012 }
1013 __kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize),
1014 KMP_ERR(status), KMP_HNT(IncreaseMonitorStackSize),
1015 __kmp_msg_null);
1016 }
1017 if (status == ENOMEM) {
1018 __kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize),
1019 KMP_ERR(status), KMP_HNT(DecreaseMonitorStackSize),
1020 __kmp_msg_null);
1021 }
1022#endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1023 if (status == EAGAIN) {
1024 __kmp_fatal(KMP_MSG(NoResourcesForMonitorThread), KMP_ERR(status),
1025 KMP_HNT(DecreaseNumberOfThreadsInUse), __kmp_msg_null);
1026 }
1027 KMP_SYSFAIL("pthread_create", status);
1028 }
1029
1030 th->th.th_info.ds.ds_thread = handle;
1031
1032#if KMP_REAL_TIME_FIX
1033 // Wait for the monitor thread is really started and set its *priority*.
1034 KMP_DEBUG_ASSERT(sizeof(kmp_uint32) ==
1035 sizeof(__kmp_global.g.g_time.dt.t_value));
1036 __kmp_wait_4((kmp_uint32 volatile *)&__kmp_global.g.g_time.dt.t_value, -1,
1037 &__kmp_neq_4, NULL);
1038#endif // KMP_REAL_TIME_FIX
1039
1040#ifdef KMP_THREAD_ATTR
1041 status = pthread_attr_destroy(&thread_attr);
1042 if (status != 0) {
1043 kmp_msg_t err_code = KMP_ERR(status);
1044 __kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code,
1045 __kmp_msg_null);
1046 if (__kmp_generate_warnings == kmp_warnings_off) {
1047 __kmp_str_free(&err_code.str);
1048 }
1049 }
1050#endif
1051
1052 KMP_MB(); /* Flush all pending memory write invalidates. */
1053
1054 KA_TRACE(10, ("__kmp_create_monitor: monitor created %#.8lx\n",
1055 th->th.th_info.ds.ds_thread));
1056
1057} // __kmp_create_monitor
1058#endif // KMP_USE_MONITOR
1059
1060void __kmp_exit_thread(int exit_status) {
1061#if KMP_OS_WASI
1062// TODO: the wasm32-wasi-threads target does not yet support pthread_exit.
1063#else
1064 pthread_exit((void *)(intptr_t)exit_status);
1065#endif
1066} // __kmp_exit_thread
1067
1068#if KMP_USE_MONITOR
1069void __kmp_resume_monitor();
1070
1071extern "C" void __kmp_reap_monitor(kmp_info_t *th) {
1072 int status;
1073 void *exit_val;
1074
1075 KA_TRACE(10, ("__kmp_reap_monitor: try to reap monitor thread with handle"
1076 " %#.8lx\n",
1077 th->th.th_info.ds.ds_thread));
1078
1079 // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
1080 // If both tid and gtid are 0, it means the monitor did not ever start.
1081 // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
1082 KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid);
1083 if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) {
1084 KA_TRACE(10, ("__kmp_reap_monitor: monitor did not start, returning\n"));
1085 return;
1086 }
1087
1088 KMP_MB(); /* Flush all pending memory write invalidates. */
1089
1090 /* First, check to see whether the monitor thread exists to wake it up. This
1091 is to avoid performance problem when the monitor sleeps during
1092 blocktime-size interval */
1093
1094 status = pthread_kill(th->th.th_info.ds.ds_thread, 0);
1095 if (status != ESRCH) {
1096 __kmp_resume_monitor(); // Wake up the monitor thread
1097 }
1098 KA_TRACE(10, ("__kmp_reap_monitor: try to join with monitor\n"));
1099 status = pthread_join(th->th.th_info.ds.ds_thread, &exit_val);
1100 if (exit_val != th) {
1101 __kmp_fatal(KMP_MSG(ReapMonitorError), KMP_ERR(status), __kmp_msg_null);
1102 }
1103
1104 th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1105 th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1106
1107 KA_TRACE(10, ("__kmp_reap_monitor: done reaping monitor thread with handle"
1108 " %#.8lx\n",
1109 th->th.th_info.ds.ds_thread));
1110
1111 KMP_MB(); /* Flush all pending memory write invalidates. */
1112}
1113#else
1114// Empty symbol to export (see exports_so.txt) when
1115// monitor thread feature is disabled
1116extern "C" void __kmp_reap_monitor(kmp_info_t *th) { (void)th; }
1117#endif // KMP_USE_MONITOR
1118
1119void __kmp_reap_worker(kmp_info_t *th) {
1120 int status;
1121 void *exit_val;
1122
1123 KMP_MB(); /* Flush all pending memory write invalidates. */
1124
1125 KA_TRACE(
1126 10, ("__kmp_reap_worker: try to reap T#%d\n", th->th.th_info.ds.ds_gtid));
1127
1128 status = pthread_join(th->th.th_info.ds.ds_thread, &exit_val);
1129#ifdef KMP_DEBUG
1130 /* Don't expose these to the user until we understand when they trigger */
1131 if (status != 0) {
1132 __kmp_fatal(KMP_MSG(ReapWorkerError), KMP_ERR(status), __kmp_msg_null);
1133 }
1134 if (exit_val != th) {
1135 KA_TRACE(10, ("__kmp_reap_worker: worker T#%d did not reap properly, "
1136 "exit_val = %p\n",
1137 th->th.th_info.ds.ds_gtid, exit_val));
1138 }
1139#else
1140 (void)status; // unused variable
1141#endif /* KMP_DEBUG */
1142
1143 KA_TRACE(10, ("__kmp_reap_worker: done reaping T#%d\n",
1144 th->th.th_info.ds.ds_gtid));
1145
1146 KMP_MB(); /* Flush all pending memory write invalidates. */
1147}
1148
1149#if KMP_HANDLE_SIGNALS
1150
1151static void __kmp_null_handler(int signo) {
1152 // Do nothing, for doing SIG_IGN-type actions.
1153} // __kmp_null_handler
1154
1155static void __kmp_team_handler(int signo) {
1156 if (__kmp_global.g.g_abort == 0) {
1157/* Stage 1 signal handler, let's shut down all of the threads */
1158#ifdef KMP_DEBUG
1159 __kmp_debug_printf("__kmp_team_handler: caught signal = %d\n", signo);
1160#endif
1161 switch (signo) {
1162 case SIGHUP:
1163 case SIGINT:
1164 case SIGQUIT:
1165 case SIGILL:
1166 case SIGABRT:
1167 case SIGFPE:
1168 case SIGBUS:
1169 case SIGSEGV:
1170#ifdef SIGSYS
1171 case SIGSYS:
1172#endif
1173 case SIGTERM:
1174 if (__kmp_debug_buf) {
1175 __kmp_dump_debug_buffer();
1176 }
1177 __kmp_unregister_library(); // cleanup shared memory
1178 KMP_MB(); // Flush all pending memory write invalidates.
1179 TCW_4(__kmp_global.g.g_abort, signo);
1180 KMP_MB(); // Flush all pending memory write invalidates.
1181 TCW_4(__kmp_global.g.g_done, TRUE);
1182 KMP_MB(); // Flush all pending memory write invalidates.
1183 break;
1184 default:
1185#ifdef KMP_DEBUG
1186 __kmp_debug_printf("__kmp_team_handler: unknown signal type");
1187#endif
1188 break;
1189 }
1190 }
1191} // __kmp_team_handler
1192
1193static void __kmp_sigaction(int signum, const struct sigaction *act,
1194 struct sigaction *oldact) {
1195 int rc = sigaction(signum, act, oldact);
1196 KMP_CHECK_SYSFAIL_ERRNO("sigaction", rc);
1197}
1198
1199static void __kmp_install_one_handler(int sig, sig_func_t handler_func,
1200 int parallel_init) {
1201 KMP_MB(); // Flush all pending memory write invalidates.
1202 KB_TRACE(60,
1203 ("__kmp_install_one_handler( %d, ..., %d )\n", sig, parallel_init));
1204 if (parallel_init) {
1205 struct sigaction new_action;
1206 struct sigaction old_action;
1207 new_action.sa_handler = handler_func;
1208 new_action.sa_flags = 0;
1209 sigfillset(&new_action.sa_mask);
1210 __kmp_sigaction(sig, &new_action, &old_action);
1211 if (old_action.sa_handler == __kmp_sighldrs[sig].sa_handler) {
1212 sigaddset(&__kmp_sigset, sig);
1213 } else {
1214 // Restore/keep user's handler if one previously installed.
1215 __kmp_sigaction(sig, &old_action, NULL);
1216 }
1217 } else {
1218 // Save initial/system signal handlers to see if user handlers installed.
1219 __kmp_sigaction(sig, NULL, &__kmp_sighldrs[sig]);
1220 }
1221 KMP_MB(); // Flush all pending memory write invalidates.
1222} // __kmp_install_one_handler
1223
1224static void __kmp_remove_one_handler(int sig) {
1225 KB_TRACE(60, ("__kmp_remove_one_handler( %d )\n", sig));
1226 if (sigismember(&__kmp_sigset, sig)) {
1227 struct sigaction old;
1228 KMP_MB(); // Flush all pending memory write invalidates.
1229 __kmp_sigaction(sig, &__kmp_sighldrs[sig], &old);
1230 if ((old.sa_handler != __kmp_team_handler) &&
1231 (old.sa_handler != __kmp_null_handler)) {
1232 // Restore the users signal handler.
1233 KB_TRACE(10, ("__kmp_remove_one_handler: oops, not our handler, "
1234 "restoring: sig=%d\n",
1235 sig));
1236 __kmp_sigaction(sig, &old, NULL);
1237 }
1238 sigdelset(&__kmp_sigset, sig);
1239 KMP_MB(); // Flush all pending memory write invalidates.
1240 }
1241} // __kmp_remove_one_handler
1242
1243void __kmp_install_signals(int parallel_init) {
1244 KB_TRACE(10, ("__kmp_install_signals( %d )\n", parallel_init));
1245 if (__kmp_handle_signals || !parallel_init) {
1246 // If ! parallel_init, we do not install handlers, just save original
1247 // handlers. Let us do it even __handle_signals is 0.
1248 sigemptyset(&__kmp_sigset);
1249 __kmp_install_one_handler(SIGHUP, __kmp_team_handler, parallel_init);
1250 __kmp_install_one_handler(SIGINT, __kmp_team_handler, parallel_init);
1251 __kmp_install_one_handler(SIGQUIT, __kmp_team_handler, parallel_init);
1252 __kmp_install_one_handler(SIGILL, __kmp_team_handler, parallel_init);
1253 __kmp_install_one_handler(SIGABRT, __kmp_team_handler, parallel_init);
1254 __kmp_install_one_handler(SIGFPE, __kmp_team_handler, parallel_init);
1255 __kmp_install_one_handler(SIGBUS, __kmp_team_handler, parallel_init);
1256 __kmp_install_one_handler(SIGSEGV, __kmp_team_handler, parallel_init);
1257#ifdef SIGSYS
1258 __kmp_install_one_handler(SIGSYS, __kmp_team_handler, parallel_init);
1259#endif // SIGSYS
1260 __kmp_install_one_handler(SIGTERM, __kmp_team_handler, parallel_init);
1261#ifdef SIGPIPE
1262 __kmp_install_one_handler(SIGPIPE, __kmp_team_handler, parallel_init);
1263#endif // SIGPIPE
1264 }
1265} // __kmp_install_signals
1266
1267void __kmp_remove_signals(void) {
1268 int sig;
1269 KB_TRACE(10, ("__kmp_remove_signals()\n"));
1270 for (sig = 1; sig < NSIG; ++sig) {
1271 __kmp_remove_one_handler(sig);
1272 }
1273} // __kmp_remove_signals
1274
1275#endif // KMP_HANDLE_SIGNALS
1276
1277void __kmp_enable(int new_state) {
1278#ifdef KMP_CANCEL_THREADS
1279 int status, old_state;
1280 status = pthread_setcancelstate(new_state, &old_state);
1281 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
1282 KMP_DEBUG_ASSERT(old_state == PTHREAD_CANCEL_DISABLE);
1283#endif
1284}
1285
1286void __kmp_disable(int *old_state) {
1287#ifdef KMP_CANCEL_THREADS
1288 int status;
1289 status = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, old_state);
1290 KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
1291#endif
1292}
1293
1294static void __kmp_atfork_prepare(void) {
1295 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
1296 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
1297}
1298
1299static void __kmp_atfork_parent(void) {
1300 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1301 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
1302}
1303
1304/* Reset the library so execution in the child starts "all over again" with
1305 clean data structures in initial states. Don't worry about freeing memory
1306 allocated by parent, just abandon it to be safe. */
1307static void __kmp_atfork_child(void) {
1308 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1309 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
1310 /* TODO make sure this is done right for nested/sibling */
1311 // ATT: Memory leaks are here? TODO: Check it and fix.
1312 /* KMP_ASSERT( 0 ); */
1313
1314 ++__kmp_fork_count;
1315
1316#if KMP_AFFINITY_SUPPORTED
1317#if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \
1318 KMP_OS_AIX
1319 // reset the affinity in the child to the initial thread
1320 // affinity in the parent
1321 kmp_set_thread_affinity_mask_initial();
1322#endif
1323 // Set default not to bind threads tightly in the child (we're expecting
1324 // over-subscription after the fork and this can improve things for
1325 // scripting languages that use OpenMP inside process-parallel code).
1326 if (__kmp_nested_proc_bind.bind_types != NULL) {
1327 __kmp_nested_proc_bind.bind_types[0] = proc_bind_false;
1328 }
1329 for (kmp_affinity_t *affinity : __kmp_affinities)
1330 *affinity = KMP_AFFINITY_INIT(affinity->env_var);
1331 __kmp_affin_fullMask = nullptr;
1332 __kmp_affin_origMask = nullptr;
1333 __kmp_topology = nullptr;
1334#endif // KMP_AFFINITY_SUPPORTED
1335
1336#if KMP_USE_MONITOR
1337 __kmp_init_monitor = 0;
1338#endif
1339 __kmp_init_parallel = FALSE;
1340 __kmp_init_middle = FALSE;
1341 __kmp_init_serial = FALSE;
1342 TCW_4(__kmp_init_gtid, FALSE);
1343 __kmp_init_common = FALSE;
1344
1345 TCW_4(__kmp_init_user_locks, FALSE);
1346#if !KMP_USE_DYNAMIC_LOCK
1347 __kmp_user_lock_table.used = 1;
1348 __kmp_user_lock_table.allocated = 0;
1349 __kmp_user_lock_table.table = NULL;
1350 __kmp_lock_blocks = NULL;
1351#endif
1352
1353 __kmp_all_nth = 0;
1354 TCW_4(__kmp_nth, 0);
1355
1356 __kmp_thread_pool = NULL;
1357 __kmp_thread_pool_insert_pt = NULL;
1358 __kmp_team_pool = NULL;
1359
1360 /* Must actually zero all the *cache arguments passed to __kmpc_threadprivate
1361 here so threadprivate doesn't use stale data */
1362 KA_TRACE(10, ("__kmp_atfork_child: checking cache address list %p\n",
1363 __kmp_threadpriv_cache_list));
1364
1365 while (__kmp_threadpriv_cache_list != NULL) {
1366
1367 if (*__kmp_threadpriv_cache_list->addr != NULL) {
1368 KC_TRACE(50, ("__kmp_atfork_child: zeroing cache at address %p\n",
1369 &(*__kmp_threadpriv_cache_list->addr)));
1370
1371 *__kmp_threadpriv_cache_list->addr = NULL;
1372 }
1373 __kmp_threadpriv_cache_list = __kmp_threadpriv_cache_list->next;
1374 }
1375
1376 __kmp_init_runtime = FALSE;
1377
1378 /* reset statically initialized locks */
1379 __kmp_init_bootstrap_lock(&__kmp_initz_lock);
1380 __kmp_init_bootstrap_lock(&__kmp_stdio_lock);
1381 __kmp_init_bootstrap_lock(&__kmp_console_lock);
1382 __kmp_init_bootstrap_lock(&__kmp_task_team_lock);
1383
1384#if USE_ITT_BUILD
1385 __kmp_itt_reset(); // reset ITT's global state
1386#endif /* USE_ITT_BUILD */
1387
1388 {
1389 // Child process often get terminated without any use of OpenMP. That might
1390 // cause mapped shared memory file to be left unattended. Thus we postpone
1391 // library registration till middle initialization in the child process.
1392 __kmp_need_register_serial = FALSE;
1393 __kmp_serial_initialize();
1394 }
1395
1396 /* This is necessary to make sure no stale data is left around */
1397 /* AC: customers complain that we use unsafe routines in the atfork
1398 handler. Mathworks: dlsym() is unsafe. We call dlsym and dlopen
1399 in dynamic_link when check the presence of shared tbbmalloc library.
1400 Suggestion is to make the library initialization lazier, similar
1401 to what done for __kmpc_begin(). */
1402 // TODO: synchronize all static initializations with regular library
1403 // startup; look at kmp_global.cpp and etc.
1404 //__kmp_internal_begin ();
1405}
1406
1407void __kmp_register_atfork(void) {
1408 if (__kmp_need_register_atfork) {
1409#if !KMP_OS_WASI
1410 int status = pthread_atfork(__kmp_atfork_prepare, __kmp_atfork_parent,
1411 __kmp_atfork_child);
1412 KMP_CHECK_SYSFAIL("pthread_atfork", status);
1413#endif
1414 __kmp_need_register_atfork = FALSE;
1415 }
1416}
1417
1418void __kmp_suspend_initialize(void) {
1419 int status;
1420 status = pthread_mutexattr_init(&__kmp_suspend_mutex_attr);
1421 KMP_CHECK_SYSFAIL("pthread_mutexattr_init", status);
1422 status = pthread_condattr_init(&__kmp_suspend_cond_attr);
1423 KMP_CHECK_SYSFAIL("pthread_condattr_init", status);
1424}
1425
1426void __kmp_suspend_initialize_thread(kmp_info_t *th) {
1427 int old_value = KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init_count);
1428 int new_value = __kmp_fork_count + 1;
1429 // Return if already initialized
1430 if (old_value == new_value)
1431 return;
1432 // Wait, then return if being initialized
1433 if (old_value == -1 || !__kmp_atomic_compare_store(
1434 &th->th.th_suspend_init_count, old_value, -1)) {
1435 while (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) != new_value) {
1436 KMP_CPU_PAUSE();
1437 }
1438 } else {
1439 // Claim to be the initializer and do initializations
1440 int status;
1441 status = pthread_cond_init(&th->th.th_suspend_cv.c_cond,
1442 &__kmp_suspend_cond_attr);
1443 KMP_CHECK_SYSFAIL("pthread_cond_init", status);
1444 status = pthread_mutex_init(&th->th.th_suspend_mx.m_mutex,
1445 &__kmp_suspend_mutex_attr);
1446 KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
1447 KMP_ATOMIC_ST_REL(&th->th.th_suspend_init_count, new_value);
1448 }
1449}
1450
1451void __kmp_suspend_uninitialize_thread(kmp_info_t *th) {
1452 if (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) > __kmp_fork_count) {
1453 /* this means we have initialize the suspension pthread objects for this
1454 thread in this instance of the process */
1455 int status;
1456
1457 status = pthread_cond_destroy(&th->th.th_suspend_cv.c_cond);
1458 if (status != 0 && status != EBUSY) {
1459 KMP_SYSFAIL("pthread_cond_destroy", status);
1460 }
1461 status = pthread_mutex_destroy(&th->th.th_suspend_mx.m_mutex);
1462 if (status != 0 && status != EBUSY) {
1463 KMP_SYSFAIL("pthread_mutex_destroy", status);
1464 }
1465 --th->th.th_suspend_init_count;
1466 KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init_count) ==
1467 __kmp_fork_count);
1468 }
1469}
1470
1471// return true if lock obtained, false otherwise
1472int __kmp_try_suspend_mx(kmp_info_t *th) {
1473 return (pthread_mutex_trylock(&th->th.th_suspend_mx.m_mutex) == 0);
1474}
1475
1476void __kmp_lock_suspend_mx(kmp_info_t *th) {
1477 int status = pthread_mutex_lock(&th->th.th_suspend_mx.m_mutex);
1478 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
1479}
1480
1481void __kmp_unlock_suspend_mx(kmp_info_t *th) {
1482 int status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex);
1483 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
1484}
1485
1486/* This routine puts the calling thread to sleep after setting the
1487 sleep bit for the indicated flag variable to true. */
1488template <class C>
1489static inline void __kmp_suspend_template(int th_gtid, C *flag) {
1490 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_suspend);
1491 kmp_info_t *th = __kmp_threads[th_gtid];
1492 int status;
1493 typename C::flag_t old_spin;
1494
1495 KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid,
1496 flag->get()));
1497
1498 __kmp_suspend_initialize_thread(th);
1499
1500 __kmp_lock_suspend_mx(th);
1501
1502 KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n",
1503 th_gtid, flag->get()));
1504
1505 /* TODO: shouldn't this use release semantics to ensure that
1506 __kmp_suspend_initialize_thread gets called first? */
1507 old_spin = flag->set_sleeping();
1508 TCW_PTR(th->th.th_sleep_loc, (void *)flag);
1509 th->th.th_sleep_loc_type = flag->get_type();
1510 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
1511 __kmp_pause_status != kmp_soft_paused) {
1512 flag->unset_sleeping();
1513 TCW_PTR(th->th.th_sleep_loc, NULL);
1514 th->th.th_sleep_loc_type = flag_unset;
1515 __kmp_unlock_suspend_mx(th);
1516 return;
1517 }
1518 KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%x,"
1519 " was %x\n",
1520 th_gtid, flag->get(), flag->load(), old_spin));
1521
1522 if (flag->done_check_val(old_spin) || flag->done_check()) {
1523 flag->unset_sleeping();
1524 TCW_PTR(th->th.th_sleep_loc, NULL);
1525 th->th.th_sleep_loc_type = flag_unset;
1526 KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit "
1527 "for spin(%p)\n",
1528 th_gtid, flag->get()));
1529 } else {
1530 /* Encapsulate in a loop as the documentation states that this may
1531 "with low probability" return when the condition variable has
1532 not been signaled or broadcast */
1533 int deactivated = FALSE;
1534
1535 while (flag->is_sleeping()) {
1536#ifdef DEBUG_SUSPEND
1537 char buffer[128];
1538 __kmp_suspend_count++;
1539 __kmp_print_cond(buffer, &th->th.th_suspend_cv);
1540 __kmp_printf("__kmp_suspend_template: suspending T#%d: %s\n", th_gtid,
1541 buffer);
1542#endif
1543 // Mark the thread as no longer active (only in the first iteration of the
1544 // loop).
1545 if (!deactivated) {
1546 th->th.th_active = FALSE;
1547 if (th->th.th_active_in_pool) {
1548 th->th.th_active_in_pool = FALSE;
1549 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
1550 KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
1551 }
1552 deactivated = TRUE;
1553 }
1554
1555 KMP_DEBUG_ASSERT(th->th.th_sleep_loc);
1556 KMP_DEBUG_ASSERT(flag->get_type() == th->th.th_sleep_loc_type);
1557
1558#if USE_SUSPEND_TIMEOUT
1559 struct timespec now;
1560 struct timeval tval;
1561 int msecs;
1562
1563 status = gettimeofday(&tval, NULL);
1564 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
1565 TIMEVAL_TO_TIMESPEC(&tval, &now);
1566
1567 msecs = (4 * __kmp_dflt_blocktime) + 200;
1568 now.tv_sec += msecs / 1000;
1569 now.tv_nsec += (msecs % 1000) * 1000;
1570
1571 KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform "
1572 "pthread_cond_timedwait\n",
1573 th_gtid));
1574 status = pthread_cond_timedwait(&th->th.th_suspend_cv.c_cond,
1575 &th->th.th_suspend_mx.m_mutex, &now);
1576#else
1577 KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform"
1578 " pthread_cond_wait\n",
1579 th_gtid));
1580 status = pthread_cond_wait(&th->th.th_suspend_cv.c_cond,
1581 &th->th.th_suspend_mx.m_mutex);
1582#endif // USE_SUSPEND_TIMEOUT
1583
1584 if ((status != 0) && (status != EINTR) && (status != ETIMEDOUT)) {
1585 KMP_SYSFAIL("pthread_cond_wait", status);
1586 }
1587
1588 KMP_DEBUG_ASSERT(flag->get_type() == flag->get_ptr_type());
1589
1590 if (!flag->is_sleeping() &&
1591 ((status == EINTR) || (status == ETIMEDOUT))) {
1592 // if interrupt or timeout, and thread is no longer sleeping, we need to
1593 // make sure sleep_loc gets reset; however, this shouldn't be needed if
1594 // we woke up with resume
1595 flag->unset_sleeping();
1596 TCW_PTR(th->th.th_sleep_loc, NULL);
1597 th->th.th_sleep_loc_type = flag_unset;
1598 }
1599#ifdef KMP_DEBUG
1600 if (status == ETIMEDOUT) {
1601 if (flag->is_sleeping()) {
1602 KF_TRACE(100,
1603 ("__kmp_suspend_template: T#%d timeout wakeup\n", th_gtid));
1604 } else {
1605 KF_TRACE(2, ("__kmp_suspend_template: T#%d timeout wakeup, sleep bit "
1606 "not set!\n",
1607 th_gtid));
1608 TCW_PTR(th->th.th_sleep_loc, NULL);
1609 th->th.th_sleep_loc_type = flag_unset;
1610 }
1611 } else if (flag->is_sleeping()) {
1612 KF_TRACE(100,
1613 ("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid));
1614 }
1615#endif
1616 } // while
1617
1618 // Mark the thread as active again (if it was previous marked as inactive)
1619 if (deactivated) {
1620 th->th.th_active = TRUE;
1621 if (TCR_4(th->th.th_in_pool)) {
1622 KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
1623 th->th.th_active_in_pool = TRUE;
1624 }
1625 }
1626 }
1627 // We may have had the loop variable set before entering the loop body;
1628 // so we need to reset sleep_loc.
1629 TCW_PTR(th->th.th_sleep_loc, NULL);
1630 th->th.th_sleep_loc_type = flag_unset;
1631
1632 KMP_DEBUG_ASSERT(!flag->is_sleeping());
1633 KMP_DEBUG_ASSERT(!th->th.th_sleep_loc);
1634#ifdef DEBUG_SUSPEND
1635 {
1636 char buffer[128];
1637 __kmp_print_cond(buffer, &th->th.th_suspend_cv);
1638 __kmp_printf("__kmp_suspend_template: T#%d has awakened: %s\n", th_gtid,
1639 buffer);
1640 }
1641#endif
1642
1643 __kmp_unlock_suspend_mx(th);
1644 KF_TRACE(30, ("__kmp_suspend_template: T#%d exit\n", th_gtid));
1645}
1646
1647template <bool C, bool S>
1648void __kmp_suspend_32(int th_gtid, kmp_flag_32<C, S> *flag) {
1649 __kmp_suspend_template(th_gtid, flag);
1650}
1651template <bool C, bool S>
1652void __kmp_suspend_64(int th_gtid, kmp_flag_64<C, S> *flag) {
1653 __kmp_suspend_template(th_gtid, flag);
1654}
1655template <bool C, bool S>
1656void __kmp_atomic_suspend_64(int th_gtid, kmp_atomic_flag_64<C, S> *flag) {
1657 __kmp_suspend_template(th_gtid, flag);
1658}
1659void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
1660 __kmp_suspend_template(th_gtid, flag);
1661}
1662
1663template void __kmp_suspend_32<false, false>(int, kmp_flag_32<false, false> *);
1664template void __kmp_suspend_64<false, true>(int, kmp_flag_64<false, true> *);
1665template void __kmp_suspend_64<true, false>(int, kmp_flag_64<true, false> *);
1666template void
1667__kmp_atomic_suspend_64<false, true>(int, kmp_atomic_flag_64<false, true> *);
1668template void
1669__kmp_atomic_suspend_64<true, false>(int, kmp_atomic_flag_64<true, false> *);
1670
1671/* This routine signals the thread specified by target_gtid to wake up
1672 after setting the sleep bit indicated by the flag argument to FALSE.
1673 The target thread must already have called __kmp_suspend_template() */
1674template <class C>
1675static inline void __kmp_resume_template(int target_gtid, C *flag) {
1676 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
1677 kmp_info_t *th = __kmp_threads[target_gtid];
1678 int status;
1679
1680#ifdef KMP_DEBUG
1681 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1682#endif
1683
1684 KF_TRACE(30, ("__kmp_resume_template: T#%d wants to wakeup T#%d enter\n",
1685 gtid, target_gtid));
1686 KMP_DEBUG_ASSERT(gtid != target_gtid);
1687
1688 __kmp_suspend_initialize_thread(th);
1689
1690 __kmp_lock_suspend_mx(th);
1691
1692 if (!flag || flag != th->th.th_sleep_loc) {
1693 // coming from __kmp_null_resume_wrapper, or thread is now sleeping on a
1694 // different location; wake up at new location
1695 flag = (C *)CCAST(void *, th->th.th_sleep_loc);
1696 }
1697
1698 // First, check if the flag is null or its type has changed. If so, someone
1699 // else woke it up.
1700 if (!flag) { // Thread doesn't appear to be sleeping on anything
1701 KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
1702 "awake: flag(%p)\n",
1703 gtid, target_gtid, (void *)NULL));
1704 __kmp_unlock_suspend_mx(th);
1705 return;
1706 } else if (flag->get_type() != th->th.th_sleep_loc_type) {
1707 // Flag type does not appear to match this function template; possibly the
1708 // thread is sleeping on something else. Try null resume again.
1709 KF_TRACE(
1710 5,
1711 ("__kmp_resume_template: T#%d retrying, thread T#%d Mismatch flag(%p), "
1712 "spin(%p) type=%d ptr_type=%d\n",
1713 gtid, target_gtid, flag, flag->get(), flag->get_type(),
1714 th->th.th_sleep_loc_type));
1715 __kmp_unlock_suspend_mx(th);
1716 __kmp_null_resume_wrapper(th);
1717 return;
1718 } else { // if multiple threads are sleeping, flag should be internally
1719 // referring to a specific thread here
1720 if (!flag->is_sleeping()) {
1721 KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
1722 "awake: flag(%p): %u\n",
1723 gtid, target_gtid, flag->get(), (unsigned int)flag->load()));
1724 __kmp_unlock_suspend_mx(th);
1725 return;
1726 }
1727 }
1728 KMP_DEBUG_ASSERT(flag);
1729 flag->unset_sleeping();
1730 TCW_PTR(th->th.th_sleep_loc, NULL);
1731 th->th.th_sleep_loc_type = flag_unset;
1732
1733 KF_TRACE(5, ("__kmp_resume_template: T#%d about to wakeup T#%d, reset "
1734 "sleep bit for flag's loc(%p): %u\n",
1735 gtid, target_gtid, flag->get(), (unsigned int)flag->load()));
1736
1737#ifdef DEBUG_SUSPEND
1738 {
1739 char buffer[128];
1740 __kmp_print_cond(buffer, &th->th.th_suspend_cv);
1741 __kmp_printf("__kmp_resume_template: T#%d resuming T#%d: %s\n", gtid,
1742 target_gtid, buffer);
1743 }
1744#endif
1745 status = pthread_cond_signal(&th->th.th_suspend_cv.c_cond);
1746 KMP_CHECK_SYSFAIL("pthread_cond_signal", status);
1747 __kmp_unlock_suspend_mx(th);
1748 KF_TRACE(30, ("__kmp_resume_template: T#%d exiting after signaling wake up"
1749 " for T#%d\n",
1750 gtid, target_gtid));
1751}
1752
1753template <bool C, bool S>
1754void __kmp_resume_32(int target_gtid, kmp_flag_32<C, S> *flag) {
1755 __kmp_resume_template(target_gtid, flag);
1756}
1757template <bool C, bool S>
1758void __kmp_resume_64(int target_gtid, kmp_flag_64<C, S> *flag) {
1759 __kmp_resume_template(target_gtid, flag);
1760}
1761template <bool C, bool S>
1762void __kmp_atomic_resume_64(int target_gtid, kmp_atomic_flag_64<C, S> *flag) {
1763 __kmp_resume_template(target_gtid, flag);
1764}
1765void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
1766 __kmp_resume_template(target_gtid, flag);
1767}
1768
1769template void __kmp_resume_32<false, true>(int, kmp_flag_32<false, true> *);
1770template void __kmp_resume_32<false, false>(int, kmp_flag_32<false, false> *);
1771template void __kmp_resume_64<false, true>(int, kmp_flag_64<false, true> *);
1772template void
1773__kmp_atomic_resume_64<false, true>(int, kmp_atomic_flag_64<false, true> *);
1774
1775#if KMP_USE_MONITOR
1776void __kmp_resume_monitor() {
1777 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
1778 int status;
1779#ifdef KMP_DEBUG
1780 int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1781 KF_TRACE(30, ("__kmp_resume_monitor: T#%d wants to wakeup T#%d enter\n", gtid,
1782 KMP_GTID_MONITOR));
1783 KMP_DEBUG_ASSERT(gtid != KMP_GTID_MONITOR);
1784#endif
1785 status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex);
1786 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
1787#ifdef DEBUG_SUSPEND
1788 {
1789 char buffer[128];
1790 __kmp_print_cond(buffer, &__kmp_wait_cv.c_cond);
1791 __kmp_printf("__kmp_resume_monitor: T#%d resuming T#%d: %s\n", gtid,
1792 KMP_GTID_MONITOR, buffer);
1793 }
1794#endif
1795 status = pthread_cond_signal(&__kmp_wait_cv.c_cond);
1796 KMP_CHECK_SYSFAIL("pthread_cond_signal", status);
1797 status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex);
1798 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
1799 KF_TRACE(30, ("__kmp_resume_monitor: T#%d exiting after signaling wake up"
1800 " for T#%d\n",
1801 gtid, KMP_GTID_MONITOR));
1802}
1803#endif // KMP_USE_MONITOR
1804
1805void __kmp_yield() { sched_yield(); }
1806
1807void __kmp_gtid_set_specific(int gtid) {
1808 if (__kmp_init_gtid) {
1809 int status;
1810 status = pthread_setspecific(__kmp_gtid_threadprivate_key,
1811 (void *)(intptr_t)(gtid + 1));
1812 KMP_CHECK_SYSFAIL("pthread_setspecific", status);
1813 } else {
1814 KA_TRACE(50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n"));
1815 }
1816}
1817
1818int __kmp_gtid_get_specific() {
1819 int gtid;
1820 if (!__kmp_init_gtid) {
1821 KA_TRACE(50, ("__kmp_gtid_get_specific: runtime shutdown, returning "
1822 "KMP_GTID_SHUTDOWN\n"));
1823 return KMP_GTID_SHUTDOWN;
1824 }
1825 gtid = (int)(size_t)pthread_getspecific(__kmp_gtid_threadprivate_key);
1826 if (gtid == 0) {
1827 gtid = KMP_GTID_DNE;
1828 } else {
1829 gtid--;
1830 }
1831 KA_TRACE(50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
1832 __kmp_gtid_threadprivate_key, gtid));
1833 return gtid;
1834}
1835
1836double __kmp_read_cpu_time(void) {
1837 /*clock_t t;*/
1838 struct tms buffer;
1839
1840 /*t =*/times(&buffer);
1841
1842 return (double)(buffer.tms_utime + buffer.tms_cutime) /
1843 (double)CLOCKS_PER_SEC;
1844}
1845
1846int __kmp_read_system_info(struct kmp_sys_info *info) {
1847 int status;
1848 struct rusage r_usage;
1849
1850 memset(info, 0, sizeof(*info));
1851
1852 status = getrusage(RUSAGE_SELF, &r_usage);
1853 KMP_CHECK_SYSFAIL_ERRNO("getrusage", status);
1854
1855#if !KMP_OS_WASI
1856 // The maximum resident set size utilized (in kilobytes)
1857 info->maxrss = r_usage.ru_maxrss;
1858 // The number of page faults serviced without any I/O
1859 info->minflt = r_usage.ru_minflt;
1860 // The number of page faults serviced that required I/O
1861 info->majflt = r_usage.ru_majflt;
1862 // The number of times a process was "swapped" out of memory
1863 info->nswap = r_usage.ru_nswap;
1864 // The number of times the file system had to perform input
1865 info->inblock = r_usage.ru_inblock;
1866 // The number of times the file system had to perform output
1867 info->oublock = r_usage.ru_oublock;
1868 // The number of times a context switch was voluntarily
1869 info->nvcsw = r_usage.ru_nvcsw;
1870 // The number of times a context switch was forced
1871 info->nivcsw = r_usage.ru_nivcsw;
1872#endif
1873
1874 return (status != 0);
1875}
1876
1877void __kmp_read_system_time(double *delta) {
1878 double t_ns;
1879 struct timeval tval;
1880 struct timespec stop;
1881 int status;
1882
1883 status = gettimeofday(&tval, NULL);
1884 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
1885 TIMEVAL_TO_TIMESPEC(&tval, &stop);
1886 t_ns = (double)(TS2NS(stop) - TS2NS(__kmp_sys_timer_data.start));
1887 *delta = (t_ns * 1e-9);
1888}
1889
1890void __kmp_clear_system_time(void) {
1891 struct timeval tval;
1892 int status;
1893 status = gettimeofday(&tval, NULL);
1894 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
1895 TIMEVAL_TO_TIMESPEC(&tval, &__kmp_sys_timer_data.start);
1896}
1897
1898static int __kmp_get_xproc(void) {
1899
1900 int r = 0;
1901
1902#if KMP_OS_LINUX
1903
1904 __kmp_type_convert(sysconf(_SC_NPROCESSORS_CONF), &(r));
1905
1906#elif KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_OPENBSD || \
1907 KMP_OS_HAIKU || KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_WASI || KMP_OS_AIX
1908
1909 __kmp_type_convert(sysconf(_SC_NPROCESSORS_ONLN), &(r));
1910
1911#elif KMP_OS_DARWIN
1912
1913 size_t len = sizeof(r);
1914 sysctlbyname("hw.logicalcpu", &r, &len, NULL, 0);
1915
1916#else
1917
1918#error "Unknown or unsupported OS."
1919
1920#endif
1921
1922 return r > 0 ? r : 2; /* guess value of 2 if OS told us 0 */
1923
1924} // __kmp_get_xproc
1925
1926int __kmp_read_from_file(char const *path, char const *format, ...) {
1927 int result;
1928 va_list args;
1929
1930 va_start(args, format);
1931 FILE *f = fopen(path, "rb");
1932 if (f == NULL) {
1933 va_end(args);
1934 return 0;
1935 }
1936 result = vfscanf(f, format, args);
1937 fclose(f);
1938 va_end(args);
1939
1940 return result;
1941}
1942
1943void __kmp_runtime_initialize(void) {
1944 int status;
1945 pthread_mutexattr_t mutex_attr;
1946 pthread_condattr_t cond_attr;
1947
1948 if (__kmp_init_runtime) {
1949 return;
1950 }
1951
1952#if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1953 if (!__kmp_cpuinfo.initialized) {
1954 __kmp_query_cpuid(&__kmp_cpuinfo);
1955 }
1956#endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
1957
1958 __kmp_xproc = __kmp_get_xproc();
1959
1960#if !KMP_32_BIT_ARCH
1961 struct rlimit rlim;
1962 // read stack size of calling thread, save it as default for worker threads;
1963 // this should be done before reading environment variables
1964 status = getrlimit(RLIMIT_STACK, &rlim);
1965 if (status == 0) { // success?
1966 __kmp_stksize = rlim.rlim_cur;
1967 __kmp_check_stksize(&__kmp_stksize); // check value and adjust if needed
1968 }
1969#endif /* KMP_32_BIT_ARCH */
1970
1971 if (sysconf(_SC_THREADS)) {
1972
1973 /* Query the maximum number of threads */
1974 __kmp_type_convert(sysconf(_SC_THREAD_THREADS_MAX), &(__kmp_sys_max_nth));
1975#ifdef __ve__
1976 if (__kmp_sys_max_nth == -1) {
1977 // VE's pthread supports only up to 64 threads per a VE process.
1978 // So we use that KMP_MAX_NTH (predefined as 64) here.
1979 __kmp_sys_max_nth = KMP_MAX_NTH;
1980 }
1981#else
1982 if (__kmp_sys_max_nth == -1) {
1983 /* Unlimited threads for NPTL */
1984 __kmp_sys_max_nth = INT_MAX;
1985 } else if (__kmp_sys_max_nth <= 1) {
1986 /* Can't tell, just use PTHREAD_THREADS_MAX */
1987 __kmp_sys_max_nth = KMP_MAX_NTH;
1988 }
1989#endif
1990
1991 /* Query the minimum stack size */
1992 __kmp_sys_min_stksize = sysconf(_SC_THREAD_STACK_MIN);
1993 if (__kmp_sys_min_stksize <= 1) {
1994 __kmp_sys_min_stksize = KMP_MIN_STKSIZE;
1995 }
1996 }
1997
1998 /* Set up minimum number of threads to switch to TLS gtid */
1999 __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
2000
2001 status = pthread_key_create(&__kmp_gtid_threadprivate_key,
2002 __kmp_internal_end_dest);
2003 KMP_CHECK_SYSFAIL("pthread_key_create", status);
2004 status = pthread_mutexattr_init(&mutex_attr);
2005 KMP_CHECK_SYSFAIL("pthread_mutexattr_init", status);
2006 status = pthread_mutex_init(&__kmp_wait_mx.m_mutex, &mutex_attr);
2007 KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2008 status = pthread_mutexattr_destroy(&mutex_attr);
2009 KMP_CHECK_SYSFAIL("pthread_mutexattr_destroy", status);
2010 status = pthread_condattr_init(&cond_attr);
2011 KMP_CHECK_SYSFAIL("pthread_condattr_init", status);
2012 status = pthread_cond_init(&__kmp_wait_cv.c_cond, &cond_attr);
2013 KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2014 status = pthread_condattr_destroy(&cond_attr);
2015 KMP_CHECK_SYSFAIL("pthread_condattr_destroy", status);
2016#if USE_ITT_BUILD
2017 __kmp_itt_initialize();
2018#endif /* USE_ITT_BUILD */
2019
2020 __kmp_init_runtime = TRUE;
2021}
2022
2023void __kmp_runtime_destroy(void) {
2024 int status;
2025
2026 if (!__kmp_init_runtime) {
2027 return; // Nothing to do.
2028 }
2029
2030#if USE_ITT_BUILD
2031 __kmp_itt_destroy();
2032#endif /* USE_ITT_BUILD */
2033
2034 status = pthread_key_delete(__kmp_gtid_threadprivate_key);
2035 KMP_CHECK_SYSFAIL("pthread_key_delete", status);
2036
2037 status = pthread_mutex_destroy(&__kmp_wait_mx.m_mutex);
2038 if (status != 0 && status != EBUSY) {
2039 KMP_SYSFAIL("pthread_mutex_destroy", status);
2040 }
2041 status = pthread_cond_destroy(&__kmp_wait_cv.c_cond);
2042 if (status != 0 && status != EBUSY) {
2043 KMP_SYSFAIL("pthread_cond_destroy", status);
2044 }
2045#if KMP_AFFINITY_SUPPORTED
2046 __kmp_affinity_uninitialize();
2047#endif
2048
2049 __kmp_init_runtime = FALSE;
2050}
2051
2052/* Put the thread to sleep for a time period */
2053/* NOTE: not currently used anywhere */
2054void __kmp_thread_sleep(int millis) { sleep((millis + 500) / 1000); }
2055
2056/* Calculate the elapsed wall clock time for the user */
2057void __kmp_elapsed(double *t) {
2058 int status;
2059#ifdef FIX_SGI_CLOCK
2060 struct timespec ts;
2061
2062 status = clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts);
2063 KMP_CHECK_SYSFAIL_ERRNO("clock_gettime", status);
2064 *t =
2065 (double)ts.tv_nsec * (1.0 / (double)KMP_NSEC_PER_SEC) + (double)ts.tv_sec;
2066#else
2067 struct timeval tv;
2068
2069 status = gettimeofday(&tv, NULL);
2070 KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
2071 *t =
2072 (double)tv.tv_usec * (1.0 / (double)KMP_USEC_PER_SEC) + (double)tv.tv_sec;
2073#endif
2074}
2075
2076/* Calculate the elapsed wall clock tick for the user */
2077void __kmp_elapsed_tick(double *t) { *t = 1 / (double)CLOCKS_PER_SEC; }
2078
2079/* Return the current time stamp in nsec */
2080kmp_uint64 __kmp_now_nsec() {
2081 struct timeval t;
2082 gettimeofday(&t, NULL);
2083 kmp_uint64 nsec = (kmp_uint64)KMP_NSEC_PER_SEC * (kmp_uint64)t.tv_sec +
2084 (kmp_uint64)1000 * (kmp_uint64)t.tv_usec;
2085 return nsec;
2086}
2087
2088#if KMP_ARCH_X86 || KMP_ARCH_X86_64
2089/* Measure clock ticks per millisecond */
2090void __kmp_initialize_system_tick() {
2091 kmp_uint64 now, nsec2, diff;
2092 kmp_uint64 delay = 1000000; // ~450 usec on most machines.
2093 kmp_uint64 nsec = __kmp_now_nsec();
2094 kmp_uint64 goal = __kmp_hardware_timestamp() + delay;
2095 while ((now = __kmp_hardware_timestamp()) < goal)
2096 ;
2097 nsec2 = __kmp_now_nsec();
2098 diff = nsec2 - nsec;
2099 if (diff > 0) {
2100 double tpus = 1000.0 * (double)(delay + (now - goal)) / (double)diff;
2101 if (tpus > 0.0) {
2102 __kmp_ticks_per_msec = (kmp_uint64)(tpus * 1000.0);
2103 __kmp_ticks_per_usec = (kmp_uint64)tpus;
2104 }
2105 }
2106}
2107#endif
2108
2109/* Determine whether the given address is mapped into the current address
2110 space. */
2111
2112int __kmp_is_address_mapped(void *addr) {
2113
2114 int found = 0;
2115 int rc;
2116
2117#if KMP_OS_LINUX || KMP_OS_HURD
2118
2119 /* On GNUish OSes, read the /proc/<pid>/maps pseudo-file to get all the
2120 address ranges mapped into the address space. */
2121
2122 char *name = __kmp_str_format("/proc/%d/maps", getpid());
2123 FILE *file = NULL;
2124
2125 file = fopen(name, "r");
2126 KMP_ASSERT(file != NULL);
2127
2128 for (;;) {
2129
2130 void *beginning = NULL;
2131 void *ending = NULL;
2132 char perms[5];
2133
2134 rc = fscanf(file, "%p-%p %4s %*[^\n]\n", &beginning, &ending, perms);
2135 if (rc == EOF) {
2136 break;
2137 }
2138 KMP_ASSERT(rc == 3 &&
2139 KMP_STRLEN(perms) == 4); // Make sure all fields are read.
2140
2141 // Ending address is not included in the region, but beginning is.
2142 if ((addr >= beginning) && (addr < ending)) {
2143 perms[2] = 0; // 3th and 4th character does not matter.
2144 if (strcmp(perms, "rw") == 0) {
2145 // Memory we are looking for should be readable and writable.
2146 found = 1;
2147 }
2148 break;
2149 }
2150 }
2151
2152 // Free resources.
2153 fclose(file);
2154 KMP_INTERNAL_FREE(name);
2155#elif KMP_OS_FREEBSD
2156 char *buf;
2157 size_t lstsz;
2158 int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, getpid()};
2159 rc = sysctl(mib, 4, NULL, &lstsz, NULL, 0);
2160 if (rc < 0)
2161 return 0;
2162 // We pass from number of vm entry's semantic
2163 // to size of whole entry map list.
2164 lstsz = lstsz * 4 / 3;
2165 buf = reinterpret_cast<char *>(KMP_INTERNAL_MALLOC(lstsz));
2166 rc = sysctl(mib, 4, buf, &lstsz, NULL, 0);
2167 if (rc < 0) {
2168 KMP_INTERNAL_FREE(buf);
2169 return 0;
2170 }
2171
2172 char *lw = buf;
2173 char *up = buf + lstsz;
2174
2175 while (lw < up) {
2176 struct kinfo_vmentry *cur = reinterpret_cast<struct kinfo_vmentry *>(lw);
2177 size_t cursz = cur->kve_structsize;
2178 if (cursz == 0)
2179 break;
2180 void *start = reinterpret_cast<void *>(cur->kve_start);
2181 void *end = reinterpret_cast<void *>(cur->kve_end);
2182 // Readable/Writable addresses within current map entry
2183 if ((addr >= start) && (addr < end)) {
2184 if ((cur->kve_protection & KVME_PROT_READ) != 0 &&
2185 (cur->kve_protection & KVME_PROT_WRITE) != 0) {
2186 found = 1;
2187 break;
2188 }
2189 }
2190 lw += cursz;
2191 }
2192 KMP_INTERNAL_FREE(buf);
2193#elif KMP_OS_DRAGONFLY
2194 char err[_POSIX2_LINE_MAX];
2195 kinfo_proc *proc;
2196 vmspace sp;
2197 vm_map *cur;
2198 vm_map_entry entry, *c;
2199 struct proc p;
2200 kvm_t *fd;
2201 uintptr_t uaddr;
2202 int num;
2203
2204 fd = kvm_openfiles(nullptr, nullptr, nullptr, O_RDONLY, err);
2205 if (!fd) {
2206 return 0;
2207 }
2208
2209 proc = kvm_getprocs(fd, KERN_PROC_PID, getpid(), &num);
2210
2211 if (kvm_read(fd, static_cast<uintptr_t>(proc->kp_paddr), &p, sizeof(p)) !=
2212 sizeof(p) ||
2213 kvm_read(fd, reinterpret_cast<uintptr_t>(p.p_vmspace), &sp, sizeof(sp)) !=
2214 sizeof(sp)) {
2215 kvm_close(fd);
2216 return 0;
2217 }
2218
2219 (void)rc;
2220 cur = &sp.vm_map;
2221 uaddr = reinterpret_cast<uintptr_t>(addr);
2222 for (c = kvm_vm_map_entry_first(fd, cur, &entry); c;
2223 c = kvm_vm_map_entry_next(fd, c, &entry)) {
2224 if ((uaddr >= entry.ba.start) && (uaddr <= entry.ba.end)) {
2225 if ((entry.protection & VM_PROT_READ) != 0 &&
2226 (entry.protection & VM_PROT_WRITE) != 0) {
2227 found = 1;
2228 break;
2229 }
2230 }
2231 }
2232
2233 kvm_close(fd);
2234#elif KMP_OS_SOLARIS
2235 prmap_t *cur, *map;
2236 void *buf;
2237 uintptr_t uaddr;
2238 ssize_t rd;
2239 int err;
2240 int file;
2241
2242 pid_t pid = getpid();
2243 struct ps_prochandle *fd = Pgrab(pid, PGRAB_RDONLY, &err);
2244 ;
2245
2246 if (!fd) {
2247 return 0;
2248 }
2249
2250 char *name = __kmp_str_format("/proc/%d/map", pid);
2251 size_t sz = (1 << 20);
2252 file = open(name, O_RDONLY);
2253 if (file == -1) {
2254 KMP_INTERNAL_FREE(name);
2255 return 0;
2256 }
2257
2258 buf = KMP_INTERNAL_MALLOC(sz);
2259
2260 while (sz > 0 && (rd = pread(file, buf, sz, 0)) == sz) {
2261 void *newbuf;
2262 sz <<= 1;
2263 newbuf = KMP_INTERNAL_REALLOC(buf, sz);
2264 buf = newbuf;
2265 }
2266
2267 map = reinterpret_cast<prmap_t *>(buf);
2268 uaddr = reinterpret_cast<uintptr_t>(addr);
2269
2270 for (cur = map; rd > 0; cur++, rd = -sizeof(*map)) {
2271 if ((uaddr >= cur->pr_vaddr) && (uaddr < cur->pr_vaddr)) {
2272 if ((cur->pr_mflags & MA_READ) != 0 && (cur->pr_mflags & MA_WRITE) != 0) {
2273 found = 1;
2274 break;
2275 }
2276 }
2277 }
2278
2279 KMP_INTERNAL_FREE(map);
2280 close(file);
2281 KMP_INTERNAL_FREE(name);
2282#elif KMP_OS_DARWIN
2283
2284 /* On OS X*, /proc pseudo filesystem is not available. Try to read memory
2285 using vm interface. */
2286
2287 int buffer;
2288 vm_size_t count;
2289 rc = vm_read_overwrite(
2290 mach_task_self(), // Task to read memory of.
2291 (vm_address_t)(addr), // Address to read from.
2292 1, // Number of bytes to be read.
2293 (vm_address_t)(&buffer), // Address of buffer to save read bytes in.
2294 &count // Address of var to save number of read bytes in.
2295 );
2296 if (rc == 0) {
2297 // Memory successfully read.
2298 found = 1;
2299 }
2300
2301#elif KMP_OS_NETBSD
2302
2303 int mib[5];
2304 mib[0] = CTL_VM;
2305 mib[1] = VM_PROC;
2306 mib[2] = VM_PROC_MAP;
2307 mib[3] = getpid();
2308 mib[4] = sizeof(struct kinfo_vmentry);
2309
2310 size_t size;
2311 rc = sysctl(mib, __arraycount(mib), NULL, &size, NULL, 0);
2312 KMP_ASSERT(!rc);
2313 KMP_ASSERT(size);
2314
2315 size = size * 4 / 3;
2316 struct kinfo_vmentry *kiv = (struct kinfo_vmentry *)KMP_INTERNAL_MALLOC(size);
2317 KMP_ASSERT(kiv);
2318
2319 rc = sysctl(mib, __arraycount(mib), kiv, &size, NULL, 0);
2320 KMP_ASSERT(!rc);
2321 KMP_ASSERT(size);
2322
2323 for (size_t i = 0; i < size; i++) {
2324 if (kiv[i].kve_start >= (uint64_t)addr &&
2325 kiv[i].kve_end <= (uint64_t)addr) {
2326 found = 1;
2327 break;
2328 }
2329 }
2330 KMP_INTERNAL_FREE(kiv);
2331#elif KMP_OS_OPENBSD
2332
2333 int mib[3];
2334 mib[0] = CTL_KERN;
2335 mib[1] = KERN_PROC_VMMAP;
2336 mib[2] = getpid();
2337
2338 size_t size;
2339 uint64_t end;
2340 rc = sysctl(mib, 3, NULL, &size, NULL, 0);
2341 KMP_ASSERT(!rc);
2342 KMP_ASSERT(size);
2343 end = size;
2344
2345 struct kinfo_vmentry kiv = {.kve_start = 0};
2346
2347 while ((rc = sysctl(mib, 3, &kiv, &size, NULL, 0)) == 0) {
2348 KMP_ASSERT(size);
2349 if (kiv.kve_end == end)
2350 break;
2351
2352 if (kiv.kve_start >= (uint64_t)addr && kiv.kve_end <= (uint64_t)addr) {
2353 found = 1;
2354 break;
2355 }
2356 kiv.kve_start += 1;
2357 }
2358#elif KMP_OS_WASI
2359 found = (int)addr < (__builtin_wasm_memory_size(0) * PAGESIZE);
2360#elif KMP_OS_AIX
2361
2362 uint32_t loadQueryBufSize = 4096u; // Default loadquery buffer size.
2363 char *loadQueryBuf;
2364
2365 for (;;) {
2366 loadQueryBuf = (char *)KMP_INTERNAL_MALLOC(loadQueryBufSize);
2367 if (loadQueryBuf == NULL) {
2368 return 0;
2369 }
2370
2371 rc = loadquery(L_GETXINFO | L_IGNOREUNLOAD, loadQueryBuf, loadQueryBufSize);
2372 if (rc < 0) {
2373 KMP_INTERNAL_FREE(loadQueryBuf);
2374 if (errno != ENOMEM) {
2375 return 0;
2376 }
2377 // errno == ENOMEM; double the size.
2378 loadQueryBufSize <<= 1;
2379 continue;
2380 }
2381 // Obtained the load info successfully.
2382 break;
2383 }
2384
2385 struct ld_xinfo *curLdInfo = (struct ld_xinfo *)loadQueryBuf;
2386
2387 // Loop through the load info to find if there is a match.
2388 for (;;) {
2389 uintptr_t curDataStart = (uintptr_t)curLdInfo->ldinfo_dataorg;
2390 uintptr_t curDataEnd = curDataStart + curLdInfo->ldinfo_datasize;
2391
2392 // The data segment is readable and writable.
2393 if (curDataStart <= (uintptr_t)addr && (uintptr_t)addr < curDataEnd) {
2394 found = 1;
2395 break;
2396 }
2397 if (curLdInfo->ldinfo_next == 0u) {
2398 // Reached the end of load info.
2399 break;
2400 }
2401 curLdInfo = (struct ld_xinfo *)((char *)curLdInfo + curLdInfo->ldinfo_next);
2402 }
2403 KMP_INTERNAL_FREE(loadQueryBuf);
2404
2405#elif KMP_OS_HAIKU
2406
2407 found = 1;
2408#else
2409
2410#error "Unknown or unsupported OS"
2411
2412#endif
2413
2414 return found;
2415
2416} // __kmp_is_address_mapped
2417
2418#ifdef USE_LOAD_BALANCE
2419
2420#if KMP_OS_DARWIN || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
2421 KMP_OS_OPENBSD || KMP_OS_SOLARIS
2422
2423// The function returns the rounded value of the system load average
2424// during given time interval which depends on the value of
2425// __kmp_load_balance_interval variable (default is 60 sec, other values
2426// may be 300 sec or 900 sec).
2427// It returns -1 in case of error.
2428int __kmp_get_load_balance(int max) {
2429 double averages[3];
2430 int ret_avg = 0;
2431
2432 int res = getloadavg(averages, 3);
2433
2434 // Check __kmp_load_balance_interval to determine which of averages to use.
2435 // getloadavg() may return the number of samples less than requested that is
2436 // less than 3.
2437 if (__kmp_load_balance_interval < 180 && (res >= 1)) {
2438 ret_avg = (int)averages[0]; // 1 min
2439 } else if ((__kmp_load_balance_interval >= 180 &&
2440 __kmp_load_balance_interval < 600) &&
2441 (res >= 2)) {
2442 ret_avg = (int)averages[1]; // 5 min
2443 } else if ((__kmp_load_balance_interval >= 600) && (res == 3)) {
2444 ret_avg = (int)averages[2]; // 15 min
2445 } else { // Error occurred
2446 return -1;
2447 }
2448
2449 return ret_avg;
2450}
2451
2452#elif KMP_OS_AIX
2453
2454// The function returns number of running (not sleeping) threads, or -1 in case
2455// of error.
2456int __kmp_get_load_balance(int max) {
2457
2458 static int glb_running_threads = 0; // Saved count of the running threads for
2459 // the thread balance algorithm.
2460 static double glb_call_time = 0; // Thread balance algorithm call time.
2461 int running_threads = 0; // Number of running threads in the system.
2462
2463 double call_time = 0.0;
2464
2465 __kmp_elapsed(&call_time);
2466
2467 if (glb_call_time &&
2468 (call_time - glb_call_time < __kmp_load_balance_interval))
2469 return glb_running_threads;
2470
2471 glb_call_time = call_time;
2472
2473 if (max <= 0) {
2474 max = INT_MAX;
2475 }
2476
2477 // Check how many perfstat_cpu_t structures are available.
2478 int logical_cpus = perfstat_cpu(NULL, NULL, sizeof(perfstat_cpu_t), 0);
2479 if (logical_cpus <= 0) {
2480 glb_call_time = -1;
2481 return -1;
2482 }
2483
2484 perfstat_cpu_t *cpu_stat = (perfstat_cpu_t *)KMP_INTERNAL_MALLOC(
2485 logical_cpus * sizeof(perfstat_cpu_t));
2486 if (cpu_stat == NULL) {
2487 glb_call_time = -1;
2488 return -1;
2489 }
2490
2491 // Set first CPU as the name of the first logical CPU for which the info is
2492 // desired.
2493 perfstat_id_t first_cpu_name;
2494 strcpy(first_cpu_name.name, FIRST_CPU);
2495
2496 // Get the stat info of logical CPUs.
2497 int rc = perfstat_cpu(&first_cpu_name, cpu_stat, sizeof(perfstat_cpu_t),
2498 logical_cpus);
2499 KMP_DEBUG_ASSERT(rc == logical_cpus);
2500 if (rc <= 0) {
2501 KMP_INTERNAL_FREE(cpu_stat);
2502 glb_call_time = -1;
2503 return -1;
2504 }
2505 for (int i = 0; i < logical_cpus; ++i) {
2506 running_threads += cpu_stat[i].runque;
2507 if (running_threads >= max)
2508 break;
2509 }
2510
2511 // There _might_ be a timing hole where the thread executing this
2512 // code gets skipped in the load balance, and running_threads is 0.
2513 // Assert in the debug builds only!!!
2514 KMP_DEBUG_ASSERT(running_threads > 0);
2515 if (running_threads <= 0)
2516 running_threads = 1;
2517
2518 KMP_INTERNAL_FREE(cpu_stat);
2519
2520 glb_running_threads = running_threads;
2521
2522 return running_threads;
2523}
2524
2525#elif KMP_OS_HAIKU
2526
2527int __kmp_get_load_balance(int max) { return -1; }
2528
2529#else // Linux* OS
2530
2531// The function returns number of running (not sleeping) threads, or -1 in case
2532// of error. Error could be reported if Linux* OS kernel too old (without
2533// "/proc" support). Counting running threads stops if max running threads
2534// encountered.
2535int __kmp_get_load_balance(int max) {
2536 static int permanent_error = 0;
2537 static int glb_running_threads = 0; // Saved count of the running threads for
2538 // the thread balance algorithm
2539 static double glb_call_time = 0; /* Thread balance algorithm call time */
2540
2541 int running_threads = 0; // Number of running threads in the system.
2542
2543 DIR *proc_dir = NULL; // Handle of "/proc/" directory.
2544 struct dirent *proc_entry = NULL;
2545
2546 kmp_str_buf_t task_path; // "/proc/<pid>/task/<tid>/" path.
2547 DIR *task_dir = NULL; // Handle of "/proc/<pid>/task/<tid>/" directory.
2548 struct dirent *task_entry = NULL;
2549 int task_path_fixed_len;
2550
2551 kmp_str_buf_t stat_path; // "/proc/<pid>/task/<tid>/stat" path.
2552 int stat_file = -1;
2553 int stat_path_fixed_len;
2554
2555#ifdef KMP_DEBUG
2556 int total_processes = 0; // Total number of processes in system.
2557#endif
2558
2559 double call_time = 0.0;
2560
2561 __kmp_str_buf_init(&task_path);
2562 __kmp_str_buf_init(&stat_path);
2563
2564 __kmp_elapsed(&call_time);
2565
2566 if (glb_call_time &&
2567 (call_time - glb_call_time < __kmp_load_balance_interval)) {
2568 running_threads = glb_running_threads;
2569 goto finish;
2570 }
2571
2572 glb_call_time = call_time;
2573
2574 // Do not spend time on scanning "/proc/" if we have a permanent error.
2575 if (permanent_error) {
2576 running_threads = -1;
2577 goto finish;
2578 }
2579
2580 if (max <= 0) {
2581 max = INT_MAX;
2582 }
2583
2584 // Open "/proc/" directory.
2585 proc_dir = opendir("/proc");
2586 if (proc_dir == NULL) {
2587 // Cannot open "/proc/". Probably the kernel does not support it. Return an
2588 // error now and in subsequent calls.
2589 running_threads = -1;
2590 permanent_error = 1;
2591 goto finish;
2592 }
2593
2594 // Initialize fixed part of task_path. This part will not change.
2595 __kmp_str_buf_cat(&task_path, "/proc/", 6);
2596 task_path_fixed_len = task_path.used; // Remember number of used characters.
2597
2598 proc_entry = readdir(proc_dir);
2599 while (proc_entry != NULL) {
2600 // Proc entry is a directory and name starts with a digit. Assume it is a
2601 // process' directory.
2602 if (proc_entry->d_type == DT_DIR && isdigit(proc_entry->d_name[0])) {
2603
2604#ifdef KMP_DEBUG
2605 ++total_processes;
2606#endif
2607 // Make sure init process is the very first in "/proc", so we can replace
2608 // strcmp( proc_entry->d_name, "1" ) == 0 with simpler total_processes ==
2609 // 1. We are going to check that total_processes == 1 => d_name == "1" is
2610 // true (where "=>" is implication). Since C++ does not have => operator,
2611 // let us replace it with its equivalent: a => b == ! a || b.
2612 KMP_DEBUG_ASSERT(total_processes != 1 ||
2613 strcmp(proc_entry->d_name, "1") == 0);
2614
2615 // Construct task_path.
2616 task_path.used = task_path_fixed_len; // Reset task_path to "/proc/".
2617 __kmp_str_buf_cat(&task_path, proc_entry->d_name,
2618 KMP_STRLEN(proc_entry->d_name));
2619 __kmp_str_buf_cat(&task_path, "/task", 5);
2620
2621 task_dir = opendir(task_path.str);
2622 if (task_dir == NULL) {
2623 // Process can finish between reading "/proc/" directory entry and
2624 // opening process' "task/" directory. So, in general case we should not
2625 // complain, but have to skip this process and read the next one. But on
2626 // systems with no "task/" support we will spend lot of time to scan
2627 // "/proc/" tree again and again without any benefit. "init" process
2628 // (its pid is 1) should exist always, so, if we cannot open
2629 // "/proc/1/task/" directory, it means "task/" is not supported by
2630 // kernel. Report an error now and in the future.
2631 if (strcmp(proc_entry->d_name, "1") == 0) {
2632 running_threads = -1;
2633 permanent_error = 1;
2634 goto finish;
2635 }
2636 } else {
2637 // Construct fixed part of stat file path.
2638 __kmp_str_buf_clear(&stat_path);
2639 __kmp_str_buf_cat(&stat_path, task_path.str, task_path.used);
2640 __kmp_str_buf_cat(&stat_path, "/", 1);
2641 stat_path_fixed_len = stat_path.used;
2642
2643 task_entry = readdir(task_dir);
2644 while (task_entry != NULL) {
2645 // It is a directory and name starts with a digit.
2646 if (proc_entry->d_type == DT_DIR && isdigit(task_entry->d_name[0])) {
2647
2648 // Construct complete stat file path. Easiest way would be:
2649 // __kmp_str_buf_print( & stat_path, "%s/%s/stat", task_path.str,
2650 // task_entry->d_name );
2651 // but seriae of __kmp_str_buf_cat works a bit faster.
2652 stat_path.used =
2653 stat_path_fixed_len; // Reset stat path to its fixed part.
2654 __kmp_str_buf_cat(&stat_path, task_entry->d_name,
2655 KMP_STRLEN(task_entry->d_name));
2656 __kmp_str_buf_cat(&stat_path, "/stat", 5);
2657
2658 // Note: Low-level API (open/read/close) is used. High-level API
2659 // (fopen/fclose) works ~ 30 % slower.
2660 stat_file = open(stat_path.str, O_RDONLY);
2661 if (stat_file == -1) {
2662 // We cannot report an error because task (thread) can terminate
2663 // just before reading this file.
2664 } else {
2665 /* Content of "stat" file looks like:
2666 24285 (program) S ...
2667
2668 It is a single line (if program name does not include funny
2669 symbols). First number is a thread id, then name of executable
2670 file name in paretheses, then state of the thread. We need just
2671 thread state.
2672
2673 Good news: Length of program name is 15 characters max. Longer
2674 names are truncated.
2675
2676 Thus, we need rather short buffer: 15 chars for program name +
2677 2 parenthesis, + 3 spaces + ~7 digits of pid = 37.
2678
2679 Bad news: Program name may contain special symbols like space,
2680 closing parenthesis, or even new line. This makes parsing
2681 "stat" file not 100 % reliable. In case of fanny program names
2682 parsing may fail (report incorrect thread state).
2683
2684 Parsing "status" file looks more promissing (due to different
2685 file structure and escaping special symbols) but reading and
2686 parsing of "status" file works slower.
2687 -- ln
2688 */
2689 char buffer[65];
2690 ssize_t len;
2691 len = read(stat_file, buffer, sizeof(buffer) - 1);
2692 if (len >= 0) {
2693 buffer[len] = 0;
2694 // Using scanf:
2695 // sscanf( buffer, "%*d (%*s) %c ", & state );
2696 // looks very nice, but searching for a closing parenthesis
2697 // works a bit faster.
2698 char *close_parent = strstr(buffer, ") ");
2699 if (close_parent != NULL) {
2700 char state = *(close_parent + 2);
2701 if (state == 'R') {
2702 ++running_threads;
2703 if (running_threads >= max) {
2704 goto finish;
2705 }
2706 }
2707 }
2708 }
2709 close(stat_file);
2710 stat_file = -1;
2711 }
2712 }
2713 task_entry = readdir(task_dir);
2714 }
2715 closedir(task_dir);
2716 task_dir = NULL;
2717 }
2718 }
2719 proc_entry = readdir(proc_dir);
2720 }
2721
2722 // There _might_ be a timing hole where the thread executing this
2723 // code get skipped in the load balance, and running_threads is 0.
2724 // Assert in the debug builds only!!!
2725 KMP_DEBUG_ASSERT(running_threads > 0);
2726 if (running_threads <= 0) {
2727 running_threads = 1;
2728 }
2729
2730finish: // Clean up and exit.
2731 if (proc_dir != NULL) {
2732 closedir(proc_dir);
2733 }
2734 __kmp_str_buf_free(&task_path);
2735 if (task_dir != NULL) {
2736 closedir(task_dir);
2737 }
2738 __kmp_str_buf_free(&stat_path);
2739 if (stat_file != -1) {
2740 close(stat_file);
2741 }
2742
2743 glb_running_threads = running_threads;
2744
2745 return running_threads;
2746
2747} // __kmp_get_load_balance
2748
2749#endif // KMP_OS_DARWIN
2750
2751#endif // USE_LOAD_BALANCE
2752
2753#if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || \
2754 ((KMP_OS_LINUX || KMP_OS_DARWIN) && KMP_ARCH_AARCH64) || \
2755 KMP_ARCH_PPC64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 || \
2756 KMP_ARCH_ARM || KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_PPC_XCOFF || \
2757 KMP_ARCH_AARCH64_32)
2758
2759// Because WebAssembly will use `call_indirect` to invoke the microtask and
2760// WebAssembly indirect calls check that the called signature is a precise
2761// match, we need to cast each microtask function pointer back from `void *` to
2762// its original type.
2763typedef void (*microtask_t0)(int *, int *);
2764typedef void (*microtask_t1)(int *, int *, void *);
2765typedef void (*microtask_t2)(int *, int *, void *, void *);
2766typedef void (*microtask_t3)(int *, int *, void *, void *, void *);
2767typedef void (*microtask_t4)(int *, int *, void *, void *, void *, void *);
2768typedef void (*microtask_t5)(int *, int *, void *, void *, void *, void *,
2769 void *);
2770typedef void (*microtask_t6)(int *, int *, void *, void *, void *, void *,
2771 void *, void *);
2772typedef void (*microtask_t7)(int *, int *, void *, void *, void *, void *,
2773 void *, void *, void *);
2774typedef void (*microtask_t8)(int *, int *, void *, void *, void *, void *,
2775 void *, void *, void *, void *);
2776typedef void (*microtask_t9)(int *, int *, void *, void *, void *, void *,
2777 void *, void *, void *, void *, void *);
2778typedef void (*microtask_t10)(int *, int *, void *, void *, void *, void *,
2779 void *, void *, void *, void *, void *, void *);
2780typedef void (*microtask_t11)(int *, int *, void *, void *, void *, void *,
2781 void *, void *, void *, void *, void *, void *,
2782 void *);
2783typedef void (*microtask_t12)(int *, int *, void *, void *, void *, void *,
2784 void *, void *, void *, void *, void *, void *,
2785 void *, void *);
2786typedef void (*microtask_t13)(int *, int *, void *, void *, void *, void *,
2787 void *, void *, void *, void *, void *, void *,
2788 void *, void *, void *);
2789typedef void (*microtask_t14)(int *, int *, void *, void *, void *, void *,
2790 void *, void *, void *, void *, void *, void *,
2791 void *, void *, void *, void *);
2792typedef void (*microtask_t15)(int *, int *, void *, void *, void *, void *,
2793 void *, void *, void *, void *, void *, void *,
2794 void *, void *, void *, void *, void *);
2795
2796// we really only need the case with 1 argument, because CLANG always build
2797// a struct of pointers to shared variables referenced in the outlined function
2798int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
2799 void *p_argv[]
2800#if OMPT_SUPPORT
2801 ,
2802 void **exit_frame_ptr
2803#endif
2804) {
2805#if OMPT_SUPPORT
2806 *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
2807#endif
2808
2809 switch (argc) {
2810 default:
2811 fprintf(stderr, "Too many args to microtask: %d!\n", argc);
2812 fflush(stderr);
2813 exit(-1);
2814 case 0:
2815 (*(microtask_t0)pkfn)(&gtid, &tid);
2816 break;
2817 case 1:
2818 (*(microtask_t1)pkfn)(&gtid, &tid, p_argv[0]);
2819 break;
2820 case 2:
2821 (*(microtask_t2)pkfn)(&gtid, &tid, p_argv[0], p_argv[1]);
2822 break;
2823 case 3:
2824 (*(microtask_t3)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2]);
2825 break;
2826 case 4:
2827 (*(microtask_t4)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2828 p_argv[3]);
2829 break;
2830 case 5:
2831 (*(microtask_t5)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2832 p_argv[3], p_argv[4]);
2833 break;
2834 case 6:
2835 (*(microtask_t6)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2836 p_argv[3], p_argv[4], p_argv[5]);
2837 break;
2838 case 7:
2839 (*(microtask_t7)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2840 p_argv[3], p_argv[4], p_argv[5], p_argv[6]);
2841 break;
2842 case 8:
2843 (*(microtask_t8)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2844 p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2845 p_argv[7]);
2846 break;
2847 case 9:
2848 (*(microtask_t9)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2849 p_argv[3], p_argv[4], p_argv[5], p_argv[6], p_argv[7],
2850 p_argv[8]);
2851 break;
2852 case 10:
2853 (*(microtask_t10)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2854 p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2855 p_argv[7], p_argv[8], p_argv[9]);
2856 break;
2857 case 11:
2858 (*(microtask_t11)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2859 p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2860 p_argv[7], p_argv[8], p_argv[9], p_argv[10]);
2861 break;
2862 case 12:
2863 (*(microtask_t12)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2864 p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2865 p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2866 p_argv[11]);
2867 break;
2868 case 13:
2869 (*(microtask_t13)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2870 p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2871 p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2872 p_argv[11], p_argv[12]);
2873 break;
2874 case 14:
2875 (*(microtask_t14)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2876 p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2877 p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2878 p_argv[11], p_argv[12], p_argv[13]);
2879 break;
2880 case 15:
2881 (*(microtask_t15)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2882 p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2883 p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2884 p_argv[11], p_argv[12], p_argv[13], p_argv[14]);
2885 break;
2886 }
2887
2888 return 1;
2889}
2890
2891#endif
2892
2893#if KMP_OS_LINUX
2894// Functions for hidden helper task
2895namespace {
2896// Condition variable for initializing hidden helper team
2897pthread_cond_t hidden_helper_threads_initz_cond_var;
2898pthread_mutex_t hidden_helper_threads_initz_lock;
2899volatile int hidden_helper_initz_signaled = FALSE;
2900
2901// Condition variable for deinitializing hidden helper team
2902pthread_cond_t hidden_helper_threads_deinitz_cond_var;
2903pthread_mutex_t hidden_helper_threads_deinitz_lock;
2904volatile int hidden_helper_deinitz_signaled = FALSE;
2905
2906// Condition variable for the wrapper function of main thread
2907pthread_cond_t hidden_helper_main_thread_cond_var;
2908pthread_mutex_t hidden_helper_main_thread_lock;
2909volatile int hidden_helper_main_thread_signaled = FALSE;
2910
2911// Semaphore for worker threads. We don't use condition variable here in case
2912// that when multiple signals are sent at the same time, only one thread might
2913// be waken.
2914sem_t hidden_helper_task_sem;
2915} // namespace
2916
2917void __kmp_hidden_helper_worker_thread_wait() {
2918 int status = sem_wait(&hidden_helper_task_sem);
2919 KMP_CHECK_SYSFAIL("sem_wait", status);
2920}
2921
2922void __kmp_do_initialize_hidden_helper_threads() {
2923 // Initialize condition variable
2924 int status =
2925 pthread_cond_init(&hidden_helper_threads_initz_cond_var, nullptr);
2926 KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2927
2928 status = pthread_cond_init(&hidden_helper_threads_deinitz_cond_var, nullptr);
2929 KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2930
2931 status = pthread_cond_init(&hidden_helper_main_thread_cond_var, nullptr);
2932 KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2933
2934 status = pthread_mutex_init(&hidden_helper_threads_initz_lock, nullptr);
2935 KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2936
2937 status = pthread_mutex_init(&hidden_helper_threads_deinitz_lock, nullptr);
2938 KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2939
2940 status = pthread_mutex_init(&hidden_helper_main_thread_lock, nullptr);
2941 KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2942
2943 // Initialize the semaphore
2944 status = sem_init(&hidden_helper_task_sem, 0, 0);
2945 KMP_CHECK_SYSFAIL("sem_init", status);
2946
2947 // Create a new thread to finish initialization
2948 pthread_t handle;
2949 status = pthread_create(
2950 &handle, nullptr,
2951 [](void *) -> void * {
2952 __kmp_hidden_helper_threads_initz_routine();
2953 return nullptr;
2954 },
2955 nullptr);
2956 KMP_CHECK_SYSFAIL("pthread_create", status);
2957}
2958
2959void __kmp_hidden_helper_threads_initz_wait() {
2960 // Initial thread waits here for the completion of the initialization. The
2961 // condition variable will be notified by main thread of hidden helper teams.
2962 int status = pthread_mutex_lock(&hidden_helper_threads_initz_lock);
2963 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2964
2965 if (!TCR_4(hidden_helper_initz_signaled)) {
2966 status = pthread_cond_wait(&hidden_helper_threads_initz_cond_var,
2967 &hidden_helper_threads_initz_lock);
2968 KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2969 }
2970
2971 status = pthread_mutex_unlock(&hidden_helper_threads_initz_lock);
2972 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2973}
2974
2975void __kmp_hidden_helper_initz_release() {
2976 // After all initialization, reset __kmp_init_hidden_helper_threads to false.
2977 int status = pthread_mutex_lock(&hidden_helper_threads_initz_lock);
2978 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2979
2980 status = pthread_cond_signal(&hidden_helper_threads_initz_cond_var);
2981 KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2982
2983 TCW_SYNC_4(hidden_helper_initz_signaled, TRUE);
2984
2985 status = pthread_mutex_unlock(&hidden_helper_threads_initz_lock);
2986 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2987}
2988
2989void __kmp_hidden_helper_main_thread_wait() {
2990 // The main thread of hidden helper team will be blocked here. The
2991 // condition variable can only be signal in the destructor of RTL.
2992 int status = pthread_mutex_lock(&hidden_helper_main_thread_lock);
2993 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2994
2995 if (!TCR_4(hidden_helper_main_thread_signaled)) {
2996 status = pthread_cond_wait(&hidden_helper_main_thread_cond_var,
2997 &hidden_helper_main_thread_lock);
2998 KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2999 }
3000
3001 status = pthread_mutex_unlock(&hidden_helper_main_thread_lock);
3002 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
3003}
3004
3005void __kmp_hidden_helper_main_thread_release() {
3006 // The initial thread of OpenMP RTL should call this function to wake up the
3007 // main thread of hidden helper team.
3008 int status = pthread_mutex_lock(&hidden_helper_main_thread_lock);
3009 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
3010
3011 status = pthread_cond_signal(&hidden_helper_main_thread_cond_var);
3012 KMP_CHECK_SYSFAIL("pthread_cond_signal", status);
3013
3014 // The hidden helper team is done here
3015 TCW_SYNC_4(hidden_helper_main_thread_signaled, TRUE);
3016
3017 status = pthread_mutex_unlock(&hidden_helper_main_thread_lock);
3018 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
3019}
3020
3021void __kmp_hidden_helper_worker_thread_signal() {
3022 int status = sem_post(&hidden_helper_task_sem);
3023 KMP_CHECK_SYSFAIL("sem_post", status);
3024}
3025
3026void __kmp_hidden_helper_threads_deinitz_wait() {
3027 // Initial thread waits here for the completion of the deinitialization. The
3028 // condition variable will be notified by main thread of hidden helper teams.
3029 int status = pthread_mutex_lock(&hidden_helper_threads_deinitz_lock);
3030 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
3031
3032 if (!TCR_4(hidden_helper_deinitz_signaled)) {
3033 status = pthread_cond_wait(&hidden_helper_threads_deinitz_cond_var,
3034 &hidden_helper_threads_deinitz_lock);
3035 KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
3036 }
3037
3038 status = pthread_mutex_unlock(&hidden_helper_threads_deinitz_lock);
3039 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
3040}
3041
3042void __kmp_hidden_helper_threads_deinitz_release() {
3043 int status = pthread_mutex_lock(&hidden_helper_threads_deinitz_lock);
3044 KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
3045
3046 status = pthread_cond_signal(&hidden_helper_threads_deinitz_cond_var);
3047 KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
3048
3049 TCW_SYNC_4(hidden_helper_deinitz_signaled, TRUE);
3050
3051 status = pthread_mutex_unlock(&hidden_helper_threads_deinitz_lock);
3052 KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
3053}
3054#else // KMP_OS_LINUX
3055void __kmp_hidden_helper_worker_thread_wait() {
3056 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3057}
3058
3059void __kmp_do_initialize_hidden_helper_threads() {
3060 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3061}
3062
3063void __kmp_hidden_helper_threads_initz_wait() {
3064 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3065}
3066
3067void __kmp_hidden_helper_initz_release() {
3068 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3069}
3070
3071void __kmp_hidden_helper_main_thread_wait() {
3072 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3073}
3074
3075void __kmp_hidden_helper_main_thread_release() {
3076 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3077}
3078
3079void __kmp_hidden_helper_worker_thread_signal() {
3080 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3081}
3082
3083void __kmp_hidden_helper_threads_deinitz_wait() {
3084 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3085}
3086
3087void __kmp_hidden_helper_threads_deinitz_release() {
3088 KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3089}
3090#endif // KMP_OS_LINUX
3091
3092bool __kmp_detect_shm() {
3093 DIR *dir = opendir("/dev/shm");
3094 if (dir) { // /dev/shm exists
3095 closedir(dir);
3096 return true;
3097 } else if (ENOENT == errno) { // /dev/shm does not exist
3098 return false;
3099 } else { // opendir() failed
3100 return false;
3101 }
3102}
3103
3104bool __kmp_detect_tmp() {
3105 DIR *dir = opendir("/tmp");
3106 if (dir) { // /tmp exists
3107 closedir(dir);
3108 return true;
3109 } else if (ENOENT == errno) { // /tmp does not exist
3110 return false;
3111 } else { // opendir() failed
3112 return false;
3113 }
3114}
3115
3116// end of file //
#define KMP_INIT_PARTITIONED_TIMERS(name)
Initializes the partitioned timers to begin with name.
Definition kmp_stats.h:940