LLVM OpenMP* Runtime Library
z_Linux_util.cpp
1 /*
2  * z_Linux_util.cpp -- platform specific routines.
3  */
4 
5 //===----------------------------------------------------------------------===//
6 //
7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8 // See https://llvm.org/LICENSE.txt for license information.
9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "kmp.h"
14 #include "kmp_affinity.h"
15 #include "kmp_i18n.h"
16 #include "kmp_io.h"
17 #include "kmp_itt.h"
18 #include "kmp_lock.h"
19 #include "kmp_stats.h"
20 #include "kmp_str.h"
21 #include "kmp_wait_release.h"
22 #include "kmp_wrapper_getpid.h"
23 
24 #if !KMP_OS_DRAGONFLY && !KMP_OS_FREEBSD && !KMP_OS_NETBSD && !KMP_OS_OPENBSD
25 #include <alloca.h>
26 #endif
27 #include <math.h> // HUGE_VAL.
28 #if KMP_OS_LINUX
29 #include <semaphore.h>
30 #endif // KMP_OS_LINUX
31 #include <sys/resource.h>
32 #if KMP_OS_AIX
33 #include <sys/ldr.h>
34 #include <libperfstat.h>
35 #elif !KMP_OS_HAIKU
36 #include <sys/syscall.h>
37 #endif
38 #include <sys/time.h>
39 #include <sys/times.h>
40 #include <unistd.h>
41 
42 #if KMP_OS_LINUX
43 #include <sys/sysinfo.h>
44 #if KMP_USE_FUTEX
45 // We should really include <futex.h>, but that causes compatibility problems on
46 // different Linux* OS distributions that either require that you include (or
47 // break when you try to include) <pci/types.h>. Since all we need is the two
48 // macros below (which are part of the kernel ABI, so can't change) we just
49 // define the constants here and don't include <futex.h>
50 #ifndef FUTEX_WAIT
51 #define FUTEX_WAIT 0
52 #endif
53 #ifndef FUTEX_WAKE
54 #define FUTEX_WAKE 1
55 #endif
56 #endif
57 #elif KMP_OS_DARWIN
58 #include <mach/mach.h>
59 #include <sys/sysctl.h>
60 #elif KMP_OS_DRAGONFLY || KMP_OS_FREEBSD
61 #include <sys/types.h>
62 #include <sys/sysctl.h>
63 #include <sys/user.h>
64 #include <pthread_np.h>
65 #if KMP_OS_DRAGONFLY
66 #include <kvm.h>
67 #endif
68 #elif KMP_OS_NETBSD || KMP_OS_OPENBSD
69 #include <sys/types.h>
70 #include <sys/sysctl.h>
71 #if KMP_OS_NETBSD
72 #include <sched.h>
73 #endif
74 #if KMP_OS_OPENBSD
75 #include <pthread_np.h>
76 #endif
77 #elif KMP_OS_SOLARIS
78 #include <libproc.h>
79 #include <procfs.h>
80 #include <thread.h>
81 #include <sys/loadavg.h>
82 #endif
83 
84 #include <ctype.h>
85 #include <dirent.h>
86 #include <fcntl.h>
87 
88 struct kmp_sys_timer {
89  struct timespec start;
90 };
91 
92 #ifndef TIMEVAL_TO_TIMESPEC
93 // Convert timeval to timespec.
94 #define TIMEVAL_TO_TIMESPEC(tv, ts) \
95  do { \
96  (ts)->tv_sec = (tv)->tv_sec; \
97  (ts)->tv_nsec = (tv)->tv_usec * 1000; \
98  } while (0)
99 #endif
100 
101 // Convert timespec to nanoseconds.
102 #define TS2NS(timespec) \
103  (((timespec).tv_sec * (long int)1e9) + (timespec).tv_nsec)
104 
105 static struct kmp_sys_timer __kmp_sys_timer_data;
106 
107 #if KMP_HANDLE_SIGNALS
108 typedef void (*sig_func_t)(int);
109 STATIC_EFI2_WORKAROUND struct sigaction __kmp_sighldrs[NSIG];
110 static sigset_t __kmp_sigset;
111 #endif
112 
113 static int __kmp_init_runtime = FALSE;
114 
115 static int __kmp_fork_count = 0;
116 
117 static pthread_condattr_t __kmp_suspend_cond_attr;
118 static pthread_mutexattr_t __kmp_suspend_mutex_attr;
119 
120 static kmp_cond_align_t __kmp_wait_cv;
121 static kmp_mutex_align_t __kmp_wait_mx;
122 
123 kmp_uint64 __kmp_ticks_per_msec = 1000000;
124 kmp_uint64 __kmp_ticks_per_usec = 1000;
125 
126 #ifdef DEBUG_SUSPEND
127 static void __kmp_print_cond(char *buffer, kmp_cond_align_t *cond) {
128  KMP_SNPRINTF(buffer, 128, "(cond (lock (%ld, %d)), (descr (%p)))",
129  cond->c_cond.__c_lock.__status, cond->c_cond.__c_lock.__spinlock,
130  cond->c_cond.__c_waiting);
131 }
132 #endif
133 
134 #if ((KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \
135  KMP_OS_AIX) && \
136  KMP_AFFINITY_SUPPORTED)
137 
138 /* Affinity support */
139 
140 void __kmp_affinity_bind_thread(int which) {
141  KMP_ASSERT2(KMP_AFFINITY_CAPABLE(),
142  "Illegal set affinity operation when not capable");
143 
144  kmp_affin_mask_t *mask;
145  KMP_CPU_ALLOC_ON_STACK(mask);
146  KMP_CPU_ZERO(mask);
147  KMP_CPU_SET(which, mask);
148  __kmp_set_system_affinity(mask, TRUE);
149  KMP_CPU_FREE_FROM_STACK(mask);
150 }
151 
152 #if KMP_OS_AIX
153 void __kmp_affinity_determine_capable(const char *env_var) {
154  // All versions of AIX support bindprocessor().
155 
156  size_t mask_size = __kmp_xproc / CHAR_BIT;
157  // Round up to byte boundary.
158  if (__kmp_xproc % CHAR_BIT)
159  ++mask_size;
160 
161  // Round up to the mask_size_type boundary.
162  if (mask_size % sizeof(__kmp_affin_mask_size))
163  mask_size += sizeof(__kmp_affin_mask_size) -
164  mask_size % sizeof(__kmp_affin_mask_size);
165  KMP_AFFINITY_ENABLE(mask_size);
166  KA_TRACE(10,
167  ("__kmp_affinity_determine_capable: "
168  "AIX OS affinity interface bindprocessor functional (mask size = "
169  "%" KMP_SIZE_T_SPEC ").\n",
170  __kmp_affin_mask_size));
171 }
172 
173 #else // !KMP_OS_AIX
174 
175 /* Determine if we can access affinity functionality on this version of
176  * Linux* OS by checking __NR_sched_{get,set}affinity system calls, and set
177  * __kmp_affin_mask_size to the appropriate value (0 means not capable). */
178 void __kmp_affinity_determine_capable(const char *env_var) {
179  // Check and see if the OS supports thread affinity.
180 
181 #if KMP_OS_LINUX
182 #define KMP_CPU_SET_SIZE_LIMIT (1024 * 1024)
183 #define KMP_CPU_SET_TRY_SIZE CACHE_LINE
184 #elif KMP_OS_FREEBSD || KMP_OS_DRAGONFLY
185 #define KMP_CPU_SET_SIZE_LIMIT (sizeof(cpuset_t))
186 #elif KMP_OS_NETBSD
187 #define KMP_CPU_SET_SIZE_LIMIT (256)
188 #endif
189 
190  int verbose = __kmp_affinity.flags.verbose;
191  int warnings = __kmp_affinity.flags.warnings;
192  enum affinity_type type = __kmp_affinity.type;
193 
194 #if KMP_OS_LINUX
195  long gCode;
196  unsigned char *buf;
197  buf = (unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT);
198 
199  // If the syscall returns a suggestion for the size,
200  // then we don't have to search for an appropriate size.
201  gCode = syscall(__NR_sched_getaffinity, 0, KMP_CPU_SET_TRY_SIZE, buf);
202  KA_TRACE(30, ("__kmp_affinity_determine_capable: "
203  "initial getaffinity call returned %ld errno = %d\n",
204  gCode, errno));
205 
206  if (gCode < 0 && errno != EINVAL) {
207  // System call not supported
208  if (verbose ||
209  (warnings && (type != affinity_none) && (type != affinity_default) &&
210  (type != affinity_disabled))) {
211  int error = errno;
212  kmp_msg_t err_code = KMP_ERR(error);
213  __kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var),
214  err_code, __kmp_msg_null);
215  if (__kmp_generate_warnings == kmp_warnings_off) {
216  __kmp_str_free(&err_code.str);
217  }
218  }
219  KMP_AFFINITY_DISABLE();
220  KMP_INTERNAL_FREE(buf);
221  return;
222  } else if (gCode > 0) {
223  // The optimal situation: the OS returns the size of the buffer it expects.
224  KMP_AFFINITY_ENABLE(gCode);
225  KA_TRACE(10, ("__kmp_affinity_determine_capable: "
226  "affinity supported (mask size %d)\n",
227  (int)__kmp_affin_mask_size));
228  KMP_INTERNAL_FREE(buf);
229  return;
230  }
231 
232  // Call the getaffinity system call repeatedly with increasing set sizes
233  // until we succeed, or reach an upper bound on the search.
234  KA_TRACE(30, ("__kmp_affinity_determine_capable: "
235  "searching for proper set size\n"));
236  int size;
237  for (size = 1; size <= KMP_CPU_SET_SIZE_LIMIT; size *= 2) {
238  gCode = syscall(__NR_sched_getaffinity, 0, size, buf);
239  KA_TRACE(30, ("__kmp_affinity_determine_capable: "
240  "getaffinity for mask size %ld returned %ld errno = %d\n",
241  size, gCode, errno));
242 
243  if (gCode < 0) {
244  if (errno == ENOSYS) {
245  // We shouldn't get here
246  KA_TRACE(30, ("__kmp_affinity_determine_capable: "
247  "inconsistent OS call behavior: errno == ENOSYS for mask "
248  "size %d\n",
249  size));
250  if (verbose ||
251  (warnings && (type != affinity_none) &&
252  (type != affinity_default) && (type != affinity_disabled))) {
253  int error = errno;
254  kmp_msg_t err_code = KMP_ERR(error);
255  __kmp_msg(kmp_ms_warning, KMP_MSG(GetAffSysCallNotSupported, env_var),
256  err_code, __kmp_msg_null);
257  if (__kmp_generate_warnings == kmp_warnings_off) {
258  __kmp_str_free(&err_code.str);
259  }
260  }
261  KMP_AFFINITY_DISABLE();
262  KMP_INTERNAL_FREE(buf);
263  return;
264  }
265  continue;
266  }
267 
268  KMP_AFFINITY_ENABLE(gCode);
269  KA_TRACE(10, ("__kmp_affinity_determine_capable: "
270  "affinity supported (mask size %d)\n",
271  (int)__kmp_affin_mask_size));
272  KMP_INTERNAL_FREE(buf);
273  return;
274  }
275 #elif KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY
276  long gCode;
277  unsigned char *buf;
278  buf = (unsigned char *)KMP_INTERNAL_MALLOC(KMP_CPU_SET_SIZE_LIMIT);
279  gCode = pthread_getaffinity_np(pthread_self(), KMP_CPU_SET_SIZE_LIMIT,
280  reinterpret_cast<cpuset_t *>(buf));
281  KA_TRACE(30, ("__kmp_affinity_determine_capable: "
282  "initial getaffinity call returned %d errno = %d\n",
283  gCode, errno));
284  if (gCode == 0) {
285  KMP_AFFINITY_ENABLE(KMP_CPU_SET_SIZE_LIMIT);
286  KA_TRACE(10, ("__kmp_affinity_determine_capable: "
287  "affinity supported (mask size %d)\n",
288  (int)__kmp_affin_mask_size));
289  KMP_INTERNAL_FREE(buf);
290  return;
291  }
292 #endif
293  KMP_INTERNAL_FREE(buf);
294 
295  // Affinity is not supported
296  KMP_AFFINITY_DISABLE();
297  KA_TRACE(10, ("__kmp_affinity_determine_capable: "
298  "cannot determine mask size - affinity not supported\n"));
299  if (verbose || (warnings && (type != affinity_none) &&
300  (type != affinity_default) && (type != affinity_disabled))) {
301  KMP_WARNING(AffCantGetMaskSize, env_var);
302  }
303 }
304 #endif // KMP_OS_AIX
305 #endif // (KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
306  KMP_OS_DRAGONFLY || KMP_OS_AIX) && KMP_AFFINITY_SUPPORTED
307 
308 #if KMP_USE_FUTEX
309 
310 int __kmp_futex_determine_capable() {
311  int loc = 0;
312  long rc = syscall(__NR_futex, &loc, FUTEX_WAKE, 1, NULL, NULL, 0);
313  int retval = (rc == 0) || (errno != ENOSYS);
314 
315  KA_TRACE(10,
316  ("__kmp_futex_determine_capable: rc = %d errno = %d\n", rc, errno));
317  KA_TRACE(10, ("__kmp_futex_determine_capable: futex syscall%s supported\n",
318  retval ? "" : " not"));
319 
320  return retval;
321 }
322 
323 #endif // KMP_USE_FUTEX
324 
325 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_WASM) && (!KMP_ASM_INTRINS)
326 /* Only 32-bit "add-exchange" instruction on IA-32 architecture causes us to
327  use compare_and_store for these routines */
328 
329 kmp_int8 __kmp_test_then_or8(volatile kmp_int8 *p, kmp_int8 d) {
330  kmp_int8 old_value, new_value;
331 
332  old_value = TCR_1(*p);
333  new_value = old_value | d;
334 
335  while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
336  KMP_CPU_PAUSE();
337  old_value = TCR_1(*p);
338  new_value = old_value | d;
339  }
340  return old_value;
341 }
342 
343 kmp_int8 __kmp_test_then_and8(volatile kmp_int8 *p, kmp_int8 d) {
344  kmp_int8 old_value, new_value;
345 
346  old_value = TCR_1(*p);
347  new_value = old_value & d;
348 
349  while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
350  KMP_CPU_PAUSE();
351  old_value = TCR_1(*p);
352  new_value = old_value & d;
353  }
354  return old_value;
355 }
356 
357 kmp_uint32 __kmp_test_then_or32(volatile kmp_uint32 *p, kmp_uint32 d) {
358  kmp_uint32 old_value, new_value;
359 
360  old_value = TCR_4(*p);
361  new_value = old_value | d;
362 
363  while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) {
364  KMP_CPU_PAUSE();
365  old_value = TCR_4(*p);
366  new_value = old_value | d;
367  }
368  return old_value;
369 }
370 
371 kmp_uint32 __kmp_test_then_and32(volatile kmp_uint32 *p, kmp_uint32 d) {
372  kmp_uint32 old_value, new_value;
373 
374  old_value = TCR_4(*p);
375  new_value = old_value & d;
376 
377  while (!KMP_COMPARE_AND_STORE_REL32(p, old_value, new_value)) {
378  KMP_CPU_PAUSE();
379  old_value = TCR_4(*p);
380  new_value = old_value & d;
381  }
382  return old_value;
383 }
384 
385 #if KMP_ARCH_X86 || KMP_ARCH_WASM
386 kmp_int8 __kmp_test_then_add8(volatile kmp_int8 *p, kmp_int8 d) {
387  kmp_int8 old_value, new_value;
388 
389  old_value = TCR_1(*p);
390  new_value = old_value + d;
391 
392  while (!KMP_COMPARE_AND_STORE_REL8(p, old_value, new_value)) {
393  KMP_CPU_PAUSE();
394  old_value = TCR_1(*p);
395  new_value = old_value + d;
396  }
397  return old_value;
398 }
399 
400 kmp_int64 __kmp_test_then_add64(volatile kmp_int64 *p, kmp_int64 d) {
401  kmp_int64 old_value, new_value;
402 
403  old_value = TCR_8(*p);
404  new_value = old_value + d;
405 
406  while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
407  KMP_CPU_PAUSE();
408  old_value = TCR_8(*p);
409  new_value = old_value + d;
410  }
411  return old_value;
412 }
413 #endif /* KMP_ARCH_X86 */
414 
415 kmp_uint64 __kmp_test_then_or64(volatile kmp_uint64 *p, kmp_uint64 d) {
416  kmp_uint64 old_value, new_value;
417 
418  old_value = TCR_8(*p);
419  new_value = old_value | d;
420  while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
421  KMP_CPU_PAUSE();
422  old_value = TCR_8(*p);
423  new_value = old_value | d;
424  }
425  return old_value;
426 }
427 
428 kmp_uint64 __kmp_test_then_and64(volatile kmp_uint64 *p, kmp_uint64 d) {
429  kmp_uint64 old_value, new_value;
430 
431  old_value = TCR_8(*p);
432  new_value = old_value & d;
433  while (!KMP_COMPARE_AND_STORE_REL64(p, old_value, new_value)) {
434  KMP_CPU_PAUSE();
435  old_value = TCR_8(*p);
436  new_value = old_value & d;
437  }
438  return old_value;
439 }
440 
441 #endif /* (KMP_ARCH_X86 || KMP_ARCH_X86_64) && (! KMP_ASM_INTRINS) */
442 
443 void __kmp_terminate_thread(int gtid) {
444  int status;
445  kmp_info_t *th = __kmp_threads[gtid];
446 
447  if (!th)
448  return;
449 
450 #ifdef KMP_CANCEL_THREADS
451  KA_TRACE(10, ("__kmp_terminate_thread: kill (%d)\n", gtid));
452  status = pthread_cancel(th->th.th_info.ds.ds_thread);
453  if (status != 0 && status != ESRCH) {
454  __kmp_fatal(KMP_MSG(CantTerminateWorkerThread), KMP_ERR(status),
455  __kmp_msg_null);
456  }
457 #endif
458  KMP_YIELD(TRUE);
459 } //
460 
461 /* Set thread stack info.
462  If values are unreasonable, assume call failed and use incremental stack
463  refinement method instead. Returns TRUE if the stack parameters could be
464  determined exactly, FALSE if incremental refinement is necessary. */
465 static kmp_int32 __kmp_set_stack_info(int gtid, kmp_info_t *th) {
466  int stack_data;
467 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
468  KMP_OS_HAIKU || KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_AIX
469  int status;
470  size_t size = 0;
471  void *addr = 0;
472 
473  /* Always do incremental stack refinement for ubermaster threads since the
474  initial thread stack range can be reduced by sibling thread creation so
475  pthread_attr_getstack may cause thread gtid aliasing */
476  if (!KMP_UBER_GTID(gtid)) {
477 
478 #if KMP_OS_SOLARIS
479  stack_t s;
480  if ((status = thr_stksegment(&s)) < 0) {
481  KMP_CHECK_SYSFAIL("thr_stksegment", status);
482  }
483 
484  addr = s.ss_sp;
485  size = s.ss_size;
486  KA_TRACE(60, ("__kmp_set_stack_info: T#%d thr_stksegment returned size:"
487  " %lu, low addr: %p\n",
488  gtid, size, addr));
489 #else
490  pthread_attr_t attr;
491  /* Fetch the real thread attributes */
492  status = pthread_attr_init(&attr);
493  KMP_CHECK_SYSFAIL("pthread_attr_init", status);
494 #if KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD
495  status = pthread_attr_get_np(pthread_self(), &attr);
496  KMP_CHECK_SYSFAIL("pthread_attr_get_np", status);
497 #else
498  status = pthread_getattr_np(pthread_self(), &attr);
499  KMP_CHECK_SYSFAIL("pthread_getattr_np", status);
500 #endif
501  status = pthread_attr_getstack(&attr, &addr, &size);
502  KMP_CHECK_SYSFAIL("pthread_attr_getstack", status);
503  KA_TRACE(60,
504  ("__kmp_set_stack_info: T#%d pthread_attr_getstack returned size:"
505  " %lu, low addr: %p\n",
506  gtid, size, addr));
507  status = pthread_attr_destroy(&attr);
508  KMP_CHECK_SYSFAIL("pthread_attr_destroy", status);
509 #endif
510  }
511 
512  if (size != 0 && addr != 0) { // was stack parameter determination successful?
513  /* Store the correct base and size */
514  TCW_PTR(th->th.th_info.ds.ds_stackbase, (((char *)addr) + size));
515  TCW_PTR(th->th.th_info.ds.ds_stacksize, size);
516  TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
517  return TRUE;
518  }
519 #endif /* KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD \
520  || KMP_OS_HAIKU || KMP_OS_HURD || KMP_OS_SOLARIS */
521  /* Use incremental refinement starting from initial conservative estimate */
522  TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
523  TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
524  TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
525  return FALSE;
526 }
527 
528 static void *__kmp_launch_worker(void *thr) {
529  int status, old_type, old_state;
530 #ifdef KMP_BLOCK_SIGNALS
531  sigset_t new_set, old_set;
532 #endif /* KMP_BLOCK_SIGNALS */
533  void *exit_val;
534 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
535  KMP_OS_OPENBSD || KMP_OS_HAIKU || KMP_OS_HURD || KMP_OS_SOLARIS || \
536  KMP_OS_AIX
537  void *volatile padding = 0;
538 #endif
539  int gtid;
540 
541  gtid = ((kmp_info_t *)thr)->th.th_info.ds.ds_gtid;
542  __kmp_gtid_set_specific(gtid);
543 #ifdef KMP_TDATA_GTID
544  __kmp_gtid = gtid;
545 #endif
546 #if KMP_STATS_ENABLED
547  // set thread local index to point to thread-specific stats
548  __kmp_stats_thread_ptr = ((kmp_info_t *)thr)->th.th_stats;
549  __kmp_stats_thread_ptr->startLife();
550  KMP_SET_THREAD_STATE(IDLE);
551  KMP_INIT_PARTITIONED_TIMERS(OMP_idle);
552 #endif
553 
554 #if USE_ITT_BUILD
555  __kmp_itt_thread_name(gtid);
556 #endif /* USE_ITT_BUILD */
557 
558 #if KMP_AFFINITY_SUPPORTED
559  __kmp_affinity_bind_init_mask(gtid);
560 #endif
561 
562 #ifdef KMP_CANCEL_THREADS
563  status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type);
564  KMP_CHECK_SYSFAIL("pthread_setcanceltype", status);
565  // josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads?
566  status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state);
567  KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
568 #endif
569 
570 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
571  // Set FP control regs to be a copy of the parallel initialization thread's.
572  __kmp_clear_x87_fpu_status_word();
573  __kmp_load_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
574  __kmp_load_mxcsr(&__kmp_init_mxcsr);
575 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
576 
577 #ifdef KMP_BLOCK_SIGNALS
578  status = sigfillset(&new_set);
579  KMP_CHECK_SYSFAIL_ERRNO("sigfillset", status);
580  status = pthread_sigmask(SIG_BLOCK, &new_set, &old_set);
581  KMP_CHECK_SYSFAIL("pthread_sigmask", status);
582 #endif /* KMP_BLOCK_SIGNALS */
583 
584 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
585  KMP_OS_OPENBSD || KMP_OS_HAIKU || KMP_OS_HURD || KMP_OS_SOLARIS || \
586  KMP_OS_AIX
587  if (__kmp_stkoffset > 0 && gtid > 0) {
588  padding = KMP_ALLOCA(gtid * __kmp_stkoffset);
589  (void)padding;
590  }
591 #endif
592 
593  KMP_MB();
594  __kmp_set_stack_info(gtid, (kmp_info_t *)thr);
595 
596  __kmp_check_stack_overlap((kmp_info_t *)thr);
597 
598  exit_val = __kmp_launch_thread((kmp_info_t *)thr);
599 
600 #ifdef KMP_BLOCK_SIGNALS
601  status = pthread_sigmask(SIG_SETMASK, &old_set, NULL);
602  KMP_CHECK_SYSFAIL("pthread_sigmask", status);
603 #endif /* KMP_BLOCK_SIGNALS */
604 
605  return exit_val;
606 }
607 
608 #if KMP_USE_MONITOR
609 /* The monitor thread controls all of the threads in the complex */
610 
611 static void *__kmp_launch_monitor(void *thr) {
612  int status, old_type, old_state;
613 #ifdef KMP_BLOCK_SIGNALS
614  sigset_t new_set;
615 #endif /* KMP_BLOCK_SIGNALS */
616  struct timespec interval;
617 
618  KMP_MB(); /* Flush all pending memory write invalidates. */
619 
620  KA_TRACE(10, ("__kmp_launch_monitor: #1 launched\n"));
621 
622  /* register us as the monitor thread */
623  __kmp_gtid_set_specific(KMP_GTID_MONITOR);
624 #ifdef KMP_TDATA_GTID
625  __kmp_gtid = KMP_GTID_MONITOR;
626 #endif
627 
628  KMP_MB();
629 
630 #if USE_ITT_BUILD
631  // Instruct Intel(R) Threading Tools to ignore monitor thread.
632  __kmp_itt_thread_ignore();
633 #endif /* USE_ITT_BUILD */
634 
635  __kmp_set_stack_info(((kmp_info_t *)thr)->th.th_info.ds.ds_gtid,
636  (kmp_info_t *)thr);
637 
638  __kmp_check_stack_overlap((kmp_info_t *)thr);
639 
640 #ifdef KMP_CANCEL_THREADS
641  status = pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, &old_type);
642  KMP_CHECK_SYSFAIL("pthread_setcanceltype", status);
643  // josh todo: isn't PTHREAD_CANCEL_ENABLE default for newly-created threads?
644  status = pthread_setcancelstate(PTHREAD_CANCEL_ENABLE, &old_state);
645  KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
646 #endif
647 
648 #if KMP_REAL_TIME_FIX
649  // This is a potential fix which allows application with real-time scheduling
650  // policy work. However, decision about the fix is not made yet, so it is
651  // disabled by default.
652  { // Are program started with real-time scheduling policy?
653  int sched = sched_getscheduler(0);
654  if (sched == SCHED_FIFO || sched == SCHED_RR) {
655  // Yes, we are a part of real-time application. Try to increase the
656  // priority of the monitor.
657  struct sched_param param;
658  int max_priority = sched_get_priority_max(sched);
659  int rc;
660  KMP_WARNING(RealTimeSchedNotSupported);
661  sched_getparam(0, &param);
662  if (param.sched_priority < max_priority) {
663  param.sched_priority += 1;
664  rc = sched_setscheduler(0, sched, &param);
665  if (rc != 0) {
666  int error = errno;
667  kmp_msg_t err_code = KMP_ERR(error);
668  __kmp_msg(kmp_ms_warning, KMP_MSG(CantChangeMonitorPriority),
669  err_code, KMP_MSG(MonitorWillStarve), __kmp_msg_null);
670  if (__kmp_generate_warnings == kmp_warnings_off) {
671  __kmp_str_free(&err_code.str);
672  }
673  }
674  } else {
675  // We cannot abort here, because number of CPUs may be enough for all
676  // the threads, including the monitor thread, so application could
677  // potentially work...
678  __kmp_msg(kmp_ms_warning, KMP_MSG(RunningAtMaxPriority),
679  KMP_MSG(MonitorWillStarve), KMP_HNT(RunningAtMaxPriority),
680  __kmp_msg_null);
681  }
682  }
683  // AC: free thread that waits for monitor started
684  TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
685  }
686 #endif // KMP_REAL_TIME_FIX
687 
688  KMP_MB(); /* Flush all pending memory write invalidates. */
689 
690  if (__kmp_monitor_wakeups == 1) {
691  interval.tv_sec = 1;
692  interval.tv_nsec = 0;
693  } else {
694  interval.tv_sec = 0;
695  interval.tv_nsec = (KMP_NSEC_PER_SEC / __kmp_monitor_wakeups);
696  }
697 
698  KA_TRACE(10, ("__kmp_launch_monitor: #2 monitor\n"));
699 
700  while (!TCR_4(__kmp_global.g.g_done)) {
701  struct timespec now;
702  struct timeval tval;
703 
704  /* This thread monitors the state of the system */
705 
706  KA_TRACE(15, ("__kmp_launch_monitor: update\n"));
707 
708  status = gettimeofday(&tval, NULL);
709  KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
710  TIMEVAL_TO_TIMESPEC(&tval, &now);
711 
712  now.tv_sec += interval.tv_sec;
713  now.tv_nsec += interval.tv_nsec;
714 
715  if (now.tv_nsec >= KMP_NSEC_PER_SEC) {
716  now.tv_sec += 1;
717  now.tv_nsec -= KMP_NSEC_PER_SEC;
718  }
719 
720  status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex);
721  KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
722  // AC: the monitor should not fall asleep if g_done has been set
723  if (!TCR_4(__kmp_global.g.g_done)) { // check once more under mutex
724  status = pthread_cond_timedwait(&__kmp_wait_cv.c_cond,
725  &__kmp_wait_mx.m_mutex, &now);
726  if (status != 0) {
727  if (status != ETIMEDOUT && status != EINTR) {
728  KMP_SYSFAIL("pthread_cond_timedwait", status);
729  }
730  }
731  }
732  status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex);
733  KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
734 
735  TCW_4(__kmp_global.g.g_time.dt.t_value,
736  TCR_4(__kmp_global.g.g_time.dt.t_value) + 1);
737 
738  KMP_MB(); /* Flush all pending memory write invalidates. */
739  }
740 
741  KA_TRACE(10, ("__kmp_launch_monitor: #3 cleanup\n"));
742 
743 #ifdef KMP_BLOCK_SIGNALS
744  status = sigfillset(&new_set);
745  KMP_CHECK_SYSFAIL_ERRNO("sigfillset", status);
746  status = pthread_sigmask(SIG_UNBLOCK, &new_set, NULL);
747  KMP_CHECK_SYSFAIL("pthread_sigmask", status);
748 #endif /* KMP_BLOCK_SIGNALS */
749 
750  KA_TRACE(10, ("__kmp_launch_monitor: #4 finished\n"));
751 
752  if (__kmp_global.g.g_abort != 0) {
753  /* now we need to terminate the worker threads */
754  /* the value of t_abort is the signal we caught */
755 
756  int gtid;
757 
758  KA_TRACE(10, ("__kmp_launch_monitor: #5 terminate sig=%d\n",
759  __kmp_global.g.g_abort));
760 
761  /* terminate the OpenMP worker threads */
762  /* TODO this is not valid for sibling threads!!
763  * the uber master might not be 0 anymore.. */
764  for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
765  __kmp_terminate_thread(gtid);
766 
767  __kmp_cleanup();
768 
769  KA_TRACE(10, ("__kmp_launch_monitor: #6 raise sig=%d\n",
770  __kmp_global.g.g_abort));
771 
772  if (__kmp_global.g.g_abort > 0)
773  raise(__kmp_global.g.g_abort);
774  }
775 
776  KA_TRACE(10, ("__kmp_launch_monitor: #7 exit\n"));
777 
778  return thr;
779 }
780 #endif // KMP_USE_MONITOR
781 
782 void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size) {
783  pthread_t handle;
784  pthread_attr_t thread_attr;
785  int status;
786 
787  th->th.th_info.ds.ds_gtid = gtid;
788 
789 #if KMP_STATS_ENABLED
790  // sets up worker thread stats
791  __kmp_acquire_tas_lock(&__kmp_stats_lock, gtid);
792 
793  // th->th.th_stats is used to transfer thread-specific stats-pointer to
794  // __kmp_launch_worker. So when thread is created (goes into
795  // __kmp_launch_worker) it will set its thread local pointer to
796  // th->th.th_stats
797  if (!KMP_UBER_GTID(gtid)) {
798  th->th.th_stats = __kmp_stats_list->push_back(gtid);
799  } else {
800  // For root threads, __kmp_stats_thread_ptr is set in __kmp_register_root(),
801  // so set the th->th.th_stats field to it.
802  th->th.th_stats = __kmp_stats_thread_ptr;
803  }
804  __kmp_release_tas_lock(&__kmp_stats_lock, gtid);
805 
806 #endif // KMP_STATS_ENABLED
807 
808  if (KMP_UBER_GTID(gtid)) {
809  KA_TRACE(10, ("__kmp_create_worker: uber thread (%d)\n", gtid));
810  th->th.th_info.ds.ds_thread = pthread_self();
811  __kmp_set_stack_info(gtid, th);
812  __kmp_check_stack_overlap(th);
813  return;
814  }
815 
816  KA_TRACE(10, ("__kmp_create_worker: try to create thread (%d)\n", gtid));
817 
818  KMP_MB(); /* Flush all pending memory write invalidates. */
819 
820 #ifdef KMP_THREAD_ATTR
821  status = pthread_attr_init(&thread_attr);
822  if (status != 0) {
823  __kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null);
824  }
825  status = pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE);
826  if (status != 0) {
827  __kmp_fatal(KMP_MSG(CantSetWorkerState), KMP_ERR(status), __kmp_msg_null);
828  }
829 
830  /* Set stack size for this thread now.
831  The multiple of 2 is there because on some machines, requesting an unusual
832  stacksize causes the thread to have an offset before the dummy alloca()
833  takes place to create the offset. Since we want the user to have a
834  sufficient stacksize AND support a stack offset, we alloca() twice the
835  offset so that the upcoming alloca() does not eliminate any premade offset,
836  and also gives the user the stack space they requested for all threads */
837  stack_size += gtid * __kmp_stkoffset * 2;
838 
839  KA_TRACE(10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
840  "__kmp_stksize = %lu bytes, final stacksize = %lu bytes\n",
841  gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size));
842 
843 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
844  status = pthread_attr_setstacksize(&thread_attr, stack_size);
845 #ifdef KMP_BACKUP_STKSIZE
846  if (status != 0) {
847  if (!__kmp_env_stksize) {
848  stack_size = KMP_BACKUP_STKSIZE + gtid * __kmp_stkoffset;
849  __kmp_stksize = KMP_BACKUP_STKSIZE;
850  KA_TRACE(10, ("__kmp_create_worker: T#%d, default stacksize = %lu bytes, "
851  "__kmp_stksize = %lu bytes, (backup) final stacksize = %lu "
852  "bytes\n",
853  gtid, KMP_DEFAULT_STKSIZE, __kmp_stksize, stack_size));
854  status = pthread_attr_setstacksize(&thread_attr, stack_size);
855  }
856  }
857 #endif /* KMP_BACKUP_STKSIZE */
858  if (status != 0) {
859  __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
860  KMP_HNT(ChangeWorkerStackSize), __kmp_msg_null);
861  }
862 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
863 
864 #endif /* KMP_THREAD_ATTR */
865 
866  status =
867  pthread_create(&handle, &thread_attr, __kmp_launch_worker, (void *)th);
868  if (status != 0 || !handle) { // ??? Why do we check handle??
869 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
870  if (status == EINVAL) {
871  __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
872  KMP_HNT(IncreaseWorkerStackSize), __kmp_msg_null);
873  }
874  if (status == ENOMEM) {
875  __kmp_fatal(KMP_MSG(CantSetWorkerStackSize, stack_size), KMP_ERR(status),
876  KMP_HNT(DecreaseWorkerStackSize), __kmp_msg_null);
877  }
878 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
879  if (status == EAGAIN) {
880  __kmp_fatal(KMP_MSG(NoResourcesForWorkerThread), KMP_ERR(status),
881  KMP_HNT(Decrease_NUM_THREADS), __kmp_msg_null);
882  }
883  KMP_SYSFAIL("pthread_create", status);
884  }
885 
886  // Rename worker threads for improved debuggability
887  if (!KMP_UBER_GTID(gtid)) {
888 #if defined(LIBOMP_HAVE_PTHREAD_SET_NAME_NP)
889  pthread_set_name_np(handle, "openmp_worker");
890 #elif defined(LIBOMP_HAVE_PTHREAD_SETNAME_NP) && !KMP_OS_DARWIN
891 #if KMP_OS_NETBSD
892  pthread_setname_np(handle, "%s", const_cast<char *>("openmp_worker"));
893 #else
894  pthread_setname_np(handle, "openmp_worker");
895 #endif
896 #endif
897  }
898 
899  th->th.th_info.ds.ds_thread = handle;
900 
901 #ifdef KMP_THREAD_ATTR
902  status = pthread_attr_destroy(&thread_attr);
903  if (status) {
904  kmp_msg_t err_code = KMP_ERR(status);
905  __kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code,
906  __kmp_msg_null);
907  if (__kmp_generate_warnings == kmp_warnings_off) {
908  __kmp_str_free(&err_code.str);
909  }
910  }
911 #endif /* KMP_THREAD_ATTR */
912 
913  KMP_MB(); /* Flush all pending memory write invalidates. */
914 
915  KA_TRACE(10, ("__kmp_create_worker: done creating thread (%d)\n", gtid));
916 
917 } // __kmp_create_worker
918 
919 #if KMP_USE_MONITOR
920 void __kmp_create_monitor(kmp_info_t *th) {
921  pthread_t handle;
922  pthread_attr_t thread_attr;
923  size_t size;
924  int status;
925  int auto_adj_size = FALSE;
926 
927  if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
928  // We don't need monitor thread in case of MAX_BLOCKTIME
929  KA_TRACE(10, ("__kmp_create_monitor: skipping monitor thread because of "
930  "MAX blocktime\n"));
931  th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op
932  th->th.th_info.ds.ds_gtid = 0;
933  return;
934  }
935  KA_TRACE(10, ("__kmp_create_monitor: try to create monitor\n"));
936 
937  KMP_MB(); /* Flush all pending memory write invalidates. */
938 
939  th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
940  th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
941 #if KMP_REAL_TIME_FIX
942  TCW_4(__kmp_global.g.g_time.dt.t_value,
943  -1); // Will use it for synchronization a bit later.
944 #else
945  TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
946 #endif // KMP_REAL_TIME_FIX
947 
948 #ifdef KMP_THREAD_ATTR
949  if (__kmp_monitor_stksize == 0) {
950  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
951  auto_adj_size = TRUE;
952  }
953  status = pthread_attr_init(&thread_attr);
954  if (status != 0) {
955  __kmp_fatal(KMP_MSG(CantInitThreadAttrs), KMP_ERR(status), __kmp_msg_null);
956  }
957  status = pthread_attr_setdetachstate(&thread_attr, PTHREAD_CREATE_JOINABLE);
958  if (status != 0) {
959  __kmp_fatal(KMP_MSG(CantSetMonitorState), KMP_ERR(status), __kmp_msg_null);
960  }
961 
962 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
963  status = pthread_attr_getstacksize(&thread_attr, &size);
964  KMP_CHECK_SYSFAIL("pthread_attr_getstacksize", status);
965 #else
966  size = __kmp_sys_min_stksize;
967 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
968 #endif /* KMP_THREAD_ATTR */
969 
970  if (__kmp_monitor_stksize == 0) {
971  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
972  }
973  if (__kmp_monitor_stksize < __kmp_sys_min_stksize) {
974  __kmp_monitor_stksize = __kmp_sys_min_stksize;
975  }
976 
977  KA_TRACE(10, ("__kmp_create_monitor: default stacksize = %lu bytes,"
978  "requested stacksize = %lu bytes\n",
979  size, __kmp_monitor_stksize));
980 
981 retry:
982 
983 /* Set stack size for this thread now. */
984 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
985  KA_TRACE(10, ("__kmp_create_monitor: setting stacksize = %lu bytes,",
986  __kmp_monitor_stksize));
987  status = pthread_attr_setstacksize(&thread_attr, __kmp_monitor_stksize);
988  if (status != 0) {
989  if (auto_adj_size) {
990  __kmp_monitor_stksize *= 2;
991  goto retry;
992  }
993  kmp_msg_t err_code = KMP_ERR(status);
994  __kmp_msg(kmp_ms_warning, // should this be fatal? BB
995  KMP_MSG(CantSetMonitorStackSize, (long int)__kmp_monitor_stksize),
996  err_code, KMP_HNT(ChangeMonitorStackSize), __kmp_msg_null);
997  if (__kmp_generate_warnings == kmp_warnings_off) {
998  __kmp_str_free(&err_code.str);
999  }
1000  }
1001 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1002 
1003  status =
1004  pthread_create(&handle, &thread_attr, __kmp_launch_monitor, (void *)th);
1005 
1006  if (status != 0) {
1007 #ifdef _POSIX_THREAD_ATTR_STACKSIZE
1008  if (status == EINVAL) {
1009  if (auto_adj_size && (__kmp_monitor_stksize < (size_t)0x40000000)) {
1010  __kmp_monitor_stksize *= 2;
1011  goto retry;
1012  }
1013  __kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize),
1014  KMP_ERR(status), KMP_HNT(IncreaseMonitorStackSize),
1015  __kmp_msg_null);
1016  }
1017  if (status == ENOMEM) {
1018  __kmp_fatal(KMP_MSG(CantSetMonitorStackSize, __kmp_monitor_stksize),
1019  KMP_ERR(status), KMP_HNT(DecreaseMonitorStackSize),
1020  __kmp_msg_null);
1021  }
1022 #endif /* _POSIX_THREAD_ATTR_STACKSIZE */
1023  if (status == EAGAIN) {
1024  __kmp_fatal(KMP_MSG(NoResourcesForMonitorThread), KMP_ERR(status),
1025  KMP_HNT(DecreaseNumberOfThreadsInUse), __kmp_msg_null);
1026  }
1027  KMP_SYSFAIL("pthread_create", status);
1028  }
1029 
1030  th->th.th_info.ds.ds_thread = handle;
1031 
1032 #if KMP_REAL_TIME_FIX
1033  // Wait for the monitor thread is really started and set its *priority*.
1034  KMP_DEBUG_ASSERT(sizeof(kmp_uint32) ==
1035  sizeof(__kmp_global.g.g_time.dt.t_value));
1036  __kmp_wait_4((kmp_uint32 volatile *)&__kmp_global.g.g_time.dt.t_value, -1,
1037  &__kmp_neq_4, NULL);
1038 #endif // KMP_REAL_TIME_FIX
1039 
1040 #ifdef KMP_THREAD_ATTR
1041  status = pthread_attr_destroy(&thread_attr);
1042  if (status != 0) {
1043  kmp_msg_t err_code = KMP_ERR(status);
1044  __kmp_msg(kmp_ms_warning, KMP_MSG(CantDestroyThreadAttrs), err_code,
1045  __kmp_msg_null);
1046  if (__kmp_generate_warnings == kmp_warnings_off) {
1047  __kmp_str_free(&err_code.str);
1048  }
1049  }
1050 #endif
1051 
1052  KMP_MB(); /* Flush all pending memory write invalidates. */
1053 
1054  KA_TRACE(10, ("__kmp_create_monitor: monitor created %#.8lx\n",
1055  th->th.th_info.ds.ds_thread));
1056 
1057 } // __kmp_create_monitor
1058 #endif // KMP_USE_MONITOR
1059 
1060 void __kmp_exit_thread(int exit_status) {
1061 #if KMP_OS_WASI
1062 // TODO: the wasm32-wasi-threads target does not yet support pthread_exit.
1063 #else
1064  pthread_exit((void *)(intptr_t)exit_status);
1065 #endif
1066 } // __kmp_exit_thread
1067 
1068 #if KMP_USE_MONITOR
1069 void __kmp_resume_monitor();
1070 
1071 extern "C" void __kmp_reap_monitor(kmp_info_t *th) {
1072  int status;
1073  void *exit_val;
1074 
1075  KA_TRACE(10, ("__kmp_reap_monitor: try to reap monitor thread with handle"
1076  " %#.8lx\n",
1077  th->th.th_info.ds.ds_thread));
1078 
1079  // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
1080  // If both tid and gtid are 0, it means the monitor did not ever start.
1081  // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
1082  KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid);
1083  if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) {
1084  KA_TRACE(10, ("__kmp_reap_monitor: monitor did not start, returning\n"));
1085  return;
1086  }
1087 
1088  KMP_MB(); /* Flush all pending memory write invalidates. */
1089 
1090  /* First, check to see whether the monitor thread exists to wake it up. This
1091  is to avoid performance problem when the monitor sleeps during
1092  blocktime-size interval */
1093 
1094  status = pthread_kill(th->th.th_info.ds.ds_thread, 0);
1095  if (status != ESRCH) {
1096  __kmp_resume_monitor(); // Wake up the monitor thread
1097  }
1098  KA_TRACE(10, ("__kmp_reap_monitor: try to join with monitor\n"));
1099  status = pthread_join(th->th.th_info.ds.ds_thread, &exit_val);
1100  if (exit_val != th) {
1101  __kmp_fatal(KMP_MSG(ReapMonitorError), KMP_ERR(status), __kmp_msg_null);
1102  }
1103 
1104  th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1105  th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1106 
1107  KA_TRACE(10, ("__kmp_reap_monitor: done reaping monitor thread with handle"
1108  " %#.8lx\n",
1109  th->th.th_info.ds.ds_thread));
1110 
1111  KMP_MB(); /* Flush all pending memory write invalidates. */
1112 }
1113 #else
1114 // Empty symbol to export (see exports_so.txt) when
1115 // monitor thread feature is disabled
1116 extern "C" void __kmp_reap_monitor(kmp_info_t *th) { (void)th; }
1117 #endif // KMP_USE_MONITOR
1118 
1119 void __kmp_reap_worker(kmp_info_t *th) {
1120  int status;
1121  void *exit_val;
1122 
1123  KMP_MB(); /* Flush all pending memory write invalidates. */
1124 
1125  KA_TRACE(
1126  10, ("__kmp_reap_worker: try to reap T#%d\n", th->th.th_info.ds.ds_gtid));
1127 
1128  status = pthread_join(th->th.th_info.ds.ds_thread, &exit_val);
1129 #ifdef KMP_DEBUG
1130  /* Don't expose these to the user until we understand when they trigger */
1131  if (status != 0) {
1132  __kmp_fatal(KMP_MSG(ReapWorkerError), KMP_ERR(status), __kmp_msg_null);
1133  }
1134  if (exit_val != th) {
1135  KA_TRACE(10, ("__kmp_reap_worker: worker T#%d did not reap properly, "
1136  "exit_val = %p\n",
1137  th->th.th_info.ds.ds_gtid, exit_val));
1138  }
1139 #else
1140  (void)status; // unused variable
1141 #endif /* KMP_DEBUG */
1142 
1143  KA_TRACE(10, ("__kmp_reap_worker: done reaping T#%d\n",
1144  th->th.th_info.ds.ds_gtid));
1145 
1146  KMP_MB(); /* Flush all pending memory write invalidates. */
1147 }
1148 
1149 #if KMP_HANDLE_SIGNALS
1150 
1151 static void __kmp_null_handler(int signo) {
1152  // Do nothing, for doing SIG_IGN-type actions.
1153 } // __kmp_null_handler
1154 
1155 static void __kmp_team_handler(int signo) {
1156  if (__kmp_global.g.g_abort == 0) {
1157 /* Stage 1 signal handler, let's shut down all of the threads */
1158 #ifdef KMP_DEBUG
1159  __kmp_debug_printf("__kmp_team_handler: caught signal = %d\n", signo);
1160 #endif
1161  switch (signo) {
1162  case SIGHUP:
1163  case SIGINT:
1164  case SIGQUIT:
1165  case SIGILL:
1166  case SIGABRT:
1167  case SIGFPE:
1168  case SIGBUS:
1169  case SIGSEGV:
1170 #ifdef SIGSYS
1171  case SIGSYS:
1172 #endif
1173  case SIGTERM:
1174  if (__kmp_debug_buf) {
1175  __kmp_dump_debug_buffer();
1176  }
1177  __kmp_unregister_library(); // cleanup shared memory
1178  KMP_MB(); // Flush all pending memory write invalidates.
1179  TCW_4(__kmp_global.g.g_abort, signo);
1180  KMP_MB(); // Flush all pending memory write invalidates.
1181  TCW_4(__kmp_global.g.g_done, TRUE);
1182  KMP_MB(); // Flush all pending memory write invalidates.
1183  break;
1184  default:
1185 #ifdef KMP_DEBUG
1186  __kmp_debug_printf("__kmp_team_handler: unknown signal type");
1187 #endif
1188  break;
1189  }
1190  }
1191 } // __kmp_team_handler
1192 
1193 static void __kmp_sigaction(int signum, const struct sigaction *act,
1194  struct sigaction *oldact) {
1195  int rc = sigaction(signum, act, oldact);
1196  KMP_CHECK_SYSFAIL_ERRNO("sigaction", rc);
1197 }
1198 
1199 static void __kmp_install_one_handler(int sig, sig_func_t handler_func,
1200  int parallel_init) {
1201  KMP_MB(); // Flush all pending memory write invalidates.
1202  KB_TRACE(60,
1203  ("__kmp_install_one_handler( %d, ..., %d )\n", sig, parallel_init));
1204  if (parallel_init) {
1205  struct sigaction new_action;
1206  struct sigaction old_action;
1207  new_action.sa_handler = handler_func;
1208  new_action.sa_flags = 0;
1209  sigfillset(&new_action.sa_mask);
1210  __kmp_sigaction(sig, &new_action, &old_action);
1211  if (old_action.sa_handler == __kmp_sighldrs[sig].sa_handler) {
1212  sigaddset(&__kmp_sigset, sig);
1213  } else {
1214  // Restore/keep user's handler if one previously installed.
1215  __kmp_sigaction(sig, &old_action, NULL);
1216  }
1217  } else {
1218  // Save initial/system signal handlers to see if user handlers installed.
1219  __kmp_sigaction(sig, NULL, &__kmp_sighldrs[sig]);
1220  }
1221  KMP_MB(); // Flush all pending memory write invalidates.
1222 } // __kmp_install_one_handler
1223 
1224 static void __kmp_remove_one_handler(int sig) {
1225  KB_TRACE(60, ("__kmp_remove_one_handler( %d )\n", sig));
1226  if (sigismember(&__kmp_sigset, sig)) {
1227  struct sigaction old;
1228  KMP_MB(); // Flush all pending memory write invalidates.
1229  __kmp_sigaction(sig, &__kmp_sighldrs[sig], &old);
1230  if ((old.sa_handler != __kmp_team_handler) &&
1231  (old.sa_handler != __kmp_null_handler)) {
1232  // Restore the users signal handler.
1233  KB_TRACE(10, ("__kmp_remove_one_handler: oops, not our handler, "
1234  "restoring: sig=%d\n",
1235  sig));
1236  __kmp_sigaction(sig, &old, NULL);
1237  }
1238  sigdelset(&__kmp_sigset, sig);
1239  KMP_MB(); // Flush all pending memory write invalidates.
1240  }
1241 } // __kmp_remove_one_handler
1242 
1243 void __kmp_install_signals(int parallel_init) {
1244  KB_TRACE(10, ("__kmp_install_signals( %d )\n", parallel_init));
1245  if (__kmp_handle_signals || !parallel_init) {
1246  // If ! parallel_init, we do not install handlers, just save original
1247  // handlers. Let us do it even __handle_signals is 0.
1248  sigemptyset(&__kmp_sigset);
1249  __kmp_install_one_handler(SIGHUP, __kmp_team_handler, parallel_init);
1250  __kmp_install_one_handler(SIGINT, __kmp_team_handler, parallel_init);
1251  __kmp_install_one_handler(SIGQUIT, __kmp_team_handler, parallel_init);
1252  __kmp_install_one_handler(SIGILL, __kmp_team_handler, parallel_init);
1253  __kmp_install_one_handler(SIGABRT, __kmp_team_handler, parallel_init);
1254  __kmp_install_one_handler(SIGFPE, __kmp_team_handler, parallel_init);
1255  __kmp_install_one_handler(SIGBUS, __kmp_team_handler, parallel_init);
1256  __kmp_install_one_handler(SIGSEGV, __kmp_team_handler, parallel_init);
1257 #ifdef SIGSYS
1258  __kmp_install_one_handler(SIGSYS, __kmp_team_handler, parallel_init);
1259 #endif // SIGSYS
1260  __kmp_install_one_handler(SIGTERM, __kmp_team_handler, parallel_init);
1261 #ifdef SIGPIPE
1262  __kmp_install_one_handler(SIGPIPE, __kmp_team_handler, parallel_init);
1263 #endif // SIGPIPE
1264  }
1265 } // __kmp_install_signals
1266 
1267 void __kmp_remove_signals(void) {
1268  int sig;
1269  KB_TRACE(10, ("__kmp_remove_signals()\n"));
1270  for (sig = 1; sig < NSIG; ++sig) {
1271  __kmp_remove_one_handler(sig);
1272  }
1273 } // __kmp_remove_signals
1274 
1275 #endif // KMP_HANDLE_SIGNALS
1276 
1277 void __kmp_enable(int new_state) {
1278 #ifdef KMP_CANCEL_THREADS
1279  int status, old_state;
1280  status = pthread_setcancelstate(new_state, &old_state);
1281  KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
1282  KMP_DEBUG_ASSERT(old_state == PTHREAD_CANCEL_DISABLE);
1283 #endif
1284 }
1285 
1286 void __kmp_disable(int *old_state) {
1287 #ifdef KMP_CANCEL_THREADS
1288  int status;
1289  status = pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, old_state);
1290  KMP_CHECK_SYSFAIL("pthread_setcancelstate", status);
1291 #endif
1292 }
1293 
1294 static void __kmp_atfork_prepare(void) {
1295  __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
1296  __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
1297 }
1298 
1299 static void __kmp_atfork_parent(void) {
1300  __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1301  __kmp_release_bootstrap_lock(&__kmp_initz_lock);
1302 }
1303 
1304 /* Reset the library so execution in the child starts "all over again" with
1305  clean data structures in initial states. Don't worry about freeing memory
1306  allocated by parent, just abandon it to be safe. */
1307 static void __kmp_atfork_child(void) {
1308  __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1309  __kmp_release_bootstrap_lock(&__kmp_initz_lock);
1310  /* TODO make sure this is done right for nested/sibling */
1311  // ATT: Memory leaks are here? TODO: Check it and fix.
1312  /* KMP_ASSERT( 0 ); */
1313 
1314  ++__kmp_fork_count;
1315 
1316 #if KMP_AFFINITY_SUPPORTED
1317 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_DRAGONFLY || \
1318  KMP_OS_AIX
1319  // reset the affinity in the child to the initial thread
1320  // affinity in the parent
1321  kmp_set_thread_affinity_mask_initial();
1322 #endif
1323  // Set default not to bind threads tightly in the child (we're expecting
1324  // over-subscription after the fork and this can improve things for
1325  // scripting languages that use OpenMP inside process-parallel code).
1326  if (__kmp_nested_proc_bind.bind_types != NULL) {
1327  __kmp_nested_proc_bind.bind_types[0] = proc_bind_false;
1328  }
1329  for (kmp_affinity_t *affinity : __kmp_affinities)
1330  *affinity = KMP_AFFINITY_INIT(affinity->env_var);
1331  __kmp_affin_fullMask = nullptr;
1332  __kmp_affin_origMask = nullptr;
1333  __kmp_topology = nullptr;
1334 #endif // KMP_AFFINITY_SUPPORTED
1335 
1336 #if KMP_USE_MONITOR
1337  __kmp_init_monitor = 0;
1338 #endif
1339  __kmp_init_parallel = FALSE;
1340  __kmp_init_middle = FALSE;
1341  __kmp_init_serial = FALSE;
1342  TCW_4(__kmp_init_gtid, FALSE);
1343  __kmp_init_common = FALSE;
1344 
1345  TCW_4(__kmp_init_user_locks, FALSE);
1346 #if !KMP_USE_DYNAMIC_LOCK
1347  __kmp_user_lock_table.used = 1;
1348  __kmp_user_lock_table.allocated = 0;
1349  __kmp_user_lock_table.table = NULL;
1350  __kmp_lock_blocks = NULL;
1351 #endif
1352 
1353  __kmp_all_nth = 0;
1354  TCW_4(__kmp_nth, 0);
1355 
1356  __kmp_thread_pool = NULL;
1357  __kmp_thread_pool_insert_pt = NULL;
1358  __kmp_team_pool = NULL;
1359 
1360  /* Must actually zero all the *cache arguments passed to __kmpc_threadprivate
1361  here so threadprivate doesn't use stale data */
1362  KA_TRACE(10, ("__kmp_atfork_child: checking cache address list %p\n",
1363  __kmp_threadpriv_cache_list));
1364 
1365  while (__kmp_threadpriv_cache_list != NULL) {
1366 
1367  if (*__kmp_threadpriv_cache_list->addr != NULL) {
1368  KC_TRACE(50, ("__kmp_atfork_child: zeroing cache at address %p\n",
1369  &(*__kmp_threadpriv_cache_list->addr)));
1370 
1371  *__kmp_threadpriv_cache_list->addr = NULL;
1372  }
1373  __kmp_threadpriv_cache_list = __kmp_threadpriv_cache_list->next;
1374  }
1375 
1376  __kmp_init_runtime = FALSE;
1377 
1378  /* reset statically initialized locks */
1379  __kmp_init_bootstrap_lock(&__kmp_initz_lock);
1380  __kmp_init_bootstrap_lock(&__kmp_stdio_lock);
1381  __kmp_init_bootstrap_lock(&__kmp_console_lock);
1382  __kmp_init_bootstrap_lock(&__kmp_task_team_lock);
1383 
1384 #if USE_ITT_BUILD
1385  __kmp_itt_reset(); // reset ITT's global state
1386 #endif /* USE_ITT_BUILD */
1387 
1388  {
1389  // Child process often get terminated without any use of OpenMP. That might
1390  // cause mapped shared memory file to be left unattended. Thus we postpone
1391  // library registration till middle initialization in the child process.
1392  __kmp_need_register_serial = FALSE;
1393  __kmp_serial_initialize();
1394  }
1395 
1396  /* This is necessary to make sure no stale data is left around */
1397  /* AC: customers complain that we use unsafe routines in the atfork
1398  handler. Mathworks: dlsym() is unsafe. We call dlsym and dlopen
1399  in dynamic_link when check the presence of shared tbbmalloc library.
1400  Suggestion is to make the library initialization lazier, similar
1401  to what done for __kmpc_begin(). */
1402  // TODO: synchronize all static initializations with regular library
1403  // startup; look at kmp_global.cpp and etc.
1404  //__kmp_internal_begin ();
1405 }
1406 
1407 void __kmp_register_atfork(void) {
1408  if (__kmp_need_register_atfork) {
1409 #if !KMP_OS_WASI
1410  int status = pthread_atfork(__kmp_atfork_prepare, __kmp_atfork_parent,
1411  __kmp_atfork_child);
1412  KMP_CHECK_SYSFAIL("pthread_atfork", status);
1413 #endif
1414  __kmp_need_register_atfork = FALSE;
1415  }
1416 }
1417 
1418 void __kmp_suspend_initialize(void) {
1419  int status;
1420  status = pthread_mutexattr_init(&__kmp_suspend_mutex_attr);
1421  KMP_CHECK_SYSFAIL("pthread_mutexattr_init", status);
1422  status = pthread_condattr_init(&__kmp_suspend_cond_attr);
1423  KMP_CHECK_SYSFAIL("pthread_condattr_init", status);
1424 }
1425 
1426 void __kmp_suspend_initialize_thread(kmp_info_t *th) {
1427  int old_value = KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init_count);
1428  int new_value = __kmp_fork_count + 1;
1429  // Return if already initialized
1430  if (old_value == new_value)
1431  return;
1432  // Wait, then return if being initialized
1433  if (old_value == -1 || !__kmp_atomic_compare_store(
1434  &th->th.th_suspend_init_count, old_value, -1)) {
1435  while (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) != new_value) {
1436  KMP_CPU_PAUSE();
1437  }
1438  } else {
1439  // Claim to be the initializer and do initializations
1440  int status;
1441  status = pthread_cond_init(&th->th.th_suspend_cv.c_cond,
1442  &__kmp_suspend_cond_attr);
1443  KMP_CHECK_SYSFAIL("pthread_cond_init", status);
1444  status = pthread_mutex_init(&th->th.th_suspend_mx.m_mutex,
1445  &__kmp_suspend_mutex_attr);
1446  KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
1447  KMP_ATOMIC_ST_REL(&th->th.th_suspend_init_count, new_value);
1448  }
1449 }
1450 
1451 void __kmp_suspend_uninitialize_thread(kmp_info_t *th) {
1452  if (KMP_ATOMIC_LD_ACQ(&th->th.th_suspend_init_count) > __kmp_fork_count) {
1453  /* this means we have initialize the suspension pthread objects for this
1454  thread in this instance of the process */
1455  int status;
1456 
1457  status = pthread_cond_destroy(&th->th.th_suspend_cv.c_cond);
1458  if (status != 0 && status != EBUSY) {
1459  KMP_SYSFAIL("pthread_cond_destroy", status);
1460  }
1461  status = pthread_mutex_destroy(&th->th.th_suspend_mx.m_mutex);
1462  if (status != 0 && status != EBUSY) {
1463  KMP_SYSFAIL("pthread_mutex_destroy", status);
1464  }
1465  --th->th.th_suspend_init_count;
1466  KMP_DEBUG_ASSERT(KMP_ATOMIC_LD_RLX(&th->th.th_suspend_init_count) ==
1467  __kmp_fork_count);
1468  }
1469 }
1470 
1471 // return true if lock obtained, false otherwise
1472 int __kmp_try_suspend_mx(kmp_info_t *th) {
1473  return (pthread_mutex_trylock(&th->th.th_suspend_mx.m_mutex) == 0);
1474 }
1475 
1476 void __kmp_lock_suspend_mx(kmp_info_t *th) {
1477  int status = pthread_mutex_lock(&th->th.th_suspend_mx.m_mutex);
1478  KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
1479 }
1480 
1481 void __kmp_unlock_suspend_mx(kmp_info_t *th) {
1482  int status = pthread_mutex_unlock(&th->th.th_suspend_mx.m_mutex);
1483  KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
1484 }
1485 
1486 /* This routine puts the calling thread to sleep after setting the
1487  sleep bit for the indicated flag variable to true. */
1488 template <class C>
1489 static inline void __kmp_suspend_template(int th_gtid, C *flag) {
1490  KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_suspend);
1491  kmp_info_t *th = __kmp_threads[th_gtid];
1492  int status;
1493  typename C::flag_t old_spin;
1494 
1495  KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag = %p\n", th_gtid,
1496  flag->get()));
1497 
1498  __kmp_suspend_initialize_thread(th);
1499 
1500  __kmp_lock_suspend_mx(th);
1501 
1502  KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for spin(%p)\n",
1503  th_gtid, flag->get()));
1504 
1505  /* TODO: shouldn't this use release semantics to ensure that
1506  __kmp_suspend_initialize_thread gets called first? */
1507  old_spin = flag->set_sleeping();
1508  TCW_PTR(th->th.th_sleep_loc, (void *)flag);
1509  th->th.th_sleep_loc_type = flag->get_type();
1510  if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
1511  __kmp_pause_status != kmp_soft_paused) {
1512  flag->unset_sleeping();
1513  TCW_PTR(th->th.th_sleep_loc, NULL);
1514  th->th.th_sleep_loc_type = flag_unset;
1515  __kmp_unlock_suspend_mx(th);
1516  return;
1517  }
1518  KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for spin(%p)==%x,"
1519  " was %x\n",
1520  th_gtid, flag->get(), flag->load(), old_spin));
1521 
1522  if (flag->done_check_val(old_spin) || flag->done_check()) {
1523  flag->unset_sleeping();
1524  TCW_PTR(th->th.th_sleep_loc, NULL);
1525  th->th.th_sleep_loc_type = flag_unset;
1526  KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit "
1527  "for spin(%p)\n",
1528  th_gtid, flag->get()));
1529  } else {
1530  /* Encapsulate in a loop as the documentation states that this may
1531  "with low probability" return when the condition variable has
1532  not been signaled or broadcast */
1533  int deactivated = FALSE;
1534 
1535  while (flag->is_sleeping()) {
1536 #ifdef DEBUG_SUSPEND
1537  char buffer[128];
1538  __kmp_suspend_count++;
1539  __kmp_print_cond(buffer, &th->th.th_suspend_cv);
1540  __kmp_printf("__kmp_suspend_template: suspending T#%d: %s\n", th_gtid,
1541  buffer);
1542 #endif
1543  // Mark the thread as no longer active (only in the first iteration of the
1544  // loop).
1545  if (!deactivated) {
1546  th->th.th_active = FALSE;
1547  if (th->th.th_active_in_pool) {
1548  th->th.th_active_in_pool = FALSE;
1549  KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
1550  KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
1551  }
1552  deactivated = TRUE;
1553  }
1554 
1555  KMP_DEBUG_ASSERT(th->th.th_sleep_loc);
1556  KMP_DEBUG_ASSERT(flag->get_type() == th->th.th_sleep_loc_type);
1557 
1558 #if USE_SUSPEND_TIMEOUT
1559  struct timespec now;
1560  struct timeval tval;
1561  int msecs;
1562 
1563  status = gettimeofday(&tval, NULL);
1564  KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
1565  TIMEVAL_TO_TIMESPEC(&tval, &now);
1566 
1567  msecs = (4 * __kmp_dflt_blocktime) + 200;
1568  now.tv_sec += msecs / 1000;
1569  now.tv_nsec += (msecs % 1000) * 1000;
1570 
1571  KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform "
1572  "pthread_cond_timedwait\n",
1573  th_gtid));
1574  status = pthread_cond_timedwait(&th->th.th_suspend_cv.c_cond,
1575  &th->th.th_suspend_mx.m_mutex, &now);
1576 #else
1577  KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform"
1578  " pthread_cond_wait\n",
1579  th_gtid));
1580  status = pthread_cond_wait(&th->th.th_suspend_cv.c_cond,
1581  &th->th.th_suspend_mx.m_mutex);
1582 #endif // USE_SUSPEND_TIMEOUT
1583 
1584  if ((status != 0) && (status != EINTR) && (status != ETIMEDOUT)) {
1585  KMP_SYSFAIL("pthread_cond_wait", status);
1586  }
1587 
1588  KMP_DEBUG_ASSERT(flag->get_type() == flag->get_ptr_type());
1589 
1590  if (!flag->is_sleeping() &&
1591  ((status == EINTR) || (status == ETIMEDOUT))) {
1592  // if interrupt or timeout, and thread is no longer sleeping, we need to
1593  // make sure sleep_loc gets reset; however, this shouldn't be needed if
1594  // we woke up with resume
1595  flag->unset_sleeping();
1596  TCW_PTR(th->th.th_sleep_loc, NULL);
1597  th->th.th_sleep_loc_type = flag_unset;
1598  }
1599 #ifdef KMP_DEBUG
1600  if (status == ETIMEDOUT) {
1601  if (flag->is_sleeping()) {
1602  KF_TRACE(100,
1603  ("__kmp_suspend_template: T#%d timeout wakeup\n", th_gtid));
1604  } else {
1605  KF_TRACE(2, ("__kmp_suspend_template: T#%d timeout wakeup, sleep bit "
1606  "not set!\n",
1607  th_gtid));
1608  TCW_PTR(th->th.th_sleep_loc, NULL);
1609  th->th.th_sleep_loc_type = flag_unset;
1610  }
1611  } else if (flag->is_sleeping()) {
1612  KF_TRACE(100,
1613  ("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid));
1614  }
1615 #endif
1616  } // while
1617 
1618  // Mark the thread as active again (if it was previous marked as inactive)
1619  if (deactivated) {
1620  th->th.th_active = TRUE;
1621  if (TCR_4(th->th.th_in_pool)) {
1622  KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
1623  th->th.th_active_in_pool = TRUE;
1624  }
1625  }
1626  }
1627  // We may have had the loop variable set before entering the loop body;
1628  // so we need to reset sleep_loc.
1629  TCW_PTR(th->th.th_sleep_loc, NULL);
1630  th->th.th_sleep_loc_type = flag_unset;
1631 
1632  KMP_DEBUG_ASSERT(!flag->is_sleeping());
1633  KMP_DEBUG_ASSERT(!th->th.th_sleep_loc);
1634 #ifdef DEBUG_SUSPEND
1635  {
1636  char buffer[128];
1637  __kmp_print_cond(buffer, &th->th.th_suspend_cv);
1638  __kmp_printf("__kmp_suspend_template: T#%d has awakened: %s\n", th_gtid,
1639  buffer);
1640  }
1641 #endif
1642 
1643  __kmp_unlock_suspend_mx(th);
1644  KF_TRACE(30, ("__kmp_suspend_template: T#%d exit\n", th_gtid));
1645 }
1646 
1647 template <bool C, bool S>
1648 void __kmp_suspend_32(int th_gtid, kmp_flag_32<C, S> *flag) {
1649  __kmp_suspend_template(th_gtid, flag);
1650 }
1651 template <bool C, bool S>
1652 void __kmp_suspend_64(int th_gtid, kmp_flag_64<C, S> *flag) {
1653  __kmp_suspend_template(th_gtid, flag);
1654 }
1655 template <bool C, bool S>
1656 void __kmp_atomic_suspend_64(int th_gtid, kmp_atomic_flag_64<C, S> *flag) {
1657  __kmp_suspend_template(th_gtid, flag);
1658 }
1659 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
1660  __kmp_suspend_template(th_gtid, flag);
1661 }
1662 
1663 template void __kmp_suspend_32<false, false>(int, kmp_flag_32<false, false> *);
1664 template void __kmp_suspend_64<false, true>(int, kmp_flag_64<false, true> *);
1665 template void __kmp_suspend_64<true, false>(int, kmp_flag_64<true, false> *);
1666 template void
1667 __kmp_atomic_suspend_64<false, true>(int, kmp_atomic_flag_64<false, true> *);
1668 template void
1669 __kmp_atomic_suspend_64<true, false>(int, kmp_atomic_flag_64<true, false> *);
1670 
1671 /* This routine signals the thread specified by target_gtid to wake up
1672  after setting the sleep bit indicated by the flag argument to FALSE.
1673  The target thread must already have called __kmp_suspend_template() */
1674 template <class C>
1675 static inline void __kmp_resume_template(int target_gtid, C *flag) {
1676  KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
1677  kmp_info_t *th = __kmp_threads[target_gtid];
1678  int status;
1679 
1680 #ifdef KMP_DEBUG
1681  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1682 #endif
1683 
1684  KF_TRACE(30, ("__kmp_resume_template: T#%d wants to wakeup T#%d enter\n",
1685  gtid, target_gtid));
1686  KMP_DEBUG_ASSERT(gtid != target_gtid);
1687 
1688  __kmp_suspend_initialize_thread(th);
1689 
1690  __kmp_lock_suspend_mx(th);
1691 
1692  if (!flag || flag != th->th.th_sleep_loc) {
1693  // coming from __kmp_null_resume_wrapper, or thread is now sleeping on a
1694  // different location; wake up at new location
1695  flag = (C *)CCAST(void *, th->th.th_sleep_loc);
1696  }
1697 
1698  // First, check if the flag is null or its type has changed. If so, someone
1699  // else woke it up.
1700  if (!flag) { // Thread doesn't appear to be sleeping on anything
1701  KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
1702  "awake: flag(%p)\n",
1703  gtid, target_gtid, (void *)NULL));
1704  __kmp_unlock_suspend_mx(th);
1705  return;
1706  } else if (flag->get_type() != th->th.th_sleep_loc_type) {
1707  // Flag type does not appear to match this function template; possibly the
1708  // thread is sleeping on something else. Try null resume again.
1709  KF_TRACE(
1710  5,
1711  ("__kmp_resume_template: T#%d retrying, thread T#%d Mismatch flag(%p), "
1712  "spin(%p) type=%d ptr_type=%d\n",
1713  gtid, target_gtid, flag, flag->get(), flag->get_type(),
1714  th->th.th_sleep_loc_type));
1715  __kmp_unlock_suspend_mx(th);
1716  __kmp_null_resume_wrapper(th);
1717  return;
1718  } else { // if multiple threads are sleeping, flag should be internally
1719  // referring to a specific thread here
1720  if (!flag->is_sleeping()) {
1721  KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
1722  "awake: flag(%p): %u\n",
1723  gtid, target_gtid, flag->get(), (unsigned int)flag->load()));
1724  __kmp_unlock_suspend_mx(th);
1725  return;
1726  }
1727  }
1728  KMP_DEBUG_ASSERT(flag);
1729  flag->unset_sleeping();
1730  TCW_PTR(th->th.th_sleep_loc, NULL);
1731  th->th.th_sleep_loc_type = flag_unset;
1732 
1733  KF_TRACE(5, ("__kmp_resume_template: T#%d about to wakeup T#%d, reset "
1734  "sleep bit for flag's loc(%p): %u\n",
1735  gtid, target_gtid, flag->get(), (unsigned int)flag->load()));
1736 
1737 #ifdef DEBUG_SUSPEND
1738  {
1739  char buffer[128];
1740  __kmp_print_cond(buffer, &th->th.th_suspend_cv);
1741  __kmp_printf("__kmp_resume_template: T#%d resuming T#%d: %s\n", gtid,
1742  target_gtid, buffer);
1743  }
1744 #endif
1745  status = pthread_cond_signal(&th->th.th_suspend_cv.c_cond);
1746  KMP_CHECK_SYSFAIL("pthread_cond_signal", status);
1747  __kmp_unlock_suspend_mx(th);
1748  KF_TRACE(30, ("__kmp_resume_template: T#%d exiting after signaling wake up"
1749  " for T#%d\n",
1750  gtid, target_gtid));
1751 }
1752 
1753 template <bool C, bool S>
1754 void __kmp_resume_32(int target_gtid, kmp_flag_32<C, S> *flag) {
1755  __kmp_resume_template(target_gtid, flag);
1756 }
1757 template <bool C, bool S>
1758 void __kmp_resume_64(int target_gtid, kmp_flag_64<C, S> *flag) {
1759  __kmp_resume_template(target_gtid, flag);
1760 }
1761 template <bool C, bool S>
1762 void __kmp_atomic_resume_64(int target_gtid, kmp_atomic_flag_64<C, S> *flag) {
1763  __kmp_resume_template(target_gtid, flag);
1764 }
1765 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
1766  __kmp_resume_template(target_gtid, flag);
1767 }
1768 
1769 template void __kmp_resume_32<false, true>(int, kmp_flag_32<false, true> *);
1770 template void __kmp_resume_32<false, false>(int, kmp_flag_32<false, false> *);
1771 template void __kmp_resume_64<false, true>(int, kmp_flag_64<false, true> *);
1772 template void
1773 __kmp_atomic_resume_64<false, true>(int, kmp_atomic_flag_64<false, true> *);
1774 
1775 #if KMP_USE_MONITOR
1776 void __kmp_resume_monitor() {
1777  KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(USER_resume);
1778  int status;
1779 #ifdef KMP_DEBUG
1780  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
1781  KF_TRACE(30, ("__kmp_resume_monitor: T#%d wants to wakeup T#%d enter\n", gtid,
1782  KMP_GTID_MONITOR));
1783  KMP_DEBUG_ASSERT(gtid != KMP_GTID_MONITOR);
1784 #endif
1785  status = pthread_mutex_lock(&__kmp_wait_mx.m_mutex);
1786  KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
1787 #ifdef DEBUG_SUSPEND
1788  {
1789  char buffer[128];
1790  __kmp_print_cond(buffer, &__kmp_wait_cv.c_cond);
1791  __kmp_printf("__kmp_resume_monitor: T#%d resuming T#%d: %s\n", gtid,
1792  KMP_GTID_MONITOR, buffer);
1793  }
1794 #endif
1795  status = pthread_cond_signal(&__kmp_wait_cv.c_cond);
1796  KMP_CHECK_SYSFAIL("pthread_cond_signal", status);
1797  status = pthread_mutex_unlock(&__kmp_wait_mx.m_mutex);
1798  KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
1799  KF_TRACE(30, ("__kmp_resume_monitor: T#%d exiting after signaling wake up"
1800  " for T#%d\n",
1801  gtid, KMP_GTID_MONITOR));
1802 }
1803 #endif // KMP_USE_MONITOR
1804 
1805 void __kmp_yield() { sched_yield(); }
1806 
1807 void __kmp_gtid_set_specific(int gtid) {
1808  if (__kmp_init_gtid) {
1809  int status;
1810  status = pthread_setspecific(__kmp_gtid_threadprivate_key,
1811  (void *)(intptr_t)(gtid + 1));
1812  KMP_CHECK_SYSFAIL("pthread_setspecific", status);
1813  } else {
1814  KA_TRACE(50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n"));
1815  }
1816 }
1817 
1818 int __kmp_gtid_get_specific() {
1819  int gtid;
1820  if (!__kmp_init_gtid) {
1821  KA_TRACE(50, ("__kmp_gtid_get_specific: runtime shutdown, returning "
1822  "KMP_GTID_SHUTDOWN\n"));
1823  return KMP_GTID_SHUTDOWN;
1824  }
1825  gtid = (int)(size_t)pthread_getspecific(__kmp_gtid_threadprivate_key);
1826  if (gtid == 0) {
1827  gtid = KMP_GTID_DNE;
1828  } else {
1829  gtid--;
1830  }
1831  KA_TRACE(50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
1832  __kmp_gtid_threadprivate_key, gtid));
1833  return gtid;
1834 }
1835 
1836 double __kmp_read_cpu_time(void) {
1837  /*clock_t t;*/
1838  struct tms buffer;
1839 
1840  /*t =*/times(&buffer);
1841 
1842  return (double)(buffer.tms_utime + buffer.tms_cutime) /
1843  (double)CLOCKS_PER_SEC;
1844 }
1845 
1846 int __kmp_read_system_info(struct kmp_sys_info *info) {
1847  int status;
1848  struct rusage r_usage;
1849 
1850  memset(info, 0, sizeof(*info));
1851 
1852  status = getrusage(RUSAGE_SELF, &r_usage);
1853  KMP_CHECK_SYSFAIL_ERRNO("getrusage", status);
1854 
1855 #if !KMP_OS_WASI
1856  // The maximum resident set size utilized (in kilobytes)
1857  info->maxrss = r_usage.ru_maxrss;
1858  // The number of page faults serviced without any I/O
1859  info->minflt = r_usage.ru_minflt;
1860  // The number of page faults serviced that required I/O
1861  info->majflt = r_usage.ru_majflt;
1862  // The number of times a process was "swapped" out of memory
1863  info->nswap = r_usage.ru_nswap;
1864  // The number of times the file system had to perform input
1865  info->inblock = r_usage.ru_inblock;
1866  // The number of times the file system had to perform output
1867  info->oublock = r_usage.ru_oublock;
1868  // The number of times a context switch was voluntarily
1869  info->nvcsw = r_usage.ru_nvcsw;
1870  // The number of times a context switch was forced
1871  info->nivcsw = r_usage.ru_nivcsw;
1872 #endif
1873 
1874  return (status != 0);
1875 }
1876 
1877 void __kmp_read_system_time(double *delta) {
1878  double t_ns;
1879  struct timeval tval;
1880  struct timespec stop;
1881  int status;
1882 
1883  status = gettimeofday(&tval, NULL);
1884  KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
1885  TIMEVAL_TO_TIMESPEC(&tval, &stop);
1886  t_ns = (double)(TS2NS(stop) - TS2NS(__kmp_sys_timer_data.start));
1887  *delta = (t_ns * 1e-9);
1888 }
1889 
1890 void __kmp_clear_system_time(void) {
1891  struct timeval tval;
1892  int status;
1893  status = gettimeofday(&tval, NULL);
1894  KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
1895  TIMEVAL_TO_TIMESPEC(&tval, &__kmp_sys_timer_data.start);
1896 }
1897 
1898 static int __kmp_get_xproc(void) {
1899 
1900  int r = 0;
1901 
1902 #if KMP_OS_LINUX
1903 
1904  __kmp_type_convert(sysconf(_SC_NPROCESSORS_CONF), &(r));
1905 
1906 #elif KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || KMP_OS_OPENBSD || \
1907  KMP_OS_HAIKU || KMP_OS_HURD || KMP_OS_SOLARIS || KMP_OS_WASI || KMP_OS_AIX
1908 
1909  __kmp_type_convert(sysconf(_SC_NPROCESSORS_ONLN), &(r));
1910 
1911 #elif KMP_OS_DARWIN
1912 
1913  size_t len = sizeof(r);
1914  sysctlbyname("hw.logicalcpu", &r, &len, NULL, 0);
1915 
1916 #else
1917 
1918 #error "Unknown or unsupported OS."
1919 
1920 #endif
1921 
1922  return r > 0 ? r : 2; /* guess value of 2 if OS told us 0 */
1923 
1924 } // __kmp_get_xproc
1925 
1926 int __kmp_read_from_file(char const *path, char const *format, ...) {
1927  int result;
1928  va_list args;
1929 
1930  va_start(args, format);
1931  FILE *f = fopen(path, "rb");
1932  if (f == NULL) {
1933  va_end(args);
1934  return 0;
1935  }
1936  result = vfscanf(f, format, args);
1937  fclose(f);
1938  va_end(args);
1939 
1940  return result;
1941 }
1942 
1943 void __kmp_runtime_initialize(void) {
1944  int status;
1945  pthread_mutexattr_t mutex_attr;
1946  pthread_condattr_t cond_attr;
1947 
1948  if (__kmp_init_runtime) {
1949  return;
1950  }
1951 
1952 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
1953  if (!__kmp_cpuinfo.initialized) {
1954  __kmp_query_cpuid(&__kmp_cpuinfo);
1955  }
1956 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
1957 
1958  __kmp_xproc = __kmp_get_xproc();
1959 
1960 #if !KMP_32_BIT_ARCH
1961  struct rlimit rlim;
1962  // read stack size of calling thread, save it as default for worker threads;
1963  // this should be done before reading environment variables
1964  status = getrlimit(RLIMIT_STACK, &rlim);
1965  if (status == 0) { // success?
1966  __kmp_stksize = rlim.rlim_cur;
1967  __kmp_check_stksize(&__kmp_stksize); // check value and adjust if needed
1968  }
1969 #endif /* KMP_32_BIT_ARCH */
1970 
1971  if (sysconf(_SC_THREADS)) {
1972 
1973  /* Query the maximum number of threads */
1974  __kmp_type_convert(sysconf(_SC_THREAD_THREADS_MAX), &(__kmp_sys_max_nth));
1975 #ifdef __ve__
1976  if (__kmp_sys_max_nth == -1) {
1977  // VE's pthread supports only up to 64 threads per a VE process.
1978  // So we use that KMP_MAX_NTH (predefined as 64) here.
1979  __kmp_sys_max_nth = KMP_MAX_NTH;
1980  }
1981 #else
1982  if (__kmp_sys_max_nth == -1) {
1983  /* Unlimited threads for NPTL */
1984  __kmp_sys_max_nth = INT_MAX;
1985  } else if (__kmp_sys_max_nth <= 1) {
1986  /* Can't tell, just use PTHREAD_THREADS_MAX */
1987  __kmp_sys_max_nth = KMP_MAX_NTH;
1988  }
1989 #endif
1990 
1991  /* Query the minimum stack size */
1992  __kmp_sys_min_stksize = sysconf(_SC_THREAD_STACK_MIN);
1993  if (__kmp_sys_min_stksize <= 1) {
1994  __kmp_sys_min_stksize = KMP_MIN_STKSIZE;
1995  }
1996  }
1997 
1998  /* Set up minimum number of threads to switch to TLS gtid */
1999  __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
2000 
2001  status = pthread_key_create(&__kmp_gtid_threadprivate_key,
2002  __kmp_internal_end_dest);
2003  KMP_CHECK_SYSFAIL("pthread_key_create", status);
2004  status = pthread_mutexattr_init(&mutex_attr);
2005  KMP_CHECK_SYSFAIL("pthread_mutexattr_init", status);
2006  status = pthread_mutex_init(&__kmp_wait_mx.m_mutex, &mutex_attr);
2007  KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2008  status = pthread_mutexattr_destroy(&mutex_attr);
2009  KMP_CHECK_SYSFAIL("pthread_mutexattr_destroy", status);
2010  status = pthread_condattr_init(&cond_attr);
2011  KMP_CHECK_SYSFAIL("pthread_condattr_init", status);
2012  status = pthread_cond_init(&__kmp_wait_cv.c_cond, &cond_attr);
2013  KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2014  status = pthread_condattr_destroy(&cond_attr);
2015  KMP_CHECK_SYSFAIL("pthread_condattr_destroy", status);
2016 #if USE_ITT_BUILD
2017  __kmp_itt_initialize();
2018 #endif /* USE_ITT_BUILD */
2019 
2020  __kmp_init_runtime = TRUE;
2021 }
2022 
2023 void __kmp_runtime_destroy(void) {
2024  int status;
2025 
2026  if (!__kmp_init_runtime) {
2027  return; // Nothing to do.
2028  }
2029 
2030 #if USE_ITT_BUILD
2031  __kmp_itt_destroy();
2032 #endif /* USE_ITT_BUILD */
2033 
2034  status = pthread_key_delete(__kmp_gtid_threadprivate_key);
2035  KMP_CHECK_SYSFAIL("pthread_key_delete", status);
2036 
2037  status = pthread_mutex_destroy(&__kmp_wait_mx.m_mutex);
2038  if (status != 0 && status != EBUSY) {
2039  KMP_SYSFAIL("pthread_mutex_destroy", status);
2040  }
2041  status = pthread_cond_destroy(&__kmp_wait_cv.c_cond);
2042  if (status != 0 && status != EBUSY) {
2043  KMP_SYSFAIL("pthread_cond_destroy", status);
2044  }
2045 #if KMP_AFFINITY_SUPPORTED
2046  __kmp_affinity_uninitialize();
2047 #endif
2048 
2049  __kmp_init_runtime = FALSE;
2050 }
2051 
2052 /* Put the thread to sleep for a time period */
2053 /* NOTE: not currently used anywhere */
2054 void __kmp_thread_sleep(int millis) { sleep((millis + 500) / 1000); }
2055 
2056 /* Calculate the elapsed wall clock time for the user */
2057 void __kmp_elapsed(double *t) {
2058  int status;
2059 #ifdef FIX_SGI_CLOCK
2060  struct timespec ts;
2061 
2062  status = clock_gettime(CLOCK_PROCESS_CPUTIME_ID, &ts);
2063  KMP_CHECK_SYSFAIL_ERRNO("clock_gettime", status);
2064  *t =
2065  (double)ts.tv_nsec * (1.0 / (double)KMP_NSEC_PER_SEC) + (double)ts.tv_sec;
2066 #else
2067  struct timeval tv;
2068 
2069  status = gettimeofday(&tv, NULL);
2070  KMP_CHECK_SYSFAIL_ERRNO("gettimeofday", status);
2071  *t =
2072  (double)tv.tv_usec * (1.0 / (double)KMP_USEC_PER_SEC) + (double)tv.tv_sec;
2073 #endif
2074 }
2075 
2076 /* Calculate the elapsed wall clock tick for the user */
2077 void __kmp_elapsed_tick(double *t) { *t = 1 / (double)CLOCKS_PER_SEC; }
2078 
2079 /* Return the current time stamp in nsec */
2080 kmp_uint64 __kmp_now_nsec() {
2081  struct timeval t;
2082  gettimeofday(&t, NULL);
2083  kmp_uint64 nsec = (kmp_uint64)KMP_NSEC_PER_SEC * (kmp_uint64)t.tv_sec +
2084  (kmp_uint64)1000 * (kmp_uint64)t.tv_usec;
2085  return nsec;
2086 }
2087 
2088 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
2089 /* Measure clock ticks per millisecond */
2090 void __kmp_initialize_system_tick() {
2091  kmp_uint64 now, nsec2, diff;
2092  kmp_uint64 delay = 1000000; // ~450 usec on most machines.
2093  kmp_uint64 nsec = __kmp_now_nsec();
2094  kmp_uint64 goal = __kmp_hardware_timestamp() + delay;
2095  while ((now = __kmp_hardware_timestamp()) < goal)
2096  ;
2097  nsec2 = __kmp_now_nsec();
2098  diff = nsec2 - nsec;
2099  if (diff > 0) {
2100  double tpus = 1000.0 * (double)(delay + (now - goal)) / (double)diff;
2101  if (tpus > 0.0) {
2102  __kmp_ticks_per_msec = (kmp_uint64)(tpus * 1000.0);
2103  __kmp_ticks_per_usec = (kmp_uint64)tpus;
2104  }
2105  }
2106 }
2107 #endif
2108 
2109 /* Determine whether the given address is mapped into the current address
2110  space. */
2111 
2112 int __kmp_is_address_mapped(void *addr) {
2113 
2114  int found = 0;
2115  int rc;
2116 
2117 #if KMP_OS_LINUX || KMP_OS_HURD
2118 
2119  /* On GNUish OSes, read the /proc/<pid>/maps pseudo-file to get all the
2120  address ranges mapped into the address space. */
2121 
2122  char *name = __kmp_str_format("/proc/%d/maps", getpid());
2123  FILE *file = NULL;
2124 
2125  file = fopen(name, "r");
2126  KMP_ASSERT(file != NULL);
2127 
2128  for (;;) {
2129 
2130  void *beginning = NULL;
2131  void *ending = NULL;
2132  char perms[5];
2133 
2134  rc = fscanf(file, "%p-%p %4s %*[^\n]\n", &beginning, &ending, perms);
2135  if (rc == EOF) {
2136  break;
2137  }
2138  KMP_ASSERT(rc == 3 &&
2139  KMP_STRLEN(perms) == 4); // Make sure all fields are read.
2140 
2141  // Ending address is not included in the region, but beginning is.
2142  if ((addr >= beginning) && (addr < ending)) {
2143  perms[2] = 0; // 3th and 4th character does not matter.
2144  if (strcmp(perms, "rw") == 0) {
2145  // Memory we are looking for should be readable and writable.
2146  found = 1;
2147  }
2148  break;
2149  }
2150  }
2151 
2152  // Free resources.
2153  fclose(file);
2154  KMP_INTERNAL_FREE(name);
2155 #elif KMP_OS_FREEBSD
2156  char *buf;
2157  size_t lstsz;
2158  int mib[] = {CTL_KERN, KERN_PROC, KERN_PROC_VMMAP, getpid()};
2159  rc = sysctl(mib, 4, NULL, &lstsz, NULL, 0);
2160  if (rc < 0)
2161  return 0;
2162  // We pass from number of vm entry's semantic
2163  // to size of whole entry map list.
2164  lstsz = lstsz * 4 / 3;
2165  buf = reinterpret_cast<char *>(KMP_INTERNAL_MALLOC(lstsz));
2166  rc = sysctl(mib, 4, buf, &lstsz, NULL, 0);
2167  if (rc < 0) {
2168  KMP_INTERNAL_FREE(buf);
2169  return 0;
2170  }
2171 
2172  char *lw = buf;
2173  char *up = buf + lstsz;
2174 
2175  while (lw < up) {
2176  struct kinfo_vmentry *cur = reinterpret_cast<struct kinfo_vmentry *>(lw);
2177  size_t cursz = cur->kve_structsize;
2178  if (cursz == 0)
2179  break;
2180  void *start = reinterpret_cast<void *>(cur->kve_start);
2181  void *end = reinterpret_cast<void *>(cur->kve_end);
2182  // Readable/Writable addresses within current map entry
2183  if ((addr >= start) && (addr < end)) {
2184  if ((cur->kve_protection & KVME_PROT_READ) != 0 &&
2185  (cur->kve_protection & KVME_PROT_WRITE) != 0) {
2186  found = 1;
2187  break;
2188  }
2189  }
2190  lw += cursz;
2191  }
2192  KMP_INTERNAL_FREE(buf);
2193 #elif KMP_OS_DRAGONFLY
2194  char err[_POSIX2_LINE_MAX];
2195  kinfo_proc *proc;
2196  vmspace sp;
2197  vm_map *cur;
2198  vm_map_entry entry, *c;
2199  struct proc p;
2200  kvm_t *fd;
2201  uintptr_t uaddr;
2202  int num;
2203 
2204  fd = kvm_openfiles(nullptr, nullptr, nullptr, O_RDONLY, err);
2205  if (!fd) {
2206  return 0;
2207  }
2208 
2209  proc = kvm_getprocs(fd, KERN_PROC_PID, getpid(), &num);
2210 
2211  if (kvm_read(fd, static_cast<uintptr_t>(proc->kp_paddr), &p, sizeof(p)) !=
2212  sizeof(p) ||
2213  kvm_read(fd, reinterpret_cast<uintptr_t>(p.p_vmspace), &sp, sizeof(sp)) !=
2214  sizeof(sp)) {
2215  kvm_close(fd);
2216  return 0;
2217  }
2218 
2219  (void)rc;
2220  cur = &sp.vm_map;
2221  uaddr = reinterpret_cast<uintptr_t>(addr);
2222  for (c = kvm_vm_map_entry_first(fd, cur, &entry); c;
2223  c = kvm_vm_map_entry_next(fd, c, &entry)) {
2224  if ((uaddr >= entry.ba.start) && (uaddr <= entry.ba.end)) {
2225  if ((entry.protection & VM_PROT_READ) != 0 &&
2226  (entry.protection & VM_PROT_WRITE) != 0) {
2227  found = 1;
2228  break;
2229  }
2230  }
2231  }
2232 
2233  kvm_close(fd);
2234 #elif KMP_OS_SOLARIS
2235  prmap_t *cur, *map;
2236  void *buf;
2237  uintptr_t uaddr;
2238  ssize_t rd;
2239  int err;
2240  int file;
2241 
2242  pid_t pid = getpid();
2243  struct ps_prochandle *fd = Pgrab(pid, PGRAB_RDONLY, &err);
2244  ;
2245 
2246  if (!fd) {
2247  return 0;
2248  }
2249 
2250  char *name = __kmp_str_format("/proc/%d/map", pid);
2251  size_t sz = (1 << 20);
2252  file = open(name, O_RDONLY);
2253  if (file == -1) {
2254  KMP_INTERNAL_FREE(name);
2255  return 0;
2256  }
2257 
2258  buf = KMP_INTERNAL_MALLOC(sz);
2259 
2260  while (sz > 0 && (rd = pread(file, buf, sz, 0)) == sz) {
2261  void *newbuf;
2262  sz <<= 1;
2263  newbuf = KMP_INTERNAL_REALLOC(buf, sz);
2264  buf = newbuf;
2265  }
2266 
2267  map = reinterpret_cast<prmap_t *>(buf);
2268  uaddr = reinterpret_cast<uintptr_t>(addr);
2269 
2270  for (cur = map; rd > 0; cur++, rd = -sizeof(*map)) {
2271  if ((uaddr >= cur->pr_vaddr) && (uaddr < cur->pr_vaddr)) {
2272  if ((cur->pr_mflags & MA_READ) != 0 && (cur->pr_mflags & MA_WRITE) != 0) {
2273  found = 1;
2274  break;
2275  }
2276  }
2277  }
2278 
2279  KMP_INTERNAL_FREE(map);
2280  close(file);
2281  KMP_INTERNAL_FREE(name);
2282 #elif KMP_OS_DARWIN
2283 
2284  /* On OS X*, /proc pseudo filesystem is not available. Try to read memory
2285  using vm interface. */
2286 
2287  int buffer;
2288  vm_size_t count;
2289  rc = vm_read_overwrite(
2290  mach_task_self(), // Task to read memory of.
2291  (vm_address_t)(addr), // Address to read from.
2292  1, // Number of bytes to be read.
2293  (vm_address_t)(&buffer), // Address of buffer to save read bytes in.
2294  &count // Address of var to save number of read bytes in.
2295  );
2296  if (rc == 0) {
2297  // Memory successfully read.
2298  found = 1;
2299  }
2300 
2301 #elif KMP_OS_NETBSD
2302 
2303  int mib[5];
2304  mib[0] = CTL_VM;
2305  mib[1] = VM_PROC;
2306  mib[2] = VM_PROC_MAP;
2307  mib[3] = getpid();
2308  mib[4] = sizeof(struct kinfo_vmentry);
2309 
2310  size_t size;
2311  rc = sysctl(mib, __arraycount(mib), NULL, &size, NULL, 0);
2312  KMP_ASSERT(!rc);
2313  KMP_ASSERT(size);
2314 
2315  size = size * 4 / 3;
2316  struct kinfo_vmentry *kiv = (struct kinfo_vmentry *)KMP_INTERNAL_MALLOC(size);
2317  KMP_ASSERT(kiv);
2318 
2319  rc = sysctl(mib, __arraycount(mib), kiv, &size, NULL, 0);
2320  KMP_ASSERT(!rc);
2321  KMP_ASSERT(size);
2322 
2323  for (size_t i = 0; i < size; i++) {
2324  if (kiv[i].kve_start >= (uint64_t)addr &&
2325  kiv[i].kve_end <= (uint64_t)addr) {
2326  found = 1;
2327  break;
2328  }
2329  }
2330  KMP_INTERNAL_FREE(kiv);
2331 #elif KMP_OS_OPENBSD
2332 
2333  int mib[3];
2334  mib[0] = CTL_KERN;
2335  mib[1] = KERN_PROC_VMMAP;
2336  mib[2] = getpid();
2337 
2338  size_t size;
2339  uint64_t end;
2340  rc = sysctl(mib, 3, NULL, &size, NULL, 0);
2341  KMP_ASSERT(!rc);
2342  KMP_ASSERT(size);
2343  end = size;
2344 
2345  struct kinfo_vmentry kiv = {.kve_start = 0};
2346 
2347  while ((rc = sysctl(mib, 3, &kiv, &size, NULL, 0)) == 0) {
2348  KMP_ASSERT(size);
2349  if (kiv.kve_end == end)
2350  break;
2351 
2352  if (kiv.kve_start >= (uint64_t)addr && kiv.kve_end <= (uint64_t)addr) {
2353  found = 1;
2354  break;
2355  }
2356  kiv.kve_start += 1;
2357  }
2358 #elif KMP_OS_WASI
2359  found = (int)addr < (__builtin_wasm_memory_size(0) * PAGESIZE);
2360 #elif KMP_OS_AIX
2361 
2362  uint32_t loadQueryBufSize = 4096u; // Default loadquery buffer size.
2363  char *loadQueryBuf;
2364 
2365  for (;;) {
2366  loadQueryBuf = (char *)KMP_INTERNAL_MALLOC(loadQueryBufSize);
2367  if (loadQueryBuf == NULL) {
2368  return 0;
2369  }
2370 
2371  rc = loadquery(L_GETXINFO | L_IGNOREUNLOAD, loadQueryBuf, loadQueryBufSize);
2372  if (rc < 0) {
2373  KMP_INTERNAL_FREE(loadQueryBuf);
2374  if (errno != ENOMEM) {
2375  return 0;
2376  }
2377  // errno == ENOMEM; double the size.
2378  loadQueryBufSize <<= 1;
2379  continue;
2380  }
2381  // Obtained the load info successfully.
2382  break;
2383  }
2384 
2385  struct ld_xinfo *curLdInfo = (struct ld_xinfo *)loadQueryBuf;
2386 
2387  // Loop through the load info to find if there is a match.
2388  for (;;) {
2389  uintptr_t curDataStart = (uintptr_t)curLdInfo->ldinfo_dataorg;
2390  uintptr_t curDataEnd = curDataStart + curLdInfo->ldinfo_datasize;
2391 
2392  // The data segment is readable and writable.
2393  if (curDataStart <= (uintptr_t)addr && (uintptr_t)addr < curDataEnd) {
2394  found = 1;
2395  break;
2396  }
2397  if (curLdInfo->ldinfo_next == 0u) {
2398  // Reached the end of load info.
2399  break;
2400  }
2401  curLdInfo = (struct ld_xinfo *)((char *)curLdInfo + curLdInfo->ldinfo_next);
2402  }
2403  KMP_INTERNAL_FREE(loadQueryBuf);
2404 
2405 #elif KMP_OS_HAIKU
2406 
2407  found = 1;
2408 #else
2409 
2410 #error "Unknown or unsupported OS"
2411 
2412 #endif
2413 
2414  return found;
2415 
2416 } // __kmp_is_address_mapped
2417 
2418 #ifdef USE_LOAD_BALANCE
2419 
2420 #if KMP_OS_DARWIN || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
2421  KMP_OS_OPENBSD || KMP_OS_SOLARIS
2422 
2423 // The function returns the rounded value of the system load average
2424 // during given time interval which depends on the value of
2425 // __kmp_load_balance_interval variable (default is 60 sec, other values
2426 // may be 300 sec or 900 sec).
2427 // It returns -1 in case of error.
2428 int __kmp_get_load_balance(int max) {
2429  double averages[3];
2430  int ret_avg = 0;
2431 
2432  int res = getloadavg(averages, 3);
2433 
2434  // Check __kmp_load_balance_interval to determine which of averages to use.
2435  // getloadavg() may return the number of samples less than requested that is
2436  // less than 3.
2437  if (__kmp_load_balance_interval < 180 && (res >= 1)) {
2438  ret_avg = (int)averages[0]; // 1 min
2439  } else if ((__kmp_load_balance_interval >= 180 &&
2440  __kmp_load_balance_interval < 600) &&
2441  (res >= 2)) {
2442  ret_avg = (int)averages[1]; // 5 min
2443  } else if ((__kmp_load_balance_interval >= 600) && (res == 3)) {
2444  ret_avg = (int)averages[2]; // 15 min
2445  } else { // Error occurred
2446  return -1;
2447  }
2448 
2449  return ret_avg;
2450 }
2451 
2452 #elif KMP_OS_AIX
2453 
2454 // The function returns number of running (not sleeping) threads, or -1 in case
2455 // of error.
2456 int __kmp_get_load_balance(int max) {
2457 
2458  static int glb_running_threads = 0; // Saved count of the running threads for
2459  // the thread balance algorithm.
2460  static double glb_call_time = 0; // Thread balance algorithm call time.
2461  int running_threads = 0; // Number of running threads in the system.
2462 
2463  double call_time = 0.0;
2464 
2465  __kmp_elapsed(&call_time);
2466 
2467  if (glb_call_time &&
2468  (call_time - glb_call_time < __kmp_load_balance_interval))
2469  return glb_running_threads;
2470 
2471  glb_call_time = call_time;
2472 
2473  if (max <= 0) {
2474  max = INT_MAX;
2475  }
2476 
2477  // Check how many perfstat_cpu_t structures are available.
2478  int logical_cpus = perfstat_cpu(NULL, NULL, sizeof(perfstat_cpu_t), 0);
2479  if (logical_cpus <= 0) {
2480  glb_call_time = -1;
2481  return -1;
2482  }
2483 
2484  perfstat_cpu_t *cpu_stat = (perfstat_cpu_t *)KMP_INTERNAL_MALLOC(
2485  logical_cpus * sizeof(perfstat_cpu_t));
2486  if (cpu_stat == NULL) {
2487  glb_call_time = -1;
2488  return -1;
2489  }
2490 
2491  // Set first CPU as the name of the first logical CPU for which the info is
2492  // desired.
2493  perfstat_id_t first_cpu_name;
2494  strcpy(first_cpu_name.name, FIRST_CPU);
2495 
2496  // Get the stat info of logical CPUs.
2497  int rc = perfstat_cpu(&first_cpu_name, cpu_stat, sizeof(perfstat_cpu_t),
2498  logical_cpus);
2499  KMP_DEBUG_ASSERT(rc == logical_cpus);
2500  if (rc <= 0) {
2501  KMP_INTERNAL_FREE(cpu_stat);
2502  glb_call_time = -1;
2503  return -1;
2504  }
2505  for (int i = 0; i < logical_cpus; ++i) {
2506  running_threads += cpu_stat[i].runque;
2507  if (running_threads >= max)
2508  break;
2509  }
2510 
2511  // There _might_ be a timing hole where the thread executing this
2512  // code gets skipped in the load balance, and running_threads is 0.
2513  // Assert in the debug builds only!!!
2514  KMP_DEBUG_ASSERT(running_threads > 0);
2515  if (running_threads <= 0)
2516  running_threads = 1;
2517 
2518  KMP_INTERNAL_FREE(cpu_stat);
2519 
2520  glb_running_threads = running_threads;
2521 
2522  return running_threads;
2523 }
2524 
2525 #else // Linux* OS
2526 
2527 // The function returns number of running (not sleeping) threads, or -1 in case
2528 // of error. Error could be reported if Linux* OS kernel too old (without
2529 // "/proc" support). Counting running threads stops if max running threads
2530 // encountered.
2531 int __kmp_get_load_balance(int max) {
2532  static int permanent_error = 0;
2533  static int glb_running_threads = 0; // Saved count of the running threads for
2534  // the thread balance algorithm
2535  static double glb_call_time = 0; /* Thread balance algorithm call time */
2536 
2537  int running_threads = 0; // Number of running threads in the system.
2538 
2539  DIR *proc_dir = NULL; // Handle of "/proc/" directory.
2540  struct dirent *proc_entry = NULL;
2541 
2542  kmp_str_buf_t task_path; // "/proc/<pid>/task/<tid>/" path.
2543  DIR *task_dir = NULL; // Handle of "/proc/<pid>/task/<tid>/" directory.
2544  struct dirent *task_entry = NULL;
2545  int task_path_fixed_len;
2546 
2547  kmp_str_buf_t stat_path; // "/proc/<pid>/task/<tid>/stat" path.
2548  int stat_file = -1;
2549  int stat_path_fixed_len;
2550 
2551 #ifdef KMP_DEBUG
2552  int total_processes = 0; // Total number of processes in system.
2553 #endif
2554 
2555  double call_time = 0.0;
2556 
2557  __kmp_str_buf_init(&task_path);
2558  __kmp_str_buf_init(&stat_path);
2559 
2560  __kmp_elapsed(&call_time);
2561 
2562  if (glb_call_time &&
2563  (call_time - glb_call_time < __kmp_load_balance_interval)) {
2564  running_threads = glb_running_threads;
2565  goto finish;
2566  }
2567 
2568  glb_call_time = call_time;
2569 
2570  // Do not spend time on scanning "/proc/" if we have a permanent error.
2571  if (permanent_error) {
2572  running_threads = -1;
2573  goto finish;
2574  }
2575 
2576  if (max <= 0) {
2577  max = INT_MAX;
2578  }
2579 
2580  // Open "/proc/" directory.
2581  proc_dir = opendir("/proc");
2582  if (proc_dir == NULL) {
2583  // Cannot open "/proc/". Probably the kernel does not support it. Return an
2584  // error now and in subsequent calls.
2585  running_threads = -1;
2586  permanent_error = 1;
2587  goto finish;
2588  }
2589 
2590  // Initialize fixed part of task_path. This part will not change.
2591  __kmp_str_buf_cat(&task_path, "/proc/", 6);
2592  task_path_fixed_len = task_path.used; // Remember number of used characters.
2593 
2594  proc_entry = readdir(proc_dir);
2595  while (proc_entry != NULL) {
2596  // Proc entry is a directory and name starts with a digit. Assume it is a
2597  // process' directory.
2598  if (proc_entry->d_type == DT_DIR && isdigit(proc_entry->d_name[0])) {
2599 
2600 #ifdef KMP_DEBUG
2601  ++total_processes;
2602 #endif
2603  // Make sure init process is the very first in "/proc", so we can replace
2604  // strcmp( proc_entry->d_name, "1" ) == 0 with simpler total_processes ==
2605  // 1. We are going to check that total_processes == 1 => d_name == "1" is
2606  // true (where "=>" is implication). Since C++ does not have => operator,
2607  // let us replace it with its equivalent: a => b == ! a || b.
2608  KMP_DEBUG_ASSERT(total_processes != 1 ||
2609  strcmp(proc_entry->d_name, "1") == 0);
2610 
2611  // Construct task_path.
2612  task_path.used = task_path_fixed_len; // Reset task_path to "/proc/".
2613  __kmp_str_buf_cat(&task_path, proc_entry->d_name,
2614  KMP_STRLEN(proc_entry->d_name));
2615  __kmp_str_buf_cat(&task_path, "/task", 5);
2616 
2617  task_dir = opendir(task_path.str);
2618  if (task_dir == NULL) {
2619  // Process can finish between reading "/proc/" directory entry and
2620  // opening process' "task/" directory. So, in general case we should not
2621  // complain, but have to skip this process and read the next one. But on
2622  // systems with no "task/" support we will spend lot of time to scan
2623  // "/proc/" tree again and again without any benefit. "init" process
2624  // (its pid is 1) should exist always, so, if we cannot open
2625  // "/proc/1/task/" directory, it means "task/" is not supported by
2626  // kernel. Report an error now and in the future.
2627  if (strcmp(proc_entry->d_name, "1") == 0) {
2628  running_threads = -1;
2629  permanent_error = 1;
2630  goto finish;
2631  }
2632  } else {
2633  // Construct fixed part of stat file path.
2634  __kmp_str_buf_clear(&stat_path);
2635  __kmp_str_buf_cat(&stat_path, task_path.str, task_path.used);
2636  __kmp_str_buf_cat(&stat_path, "/", 1);
2637  stat_path_fixed_len = stat_path.used;
2638 
2639  task_entry = readdir(task_dir);
2640  while (task_entry != NULL) {
2641  // It is a directory and name starts with a digit.
2642  if (proc_entry->d_type == DT_DIR && isdigit(task_entry->d_name[0])) {
2643 
2644  // Construct complete stat file path. Easiest way would be:
2645  // __kmp_str_buf_print( & stat_path, "%s/%s/stat", task_path.str,
2646  // task_entry->d_name );
2647  // but seriae of __kmp_str_buf_cat works a bit faster.
2648  stat_path.used =
2649  stat_path_fixed_len; // Reset stat path to its fixed part.
2650  __kmp_str_buf_cat(&stat_path, task_entry->d_name,
2651  KMP_STRLEN(task_entry->d_name));
2652  __kmp_str_buf_cat(&stat_path, "/stat", 5);
2653 
2654  // Note: Low-level API (open/read/close) is used. High-level API
2655  // (fopen/fclose) works ~ 30 % slower.
2656  stat_file = open(stat_path.str, O_RDONLY);
2657  if (stat_file == -1) {
2658  // We cannot report an error because task (thread) can terminate
2659  // just before reading this file.
2660  } else {
2661  /* Content of "stat" file looks like:
2662  24285 (program) S ...
2663 
2664  It is a single line (if program name does not include funny
2665  symbols). First number is a thread id, then name of executable
2666  file name in paretheses, then state of the thread. We need just
2667  thread state.
2668 
2669  Good news: Length of program name is 15 characters max. Longer
2670  names are truncated.
2671 
2672  Thus, we need rather short buffer: 15 chars for program name +
2673  2 parenthesis, + 3 spaces + ~7 digits of pid = 37.
2674 
2675  Bad news: Program name may contain special symbols like space,
2676  closing parenthesis, or even new line. This makes parsing
2677  "stat" file not 100 % reliable. In case of fanny program names
2678  parsing may fail (report incorrect thread state).
2679 
2680  Parsing "status" file looks more promissing (due to different
2681  file structure and escaping special symbols) but reading and
2682  parsing of "status" file works slower.
2683  -- ln
2684  */
2685  char buffer[65];
2686  ssize_t len;
2687  len = read(stat_file, buffer, sizeof(buffer) - 1);
2688  if (len >= 0) {
2689  buffer[len] = 0;
2690  // Using scanf:
2691  // sscanf( buffer, "%*d (%*s) %c ", & state );
2692  // looks very nice, but searching for a closing parenthesis
2693  // works a bit faster.
2694  char *close_parent = strstr(buffer, ") ");
2695  if (close_parent != NULL) {
2696  char state = *(close_parent + 2);
2697  if (state == 'R') {
2698  ++running_threads;
2699  if (running_threads >= max) {
2700  goto finish;
2701  }
2702  }
2703  }
2704  }
2705  close(stat_file);
2706  stat_file = -1;
2707  }
2708  }
2709  task_entry = readdir(task_dir);
2710  }
2711  closedir(task_dir);
2712  task_dir = NULL;
2713  }
2714  }
2715  proc_entry = readdir(proc_dir);
2716  }
2717 
2718  // There _might_ be a timing hole where the thread executing this
2719  // code get skipped in the load balance, and running_threads is 0.
2720  // Assert in the debug builds only!!!
2721  KMP_DEBUG_ASSERT(running_threads > 0);
2722  if (running_threads <= 0) {
2723  running_threads = 1;
2724  }
2725 
2726 finish: // Clean up and exit.
2727  if (proc_dir != NULL) {
2728  closedir(proc_dir);
2729  }
2730  __kmp_str_buf_free(&task_path);
2731  if (task_dir != NULL) {
2732  closedir(task_dir);
2733  }
2734  __kmp_str_buf_free(&stat_path);
2735  if (stat_file != -1) {
2736  close(stat_file);
2737  }
2738 
2739  glb_running_threads = running_threads;
2740 
2741  return running_threads;
2742 
2743 } // __kmp_get_load_balance
2744 
2745 #endif // KMP_OS_DARWIN
2746 
2747 #endif // USE_LOAD_BALANCE
2748 
2749 #if !(KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_MIC || \
2750  ((KMP_OS_LINUX || KMP_OS_DARWIN) && KMP_ARCH_AARCH64) || \
2751  KMP_ARCH_PPC64 || KMP_ARCH_RISCV64 || KMP_ARCH_LOONGARCH64 || \
2752  KMP_ARCH_ARM || KMP_ARCH_VE || KMP_ARCH_S390X || KMP_ARCH_PPC_XCOFF || \
2753  KMP_ARCH_AARCH64_32)
2754 
2755 // Because WebAssembly will use `call_indirect` to invoke the microtask and
2756 // WebAssembly indirect calls check that the called signature is a precise
2757 // match, we need to cast each microtask function pointer back from `void *` to
2758 // its original type.
2759 typedef void (*microtask_t0)(int *, int *);
2760 typedef void (*microtask_t1)(int *, int *, void *);
2761 typedef void (*microtask_t2)(int *, int *, void *, void *);
2762 typedef void (*microtask_t3)(int *, int *, void *, void *, void *);
2763 typedef void (*microtask_t4)(int *, int *, void *, void *, void *, void *);
2764 typedef void (*microtask_t5)(int *, int *, void *, void *, void *, void *,
2765  void *);
2766 typedef void (*microtask_t6)(int *, int *, void *, void *, void *, void *,
2767  void *, void *);
2768 typedef void (*microtask_t7)(int *, int *, void *, void *, void *, void *,
2769  void *, void *, void *);
2770 typedef void (*microtask_t8)(int *, int *, void *, void *, void *, void *,
2771  void *, void *, void *, void *);
2772 typedef void (*microtask_t9)(int *, int *, void *, void *, void *, void *,
2773  void *, void *, void *, void *, void *);
2774 typedef void (*microtask_t10)(int *, int *, void *, void *, void *, void *,
2775  void *, void *, void *, void *, void *, void *);
2776 typedef void (*microtask_t11)(int *, int *, void *, void *, void *, void *,
2777  void *, void *, void *, void *, void *, void *,
2778  void *);
2779 typedef void (*microtask_t12)(int *, int *, void *, void *, void *, void *,
2780  void *, void *, void *, void *, void *, void *,
2781  void *, void *);
2782 typedef void (*microtask_t13)(int *, int *, void *, void *, void *, void *,
2783  void *, void *, void *, void *, void *, void *,
2784  void *, void *, void *);
2785 typedef void (*microtask_t14)(int *, int *, void *, void *, void *, void *,
2786  void *, void *, void *, void *, void *, void *,
2787  void *, void *, void *, void *);
2788 typedef void (*microtask_t15)(int *, int *, void *, void *, void *, void *,
2789  void *, void *, void *, void *, void *, void *,
2790  void *, void *, void *, void *, void *);
2791 
2792 // we really only need the case with 1 argument, because CLANG always build
2793 // a struct of pointers to shared variables referenced in the outlined function
2794 int __kmp_invoke_microtask(microtask_t pkfn, int gtid, int tid, int argc,
2795  void *p_argv[]
2796 #if OMPT_SUPPORT
2797  ,
2798  void **exit_frame_ptr
2799 #endif
2800 ) {
2801 #if OMPT_SUPPORT
2802  *exit_frame_ptr = OMPT_GET_FRAME_ADDRESS(0);
2803 #endif
2804 
2805  switch (argc) {
2806  default:
2807  fprintf(stderr, "Too many args to microtask: %d!\n", argc);
2808  fflush(stderr);
2809  exit(-1);
2810  case 0:
2811  (*(microtask_t0)pkfn)(&gtid, &tid);
2812  break;
2813  case 1:
2814  (*(microtask_t1)pkfn)(&gtid, &tid, p_argv[0]);
2815  break;
2816  case 2:
2817  (*(microtask_t2)pkfn)(&gtid, &tid, p_argv[0], p_argv[1]);
2818  break;
2819  case 3:
2820  (*(microtask_t3)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2]);
2821  break;
2822  case 4:
2823  (*(microtask_t4)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2824  p_argv[3]);
2825  break;
2826  case 5:
2827  (*(microtask_t5)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2828  p_argv[3], p_argv[4]);
2829  break;
2830  case 6:
2831  (*(microtask_t6)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2832  p_argv[3], p_argv[4], p_argv[5]);
2833  break;
2834  case 7:
2835  (*(microtask_t7)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2836  p_argv[3], p_argv[4], p_argv[5], p_argv[6]);
2837  break;
2838  case 8:
2839  (*(microtask_t8)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2840  p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2841  p_argv[7]);
2842  break;
2843  case 9:
2844  (*(microtask_t9)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2845  p_argv[3], p_argv[4], p_argv[5], p_argv[6], p_argv[7],
2846  p_argv[8]);
2847  break;
2848  case 10:
2849  (*(microtask_t10)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2850  p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2851  p_argv[7], p_argv[8], p_argv[9]);
2852  break;
2853  case 11:
2854  (*(microtask_t11)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2855  p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2856  p_argv[7], p_argv[8], p_argv[9], p_argv[10]);
2857  break;
2858  case 12:
2859  (*(microtask_t12)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2860  p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2861  p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2862  p_argv[11]);
2863  break;
2864  case 13:
2865  (*(microtask_t13)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2866  p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2867  p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2868  p_argv[11], p_argv[12]);
2869  break;
2870  case 14:
2871  (*(microtask_t14)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2872  p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2873  p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2874  p_argv[11], p_argv[12], p_argv[13]);
2875  break;
2876  case 15:
2877  (*(microtask_t15)pkfn)(&gtid, &tid, p_argv[0], p_argv[1], p_argv[2],
2878  p_argv[3], p_argv[4], p_argv[5], p_argv[6],
2879  p_argv[7], p_argv[8], p_argv[9], p_argv[10],
2880  p_argv[11], p_argv[12], p_argv[13], p_argv[14]);
2881  break;
2882  }
2883 
2884  return 1;
2885 }
2886 
2887 #endif
2888 
2889 #if KMP_OS_LINUX
2890 // Functions for hidden helper task
2891 namespace {
2892 // Condition variable for initializing hidden helper team
2893 pthread_cond_t hidden_helper_threads_initz_cond_var;
2894 pthread_mutex_t hidden_helper_threads_initz_lock;
2895 volatile int hidden_helper_initz_signaled = FALSE;
2896 
2897 // Condition variable for deinitializing hidden helper team
2898 pthread_cond_t hidden_helper_threads_deinitz_cond_var;
2899 pthread_mutex_t hidden_helper_threads_deinitz_lock;
2900 volatile int hidden_helper_deinitz_signaled = FALSE;
2901 
2902 // Condition variable for the wrapper function of main thread
2903 pthread_cond_t hidden_helper_main_thread_cond_var;
2904 pthread_mutex_t hidden_helper_main_thread_lock;
2905 volatile int hidden_helper_main_thread_signaled = FALSE;
2906 
2907 // Semaphore for worker threads. We don't use condition variable here in case
2908 // that when multiple signals are sent at the same time, only one thread might
2909 // be waken.
2910 sem_t hidden_helper_task_sem;
2911 } // namespace
2912 
2913 void __kmp_hidden_helper_worker_thread_wait() {
2914  int status = sem_wait(&hidden_helper_task_sem);
2915  KMP_CHECK_SYSFAIL("sem_wait", status);
2916 }
2917 
2918 void __kmp_do_initialize_hidden_helper_threads() {
2919  // Initialize condition variable
2920  int status =
2921  pthread_cond_init(&hidden_helper_threads_initz_cond_var, nullptr);
2922  KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2923 
2924  status = pthread_cond_init(&hidden_helper_threads_deinitz_cond_var, nullptr);
2925  KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2926 
2927  status = pthread_cond_init(&hidden_helper_main_thread_cond_var, nullptr);
2928  KMP_CHECK_SYSFAIL("pthread_cond_init", status);
2929 
2930  status = pthread_mutex_init(&hidden_helper_threads_initz_lock, nullptr);
2931  KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2932 
2933  status = pthread_mutex_init(&hidden_helper_threads_deinitz_lock, nullptr);
2934  KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2935 
2936  status = pthread_mutex_init(&hidden_helper_main_thread_lock, nullptr);
2937  KMP_CHECK_SYSFAIL("pthread_mutex_init", status);
2938 
2939  // Initialize the semaphore
2940  status = sem_init(&hidden_helper_task_sem, 0, 0);
2941  KMP_CHECK_SYSFAIL("sem_init", status);
2942 
2943  // Create a new thread to finish initialization
2944  pthread_t handle;
2945  status = pthread_create(
2946  &handle, nullptr,
2947  [](void *) -> void * {
2948  __kmp_hidden_helper_threads_initz_routine();
2949  return nullptr;
2950  },
2951  nullptr);
2952  KMP_CHECK_SYSFAIL("pthread_create", status);
2953 }
2954 
2955 void __kmp_hidden_helper_threads_initz_wait() {
2956  // Initial thread waits here for the completion of the initialization. The
2957  // condition variable will be notified by main thread of hidden helper teams.
2958  int status = pthread_mutex_lock(&hidden_helper_threads_initz_lock);
2959  KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2960 
2961  if (!TCR_4(hidden_helper_initz_signaled)) {
2962  status = pthread_cond_wait(&hidden_helper_threads_initz_cond_var,
2963  &hidden_helper_threads_initz_lock);
2964  KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2965  }
2966 
2967  status = pthread_mutex_unlock(&hidden_helper_threads_initz_lock);
2968  KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2969 }
2970 
2971 void __kmp_hidden_helper_initz_release() {
2972  // After all initialization, reset __kmp_init_hidden_helper_threads to false.
2973  int status = pthread_mutex_lock(&hidden_helper_threads_initz_lock);
2974  KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2975 
2976  status = pthread_cond_signal(&hidden_helper_threads_initz_cond_var);
2977  KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2978 
2979  TCW_SYNC_4(hidden_helper_initz_signaled, TRUE);
2980 
2981  status = pthread_mutex_unlock(&hidden_helper_threads_initz_lock);
2982  KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2983 }
2984 
2985 void __kmp_hidden_helper_main_thread_wait() {
2986  // The main thread of hidden helper team will be blocked here. The
2987  // condition variable can only be signal in the destructor of RTL.
2988  int status = pthread_mutex_lock(&hidden_helper_main_thread_lock);
2989  KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
2990 
2991  if (!TCR_4(hidden_helper_main_thread_signaled)) {
2992  status = pthread_cond_wait(&hidden_helper_main_thread_cond_var,
2993  &hidden_helper_main_thread_lock);
2994  KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
2995  }
2996 
2997  status = pthread_mutex_unlock(&hidden_helper_main_thread_lock);
2998  KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
2999 }
3000 
3001 void __kmp_hidden_helper_main_thread_release() {
3002  // The initial thread of OpenMP RTL should call this function to wake up the
3003  // main thread of hidden helper team.
3004  int status = pthread_mutex_lock(&hidden_helper_main_thread_lock);
3005  KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
3006 
3007  status = pthread_cond_signal(&hidden_helper_main_thread_cond_var);
3008  KMP_CHECK_SYSFAIL("pthread_cond_signal", status);
3009 
3010  // The hidden helper team is done here
3011  TCW_SYNC_4(hidden_helper_main_thread_signaled, TRUE);
3012 
3013  status = pthread_mutex_unlock(&hidden_helper_main_thread_lock);
3014  KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
3015 }
3016 
3017 void __kmp_hidden_helper_worker_thread_signal() {
3018  int status = sem_post(&hidden_helper_task_sem);
3019  KMP_CHECK_SYSFAIL("sem_post", status);
3020 }
3021 
3022 void __kmp_hidden_helper_threads_deinitz_wait() {
3023  // Initial thread waits here for the completion of the deinitialization. The
3024  // condition variable will be notified by main thread of hidden helper teams.
3025  int status = pthread_mutex_lock(&hidden_helper_threads_deinitz_lock);
3026  KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
3027 
3028  if (!TCR_4(hidden_helper_deinitz_signaled)) {
3029  status = pthread_cond_wait(&hidden_helper_threads_deinitz_cond_var,
3030  &hidden_helper_threads_deinitz_lock);
3031  KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
3032  }
3033 
3034  status = pthread_mutex_unlock(&hidden_helper_threads_deinitz_lock);
3035  KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
3036 }
3037 
3038 void __kmp_hidden_helper_threads_deinitz_release() {
3039  int status = pthread_mutex_lock(&hidden_helper_threads_deinitz_lock);
3040  KMP_CHECK_SYSFAIL("pthread_mutex_lock", status);
3041 
3042  status = pthread_cond_signal(&hidden_helper_threads_deinitz_cond_var);
3043  KMP_CHECK_SYSFAIL("pthread_cond_wait", status);
3044 
3045  TCW_SYNC_4(hidden_helper_deinitz_signaled, TRUE);
3046 
3047  status = pthread_mutex_unlock(&hidden_helper_threads_deinitz_lock);
3048  KMP_CHECK_SYSFAIL("pthread_mutex_unlock", status);
3049 }
3050 #else // KMP_OS_LINUX
3051 void __kmp_hidden_helper_worker_thread_wait() {
3052  KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3053 }
3054 
3055 void __kmp_do_initialize_hidden_helper_threads() {
3056  KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3057 }
3058 
3059 void __kmp_hidden_helper_threads_initz_wait() {
3060  KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3061 }
3062 
3063 void __kmp_hidden_helper_initz_release() {
3064  KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3065 }
3066 
3067 void __kmp_hidden_helper_main_thread_wait() {
3068  KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3069 }
3070 
3071 void __kmp_hidden_helper_main_thread_release() {
3072  KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3073 }
3074 
3075 void __kmp_hidden_helper_worker_thread_signal() {
3076  KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3077 }
3078 
3079 void __kmp_hidden_helper_threads_deinitz_wait() {
3080  KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3081 }
3082 
3083 void __kmp_hidden_helper_threads_deinitz_release() {
3084  KMP_ASSERT(0 && "Hidden helper task is not supported on this OS");
3085 }
3086 #endif // KMP_OS_LINUX
3087 
3088 bool __kmp_detect_shm() {
3089  DIR *dir = opendir("/dev/shm");
3090  if (dir) { // /dev/shm exists
3091  closedir(dir);
3092  return true;
3093  } else if (ENOENT == errno) { // /dev/shm does not exist
3094  return false;
3095  } else { // opendir() failed
3096  return false;
3097  }
3098 }
3099 
3100 bool __kmp_detect_tmp() {
3101  DIR *dir = opendir("/tmp");
3102  if (dir) { // /tmp exists
3103  closedir(dir);
3104  return true;
3105  } else if (ENOENT == errno) { // /tmp does not exist
3106  return false;
3107  } else { // opendir() failed
3108  return false;
3109  }
3110 }
3111 
3112 // end of file //
#define KMP_INIT_PARTITIONED_TIMERS(name)
Initializes the partitioned timers to begin with name.
Definition: kmp_stats.h:940