LLVM OpenMP* Runtime Library
 All Classes Functions Variables Typedefs Enumerations Enumerator Modules Pages
z_Windows_NT_util.cpp
1 /*
2  * z_Windows_NT_util.cpp -- platform specific routines.
3  */
4 
5 //===----------------------------------------------------------------------===//
6 //
7 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
8 // See https://llvm.org/LICENSE.txt for license information.
9 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "kmp.h"
14 #include "kmp_affinity.h"
15 #include "kmp_i18n.h"
16 #include "kmp_io.h"
17 #include "kmp_itt.h"
18 #include "kmp_wait_release.h"
19 
20 /* This code is related to NtQuerySystemInformation() function. This function
21  is used in the Load balance algorithm for OMP_DYNAMIC=true to find the
22  number of running threads in the system. */
23 
24 #include <ntsecapi.h> // UNICODE_STRING
25 #include <ntstatus.h>
26 
27 enum SYSTEM_INFORMATION_CLASS {
28  SystemProcessInformation = 5
29 }; // SYSTEM_INFORMATION_CLASS
30 
31 struct CLIENT_ID {
32  HANDLE UniqueProcess;
33  HANDLE UniqueThread;
34 }; // struct CLIENT_ID
35 
36 enum THREAD_STATE {
37  StateInitialized,
38  StateReady,
39  StateRunning,
40  StateStandby,
41  StateTerminated,
42  StateWait,
43  StateTransition,
44  StateUnknown
45 }; // enum THREAD_STATE
46 
47 struct VM_COUNTERS {
48  SIZE_T PeakVirtualSize;
49  SIZE_T VirtualSize;
50  ULONG PageFaultCount;
51  SIZE_T PeakWorkingSetSize;
52  SIZE_T WorkingSetSize;
53  SIZE_T QuotaPeakPagedPoolUsage;
54  SIZE_T QuotaPagedPoolUsage;
55  SIZE_T QuotaPeakNonPagedPoolUsage;
56  SIZE_T QuotaNonPagedPoolUsage;
57  SIZE_T PagefileUsage;
58  SIZE_T PeakPagefileUsage;
59  SIZE_T PrivatePageCount;
60 }; // struct VM_COUNTERS
61 
62 struct SYSTEM_THREAD {
63  LARGE_INTEGER KernelTime;
64  LARGE_INTEGER UserTime;
65  LARGE_INTEGER CreateTime;
66  ULONG WaitTime;
67  LPVOID StartAddress;
68  CLIENT_ID ClientId;
69  DWORD Priority;
70  LONG BasePriority;
71  ULONG ContextSwitchCount;
72  THREAD_STATE State;
73  ULONG WaitReason;
74 }; // SYSTEM_THREAD
75 
76 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, KernelTime) == 0);
77 #if KMP_ARCH_X86
78 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 28);
79 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 52);
80 #else
81 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, StartAddress) == 32);
82 KMP_BUILD_ASSERT(offsetof(SYSTEM_THREAD, State) == 68);
83 #endif
84 
85 struct SYSTEM_PROCESS_INFORMATION {
86  ULONG NextEntryOffset;
87  ULONG NumberOfThreads;
88  LARGE_INTEGER Reserved[3];
89  LARGE_INTEGER CreateTime;
90  LARGE_INTEGER UserTime;
91  LARGE_INTEGER KernelTime;
92  UNICODE_STRING ImageName;
93  DWORD BasePriority;
94  HANDLE ProcessId;
95  HANDLE ParentProcessId;
96  ULONG HandleCount;
97  ULONG Reserved2[2];
98  VM_COUNTERS VMCounters;
99  IO_COUNTERS IOCounters;
100  SYSTEM_THREAD Threads[1];
101 }; // SYSTEM_PROCESS_INFORMATION
102 typedef SYSTEM_PROCESS_INFORMATION *PSYSTEM_PROCESS_INFORMATION;
103 
104 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, NextEntryOffset) == 0);
105 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, CreateTime) == 32);
106 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ImageName) == 56);
107 #if KMP_ARCH_X86
108 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 68);
109 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 76);
110 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 88);
111 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 136);
112 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 184);
113 #else
114 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, ProcessId) == 80);
115 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, HandleCount) == 96);
116 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, VMCounters) == 112);
117 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, IOCounters) == 208);
118 KMP_BUILD_ASSERT(offsetof(SYSTEM_PROCESS_INFORMATION, Threads) == 256);
119 #endif
120 
121 typedef NTSTATUS(NTAPI *NtQuerySystemInformation_t)(SYSTEM_INFORMATION_CLASS,
122  PVOID, ULONG, PULONG);
123 NtQuerySystemInformation_t NtQuerySystemInformation = NULL;
124 
125 HMODULE ntdll = NULL;
126 
127 /* End of NtQuerySystemInformation()-related code */
128 
129 static HMODULE kernel32 = NULL;
130 
131 #if KMP_HANDLE_SIGNALS
132 typedef void (*sig_func_t)(int);
133 static sig_func_t __kmp_sighldrs[NSIG];
134 static int __kmp_siginstalled[NSIG];
135 #endif
136 
137 #if KMP_USE_MONITOR
138 static HANDLE __kmp_monitor_ev;
139 #endif
140 static kmp_int64 __kmp_win32_time;
141 double __kmp_win32_tick;
142 
143 int __kmp_init_runtime = FALSE;
144 CRITICAL_SECTION __kmp_win32_section;
145 
146 void __kmp_win32_mutex_init(kmp_win32_mutex_t *mx) {
147  InitializeCriticalSection(&mx->cs);
148 #if USE_ITT_BUILD
149  __kmp_itt_system_object_created(&mx->cs, "Critical Section");
150 #endif /* USE_ITT_BUILD */
151 }
152 
153 void __kmp_win32_mutex_destroy(kmp_win32_mutex_t *mx) {
154  DeleteCriticalSection(&mx->cs);
155 }
156 
157 void __kmp_win32_mutex_lock(kmp_win32_mutex_t *mx) {
158  EnterCriticalSection(&mx->cs);
159 }
160 
161 int __kmp_win32_mutex_trylock(kmp_win32_mutex_t *mx) {
162  return TryEnterCriticalSection(&mx->cs);
163 }
164 
165 void __kmp_win32_mutex_unlock(kmp_win32_mutex_t *mx) {
166  LeaveCriticalSection(&mx->cs);
167 }
168 
169 void __kmp_win32_cond_init(kmp_win32_cond_t *cv) {
170  cv->waiters_count_ = 0;
171  cv->wait_generation_count_ = 0;
172  cv->release_count_ = 0;
173 
174  /* Initialize the critical section */
175  __kmp_win32_mutex_init(&cv->waiters_count_lock_);
176 
177  /* Create a manual-reset event. */
178  cv->event_ = CreateEvent(NULL, // no security
179  TRUE, // manual-reset
180  FALSE, // non-signaled initially
181  NULL); // unnamed
182 #if USE_ITT_BUILD
183  __kmp_itt_system_object_created(cv->event_, "Event");
184 #endif /* USE_ITT_BUILD */
185 }
186 
187 void __kmp_win32_cond_destroy(kmp_win32_cond_t *cv) {
188  __kmp_win32_mutex_destroy(&cv->waiters_count_lock_);
189  __kmp_free_handle(cv->event_);
190  memset(cv, '\0', sizeof(*cv));
191 }
192 
193 /* TODO associate cv with a team instead of a thread so as to optimize
194  the case where we wake up a whole team */
195 
196 void __kmp_win32_cond_wait(kmp_win32_cond_t *cv, kmp_win32_mutex_t *mx,
197  kmp_info_t *th, int need_decrease_load) {
198  int my_generation;
199  int last_waiter;
200 
201  /* Avoid race conditions */
202  __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
203 
204  /* Increment count of waiters */
205  cv->waiters_count_++;
206 
207  /* Store current generation in our activation record. */
208  my_generation = cv->wait_generation_count_;
209 
210  __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
211  __kmp_win32_mutex_unlock(mx);
212 
213  for (;;) {
214  int wait_done;
215 
216  /* Wait until the event is signaled */
217  WaitForSingleObject(cv->event_, INFINITE);
218 
219  __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
220 
221  /* Exit the loop when the <cv->event_> is signaled and there are still
222  waiting threads from this <wait_generation> that haven't been released
223  from this wait yet. */
224  wait_done = (cv->release_count_ > 0) &&
225  (cv->wait_generation_count_ != my_generation);
226 
227  __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
228 
229  /* there used to be a semicolon after the if statement, it looked like a
230  bug, so i removed it */
231  if (wait_done)
232  break;
233  }
234 
235  __kmp_win32_mutex_lock(mx);
236  __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
237 
238  cv->waiters_count_--;
239  cv->release_count_--;
240 
241  last_waiter = (cv->release_count_ == 0);
242 
243  __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
244 
245  if (last_waiter) {
246  /* We're the last waiter to be notified, so reset the manual event. */
247  ResetEvent(cv->event_);
248  }
249 }
250 
251 void __kmp_win32_cond_broadcast(kmp_win32_cond_t *cv) {
252  __kmp_win32_mutex_lock(&cv->waiters_count_lock_);
253 
254  if (cv->waiters_count_ > 0) {
255  SetEvent(cv->event_);
256  /* Release all the threads in this generation. */
257 
258  cv->release_count_ = cv->waiters_count_;
259 
260  /* Start a new generation. */
261  cv->wait_generation_count_++;
262  }
263 
264  __kmp_win32_mutex_unlock(&cv->waiters_count_lock_);
265 }
266 
267 void __kmp_win32_cond_signal(kmp_win32_cond_t *cv) {
268  __kmp_win32_cond_broadcast(cv);
269 }
270 
271 void __kmp_enable(int new_state) {
272  if (__kmp_init_runtime)
273  LeaveCriticalSection(&__kmp_win32_section);
274 }
275 
276 void __kmp_disable(int *old_state) {
277  *old_state = 0;
278 
279  if (__kmp_init_runtime)
280  EnterCriticalSection(&__kmp_win32_section);
281 }
282 
283 void __kmp_suspend_initialize(void) { /* do nothing */
284 }
285 
286 static void __kmp_suspend_initialize_thread(kmp_info_t *th) {
287  if (!TCR_4(th->th.th_suspend_init)) {
288  /* this means we haven't initialized the suspension pthread objects for this
289  thread in this instance of the process */
290  __kmp_win32_cond_init(&th->th.th_suspend_cv);
291  __kmp_win32_mutex_init(&th->th.th_suspend_mx);
292  TCW_4(th->th.th_suspend_init, TRUE);
293  }
294 }
295 
296 void __kmp_suspend_uninitialize_thread(kmp_info_t *th) {
297  if (TCR_4(th->th.th_suspend_init)) {
298  /* this means we have initialize the suspension pthread objects for this
299  thread in this instance of the process */
300  __kmp_win32_cond_destroy(&th->th.th_suspend_cv);
301  __kmp_win32_mutex_destroy(&th->th.th_suspend_mx);
302  TCW_4(th->th.th_suspend_init, FALSE);
303  }
304 }
305 
306 int __kmp_try_suspend_mx(kmp_info_t *th) {
307  return __kmp_win32_mutex_trylock(&th->th.th_suspend_mx);
308 }
309 
310 void __kmp_lock_suspend_mx(kmp_info_t *th) {
311  __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
312 }
313 
314 void __kmp_unlock_suspend_mx(kmp_info_t *th) {
315  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
316 }
317 
318 /* This routine puts the calling thread to sleep after setting the
319  sleep bit for the indicated flag variable to true. */
320 template <class C>
321 static inline void __kmp_suspend_template(int th_gtid, C *flag) {
322  kmp_info_t *th = __kmp_threads[th_gtid];
323  int status;
324  typename C::flag_t old_spin;
325 
326  KF_TRACE(30, ("__kmp_suspend_template: T#%d enter for flag's loc(%p)\n",
327  th_gtid, flag->get()));
328 
329  __kmp_suspend_initialize_thread(th);
330  __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
331 
332  KF_TRACE(10, ("__kmp_suspend_template: T#%d setting sleep bit for flag's"
333  " loc(%p)\n",
334  th_gtid, flag->get()));
335 
336  /* TODO: shouldn't this use release semantics to ensure that
337  __kmp_suspend_initialize_thread gets called first? */
338  old_spin = flag->set_sleeping();
339 #if OMP_50_ENABLED
340  if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME &&
341  __kmp_pause_status != kmp_soft_paused) {
342  flag->unset_sleeping();
343  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
344  return;
345  }
346 #endif
347 
348  KF_TRACE(5, ("__kmp_suspend_template: T#%d set sleep bit for flag's"
349  " loc(%p)==%d\n",
350  th_gtid, flag->get(), *(flag->get())));
351 
352  if (flag->done_check_val(old_spin)) {
353  old_spin = flag->unset_sleeping();
354  KF_TRACE(5, ("__kmp_suspend_template: T#%d false alarm, reset sleep bit "
355  "for flag's loc(%p)\n",
356  th_gtid, flag->get()));
357  } else {
358 #ifdef DEBUG_SUSPEND
359  __kmp_suspend_count++;
360 #endif
361  /* Encapsulate in a loop as the documentation states that this may "with
362  low probability" return when the condition variable has not been signaled
363  or broadcast */
364  int deactivated = FALSE;
365  TCW_PTR(th->th.th_sleep_loc, (void *)flag);
366  while (flag->is_sleeping()) {
367  KF_TRACE(15, ("__kmp_suspend_template: T#%d about to perform "
368  "kmp_win32_cond_wait()\n",
369  th_gtid));
370  // Mark the thread as no longer active (only in the first iteration of the
371  // loop).
372  if (!deactivated) {
373  th->th.th_active = FALSE;
374  if (th->th.th_active_in_pool) {
375  th->th.th_active_in_pool = FALSE;
376  KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
377  KMP_DEBUG_ASSERT(TCR_4(__kmp_thread_pool_active_nth) >= 0);
378  }
379  deactivated = TRUE;
380 
381  __kmp_win32_cond_wait(&th->th.th_suspend_cv, &th->th.th_suspend_mx, 0,
382  0);
383  } else {
384  __kmp_win32_cond_wait(&th->th.th_suspend_cv, &th->th.th_suspend_mx, 0,
385  0);
386  }
387 
388 #ifdef KMP_DEBUG
389  if (flag->is_sleeping()) {
390  KF_TRACE(100,
391  ("__kmp_suspend_template: T#%d spurious wakeup\n", th_gtid));
392  }
393 #endif /* KMP_DEBUG */
394 
395  } // while
396 
397  // Mark the thread as active again (if it was previous marked as inactive)
398  if (deactivated) {
399  th->th.th_active = TRUE;
400  if (TCR_4(th->th.th_in_pool)) {
401  KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
402  th->th.th_active_in_pool = TRUE;
403  }
404  }
405  }
406 
407  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
408 
409  KF_TRACE(30, ("__kmp_suspend_template: T#%d exit\n", th_gtid));
410 }
411 
412 void __kmp_suspend_32(int th_gtid, kmp_flag_32 *flag) {
413  __kmp_suspend_template(th_gtid, flag);
414 }
415 void __kmp_suspend_64(int th_gtid, kmp_flag_64 *flag) {
416  __kmp_suspend_template(th_gtid, flag);
417 }
418 void __kmp_suspend_oncore(int th_gtid, kmp_flag_oncore *flag) {
419  __kmp_suspend_template(th_gtid, flag);
420 }
421 
422 /* This routine signals the thread specified by target_gtid to wake up
423  after setting the sleep bit indicated by the flag argument to FALSE */
424 template <class C>
425 static inline void __kmp_resume_template(int target_gtid, C *flag) {
426  kmp_info_t *th = __kmp_threads[target_gtid];
427  int status;
428 
429 #ifdef KMP_DEBUG
430  int gtid = TCR_4(__kmp_init_gtid) ? __kmp_get_gtid() : -1;
431 #endif
432 
433  KF_TRACE(30, ("__kmp_resume_template: T#%d wants to wakeup T#%d enter\n",
434  gtid, target_gtid));
435 
436  __kmp_suspend_initialize_thread(th);
437  __kmp_win32_mutex_lock(&th->th.th_suspend_mx);
438 
439  if (!flag) { // coming from __kmp_null_resume_wrapper
440  flag = (C *)th->th.th_sleep_loc;
441  }
442 
443  // First, check if the flag is null or its type has changed. If so, someone
444  // else woke it up.
445  if (!flag || flag->get_type() != flag->get_ptr_type()) { // get_ptr_type
446  // simply shows what
447  // flag was cast to
448  KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
449  "awake: flag's loc(%p)\n",
450  gtid, target_gtid, NULL));
451  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
452  return;
453  } else {
454  typename C::flag_t old_spin = flag->unset_sleeping();
455  if (!flag->is_sleeping_val(old_spin)) {
456  KF_TRACE(5, ("__kmp_resume_template: T#%d exiting, thread T#%d already "
457  "awake: flag's loc(%p): %u => %u\n",
458  gtid, target_gtid, flag->get(), old_spin, *(flag->get())));
459  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
460  return;
461  }
462  }
463  TCW_PTR(th->th.th_sleep_loc, NULL);
464  KF_TRACE(5, ("__kmp_resume_template: T#%d about to wakeup T#%d, reset sleep "
465  "bit for flag's loc(%p)\n",
466  gtid, target_gtid, flag->get()));
467 
468  __kmp_win32_cond_signal(&th->th.th_suspend_cv);
469  __kmp_win32_mutex_unlock(&th->th.th_suspend_mx);
470 
471  KF_TRACE(30, ("__kmp_resume_template: T#%d exiting after signaling wake up"
472  " for T#%d\n",
473  gtid, target_gtid));
474 }
475 
476 void __kmp_resume_32(int target_gtid, kmp_flag_32 *flag) {
477  __kmp_resume_template(target_gtid, flag);
478 }
479 void __kmp_resume_64(int target_gtid, kmp_flag_64 *flag) {
480  __kmp_resume_template(target_gtid, flag);
481 }
482 void __kmp_resume_oncore(int target_gtid, kmp_flag_oncore *flag) {
483  __kmp_resume_template(target_gtid, flag);
484 }
485 
486 void __kmp_yield(int cond) {
487  if (cond)
488  Sleep(0);
489 }
490 
491 void __kmp_gtid_set_specific(int gtid) {
492  if (__kmp_init_gtid) {
493  KA_TRACE(50, ("__kmp_gtid_set_specific: T#%d key:%d\n", gtid,
494  __kmp_gtid_threadprivate_key));
495  if (!TlsSetValue(__kmp_gtid_threadprivate_key, (LPVOID)(gtid + 1)))
496  KMP_FATAL(TLSSetValueFailed);
497  } else {
498  KA_TRACE(50, ("__kmp_gtid_set_specific: runtime shutdown, returning\n"));
499  }
500 }
501 
502 int __kmp_gtid_get_specific() {
503  int gtid;
504  if (!__kmp_init_gtid) {
505  KA_TRACE(50, ("__kmp_gtid_get_specific: runtime shutdown, returning "
506  "KMP_GTID_SHUTDOWN\n"));
507  return KMP_GTID_SHUTDOWN;
508  }
509  gtid = (int)(kmp_intptr_t)TlsGetValue(__kmp_gtid_threadprivate_key);
510  if (gtid == 0) {
511  gtid = KMP_GTID_DNE;
512  } else {
513  gtid--;
514  }
515  KA_TRACE(50, ("__kmp_gtid_get_specific: key:%d gtid:%d\n",
516  __kmp_gtid_threadprivate_key, gtid));
517  return gtid;
518 }
519 
520 void __kmp_affinity_bind_thread(int proc) {
521  if (__kmp_num_proc_groups > 1) {
522  // Form the GROUP_AFFINITY struct directly, rather than filling
523  // out a bit vector and calling __kmp_set_system_affinity().
524  GROUP_AFFINITY ga;
525  KMP_DEBUG_ASSERT((proc >= 0) && (proc < (__kmp_num_proc_groups * CHAR_BIT *
526  sizeof(DWORD_PTR))));
527  ga.Group = proc / (CHAR_BIT * sizeof(DWORD_PTR));
528  ga.Mask = (unsigned long long)1 << (proc % (CHAR_BIT * sizeof(DWORD_PTR)));
529  ga.Reserved[0] = ga.Reserved[1] = ga.Reserved[2] = 0;
530 
531  KMP_DEBUG_ASSERT(__kmp_SetThreadGroupAffinity != NULL);
532  if (__kmp_SetThreadGroupAffinity(GetCurrentThread(), &ga, NULL) == 0) {
533  DWORD error = GetLastError();
534  if (__kmp_affinity_verbose) { // AC: continue silently if not verbose
535  kmp_msg_t err_code = KMP_ERR(error);
536  __kmp_msg(kmp_ms_warning, KMP_MSG(CantSetThreadAffMask), err_code,
537  __kmp_msg_null);
538  if (__kmp_generate_warnings == kmp_warnings_off) {
539  __kmp_str_free(&err_code.str);
540  }
541  }
542  }
543  } else {
544  kmp_affin_mask_t *mask;
545  KMP_CPU_ALLOC_ON_STACK(mask);
546  KMP_CPU_ZERO(mask);
547  KMP_CPU_SET(proc, mask);
548  __kmp_set_system_affinity(mask, TRUE);
549  KMP_CPU_FREE_FROM_STACK(mask);
550  }
551 }
552 
553 void __kmp_affinity_determine_capable(const char *env_var) {
554 // All versions of Windows* OS (since Win '95) support SetThreadAffinityMask().
555 
556 #if KMP_GROUP_AFFINITY
557  KMP_AFFINITY_ENABLE(__kmp_num_proc_groups * sizeof(DWORD_PTR));
558 #else
559  KMP_AFFINITY_ENABLE(sizeof(DWORD_PTR));
560 #endif
561 
562  KA_TRACE(10, ("__kmp_affinity_determine_capable: "
563  "Windows* OS affinity interface functional (mask size = "
564  "%" KMP_SIZE_T_SPEC ").\n",
565  __kmp_affin_mask_size));
566 }
567 
568 double __kmp_read_cpu_time(void) {
569  FILETIME CreationTime, ExitTime, KernelTime, UserTime;
570  int status;
571  double cpu_time;
572 
573  cpu_time = 0;
574 
575  status = GetProcessTimes(GetCurrentProcess(), &CreationTime, &ExitTime,
576  &KernelTime, &UserTime);
577 
578  if (status) {
579  double sec = 0;
580 
581  sec += KernelTime.dwHighDateTime;
582  sec += UserTime.dwHighDateTime;
583 
584  /* Shift left by 32 bits */
585  sec *= (double)(1 << 16) * (double)(1 << 16);
586 
587  sec += KernelTime.dwLowDateTime;
588  sec += UserTime.dwLowDateTime;
589 
590  cpu_time += (sec * 100.0) / KMP_NSEC_PER_SEC;
591  }
592 
593  return cpu_time;
594 }
595 
596 int __kmp_read_system_info(struct kmp_sys_info *info) {
597  info->maxrss = 0; /* the maximum resident set size utilized (in kilobytes) */
598  info->minflt = 0; /* the number of page faults serviced without any I/O */
599  info->majflt = 0; /* the number of page faults serviced that required I/O */
600  info->nswap = 0; // the number of times a process was "swapped" out of memory
601  info->inblock = 0; // the number of times the file system had to perform input
602  info->oublock = 0; // number of times the file system had to perform output
603  info->nvcsw = 0; /* the number of times a context switch was voluntarily */
604  info->nivcsw = 0; /* the number of times a context switch was forced */
605 
606  return 1;
607 }
608 
609 void __kmp_runtime_initialize(void) {
610  SYSTEM_INFO info;
611  kmp_str_buf_t path;
612  UINT path_size;
613 
614  if (__kmp_init_runtime) {
615  return;
616  }
617 
618 #if KMP_DYNAMIC_LIB
619  /* Pin dynamic library for the lifetime of application */
620  {
621  // First, turn off error message boxes
622  UINT err_mode = SetErrorMode(SEM_FAILCRITICALERRORS);
623  HMODULE h;
624  BOOL ret = GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS |
625  GET_MODULE_HANDLE_EX_FLAG_PIN,
626  (LPCTSTR)&__kmp_serial_initialize, &h);
627  KMP_DEBUG_ASSERT2(h && ret, "OpenMP RTL cannot find itself loaded");
628  SetErrorMode(err_mode); // Restore error mode
629  KA_TRACE(10, ("__kmp_runtime_initialize: dynamic library pinned\n"));
630  }
631 #endif
632 
633  InitializeCriticalSection(&__kmp_win32_section);
634 #if USE_ITT_BUILD
635  __kmp_itt_system_object_created(&__kmp_win32_section, "Critical Section");
636 #endif /* USE_ITT_BUILD */
637  __kmp_initialize_system_tick();
638 
639 #if (KMP_ARCH_X86 || KMP_ARCH_X86_64)
640  if (!__kmp_cpuinfo.initialized) {
641  __kmp_query_cpuid(&__kmp_cpuinfo);
642  }
643 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
644 
645 /* Set up minimum number of threads to switch to TLS gtid */
646 #if KMP_OS_WINDOWS && !KMP_DYNAMIC_LIB
647  // Windows* OS, static library.
648  /* New thread may use stack space previously used by another thread,
649  currently terminated. On Windows* OS, in case of static linking, we do not
650  know the moment of thread termination, and our structures (__kmp_threads
651  and __kmp_root arrays) are still keep info about dead threads. This leads
652  to problem in __kmp_get_global_thread_id() function: it wrongly finds gtid
653  (by searching through stack addresses of all known threads) for
654  unregistered foreign tread.
655 
656  Setting __kmp_tls_gtid_min to 0 workarounds this problem:
657  __kmp_get_global_thread_id() does not search through stacks, but get gtid
658  from TLS immediately.
659  --ln
660  */
661  __kmp_tls_gtid_min = 0;
662 #else
663  __kmp_tls_gtid_min = KMP_TLS_GTID_MIN;
664 #endif
665 
666  /* for the static library */
667  if (!__kmp_gtid_threadprivate_key) {
668  __kmp_gtid_threadprivate_key = TlsAlloc();
669  if (__kmp_gtid_threadprivate_key == TLS_OUT_OF_INDEXES) {
670  KMP_FATAL(TLSOutOfIndexes);
671  }
672  }
673 
674  // Load ntdll.dll.
675  /* Simple GetModuleHandle( "ntdll.dl" ) is not suitable due to security issue
676  (see http://www.microsoft.com/technet/security/advisory/2269637.mspx). We
677  have to specify full path to the library. */
678  __kmp_str_buf_init(&path);
679  path_size = GetSystemDirectory(path.str, path.size);
680  KMP_DEBUG_ASSERT(path_size > 0);
681  if (path_size >= path.size) {
682  // Buffer is too short. Expand the buffer and try again.
683  __kmp_str_buf_reserve(&path, path_size);
684  path_size = GetSystemDirectory(path.str, path.size);
685  KMP_DEBUG_ASSERT(path_size > 0);
686  }
687  if (path_size > 0 && path_size < path.size) {
688  // Now we have system directory name in the buffer.
689  // Append backslash and name of dll to form full path,
690  path.used = path_size;
691  __kmp_str_buf_print(&path, "\\%s", "ntdll.dll");
692 
693  // Now load ntdll using full path.
694  ntdll = GetModuleHandle(path.str);
695  }
696 
697  KMP_DEBUG_ASSERT(ntdll != NULL);
698  if (ntdll != NULL) {
699  NtQuerySystemInformation = (NtQuerySystemInformation_t)GetProcAddress(
700  ntdll, "NtQuerySystemInformation");
701  }
702  KMP_DEBUG_ASSERT(NtQuerySystemInformation != NULL);
703 
704 #if KMP_GROUP_AFFINITY
705  // Load kernel32.dll.
706  // Same caveat - must use full system path name.
707  if (path_size > 0 && path_size < path.size) {
708  // Truncate the buffer back to just the system path length,
709  // discarding "\\ntdll.dll", and replacing it with "kernel32.dll".
710  path.used = path_size;
711  __kmp_str_buf_print(&path, "\\%s", "kernel32.dll");
712 
713  // Load kernel32.dll using full path.
714  kernel32 = GetModuleHandle(path.str);
715  KA_TRACE(10, ("__kmp_runtime_initialize: kernel32.dll = %s\n", path.str));
716 
717  // Load the function pointers to kernel32.dll routines
718  // that may or may not exist on this system.
719  if (kernel32 != NULL) {
720  __kmp_GetActiveProcessorCount =
721  (kmp_GetActiveProcessorCount_t)GetProcAddress(
722  kernel32, "GetActiveProcessorCount");
723  __kmp_GetActiveProcessorGroupCount =
724  (kmp_GetActiveProcessorGroupCount_t)GetProcAddress(
725  kernel32, "GetActiveProcessorGroupCount");
726  __kmp_GetThreadGroupAffinity =
727  (kmp_GetThreadGroupAffinity_t)GetProcAddress(
728  kernel32, "GetThreadGroupAffinity");
729  __kmp_SetThreadGroupAffinity =
730  (kmp_SetThreadGroupAffinity_t)GetProcAddress(
731  kernel32, "SetThreadGroupAffinity");
732 
733  KA_TRACE(10, ("__kmp_runtime_initialize: __kmp_GetActiveProcessorCount"
734  " = %p\n",
735  __kmp_GetActiveProcessorCount));
736  KA_TRACE(10, ("__kmp_runtime_initialize: "
737  "__kmp_GetActiveProcessorGroupCount = %p\n",
738  __kmp_GetActiveProcessorGroupCount));
739  KA_TRACE(10, ("__kmp_runtime_initialize:__kmp_GetThreadGroupAffinity"
740  " = %p\n",
741  __kmp_GetThreadGroupAffinity));
742  KA_TRACE(10, ("__kmp_runtime_initialize: __kmp_SetThreadGroupAffinity"
743  " = %p\n",
744  __kmp_SetThreadGroupAffinity));
745  KA_TRACE(10, ("__kmp_runtime_initialize: sizeof(kmp_affin_mask_t) = %d\n",
746  sizeof(kmp_affin_mask_t)));
747 
748  // See if group affinity is supported on this system.
749  // If so, calculate the #groups and #procs.
750  //
751  // Group affinity was introduced with Windows* 7 OS and
752  // Windows* Server 2008 R2 OS.
753  if ((__kmp_GetActiveProcessorCount != NULL) &&
754  (__kmp_GetActiveProcessorGroupCount != NULL) &&
755  (__kmp_GetThreadGroupAffinity != NULL) &&
756  (__kmp_SetThreadGroupAffinity != NULL) &&
757  ((__kmp_num_proc_groups = __kmp_GetActiveProcessorGroupCount()) >
758  1)) {
759  // Calculate the total number of active OS procs.
760  int i;
761 
762  KA_TRACE(10, ("__kmp_runtime_initialize: %d processor groups"
763  " detected\n",
764  __kmp_num_proc_groups));
765 
766  __kmp_xproc = 0;
767 
768  for (i = 0; i < __kmp_num_proc_groups; i++) {
769  DWORD size = __kmp_GetActiveProcessorCount(i);
770  __kmp_xproc += size;
771  KA_TRACE(10, ("__kmp_runtime_initialize: proc group %d size = %d\n",
772  i, size));
773  }
774  } else {
775  KA_TRACE(10, ("__kmp_runtime_initialize: %d processor groups"
776  " detected\n",
777  __kmp_num_proc_groups));
778  }
779  }
780  }
781  if (__kmp_num_proc_groups <= 1) {
782  GetSystemInfo(&info);
783  __kmp_xproc = info.dwNumberOfProcessors;
784  }
785 #else
786  GetSystemInfo(&info);
787  __kmp_xproc = info.dwNumberOfProcessors;
788 #endif /* KMP_GROUP_AFFINITY */
789 
790  // If the OS said there were 0 procs, take a guess and use a value of 2.
791  // This is done for Linux* OS, also. Do we need error / warning?
792  if (__kmp_xproc <= 0) {
793  __kmp_xproc = 2;
794  }
795 
796  KA_TRACE(5,
797  ("__kmp_runtime_initialize: total processors = %d\n", __kmp_xproc));
798 
799  __kmp_str_buf_free(&path);
800 
801 #if USE_ITT_BUILD
802  __kmp_itt_initialize();
803 #endif /* USE_ITT_BUILD */
804 
805  __kmp_init_runtime = TRUE;
806 } // __kmp_runtime_initialize
807 
808 void __kmp_runtime_destroy(void) {
809  if (!__kmp_init_runtime) {
810  return;
811  }
812 
813 #if USE_ITT_BUILD
814  __kmp_itt_destroy();
815 #endif /* USE_ITT_BUILD */
816 
817  /* we can't DeleteCriticalsection( & __kmp_win32_section ); */
818  /* due to the KX_TRACE() commands */
819  KA_TRACE(40, ("__kmp_runtime_destroy\n"));
820 
821  if (__kmp_gtid_threadprivate_key) {
822  TlsFree(__kmp_gtid_threadprivate_key);
823  __kmp_gtid_threadprivate_key = 0;
824  }
825 
826  __kmp_affinity_uninitialize();
827  DeleteCriticalSection(&__kmp_win32_section);
828 
829  ntdll = NULL;
830  NtQuerySystemInformation = NULL;
831 
832 #if KMP_ARCH_X86_64
833  kernel32 = NULL;
834  __kmp_GetActiveProcessorCount = NULL;
835  __kmp_GetActiveProcessorGroupCount = NULL;
836  __kmp_GetThreadGroupAffinity = NULL;
837  __kmp_SetThreadGroupAffinity = NULL;
838 #endif // KMP_ARCH_X86_64
839 
840  __kmp_init_runtime = FALSE;
841 }
842 
843 void __kmp_terminate_thread(int gtid) {
844  kmp_info_t *th = __kmp_threads[gtid];
845 
846  if (!th)
847  return;
848 
849  KA_TRACE(10, ("__kmp_terminate_thread: kill (%d)\n", gtid));
850 
851  if (TerminateThread(th->th.th_info.ds.ds_thread, (DWORD)-1) == FALSE) {
852  /* It's OK, the thread may have exited already */
853  }
854  __kmp_free_handle(th->th.th_info.ds.ds_thread);
855 }
856 
857 void __kmp_clear_system_time(void) {
858  BOOL status;
859  LARGE_INTEGER time;
860  status = QueryPerformanceCounter(&time);
861  __kmp_win32_time = (kmp_int64)time.QuadPart;
862 }
863 
864 void __kmp_initialize_system_tick(void) {
865  {
866  BOOL status;
867  LARGE_INTEGER freq;
868 
869  status = QueryPerformanceFrequency(&freq);
870  if (!status) {
871  DWORD error = GetLastError();
872  __kmp_fatal(KMP_MSG(FunctionError, "QueryPerformanceFrequency()"),
873  KMP_ERR(error), __kmp_msg_null);
874 
875  } else {
876  __kmp_win32_tick = ((double)1.0) / (double)freq.QuadPart;
877  }
878  }
879 }
880 
881 /* Calculate the elapsed wall clock time for the user */
882 
883 void __kmp_elapsed(double *t) {
884  BOOL status;
885  LARGE_INTEGER now;
886  status = QueryPerformanceCounter(&now);
887  *t = ((double)now.QuadPart) * __kmp_win32_tick;
888 }
889 
890 /* Calculate the elapsed wall clock tick for the user */
891 
892 void __kmp_elapsed_tick(double *t) { *t = __kmp_win32_tick; }
893 
894 void __kmp_read_system_time(double *delta) {
895  if (delta != NULL) {
896  BOOL status;
897  LARGE_INTEGER now;
898 
899  status = QueryPerformanceCounter(&now);
900 
901  *delta = ((double)(((kmp_int64)now.QuadPart) - __kmp_win32_time)) *
902  __kmp_win32_tick;
903  }
904 }
905 
906 /* Return the current time stamp in nsec */
907 kmp_uint64 __kmp_now_nsec() {
908  LARGE_INTEGER now;
909  QueryPerformanceCounter(&now);
910  return 1e9 * __kmp_win32_tick * now.QuadPart;
911 }
912 
913 extern "C"
914 void *__stdcall __kmp_launch_worker(void *arg) {
915  volatile void *stack_data;
916  void *exit_val;
917  void *padding = 0;
918  kmp_info_t *this_thr = (kmp_info_t *)arg;
919  int gtid;
920 
921  gtid = this_thr->th.th_info.ds.ds_gtid;
922  __kmp_gtid_set_specific(gtid);
923 #ifdef KMP_TDATA_GTID
924 #error "This define causes problems with LoadLibrary() + declspec(thread) " \
925  "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
926  "reference: http://support.microsoft.com/kb/118816"
927 //__kmp_gtid = gtid;
928 #endif
929 
930 #if USE_ITT_BUILD
931  __kmp_itt_thread_name(gtid);
932 #endif /* USE_ITT_BUILD */
933 
934  __kmp_affinity_set_init_mask(gtid, FALSE);
935 
936 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
937  // Set FP control regs to be a copy of the parallel initialization thread's.
938  __kmp_clear_x87_fpu_status_word();
939  __kmp_load_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
940  __kmp_load_mxcsr(&__kmp_init_mxcsr);
941 #endif /* KMP_ARCH_X86 || KMP_ARCH_X86_64 */
942 
943  if (__kmp_stkoffset > 0 && gtid > 0) {
944  padding = KMP_ALLOCA(gtid * __kmp_stkoffset);
945  }
946 
947  KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive);
948  this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
949  TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE);
950 
951  if (TCR_4(__kmp_gtid_mode) <
952  2) { // check stack only if it is used to get gtid
953  TCW_PTR(this_thr->th.th_info.ds.ds_stackbase, &stack_data);
954  KMP_ASSERT(this_thr->th.th_info.ds.ds_stackgrow == FALSE);
955  __kmp_check_stack_overlap(this_thr);
956  }
957  KMP_MB();
958  exit_val = __kmp_launch_thread(this_thr);
959  KMP_FSYNC_RELEASING(&this_thr->th.th_info.ds.ds_alive);
960  TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE);
961  KMP_MB();
962  return exit_val;
963 }
964 
965 #if KMP_USE_MONITOR
966 /* The monitor thread controls all of the threads in the complex */
967 
968 void *__stdcall __kmp_launch_monitor(void *arg) {
969  DWORD wait_status;
970  kmp_thread_t monitor;
971  int status;
972  int interval;
973  kmp_info_t *this_thr = (kmp_info_t *)arg;
974 
975  KMP_DEBUG_ASSERT(__kmp_init_monitor);
976  TCW_4(__kmp_init_monitor, 2); // AC: Signal library that monitor has started
977  // TODO: hide "2" in enum (like {true,false,started})
978  this_thr->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
979  TCW_4(this_thr->th.th_info.ds.ds_alive, TRUE);
980 
981  KMP_MB(); /* Flush all pending memory write invalidates. */
982  KA_TRACE(10, ("__kmp_launch_monitor: launched\n"));
983 
984  monitor = GetCurrentThread();
985 
986  /* set thread priority */
987  status = SetThreadPriority(monitor, THREAD_PRIORITY_HIGHEST);
988  if (!status) {
989  DWORD error = GetLastError();
990  __kmp_fatal(KMP_MSG(CantSetThreadPriority), KMP_ERR(error), __kmp_msg_null);
991  }
992 
993  /* register us as monitor */
994  __kmp_gtid_set_specific(KMP_GTID_MONITOR);
995 #ifdef KMP_TDATA_GTID
996 #error "This define causes problems with LoadLibrary() + declspec(thread) " \
997  "on Windows* OS. See CQ50564, tests kmp_load_library*.c and this MSDN " \
998  "reference: http://support.microsoft.com/kb/118816"
999 //__kmp_gtid = KMP_GTID_MONITOR;
1000 #endif
1001 
1002 #if USE_ITT_BUILD
1003  __kmp_itt_thread_ignore(); // Instruct Intel(R) Threading Tools to ignore
1004 // monitor thread.
1005 #endif /* USE_ITT_BUILD */
1006 
1007  KMP_MB(); /* Flush all pending memory write invalidates. */
1008 
1009  interval = (1000 / __kmp_monitor_wakeups); /* in milliseconds */
1010 
1011  while (!TCR_4(__kmp_global.g.g_done)) {
1012  /* This thread monitors the state of the system */
1013 
1014  KA_TRACE(15, ("__kmp_launch_monitor: update\n"));
1015 
1016  wait_status = WaitForSingleObject(__kmp_monitor_ev, interval);
1017 
1018  if (wait_status == WAIT_TIMEOUT) {
1019  TCW_4(__kmp_global.g.g_time.dt.t_value,
1020  TCR_4(__kmp_global.g.g_time.dt.t_value) + 1);
1021  }
1022 
1023  KMP_MB(); /* Flush all pending memory write invalidates. */
1024  }
1025 
1026  KA_TRACE(10, ("__kmp_launch_monitor: finished\n"));
1027 
1028  status = SetThreadPriority(monitor, THREAD_PRIORITY_NORMAL);
1029  if (!status) {
1030  DWORD error = GetLastError();
1031  __kmp_fatal(KMP_MSG(CantSetThreadPriority), KMP_ERR(error), __kmp_msg_null);
1032  }
1033 
1034  if (__kmp_global.g.g_abort != 0) {
1035  /* now we need to terminate the worker threads */
1036  /* the value of t_abort is the signal we caught */
1037  int gtid;
1038 
1039  KA_TRACE(10, ("__kmp_launch_monitor: terminate sig=%d\n",
1040  (__kmp_global.g.g_abort)));
1041 
1042  /* terminate the OpenMP worker threads */
1043  /* TODO this is not valid for sibling threads!!
1044  * the uber master might not be 0 anymore.. */
1045  for (gtid = 1; gtid < __kmp_threads_capacity; ++gtid)
1046  __kmp_terminate_thread(gtid);
1047 
1048  __kmp_cleanup();
1049 
1050  Sleep(0);
1051 
1052  KA_TRACE(10,
1053  ("__kmp_launch_monitor: raise sig=%d\n", __kmp_global.g.g_abort));
1054 
1055  if (__kmp_global.g.g_abort > 0) {
1056  raise(__kmp_global.g.g_abort);
1057  }
1058  }
1059 
1060  TCW_4(this_thr->th.th_info.ds.ds_alive, FALSE);
1061 
1062  KMP_MB();
1063  return arg;
1064 }
1065 #endif
1066 
1067 void __kmp_create_worker(int gtid, kmp_info_t *th, size_t stack_size) {
1068  kmp_thread_t handle;
1069  DWORD idThread;
1070 
1071  KA_TRACE(10, ("__kmp_create_worker: try to create thread (%d)\n", gtid));
1072 
1073  th->th.th_info.ds.ds_gtid = gtid;
1074 
1075  if (KMP_UBER_GTID(gtid)) {
1076  int stack_data;
1077 
1078  /* TODO: GetCurrentThread() returns a pseudo-handle that is unsuitable for
1079  other threads to use. Is it appropriate to just use GetCurrentThread?
1080  When should we close this handle? When unregistering the root? */
1081  {
1082  BOOL rc;
1083  rc = DuplicateHandle(GetCurrentProcess(), GetCurrentThread(),
1084  GetCurrentProcess(), &th->th.th_info.ds.ds_thread, 0,
1085  FALSE, DUPLICATE_SAME_ACCESS);
1086  KMP_ASSERT(rc);
1087  KA_TRACE(10, (" __kmp_create_worker: ROOT Handle duplicated, th = %p, "
1088  "handle = %" KMP_UINTPTR_SPEC "\n",
1089  (LPVOID)th, th->th.th_info.ds.ds_thread));
1090  th->th.th_info.ds.ds_thread_id = GetCurrentThreadId();
1091  }
1092  if (TCR_4(__kmp_gtid_mode) < 2) { // check stack only if used to get gtid
1093  /* we will dynamically update the stack range if gtid_mode == 1 */
1094  TCW_PTR(th->th.th_info.ds.ds_stackbase, &stack_data);
1095  TCW_PTR(th->th.th_info.ds.ds_stacksize, 0);
1096  TCW_4(th->th.th_info.ds.ds_stackgrow, TRUE);
1097  __kmp_check_stack_overlap(th);
1098  }
1099  } else {
1100  KMP_MB(); /* Flush all pending memory write invalidates. */
1101 
1102  /* Set stack size for this thread now. */
1103  KA_TRACE(10,
1104  ("__kmp_create_worker: stack_size = %" KMP_SIZE_T_SPEC " bytes\n",
1105  stack_size));
1106 
1107  stack_size += gtid * __kmp_stkoffset;
1108 
1109  TCW_PTR(th->th.th_info.ds.ds_stacksize, stack_size);
1110  TCW_4(th->th.th_info.ds.ds_stackgrow, FALSE);
1111 
1112  KA_TRACE(10,
1113  ("__kmp_create_worker: (before) stack_size = %" KMP_SIZE_T_SPEC
1114  " bytes, &__kmp_launch_worker = %p, th = %p, &idThread = %p\n",
1115  (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker,
1116  (LPVOID)th, &idThread));
1117 
1118  handle = CreateThread(
1119  NULL, (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)__kmp_launch_worker,
1120  (LPVOID)th, STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread);
1121 
1122  KA_TRACE(10,
1123  ("__kmp_create_worker: (after) stack_size = %" KMP_SIZE_T_SPEC
1124  " bytes, &__kmp_launch_worker = %p, th = %p, "
1125  "idThread = %u, handle = %" KMP_UINTPTR_SPEC "\n",
1126  (SIZE_T)stack_size, (LPTHREAD_START_ROUTINE)&__kmp_launch_worker,
1127  (LPVOID)th, idThread, handle));
1128 
1129  if (handle == 0) {
1130  DWORD error = GetLastError();
1131  __kmp_fatal(KMP_MSG(CantCreateThread), KMP_ERR(error), __kmp_msg_null);
1132  } else {
1133  th->th.th_info.ds.ds_thread = handle;
1134  }
1135 
1136  KMP_MB(); /* Flush all pending memory write invalidates. */
1137  }
1138 
1139  KA_TRACE(10, ("__kmp_create_worker: done creating thread (%d)\n", gtid));
1140 }
1141 
1142 int __kmp_still_running(kmp_info_t *th) {
1143  return (WAIT_TIMEOUT == WaitForSingleObject(th->th.th_info.ds.ds_thread, 0));
1144 }
1145 
1146 #if KMP_USE_MONITOR
1147 void __kmp_create_monitor(kmp_info_t *th) {
1148  kmp_thread_t handle;
1149  DWORD idThread;
1150  int ideal, new_ideal;
1151 
1152  if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
1153  // We don't need monitor thread in case of MAX_BLOCKTIME
1154  KA_TRACE(10, ("__kmp_create_monitor: skipping monitor thread because of "
1155  "MAX blocktime\n"));
1156  th->th.th_info.ds.ds_tid = 0; // this makes reap_monitor no-op
1157  th->th.th_info.ds.ds_gtid = 0;
1158  TCW_4(__kmp_init_monitor, 2); // Signal to stop waiting for monitor creation
1159  return;
1160  }
1161  KA_TRACE(10, ("__kmp_create_monitor: try to create monitor\n"));
1162 
1163  KMP_MB(); /* Flush all pending memory write invalidates. */
1164 
1165  __kmp_monitor_ev = CreateEvent(NULL, TRUE, FALSE, NULL);
1166  if (__kmp_monitor_ev == NULL) {
1167  DWORD error = GetLastError();
1168  __kmp_fatal(KMP_MSG(CantCreateEvent), KMP_ERR(error), __kmp_msg_null);
1169  }
1170 #if USE_ITT_BUILD
1171  __kmp_itt_system_object_created(__kmp_monitor_ev, "Event");
1172 #endif /* USE_ITT_BUILD */
1173 
1174  th->th.th_info.ds.ds_tid = KMP_GTID_MONITOR;
1175  th->th.th_info.ds.ds_gtid = KMP_GTID_MONITOR;
1176 
1177  // FIXME - on Windows* OS, if __kmp_monitor_stksize = 0, figure out how
1178  // to automatically expand stacksize based on CreateThread error code.
1179  if (__kmp_monitor_stksize == 0) {
1180  __kmp_monitor_stksize = KMP_DEFAULT_MONITOR_STKSIZE;
1181  }
1182  if (__kmp_monitor_stksize < __kmp_sys_min_stksize) {
1183  __kmp_monitor_stksize = __kmp_sys_min_stksize;
1184  }
1185 
1186  KA_TRACE(10, ("__kmp_create_monitor: requested stacksize = %d bytes\n",
1187  (int)__kmp_monitor_stksize));
1188 
1189  TCW_4(__kmp_global.g.g_time.dt.t_value, 0);
1190 
1191  handle =
1192  CreateThread(NULL, (SIZE_T)__kmp_monitor_stksize,
1193  (LPTHREAD_START_ROUTINE)__kmp_launch_monitor, (LPVOID)th,
1194  STACK_SIZE_PARAM_IS_A_RESERVATION, &idThread);
1195  if (handle == 0) {
1196  DWORD error = GetLastError();
1197  __kmp_fatal(KMP_MSG(CantCreateThread), KMP_ERR(error), __kmp_msg_null);
1198  } else
1199  th->th.th_info.ds.ds_thread = handle;
1200 
1201  KMP_MB(); /* Flush all pending memory write invalidates. */
1202 
1203  KA_TRACE(10, ("__kmp_create_monitor: monitor created %p\n",
1204  (void *)th->th.th_info.ds.ds_thread));
1205 }
1206 #endif
1207 
1208 /* Check to see if thread is still alive.
1209  NOTE: The ExitProcess(code) system call causes all threads to Terminate
1210  with a exit_val = code. Because of this we can not rely on exit_val having
1211  any particular value. So this routine may return STILL_ALIVE in exit_val
1212  even after the thread is dead. */
1213 
1214 int __kmp_is_thread_alive(kmp_info_t *th, DWORD *exit_val) {
1215  DWORD rc;
1216  rc = GetExitCodeThread(th->th.th_info.ds.ds_thread, exit_val);
1217  if (rc == 0) {
1218  DWORD error = GetLastError();
1219  __kmp_fatal(KMP_MSG(FunctionError, "GetExitCodeThread()"), KMP_ERR(error),
1220  __kmp_msg_null);
1221  }
1222  return (*exit_val == STILL_ACTIVE);
1223 }
1224 
1225 void __kmp_exit_thread(int exit_status) {
1226  ExitThread(exit_status);
1227 } // __kmp_exit_thread
1228 
1229 // This is a common part for both __kmp_reap_worker() and __kmp_reap_monitor().
1230 static void __kmp_reap_common(kmp_info_t *th) {
1231  DWORD exit_val;
1232 
1233  KMP_MB(); /* Flush all pending memory write invalidates. */
1234 
1235  KA_TRACE(
1236  10, ("__kmp_reap_common: try to reap (%d)\n", th->th.th_info.ds.ds_gtid));
1237 
1238  /* 2006-10-19:
1239  There are two opposite situations:
1240  1. Windows* OS keep thread alive after it resets ds_alive flag and
1241  exits from thread function. (For example, see C70770/Q394281 "unloading of
1242  dll based on OMP is very slow".)
1243  2. Windows* OS may kill thread before it resets ds_alive flag.
1244 
1245  Right solution seems to be waiting for *either* thread termination *or*
1246  ds_alive resetting. */
1247  {
1248  // TODO: This code is very similar to KMP_WAIT_YIELD. Need to generalize
1249  // KMP_WAIT_YIELD to cover this usage also.
1250  void *obj = NULL;
1251  kmp_uint32 spins;
1252 #if USE_ITT_BUILD
1253  KMP_FSYNC_SPIN_INIT(obj, (void *)&th->th.th_info.ds.ds_alive);
1254 #endif /* USE_ITT_BUILD */
1255  KMP_INIT_YIELD(spins);
1256  do {
1257 #if USE_ITT_BUILD
1258  KMP_FSYNC_SPIN_PREPARE(obj);
1259 #endif /* USE_ITT_BUILD */
1260  __kmp_is_thread_alive(th, &exit_val);
1261  KMP_YIELD(TCR_4(__kmp_nth) > __kmp_avail_proc);
1262  KMP_YIELD_SPIN(spins);
1263  } while (exit_val == STILL_ACTIVE && TCR_4(th->th.th_info.ds.ds_alive));
1264 #if USE_ITT_BUILD
1265  if (exit_val == STILL_ACTIVE) {
1266  KMP_FSYNC_CANCEL(obj);
1267  } else {
1268  KMP_FSYNC_SPIN_ACQUIRED(obj);
1269  }
1270 #endif /* USE_ITT_BUILD */
1271  }
1272 
1273  __kmp_free_handle(th->th.th_info.ds.ds_thread);
1274 
1275  /* NOTE: The ExitProcess(code) system call causes all threads to Terminate
1276  with a exit_val = code. Because of this we can not rely on exit_val having
1277  any particular value. */
1278  if (exit_val == STILL_ACTIVE) {
1279  KA_TRACE(1, ("__kmp_reap_common: thread still active.\n"));
1280  } else if ((void *)exit_val != (void *)th) {
1281  KA_TRACE(1, ("__kmp_reap_common: ExitProcess / TerminateThread used?\n"));
1282  }
1283 
1284  KA_TRACE(10,
1285  ("__kmp_reap_common: done reaping (%d), handle = %" KMP_UINTPTR_SPEC
1286  "\n",
1287  th->th.th_info.ds.ds_gtid, th->th.th_info.ds.ds_thread));
1288 
1289  th->th.th_info.ds.ds_thread = 0;
1290  th->th.th_info.ds.ds_tid = KMP_GTID_DNE;
1291  th->th.th_info.ds.ds_gtid = KMP_GTID_DNE;
1292  th->th.th_info.ds.ds_thread_id = 0;
1293 
1294  KMP_MB(); /* Flush all pending memory write invalidates. */
1295 }
1296 
1297 #if KMP_USE_MONITOR
1298 void __kmp_reap_monitor(kmp_info_t *th) {
1299  int status;
1300 
1301  KA_TRACE(10, ("__kmp_reap_monitor: try to reap %p\n",
1302  (void *)th->th.th_info.ds.ds_thread));
1303 
1304  // If monitor has been created, its tid and gtid should be KMP_GTID_MONITOR.
1305  // If both tid and gtid are 0, it means the monitor did not ever start.
1306  // If both tid and gtid are KMP_GTID_DNE, the monitor has been shut down.
1307  KMP_DEBUG_ASSERT(th->th.th_info.ds.ds_tid == th->th.th_info.ds.ds_gtid);
1308  if (th->th.th_info.ds.ds_gtid != KMP_GTID_MONITOR) {
1309  KA_TRACE(10, ("__kmp_reap_monitor: monitor did not start, returning\n"));
1310  return;
1311  }
1312 
1313  KMP_MB(); /* Flush all pending memory write invalidates. */
1314 
1315  status = SetEvent(__kmp_monitor_ev);
1316  if (status == FALSE) {
1317  DWORD error = GetLastError();
1318  __kmp_fatal(KMP_MSG(CantSetEvent), KMP_ERR(error), __kmp_msg_null);
1319  }
1320  KA_TRACE(10, ("__kmp_reap_monitor: reaping thread (%d)\n",
1321  th->th.th_info.ds.ds_gtid));
1322  __kmp_reap_common(th);
1323 
1324  __kmp_free_handle(__kmp_monitor_ev);
1325 
1326  KMP_MB(); /* Flush all pending memory write invalidates. */
1327 }
1328 #endif
1329 
1330 void __kmp_reap_worker(kmp_info_t *th) {
1331  KA_TRACE(10, ("__kmp_reap_worker: reaping thread (%d)\n",
1332  th->th.th_info.ds.ds_gtid));
1333  __kmp_reap_common(th);
1334 }
1335 
1336 #if KMP_HANDLE_SIGNALS
1337 
1338 static void __kmp_team_handler(int signo) {
1339  if (__kmp_global.g.g_abort == 0) {
1340  // Stage 1 signal handler, let's shut down all of the threads.
1341  if (__kmp_debug_buf) {
1342  __kmp_dump_debug_buffer();
1343  }
1344  KMP_MB(); // Flush all pending memory write invalidates.
1345  TCW_4(__kmp_global.g.g_abort, signo);
1346  KMP_MB(); // Flush all pending memory write invalidates.
1347  TCW_4(__kmp_global.g.g_done, TRUE);
1348  KMP_MB(); // Flush all pending memory write invalidates.
1349  }
1350 } // __kmp_team_handler
1351 
1352 static sig_func_t __kmp_signal(int signum, sig_func_t handler) {
1353  sig_func_t old = signal(signum, handler);
1354  if (old == SIG_ERR) {
1355  int error = errno;
1356  __kmp_fatal(KMP_MSG(FunctionError, "signal"), KMP_ERR(error),
1357  __kmp_msg_null);
1358  }
1359  return old;
1360 }
1361 
1362 static void __kmp_install_one_handler(int sig, sig_func_t handler,
1363  int parallel_init) {
1364  sig_func_t old;
1365  KMP_MB(); /* Flush all pending memory write invalidates. */
1366  KB_TRACE(60, ("__kmp_install_one_handler: called: sig=%d\n", sig));
1367  if (parallel_init) {
1368  old = __kmp_signal(sig, handler);
1369  // SIG_DFL on Windows* OS in NULL or 0.
1370  if (old == __kmp_sighldrs[sig]) {
1371  __kmp_siginstalled[sig] = 1;
1372  } else { // Restore/keep user's handler if one previously installed.
1373  old = __kmp_signal(sig, old);
1374  }
1375  } else {
1376  // Save initial/system signal handlers to see if user handlers installed.
1377  // 2009-09-23: It is a dead code. On Windows* OS __kmp_install_signals
1378  // called once with parallel_init == TRUE.
1379  old = __kmp_signal(sig, SIG_DFL);
1380  __kmp_sighldrs[sig] = old;
1381  __kmp_signal(sig, old);
1382  }
1383  KMP_MB(); /* Flush all pending memory write invalidates. */
1384 } // __kmp_install_one_handler
1385 
1386 static void __kmp_remove_one_handler(int sig) {
1387  if (__kmp_siginstalled[sig]) {
1388  sig_func_t old;
1389  KMP_MB(); // Flush all pending memory write invalidates.
1390  KB_TRACE(60, ("__kmp_remove_one_handler: called: sig=%d\n", sig));
1391  old = __kmp_signal(sig, __kmp_sighldrs[sig]);
1392  if (old != __kmp_team_handler) {
1393  KB_TRACE(10, ("__kmp_remove_one_handler: oops, not our handler, "
1394  "restoring: sig=%d\n",
1395  sig));
1396  old = __kmp_signal(sig, old);
1397  }
1398  __kmp_sighldrs[sig] = NULL;
1399  __kmp_siginstalled[sig] = 0;
1400  KMP_MB(); // Flush all pending memory write invalidates.
1401  }
1402 } // __kmp_remove_one_handler
1403 
1404 void __kmp_install_signals(int parallel_init) {
1405  KB_TRACE(10, ("__kmp_install_signals: called\n"));
1406  if (!__kmp_handle_signals) {
1407  KB_TRACE(10, ("__kmp_install_signals: KMP_HANDLE_SIGNALS is false - "
1408  "handlers not installed\n"));
1409  return;
1410  }
1411  __kmp_install_one_handler(SIGINT, __kmp_team_handler, parallel_init);
1412  __kmp_install_one_handler(SIGILL, __kmp_team_handler, parallel_init);
1413  __kmp_install_one_handler(SIGABRT, __kmp_team_handler, parallel_init);
1414  __kmp_install_one_handler(SIGFPE, __kmp_team_handler, parallel_init);
1415  __kmp_install_one_handler(SIGSEGV, __kmp_team_handler, parallel_init);
1416  __kmp_install_one_handler(SIGTERM, __kmp_team_handler, parallel_init);
1417 } // __kmp_install_signals
1418 
1419 void __kmp_remove_signals(void) {
1420  int sig;
1421  KB_TRACE(10, ("__kmp_remove_signals: called\n"));
1422  for (sig = 1; sig < NSIG; ++sig) {
1423  __kmp_remove_one_handler(sig);
1424  }
1425 } // __kmp_remove_signals
1426 
1427 #endif // KMP_HANDLE_SIGNALS
1428 
1429 /* Put the thread to sleep for a time period */
1430 void __kmp_thread_sleep(int millis) {
1431  DWORD status;
1432 
1433  status = SleepEx((DWORD)millis, FALSE);
1434  if (status) {
1435  DWORD error = GetLastError();
1436  __kmp_fatal(KMP_MSG(FunctionError, "SleepEx()"), KMP_ERR(error),
1437  __kmp_msg_null);
1438  }
1439 }
1440 
1441 // Determine whether the given address is mapped into the current address space.
1442 int __kmp_is_address_mapped(void *addr) {
1443  DWORD status;
1444  MEMORY_BASIC_INFORMATION lpBuffer;
1445  SIZE_T dwLength;
1446 
1447  dwLength = sizeof(MEMORY_BASIC_INFORMATION);
1448 
1449  status = VirtualQuery(addr, &lpBuffer, dwLength);
1450 
1451  return !(((lpBuffer.State == MEM_RESERVE) || (lpBuffer.State == MEM_FREE)) ||
1452  ((lpBuffer.Protect == PAGE_NOACCESS) ||
1453  (lpBuffer.Protect == PAGE_EXECUTE)));
1454 }
1455 
1456 kmp_uint64 __kmp_hardware_timestamp(void) {
1457  kmp_uint64 r = 0;
1458 
1459  QueryPerformanceCounter((LARGE_INTEGER *)&r);
1460  return r;
1461 }
1462 
1463 /* Free handle and check the error code */
1464 void __kmp_free_handle(kmp_thread_t tHandle) {
1465  /* called with parameter type HANDLE also, thus suppose kmp_thread_t defined
1466  * as HANDLE */
1467  BOOL rc;
1468  rc = CloseHandle(tHandle);
1469  if (!rc) {
1470  DWORD error = GetLastError();
1471  __kmp_fatal(KMP_MSG(CantCloseHandle), KMP_ERR(error), __kmp_msg_null);
1472  }
1473 }
1474 
1475 int __kmp_get_load_balance(int max) {
1476  static ULONG glb_buff_size = 100 * 1024;
1477 
1478  // Saved count of the running threads for the thread balance algortihm
1479  static int glb_running_threads = 0;
1480  static double glb_call_time = 0; /* Thread balance algorithm call time */
1481 
1482  int running_threads = 0; // Number of running threads in the system.
1483  NTSTATUS status = 0;
1484  ULONG buff_size = 0;
1485  ULONG info_size = 0;
1486  void *buffer = NULL;
1487  PSYSTEM_PROCESS_INFORMATION spi = NULL;
1488  int first_time = 1;
1489 
1490  double call_time = 0.0; // start, finish;
1491 
1492  __kmp_elapsed(&call_time);
1493 
1494  if (glb_call_time &&
1495  (call_time - glb_call_time < __kmp_load_balance_interval)) {
1496  running_threads = glb_running_threads;
1497  goto finish;
1498  }
1499  glb_call_time = call_time;
1500 
1501  // Do not spend time on running algorithm if we have a permanent error.
1502  if (NtQuerySystemInformation == NULL) {
1503  running_threads = -1;
1504  goto finish;
1505  }
1506 
1507  if (max <= 0) {
1508  max = INT_MAX;
1509  }
1510 
1511  do {
1512 
1513  if (first_time) {
1514  buff_size = glb_buff_size;
1515  } else {
1516  buff_size = 2 * buff_size;
1517  }
1518 
1519  buffer = KMP_INTERNAL_REALLOC(buffer, buff_size);
1520  if (buffer == NULL) {
1521  running_threads = -1;
1522  goto finish;
1523  }
1524  status = NtQuerySystemInformation(SystemProcessInformation, buffer,
1525  buff_size, &info_size);
1526  first_time = 0;
1527 
1528  } while (status == STATUS_INFO_LENGTH_MISMATCH);
1529  glb_buff_size = buff_size;
1530 
1531 #define CHECK(cond) \
1532  { \
1533  KMP_DEBUG_ASSERT(cond); \
1534  if (!(cond)) { \
1535  running_threads = -1; \
1536  goto finish; \
1537  } \
1538  }
1539 
1540  CHECK(buff_size >= info_size);
1541  spi = PSYSTEM_PROCESS_INFORMATION(buffer);
1542  for (;;) {
1543  ptrdiff_t offset = uintptr_t(spi) - uintptr_t(buffer);
1544  CHECK(0 <= offset &&
1545  offset + sizeof(SYSTEM_PROCESS_INFORMATION) < info_size);
1546  HANDLE pid = spi->ProcessId;
1547  ULONG num = spi->NumberOfThreads;
1548  CHECK(num >= 1);
1549  size_t spi_size =
1550  sizeof(SYSTEM_PROCESS_INFORMATION) + sizeof(SYSTEM_THREAD) * (num - 1);
1551  CHECK(offset + spi_size <
1552  info_size); // Make sure process info record fits the buffer.
1553  if (spi->NextEntryOffset != 0) {
1554  CHECK(spi_size <=
1555  spi->NextEntryOffset); // And do not overlap with the next record.
1556  }
1557  // pid == 0 corresponds to the System Idle Process. It always has running
1558  // threads on all cores. So, we don't consider the running threads of this
1559  // process.
1560  if (pid != 0) {
1561  for (int i = 0; i < num; ++i) {
1562  THREAD_STATE state = spi->Threads[i].State;
1563  // Count threads that have Ready or Running state.
1564  // !!! TODO: Why comment does not match the code???
1565  if (state == StateRunning) {
1566  ++running_threads;
1567  // Stop counting running threads if the number is already greater than
1568  // the number of available cores
1569  if (running_threads >= max) {
1570  goto finish;
1571  }
1572  }
1573  }
1574  }
1575  if (spi->NextEntryOffset == 0) {
1576  break;
1577  }
1578  spi = PSYSTEM_PROCESS_INFORMATION(uintptr_t(spi) + spi->NextEntryOffset);
1579  }
1580 
1581 #undef CHECK
1582 
1583 finish: // Clean up and exit.
1584 
1585  if (buffer != NULL) {
1586  KMP_INTERNAL_FREE(buffer);
1587  }
1588 
1589  glb_running_threads = running_threads;
1590 
1591  return running_threads;
1592 } //__kmp_get_load_balance()