14 #include "kmp_affinity.h"
15 #include "kmp_atomic.h"
16 #include "kmp_environment.h"
17 #include "kmp_error.h"
21 #include "kmp_settings.h"
22 #include "kmp_stats.h"
24 #include "kmp_wait_release.h"
25 #include "kmp_wrapper_getpid.h"
26 #include "kmp_dispatch.h"
27 #if KMP_USE_HIER_SCHED
28 #include "kmp_dispatch_hier.h"
32 #include "ompt-specific.h"
35 #include "ompd-specific.h"
38 #if OMP_PROFILING_SUPPORT
39 #include "llvm/Support/TimeProfiler.h"
40 static char *ProfileTraceFile =
nullptr;
44 #define KMP_USE_PRCTL 0
50 #include "tsan_annotations.h"
61 #if defined(KMP_GOMP_COMPAT)
62 char const __kmp_version_alt_comp[] =
63 KMP_VERSION_PREFIX
"alternative compiler support: yes";
66 char const __kmp_version_omp_api[] =
67 KMP_VERSION_PREFIX
"API version: 5.0 (201611)";
70 char const __kmp_version_lock[] =
71 KMP_VERSION_PREFIX
"lock type: run time selectable";
74 #define KMP_MIN(x, y) ((x) < (y) ? (x) : (y))
79 kmp_info_t __kmp_monitor;
84 void __kmp_cleanup(
void);
86 static void __kmp_initialize_info(kmp_info_t *, kmp_team_t *,
int tid,
88 static void __kmp_initialize_team(kmp_team_t *team,
int new_nproc,
89 kmp_internal_control_t *new_icvs,
91 #if KMP_AFFINITY_SUPPORTED
92 static void __kmp_partition_places(kmp_team_t *team,
93 int update_master_only = 0);
95 static void __kmp_do_serial_initialize(
void);
96 void __kmp_fork_barrier(
int gtid,
int tid);
97 void __kmp_join_barrier(
int gtid);
98 void __kmp_setup_icv_copy(kmp_team_t *team,
int new_nproc,
99 kmp_internal_control_t *new_icvs,
ident_t *loc);
101 #ifdef USE_LOAD_BALANCE
102 static int __kmp_load_balance_nproc(kmp_root_t *root,
int set_nproc);
105 static int __kmp_expand_threads(
int nNeed);
107 static int __kmp_unregister_root_other_thread(
int gtid);
109 static void __kmp_reap_thread(kmp_info_t *thread,
int is_root);
110 kmp_info_t *__kmp_thread_pool_insert_pt = NULL;
112 void __kmp_resize_dist_barrier(kmp_team_t *team,
int old_nthreads,
114 void __kmp_add_threads_to_team(kmp_team_t *team,
int new_nthreads);
119 int __kmp_get_global_thread_id() {
121 kmp_info_t **other_threads;
129 (
"*** __kmp_get_global_thread_id: entering, nproc=%d all_nproc=%d\n",
130 __kmp_nth, __kmp_all_nth));
137 if (!TCR_4(__kmp_init_gtid))
140 #ifdef KMP_TDATA_GTID
141 if (TCR_4(__kmp_gtid_mode) >= 3) {
142 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id: using TDATA\n"));
146 if (TCR_4(__kmp_gtid_mode) >= 2) {
147 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id: using keyed TLS\n"));
148 return __kmp_gtid_get_specific();
150 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id: using internal alg.\n"));
152 stack_addr = (
char *)&stack_data;
153 other_threads = __kmp_threads;
166 for (i = 0; i < __kmp_threads_capacity; i++) {
168 kmp_info_t *thr = (kmp_info_t *)TCR_SYNC_PTR(other_threads[i]);
172 stack_size = (size_t)TCR_PTR(thr->th.th_info.ds.ds_stacksize);
173 stack_base = (
char *)TCR_PTR(thr->th.th_info.ds.ds_stackbase);
177 if (stack_addr <= stack_base) {
178 size_t stack_diff = stack_base - stack_addr;
180 if (stack_diff <= stack_size) {
183 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == i);
191 (
"*** __kmp_get_global_thread_id: internal alg. failed to find "
192 "thread, using TLS\n"));
193 i = __kmp_gtid_get_specific();
203 if (!TCR_4(other_threads[i]->th.th_info.ds.ds_stackgrow)) {
204 KMP_FATAL(StackOverflow, i);
207 stack_base = (
char *)other_threads[i]->th.th_info.ds.ds_stackbase;
208 if (stack_addr > stack_base) {
209 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stackbase, stack_addr);
210 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,
211 other_threads[i]->th.th_info.ds.ds_stacksize + stack_addr -
214 TCW_PTR(other_threads[i]->th.th_info.ds.ds_stacksize,
215 stack_base - stack_addr);
219 if (__kmp_storage_map) {
220 char *stack_end = (
char *)other_threads[i]->th.th_info.ds.ds_stackbase;
221 char *stack_beg = stack_end - other_threads[i]->th.th_info.ds.ds_stacksize;
222 __kmp_print_storage_map_gtid(i, stack_beg, stack_end,
223 other_threads[i]->th.th_info.ds.ds_stacksize,
224 "th_%d stack (refinement)", i);
229 int __kmp_get_global_thread_id_reg() {
232 if (!__kmp_init_serial) {
235 #ifdef KMP_TDATA_GTID
236 if (TCR_4(__kmp_gtid_mode) >= 3) {
237 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id_reg: using TDATA\n"));
241 if (TCR_4(__kmp_gtid_mode) >= 2) {
242 KA_TRACE(1000, (
"*** __kmp_get_global_thread_id_reg: using keyed TLS\n"));
243 gtid = __kmp_gtid_get_specific();
246 (
"*** __kmp_get_global_thread_id_reg: using internal alg.\n"));
247 gtid = __kmp_get_global_thread_id();
251 if (gtid == KMP_GTID_DNE) {
253 (
"__kmp_get_global_thread_id_reg: Encountered new root thread. "
254 "Registering a new gtid.\n"));
255 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
256 if (!__kmp_init_serial) {
257 __kmp_do_serial_initialize();
258 gtid = __kmp_gtid_get_specific();
260 gtid = __kmp_register_root(FALSE);
262 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
266 KMP_DEBUG_ASSERT(gtid >= 0);
272 void __kmp_check_stack_overlap(kmp_info_t *th) {
274 char *stack_beg = NULL;
275 char *stack_end = NULL;
278 KA_TRACE(10, (
"__kmp_check_stack_overlap: called\n"));
279 if (__kmp_storage_map) {
280 stack_end = (
char *)th->th.th_info.ds.ds_stackbase;
281 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
283 gtid = __kmp_gtid_from_thread(th);
285 if (gtid == KMP_GTID_MONITOR) {
286 __kmp_print_storage_map_gtid(
287 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
288 "th_%s stack (%s)",
"mon",
289 (th->th.th_info.ds.ds_stackgrow) ?
"initial" :
"actual");
291 __kmp_print_storage_map_gtid(
292 gtid, stack_beg, stack_end, th->th.th_info.ds.ds_stacksize,
293 "th_%d stack (%s)", gtid,
294 (th->th.th_info.ds.ds_stackgrow) ?
"initial" :
"actual");
300 gtid = __kmp_gtid_from_thread(th);
301 if (__kmp_env_checks == TRUE && !KMP_UBER_GTID(gtid)) {
303 (
"__kmp_check_stack_overlap: performing extensive checking\n"));
304 if (stack_beg == NULL) {
305 stack_end = (
char *)th->th.th_info.ds.ds_stackbase;
306 stack_beg = stack_end - th->th.th_info.ds.ds_stacksize;
309 for (f = 0; f < __kmp_threads_capacity; f++) {
310 kmp_info_t *f_th = (kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[f]);
312 if (f_th && f_th != th) {
313 char *other_stack_end =
314 (
char *)TCR_PTR(f_th->th.th_info.ds.ds_stackbase);
315 char *other_stack_beg =
316 other_stack_end - (size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize);
317 if ((stack_beg > other_stack_beg && stack_beg < other_stack_end) ||
318 (stack_end > other_stack_beg && stack_end < other_stack_end)) {
321 if (__kmp_storage_map)
322 __kmp_print_storage_map_gtid(
323 -1, other_stack_beg, other_stack_end,
324 (
size_t)TCR_PTR(f_th->th.th_info.ds.ds_stacksize),
325 "th_%d stack (overlapped)", __kmp_gtid_from_thread(f_th));
327 __kmp_fatal(KMP_MSG(StackOverlap), KMP_HNT(ChangeStackLimit),
333 KA_TRACE(10, (
"__kmp_check_stack_overlap: returning\n"));
338 void __kmp_infinite_loop(
void) {
339 static int done = FALSE;
346 #define MAX_MESSAGE 512
348 void __kmp_print_storage_map_gtid(
int gtid,
void *p1,
void *p2,
size_t size,
349 char const *format, ...) {
350 char buffer[MAX_MESSAGE];
353 va_start(ap, format);
354 KMP_SNPRINTF(buffer,
sizeof(buffer),
"OMP storage map: %p %p%8lu %s\n", p1,
355 p2, (
unsigned long)size, format);
356 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
357 __kmp_vprintf(kmp_err, buffer, ap);
358 #if KMP_PRINT_DATA_PLACEMENT
361 if (p1 <= p2 && (
char *)p2 - (
char *)p1 == size) {
362 if (__kmp_storage_map_verbose) {
363 node = __kmp_get_host_node(p1);
365 __kmp_storage_map_verbose = FALSE;
369 int localProc = __kmp_get_cpu_from_gtid(gtid);
371 const int page_size = KMP_GET_PAGE_SIZE();
373 p1 = (
void *)((
size_t)p1 & ~((size_t)page_size - 1));
374 p2 = (
void *)(((
size_t)p2 - 1) & ~((
size_t)page_size - 1));
376 __kmp_printf_no_lock(
" GTID %d localNode %d\n", gtid,
379 __kmp_printf_no_lock(
" GTID %d\n", gtid);
388 (
char *)p1 += page_size;
389 }
while (p1 <= p2 && (node = __kmp_get_host_node(p1)) == lastNode);
390 __kmp_printf_no_lock(
" %p-%p memNode %d\n", last, (
char *)p1 - 1,
394 __kmp_printf_no_lock(
" %p-%p memNode %d\n", p1,
395 (
char *)p1 + (page_size - 1),
396 __kmp_get_host_node(p1));
398 __kmp_printf_no_lock(
" %p-%p memNode %d\n", p2,
399 (
char *)p2 + (page_size - 1),
400 __kmp_get_host_node(p2));
406 __kmp_printf_no_lock(
" %s\n", KMP_I18N_STR(StorageMapWarning));
409 __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
412 void __kmp_warn(
char const *format, ...) {
413 char buffer[MAX_MESSAGE];
416 if (__kmp_generate_warnings == kmp_warnings_off) {
420 va_start(ap, format);
422 KMP_SNPRINTF(buffer,
sizeof(buffer),
"OMP warning: %s\n", format);
423 __kmp_acquire_bootstrap_lock(&__kmp_stdio_lock);
424 __kmp_vprintf(kmp_err, buffer, ap);
425 __kmp_release_bootstrap_lock(&__kmp_stdio_lock);
430 void __kmp_abort_process() {
432 __kmp_acquire_bootstrap_lock(&__kmp_exit_lock);
434 if (__kmp_debug_buf) {
435 __kmp_dump_debug_buffer();
438 if (KMP_OS_WINDOWS) {
441 __kmp_global.g.g_abort = SIGABRT;
455 __kmp_unregister_library();
459 __kmp_infinite_loop();
460 __kmp_release_bootstrap_lock(&__kmp_exit_lock);
464 void __kmp_abort_thread(
void) {
467 __kmp_infinite_loop();
473 static void __kmp_print_thread_storage_map(kmp_info_t *thr,
int gtid) {
474 __kmp_print_storage_map_gtid(gtid, thr, thr + 1,
sizeof(kmp_info_t),
"th_%d",
477 __kmp_print_storage_map_gtid(gtid, &thr->th.th_info, &thr->th.th_team,
478 sizeof(kmp_desc_t),
"th_%d.th_info", gtid);
480 __kmp_print_storage_map_gtid(gtid, &thr->th.th_local, &thr->th.th_pri_head,
481 sizeof(kmp_local_t),
"th_%d.th_local", gtid);
483 __kmp_print_storage_map_gtid(
484 gtid, &thr->th.th_bar[0], &thr->th.th_bar[bs_last_barrier],
485 sizeof(kmp_balign_t) * bs_last_barrier,
"th_%d.th_bar", gtid);
487 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_plain_barrier],
488 &thr->th.th_bar[bs_plain_barrier + 1],
489 sizeof(kmp_balign_t),
"th_%d.th_bar[plain]",
492 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_forkjoin_barrier],
493 &thr->th.th_bar[bs_forkjoin_barrier + 1],
494 sizeof(kmp_balign_t),
"th_%d.th_bar[forkjoin]",
497 #if KMP_FAST_REDUCTION_BARRIER
498 __kmp_print_storage_map_gtid(gtid, &thr->th.th_bar[bs_reduction_barrier],
499 &thr->th.th_bar[bs_reduction_barrier + 1],
500 sizeof(kmp_balign_t),
"th_%d.th_bar[reduction]",
508 static void __kmp_print_team_storage_map(
const char *header, kmp_team_t *team,
509 int team_id,
int num_thr) {
510 int num_disp_buff = team->t.t_max_nproc > 1 ? __kmp_dispatch_num_buffers : 2;
511 __kmp_print_storage_map_gtid(-1, team, team + 1,
sizeof(kmp_team_t),
"%s_%d",
514 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[0],
515 &team->t.t_bar[bs_last_barrier],
516 sizeof(kmp_balign_team_t) * bs_last_barrier,
517 "%s_%d.t_bar", header, team_id);
519 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_plain_barrier],
520 &team->t.t_bar[bs_plain_barrier + 1],
521 sizeof(kmp_balign_team_t),
"%s_%d.t_bar[plain]",
524 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_forkjoin_barrier],
525 &team->t.t_bar[bs_forkjoin_barrier + 1],
526 sizeof(kmp_balign_team_t),
527 "%s_%d.t_bar[forkjoin]", header, team_id);
529 #if KMP_FAST_REDUCTION_BARRIER
530 __kmp_print_storage_map_gtid(-1, &team->t.t_bar[bs_reduction_barrier],
531 &team->t.t_bar[bs_reduction_barrier + 1],
532 sizeof(kmp_balign_team_t),
533 "%s_%d.t_bar[reduction]", header, team_id);
536 __kmp_print_storage_map_gtid(
537 -1, &team->t.t_dispatch[0], &team->t.t_dispatch[num_thr],
538 sizeof(kmp_disp_t) * num_thr,
"%s_%d.t_dispatch", header, team_id);
540 __kmp_print_storage_map_gtid(
541 -1, &team->t.t_threads[0], &team->t.t_threads[num_thr],
542 sizeof(kmp_info_t *) * num_thr,
"%s_%d.t_threads", header, team_id);
544 __kmp_print_storage_map_gtid(-1, &team->t.t_disp_buffer[0],
545 &team->t.t_disp_buffer[num_disp_buff],
546 sizeof(dispatch_shared_info_t) * num_disp_buff,
547 "%s_%d.t_disp_buffer", header, team_id);
550 static void __kmp_init_allocator() {
551 __kmp_init_memkind();
552 __kmp_init_target_mem();
554 static void __kmp_fini_allocator() { __kmp_fini_memkind(); }
561 BOOL WINAPI DllMain(HINSTANCE hInstDLL, DWORD fdwReason, LPVOID lpReserved) {
566 case DLL_PROCESS_ATTACH:
567 KA_TRACE(10, (
"DllMain: PROCESS_ATTACH\n"));
571 case DLL_PROCESS_DETACH:
572 KA_TRACE(10, (
"DllMain: PROCESS_DETACH T#%d\n", __kmp_gtid_get_specific()));
585 if (lpReserved == NULL)
586 __kmp_internal_end_library(__kmp_gtid_get_specific());
590 case DLL_THREAD_ATTACH:
591 KA_TRACE(10, (
"DllMain: THREAD_ATTACH\n"));
597 case DLL_THREAD_DETACH:
598 KA_TRACE(10, (
"DllMain: THREAD_DETACH T#%d\n", __kmp_gtid_get_specific()));
600 __kmp_internal_end_thread(__kmp_gtid_get_specific());
611 void __kmp_parallel_deo(
int *gtid_ref,
int *cid_ref,
ident_t *loc_ref) {
612 int gtid = *gtid_ref;
613 #ifdef BUILD_PARALLEL_ORDERED
614 kmp_team_t *team = __kmp_team_from_gtid(gtid);
617 if (__kmp_env_consistency_check) {
618 if (__kmp_threads[gtid]->th.th_root->r.r_active)
619 #if KMP_USE_DYNAMIC_LOCK
620 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL, 0);
622 __kmp_push_sync(gtid, ct_ordered_in_parallel, loc_ref, NULL);
625 #ifdef BUILD_PARALLEL_ORDERED
626 if (!team->t.t_serialized) {
628 KMP_WAIT(&team->t.t_ordered.dt.t_value, __kmp_tid_from_gtid(gtid), KMP_EQ,
636 void __kmp_parallel_dxo(
int *gtid_ref,
int *cid_ref,
ident_t *loc_ref) {
637 int gtid = *gtid_ref;
638 #ifdef BUILD_PARALLEL_ORDERED
639 int tid = __kmp_tid_from_gtid(gtid);
640 kmp_team_t *team = __kmp_team_from_gtid(gtid);
643 if (__kmp_env_consistency_check) {
644 if (__kmp_threads[gtid]->th.th_root->r.r_active)
645 __kmp_pop_sync(gtid, ct_ordered_in_parallel, loc_ref);
647 #ifdef BUILD_PARALLEL_ORDERED
648 if (!team->t.t_serialized) {
653 team->t.t_ordered.dt.t_value = ((tid + 1) % team->t.t_nproc);
663 int __kmp_enter_single(
int gtid,
ident_t *id_ref,
int push_ws) {
668 if (!TCR_4(__kmp_init_parallel))
669 __kmp_parallel_initialize();
670 __kmp_resume_if_soft_paused();
672 th = __kmp_threads[gtid];
673 team = th->th.th_team;
676 th->th.th_ident = id_ref;
678 if (team->t.t_serialized) {
681 kmp_int32 old_this = th->th.th_local.this_construct;
683 ++th->th.th_local.this_construct;
687 if (team->t.t_construct == old_this) {
688 status = __kmp_atomic_compare_store_acq(&team->t.t_construct, old_this,
689 th->th.th_local.this_construct);
692 if (__itt_metadata_add_ptr && __kmp_forkjoin_frames_mode == 3 &&
693 KMP_MASTER_GTID(gtid) && th->th.th_teams_microtask == NULL &&
694 team->t.t_active_level == 1) {
696 __kmp_itt_metadata_single(id_ref);
701 if (__kmp_env_consistency_check) {
702 if (status && push_ws) {
703 __kmp_push_workshare(gtid, ct_psingle, id_ref);
705 __kmp_check_workshare(gtid, ct_psingle, id_ref);
710 __kmp_itt_single_start(gtid);
716 void __kmp_exit_single(
int gtid) {
718 __kmp_itt_single_end(gtid);
720 if (__kmp_env_consistency_check)
721 __kmp_pop_workshare(gtid, ct_psingle, NULL);
730 static int __kmp_reserve_threads(kmp_root_t *root, kmp_team_t *parent_team,
731 int master_tid,
int set_nthreads,
735 KMP_DEBUG_ASSERT(__kmp_init_serial);
736 KMP_DEBUG_ASSERT(root && parent_team);
737 kmp_info_t *this_thr = parent_team->t.t_threads[master_tid];
741 new_nthreads = set_nthreads;
742 if (!get__dynamic_2(parent_team, master_tid)) {
745 #ifdef USE_LOAD_BALANCE
746 else if (__kmp_global.g.g_dynamic_mode == dynamic_load_balance) {
747 new_nthreads = __kmp_load_balance_nproc(root, set_nthreads);
748 if (new_nthreads == 1) {
749 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d load balance reduced "
750 "reservation to 1 thread\n",
754 if (new_nthreads < set_nthreads) {
755 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d load balance reduced "
756 "reservation to %d threads\n",
757 master_tid, new_nthreads));
761 else if (__kmp_global.g.g_dynamic_mode == dynamic_thread_limit) {
762 new_nthreads = __kmp_avail_proc - __kmp_nth +
763 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
764 if (new_nthreads <= 1) {
765 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d thread limit reduced "
766 "reservation to 1 thread\n",
770 if (new_nthreads < set_nthreads) {
771 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d thread limit reduced "
772 "reservation to %d threads\n",
773 master_tid, new_nthreads));
775 new_nthreads = set_nthreads;
777 }
else if (__kmp_global.g.g_dynamic_mode == dynamic_random) {
778 if (set_nthreads > 2) {
779 new_nthreads = __kmp_get_random(parent_team->t.t_threads[master_tid]);
780 new_nthreads = (new_nthreads % set_nthreads) + 1;
781 if (new_nthreads == 1) {
782 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d dynamic random reduced "
783 "reservation to 1 thread\n",
787 if (new_nthreads < set_nthreads) {
788 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d dynamic random reduced "
789 "reservation to %d threads\n",
790 master_tid, new_nthreads));
798 if (__kmp_nth + new_nthreads -
799 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
801 int tl_nthreads = __kmp_max_nth - __kmp_nth +
802 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
803 if (tl_nthreads <= 0) {
808 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
809 __kmp_reserve_warn = 1;
810 __kmp_msg(kmp_ms_warning,
811 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads),
812 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
814 if (tl_nthreads == 1) {
815 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT "
816 "reduced reservation to 1 thread\n",
820 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d KMP_DEVICE_THREAD_LIMIT reduced "
821 "reservation to %d threads\n",
822 master_tid, tl_nthreads));
823 new_nthreads = tl_nthreads;
827 int cg_nthreads = this_thr->th.th_cg_roots->cg_nthreads;
828 int max_cg_threads = this_thr->th.th_cg_roots->cg_thread_limit;
829 if (cg_nthreads + new_nthreads -
830 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
832 int tl_nthreads = max_cg_threads - cg_nthreads +
833 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
834 if (tl_nthreads <= 0) {
839 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
840 __kmp_reserve_warn = 1;
841 __kmp_msg(kmp_ms_warning,
842 KMP_MSG(CantFormThrTeam, set_nthreads, tl_nthreads),
843 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
845 if (tl_nthreads == 1) {
846 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT "
847 "reduced reservation to 1 thread\n",
851 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d OMP_THREAD_LIMIT reduced "
852 "reservation to %d threads\n",
853 master_tid, tl_nthreads));
854 new_nthreads = tl_nthreads;
860 capacity = __kmp_threads_capacity;
861 if (TCR_PTR(__kmp_threads[0]) == NULL) {
867 if (__kmp_enable_hidden_helper && !TCR_4(__kmp_init_hidden_helper_threads)) {
868 capacity -= __kmp_hidden_helper_threads_num;
870 if (__kmp_nth + new_nthreads -
871 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) >
874 int slotsRequired = __kmp_nth + new_nthreads -
875 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc) -
877 int slotsAdded = __kmp_expand_threads(slotsRequired);
878 if (slotsAdded < slotsRequired) {
880 new_nthreads -= (slotsRequired - slotsAdded);
881 KMP_ASSERT(new_nthreads >= 1);
884 if (!get__dynamic_2(parent_team, master_tid) && (!__kmp_reserve_warn)) {
885 __kmp_reserve_warn = 1;
886 if (__kmp_tp_cached) {
887 __kmp_msg(kmp_ms_warning,
888 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads),
889 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity),
890 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null);
892 __kmp_msg(kmp_ms_warning,
893 KMP_MSG(CantFormThrTeam, set_nthreads, new_nthreads),
894 KMP_HNT(SystemLimitOnThreads), __kmp_msg_null);
901 if (new_nthreads == 1) {
903 (
"__kmp_reserve_threads: T#%d serializing team after reclaiming "
904 "dead roots and rechecking; requested %d threads\n",
905 __kmp_get_gtid(), set_nthreads));
907 KC_TRACE(10, (
"__kmp_reserve_threads: T#%d allocating %d threads; requested"
909 __kmp_get_gtid(), new_nthreads, set_nthreads));
918 static void __kmp_fork_team_threads(kmp_root_t *root, kmp_team_t *team,
919 kmp_info_t *master_th,
int master_gtid) {
923 KA_TRACE(10, (
"__kmp_fork_team_threads: new_nprocs = %d\n", team->t.t_nproc));
924 KMP_DEBUG_ASSERT(master_gtid == __kmp_get_gtid());
928 master_th->th.th_info.ds.ds_tid = 0;
929 master_th->th.th_team = team;
930 master_th->th.th_team_nproc = team->t.t_nproc;
931 master_th->th.th_team_master = master_th;
932 master_th->th.th_team_serialized = FALSE;
933 master_th->th.th_dispatch = &team->t.t_dispatch[0];
936 #if KMP_NESTED_HOT_TEAMS
938 kmp_hot_team_ptr_t *hot_teams = master_th->th.th_hot_teams;
941 int level = team->t.t_active_level - 1;
942 if (master_th->th.th_teams_microtask) {
943 if (master_th->th.th_teams_size.nteams > 1) {
947 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
948 master_th->th.th_teams_level == team->t.t_level) {
953 if (level < __kmp_hot_teams_max_level) {
954 if (hot_teams[level].hot_team) {
956 KMP_DEBUG_ASSERT(hot_teams[level].hot_team == team);
960 hot_teams[level].hot_team = team;
961 hot_teams[level].hot_team_nth = team->t.t_nproc;
968 use_hot_team = team == root->r.r_hot_team;
973 team->t.t_threads[0] = master_th;
974 __kmp_initialize_info(master_th, team, 0, master_gtid);
977 for (i = 1; i < team->t.t_nproc; i++) {
980 kmp_info_t *thr = __kmp_allocate_thread(root, team, i);
981 team->t.t_threads[i] = thr;
982 KMP_DEBUG_ASSERT(thr);
983 KMP_DEBUG_ASSERT(thr->th.th_team == team);
985 KA_TRACE(20, (
"__kmp_fork_team_threads: T#%d(%d:%d) init arrived "
986 "T#%d(%d:%d) join =%llu, plain=%llu\n",
987 __kmp_gtid_from_tid(0, team), team->t.t_id, 0,
988 __kmp_gtid_from_tid(i, team), team->t.t_id, i,
989 team->t.t_bar[bs_forkjoin_barrier].b_arrived,
990 team->t.t_bar[bs_plain_barrier].b_arrived));
991 thr->th.th_teams_microtask = master_th->th.th_teams_microtask;
992 thr->th.th_teams_level = master_th->th.th_teams_level;
993 thr->th.th_teams_size = master_th->th.th_teams_size;
996 kmp_balign_t *balign = team->t.t_threads[i]->th.th_bar;
997 for (b = 0; b < bs_last_barrier; ++b) {
998 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
999 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
1001 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
1007 #if KMP_AFFINITY_SUPPORTED
1008 __kmp_partition_places(team);
1012 if (__kmp_display_affinity && team->t.t_display_affinity != 1) {
1013 for (i = 0; i < team->t.t_nproc; i++) {
1014 kmp_info_t *thr = team->t.t_threads[i];
1015 if (thr->th.th_prev_num_threads != team->t.t_nproc ||
1016 thr->th.th_prev_level != team->t.t_level) {
1017 team->t.t_display_affinity = 1;
1026 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1030 inline static void propagateFPControl(kmp_team_t *team) {
1031 if (__kmp_inherit_fp_control) {
1032 kmp_int16 x87_fpu_control_word;
1036 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
1037 __kmp_store_mxcsr(&mxcsr);
1038 mxcsr &= KMP_X86_MXCSR_MASK;
1049 KMP_CHECK_UPDATE(team->t.t_x87_fpu_control_word, x87_fpu_control_word);
1050 KMP_CHECK_UPDATE(team->t.t_mxcsr, mxcsr);
1053 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, TRUE);
1057 KMP_CHECK_UPDATE(team->t.t_fp_control_saved, FALSE);
1063 inline static void updateHWFPControl(kmp_team_t *team) {
1064 if (__kmp_inherit_fp_control && team->t.t_fp_control_saved) {
1067 kmp_int16 x87_fpu_control_word;
1069 __kmp_store_x87_fpu_control_word(&x87_fpu_control_word);
1070 __kmp_store_mxcsr(&mxcsr);
1071 mxcsr &= KMP_X86_MXCSR_MASK;
1073 if (team->t.t_x87_fpu_control_word != x87_fpu_control_word) {
1074 __kmp_clear_x87_fpu_status_word();
1075 __kmp_load_x87_fpu_control_word(&team->t.t_x87_fpu_control_word);
1078 if (team->t.t_mxcsr != mxcsr) {
1079 __kmp_load_mxcsr(&team->t.t_mxcsr);
1084 #define propagateFPControl(x) ((void)0)
1085 #define updateHWFPControl(x) ((void)0)
1088 static void __kmp_alloc_argv_entries(
int argc, kmp_team_t *team,
1093 void __kmp_serialized_parallel(
ident_t *loc, kmp_int32 global_tid) {
1094 kmp_info_t *this_thr;
1095 kmp_team_t *serial_team;
1097 KC_TRACE(10, (
"__kmpc_serialized_parallel: called by T#%d\n", global_tid));
1104 if (!TCR_4(__kmp_init_parallel))
1105 __kmp_parallel_initialize();
1106 __kmp_resume_if_soft_paused();
1108 this_thr = __kmp_threads[global_tid];
1109 serial_team = this_thr->th.th_serial_team;
1112 KMP_DEBUG_ASSERT(serial_team);
1115 if (__kmp_tasking_mode != tskm_immediate_exec) {
1117 this_thr->th.th_task_team ==
1118 this_thr->th.th_team->t.t_task_team[this_thr->th.th_task_state]);
1119 KMP_DEBUG_ASSERT(serial_team->t.t_task_team[this_thr->th.th_task_state] ==
1121 KA_TRACE(20, (
"__kmpc_serialized_parallel: T#%d pushing task_team %p / "
1122 "team %p, new task_team = NULL\n",
1123 global_tid, this_thr->th.th_task_team, this_thr->th.th_team));
1124 this_thr->th.th_task_team = NULL;
1127 kmp_proc_bind_t proc_bind = this_thr->th.th_set_proc_bind;
1128 if (this_thr->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
1129 proc_bind = proc_bind_false;
1130 }
else if (proc_bind == proc_bind_default) {
1133 proc_bind = this_thr->th.th_current_task->td_icvs.proc_bind;
1136 this_thr->th.th_set_proc_bind = proc_bind_default;
1139 ompt_data_t ompt_parallel_data = ompt_data_none;
1140 void *codeptr = OMPT_LOAD_RETURN_ADDRESS(global_tid);
1141 if (ompt_enabled.enabled &&
1142 this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
1144 ompt_task_info_t *parent_task_info;
1145 parent_task_info = OMPT_CUR_TASK_INFO(this_thr);
1147 parent_task_info->frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0);
1148 if (ompt_enabled.ompt_callback_parallel_begin) {
1151 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)(
1152 &(parent_task_info->task_data), &(parent_task_info->frame),
1153 &ompt_parallel_data, team_size,
1154 ompt_parallel_invoker_program | ompt_parallel_team, codeptr);
1159 if (this_thr->th.th_team != serial_team) {
1161 int level = this_thr->th.th_team->t.t_level;
1163 if (serial_team->t.t_serialized) {
1166 kmp_team_t *new_team;
1168 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
1171 __kmp_allocate_team(this_thr->th.th_root, 1, 1,
1175 proc_bind, &this_thr->th.th_current_task->td_icvs,
1176 0 USE_NESTED_HOT_ARG(NULL));
1177 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1178 KMP_ASSERT(new_team);
1181 new_team->t.t_threads[0] = this_thr;
1182 new_team->t.t_parent = this_thr->th.th_team;
1183 serial_team = new_team;
1184 this_thr->th.th_serial_team = serial_team;
1188 (
"__kmpc_serialized_parallel: T#%d allocated new serial team %p\n",
1189 global_tid, serial_team));
1197 (
"__kmpc_serialized_parallel: T#%d reusing cached serial team %p\n",
1198 global_tid, serial_team));
1202 KMP_DEBUG_ASSERT(serial_team->t.t_threads);
1203 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
1204 KMP_DEBUG_ASSERT(this_thr->th.th_team != serial_team);
1205 serial_team->t.t_ident = loc;
1206 serial_team->t.t_serialized = 1;
1207 serial_team->t.t_nproc = 1;
1208 serial_team->t.t_parent = this_thr->th.th_team;
1209 serial_team->t.t_sched.sched = this_thr->th.th_team->t.t_sched.sched;
1210 this_thr->th.th_team = serial_team;
1211 serial_team->t.t_master_tid = this_thr->th.th_info.ds.ds_tid;
1213 KF_TRACE(10, (
"__kmpc_serialized_parallel: T#%d curtask=%p\n", global_tid,
1214 this_thr->th.th_current_task));
1215 KMP_ASSERT(this_thr->th.th_current_task->td_flags.executing == 1);
1216 this_thr->th.th_current_task->td_flags.executing = 0;
1218 __kmp_push_current_task_to_thread(this_thr, serial_team, 0);
1223 copy_icvs(&this_thr->th.th_current_task->td_icvs,
1224 &this_thr->th.th_current_task->td_parent->td_icvs);
1228 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
1229 this_thr->th.th_current_task->td_icvs.nproc =
1230 __kmp_nested_nth.nth[level + 1];
1233 if (__kmp_nested_proc_bind.used &&
1234 (level + 1 < __kmp_nested_proc_bind.used)) {
1235 this_thr->th.th_current_task->td_icvs.proc_bind =
1236 __kmp_nested_proc_bind.bind_types[level + 1];
1240 serial_team->t.t_pkfn = (microtask_t)(~0);
1242 this_thr->th.th_info.ds.ds_tid = 0;
1245 this_thr->th.th_team_nproc = 1;
1246 this_thr->th.th_team_master = this_thr;
1247 this_thr->th.th_team_serialized = 1;
1249 serial_team->t.t_level = serial_team->t.t_parent->t.t_level + 1;
1250 serial_team->t.t_active_level = serial_team->t.t_parent->t.t_active_level;
1251 serial_team->t.t_def_allocator = this_thr->th.th_def_allocator;
1253 propagateFPControl(serial_team);
1256 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);
1257 if (!serial_team->t.t_dispatch->th_disp_buffer) {
1258 serial_team->t.t_dispatch->th_disp_buffer =
1259 (dispatch_private_info_t *)__kmp_allocate(
1260 sizeof(dispatch_private_info_t));
1262 this_thr->th.th_dispatch = serial_team->t.t_dispatch;
1269 KMP_DEBUG_ASSERT(this_thr->th.th_team == serial_team);
1270 KMP_DEBUG_ASSERT(serial_team->t.t_threads);
1271 KMP_DEBUG_ASSERT(serial_team->t.t_threads[0] == this_thr);
1272 ++serial_team->t.t_serialized;
1273 this_thr->th.th_team_serialized = serial_team->t.t_serialized;
1276 int level = this_thr->th.th_team->t.t_level;
1279 if (__kmp_nested_nth.used && (level + 1 < __kmp_nested_nth.used)) {
1280 this_thr->th.th_current_task->td_icvs.nproc =
1281 __kmp_nested_nth.nth[level + 1];
1283 serial_team->t.t_level++;
1284 KF_TRACE(10, (
"__kmpc_serialized_parallel: T#%d increasing nesting level "
1285 "of serial team %p to %d\n",
1286 global_tid, serial_team, serial_team->t.t_level));
1289 KMP_DEBUG_ASSERT(serial_team->t.t_dispatch);
1291 dispatch_private_info_t *disp_buffer =
1292 (dispatch_private_info_t *)__kmp_allocate(
1293 sizeof(dispatch_private_info_t));
1294 disp_buffer->next = serial_team->t.t_dispatch->th_disp_buffer;
1295 serial_team->t.t_dispatch->th_disp_buffer = disp_buffer;
1297 this_thr->th.th_dispatch = serial_team->t.t_dispatch;
1301 KMP_CHECK_UPDATE(serial_team->t.t_cancel_request, cancel_noreq);
1305 if (__kmp_display_affinity) {
1306 if (this_thr->th.th_prev_level != serial_team->t.t_level ||
1307 this_thr->th.th_prev_num_threads != 1) {
1309 __kmp_aux_display_affinity(global_tid, NULL);
1310 this_thr->th.th_prev_level = serial_team->t.t_level;
1311 this_thr->th.th_prev_num_threads = 1;
1315 if (__kmp_env_consistency_check)
1316 __kmp_push_parallel(global_tid, NULL);
1318 serial_team->t.ompt_team_info.master_return_address = codeptr;
1319 if (ompt_enabled.enabled &&
1320 this_thr->th.ompt_thread_info.state != ompt_state_overhead) {
1321 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame.ptr =
1322 OMPT_GET_FRAME_ADDRESS(0);
1324 ompt_lw_taskteam_t lw_taskteam;
1325 __ompt_lw_taskteam_init(&lw_taskteam, this_thr, global_tid,
1326 &ompt_parallel_data, codeptr);
1328 __ompt_lw_taskteam_link(&lw_taskteam, this_thr, 1);
1332 if (ompt_enabled.ompt_callback_implicit_task) {
1333 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1334 ompt_scope_begin, OMPT_CUR_TEAM_DATA(this_thr),
1335 OMPT_CUR_TASK_DATA(this_thr), 1, __kmp_tid_from_gtid(global_tid),
1336 ompt_task_implicit);
1337 OMPT_CUR_TASK_INFO(this_thr)->thread_num =
1338 __kmp_tid_from_gtid(global_tid);
1342 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
1343 OMPT_CUR_TASK_INFO(this_thr)->frame.exit_frame.ptr =
1344 OMPT_GET_FRAME_ADDRESS(0);
1351 int __kmp_fork_call(
ident_t *loc,
int gtid,
1352 enum fork_context_e call_context,
1353 kmp_int32 argc, microtask_t microtask, launch_t invoker,
1358 int master_this_cons;
1360 kmp_team_t *parent_team;
1361 kmp_info_t *master_th;
1365 int master_set_numthreads;
1369 #if KMP_NESTED_HOT_TEAMS
1370 kmp_hot_team_ptr_t **p_hot_teams;
1373 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_fork_call);
1376 KA_TRACE(20, (
"__kmp_fork_call: enter T#%d\n", gtid));
1377 if (__kmp_stkpadding > 0 && __kmp_root[gtid] != NULL) {
1380 void *dummy = KMP_ALLOCA(__kmp_stkpadding);
1382 if (__kmp_stkpadding > KMP_MAX_STKPADDING)
1383 __kmp_stkpadding += (short)((kmp_int64)dummy);
1389 if (!TCR_4(__kmp_init_parallel))
1390 __kmp_parallel_initialize();
1391 __kmp_resume_if_soft_paused();
1394 master_th = __kmp_threads[gtid];
1396 parent_team = master_th->th.th_team;
1397 master_tid = master_th->th.th_info.ds.ds_tid;
1398 master_this_cons = master_th->th.th_local.this_construct;
1399 root = master_th->th.th_root;
1400 master_active = root->r.r_active;
1401 master_set_numthreads = master_th->th.th_set_nproc;
1404 ompt_data_t ompt_parallel_data = ompt_data_none;
1405 ompt_data_t *parent_task_data;
1406 ompt_frame_t *ompt_frame;
1407 ompt_data_t *implicit_task_data;
1408 void *return_address = NULL;
1410 if (ompt_enabled.enabled) {
1411 __ompt_get_task_info_internal(0, NULL, &parent_task_data, &ompt_frame,
1413 return_address = OMPT_LOAD_RETURN_ADDRESS(gtid);
1418 __kmp_assign_root_init_mask();
1421 level = parent_team->t.t_level;
1423 active_level = parent_team->t.t_active_level;
1425 teams_level = master_th->th.th_teams_level;
1426 #if KMP_NESTED_HOT_TEAMS
1427 p_hot_teams = &master_th->th.th_hot_teams;
1428 if (*p_hot_teams == NULL && __kmp_hot_teams_max_level > 0) {
1429 *p_hot_teams = (kmp_hot_team_ptr_t *)__kmp_allocate(
1430 sizeof(kmp_hot_team_ptr_t) * __kmp_hot_teams_max_level);
1431 (*p_hot_teams)[0].hot_team = root->r.r_hot_team;
1433 (*p_hot_teams)[0].hot_team_nth = 1;
1438 if (ompt_enabled.enabled) {
1439 if (ompt_enabled.ompt_callback_parallel_begin) {
1440 int team_size = master_set_numthreads
1441 ? master_set_numthreads
1442 : get__nproc_2(parent_team, master_tid);
1443 int flags = OMPT_INVOKER(call_context) |
1444 ((microtask == (microtask_t)__kmp_teams_master)
1445 ? ompt_parallel_league
1446 : ompt_parallel_team);
1447 ompt_callbacks.ompt_callback(ompt_callback_parallel_begin)(
1448 parent_task_data, ompt_frame, &ompt_parallel_data, team_size, flags,
1451 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1455 master_th->th.th_ident = loc;
1457 if (master_th->th.th_teams_microtask && ap &&
1458 microtask != (microtask_t)__kmp_teams_master && level == teams_level) {
1462 parent_team->t.t_ident = loc;
1463 __kmp_alloc_argv_entries(argc, parent_team, TRUE);
1464 parent_team->t.t_argc = argc;
1465 argv = (
void **)parent_team->t.t_argv;
1466 for (i = argc - 1; i >= 0; --i)
1467 *argv++ = va_arg(kmp_va_deref(ap),
void *);
1469 if (parent_team == master_th->th.th_serial_team) {
1472 KMP_DEBUG_ASSERT(parent_team->t.t_serialized > 1);
1474 if (call_context == fork_context_gnu) {
1477 parent_team->t.t_serialized--;
1482 parent_team->t.t_pkfn = microtask;
1487 void **exit_frame_p;
1489 ompt_lw_taskteam_t lw_taskteam;
1491 if (ompt_enabled.enabled) {
1492 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1493 &ompt_parallel_data, return_address);
1494 exit_frame_p = &(lw_taskteam.ompt_task_info.frame.exit_frame.ptr);
1496 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1500 implicit_task_data = OMPT_CUR_TASK_DATA(master_th);
1501 if (ompt_enabled.ompt_callback_implicit_task) {
1502 OMPT_CUR_TASK_INFO(master_th)->thread_num =
1503 __kmp_tid_from_gtid(gtid);
1504 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1505 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th),
1506 implicit_task_data, 1,
1507 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit);
1511 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
1513 exit_frame_p = &dummy;
1518 parent_team->t.t_serialized--;
1521 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1522 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1523 __kmp_invoke_microtask(microtask, gtid, 0, argc, parent_team->t.t_argv
1532 if (ompt_enabled.enabled) {
1533 *exit_frame_p = NULL;
1534 OMPT_CUR_TASK_INFO(master_th)->frame.exit_frame = ompt_data_none;
1535 if (ompt_enabled.ompt_callback_implicit_task) {
1536 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1537 ompt_scope_end, NULL, implicit_task_data, 1,
1538 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit);
1540 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th);
1541 __ompt_lw_taskteam_unlink(master_th);
1542 if (ompt_enabled.ompt_callback_parallel_end) {
1543 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1544 &ompt_parallel_data, OMPT_CUR_TASK_DATA(master_th),
1545 OMPT_INVOKER(call_context) | ompt_parallel_team,
1548 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1554 parent_team->t.t_pkfn = microtask;
1555 parent_team->t.t_invoke = invoker;
1556 KMP_ATOMIC_INC(&root->r.r_in_parallel);
1557 parent_team->t.t_active_level++;
1558 parent_team->t.t_level++;
1559 parent_team->t.t_def_allocator = master_th->th.th_def_allocator;
1562 if (ompt_enabled.enabled) {
1563 ompt_lw_taskteam_t lw_taskteam;
1564 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1565 &ompt_parallel_data, return_address);
1566 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 1,
true);
1571 if (master_set_numthreads) {
1572 if (master_set_numthreads <= master_th->th.th_teams_size.nth) {
1574 kmp_info_t **other_threads = parent_team->t.t_threads;
1577 int old_proc = master_th->th.th_teams_size.nth;
1578 if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] ==
1580 __kmp_resize_dist_barrier(parent_team, old_proc,
1581 master_set_numthreads);
1582 __kmp_add_threads_to_team(parent_team, master_set_numthreads);
1584 parent_team->t.t_nproc = master_set_numthreads;
1585 for (i = 0; i < master_set_numthreads; ++i) {
1586 other_threads[i]->th.th_team_nproc = master_set_numthreads;
1590 master_th->th.th_set_nproc = 0;
1594 if (__kmp_debugging) {
1595 int nth = __kmp_omp_num_threads(loc);
1597 master_set_numthreads = nth;
1602 #if USE_ITT_BUILD && USE_ITT_NOTIFY
1603 if (((__itt_frame_submit_v3_ptr && __itt_get_timestamp_ptr) ||
1605 __kmp_forkjoin_frames_mode == 3 &&
1606 parent_team->t.t_active_level == 1
1607 && master_th->th.th_teams_size.nteams == 1) {
1608 kmp_uint64 tmp_time = __itt_get_timestamp();
1609 master_th->th.th_frame_time = tmp_time;
1610 parent_team->t.t_region_time = tmp_time;
1612 if (__itt_stack_caller_create_ptr) {
1613 KMP_DEBUG_ASSERT(parent_team->t.t_stack_id == NULL);
1615 parent_team->t.t_stack_id = __kmp_itt_stack_caller_create();
1619 KF_TRACE(10, (
"__kmp_fork_call: before internal fork: root=%p, team=%p, "
1620 "master_th=%p, gtid=%d\n",
1621 root, parent_team, master_th, gtid));
1622 __kmp_internal_fork(loc, gtid, parent_team);
1623 KF_TRACE(10, (
"__kmp_fork_call: after internal fork: root=%p, team=%p, "
1624 "master_th=%p, gtid=%d\n",
1625 root, parent_team, master_th, gtid));
1627 if (call_context == fork_context_gnu)
1631 KA_TRACE(20, (
"__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,
1632 parent_team->t.t_id, parent_team->t.t_pkfn));
1634 if (!parent_team->t.t_invoke(gtid)) {
1635 KMP_ASSERT2(0,
"cannot invoke microtask for PRIMARY thread");
1637 KA_TRACE(20, (
"__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,
1638 parent_team->t.t_id, parent_team->t.t_pkfn));
1641 KA_TRACE(20, (
"__kmp_fork_call: parallel exit T#%d\n", gtid));
1647 if (__kmp_tasking_mode != tskm_immediate_exec) {
1648 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
1649 parent_team->t.t_task_team[master_th->th.th_task_state]);
1656 int enter_teams = 0;
1657 if (parent_team->t.t_active_level >=
1658 master_th->th.th_current_task->td_icvs.max_active_levels) {
1661 enter_teams = ((ap == NULL && active_level == 0) ||
1662 (ap && teams_level > 0 && teams_level == level));
1663 nthreads = master_set_numthreads
1664 ? master_set_numthreads
1666 : get__nproc_2(parent_team, master_tid);
1671 if ((get__max_active_levels(master_th) == 1 &&
1672 (root->r.r_in_parallel && !enter_teams)) ||
1673 (__kmp_library == library_serial)) {
1674 KC_TRACE(10, (
"__kmp_fork_call: T#%d serializing team; requested %d"
1682 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
1687 nthreads = __kmp_reserve_threads(root, parent_team, master_tid,
1688 nthreads, enter_teams);
1689 if (nthreads == 1) {
1693 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
1697 KMP_DEBUG_ASSERT(nthreads > 0);
1700 master_th->th.th_set_nproc = 0;
1703 if (nthreads == 1) {
1705 #if KMP_OS_LINUX && \
1706 (KMP_ARCH_X86 || KMP_ARCH_X86_64 || KMP_ARCH_ARM || KMP_ARCH_AARCH64)
1709 void **args = (
void **)KMP_ALLOCA(argc *
sizeof(
void *));
1714 (
"__kmp_fork_call: T#%d serializing parallel region\n", gtid));
1719 master_th->th.th_serial_team->t.t_pkfn = microtask;
1722 if (call_context == fork_context_intel) {
1724 master_th->th.th_serial_team->t.t_ident = loc;
1727 master_th->th.th_serial_team->t.t_level--;
1732 void **exit_frame_p;
1733 ompt_task_info_t *task_info;
1735 ompt_lw_taskteam_t lw_taskteam;
1737 if (ompt_enabled.enabled) {
1738 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1739 &ompt_parallel_data, return_address);
1741 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1744 task_info = OMPT_CUR_TASK_INFO(master_th);
1745 exit_frame_p = &(task_info->frame.exit_frame.ptr);
1746 if (ompt_enabled.ompt_callback_implicit_task) {
1747 OMPT_CUR_TASK_INFO(master_th)->thread_num =
1748 __kmp_tid_from_gtid(gtid);
1749 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1750 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th),
1751 &(task_info->task_data), 1,
1752 OMPT_CUR_TASK_INFO(master_th)->thread_num,
1753 ompt_task_implicit);
1757 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
1759 exit_frame_p = &dummy;
1764 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1765 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1766 __kmp_invoke_microtask(microtask, gtid, 0, argc,
1767 parent_team->t.t_argv
1776 if (ompt_enabled.enabled) {
1777 *exit_frame_p = NULL;
1778 if (ompt_enabled.ompt_callback_implicit_task) {
1779 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1780 ompt_scope_end, NULL, &(task_info->task_data), 1,
1781 OMPT_CUR_TASK_INFO(master_th)->thread_num,
1782 ompt_task_implicit);
1784 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th);
1785 __ompt_lw_taskteam_unlink(master_th);
1786 if (ompt_enabled.ompt_callback_parallel_end) {
1787 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1788 &ompt_parallel_data, parent_task_data,
1789 OMPT_INVOKER(call_context) | ompt_parallel_team,
1792 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1795 }
else if (microtask == (microtask_t)__kmp_teams_master) {
1796 KMP_DEBUG_ASSERT(master_th->th.th_team ==
1797 master_th->th.th_serial_team);
1798 team = master_th->th.th_team;
1800 team->t.t_invoke = invoker;
1801 __kmp_alloc_argv_entries(argc, team, TRUE);
1802 team->t.t_argc = argc;
1803 argv = (
void **)team->t.t_argv;
1805 for (i = argc - 1; i >= 0; --i)
1806 *argv++ = va_arg(kmp_va_deref(ap),
void *);
1808 for (i = 0; i < argc; ++i)
1810 argv[i] = parent_team->t.t_argv[i];
1818 if (ompt_enabled.enabled) {
1819 ompt_task_info_t *task_info = OMPT_CUR_TASK_INFO(master_th);
1820 if (ompt_enabled.ompt_callback_implicit_task) {
1821 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1822 ompt_scope_end, NULL, &(task_info->task_data), 0,
1823 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_initial);
1825 if (ompt_enabled.ompt_callback_parallel_end) {
1826 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1827 &ompt_parallel_data, parent_task_data,
1828 OMPT_INVOKER(call_context) | ompt_parallel_league,
1831 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1836 for (i = argc - 1; i >= 0; --i)
1837 *argv++ = va_arg(kmp_va_deref(ap),
void *);
1842 void **exit_frame_p;
1843 ompt_task_info_t *task_info;
1845 ompt_lw_taskteam_t lw_taskteam;
1847 if (ompt_enabled.enabled) {
1848 __ompt_lw_taskteam_init(&lw_taskteam, master_th, gtid,
1849 &ompt_parallel_data, return_address);
1850 __ompt_lw_taskteam_link(&lw_taskteam, master_th, 0);
1852 task_info = OMPT_CUR_TASK_INFO(master_th);
1853 exit_frame_p = &(task_info->frame.exit_frame.ptr);
1856 implicit_task_data = OMPT_CUR_TASK_DATA(master_th);
1857 if (ompt_enabled.ompt_callback_implicit_task) {
1858 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1859 ompt_scope_begin, OMPT_CUR_TEAM_DATA(master_th),
1860 implicit_task_data, 1, __kmp_tid_from_gtid(gtid),
1861 ompt_task_implicit);
1862 OMPT_CUR_TASK_INFO(master_th)->thread_num =
1863 __kmp_tid_from_gtid(gtid);
1867 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
1869 exit_frame_p = &dummy;
1874 KMP_TIME_PARTITIONED_BLOCK(OMP_parallel);
1875 KMP_SET_THREAD_STATE_BLOCK(IMPLICIT_TASK);
1876 __kmp_invoke_microtask(microtask, gtid, 0, argc, args
1885 if (ompt_enabled.enabled) {
1886 *exit_frame_p = NULL;
1887 if (ompt_enabled.ompt_callback_implicit_task) {
1888 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
1889 ompt_scope_end, NULL, &(task_info->task_data), 1,
1890 OMPT_CUR_TASK_INFO(master_th)->thread_num,
1891 ompt_task_implicit);
1894 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th);
1895 __ompt_lw_taskteam_unlink(master_th);
1896 if (ompt_enabled.ompt_callback_parallel_end) {
1897 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
1898 &ompt_parallel_data, parent_task_data,
1899 OMPT_INVOKER(call_context) | ompt_parallel_team,
1902 master_th->th.ompt_thread_info.state = ompt_state_overhead;
1906 }
else if (call_context == fork_context_gnu) {
1908 ompt_lw_taskteam_t lwt;
1909 __ompt_lw_taskteam_init(&lwt, master_th, gtid, &ompt_parallel_data,
1912 lwt.ompt_task_info.frame.exit_frame = ompt_data_none;
1913 __ompt_lw_taskteam_link(&lwt, master_th, 1);
1918 KA_TRACE(20, (
"__kmp_fork_call: T#%d serial exit\n", gtid));
1921 KMP_ASSERT2(call_context < fork_context_last,
1922 "__kmp_fork_call: unknown fork_context parameter");
1925 KA_TRACE(20, (
"__kmp_fork_call: T#%d serial exit\n", gtid));
1932 KF_TRACE(10, (
"__kmp_fork_call: parent_team_aclevel=%d, master_th=%p, "
1933 "curtask=%p, curtask_max_aclevel=%d\n",
1934 parent_team->t.t_active_level, master_th,
1935 master_th->th.th_current_task,
1936 master_th->th.th_current_task->td_icvs.max_active_levels));
1940 master_th->th.th_current_task->td_flags.executing = 0;
1942 if (!master_th->th.th_teams_microtask || level > teams_level) {
1944 KMP_ATOMIC_INC(&root->r.r_in_parallel);
1948 int nthreads_icv = master_th->th.th_current_task->td_icvs.nproc;
1949 if ((level + 1 < __kmp_nested_nth.used) &&
1950 (__kmp_nested_nth.nth[level + 1] != nthreads_icv)) {
1951 nthreads_icv = __kmp_nested_nth.nth[level + 1];
1957 kmp_proc_bind_t proc_bind = master_th->th.th_set_proc_bind;
1958 kmp_proc_bind_t proc_bind_icv =
1960 if (master_th->th.th_current_task->td_icvs.proc_bind == proc_bind_false) {
1961 proc_bind = proc_bind_false;
1963 if (proc_bind == proc_bind_default) {
1966 proc_bind = master_th->th.th_current_task->td_icvs.proc_bind;
1972 if ((level + 1 < __kmp_nested_proc_bind.used) &&
1973 (__kmp_nested_proc_bind.bind_types[level + 1] !=
1974 master_th->th.th_current_task->td_icvs.proc_bind)) {
1975 proc_bind_icv = __kmp_nested_proc_bind.bind_types[level + 1];
1980 master_th->th.th_set_proc_bind = proc_bind_default;
1982 if ((nthreads_icv > 0) || (proc_bind_icv != proc_bind_default)) {
1983 kmp_internal_control_t new_icvs;
1984 copy_icvs(&new_icvs, &master_th->th.th_current_task->td_icvs);
1985 new_icvs.next = NULL;
1986 if (nthreads_icv > 0) {
1987 new_icvs.nproc = nthreads_icv;
1989 if (proc_bind_icv != proc_bind_default) {
1990 new_icvs.proc_bind = proc_bind_icv;
1994 KF_TRACE(10, (
"__kmp_fork_call: before __kmp_allocate_team\n"));
1995 team = __kmp_allocate_team(root, nthreads, nthreads,
1999 proc_bind, &new_icvs,
2000 argc USE_NESTED_HOT_ARG(master_th));
2001 if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar)
2002 copy_icvs((kmp_internal_control_t *)team->t.b->team_icvs, &new_icvs);
2005 KF_TRACE(10, (
"__kmp_fork_call: before __kmp_allocate_team\n"));
2006 team = __kmp_allocate_team(root, nthreads, nthreads,
2011 &master_th->th.th_current_task->td_icvs,
2012 argc USE_NESTED_HOT_ARG(master_th));
2013 if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar)
2014 copy_icvs((kmp_internal_control_t *)team->t.b->team_icvs,
2015 &master_th->th.th_current_task->td_icvs);
2018 10, (
"__kmp_fork_call: after __kmp_allocate_team - team = %p\n", team));
2021 KMP_CHECK_UPDATE(team->t.t_master_tid, master_tid);
2022 KMP_CHECK_UPDATE(team->t.t_master_this_cons, master_this_cons);
2023 KMP_CHECK_UPDATE(team->t.t_ident, loc);
2024 KMP_CHECK_UPDATE(team->t.t_parent, parent_team);
2025 KMP_CHECK_UPDATE_SYNC(team->t.t_pkfn, microtask);
2027 KMP_CHECK_UPDATE_SYNC(team->t.ompt_team_info.master_return_address,
2030 KMP_CHECK_UPDATE(team->t.t_invoke, invoker);
2032 if (!master_th->th.th_teams_microtask || level > teams_level) {
2033 int new_level = parent_team->t.t_level + 1;
2034 KMP_CHECK_UPDATE(team->t.t_level, new_level);
2035 new_level = parent_team->t.t_active_level + 1;
2036 KMP_CHECK_UPDATE(team->t.t_active_level, new_level);
2039 int new_level = parent_team->t.t_level;
2040 KMP_CHECK_UPDATE(team->t.t_level, new_level);
2041 new_level = parent_team->t.t_active_level;
2042 KMP_CHECK_UPDATE(team->t.t_active_level, new_level);
2044 kmp_r_sched_t new_sched = get__sched_2(parent_team, master_tid);
2046 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched);
2048 KMP_CHECK_UPDATE(team->t.t_cancel_request, cancel_noreq);
2049 KMP_CHECK_UPDATE(team->t.t_def_allocator, master_th->th.th_def_allocator);
2052 propagateFPControl(team);
2054 if (ompd_state & OMPD_ENABLE_BP)
2055 ompd_bp_parallel_begin();
2058 if (__kmp_tasking_mode != tskm_immediate_exec) {
2061 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
2062 parent_team->t.t_task_team[master_th->th.th_task_state]);
2063 KA_TRACE(20, (
"__kmp_fork_call: Primary T#%d pushing task_team %p / team "
2064 "%p, new task_team %p / team %p\n",
2065 __kmp_gtid_from_thread(master_th),
2066 master_th->th.th_task_team, parent_team,
2067 team->t.t_task_team[master_th->th.th_task_state], team));
2069 if (active_level || master_th->th.th_task_team) {
2071 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
2072 if (master_th->th.th_task_state_top >=
2073 master_th->th.th_task_state_stack_sz) {
2074 kmp_uint32 new_size = 2 * master_th->th.th_task_state_stack_sz;
2075 kmp_uint8 *old_stack, *new_stack;
2077 new_stack = (kmp_uint8 *)__kmp_allocate(new_size);
2078 for (i = 0; i < master_th->th.th_task_state_stack_sz; ++i) {
2079 new_stack[i] = master_th->th.th_task_state_memo_stack[i];
2081 for (i = master_th->th.th_task_state_stack_sz; i < new_size;
2085 old_stack = master_th->th.th_task_state_memo_stack;
2086 master_th->th.th_task_state_memo_stack = new_stack;
2087 master_th->th.th_task_state_stack_sz = new_size;
2088 __kmp_free(old_stack);
2092 .th_task_state_memo_stack[master_th->th.th_task_state_top] =
2093 master_th->th.th_task_state;
2094 master_th->th.th_task_state_top++;
2095 #if KMP_NESTED_HOT_TEAMS
2096 if (master_th->th.th_hot_teams &&
2097 active_level < __kmp_hot_teams_max_level &&
2098 team == master_th->th.th_hot_teams[active_level].hot_team) {
2100 master_th->th.th_task_state =
2102 .th_task_state_memo_stack[master_th->th.th_task_state_top];
2105 master_th->th.th_task_state = 0;
2106 #if KMP_NESTED_HOT_TEAMS
2110 #if !KMP_NESTED_HOT_TEAMS
2111 KMP_DEBUG_ASSERT((master_th->th.th_task_team == NULL) ||
2112 (team == root->r.r_hot_team));
2118 (
"__kmp_fork_call: T#%d(%d:%d)->(%d:0) created a team of %d threads\n",
2119 gtid, parent_team->t.t_id, team->t.t_master_tid, team->t.t_id,
2121 KMP_DEBUG_ASSERT(team != root->r.r_hot_team ||
2122 (team->t.t_master_tid == 0 &&
2123 (team->t.t_parent == root->r.r_root_team ||
2124 team->t.t_parent->t.t_serialized)));
2128 argv = (
void **)team->t.t_argv;
2130 for (i = argc - 1; i >= 0; --i) {
2131 void *new_argv = va_arg(kmp_va_deref(ap),
void *);
2132 KMP_CHECK_UPDATE(*argv, new_argv);
2136 for (i = 0; i < argc; ++i) {
2138 KMP_CHECK_UPDATE(argv[i], team->t.t_parent->t.t_argv[i]);
2143 KMP_CHECK_UPDATE(team->t.t_master_active, master_active);
2144 if (!root->r.r_active)
2145 root->r.r_active = TRUE;
2147 __kmp_fork_team_threads(root, team, master_th, gtid);
2148 __kmp_setup_icv_copy(team, nthreads,
2149 &master_th->th.th_current_task->td_icvs, loc);
2152 master_th->th.ompt_thread_info.state = ompt_state_work_parallel;
2155 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2158 if (team->t.t_active_level == 1
2159 && !master_th->th.th_teams_microtask) {
2161 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) &&
2162 (__kmp_forkjoin_frames_mode == 3 ||
2163 __kmp_forkjoin_frames_mode == 1)) {
2164 kmp_uint64 tmp_time = 0;
2165 if (__itt_get_timestamp_ptr)
2166 tmp_time = __itt_get_timestamp();
2168 master_th->th.th_frame_time = tmp_time;
2169 if (__kmp_forkjoin_frames_mode == 3)
2170 team->t.t_region_time = tmp_time;
2174 if ((__itt_frame_begin_v3_ptr || KMP_ITT_DEBUG) &&
2175 __kmp_forkjoin_frames && !__kmp_forkjoin_frames_mode) {
2177 __kmp_itt_region_forking(gtid, team->t.t_nproc, 0);
2183 KMP_DEBUG_ASSERT(team == __kmp_threads[gtid]->th.th_team);
2186 (
"__kmp_internal_fork : root=%p, team=%p, master_th=%p, gtid=%d\n",
2187 root, team, master_th, gtid));
2190 if (__itt_stack_caller_create_ptr) {
2193 KMP_DEBUG_ASSERT(team->t.t_stack_id == NULL);
2194 team->t.t_stack_id = __kmp_itt_stack_caller_create();
2195 }
else if (parent_team->t.t_serialized) {
2200 KMP_DEBUG_ASSERT(parent_team->t.t_stack_id == NULL);
2201 parent_team->t.t_stack_id = __kmp_itt_stack_caller_create();
2209 __kmp_internal_fork(loc, gtid, team);
2210 KF_TRACE(10, (
"__kmp_internal_fork : after : root=%p, team=%p, "
2211 "master_th=%p, gtid=%d\n",
2212 root, team, master_th, gtid));
2215 if (call_context == fork_context_gnu) {
2216 KA_TRACE(20, (
"__kmp_fork_call: parallel exit T#%d\n", gtid));
2221 KA_TRACE(20, (
"__kmp_fork_call: T#%d(%d:0) invoke microtask = %p\n", gtid,
2222 team->t.t_id, team->t.t_pkfn));
2225 #if KMP_STATS_ENABLED
2229 KMP_SET_THREAD_STATE(stats_state_e::TEAMS_REGION);
2233 if (!team->t.t_invoke(gtid)) {
2234 KMP_ASSERT2(0,
"cannot invoke microtask for PRIMARY thread");
2237 #if KMP_STATS_ENABLED
2240 KMP_SET_THREAD_STATE(previous_state);
2244 KA_TRACE(20, (
"__kmp_fork_call: T#%d(%d:0) done microtask = %p\n", gtid,
2245 team->t.t_id, team->t.t_pkfn));
2248 KA_TRACE(20, (
"__kmp_fork_call: parallel exit T#%d\n", gtid));
2250 if (ompt_enabled.enabled) {
2251 master_th->th.ompt_thread_info.state = ompt_state_overhead;
2259 static inline void __kmp_join_restore_state(kmp_info_t *thread,
2262 thread->th.ompt_thread_info.state =
2263 ((team->t.t_serialized) ? ompt_state_work_serial
2264 : ompt_state_work_parallel);
2267 static inline void __kmp_join_ompt(
int gtid, kmp_info_t *thread,
2268 kmp_team_t *team, ompt_data_t *parallel_data,
2269 int flags,
void *codeptr) {
2270 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2271 if (ompt_enabled.ompt_callback_parallel_end) {
2272 ompt_callbacks.ompt_callback(ompt_callback_parallel_end)(
2273 parallel_data, &(task_info->task_data), flags, codeptr);
2276 task_info->frame.enter_frame = ompt_data_none;
2277 __kmp_join_restore_state(thread, team);
2281 void __kmp_join_call(
ident_t *loc,
int gtid
2284 enum fork_context_e fork_context
2288 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_join_call);
2290 kmp_team_t *parent_team;
2291 kmp_info_t *master_th;
2295 KA_TRACE(20, (
"__kmp_join_call: enter T#%d\n", gtid));
2298 master_th = __kmp_threads[gtid];
2299 root = master_th->th.th_root;
2300 team = master_th->th.th_team;
2301 parent_team = team->t.t_parent;
2303 master_th->th.th_ident = loc;
2306 void *team_microtask = (
void *)team->t.t_pkfn;
2310 if (ompt_enabled.enabled &&
2311 !(team->t.t_serialized && fork_context == fork_context_gnu)) {
2312 master_th->th.ompt_thread_info.state = ompt_state_overhead;
2317 if (__kmp_tasking_mode != tskm_immediate_exec && !exit_teams) {
2318 KA_TRACE(20, (
"__kmp_join_call: T#%d, old team = %p old task_team = %p, "
2319 "th_task_team = %p\n",
2320 __kmp_gtid_from_thread(master_th), team,
2321 team->t.t_task_team[master_th->th.th_task_state],
2322 master_th->th.th_task_team));
2323 KMP_DEBUG_ASSERT(master_th->th.th_task_team ==
2324 team->t.t_task_team[master_th->th.th_task_state]);
2328 if (team->t.t_serialized) {
2329 if (master_th->th.th_teams_microtask) {
2331 int level = team->t.t_level;
2332 int tlevel = master_th->th.th_teams_level;
2333 if (level == tlevel) {
2337 }
else if (level == tlevel + 1) {
2341 team->t.t_serialized++;
2347 if (ompt_enabled.enabled) {
2348 __kmp_join_restore_state(master_th, parent_team);
2355 master_active = team->t.t_master_active;
2360 __kmp_internal_join(loc, gtid, team);
2362 if (__itt_stack_caller_create_ptr) {
2363 KMP_DEBUG_ASSERT(team->t.t_stack_id != NULL);
2365 __kmp_itt_stack_caller_destroy((__itt_caller)team->t.t_stack_id);
2366 team->t.t_stack_id = NULL;
2370 master_th->th.th_task_state =
2373 if (__itt_stack_caller_create_ptr && parent_team->t.t_serialized) {
2374 KMP_DEBUG_ASSERT(parent_team->t.t_stack_id != NULL);
2378 __kmp_itt_stack_caller_destroy((__itt_caller)parent_team->t.t_stack_id);
2379 parent_team->t.t_stack_id = NULL;
2383 if (team->t.t_nproc > 1 &&
2384 __kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
2385 team->t.b->update_num_threads(team->t.t_nproc);
2386 __kmp_add_threads_to_team(team, team->t.t_nproc);
2393 ompt_data_t *parallel_data = &(team->t.ompt_team_info.parallel_data);
2394 void *codeptr = team->t.ompt_team_info.master_return_address;
2399 if (team->t.t_active_level == 1 &&
2400 (!master_th->th.th_teams_microtask ||
2401 master_th->th.th_teams_size.nteams == 1)) {
2402 master_th->th.th_ident = loc;
2405 if ((__itt_frame_submit_v3_ptr || KMP_ITT_DEBUG) &&
2406 __kmp_forkjoin_frames_mode == 3)
2407 __kmp_itt_frame_submit(gtid, team->t.t_region_time,
2408 master_th->th.th_frame_time, 0, loc,
2409 master_th->th.th_team_nproc, 1);
2410 else if ((__itt_frame_end_v3_ptr || KMP_ITT_DEBUG) &&
2411 !__kmp_forkjoin_frames_mode && __kmp_forkjoin_frames)
2412 __kmp_itt_region_joined(gtid);
2416 if (master_th->th.th_teams_microtask && !exit_teams &&
2417 team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
2418 team->t.t_level == master_th->th.th_teams_level + 1) {
2423 ompt_data_t ompt_parallel_data = ompt_data_none;
2424 if (ompt_enabled.enabled) {
2425 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2426 if (ompt_enabled.ompt_callback_implicit_task) {
2427 int ompt_team_size = team->t.t_nproc;
2428 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
2429 ompt_scope_end, NULL, &(task_info->task_data), ompt_team_size,
2430 OMPT_CUR_TASK_INFO(master_th)->thread_num, ompt_task_implicit);
2432 task_info->frame.exit_frame = ompt_data_none;
2433 task_info->task_data = ompt_data_none;
2434 ompt_parallel_data = *OMPT_CUR_TEAM_DATA(master_th);
2435 __ompt_lw_taskteam_unlink(master_th);
2440 team->t.t_active_level--;
2441 KMP_ATOMIC_DEC(&root->r.r_in_parallel);
2447 if (master_th->th.th_team_nproc < master_th->th.th_teams_size.nth) {
2448 int old_num = master_th->th.th_team_nproc;
2449 int new_num = master_th->th.th_teams_size.nth;
2450 kmp_info_t **other_threads = team->t.t_threads;
2451 team->t.t_nproc = new_num;
2452 for (
int i = 0; i < old_num; ++i) {
2453 other_threads[i]->th.th_team_nproc = new_num;
2456 for (
int i = old_num; i < new_num; ++i) {
2458 KMP_DEBUG_ASSERT(other_threads[i]);
2459 kmp_balign_t *balign = other_threads[i]->th.th_bar;
2460 for (
int b = 0; b < bs_last_barrier; ++b) {
2461 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
2462 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
2464 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
2467 if (__kmp_tasking_mode != tskm_immediate_exec) {
2469 other_threads[i]->th.th_task_state = master_th->th.th_task_state;
2475 if (ompt_enabled.enabled) {
2476 __kmp_join_ompt(gtid, master_th, parent_team, &ompt_parallel_data,
2477 OMPT_INVOKER(fork_context) | ompt_parallel_team, codeptr);
2485 master_th->th.th_info.ds.ds_tid = team->t.t_master_tid;
2486 master_th->th.th_local.this_construct = team->t.t_master_this_cons;
2488 master_th->th.th_dispatch = &parent_team->t.t_dispatch[team->t.t_master_tid];
2493 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
2495 if (!master_th->th.th_teams_microtask ||
2496 team->t.t_level > master_th->th.th_teams_level) {
2498 KMP_ATOMIC_DEC(&root->r.r_in_parallel);
2500 KMP_DEBUG_ASSERT(root->r.r_in_parallel >= 0);
2503 if (ompt_enabled.enabled) {
2504 ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
2505 if (ompt_enabled.ompt_callback_implicit_task) {
2506 int flags = (team_microtask == (
void *)__kmp_teams_master)
2508 : ompt_task_implicit;
2509 int ompt_team_size = (flags == ompt_task_initial) ? 0 : team->t.t_nproc;
2510 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
2511 ompt_scope_end, NULL, &(task_info->task_data), ompt_team_size,
2512 OMPT_CUR_TASK_INFO(master_th)->thread_num, flags);
2514 task_info->frame.exit_frame = ompt_data_none;
2515 task_info->task_data = ompt_data_none;
2519 KF_TRACE(10, (
"__kmp_join_call1: T#%d, this_thread=%p team=%p\n", 0,
2521 __kmp_pop_current_task_from_thread(master_th);
2523 #if KMP_AFFINITY_SUPPORTED
2525 master_th->th.th_first_place = team->t.t_first_place;
2526 master_th->th.th_last_place = team->t.t_last_place;
2528 master_th->th.th_def_allocator = team->t.t_def_allocator;
2531 if (ompd_state & OMPD_ENABLE_BP)
2532 ompd_bp_parallel_end();
2534 updateHWFPControl(team);
2536 if (root->r.r_active != master_active)
2537 root->r.r_active = master_active;
2539 __kmp_free_team(root, team USE_NESTED_HOT_ARG(
2547 master_th->th.th_team = parent_team;
2548 master_th->th.th_team_nproc = parent_team->t.t_nproc;
2549 master_th->th.th_team_master = parent_team->t.t_threads[0];
2550 master_th->th.th_team_serialized = parent_team->t.t_serialized;
2553 if (parent_team->t.t_serialized &&
2554 parent_team != master_th->th.th_serial_team &&
2555 parent_team != root->r.r_root_team) {
2556 __kmp_free_team(root,
2557 master_th->th.th_serial_team USE_NESTED_HOT_ARG(NULL));
2558 master_th->th.th_serial_team = parent_team;
2561 if (__kmp_tasking_mode != tskm_immediate_exec) {
2562 if (master_th->th.th_task_state_top >
2564 KMP_DEBUG_ASSERT(master_th->th.th_task_state_memo_stack);
2566 master_th->th.th_task_state_memo_stack[master_th->th.th_task_state_top] =
2567 master_th->th.th_task_state;
2568 --master_th->th.th_task_state_top;
2570 master_th->th.th_task_state =
2572 .th_task_state_memo_stack[master_th->th.th_task_state_top];
2575 master_th->th.th_task_team =
2576 parent_team->t.t_task_team[master_th->th.th_task_state];
2578 (
"__kmp_join_call: Primary T#%d restoring task_team %p, team %p\n",
2579 __kmp_gtid_from_thread(master_th), master_th->th.th_task_team,
2586 master_th->th.th_current_task->td_flags.executing = 1;
2588 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2592 OMPT_INVOKER(fork_context) |
2593 ((team_microtask == (
void *)__kmp_teams_master) ? ompt_parallel_league
2594 : ompt_parallel_team);
2595 if (ompt_enabled.enabled) {
2596 __kmp_join_ompt(gtid, master_th, parent_team, parallel_data, flags,
2602 KA_TRACE(20, (
"__kmp_join_call: exit T#%d\n", gtid));
2607 void __kmp_save_internal_controls(kmp_info_t *thread) {
2609 if (thread->th.th_team != thread->th.th_serial_team) {
2612 if (thread->th.th_team->t.t_serialized > 1) {
2615 if (thread->th.th_team->t.t_control_stack_top == NULL) {
2618 if (thread->th.th_team->t.t_control_stack_top->serial_nesting_level !=
2619 thread->th.th_team->t.t_serialized) {
2624 kmp_internal_control_t *control =
2625 (kmp_internal_control_t *)__kmp_allocate(
2626 sizeof(kmp_internal_control_t));
2628 copy_icvs(control, &thread->th.th_current_task->td_icvs);
2630 control->serial_nesting_level = thread->th.th_team->t.t_serialized;
2632 control->next = thread->th.th_team->t.t_control_stack_top;
2633 thread->th.th_team->t.t_control_stack_top = control;
2639 void __kmp_set_num_threads(
int new_nth,
int gtid) {
2643 KF_TRACE(10, (
"__kmp_set_num_threads: new __kmp_nth = %d\n", new_nth));
2644 KMP_DEBUG_ASSERT(__kmp_init_serial);
2648 else if (new_nth > __kmp_max_nth)
2649 new_nth = __kmp_max_nth;
2652 thread = __kmp_threads[gtid];
2653 if (thread->th.th_current_task->td_icvs.nproc == new_nth)
2656 __kmp_save_internal_controls(thread);
2658 set__nproc(thread, new_nth);
2663 root = thread->th.th_root;
2664 if (__kmp_init_parallel && (!root->r.r_active) &&
2665 (root->r.r_hot_team->t.t_nproc > new_nth)
2666 #
if KMP_NESTED_HOT_TEAMS
2667 && __kmp_hot_teams_max_level && !__kmp_hot_teams_mode
2670 kmp_team_t *hot_team = root->r.r_hot_team;
2673 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
2675 if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
2676 __kmp_resize_dist_barrier(hot_team, hot_team->t.t_nproc, new_nth);
2679 for (f = new_nth; f < hot_team->t.t_nproc; f++) {
2680 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL);
2681 if (__kmp_tasking_mode != tskm_immediate_exec) {
2684 hot_team->t.t_threads[f]->th.th_task_team = NULL;
2686 __kmp_free_thread(hot_team->t.t_threads[f]);
2687 hot_team->t.t_threads[f] = NULL;
2689 hot_team->t.t_nproc = new_nth;
2690 #if KMP_NESTED_HOT_TEAMS
2691 if (thread->th.th_hot_teams) {
2692 KMP_DEBUG_ASSERT(hot_team == thread->th.th_hot_teams[0].hot_team);
2693 thread->th.th_hot_teams[0].hot_team_nth = new_nth;
2697 if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
2698 hot_team->t.b->update_num_threads(new_nth);
2699 __kmp_add_threads_to_team(hot_team, new_nth);
2702 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
2705 for (f = 0; f < new_nth; f++) {
2706 KMP_DEBUG_ASSERT(hot_team->t.t_threads[f] != NULL);
2707 hot_team->t.t_threads[f]->th.th_team_nproc = new_nth;
2710 hot_team->t.t_size_changed = -1;
2715 void __kmp_set_max_active_levels(
int gtid,
int max_active_levels) {
2718 KF_TRACE(10, (
"__kmp_set_max_active_levels: new max_active_levels for thread "
2720 gtid, max_active_levels));
2721 KMP_DEBUG_ASSERT(__kmp_init_serial);
2724 if (max_active_levels < 0) {
2725 KMP_WARNING(ActiveLevelsNegative, max_active_levels);
2730 KF_TRACE(10, (
"__kmp_set_max_active_levels: the call is ignored: new "
2731 "max_active_levels for thread %d = (%d)\n",
2732 gtid, max_active_levels));
2735 if (max_active_levels <= KMP_MAX_ACTIVE_LEVELS_LIMIT) {
2740 KMP_WARNING(ActiveLevelsExceedLimit, max_active_levels,
2741 KMP_MAX_ACTIVE_LEVELS_LIMIT);
2742 max_active_levels = KMP_MAX_ACTIVE_LEVELS_LIMIT;
2748 KF_TRACE(10, (
"__kmp_set_max_active_levels: after validation: new "
2749 "max_active_levels for thread %d = (%d)\n",
2750 gtid, max_active_levels));
2752 thread = __kmp_threads[gtid];
2754 __kmp_save_internal_controls(thread);
2756 set__max_active_levels(thread, max_active_levels);
2760 int __kmp_get_max_active_levels(
int gtid) {
2763 KF_TRACE(10, (
"__kmp_get_max_active_levels: thread %d\n", gtid));
2764 KMP_DEBUG_ASSERT(__kmp_init_serial);
2766 thread = __kmp_threads[gtid];
2767 KMP_DEBUG_ASSERT(thread->th.th_current_task);
2768 KF_TRACE(10, (
"__kmp_get_max_active_levels: thread %d, curtask=%p, "
2769 "curtask_maxaclevel=%d\n",
2770 gtid, thread->th.th_current_task,
2771 thread->th.th_current_task->td_icvs.max_active_levels));
2772 return thread->th.th_current_task->td_icvs.max_active_levels;
2776 void __kmp_set_num_teams(
int num_teams) {
2778 __kmp_nteams = num_teams;
2780 int __kmp_get_max_teams(
void) {
return __kmp_nteams; }
2782 void __kmp_set_teams_thread_limit(
int limit) {
2784 __kmp_teams_thread_limit = limit;
2786 int __kmp_get_teams_thread_limit(
void) {
return __kmp_teams_thread_limit; }
2788 KMP_BUILD_ASSERT(
sizeof(kmp_sched_t) ==
sizeof(
int));
2789 KMP_BUILD_ASSERT(
sizeof(
enum sched_type) ==
sizeof(
int));
2792 void __kmp_set_schedule(
int gtid, kmp_sched_t kind,
int chunk) {
2794 kmp_sched_t orig_kind;
2797 KF_TRACE(10, (
"__kmp_set_schedule: new schedule for thread %d = (%d, %d)\n",
2798 gtid, (
int)kind, chunk));
2799 KMP_DEBUG_ASSERT(__kmp_init_serial);
2806 kind = __kmp_sched_without_mods(kind);
2808 if (kind <= kmp_sched_lower || kind >= kmp_sched_upper ||
2809 (kind <= kmp_sched_lower_ext && kind >= kmp_sched_upper_std)) {
2811 __kmp_msg(kmp_ms_warning, KMP_MSG(ScheduleKindOutOfRange, kind),
2812 KMP_HNT(DefaultScheduleKindUsed,
"static, no chunk"),
2814 kind = kmp_sched_default;
2818 thread = __kmp_threads[gtid];
2820 __kmp_save_internal_controls(thread);
2822 if (kind < kmp_sched_upper_std) {
2823 if (kind == kmp_sched_static && chunk < KMP_DEFAULT_CHUNK) {
2826 thread->th.th_current_task->td_icvs.sched.r_sched_type =
kmp_sch_static;
2828 thread->th.th_current_task->td_icvs.sched.r_sched_type =
2829 __kmp_sch_map[kind - kmp_sched_lower - 1];
2834 thread->th.th_current_task->td_icvs.sched.r_sched_type =
2835 __kmp_sch_map[kind - kmp_sched_lower_ext + kmp_sched_upper_std -
2836 kmp_sched_lower - 2];
2838 __kmp_sched_apply_mods_intkind(
2839 orig_kind, &(thread->th.th_current_task->td_icvs.sched.r_sched_type));
2840 if (kind == kmp_sched_auto || chunk < 1) {
2842 thread->th.th_current_task->td_icvs.sched.chunk = KMP_DEFAULT_CHUNK;
2844 thread->th.th_current_task->td_icvs.sched.chunk = chunk;
2849 void __kmp_get_schedule(
int gtid, kmp_sched_t *kind,
int *chunk) {
2853 KF_TRACE(10, (
"__kmp_get_schedule: thread %d\n", gtid));
2854 KMP_DEBUG_ASSERT(__kmp_init_serial);
2856 thread = __kmp_threads[gtid];
2858 th_type = thread->th.th_current_task->td_icvs.sched.r_sched_type;
2859 switch (SCHEDULE_WITHOUT_MODIFIERS(th_type)) {
2861 case kmp_sch_static_greedy:
2862 case kmp_sch_static_balanced:
2863 *kind = kmp_sched_static;
2864 __kmp_sched_apply_mods_stdkind(kind, th_type);
2867 case kmp_sch_static_chunked:
2868 *kind = kmp_sched_static;
2870 case kmp_sch_dynamic_chunked:
2871 *kind = kmp_sched_dynamic;
2874 case kmp_sch_guided_iterative_chunked:
2875 case kmp_sch_guided_analytical_chunked:
2876 *kind = kmp_sched_guided;
2879 *kind = kmp_sched_auto;
2881 case kmp_sch_trapezoidal:
2882 *kind = kmp_sched_trapezoidal;
2884 #if KMP_STATIC_STEAL_ENABLED
2885 case kmp_sch_static_steal:
2886 *kind = kmp_sched_static_steal;
2890 KMP_FATAL(UnknownSchedulingType, th_type);
2893 __kmp_sched_apply_mods_stdkind(kind, th_type);
2894 *chunk = thread->th.th_current_task->td_icvs.sched.chunk;
2897 int __kmp_get_ancestor_thread_num(
int gtid,
int level) {
2903 KF_TRACE(10, (
"__kmp_get_ancestor_thread_num: thread %d %d\n", gtid, level));
2904 KMP_DEBUG_ASSERT(__kmp_init_serial);
2911 thr = __kmp_threads[gtid];
2912 team = thr->th.th_team;
2913 ii = team->t.t_level;
2917 if (thr->th.th_teams_microtask) {
2919 int tlevel = thr->th.th_teams_level;
2922 KMP_DEBUG_ASSERT(ii >= tlevel);
2934 return __kmp_tid_from_gtid(gtid);
2936 dd = team->t.t_serialized;
2938 while (ii > level) {
2939 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
2941 if ((team->t.t_serialized) && (!dd)) {
2942 team = team->t.t_parent;
2946 team = team->t.t_parent;
2947 dd = team->t.t_serialized;
2952 return (dd > 1) ? (0) : (team->t.t_master_tid);
2955 int __kmp_get_team_size(
int gtid,
int level) {
2961 KF_TRACE(10, (
"__kmp_get_team_size: thread %d %d\n", gtid, level));
2962 KMP_DEBUG_ASSERT(__kmp_init_serial);
2969 thr = __kmp_threads[gtid];
2970 team = thr->th.th_team;
2971 ii = team->t.t_level;
2975 if (thr->th.th_teams_microtask) {
2977 int tlevel = thr->th.th_teams_level;
2980 KMP_DEBUG_ASSERT(ii >= tlevel);
2991 while (ii > level) {
2992 for (dd = team->t.t_serialized; (dd > 0) && (ii > level); dd--, ii--) {
2994 if (team->t.t_serialized && (!dd)) {
2995 team = team->t.t_parent;
2999 team = team->t.t_parent;
3004 return team->t.t_nproc;
3007 kmp_r_sched_t __kmp_get_schedule_global() {
3012 kmp_r_sched_t r_sched;
3018 enum sched_type s = SCHEDULE_WITHOUT_MODIFIERS(__kmp_sched);
3019 enum sched_type sched_modifiers = SCHEDULE_GET_MODIFIERS(__kmp_sched);
3022 r_sched.r_sched_type = __kmp_static;
3025 r_sched.r_sched_type = __kmp_guided;
3027 r_sched.r_sched_type = __kmp_sched;
3029 SCHEDULE_SET_MODIFIERS(r_sched.r_sched_type, sched_modifiers);
3031 if (__kmp_chunk < KMP_DEFAULT_CHUNK) {
3033 r_sched.chunk = KMP_DEFAULT_CHUNK;
3035 r_sched.chunk = __kmp_chunk;
3043 static void __kmp_alloc_argv_entries(
int argc, kmp_team_t *team,
int realloc) {
3045 KMP_DEBUG_ASSERT(team);
3046 if (!realloc || argc > team->t.t_max_argc) {
3048 KA_TRACE(100, (
"__kmp_alloc_argv_entries: team %d: needed entries=%d, "
3049 "current entries=%d\n",
3050 team->t.t_id, argc, (realloc) ? team->t.t_max_argc : 0));
3052 if (realloc && team->t.t_argv != &team->t.t_inline_argv[0])
3053 __kmp_free((
void *)team->t.t_argv);
3055 if (argc <= KMP_INLINE_ARGV_ENTRIES) {
3057 team->t.t_max_argc = KMP_INLINE_ARGV_ENTRIES;
3058 KA_TRACE(100, (
"__kmp_alloc_argv_entries: team %d: inline allocate %d "
3060 team->t.t_id, team->t.t_max_argc));
3061 team->t.t_argv = &team->t.t_inline_argv[0];
3062 if (__kmp_storage_map) {
3063 __kmp_print_storage_map_gtid(
3064 -1, &team->t.t_inline_argv[0],
3065 &team->t.t_inline_argv[KMP_INLINE_ARGV_ENTRIES],
3066 (
sizeof(
void *) * KMP_INLINE_ARGV_ENTRIES),
"team_%d.t_inline_argv",
3071 team->t.t_max_argc = (argc <= (KMP_MIN_MALLOC_ARGV_ENTRIES >> 1))
3072 ? KMP_MIN_MALLOC_ARGV_ENTRIES
3074 KA_TRACE(100, (
"__kmp_alloc_argv_entries: team %d: dynamic allocate %d "
3076 team->t.t_id, team->t.t_max_argc));
3078 (
void **)__kmp_page_allocate(
sizeof(
void *) * team->t.t_max_argc);
3079 if (__kmp_storage_map) {
3080 __kmp_print_storage_map_gtid(-1, &team->t.t_argv[0],
3081 &team->t.t_argv[team->t.t_max_argc],
3082 sizeof(
void *) * team->t.t_max_argc,
3083 "team_%d.t_argv", team->t.t_id);
3089 static void __kmp_allocate_team_arrays(kmp_team_t *team,
int max_nth) {
3091 int num_disp_buff = max_nth > 1 ? __kmp_dispatch_num_buffers : 2;
3093 (kmp_info_t **)__kmp_allocate(
sizeof(kmp_info_t *) * max_nth);
3094 team->t.t_disp_buffer = (dispatch_shared_info_t *)__kmp_allocate(
3095 sizeof(dispatch_shared_info_t) * num_disp_buff);
3096 team->t.t_dispatch =
3097 (kmp_disp_t *)__kmp_allocate(
sizeof(kmp_disp_t) * max_nth);
3098 team->t.t_implicit_task_taskdata =
3099 (kmp_taskdata_t *)__kmp_allocate(
sizeof(kmp_taskdata_t) * max_nth);
3100 team->t.t_max_nproc = max_nth;
3103 for (i = 0; i < num_disp_buff; ++i) {
3104 team->t.t_disp_buffer[i].buffer_index = i;
3105 team->t.t_disp_buffer[i].doacross_buf_idx = i;
3109 static void __kmp_free_team_arrays(kmp_team_t *team) {
3112 for (i = 0; i < team->t.t_max_nproc; ++i) {
3113 if (team->t.t_dispatch[i].th_disp_buffer != NULL) {
3114 __kmp_free(team->t.t_dispatch[i].th_disp_buffer);
3115 team->t.t_dispatch[i].th_disp_buffer = NULL;
3118 #if KMP_USE_HIER_SCHED
3119 __kmp_dispatch_free_hierarchies(team);
3121 __kmp_free(team->t.t_threads);
3122 __kmp_free(team->t.t_disp_buffer);
3123 __kmp_free(team->t.t_dispatch);
3124 __kmp_free(team->t.t_implicit_task_taskdata);
3125 team->t.t_threads = NULL;
3126 team->t.t_disp_buffer = NULL;
3127 team->t.t_dispatch = NULL;
3128 team->t.t_implicit_task_taskdata = 0;
3131 static void __kmp_reallocate_team_arrays(kmp_team_t *team,
int max_nth) {
3132 kmp_info_t **oldThreads = team->t.t_threads;
3134 __kmp_free(team->t.t_disp_buffer);
3135 __kmp_free(team->t.t_dispatch);
3136 __kmp_free(team->t.t_implicit_task_taskdata);
3137 __kmp_allocate_team_arrays(team, max_nth);
3139 KMP_MEMCPY(team->t.t_threads, oldThreads,
3140 team->t.t_nproc *
sizeof(kmp_info_t *));
3142 __kmp_free(oldThreads);
3145 static kmp_internal_control_t __kmp_get_global_icvs(
void) {
3147 kmp_r_sched_t r_sched =
3148 __kmp_get_schedule_global();
3150 KMP_DEBUG_ASSERT(__kmp_nested_proc_bind.used > 0);
3152 kmp_internal_control_t g_icvs = {
3154 (kmp_int8)__kmp_global.g.g_dynamic,
3156 (kmp_int8)__kmp_env_blocktime,
3158 __kmp_dflt_blocktime,
3163 __kmp_dflt_team_nth,
3167 __kmp_dflt_max_active_levels,
3171 __kmp_nested_proc_bind.bind_types[0],
3172 __kmp_default_device,
3179 static kmp_internal_control_t __kmp_get_x_global_icvs(
const kmp_team_t *team) {
3181 kmp_internal_control_t gx_icvs;
3182 gx_icvs.serial_nesting_level =
3184 copy_icvs(&gx_icvs, &team->t.t_threads[0]->th.th_current_task->td_icvs);
3185 gx_icvs.next = NULL;
3190 static void __kmp_initialize_root(kmp_root_t *root) {
3192 kmp_team_t *root_team;
3193 kmp_team_t *hot_team;
3194 int hot_team_max_nth;
3195 kmp_r_sched_t r_sched =
3196 __kmp_get_schedule_global();
3197 kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
3198 KMP_DEBUG_ASSERT(root);
3199 KMP_ASSERT(!root->r.r_begin);
3202 __kmp_init_lock(&root->r.r_begin_lock);
3203 root->r.r_begin = FALSE;
3204 root->r.r_active = FALSE;
3205 root->r.r_in_parallel = 0;
3206 root->r.r_blocktime = __kmp_dflt_blocktime;
3207 #if KMP_AFFINITY_SUPPORTED
3208 root->r.r_affinity_assigned = FALSE;
3213 KF_TRACE(10, (
"__kmp_initialize_root: before root_team\n"));
3216 __kmp_allocate_team(root,
3222 __kmp_nested_proc_bind.bind_types[0], &r_icvs,
3224 USE_NESTED_HOT_ARG(NULL)
3229 TCW_SYNC_PTR(root_team->t.t_pkfn, (microtask_t)(~0));
3232 KF_TRACE(10, (
"__kmp_initialize_root: after root_team = %p\n", root_team));
3234 root->r.r_root_team = root_team;
3235 root_team->t.t_control_stack_top = NULL;
3238 root_team->t.t_threads[0] = NULL;
3239 root_team->t.t_nproc = 1;
3240 root_team->t.t_serialized = 1;
3242 root_team->t.t_sched.sched = r_sched.sched;
3245 (
"__kmp_initialize_root: init root team %d arrived: join=%u, plain=%u\n",
3246 root_team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
3250 KF_TRACE(10, (
"__kmp_initialize_root: before hot_team\n"));
3253 __kmp_allocate_team(root,
3255 __kmp_dflt_team_nth_ub * 2,
3259 __kmp_nested_proc_bind.bind_types[0], &r_icvs,
3261 USE_NESTED_HOT_ARG(NULL)
3263 KF_TRACE(10, (
"__kmp_initialize_root: after hot_team = %p\n", hot_team));
3265 root->r.r_hot_team = hot_team;
3266 root_team->t.t_control_stack_top = NULL;
3269 hot_team->t.t_parent = root_team;
3272 hot_team_max_nth = hot_team->t.t_max_nproc;
3273 for (f = 0; f < hot_team_max_nth; ++f) {
3274 hot_team->t.t_threads[f] = NULL;
3276 hot_team->t.t_nproc = 1;
3278 hot_team->t.t_sched.sched = r_sched.sched;
3279 hot_team->t.t_size_changed = 0;
3284 typedef struct kmp_team_list_item {
3285 kmp_team_p
const *entry;
3286 struct kmp_team_list_item *next;
3287 } kmp_team_list_item_t;
3288 typedef kmp_team_list_item_t *kmp_team_list_t;
3290 static void __kmp_print_structure_team_accum(
3291 kmp_team_list_t list,
3292 kmp_team_p
const *team
3302 KMP_DEBUG_ASSERT(list != NULL);
3307 __kmp_print_structure_team_accum(list, team->t.t_parent);
3308 __kmp_print_structure_team_accum(list, team->t.t_next_pool);
3312 while (l->next != NULL && l->entry != team) {
3315 if (l->next != NULL) {
3321 while (l->next != NULL && l->entry->t.t_id <= team->t.t_id) {
3327 kmp_team_list_item_t *item = (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(
3328 sizeof(kmp_team_list_item_t));
3335 static void __kmp_print_structure_team(
char const *title, kmp_team_p
const *team
3338 __kmp_printf(
"%s", title);
3340 __kmp_printf(
"%2x %p\n", team->t.t_id, team);
3342 __kmp_printf(
" - (nil)\n");
3346 static void __kmp_print_structure_thread(
char const *title,
3347 kmp_info_p
const *thread) {
3348 __kmp_printf(
"%s", title);
3349 if (thread != NULL) {
3350 __kmp_printf(
"%2d %p\n", thread->th.th_info.ds.ds_gtid, thread);
3352 __kmp_printf(
" - (nil)\n");
3356 void __kmp_print_structure(
void) {
3358 kmp_team_list_t list;
3362 (kmp_team_list_item_t *)KMP_INTERNAL_MALLOC(
sizeof(kmp_team_list_item_t));
3366 __kmp_printf(
"\n------------------------------\nGlobal Thread "
3367 "Table\n------------------------------\n");
3370 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3371 __kmp_printf(
"%2d", gtid);
3372 if (__kmp_threads != NULL) {
3373 __kmp_printf(
" %p", __kmp_threads[gtid]);
3375 if (__kmp_root != NULL) {
3376 __kmp_printf(
" %p", __kmp_root[gtid]);
3383 __kmp_printf(
"\n------------------------------\nThreads\n--------------------"
3385 if (__kmp_threads != NULL) {
3387 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3388 kmp_info_t
const *thread = __kmp_threads[gtid];
3389 if (thread != NULL) {
3390 __kmp_printf(
"GTID %2d %p:\n", gtid, thread);
3391 __kmp_printf(
" Our Root: %p\n", thread->th.th_root);
3392 __kmp_print_structure_team(
" Our Team: ", thread->th.th_team);
3393 __kmp_print_structure_team(
" Serial Team: ",
3394 thread->th.th_serial_team);
3395 __kmp_printf(
" Threads: %2d\n", thread->th.th_team_nproc);
3396 __kmp_print_structure_thread(
" Primary: ",
3397 thread->th.th_team_master);
3398 __kmp_printf(
" Serialized?: %2d\n", thread->th.th_team_serialized);
3399 __kmp_printf(
" Set NProc: %2d\n", thread->th.th_set_nproc);
3400 __kmp_printf(
" Set Proc Bind: %2d\n", thread->th.th_set_proc_bind);
3401 __kmp_print_structure_thread(
" Next in pool: ",
3402 thread->th.th_next_pool);
3404 __kmp_print_structure_team_accum(list, thread->th.th_team);
3405 __kmp_print_structure_team_accum(list, thread->th.th_serial_team);
3409 __kmp_printf(
"Threads array is not allocated.\n");
3413 __kmp_printf(
"\n------------------------------\nUbers\n----------------------"
3415 if (__kmp_root != NULL) {
3417 for (gtid = 0; gtid < __kmp_threads_capacity; ++gtid) {
3418 kmp_root_t
const *root = __kmp_root[gtid];
3420 __kmp_printf(
"GTID %2d %p:\n", gtid, root);
3421 __kmp_print_structure_team(
" Root Team: ", root->r.r_root_team);
3422 __kmp_print_structure_team(
" Hot Team: ", root->r.r_hot_team);
3423 __kmp_print_structure_thread(
" Uber Thread: ",
3424 root->r.r_uber_thread);
3425 __kmp_printf(
" Active?: %2d\n", root->r.r_active);
3426 __kmp_printf(
" In Parallel: %2d\n",
3427 KMP_ATOMIC_LD_RLX(&root->r.r_in_parallel));
3429 __kmp_print_structure_team_accum(list, root->r.r_root_team);
3430 __kmp_print_structure_team_accum(list, root->r.r_hot_team);
3434 __kmp_printf(
"Ubers array is not allocated.\n");
3437 __kmp_printf(
"\n------------------------------\nTeams\n----------------------"
3439 while (list->next != NULL) {
3440 kmp_team_p
const *team = list->entry;
3442 __kmp_printf(
"Team %2x %p:\n", team->t.t_id, team);
3443 __kmp_print_structure_team(
" Parent Team: ", team->t.t_parent);
3444 __kmp_printf(
" Primary TID: %2d\n", team->t.t_master_tid);
3445 __kmp_printf(
" Max threads: %2d\n", team->t.t_max_nproc);
3446 __kmp_printf(
" Levels of serial: %2d\n", team->t.t_serialized);
3447 __kmp_printf(
" Number threads: %2d\n", team->t.t_nproc);
3448 for (i = 0; i < team->t.t_nproc; ++i) {
3449 __kmp_printf(
" Thread %2d: ", i);
3450 __kmp_print_structure_thread(
"", team->t.t_threads[i]);
3452 __kmp_print_structure_team(
" Next in pool: ", team->t.t_next_pool);
3458 __kmp_printf(
"\n------------------------------\nPools\n----------------------"
3460 __kmp_print_structure_thread(
"Thread pool: ",
3461 CCAST(kmp_info_t *, __kmp_thread_pool));
3462 __kmp_print_structure_team(
"Team pool: ",
3463 CCAST(kmp_team_t *, __kmp_team_pool));
3467 while (list != NULL) {
3468 kmp_team_list_item_t *item = list;
3470 KMP_INTERNAL_FREE(item);
3479 static const unsigned __kmp_primes[] = {
3480 0x9e3779b1, 0xffe6cc59, 0x2109f6dd, 0x43977ab5, 0xba5703f5, 0xb495a877,
3481 0xe1626741, 0x79695e6b, 0xbc98c09f, 0xd5bee2b3, 0x287488f9, 0x3af18231,
3482 0x9677cd4d, 0xbe3a6929, 0xadc6a877, 0xdcf0674b, 0xbe4d6fe9, 0x5f15e201,
3483 0x99afc3fd, 0xf3f16801, 0xe222cfff, 0x24ba5fdb, 0x0620452d, 0x79f149e3,
3484 0xc8b93f49, 0x972702cd, 0xb07dd827, 0x6c97d5ed, 0x085a3d61, 0x46eb5ea7,
3485 0x3d9910ed, 0x2e687b5b, 0x29609227, 0x6eb081f1, 0x0954c4e1, 0x9d114db9,
3486 0x542acfa9, 0xb3e6bd7b, 0x0742d917, 0xe9f3ffa7, 0x54581edb, 0xf2480f45,
3487 0x0bb9288f, 0xef1affc7, 0x85fa0ca7, 0x3ccc14db, 0xe6baf34b, 0x343377f7,
3488 0x5ca19031, 0xe6d9293b, 0xf0a9f391, 0x5d2e980b, 0xfc411073, 0xc3749363,
3489 0xb892d829, 0x3549366b, 0x629750ad, 0xb98294e5, 0x892d9483, 0xc235baf3,
3490 0x3d2402a3, 0x6bdef3c9, 0xbec333cd, 0x40c9520f};
3494 unsigned short __kmp_get_random(kmp_info_t *thread) {
3495 unsigned x = thread->th.th_x;
3496 unsigned short r = (
unsigned short)(x >> 16);
3498 thread->th.th_x = x * thread->th.th_a + 1;
3500 KA_TRACE(30, (
"__kmp_get_random: THREAD: %d, RETURN: %u\n",
3501 thread->th.th_info.ds.ds_tid, r));
3507 void __kmp_init_random(kmp_info_t *thread) {
3508 unsigned seed = thread->th.th_info.ds.ds_tid;
3511 __kmp_primes[seed % (
sizeof(__kmp_primes) /
sizeof(__kmp_primes[0]))];
3512 thread->th.th_x = (seed + 1) * thread->th.th_a + 1;
3514 (
"__kmp_init_random: THREAD: %u; A: %u\n", seed, thread->th.th_a));
3520 static int __kmp_reclaim_dead_roots(
void) {
3523 for (i = 0; i < __kmp_threads_capacity; ++i) {
3524 if (KMP_UBER_GTID(i) &&
3525 !__kmp_still_running((kmp_info_t *)TCR_SYNC_PTR(__kmp_threads[i])) &&
3528 r += __kmp_unregister_root_other_thread(i);
3553 static int __kmp_expand_threads(
int nNeed) {
3555 int minimumRequiredCapacity;
3557 kmp_info_t **newThreads;
3558 kmp_root_t **newRoot;
3564 #if KMP_OS_WINDOWS && !KMP_DYNAMIC_LIB
3567 added = __kmp_reclaim_dead_roots();
3596 KMP_DEBUG_ASSERT(__kmp_sys_max_nth >= __kmp_threads_capacity);
3599 if (__kmp_sys_max_nth - __kmp_threads_capacity < nNeed) {
3603 minimumRequiredCapacity = __kmp_threads_capacity + nNeed;
3605 newCapacity = __kmp_threads_capacity;
3607 newCapacity = newCapacity <= (__kmp_sys_max_nth >> 1) ? (newCapacity << 1)
3608 : __kmp_sys_max_nth;
3609 }
while (newCapacity < minimumRequiredCapacity);
3610 newThreads = (kmp_info_t **)__kmp_allocate(
3611 (
sizeof(kmp_info_t *) +
sizeof(kmp_root_t *)) * newCapacity + CACHE_LINE);
3613 (kmp_root_t **)((
char *)newThreads +
sizeof(kmp_info_t *) * newCapacity);
3614 KMP_MEMCPY(newThreads, __kmp_threads,
3615 __kmp_threads_capacity *
sizeof(kmp_info_t *));
3616 KMP_MEMCPY(newRoot, __kmp_root,
3617 __kmp_threads_capacity *
sizeof(kmp_root_t *));
3619 kmp_info_t **temp_threads = __kmp_threads;
3620 *(kmp_info_t * *
volatile *)&__kmp_threads = newThreads;
3621 *(kmp_root_t * *
volatile *)&__kmp_root = newRoot;
3622 __kmp_free(temp_threads);
3623 added += newCapacity - __kmp_threads_capacity;
3624 *(
volatile int *)&__kmp_threads_capacity = newCapacity;
3626 if (newCapacity > __kmp_tp_capacity) {
3627 __kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock);
3628 if (__kmp_tp_cached && newCapacity > __kmp_tp_capacity) {
3629 __kmp_threadprivate_resize_cache(newCapacity);
3631 *(
volatile int *)&__kmp_tp_capacity = newCapacity;
3633 __kmp_release_bootstrap_lock(&__kmp_tp_cached_lock);
3642 int __kmp_register_root(
int initial_thread) {
3643 kmp_info_t *root_thread;
3647 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
3648 KA_TRACE(20, (
"__kmp_register_root: entered\n"));
3665 capacity = __kmp_threads_capacity;
3666 if (!initial_thread && TCR_PTR(__kmp_threads[0]) == NULL) {
3673 if (__kmp_enable_hidden_helper && !TCR_4(__kmp_init_hidden_helper_threads)) {
3674 capacity -= __kmp_hidden_helper_threads_num;
3678 if (__kmp_all_nth >= capacity && !__kmp_expand_threads(1)) {
3679 if (__kmp_tp_cached) {
3680 __kmp_fatal(KMP_MSG(CantRegisterNewThread),
3681 KMP_HNT(Set_ALL_THREADPRIVATE, __kmp_tp_capacity),
3682 KMP_HNT(PossibleSystemLimitOnThreads), __kmp_msg_null);
3684 __kmp_fatal(KMP_MSG(CantRegisterNewThread), KMP_HNT(SystemLimitOnThreads),
3694 if (TCR_4(__kmp_init_hidden_helper_threads)) {
3697 for (gtid = 1; TCR_PTR(__kmp_threads[gtid]) != NULL &&
3698 gtid <= __kmp_hidden_helper_threads_num;
3701 KMP_ASSERT(gtid <= __kmp_hidden_helper_threads_num);
3702 KA_TRACE(1, (
"__kmp_register_root: found slot in threads array for "
3703 "hidden helper thread: T#%d\n",
3709 if (initial_thread && TCR_PTR(__kmp_threads[0]) == NULL) {
3712 for (gtid = __kmp_hidden_helper_threads_num + 1;
3713 TCR_PTR(__kmp_threads[gtid]) != NULL; gtid++)
3717 1, (
"__kmp_register_root: found slot in threads array: T#%d\n", gtid));
3718 KMP_ASSERT(gtid < __kmp_threads_capacity);
3723 TCW_4(__kmp_nth, __kmp_nth + 1);
3727 if (__kmp_adjust_gtid_mode) {
3728 if (__kmp_all_nth >= __kmp_tls_gtid_min) {
3729 if (TCR_4(__kmp_gtid_mode) != 2) {
3730 TCW_4(__kmp_gtid_mode, 2);
3733 if (TCR_4(__kmp_gtid_mode) != 1) {
3734 TCW_4(__kmp_gtid_mode, 1);
3739 #ifdef KMP_ADJUST_BLOCKTIME
3742 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
3743 if (__kmp_nth > __kmp_avail_proc) {
3744 __kmp_zero_bt = TRUE;
3750 if (!(root = __kmp_root[gtid])) {
3751 root = __kmp_root[gtid] = (kmp_root_t *)__kmp_allocate(
sizeof(kmp_root_t));
3752 KMP_DEBUG_ASSERT(!root->r.r_root_team);
3755 #if KMP_STATS_ENABLED
3757 __kmp_stats_thread_ptr = __kmp_stats_list->push_back(gtid);
3758 __kmp_stats_thread_ptr->startLife();
3759 KMP_SET_THREAD_STATE(SERIAL_REGION);
3762 __kmp_initialize_root(root);
3765 if (root->r.r_uber_thread) {
3766 root_thread = root->r.r_uber_thread;
3768 root_thread = (kmp_info_t *)__kmp_allocate(
sizeof(kmp_info_t));
3769 if (__kmp_storage_map) {
3770 __kmp_print_thread_storage_map(root_thread, gtid);
3772 root_thread->th.th_info.ds.ds_gtid = gtid;
3774 root_thread->th.ompt_thread_info.thread_data = ompt_data_none;
3776 root_thread->th.th_root = root;
3777 if (__kmp_env_consistency_check) {
3778 root_thread->th.th_cons = __kmp_allocate_cons_stack(gtid);
3781 __kmp_initialize_fast_memory(root_thread);
3785 KMP_DEBUG_ASSERT(root_thread->th.th_local.bget_data == NULL);
3786 __kmp_initialize_bget(root_thread);
3788 __kmp_init_random(root_thread);
3792 if (!root_thread->th.th_serial_team) {
3793 kmp_internal_control_t r_icvs = __kmp_get_global_icvs();
3794 KF_TRACE(10, (
"__kmp_register_root: before serial_team\n"));
3795 root_thread->th.th_serial_team = __kmp_allocate_team(
3800 proc_bind_default, &r_icvs, 0 USE_NESTED_HOT_ARG(NULL));
3802 KMP_ASSERT(root_thread->th.th_serial_team);
3803 KF_TRACE(10, (
"__kmp_register_root: after serial_team = %p\n",
3804 root_thread->th.th_serial_team));
3807 TCW_SYNC_PTR(__kmp_threads[gtid], root_thread);
3809 root->r.r_root_team->t.t_threads[0] = root_thread;
3810 root->r.r_hot_team->t.t_threads[0] = root_thread;
3811 root_thread->th.th_serial_team->t.t_threads[0] = root_thread;
3813 root_thread->th.th_serial_team->t.t_serialized = 0;
3814 root->r.r_uber_thread = root_thread;
3817 __kmp_initialize_info(root_thread, root->r.r_root_team, 0, gtid);
3818 TCW_4(__kmp_init_gtid, TRUE);
3821 __kmp_gtid_set_specific(gtid);
3824 __kmp_itt_thread_name(gtid);
3827 #ifdef KMP_TDATA_GTID
3830 __kmp_create_worker(gtid, root_thread, __kmp_stksize);
3831 KMP_DEBUG_ASSERT(__kmp_gtid_get_specific() == gtid);
3833 KA_TRACE(20, (
"__kmp_register_root: T#%d init T#%d(%d:%d) arrived: join=%u, "
3835 gtid, __kmp_gtid_from_tid(0, root->r.r_hot_team),
3836 root->r.r_hot_team->t.t_id, 0, KMP_INIT_BARRIER_STATE,
3837 KMP_INIT_BARRIER_STATE));
3840 for (b = 0; b < bs_last_barrier; ++b) {
3841 root_thread->th.th_bar[b].bb.b_arrived = KMP_INIT_BARRIER_STATE;
3843 root_thread->th.th_bar[b].bb.b_worker_arrived = 0;
3847 KMP_DEBUG_ASSERT(root->r.r_hot_team->t.t_bar[bs_forkjoin_barrier].b_arrived ==
3848 KMP_INIT_BARRIER_STATE);
3850 #if KMP_AFFINITY_SUPPORTED
3851 root_thread->th.th_current_place = KMP_PLACE_UNDEFINED;
3852 root_thread->th.th_new_place = KMP_PLACE_UNDEFINED;
3853 root_thread->th.th_first_place = KMP_PLACE_UNDEFINED;
3854 root_thread->th.th_last_place = KMP_PLACE_UNDEFINED;
3856 root_thread->th.th_def_allocator = __kmp_def_allocator;
3857 root_thread->th.th_prev_level = 0;
3858 root_thread->th.th_prev_num_threads = 1;
3860 kmp_cg_root_t *tmp = (kmp_cg_root_t *)__kmp_allocate(
sizeof(kmp_cg_root_t));
3861 tmp->cg_root = root_thread;
3862 tmp->cg_thread_limit = __kmp_cg_max_nth;
3863 tmp->cg_nthreads = 1;
3864 KA_TRACE(100, (
"__kmp_register_root: Thread %p created node %p with"
3865 " cg_nthreads init to 1\n",
3868 root_thread->th.th_cg_roots = tmp;
3870 __kmp_root_counter++;
3873 if (!initial_thread && ompt_enabled.enabled) {
3875 kmp_info_t *root_thread = ompt_get_thread();
3877 ompt_set_thread_state(root_thread, ompt_state_overhead);
3879 if (ompt_enabled.ompt_callback_thread_begin) {
3880 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)(
3881 ompt_thread_initial, __ompt_get_thread_data_internal());
3883 ompt_data_t *task_data;
3884 ompt_data_t *parallel_data;
3885 __ompt_get_task_info_internal(0, NULL, &task_data, NULL, ¶llel_data,
3887 if (ompt_enabled.ompt_callback_implicit_task) {
3888 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
3889 ompt_scope_begin, parallel_data, task_data, 1, 1, ompt_task_initial);
3892 ompt_set_thread_state(root_thread, ompt_state_work_serial);
3896 if (ompd_state & OMPD_ENABLE_BP)
3897 ompd_bp_thread_begin();
3901 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
3906 #if KMP_NESTED_HOT_TEAMS
3907 static int __kmp_free_hot_teams(kmp_root_t *root, kmp_info_t *thr,
int level,
3908 const int max_level) {
3910 kmp_hot_team_ptr_t *hot_teams = thr->th.th_hot_teams;
3911 if (!hot_teams || !hot_teams[level].hot_team) {
3914 KMP_DEBUG_ASSERT(level < max_level);
3915 kmp_team_t *team = hot_teams[level].hot_team;
3916 nth = hot_teams[level].hot_team_nth;
3918 if (level < max_level - 1) {
3919 for (i = 0; i < nth; ++i) {
3920 kmp_info_t *th = team->t.t_threads[i];
3921 n += __kmp_free_hot_teams(root, th, level + 1, max_level);
3922 if (i > 0 && th->th.th_hot_teams) {
3923 __kmp_free(th->th.th_hot_teams);
3924 th->th.th_hot_teams = NULL;
3928 __kmp_free_team(root, team, NULL);
3935 static int __kmp_reset_root(
int gtid, kmp_root_t *root) {
3936 kmp_team_t *root_team = root->r.r_root_team;
3937 kmp_team_t *hot_team = root->r.r_hot_team;
3938 int n = hot_team->t.t_nproc;
3941 KMP_DEBUG_ASSERT(!root->r.r_active);
3943 root->r.r_root_team = NULL;
3944 root->r.r_hot_team = NULL;
3947 __kmp_free_team(root, root_team USE_NESTED_HOT_ARG(NULL));
3948 #if KMP_NESTED_HOT_TEAMS
3949 if (__kmp_hot_teams_max_level >
3951 for (i = 0; i < hot_team->t.t_nproc; ++i) {
3952 kmp_info_t *th = hot_team->t.t_threads[i];
3953 if (__kmp_hot_teams_max_level > 1) {
3954 n += __kmp_free_hot_teams(root, th, 1, __kmp_hot_teams_max_level);
3956 if (th->th.th_hot_teams) {
3957 __kmp_free(th->th.th_hot_teams);
3958 th->th.th_hot_teams = NULL;
3963 __kmp_free_team(root, hot_team USE_NESTED_HOT_ARG(NULL));
3968 if (__kmp_tasking_mode != tskm_immediate_exec) {
3969 __kmp_wait_to_unref_task_teams();
3975 10, (
"__kmp_reset_root: free handle, th = %p, handle = %" KMP_UINTPTR_SPEC
3977 (LPVOID) & (root->r.r_uber_thread->th),
3978 root->r.r_uber_thread->th.th_info.ds.ds_thread));
3979 __kmp_free_handle(root->r.r_uber_thread->th.th_info.ds.ds_thread);
3983 if (ompd_state & OMPD_ENABLE_BP)
3984 ompd_bp_thread_end();
3988 ompt_data_t *task_data;
3989 ompt_data_t *parallel_data;
3990 __ompt_get_task_info_internal(0, NULL, &task_data, NULL, ¶llel_data,
3992 if (ompt_enabled.ompt_callback_implicit_task) {
3993 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
3994 ompt_scope_end, parallel_data, task_data, 0, 1, ompt_task_initial);
3996 if (ompt_enabled.ompt_callback_thread_end) {
3997 ompt_callbacks.ompt_callback(ompt_callback_thread_end)(
3998 &(root->r.r_uber_thread->th.ompt_thread_info.thread_data));
4004 i = root->r.r_uber_thread->th.th_cg_roots->cg_nthreads--;
4005 KA_TRACE(100, (
"__kmp_reset_root: Thread %p decrement cg_nthreads on node %p"
4007 root->r.r_uber_thread, root->r.r_uber_thread->th.th_cg_roots,
4008 root->r.r_uber_thread->th.th_cg_roots->cg_nthreads));
4011 KMP_DEBUG_ASSERT(root->r.r_uber_thread ==
4012 root->r.r_uber_thread->th.th_cg_roots->cg_root);
4013 KMP_DEBUG_ASSERT(root->r.r_uber_thread->th.th_cg_roots->up == NULL);
4014 __kmp_free(root->r.r_uber_thread->th.th_cg_roots);
4015 root->r.r_uber_thread->th.th_cg_roots = NULL;
4017 __kmp_reap_thread(root->r.r_uber_thread, 1);
4021 root->r.r_uber_thread = NULL;
4023 root->r.r_begin = FALSE;
4028 void __kmp_unregister_root_current_thread(
int gtid) {
4029 KA_TRACE(1, (
"__kmp_unregister_root_current_thread: enter T#%d\n", gtid));
4033 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
4034 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
4035 KC_TRACE(10, (
"__kmp_unregister_root_current_thread: already finished, "
4038 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
4041 kmp_root_t *root = __kmp_root[gtid];
4043 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]);
4044 KMP_ASSERT(KMP_UBER_GTID(gtid));
4045 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root);
4046 KMP_ASSERT(root->r.r_active == FALSE);
4050 kmp_info_t *thread = __kmp_threads[gtid];
4051 kmp_team_t *team = thread->th.th_team;
4052 kmp_task_team_t *task_team = thread->th.th_task_team;
4055 if (task_team != NULL && task_team->tt.tt_found_proxy_tasks) {
4058 thread->th.ompt_thread_info.state = ompt_state_undefined;
4060 __kmp_task_team_wait(thread, team USE_ITT_BUILD_ARG(NULL));
4063 __kmp_reset_root(gtid, root);
4067 (
"__kmp_unregister_root_current_thread: T#%d unregistered\n", gtid));
4069 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
4076 static int __kmp_unregister_root_other_thread(
int gtid) {
4077 kmp_root_t *root = __kmp_root[gtid];
4080 KA_TRACE(1, (
"__kmp_unregister_root_other_thread: enter T#%d\n", gtid));
4081 KMP_DEBUG_ASSERT(__kmp_threads && __kmp_threads[gtid]);
4082 KMP_ASSERT(KMP_UBER_GTID(gtid));
4083 KMP_ASSERT(root == __kmp_threads[gtid]->th.th_root);
4084 KMP_ASSERT(root->r.r_active == FALSE);
4086 r = __kmp_reset_root(gtid, root);
4088 (
"__kmp_unregister_root_other_thread: T#%d unregistered\n", gtid));
4094 void __kmp_task_info() {
4096 kmp_int32 gtid = __kmp_entry_gtid();
4097 kmp_int32 tid = __kmp_tid_from_gtid(gtid);
4098 kmp_info_t *this_thr = __kmp_threads[gtid];
4099 kmp_team_t *steam = this_thr->th.th_serial_team;
4100 kmp_team_t *team = this_thr->th.th_team;
4103 "__kmp_task_info: gtid=%d tid=%d t_thread=%p team=%p steam=%p curtask=%p "
4105 gtid, tid, this_thr, team, steam, this_thr->th.th_current_task,
4106 team->t.t_implicit_task_taskdata[tid].td_parent);
4113 static void __kmp_initialize_info(kmp_info_t *this_thr, kmp_team_t *team,
4114 int tid,
int gtid) {
4118 KMP_DEBUG_ASSERT(this_thr != NULL);
4119 KMP_DEBUG_ASSERT(this_thr->th.th_serial_team);
4120 KMP_DEBUG_ASSERT(team);
4121 KMP_DEBUG_ASSERT(team->t.t_threads);
4122 KMP_DEBUG_ASSERT(team->t.t_dispatch);
4123 kmp_info_t *master = team->t.t_threads[0];
4124 KMP_DEBUG_ASSERT(master);
4125 KMP_DEBUG_ASSERT(master->th.th_root);
4129 TCW_SYNC_PTR(this_thr->th.th_team, team);
4131 this_thr->th.th_info.ds.ds_tid = tid;
4132 this_thr->th.th_set_nproc = 0;
4133 if (__kmp_tasking_mode != tskm_immediate_exec)
4136 this_thr->th.th_reap_state = KMP_NOT_SAFE_TO_REAP;
4138 this_thr->th.th_reap_state = KMP_SAFE_TO_REAP;
4139 this_thr->th.th_set_proc_bind = proc_bind_default;
4140 #if KMP_AFFINITY_SUPPORTED
4141 this_thr->th.th_new_place = this_thr->th.th_current_place;
4143 this_thr->th.th_root = master->th.th_root;
4146 this_thr->th.th_team_nproc = team->t.t_nproc;
4147 this_thr->th.th_team_master = master;
4148 this_thr->th.th_team_serialized = team->t.t_serialized;
4150 KMP_DEBUG_ASSERT(team->t.t_implicit_task_taskdata);
4152 KF_TRACE(10, (
"__kmp_initialize_info1: T#%d:%d this_thread=%p curtask=%p\n",
4153 tid, gtid, this_thr, this_thr->th.th_current_task));
4155 __kmp_init_implicit_task(this_thr->th.th_team_master->th.th_ident, this_thr,
4158 KF_TRACE(10, (
"__kmp_initialize_info2: T#%d:%d this_thread=%p curtask=%p\n",
4159 tid, gtid, this_thr, this_thr->th.th_current_task));
4164 this_thr->th.th_dispatch = &team->t.t_dispatch[tid];
4166 this_thr->th.th_local.this_construct = 0;
4168 if (!this_thr->th.th_pri_common) {
4169 this_thr->th.th_pri_common =
4170 (
struct common_table *)__kmp_allocate(
sizeof(
struct common_table));
4171 if (__kmp_storage_map) {
4172 __kmp_print_storage_map_gtid(
4173 gtid, this_thr->th.th_pri_common, this_thr->th.th_pri_common + 1,
4174 sizeof(
struct common_table),
"th_%d.th_pri_common\n", gtid);
4176 this_thr->th.th_pri_head = NULL;
4179 if (this_thr != master &&
4180 this_thr->th.th_cg_roots != master->th.th_cg_roots) {
4182 KMP_DEBUG_ASSERT(master->th.th_cg_roots);
4183 kmp_cg_root_t *tmp = this_thr->th.th_cg_roots;
4186 int i = tmp->cg_nthreads--;
4187 KA_TRACE(100, (
"__kmp_initialize_info: Thread %p decrement cg_nthreads"
4188 " on node %p of thread %p to %d\n",
4189 this_thr, tmp, tmp->cg_root, tmp->cg_nthreads));
4194 this_thr->th.th_cg_roots = master->th.th_cg_roots;
4196 this_thr->th.th_cg_roots->cg_nthreads++;
4197 KA_TRACE(100, (
"__kmp_initialize_info: Thread %p increment cg_nthreads on"
4198 " node %p of thread %p to %d\n",
4199 this_thr, this_thr->th.th_cg_roots,
4200 this_thr->th.th_cg_roots->cg_root,
4201 this_thr->th.th_cg_roots->cg_nthreads));
4202 this_thr->th.th_current_task->td_icvs.thread_limit =
4203 this_thr->th.th_cg_roots->cg_thread_limit;
4208 volatile kmp_disp_t *dispatch = this_thr->th.th_dispatch;
4211 sizeof(dispatch_private_info_t) *
4212 (team->t.t_max_nproc == 1 ? 1 : __kmp_dispatch_num_buffers);
4213 KD_TRACE(10, (
"__kmp_initialize_info: T#%d max_nproc: %d\n", gtid,
4214 team->t.t_max_nproc));
4215 KMP_ASSERT(dispatch);
4216 KMP_DEBUG_ASSERT(team->t.t_dispatch);
4217 KMP_DEBUG_ASSERT(dispatch == &team->t.t_dispatch[tid]);
4219 dispatch->th_disp_index = 0;
4220 dispatch->th_doacross_buf_idx = 0;
4221 if (!dispatch->th_disp_buffer) {
4222 dispatch->th_disp_buffer =
4223 (dispatch_private_info_t *)__kmp_allocate(disp_size);
4225 if (__kmp_storage_map) {
4226 __kmp_print_storage_map_gtid(
4227 gtid, &dispatch->th_disp_buffer[0],
4228 &dispatch->th_disp_buffer[team->t.t_max_nproc == 1
4230 : __kmp_dispatch_num_buffers],
4232 "th_%d.th_dispatch.th_disp_buffer "
4233 "(team_%d.t_dispatch[%d].th_disp_buffer)",
4234 gtid, team->t.t_id, gtid);
4237 memset(&dispatch->th_disp_buffer[0],
'\0', disp_size);
4240 dispatch->th_dispatch_pr_current = 0;
4241 dispatch->th_dispatch_sh_current = 0;
4243 dispatch->th_deo_fcn = 0;
4244 dispatch->th_dxo_fcn = 0;
4247 this_thr->th.th_next_pool = NULL;
4249 if (!this_thr->th.th_task_state_memo_stack) {
4251 this_thr->th.th_task_state_memo_stack =
4252 (kmp_uint8 *)__kmp_allocate(4 *
sizeof(kmp_uint8));
4253 this_thr->th.th_task_state_top = 0;
4254 this_thr->th.th_task_state_stack_sz = 4;
4255 for (i = 0; i < this_thr->th.th_task_state_stack_sz;
4257 this_thr->th.th_task_state_memo_stack[i] = 0;
4260 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
4261 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
4271 kmp_info_t *__kmp_allocate_thread(kmp_root_t *root, kmp_team_t *team,
4273 kmp_team_t *serial_team;
4274 kmp_info_t *new_thr;
4277 KA_TRACE(20, (
"__kmp_allocate_thread: T#%d\n", __kmp_get_gtid()));
4278 KMP_DEBUG_ASSERT(root && team);
4279 #if !KMP_NESTED_HOT_TEAMS
4280 KMP_DEBUG_ASSERT(KMP_MASTER_GTID(__kmp_get_gtid()));
4285 if (__kmp_thread_pool) {
4286 new_thr = CCAST(kmp_info_t *, __kmp_thread_pool);
4287 __kmp_thread_pool = (
volatile kmp_info_t *)new_thr->th.th_next_pool;
4288 if (new_thr == __kmp_thread_pool_insert_pt) {
4289 __kmp_thread_pool_insert_pt = NULL;
4291 TCW_4(new_thr->th.th_in_pool, FALSE);
4292 __kmp_suspend_initialize_thread(new_thr);
4293 __kmp_lock_suspend_mx(new_thr);
4294 if (new_thr->th.th_active_in_pool == TRUE) {
4295 KMP_DEBUG_ASSERT(new_thr->th.th_active == TRUE);
4296 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
4297 new_thr->th.th_active_in_pool = FALSE;
4299 __kmp_unlock_suspend_mx(new_thr);
4301 KA_TRACE(20, (
"__kmp_allocate_thread: T#%d using thread T#%d\n",
4302 __kmp_get_gtid(), new_thr->th.th_info.ds.ds_gtid));
4303 KMP_ASSERT(!new_thr->th.th_team);
4304 KMP_DEBUG_ASSERT(__kmp_nth < __kmp_threads_capacity);
4307 __kmp_initialize_info(new_thr, team, new_tid,
4308 new_thr->th.th_info.ds.ds_gtid);
4309 KMP_DEBUG_ASSERT(new_thr->th.th_serial_team);
4311 TCW_4(__kmp_nth, __kmp_nth + 1);
4313 new_thr->th.th_task_state = 0;
4314 new_thr->th.th_task_state_top = 0;
4315 new_thr->th.th_task_state_stack_sz = 4;
4317 if (__kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
4319 KMP_DEBUG_ASSERT(new_thr->th.th_used_in_team.load() == 0);
4323 #ifdef KMP_ADJUST_BLOCKTIME
4326 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
4327 if (__kmp_nth > __kmp_avail_proc) {
4328 __kmp_zero_bt = TRUE;
4337 kmp_balign_t *balign = new_thr->th.th_bar;
4338 for (b = 0; b < bs_last_barrier; ++b)
4339 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
4342 KF_TRACE(10, (
"__kmp_allocate_thread: T#%d using thread %p T#%d\n",
4343 __kmp_get_gtid(), new_thr, new_thr->th.th_info.ds.ds_gtid));
4350 KMP_ASSERT(__kmp_nth == __kmp_all_nth);
4351 KMP_ASSERT(__kmp_all_nth < __kmp_threads_capacity);
4356 if (!TCR_4(__kmp_init_monitor)) {
4357 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
4358 if (!TCR_4(__kmp_init_monitor)) {
4359 KF_TRACE(10, (
"before __kmp_create_monitor\n"));
4360 TCW_4(__kmp_init_monitor, 1);
4361 __kmp_create_monitor(&__kmp_monitor);
4362 KF_TRACE(10, (
"after __kmp_create_monitor\n"));
4373 while (TCR_4(__kmp_init_monitor) < 2) {
4376 KF_TRACE(10, (
"after monitor thread has started\n"));
4379 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
4386 int new_start_gtid = TCR_4(__kmp_init_hidden_helper_threads)
4388 : __kmp_hidden_helper_threads_num + 1;
4390 for (new_gtid = new_start_gtid; TCR_PTR(__kmp_threads[new_gtid]) != NULL;
4392 KMP_DEBUG_ASSERT(new_gtid < __kmp_threads_capacity);
4395 if (TCR_4(__kmp_init_hidden_helper_threads)) {
4396 KMP_DEBUG_ASSERT(new_gtid <= __kmp_hidden_helper_threads_num);
4401 new_thr = (kmp_info_t *)__kmp_allocate(
sizeof(kmp_info_t));
4403 TCW_SYNC_PTR(__kmp_threads[new_gtid], new_thr);
4405 #if USE_ITT_BUILD && USE_ITT_NOTIFY && KMP_DEBUG
4408 __itt_suppress_mark_range(
4409 __itt_suppress_range, __itt_suppress_threading_errors,
4410 &new_thr->th.th_sleep_loc,
sizeof(new_thr->th.th_sleep_loc));
4411 __itt_suppress_mark_range(
4412 __itt_suppress_range, __itt_suppress_threading_errors,
4413 &new_thr->th.th_reap_state,
sizeof(new_thr->th.th_reap_state));
4415 __itt_suppress_mark_range(
4416 __itt_suppress_range, __itt_suppress_threading_errors,
4417 &new_thr->th.th_suspend_init,
sizeof(new_thr->th.th_suspend_init));
4419 __itt_suppress_mark_range(__itt_suppress_range,
4420 __itt_suppress_threading_errors,
4421 &new_thr->th.th_suspend_init_count,
4422 sizeof(new_thr->th.th_suspend_init_count));
4425 __itt_suppress_mark_range(__itt_suppress_range,
4426 __itt_suppress_threading_errors,
4427 CCAST(kmp_uint64 *, &new_thr->th.th_bar[0].bb.b_go),
4428 sizeof(new_thr->th.th_bar[0].bb.b_go));
4429 __itt_suppress_mark_range(__itt_suppress_range,
4430 __itt_suppress_threading_errors,
4431 CCAST(kmp_uint64 *, &new_thr->th.th_bar[1].bb.b_go),
4432 sizeof(new_thr->th.th_bar[1].bb.b_go));
4433 __itt_suppress_mark_range(__itt_suppress_range,
4434 __itt_suppress_threading_errors,
4435 CCAST(kmp_uint64 *, &new_thr->th.th_bar[2].bb.b_go),
4436 sizeof(new_thr->th.th_bar[2].bb.b_go));
4438 if (__kmp_storage_map) {
4439 __kmp_print_thread_storage_map(new_thr, new_gtid);
4444 kmp_internal_control_t r_icvs = __kmp_get_x_global_icvs(team);
4445 KF_TRACE(10, (
"__kmp_allocate_thread: before th_serial/serial_team\n"));
4446 new_thr->th.th_serial_team = serial_team =
4447 (kmp_team_t *)__kmp_allocate_team(root, 1, 1,
4451 proc_bind_default, &r_icvs,
4452 0 USE_NESTED_HOT_ARG(NULL));
4454 KMP_ASSERT(serial_team);
4455 serial_team->t.t_serialized = 0;
4457 serial_team->t.t_threads[0] = new_thr;
4459 (
"__kmp_allocate_thread: after th_serial/serial_team : new_thr=%p\n",
4463 __kmp_initialize_info(new_thr, team, new_tid, new_gtid);
4466 __kmp_initialize_fast_memory(new_thr);
4470 KMP_DEBUG_ASSERT(new_thr->th.th_local.bget_data == NULL);
4471 __kmp_initialize_bget(new_thr);
4474 __kmp_init_random(new_thr);
4478 (
"__kmp_allocate_thread: T#%d init go fork=%u, plain=%u\n",
4479 __kmp_get_gtid(), KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
4482 kmp_balign_t *balign = new_thr->th.th_bar;
4483 for (b = 0; b < bs_last_barrier; ++b) {
4484 balign[b].bb.b_go = KMP_INIT_BARRIER_STATE;
4485 balign[b].bb.team = NULL;
4486 balign[b].bb.wait_flag = KMP_BARRIER_NOT_WAITING;
4487 balign[b].bb.use_oncore_barrier = 0;
4490 TCW_PTR(new_thr->th.th_sleep_loc, NULL);
4491 new_thr->th.th_sleep_loc_type = flag_unset;
4493 new_thr->th.th_spin_here = FALSE;
4494 new_thr->th.th_next_waiting = 0;
4496 new_thr->th.th_blocking =
false;
4499 #if KMP_AFFINITY_SUPPORTED
4500 new_thr->th.th_current_place = KMP_PLACE_UNDEFINED;
4501 new_thr->th.th_new_place = KMP_PLACE_UNDEFINED;
4502 new_thr->th.th_first_place = KMP_PLACE_UNDEFINED;
4503 new_thr->th.th_last_place = KMP_PLACE_UNDEFINED;
4505 new_thr->th.th_def_allocator = __kmp_def_allocator;
4506 new_thr->th.th_prev_level = 0;
4507 new_thr->th.th_prev_num_threads = 1;
4509 TCW_4(new_thr->th.th_in_pool, FALSE);
4510 new_thr->th.th_active_in_pool = FALSE;
4511 TCW_4(new_thr->th.th_active, TRUE);
4519 if (__kmp_adjust_gtid_mode) {
4520 if (__kmp_all_nth >= __kmp_tls_gtid_min) {
4521 if (TCR_4(__kmp_gtid_mode) != 2) {
4522 TCW_4(__kmp_gtid_mode, 2);
4525 if (TCR_4(__kmp_gtid_mode) != 1) {
4526 TCW_4(__kmp_gtid_mode, 1);
4531 #ifdef KMP_ADJUST_BLOCKTIME
4534 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
4535 if (__kmp_nth > __kmp_avail_proc) {
4536 __kmp_zero_bt = TRUE;
4543 10, (
"__kmp_allocate_thread: before __kmp_create_worker: %p\n", new_thr));
4544 __kmp_create_worker(new_gtid, new_thr, __kmp_stksize);
4546 (
"__kmp_allocate_thread: after __kmp_create_worker: %p\n", new_thr));
4548 KA_TRACE(20, (
"__kmp_allocate_thread: T#%d forked T#%d\n", __kmp_get_gtid(),
4559 static void __kmp_reinitialize_team(kmp_team_t *team,
4560 kmp_internal_control_t *new_icvs,
4562 KF_TRACE(10, (
"__kmp_reinitialize_team: enter this_thread=%p team=%p\n",
4563 team->t.t_threads[0], team));
4564 KMP_DEBUG_ASSERT(team && new_icvs);
4565 KMP_DEBUG_ASSERT((!TCR_4(__kmp_init_parallel)) || new_icvs->nproc);
4566 KMP_CHECK_UPDATE(team->t.t_ident, loc);
4568 KMP_CHECK_UPDATE(team->t.t_id, KMP_GEN_TEAM_ID());
4570 __kmp_init_implicit_task(loc, team->t.t_threads[0], team, 0, FALSE);
4571 copy_icvs(&team->t.t_implicit_task_taskdata[0].td_icvs, new_icvs);
4573 KF_TRACE(10, (
"__kmp_reinitialize_team: exit this_thread=%p team=%p\n",
4574 team->t.t_threads[0], team));
4580 static void __kmp_initialize_team(kmp_team_t *team,
int new_nproc,
4581 kmp_internal_control_t *new_icvs,
4583 KF_TRACE(10, (
"__kmp_initialize_team: enter: team=%p\n", team));
4586 KMP_DEBUG_ASSERT(team);
4587 KMP_DEBUG_ASSERT(new_nproc <= team->t.t_max_nproc);
4588 KMP_DEBUG_ASSERT(team->t.t_threads);
4591 team->t.t_master_tid = 0;
4593 team->t.t_serialized = new_nproc > 1 ? 0 : 1;
4594 team->t.t_nproc = new_nproc;
4597 team->t.t_next_pool = NULL;
4601 TCW_SYNC_PTR(team->t.t_pkfn, NULL);
4602 team->t.t_invoke = NULL;
4605 team->t.t_sched.sched = new_icvs->sched.sched;
4607 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
4608 team->t.t_fp_control_saved = FALSE;
4609 team->t.t_x87_fpu_control_word = 0;
4610 team->t.t_mxcsr = 0;
4613 team->t.t_construct = 0;
4615 team->t.t_ordered.dt.t_value = 0;
4616 team->t.t_master_active = FALSE;
4619 team->t.t_copypriv_data = NULL;
4622 team->t.t_copyin_counter = 0;
4625 team->t.t_control_stack_top = NULL;
4627 __kmp_reinitialize_team(team, new_icvs, loc);
4630 KF_TRACE(10, (
"__kmp_initialize_team: exit: team=%p\n", team));
4633 #if (KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED
4636 __kmp_set_thread_affinity_mask_full_tmp(kmp_affin_mask_t *old_mask) {
4637 if (KMP_AFFINITY_CAPABLE()) {
4639 if (old_mask != NULL) {
4640 status = __kmp_get_system_affinity(old_mask, TRUE);
4643 __kmp_fatal(KMP_MSG(ChangeThreadAffMaskError), KMP_ERR(error),
4647 __kmp_set_system_affinity(__kmp_affin_fullMask, TRUE);
4652 #if KMP_AFFINITY_SUPPORTED
4658 static void __kmp_partition_places(kmp_team_t *team,
int update_master_only) {
4660 if (KMP_HIDDEN_HELPER_TEAM(team))
4663 kmp_info_t *master_th = team->t.t_threads[0];
4664 KMP_DEBUG_ASSERT(master_th != NULL);
4665 kmp_proc_bind_t proc_bind = team->t.t_proc_bind;
4666 int first_place = master_th->th.th_first_place;
4667 int last_place = master_th->th.th_last_place;
4668 int masters_place = master_th->th.th_current_place;
4669 team->t.t_first_place = first_place;
4670 team->t.t_last_place = last_place;
4672 KA_TRACE(20, (
"__kmp_partition_places: enter: proc_bind = %d T#%d(%d:0) "
4673 "bound to place %d partition = [%d,%d]\n",
4674 proc_bind, __kmp_gtid_from_thread(team->t.t_threads[0]),
4675 team->t.t_id, masters_place, first_place, last_place));
4677 switch (proc_bind) {
4679 case proc_bind_default:
4682 KMP_DEBUG_ASSERT(team->t.t_nproc == 1);
4685 case proc_bind_primary: {
4687 int n_th = team->t.t_nproc;
4688 for (f = 1; f < n_th; f++) {
4689 kmp_info_t *th = team->t.t_threads[f];
4690 KMP_DEBUG_ASSERT(th != NULL);
4691 th->th.th_first_place = first_place;
4692 th->th.th_last_place = last_place;
4693 th->th.th_new_place = masters_place;
4694 if (__kmp_display_affinity && masters_place != th->th.th_current_place &&
4695 team->t.t_display_affinity != 1) {
4696 team->t.t_display_affinity = 1;
4699 KA_TRACE(100, (
"__kmp_partition_places: primary: T#%d(%d:%d) place %d "
4700 "partition = [%d,%d]\n",
4701 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,
4702 f, masters_place, first_place, last_place));
4706 case proc_bind_close: {
4708 int n_th = team->t.t_nproc;
4710 if (first_place <= last_place) {
4711 n_places = last_place - first_place + 1;
4713 n_places = __kmp_affinity_num_masks - first_place + last_place + 1;
4715 if (n_th <= n_places) {
4716 int place = masters_place;
4717 for (f = 1; f < n_th; f++) {
4718 kmp_info_t *th = team->t.t_threads[f];
4719 KMP_DEBUG_ASSERT(th != NULL);
4721 if (place == last_place) {
4722 place = first_place;
4723 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4728 th->th.th_first_place = first_place;
4729 th->th.th_last_place = last_place;
4730 th->th.th_new_place = place;
4731 if (__kmp_display_affinity && place != th->th.th_current_place &&
4732 team->t.t_display_affinity != 1) {
4733 team->t.t_display_affinity = 1;
4736 KA_TRACE(100, (
"__kmp_partition_places: close: T#%d(%d:%d) place %d "
4737 "partition = [%d,%d]\n",
4738 __kmp_gtid_from_thread(team->t.t_threads[f]),
4739 team->t.t_id, f, place, first_place, last_place));
4742 int S, rem, gap, s_count;
4743 S = n_th / n_places;
4745 rem = n_th - (S * n_places);
4746 gap = rem > 0 ? n_places / rem : n_places;
4747 int place = masters_place;
4749 for (f = 0; f < n_th; f++) {
4750 kmp_info_t *th = team->t.t_threads[f];
4751 KMP_DEBUG_ASSERT(th != NULL);
4753 th->th.th_first_place = first_place;
4754 th->th.th_last_place = last_place;
4755 th->th.th_new_place = place;
4756 if (__kmp_display_affinity && place != th->th.th_current_place &&
4757 team->t.t_display_affinity != 1) {
4758 team->t.t_display_affinity = 1;
4762 if ((s_count == S) && rem && (gap_ct == gap)) {
4764 }
else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
4766 if (place == last_place) {
4767 place = first_place;
4768 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4776 }
else if (s_count == S) {
4777 if (place == last_place) {
4778 place = first_place;
4779 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4789 (
"__kmp_partition_places: close: T#%d(%d:%d) place %d "
4790 "partition = [%d,%d]\n",
4791 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id, f,
4792 th->th.th_new_place, first_place, last_place));
4794 KMP_DEBUG_ASSERT(place == masters_place);
4798 case proc_bind_spread: {
4800 int n_th = team->t.t_nproc;
4803 if (first_place <= last_place) {
4804 n_places = last_place - first_place + 1;
4806 n_places = __kmp_affinity_num_masks - first_place + last_place + 1;
4808 if (n_th <= n_places) {
4811 if (n_places !=
static_cast<int>(__kmp_affinity_num_masks)) {
4812 int S = n_places / n_th;
4813 int s_count, rem, gap, gap_ct;
4815 place = masters_place;
4816 rem = n_places - n_th * S;
4817 gap = rem ? n_th / rem : 1;
4820 if (update_master_only == 1)
4822 for (f = 0; f < thidx; f++) {
4823 kmp_info_t *th = team->t.t_threads[f];
4824 KMP_DEBUG_ASSERT(th != NULL);
4826 th->th.th_first_place = place;
4827 th->th.th_new_place = place;
4828 if (__kmp_display_affinity && place != th->th.th_current_place &&
4829 team->t.t_display_affinity != 1) {
4830 team->t.t_display_affinity = 1;
4833 while (s_count < S) {
4834 if (place == last_place) {
4835 place = first_place;
4836 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4843 if (rem && (gap_ct == gap)) {
4844 if (place == last_place) {
4845 place = first_place;
4846 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4854 th->th.th_last_place = place;
4857 if (place == last_place) {
4858 place = first_place;
4859 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4866 (
"__kmp_partition_places: spread: T#%d(%d:%d) place %d "
4867 "partition = [%d,%d], __kmp_affinity_num_masks: %u\n",
4868 __kmp_gtid_from_thread(team->t.t_threads[f]), team->t.t_id,
4869 f, th->th.th_new_place, th->th.th_first_place,
4870 th->th.th_last_place, __kmp_affinity_num_masks));
4876 double current =
static_cast<double>(masters_place);
4878 (
static_cast<double>(n_places + 1) /
static_cast<double>(n_th));
4883 if (update_master_only == 1)
4885 for (f = 0; f < thidx; f++) {
4886 first =
static_cast<int>(current);
4887 last =
static_cast<int>(current + spacing) - 1;
4888 KMP_DEBUG_ASSERT(last >= first);
4889 if (first >= n_places) {
4890 if (masters_place) {
4893 if (first == (masters_place + 1)) {
4894 KMP_DEBUG_ASSERT(f == n_th);
4897 if (last == masters_place) {
4898 KMP_DEBUG_ASSERT(f == (n_th - 1));
4902 KMP_DEBUG_ASSERT(f == n_th);
4907 if (last >= n_places) {
4908 last = (n_places - 1);
4913 KMP_DEBUG_ASSERT(0 <= first);
4914 KMP_DEBUG_ASSERT(n_places > first);
4915 KMP_DEBUG_ASSERT(0 <= last);
4916 KMP_DEBUG_ASSERT(n_places > last);
4917 KMP_DEBUG_ASSERT(last_place >= first_place);
4918 th = team->t.t_threads[f];
4919 KMP_DEBUG_ASSERT(th);
4920 th->th.th_first_place = first;
4921 th->th.th_new_place = place;
4922 th->th.th_last_place = last;
4923 if (__kmp_display_affinity && place != th->th.th_current_place &&
4924 team->t.t_display_affinity != 1) {
4925 team->t.t_display_affinity = 1;
4928 (
"__kmp_partition_places: spread: T#%d(%d:%d) place %d "
4929 "partition = [%d,%d], spacing = %.4f\n",
4930 __kmp_gtid_from_thread(team->t.t_threads[f]),
4931 team->t.t_id, f, th->th.th_new_place,
4932 th->th.th_first_place, th->th.th_last_place, spacing));
4936 KMP_DEBUG_ASSERT(update_master_only || place == masters_place);
4938 int S, rem, gap, s_count;
4939 S = n_th / n_places;
4941 rem = n_th - (S * n_places);
4942 gap = rem > 0 ? n_places / rem : n_places;
4943 int place = masters_place;
4946 if (update_master_only == 1)
4948 for (f = 0; f < thidx; f++) {
4949 kmp_info_t *th = team->t.t_threads[f];
4950 KMP_DEBUG_ASSERT(th != NULL);
4952 th->th.th_first_place = place;
4953 th->th.th_last_place = place;
4954 th->th.th_new_place = place;
4955 if (__kmp_display_affinity && place != th->th.th_current_place &&
4956 team->t.t_display_affinity != 1) {
4957 team->t.t_display_affinity = 1;
4961 if ((s_count == S) && rem && (gap_ct == gap)) {
4963 }
else if ((s_count == S + 1) && rem && (gap_ct == gap)) {
4965 if (place == last_place) {
4966 place = first_place;
4967 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4975 }
else if (s_count == S) {
4976 if (place == last_place) {
4977 place = first_place;
4978 }
else if (place == (
int)(__kmp_affinity_num_masks - 1)) {
4987 KA_TRACE(100, (
"__kmp_partition_places: spread: T#%d(%d:%d) place %d "
4988 "partition = [%d,%d]\n",
4989 __kmp_gtid_from_thread(team->t.t_threads[f]),
4990 team->t.t_id, f, th->th.th_new_place,
4991 th->th.th_first_place, th->th.th_last_place));
4993 KMP_DEBUG_ASSERT(update_master_only || place == masters_place);
5001 KA_TRACE(20, (
"__kmp_partition_places: exit T#%d\n", team->t.t_id));
5009 __kmp_allocate_team(kmp_root_t *root,
int new_nproc,
int max_nproc,
5011 ompt_data_t ompt_parallel_data,
5013 kmp_proc_bind_t new_proc_bind,
5014 kmp_internal_control_t *new_icvs,
5015 int argc USE_NESTED_HOT_ARG(kmp_info_t *master)) {
5016 KMP_TIME_DEVELOPER_PARTITIONED_BLOCK(KMP_allocate_team);
5019 int use_hot_team = !root->r.r_active;
5022 KA_TRACE(20, (
"__kmp_allocate_team: called\n"));
5023 KMP_DEBUG_ASSERT(new_nproc >= 1 && argc >= 0);
5024 KMP_DEBUG_ASSERT(max_nproc >= new_nproc);
5027 #if KMP_NESTED_HOT_TEAMS
5028 kmp_hot_team_ptr_t *hot_teams;
5030 team = master->th.th_team;
5031 level = team->t.t_active_level;
5032 if (master->th.th_teams_microtask) {
5033 if (master->th.th_teams_size.nteams > 1 &&
5036 (microtask_t)__kmp_teams_master ||
5037 master->th.th_teams_level <
5043 hot_teams = master->th.th_hot_teams;
5044 if (level < __kmp_hot_teams_max_level && hot_teams &&
5045 hot_teams[level].hot_team) {
5053 KMP_DEBUG_ASSERT(new_nproc == 1);
5057 if (use_hot_team && new_nproc > 1) {
5058 KMP_DEBUG_ASSERT(new_nproc <= max_nproc);
5059 #if KMP_NESTED_HOT_TEAMS
5060 team = hot_teams[level].hot_team;
5062 team = root->r.r_hot_team;
5065 if (__kmp_tasking_mode != tskm_immediate_exec) {
5066 KA_TRACE(20, (
"__kmp_allocate_team: hot team task_team[0] = %p "
5067 "task_team[1] = %p before reinit\n",
5068 team->t.t_task_team[0], team->t.t_task_team[1]));
5072 if (team->t.t_nproc != new_nproc &&
5073 __kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
5075 int old_nthr = team->t.t_nproc;
5076 __kmp_resize_dist_barrier(team, old_nthr, new_nproc);
5082 if (team->t.t_nproc == new_nproc) {
5083 KA_TRACE(20, (
"__kmp_allocate_team: reusing hot team\n"));
5086 if (team->t.t_size_changed == -1) {
5087 team->t.t_size_changed = 1;
5089 KMP_CHECK_UPDATE(team->t.t_size_changed, 0);
5093 kmp_r_sched_t new_sched = new_icvs->sched;
5095 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_sched.sched);
5097 __kmp_reinitialize_team(team, new_icvs,
5098 root->r.r_uber_thread->th.th_ident);
5100 KF_TRACE(10, (
"__kmp_allocate_team2: T#%d, this_thread=%p team=%p\n", 0,
5101 team->t.t_threads[0], team));
5102 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);
5104 #if KMP_AFFINITY_SUPPORTED
5105 if ((team->t.t_size_changed == 0) &&
5106 (team->t.t_proc_bind == new_proc_bind)) {
5107 if (new_proc_bind == proc_bind_spread) {
5108 __kmp_partition_places(
5111 KA_TRACE(200, (
"__kmp_allocate_team: reusing hot team #%d bindings: "
5112 "proc_bind = %d, partition = [%d,%d]\n",
5113 team->t.t_id, new_proc_bind, team->t.t_first_place,
5114 team->t.t_last_place));
5116 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5117 __kmp_partition_places(team);
5120 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5122 }
else if (team->t.t_nproc > new_nproc) {
5124 (
"__kmp_allocate_team: decreasing hot team thread count to %d\n",
5127 team->t.t_size_changed = 1;
5128 if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
5131 __kmp_add_threads_to_team(team, new_nproc);
5133 #if KMP_NESTED_HOT_TEAMS
5134 if (__kmp_hot_teams_mode == 0) {
5137 KMP_DEBUG_ASSERT(hot_teams[level].hot_team_nth == team->t.t_nproc);
5138 hot_teams[level].hot_team_nth = new_nproc;
5141 for (f = new_nproc; f < team->t.t_nproc; f++) {
5142 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5143 if (__kmp_tasking_mode != tskm_immediate_exec) {
5146 team->t.t_threads[f]->th.th_task_team = NULL;
5148 __kmp_free_thread(team->t.t_threads[f]);
5149 team->t.t_threads[f] = NULL;
5151 #if KMP_NESTED_HOT_TEAMS
5156 for (f = new_nproc; f < team->t.t_nproc; ++f) {
5157 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5158 kmp_balign_t *balign = team->t.t_threads[f]->th.th_bar;
5159 for (
int b = 0; b < bs_last_barrier; ++b) {
5160 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG) {
5161 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG;
5163 KMP_CHECK_UPDATE(balign[b].bb.leaf_kids, 0);
5168 team->t.t_nproc = new_nproc;
5170 KMP_CHECK_UPDATE(team->t.t_sched.sched, new_icvs->sched.sched);
5171 __kmp_reinitialize_team(team, new_icvs,
5172 root->r.r_uber_thread->th.th_ident);
5175 for (f = 0; f < new_nproc; ++f) {
5176 team->t.t_threads[f]->th.th_team_nproc = new_nproc;
5181 KF_TRACE(10, (
"__kmp_allocate_team: T#%d, this_thread=%p team=%p\n", 0,
5182 team->t.t_threads[0], team));
5184 __kmp_push_current_task_to_thread(team->t.t_threads[0], team, 0);
5187 for (f = 0; f < team->t.t_nproc; f++) {
5188 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
5189 team->t.t_threads[f]->th.th_team_nproc ==
5194 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5195 #if KMP_AFFINITY_SUPPORTED
5196 __kmp_partition_places(team);
5199 #if (KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED
5200 kmp_affin_mask_t *old_mask;
5201 if (KMP_AFFINITY_CAPABLE()) {
5202 KMP_CPU_ALLOC(old_mask);
5207 (
"__kmp_allocate_team: increasing hot team thread count to %d\n",
5209 int old_nproc = team->t.t_nproc;
5210 team->t.t_size_changed = 1;
5212 #if KMP_NESTED_HOT_TEAMS
5213 int avail_threads = hot_teams[level].hot_team_nth;
5214 if (new_nproc < avail_threads)
5215 avail_threads = new_nproc;
5216 kmp_info_t **other_threads = team->t.t_threads;
5217 for (f = team->t.t_nproc; f < avail_threads; ++f) {
5221 kmp_balign_t *balign = other_threads[f]->th.th_bar;
5222 for (b = 0; b < bs_last_barrier; ++b) {
5223 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5224 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
5226 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5230 if (hot_teams[level].hot_team_nth >= new_nproc) {
5233 KMP_DEBUG_ASSERT(__kmp_hot_teams_mode == 1);
5234 team->t.t_nproc = new_nproc;
5238 team->t.t_nproc = hot_teams[level].hot_team_nth;
5239 hot_teams[level].hot_team_nth = new_nproc;
5241 if (team->t.t_max_nproc < new_nproc) {
5243 __kmp_reallocate_team_arrays(team, new_nproc);
5244 __kmp_reinitialize_team(team, new_icvs, NULL);
5247 #if (KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED
5253 __kmp_set_thread_affinity_mask_full_tmp(old_mask);
5257 for (f = team->t.t_nproc; f < new_nproc; f++) {
5258 kmp_info_t *new_worker = __kmp_allocate_thread(root, team, f);
5259 KMP_DEBUG_ASSERT(new_worker);
5260 team->t.t_threads[f] = new_worker;
5263 (
"__kmp_allocate_team: team %d init T#%d arrived: "
5264 "join=%llu, plain=%llu\n",
5265 team->t.t_id, __kmp_gtid_from_tid(f, team), team->t.t_id, f,
5266 team->t.t_bar[bs_forkjoin_barrier].b_arrived,
5267 team->t.t_bar[bs_plain_barrier].b_arrived));
5271 kmp_balign_t *balign = new_worker->th.th_bar;
5272 for (b = 0; b < bs_last_barrier; ++b) {
5273 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5274 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag !=
5275 KMP_BARRIER_PARENT_FLAG);
5277 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5283 #if (KMP_OS_LINUX || KMP_OS_FREEBSD) && KMP_AFFINITY_SUPPORTED
5284 if (KMP_AFFINITY_CAPABLE()) {
5286 __kmp_set_system_affinity(old_mask, TRUE);
5287 KMP_CPU_FREE(old_mask);
5290 #if KMP_NESTED_HOT_TEAMS
5293 if (__kmp_barrier_release_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
5296 __kmp_add_threads_to_team(team, new_nproc);
5300 __kmp_initialize_team(team, new_nproc, new_icvs,
5301 root->r.r_uber_thread->th.th_ident);
5304 KMP_DEBUG_ASSERT(team->t.t_nproc == new_nproc);
5305 for (f = 0; f < team->t.t_nproc; ++f)
5306 __kmp_initialize_info(team->t.t_threads[f], team, f,
5307 __kmp_gtid_from_tid(f, team));
5315 for (f = old_nproc; f < team->t.t_nproc; ++f)
5316 team->t.t_threads[f]->th.th_task_state =
5317 team->t.t_threads[0]->th.th_task_state_memo_stack[level];
5320 kmp_uint8 old_state = team->t.t_threads[0]->th.th_task_state;
5321 for (f = old_nproc; f < team->t.t_nproc; ++f)
5322 team->t.t_threads[f]->th.th_task_state = old_state;
5326 for (f = 0; f < team->t.t_nproc; ++f) {
5327 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
5328 team->t.t_threads[f]->th.th_team_nproc ==
5333 KMP_CHECK_UPDATE(team->t.t_proc_bind, new_proc_bind);
5334 #if KMP_AFFINITY_SUPPORTED
5335 __kmp_partition_places(team);
5339 kmp_info_t *master = team->t.t_threads[0];
5340 if (master->th.th_teams_microtask) {
5341 for (f = 1; f < new_nproc; ++f) {
5343 kmp_info_t *thr = team->t.t_threads[f];
5344 thr->th.th_teams_microtask = master->th.th_teams_microtask;
5345 thr->th.th_teams_level = master->th.th_teams_level;
5346 thr->th.th_teams_size = master->th.th_teams_size;
5349 #if KMP_NESTED_HOT_TEAMS
5353 for (f = 1; f < new_nproc; ++f) {
5354 kmp_info_t *thr = team->t.t_threads[f];
5356 kmp_balign_t *balign = thr->th.th_bar;
5357 for (b = 0; b < bs_last_barrier; ++b) {
5358 balign[b].bb.b_arrived = team->t.t_bar[b].b_arrived;
5359 KMP_DEBUG_ASSERT(balign[b].bb.wait_flag != KMP_BARRIER_PARENT_FLAG);
5361 balign[b].bb.b_worker_arrived = team->t.t_bar[b].b_team_arrived;
5369 __kmp_alloc_argv_entries(argc, team, TRUE);
5370 KMP_CHECK_UPDATE(team->t.t_argc, argc);
5374 KF_TRACE(10, (
" hot_team = %p\n", team));
5377 if (__kmp_tasking_mode != tskm_immediate_exec) {
5378 KA_TRACE(20, (
"__kmp_allocate_team: hot team task_team[0] = %p "
5379 "task_team[1] = %p after reinit\n",
5380 team->t.t_task_team[0], team->t.t_task_team[1]));
5385 __ompt_team_assign_id(team, ompt_parallel_data);
5395 for (team = CCAST(kmp_team_t *, __kmp_team_pool); (team);) {
5398 if (team->t.t_max_nproc >= max_nproc) {
5400 __kmp_team_pool = team->t.t_next_pool;
5402 if (max_nproc > 1 &&
5403 __kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
5405 team->t.b = distributedBarrier::allocate(__kmp_dflt_team_nth_ub);
5410 __kmp_initialize_team(team, new_nproc, new_icvs, NULL);
5412 KA_TRACE(20, (
"__kmp_allocate_team: setting task_team[0] %p and "
5413 "task_team[1] %p to NULL\n",
5414 &team->t.t_task_team[0], &team->t.t_task_team[1]));
5415 team->t.t_task_team[0] = NULL;
5416 team->t.t_task_team[1] = NULL;
5419 __kmp_alloc_argv_entries(argc, team, TRUE);
5420 KMP_CHECK_UPDATE(team->t.t_argc, argc);
5423 20, (
"__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",
5424 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
5427 for (b = 0; b < bs_last_barrier; ++b) {
5428 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE;
5430 team->t.t_bar[b].b_master_arrived = 0;
5431 team->t.t_bar[b].b_team_arrived = 0;
5436 team->t.t_proc_bind = new_proc_bind;
5438 KA_TRACE(20, (
"__kmp_allocate_team: using team from pool %d.\n",
5442 __ompt_team_assign_id(team, ompt_parallel_data);
5454 team = __kmp_reap_team(team);
5455 __kmp_team_pool = team;
5460 team = (kmp_team_t *)__kmp_allocate(
sizeof(kmp_team_t));
5463 team->t.t_max_nproc = max_nproc;
5464 if (max_nproc > 1 &&
5465 __kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
5467 team->t.b = distributedBarrier::allocate(__kmp_dflt_team_nth_ub);
5472 __kmp_allocate_team_arrays(team, max_nproc);
5474 KA_TRACE(20, (
"__kmp_allocate_team: making a new team\n"));
5475 __kmp_initialize_team(team, new_nproc, new_icvs, NULL);
5477 KA_TRACE(20, (
"__kmp_allocate_team: setting task_team[0] %p and task_team[1] "
5479 &team->t.t_task_team[0], &team->t.t_task_team[1]));
5480 team->t.t_task_team[0] = NULL;
5482 team->t.t_task_team[1] = NULL;
5485 if (__kmp_storage_map) {
5486 __kmp_print_team_storage_map(
"team", team, team->t.t_id, new_nproc);
5490 __kmp_alloc_argv_entries(argc, team, FALSE);
5491 team->t.t_argc = argc;
5494 (
"__kmp_allocate_team: team %d init arrived: join=%u, plain=%u\n",
5495 team->t.t_id, KMP_INIT_BARRIER_STATE, KMP_INIT_BARRIER_STATE));
5498 for (b = 0; b < bs_last_barrier; ++b) {
5499 team->t.t_bar[b].b_arrived = KMP_INIT_BARRIER_STATE;
5501 team->t.t_bar[b].b_master_arrived = 0;
5502 team->t.t_bar[b].b_team_arrived = 0;
5507 team->t.t_proc_bind = new_proc_bind;
5510 __ompt_team_assign_id(team, ompt_parallel_data);
5511 team->t.ompt_serialized_team_info = NULL;
5516 KA_TRACE(20, (
"__kmp_allocate_team: done creating a new team %d.\n",
5527 void __kmp_free_team(kmp_root_t *root,
5528 kmp_team_t *team USE_NESTED_HOT_ARG(kmp_info_t *master)) {
5530 KA_TRACE(20, (
"__kmp_free_team: T#%d freeing team %d\n", __kmp_get_gtid(),
5534 KMP_DEBUG_ASSERT(root);
5535 KMP_DEBUG_ASSERT(team);
5536 KMP_DEBUG_ASSERT(team->t.t_nproc <= team->t.t_max_nproc);
5537 KMP_DEBUG_ASSERT(team->t.t_threads);
5539 int use_hot_team = team == root->r.r_hot_team;
5540 #if KMP_NESTED_HOT_TEAMS
5542 kmp_hot_team_ptr_t *hot_teams;
5544 level = team->t.t_active_level - 1;
5545 if (master->th.th_teams_microtask) {
5546 if (master->th.th_teams_size.nteams > 1) {
5550 if (team->t.t_pkfn != (microtask_t)__kmp_teams_master &&
5551 master->th.th_teams_level == team->t.t_level) {
5556 hot_teams = master->th.th_hot_teams;
5557 if (level < __kmp_hot_teams_max_level) {
5558 KMP_DEBUG_ASSERT(team == hot_teams[level].hot_team);
5565 TCW_SYNC_PTR(team->t.t_pkfn,
5568 team->t.t_copyin_counter = 0;
5573 if (!use_hot_team) {
5574 if (__kmp_tasking_mode != tskm_immediate_exec) {
5576 for (f = 1; f < team->t.t_nproc; ++f) {
5577 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5578 kmp_info_t *th = team->t.t_threads[f];
5579 volatile kmp_uint32 *state = &th->th.th_reap_state;
5580 while (*state != KMP_SAFE_TO_REAP) {
5584 if (!__kmp_is_thread_alive(th, &ecode)) {
5585 *state = KMP_SAFE_TO_REAP;
5590 kmp_flag_64<> fl(&th->th.th_bar[bs_forkjoin_barrier].bb.b_go, th);
5591 if (fl.is_sleeping())
5592 fl.resume(__kmp_gtid_from_thread(th));
5599 for (tt_idx = 0; tt_idx < 2; ++tt_idx) {
5600 kmp_task_team_t *task_team = team->t.t_task_team[tt_idx];
5601 if (task_team != NULL) {
5602 for (f = 0; f < team->t.t_nproc; ++f) {
5603 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5604 team->t.t_threads[f]->th.th_task_team = NULL;
5608 (
"__kmp_free_team: T#%d deactivating task_team %p on team %d\n",
5609 __kmp_get_gtid(), task_team, team->t.t_id));
5610 #if KMP_NESTED_HOT_TEAMS
5611 __kmp_free_task_team(master, task_team);
5613 team->t.t_task_team[tt_idx] = NULL;
5619 team->t.t_parent = NULL;
5620 team->t.t_level = 0;
5621 team->t.t_active_level = 0;
5624 for (f = 1; f < team->t.t_nproc; ++f) {
5625 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
5626 if (__kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
5627 KMP_COMPARE_AND_STORE_ACQ32(&(team->t.t_threads[f]->th.th_used_in_team),
5630 __kmp_free_thread(team->t.t_threads[f]);
5633 if (__kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
5636 team->t.b->go_release();
5637 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
5638 for (f = 1; f < team->t.t_nproc; ++f) {
5639 if (team->t.b->sleep[f].sleep) {
5640 __kmp_atomic_resume_64(
5641 team->t.t_threads[f]->th.th_info.ds.ds_gtid,
5642 (kmp_atomic_flag_64<> *)NULL);
5647 for (
int f = 1; f < team->t.t_nproc; ++f) {
5648 while (team->t.t_threads[f]->th.th_used_in_team.load() != 0)
5654 for (f = 1; f < team->t.t_nproc; ++f) {
5655 team->t.t_threads[f] = NULL;
5658 if (team->t.t_max_nproc > 1 &&
5659 __kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
5660 distributedBarrier::deallocate(team->t.b);
5665 team->t.t_next_pool = CCAST(kmp_team_t *, __kmp_team_pool);
5666 __kmp_team_pool = (
volatile kmp_team_t *)team;
5669 KMP_DEBUG_ASSERT(team->t.t_threads[1] &&
5670 team->t.t_threads[1]->th.th_cg_roots);
5671 if (team->t.t_threads[1]->th.th_cg_roots->cg_root == team->t.t_threads[1]) {
5673 for (f = 1; f < team->t.t_nproc; ++f) {
5674 kmp_info_t *thr = team->t.t_threads[f];
5675 KMP_DEBUG_ASSERT(thr && thr->th.th_cg_roots &&
5676 thr->th.th_cg_roots->cg_root == thr);
5678 kmp_cg_root_t *tmp = thr->th.th_cg_roots;
5679 thr->th.th_cg_roots = tmp->up;
5680 KA_TRACE(100, (
"__kmp_free_team: Thread %p popping node %p and moving"
5681 " up to node %p. cg_nthreads was %d\n",
5682 thr, tmp, thr->th.th_cg_roots, tmp->cg_nthreads));
5683 int i = tmp->cg_nthreads--;
5688 if (thr->th.th_cg_roots)
5689 thr->th.th_current_task->td_icvs.thread_limit =
5690 thr->th.th_cg_roots->cg_thread_limit;
5699 kmp_team_t *__kmp_reap_team(kmp_team_t *team) {
5700 kmp_team_t *next_pool = team->t.t_next_pool;
5702 KMP_DEBUG_ASSERT(team);
5703 KMP_DEBUG_ASSERT(team->t.t_dispatch);
5704 KMP_DEBUG_ASSERT(team->t.t_disp_buffer);
5705 KMP_DEBUG_ASSERT(team->t.t_threads);
5706 KMP_DEBUG_ASSERT(team->t.t_argv);
5711 __kmp_free_team_arrays(team);
5712 if (team->t.t_argv != &team->t.t_inline_argv[0])
5713 __kmp_free((
void *)team->t.t_argv);
5745 void __kmp_free_thread(kmp_info_t *this_th) {
5749 KA_TRACE(20, (
"__kmp_free_thread: T#%d putting T#%d back on free pool.\n",
5750 __kmp_get_gtid(), this_th->th.th_info.ds.ds_gtid));
5752 KMP_DEBUG_ASSERT(this_th);
5757 kmp_balign_t *balign = this_th->th.th_bar;
5758 for (b = 0; b < bs_last_barrier; ++b) {
5759 if (balign[b].bb.wait_flag == KMP_BARRIER_PARENT_FLAG)
5760 balign[b].bb.wait_flag = KMP_BARRIER_SWITCH_TO_OWN_FLAG;
5761 balign[b].bb.team = NULL;
5762 balign[b].bb.leaf_kids = 0;
5764 this_th->th.th_task_state = 0;
5765 this_th->th.th_reap_state = KMP_SAFE_TO_REAP;
5768 TCW_PTR(this_th->th.th_team, NULL);
5769 TCW_PTR(this_th->th.th_root, NULL);
5770 TCW_PTR(this_th->th.th_dispatch, NULL);
5772 while (this_th->th.th_cg_roots) {
5773 this_th->th.th_cg_roots->cg_nthreads--;
5774 KA_TRACE(100, (
"__kmp_free_thread: Thread %p decrement cg_nthreads on node"
5775 " %p of thread %p to %d\n",
5776 this_th, this_th->th.th_cg_roots,
5777 this_th->th.th_cg_roots->cg_root,
5778 this_th->th.th_cg_roots->cg_nthreads));
5779 kmp_cg_root_t *tmp = this_th->th.th_cg_roots;
5780 if (tmp->cg_root == this_th) {
5781 KMP_DEBUG_ASSERT(tmp->cg_nthreads == 0);
5783 5, (
"__kmp_free_thread: Thread %p freeing node %p\n", this_th, tmp));
5784 this_th->th.th_cg_roots = tmp->up;
5787 if (tmp->cg_nthreads == 0) {
5790 this_th->th.th_cg_roots = NULL;
5800 __kmp_free_implicit_task(this_th);
5801 this_th->th.th_current_task = NULL;
5805 gtid = this_th->th.th_info.ds.ds_gtid;
5806 if (__kmp_thread_pool_insert_pt != NULL) {
5807 KMP_DEBUG_ASSERT(__kmp_thread_pool != NULL);
5808 if (__kmp_thread_pool_insert_pt->th.th_info.ds.ds_gtid > gtid) {
5809 __kmp_thread_pool_insert_pt = NULL;
5818 if (__kmp_thread_pool_insert_pt != NULL) {
5819 scan = &(__kmp_thread_pool_insert_pt->th.th_next_pool);
5821 scan = CCAST(kmp_info_t **, &__kmp_thread_pool);
5823 for (; (*scan != NULL) && ((*scan)->th.th_info.ds.ds_gtid < gtid);
5824 scan = &((*scan)->th.th_next_pool))
5829 TCW_PTR(this_th->th.th_next_pool, *scan);
5830 __kmp_thread_pool_insert_pt = *scan = this_th;
5831 KMP_DEBUG_ASSERT((this_th->th.th_next_pool == NULL) ||
5832 (this_th->th.th_info.ds.ds_gtid <
5833 this_th->th.th_next_pool->th.th_info.ds.ds_gtid));
5834 TCW_4(this_th->th.th_in_pool, TRUE);
5835 __kmp_suspend_initialize_thread(this_th);
5836 __kmp_lock_suspend_mx(this_th);
5837 if (this_th->th.th_active == TRUE) {
5838 KMP_ATOMIC_INC(&__kmp_thread_pool_active_nth);
5839 this_th->th.th_active_in_pool = TRUE;
5843 KMP_DEBUG_ASSERT(this_th->th.th_active_in_pool == FALSE);
5846 __kmp_unlock_suspend_mx(this_th);
5848 TCW_4(__kmp_nth, __kmp_nth - 1);
5850 #ifdef KMP_ADJUST_BLOCKTIME
5853 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
5854 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
5855 if (__kmp_nth <= __kmp_avail_proc) {
5856 __kmp_zero_bt = FALSE;
5866 void *__kmp_launch_thread(kmp_info_t *this_thr) {
5867 #if OMP_PROFILING_SUPPORT
5868 ProfileTraceFile = getenv(
"LIBOMPTARGET_PROFILE");
5870 if (ProfileTraceFile)
5871 llvm::timeTraceProfilerInitialize(500 ,
"libomptarget");
5874 int gtid = this_thr->th.th_info.ds.ds_gtid;
5876 kmp_team_t **
volatile pteam;
5879 KA_TRACE(10, (
"__kmp_launch_thread: T#%d start\n", gtid));
5881 if (__kmp_env_consistency_check) {
5882 this_thr->th.th_cons = __kmp_allocate_cons_stack(gtid);
5886 if (ompd_state & OMPD_ENABLE_BP)
5887 ompd_bp_thread_begin();
5891 ompt_data_t *thread_data =
nullptr;
5892 if (ompt_enabled.enabled) {
5893 thread_data = &(this_thr->th.ompt_thread_info.thread_data);
5894 *thread_data = ompt_data_none;
5896 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
5897 this_thr->th.ompt_thread_info.wait_id = 0;
5898 this_thr->th.ompt_thread_info.idle_frame = OMPT_GET_FRAME_ADDRESS(0);
5899 this_thr->th.ompt_thread_info.parallel_flags = 0;
5900 if (ompt_enabled.ompt_callback_thread_begin) {
5901 ompt_callbacks.ompt_callback(ompt_callback_thread_begin)(
5902 ompt_thread_worker, thread_data);
5904 this_thr->th.ompt_thread_info.state = ompt_state_idle;
5909 while (!TCR_4(__kmp_global.g.g_done)) {
5910 KMP_DEBUG_ASSERT(this_thr == __kmp_threads[gtid]);
5914 KA_TRACE(20, (
"__kmp_launch_thread: T#%d waiting for work\n", gtid));
5917 __kmp_fork_barrier(gtid, KMP_GTID_DNE);
5920 if (ompt_enabled.enabled) {
5921 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
5925 pteam = &this_thr->th.th_team;
5928 if (TCR_SYNC_PTR(*pteam) && !TCR_4(__kmp_global.g.g_done)) {
5930 if (TCR_SYNC_PTR((*pteam)->t.t_pkfn) != NULL) {
5933 (
"__kmp_launch_thread: T#%d(%d:%d) invoke microtask = %p\n",
5934 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid),
5935 (*pteam)->t.t_pkfn));
5937 updateHWFPControl(*pteam);
5940 if (ompt_enabled.enabled) {
5941 this_thr->th.ompt_thread_info.state = ompt_state_work_parallel;
5945 rc = (*pteam)->t.t_invoke(gtid);
5949 KA_TRACE(20, (
"__kmp_launch_thread: T#%d(%d:%d) done microtask = %p\n",
5950 gtid, (*pteam)->t.t_id, __kmp_tid_from_gtid(gtid),
5951 (*pteam)->t.t_pkfn));
5954 if (ompt_enabled.enabled) {
5956 __ompt_get_task_info_object(0)->frame.exit_frame = ompt_data_none;
5958 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
5962 __kmp_join_barrier(gtid);
5965 TCR_SYNC_PTR((intptr_t)__kmp_global.g.g_done);
5968 if (ompd_state & OMPD_ENABLE_BP)
5969 ompd_bp_thread_end();
5973 if (ompt_enabled.ompt_callback_thread_end) {
5974 ompt_callbacks.ompt_callback(ompt_callback_thread_end)(thread_data);
5978 this_thr->th.th_task_team = NULL;
5980 __kmp_common_destroy_gtid(gtid);
5982 KA_TRACE(10, (
"__kmp_launch_thread: T#%d done\n", gtid));
5985 #if OMP_PROFILING_SUPPORT
5986 llvm::timeTraceProfilerFinishThread();
5993 void __kmp_internal_end_dest(
void *specific_gtid) {
5996 __kmp_type_convert((kmp_intptr_t)specific_gtid - 1, >id);
5998 KA_TRACE(30, (
"__kmp_internal_end_dest: T#%d\n", gtid));
6002 __kmp_internal_end_thread(gtid);
6005 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB
6007 __attribute__((destructor))
void __kmp_internal_end_dtor(
void) {
6008 __kmp_internal_end_atexit();
6015 void __kmp_internal_end_atexit(
void) {
6016 KA_TRACE(30, (
"__kmp_internal_end_atexit\n"));
6040 __kmp_internal_end_library(-1);
6042 __kmp_close_console();
6046 static void __kmp_reap_thread(kmp_info_t *thread,
int is_root) {
6051 KMP_DEBUG_ASSERT(thread != NULL);
6053 gtid = thread->th.th_info.ds.ds_gtid;
6056 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
6059 20, (
"__kmp_reap_thread: releasing T#%d from fork barrier for reap\n",
6061 if (__kmp_barrier_gather_pattern[bs_forkjoin_barrier] == bp_dist_bar) {
6063 !KMP_COMPARE_AND_STORE_ACQ32(&(thread->th.th_used_in_team), 0, 3))
6065 __kmp_resume_32(gtid, (kmp_flag_32<false, false> *)NULL);
6069 ANNOTATE_HAPPENS_BEFORE(thread);
6070 kmp_flag_64<> flag(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go,
6072 __kmp_release_64(&flag);
6077 __kmp_reap_worker(thread);
6089 if (thread->th.th_active_in_pool) {
6090 thread->th.th_active_in_pool = FALSE;
6091 KMP_ATOMIC_DEC(&__kmp_thread_pool_active_nth);
6092 KMP_DEBUG_ASSERT(__kmp_thread_pool_active_nth >= 0);
6096 __kmp_free_implicit_task(thread);
6100 __kmp_free_fast_memory(thread);
6103 __kmp_suspend_uninitialize_thread(thread);
6105 KMP_DEBUG_ASSERT(__kmp_threads[gtid] == thread);
6106 TCW_SYNC_PTR(__kmp_threads[gtid], NULL);
6111 #ifdef KMP_ADJUST_BLOCKTIME
6114 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
6115 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
6116 if (__kmp_nth <= __kmp_avail_proc) {
6117 __kmp_zero_bt = FALSE;
6123 if (__kmp_env_consistency_check) {
6124 if (thread->th.th_cons) {
6125 __kmp_free_cons_stack(thread->th.th_cons);
6126 thread->th.th_cons = NULL;
6130 if (thread->th.th_pri_common != NULL) {
6131 __kmp_free(thread->th.th_pri_common);
6132 thread->th.th_pri_common = NULL;
6135 if (thread->th.th_task_state_memo_stack != NULL) {
6136 __kmp_free(thread->th.th_task_state_memo_stack);
6137 thread->th.th_task_state_memo_stack = NULL;
6141 if (thread->th.th_local.bget_data != NULL) {
6142 __kmp_finalize_bget(thread);
6146 #if KMP_AFFINITY_SUPPORTED
6147 if (thread->th.th_affin_mask != NULL) {
6148 KMP_CPU_FREE(thread->th.th_affin_mask);
6149 thread->th.th_affin_mask = NULL;
6153 #if KMP_USE_HIER_SCHED
6154 if (thread->th.th_hier_bar_data != NULL) {
6155 __kmp_free(thread->th.th_hier_bar_data);
6156 thread->th.th_hier_bar_data = NULL;
6160 __kmp_reap_team(thread->th.th_serial_team);
6161 thread->th.th_serial_team = NULL;
6168 static void __kmp_internal_end(
void) {
6172 __kmp_unregister_library();
6179 __kmp_reclaim_dead_roots();
6183 for (i = 0; i < __kmp_threads_capacity; i++)
6185 if (__kmp_root[i]->r.r_active)
6188 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
6190 if (i < __kmp_threads_capacity) {
6202 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
6203 if (TCR_4(__kmp_init_monitor)) {
6204 __kmp_reap_monitor(&__kmp_monitor);
6205 TCW_4(__kmp_init_monitor, 0);
6207 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
6208 KA_TRACE(10, (
"__kmp_internal_end: monitor reaped\n"));
6214 for (i = 0; i < __kmp_threads_capacity; i++) {
6215 if (__kmp_root[i]) {
6218 KMP_ASSERT(!__kmp_root[i]->r.r_active);
6227 while (__kmp_thread_pool != NULL) {
6229 kmp_info_t *thread = CCAST(kmp_info_t *, __kmp_thread_pool);
6230 __kmp_thread_pool = thread->th.th_next_pool;
6232 KMP_DEBUG_ASSERT(thread->th.th_reap_state == KMP_SAFE_TO_REAP);
6233 thread->th.th_next_pool = NULL;
6234 thread->th.th_in_pool = FALSE;
6235 __kmp_reap_thread(thread, 0);
6237 __kmp_thread_pool_insert_pt = NULL;
6240 while (__kmp_team_pool != NULL) {
6242 kmp_team_t *team = CCAST(kmp_team_t *, __kmp_team_pool);
6243 __kmp_team_pool = team->t.t_next_pool;
6245 team->t.t_next_pool = NULL;
6246 __kmp_reap_team(team);
6249 __kmp_reap_task_teams();
6256 for (i = 0; i < __kmp_threads_capacity; i++) {
6257 kmp_info_t *thr = __kmp_threads[i];
6258 while (thr && KMP_ATOMIC_LD_ACQ(&thr->th.th_blocking))
6263 for (i = 0; i < __kmp_threads_capacity; ++i) {
6270 TCW_SYNC_4(__kmp_init_common, FALSE);
6272 KA_TRACE(10, (
"__kmp_internal_end: all workers reaped\n"));
6280 __kmp_acquire_bootstrap_lock(&__kmp_monitor_lock);
6281 if (TCR_4(__kmp_init_monitor)) {
6282 __kmp_reap_monitor(&__kmp_monitor);
6283 TCW_4(__kmp_init_monitor, 0);
6285 __kmp_release_bootstrap_lock(&__kmp_monitor_lock);
6286 KA_TRACE(10, (
"__kmp_internal_end: monitor reaped\n"));
6289 TCW_4(__kmp_init_gtid, FALSE);
6298 void __kmp_internal_end_library(
int gtid_req) {
6305 if (__kmp_global.g.g_abort) {
6306 KA_TRACE(11, (
"__kmp_internal_end_library: abort, exiting\n"));
6310 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6311 KA_TRACE(10, (
"__kmp_internal_end_library: already finished\n"));
6318 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific();
6320 10, (
"__kmp_internal_end_library: enter T#%d (%d)\n", gtid, gtid_req));
6321 if (gtid == KMP_GTID_SHUTDOWN) {
6322 KA_TRACE(10, (
"__kmp_internal_end_library: !__kmp_init_runtime, system "
6323 "already shutdown\n"));
6325 }
else if (gtid == KMP_GTID_MONITOR) {
6326 KA_TRACE(10, (
"__kmp_internal_end_library: monitor thread, gtid not "
6327 "registered, or system shutdown\n"));
6329 }
else if (gtid == KMP_GTID_DNE) {
6330 KA_TRACE(10, (
"__kmp_internal_end_library: gtid not registered or system "
6333 }
else if (KMP_UBER_GTID(gtid)) {
6335 if (__kmp_root[gtid]->r.r_active) {
6336 __kmp_global.g.g_abort = -1;
6337 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
6338 __kmp_unregister_library();
6340 (
"__kmp_internal_end_library: root still active, abort T#%d\n",
6346 (
"__kmp_internal_end_library: unregistering sibling T#%d\n", gtid));
6347 __kmp_unregister_root_current_thread(gtid);
6354 #ifdef DUMP_DEBUG_ON_EXIT
6355 if (__kmp_debug_buf)
6356 __kmp_dump_debug_buffer();
6361 __kmp_unregister_library();
6366 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6369 if (__kmp_global.g.g_abort) {
6370 KA_TRACE(10, (
"__kmp_internal_end_library: abort, exiting\n"));
6372 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6375 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6376 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6385 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
6388 __kmp_internal_end();
6390 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6391 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6393 KA_TRACE(10, (
"__kmp_internal_end_library: exit\n"));
6395 #ifdef DUMP_DEBUG_ON_EXIT
6396 if (__kmp_debug_buf)
6397 __kmp_dump_debug_buffer();
6401 __kmp_close_console();
6404 __kmp_fini_allocator();
6408 void __kmp_internal_end_thread(
int gtid_req) {
6417 if (__kmp_global.g.g_abort) {
6418 KA_TRACE(11, (
"__kmp_internal_end_thread: abort, exiting\n"));
6422 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6423 KA_TRACE(10, (
"__kmp_internal_end_thread: already finished\n"));
6428 if (TCR_4(__kmp_init_hidden_helper)) {
6429 TCW_SYNC_4(__kmp_hidden_helper_team_done, TRUE);
6431 __kmp_hidden_helper_main_thread_release();
6433 __kmp_hidden_helper_threads_deinitz_wait();
6440 int gtid = (gtid_req >= 0) ? gtid_req : __kmp_gtid_get_specific();
6442 (
"__kmp_internal_end_thread: enter T#%d (%d)\n", gtid, gtid_req));
6443 if (gtid == KMP_GTID_SHUTDOWN) {
6444 KA_TRACE(10, (
"__kmp_internal_end_thread: !__kmp_init_runtime, system "
6445 "already shutdown\n"));
6447 }
else if (gtid == KMP_GTID_MONITOR) {
6448 KA_TRACE(10, (
"__kmp_internal_end_thread: monitor thread, gtid not "
6449 "registered, or system shutdown\n"));
6451 }
else if (gtid == KMP_GTID_DNE) {
6452 KA_TRACE(10, (
"__kmp_internal_end_thread: gtid not registered or system "
6456 }
else if (KMP_UBER_GTID(gtid)) {
6458 if (__kmp_root[gtid]->r.r_active) {
6459 __kmp_global.g.g_abort = -1;
6460 TCW_SYNC_4(__kmp_global.g.g_done, TRUE);
6462 (
"__kmp_internal_end_thread: root still active, abort T#%d\n",
6466 KA_TRACE(10, (
"__kmp_internal_end_thread: unregistering sibling T#%d\n",
6468 __kmp_unregister_root_current_thread(gtid);
6472 KA_TRACE(10, (
"__kmp_internal_end_thread: worker thread T#%d\n", gtid));
6475 __kmp_threads[gtid]->th.th_task_team = NULL;
6479 (
"__kmp_internal_end_thread: worker thread done, exiting T#%d\n",
6485 if (__kmp_pause_status != kmp_hard_paused)
6489 KA_TRACE(10, (
"__kmp_internal_end_thread: exiting T#%d\n", gtid_req));
6494 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
6497 if (__kmp_global.g.g_abort) {
6498 KA_TRACE(10, (
"__kmp_internal_end_thread: abort, exiting\n"));
6500 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6503 if (TCR_4(__kmp_global.g.g_done) || !__kmp_init_serial) {
6504 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6515 __kmp_acquire_bootstrap_lock(&__kmp_forkjoin_lock);
6517 for (i = 0; i < __kmp_threads_capacity; ++i) {
6518 if (KMP_UBER_GTID(i)) {
6521 (
"__kmp_internal_end_thread: remaining sibling task: gtid==%d\n", i));
6522 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6523 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6530 __kmp_internal_end();
6532 __kmp_release_bootstrap_lock(&__kmp_forkjoin_lock);
6533 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
6535 KA_TRACE(10, (
"__kmp_internal_end_thread: exit T#%d\n", gtid_req));
6537 #ifdef DUMP_DEBUG_ON_EXIT
6538 if (__kmp_debug_buf)
6539 __kmp_dump_debug_buffer();
6546 static long __kmp_registration_flag = 0;
6548 static char *__kmp_registration_str = NULL;
6551 static inline char *__kmp_reg_status_name() {
6557 #if KMP_OS_UNIX && !KMP_OS_DARWIN && KMP_DYNAMIC_LIB
6558 return __kmp_str_format(
"__KMP_REGISTERED_LIB_%d_%d", (
int)getpid(),
6561 return __kmp_str_format(
"__KMP_REGISTERED_LIB_%d", (
int)getpid());
6565 void __kmp_register_library_startup(
void) {
6567 char *name = __kmp_reg_status_name();
6573 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
6574 __kmp_initialize_system_tick();
6576 __kmp_read_system_time(&time.dtime);
6577 __kmp_registration_flag = 0xCAFE0000L | (time.ltime & 0x0000FFFFL);
6578 __kmp_registration_str =
6579 __kmp_str_format(
"%p-%lx-%s", &__kmp_registration_flag,
6580 __kmp_registration_flag, KMP_LIBRARY_FILE);
6582 KA_TRACE(50, (
"__kmp_register_library_startup: %s=\"%s\"\n", name,
6583 __kmp_registration_str));
6589 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB
6590 char *shm_name = __kmp_str_format(
"/%s", name);
6591 int shm_preexist = 0;
6593 int fd1 = shm_open(shm_name, O_CREAT | O_EXCL | O_RDWR, 0666);
6594 if ((fd1 == -1) && (errno == EEXIST)) {
6597 fd1 = shm_open(shm_name, O_RDWR, 0666);
6600 __kmp_fatal(KMP_MSG(FunctionError,
"Can't open SHM"), KMP_ERR(0),
6606 }
else if (fd1 == -1) {
6609 __kmp_fatal(KMP_MSG(FunctionError,
"Can't open SHM2"), KMP_ERR(errno),
6612 if (shm_preexist == 0) {
6614 if (ftruncate(fd1, SHM_SIZE) == -1) {
6616 __kmp_fatal(KMP_MSG(FunctionError,
"Can't set size of SHM"),
6617 KMP_ERR(errno), __kmp_msg_null);
6621 (
char *)mmap(0, SHM_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd1, 0);
6622 if (data1 == MAP_FAILED) {
6624 __kmp_fatal(KMP_MSG(FunctionError,
"Can't map SHM"), KMP_ERR(errno),
6627 if (shm_preexist == 0) {
6628 KMP_STRCPY_S(data1, SHM_SIZE, __kmp_registration_str);
6631 value = __kmp_str_format(
"%s", data1);
6632 munmap(data1, SHM_SIZE);
6636 __kmp_env_set(name, __kmp_registration_str, 0);
6638 value = __kmp_env_get(name);
6641 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) {
6648 char *flag_addr_str = NULL;
6649 char *flag_val_str = NULL;
6650 char const *file_name = NULL;
6651 __kmp_str_split(tail,
'-', &flag_addr_str, &tail);
6652 __kmp_str_split(tail,
'-', &flag_val_str, &tail);
6655 long *flag_addr = 0;
6656 unsigned long flag_val = 0;
6657 KMP_SSCANF(flag_addr_str,
"%p", RCAST(
void **, &flag_addr));
6658 KMP_SSCANF(flag_val_str,
"%lx", &flag_val);
6659 if (flag_addr != 0 && flag_val != 0 && strcmp(file_name,
"") != 0) {
6663 if (__kmp_is_address_mapped(flag_addr) && *flag_addr == flag_val) {
6677 file_name =
"unknown library";
6682 char *duplicate_ok = __kmp_env_get(
"KMP_DUPLICATE_LIB_OK");
6683 if (!__kmp_str_match_true(duplicate_ok)) {
6685 __kmp_fatal(KMP_MSG(DuplicateLibrary, KMP_LIBRARY_FILE, file_name),
6686 KMP_HNT(DuplicateLibrary), __kmp_msg_null);
6688 KMP_INTERNAL_FREE(duplicate_ok);
6689 __kmp_duplicate_library_ok = 1;
6694 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB
6696 shm_unlink(shm_name);
6699 __kmp_env_unset(name);
6703 KMP_DEBUG_ASSERT(0);
6707 KMP_INTERNAL_FREE((
void *)value);
6708 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB
6709 KMP_INTERNAL_FREE((
void *)shm_name);
6712 KMP_INTERNAL_FREE((
void *)name);
6716 void __kmp_unregister_library(
void) {
6718 char *name = __kmp_reg_status_name();
6721 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB
6722 char *shm_name = __kmp_str_format(
"/%s", name);
6723 int fd1 = shm_open(shm_name, O_RDONLY, 0666);
6728 char *data1 = (
char *)mmap(0, SHM_SIZE, PROT_READ, MAP_SHARED, fd1, 0);
6729 if (data1 != MAP_FAILED) {
6730 value = __kmp_str_format(
"%s", data1);
6731 munmap(data1, SHM_SIZE);
6735 value = __kmp_env_get(name);
6738 KMP_DEBUG_ASSERT(__kmp_registration_flag != 0);
6739 KMP_DEBUG_ASSERT(__kmp_registration_str != NULL);
6740 if (value != NULL && strcmp(value, __kmp_registration_str) == 0) {
6742 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB
6743 shm_unlink(shm_name);
6745 __kmp_env_unset(name);
6749 #if KMP_OS_UNIX && KMP_DYNAMIC_LIB
6750 KMP_INTERNAL_FREE(shm_name);
6753 KMP_INTERNAL_FREE(__kmp_registration_str);
6754 KMP_INTERNAL_FREE(value);
6755 KMP_INTERNAL_FREE(name);
6757 __kmp_registration_flag = 0;
6758 __kmp_registration_str = NULL;
6765 #if KMP_MIC_SUPPORTED
6767 static void __kmp_check_mic_type() {
6768 kmp_cpuid_t cpuid_state = {0};
6769 kmp_cpuid_t *cs_p = &cpuid_state;
6770 __kmp_x86_cpuid(1, 0, cs_p);
6772 if ((cs_p->eax & 0xff0) == 0xB10) {
6773 __kmp_mic_type = mic2;
6774 }
else if ((cs_p->eax & 0xf0ff0) == 0x50670) {
6775 __kmp_mic_type = mic3;
6777 __kmp_mic_type = non_mic;
6784 static void __kmp_user_level_mwait_init() {
6785 struct kmp_cpuid buf;
6786 __kmp_x86_cpuid(7, 0, &buf);
6787 __kmp_umwait_enabled = ((buf.ecx >> 5) & 1) && __kmp_user_level_mwait;
6788 KF_TRACE(30, (
"__kmp_user_level_mwait_init: __kmp_umwait_enabled = %d\n",
6789 __kmp_umwait_enabled));
6791 #elif KMP_HAVE_MWAIT
6792 #ifndef AT_INTELPHIUSERMWAIT
6795 #define AT_INTELPHIUSERMWAIT 10000
6800 unsigned long getauxval(
unsigned long) KMP_WEAK_ATTRIBUTE_EXTERNAL;
6801 unsigned long getauxval(
unsigned long) {
return 0; }
6803 static void __kmp_user_level_mwait_init() {
6808 if (__kmp_mic_type == mic3) {
6809 unsigned long res = getauxval(AT_INTELPHIUSERMWAIT);
6810 if ((res & 0x1) || __kmp_user_level_mwait) {
6811 __kmp_mwait_enabled = TRUE;
6812 if (__kmp_user_level_mwait) {
6813 KMP_INFORM(EnvMwaitWarn);
6816 __kmp_mwait_enabled = FALSE;
6819 KF_TRACE(30, (
"__kmp_user_level_mwait_init: __kmp_mic_type = %d, "
6820 "__kmp_mwait_enabled = %d\n",
6821 __kmp_mic_type, __kmp_mwait_enabled));
6825 static void __kmp_do_serial_initialize(
void) {
6829 KA_TRACE(10, (
"__kmp_do_serial_initialize: enter\n"));
6831 KMP_DEBUG_ASSERT(
sizeof(kmp_int32) == 4);
6832 KMP_DEBUG_ASSERT(
sizeof(kmp_uint32) == 4);
6833 KMP_DEBUG_ASSERT(
sizeof(kmp_int64) == 8);
6834 KMP_DEBUG_ASSERT(
sizeof(kmp_uint64) == 8);
6835 KMP_DEBUG_ASSERT(
sizeof(kmp_intptr_t) ==
sizeof(
void *));
6845 __kmp_validate_locks();
6848 __kmp_init_allocator();
6853 __kmp_register_library_startup();
6856 if (TCR_4(__kmp_global.g.g_done)) {
6857 KA_TRACE(10, (
"__kmp_do_serial_initialize: reinitialization of library\n"));
6860 __kmp_global.g.g_abort = 0;
6861 TCW_SYNC_4(__kmp_global.g.g_done, FALSE);
6864 #if KMP_USE_ADAPTIVE_LOCKS
6865 #if KMP_DEBUG_ADAPTIVE_LOCKS
6866 __kmp_init_speculative_stats();
6869 #if KMP_STATS_ENABLED
6872 __kmp_init_lock(&__kmp_global_lock);
6873 __kmp_init_queuing_lock(&__kmp_dispatch_lock);
6874 __kmp_init_lock(&__kmp_debug_lock);
6875 __kmp_init_atomic_lock(&__kmp_atomic_lock);
6876 __kmp_init_atomic_lock(&__kmp_atomic_lock_1i);
6877 __kmp_init_atomic_lock(&__kmp_atomic_lock_2i);
6878 __kmp_init_atomic_lock(&__kmp_atomic_lock_4i);
6879 __kmp_init_atomic_lock(&__kmp_atomic_lock_4r);
6880 __kmp_init_atomic_lock(&__kmp_atomic_lock_8i);
6881 __kmp_init_atomic_lock(&__kmp_atomic_lock_8r);
6882 __kmp_init_atomic_lock(&__kmp_atomic_lock_8c);
6883 __kmp_init_atomic_lock(&__kmp_atomic_lock_10r);
6884 __kmp_init_atomic_lock(&__kmp_atomic_lock_16r);
6885 __kmp_init_atomic_lock(&__kmp_atomic_lock_16c);
6886 __kmp_init_atomic_lock(&__kmp_atomic_lock_20c);
6887 __kmp_init_atomic_lock(&__kmp_atomic_lock_32c);
6888 __kmp_init_bootstrap_lock(&__kmp_forkjoin_lock);
6889 __kmp_init_bootstrap_lock(&__kmp_exit_lock);
6891 __kmp_init_bootstrap_lock(&__kmp_monitor_lock);
6893 __kmp_init_bootstrap_lock(&__kmp_tp_cached_lock);
6897 __kmp_runtime_initialize();
6899 #if KMP_MIC_SUPPORTED
6900 __kmp_check_mic_type();
6907 __kmp_abort_delay = 0;
6911 __kmp_dflt_team_nth_ub = __kmp_xproc;
6912 if (__kmp_dflt_team_nth_ub < KMP_MIN_NTH) {
6913 __kmp_dflt_team_nth_ub = KMP_MIN_NTH;
6915 if (__kmp_dflt_team_nth_ub > __kmp_sys_max_nth) {
6916 __kmp_dflt_team_nth_ub = __kmp_sys_max_nth;
6918 __kmp_max_nth = __kmp_sys_max_nth;
6919 __kmp_cg_max_nth = __kmp_sys_max_nth;
6920 __kmp_teams_max_nth = __kmp_xproc;
6921 if (__kmp_teams_max_nth > __kmp_sys_max_nth) {
6922 __kmp_teams_max_nth = __kmp_sys_max_nth;
6927 __kmp_dflt_blocktime = KMP_DEFAULT_BLOCKTIME;
6929 __kmp_monitor_wakeups =
6930 KMP_WAKEUPS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups);
6931 __kmp_bt_intervals =
6932 KMP_INTERVALS_FROM_BLOCKTIME(__kmp_dflt_blocktime, __kmp_monitor_wakeups);
6935 __kmp_library = library_throughput;
6937 __kmp_static = kmp_sch_static_balanced;
6944 #if KMP_FAST_REDUCTION_BARRIER
6945 #define kmp_reduction_barrier_gather_bb ((int)1)
6946 #define kmp_reduction_barrier_release_bb ((int)1)
6947 #define kmp_reduction_barrier_gather_pat __kmp_barrier_gather_pat_dflt
6948 #define kmp_reduction_barrier_release_pat __kmp_barrier_release_pat_dflt
6950 for (i = bs_plain_barrier; i < bs_last_barrier; i++) {
6951 __kmp_barrier_gather_branch_bits[i] = __kmp_barrier_gather_bb_dflt;
6952 __kmp_barrier_release_branch_bits[i] = __kmp_barrier_release_bb_dflt;
6953 __kmp_barrier_gather_pattern[i] = __kmp_barrier_gather_pat_dflt;
6954 __kmp_barrier_release_pattern[i] = __kmp_barrier_release_pat_dflt;
6955 #if KMP_FAST_REDUCTION_BARRIER
6956 if (i == bs_reduction_barrier) {
6958 __kmp_barrier_gather_branch_bits[i] = kmp_reduction_barrier_gather_bb;
6959 __kmp_barrier_release_branch_bits[i] = kmp_reduction_barrier_release_bb;
6960 __kmp_barrier_gather_pattern[i] = kmp_reduction_barrier_gather_pat;
6961 __kmp_barrier_release_pattern[i] = kmp_reduction_barrier_release_pat;
6965 #if KMP_FAST_REDUCTION_BARRIER
6966 #undef kmp_reduction_barrier_release_pat
6967 #undef kmp_reduction_barrier_gather_pat
6968 #undef kmp_reduction_barrier_release_bb
6969 #undef kmp_reduction_barrier_gather_bb
6971 #if KMP_MIC_SUPPORTED
6972 if (__kmp_mic_type == mic2) {
6974 __kmp_barrier_gather_branch_bits[bs_plain_barrier] = 3;
6975 __kmp_barrier_release_branch_bits[bs_forkjoin_barrier] =
6977 __kmp_barrier_gather_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar;
6978 __kmp_barrier_release_pattern[bs_forkjoin_barrier] = bp_hierarchical_bar;
6980 #if KMP_FAST_REDUCTION_BARRIER
6981 if (__kmp_mic_type == mic2) {
6982 __kmp_barrier_gather_pattern[bs_reduction_barrier] = bp_hierarchical_bar;
6983 __kmp_barrier_release_pattern[bs_reduction_barrier] = bp_hierarchical_bar;
6990 __kmp_env_checks = TRUE;
6992 __kmp_env_checks = FALSE;
6996 __kmp_foreign_tp = TRUE;
6998 __kmp_global.g.g_dynamic = FALSE;
6999 __kmp_global.g.g_dynamic_mode = dynamic_default;
7001 __kmp_init_nesting_mode();
7003 __kmp_env_initialize(NULL);
7005 #if KMP_HAVE_MWAIT || KMP_HAVE_UMWAIT
7006 __kmp_user_level_mwait_init();
7010 char const *val = __kmp_env_get(
"KMP_DUMP_CATALOG");
7011 if (__kmp_str_match_true(val)) {
7012 kmp_str_buf_t buffer;
7013 __kmp_str_buf_init(&buffer);
7014 __kmp_i18n_dump_catalog(&buffer);
7015 __kmp_printf(
"%s", buffer.str);
7016 __kmp_str_buf_free(&buffer);
7018 __kmp_env_free(&val);
7021 __kmp_threads_capacity =
7022 __kmp_initial_threads_capacity(__kmp_dflt_team_nth_ub);
7024 __kmp_tp_capacity = __kmp_default_tp_capacity(
7025 __kmp_dflt_team_nth_ub, __kmp_max_nth, __kmp_allThreadsSpecified);
7030 KMP_DEBUG_ASSERT(__kmp_thread_pool == NULL);
7031 KMP_DEBUG_ASSERT(__kmp_thread_pool_insert_pt == NULL);
7032 KMP_DEBUG_ASSERT(__kmp_team_pool == NULL);
7033 __kmp_thread_pool = NULL;
7034 __kmp_thread_pool_insert_pt = NULL;
7035 __kmp_team_pool = NULL;
7042 (
sizeof(kmp_info_t *) +
sizeof(kmp_root_t *)) * __kmp_threads_capacity +
7044 __kmp_threads = (kmp_info_t **)__kmp_allocate(size);
7045 __kmp_root = (kmp_root_t **)((
char *)__kmp_threads +
7046 sizeof(kmp_info_t *) * __kmp_threads_capacity);
7049 KMP_DEBUG_ASSERT(__kmp_all_nth ==
7051 KMP_DEBUG_ASSERT(__kmp_nth == 0);
7056 gtid = __kmp_register_root(TRUE);
7057 KA_TRACE(10, (
"__kmp_do_serial_initialize T#%d\n", gtid));
7058 KMP_ASSERT(KMP_UBER_GTID(gtid));
7059 KMP_ASSERT(KMP_INITIAL_GTID(gtid));
7063 __kmp_common_initialize();
7067 __kmp_register_atfork();
7070 #if !KMP_DYNAMIC_LIB
7074 int rc = atexit(__kmp_internal_end_atexit);
7076 __kmp_fatal(KMP_MSG(FunctionError,
"atexit()"), KMP_ERR(rc),
7082 #if KMP_HANDLE_SIGNALS
7088 __kmp_install_signals(FALSE);
7091 __kmp_install_signals(TRUE);
7096 __kmp_init_counter++;
7098 __kmp_init_serial = TRUE;
7100 if (__kmp_settings) {
7104 if (__kmp_display_env || __kmp_display_env_verbose) {
7105 __kmp_env_print_2();
7114 KA_TRACE(10, (
"__kmp_do_serial_initialize: exit\n"));
7117 void __kmp_serial_initialize(
void) {
7118 if (__kmp_init_serial) {
7121 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
7122 if (__kmp_init_serial) {
7123 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7126 __kmp_do_serial_initialize();
7127 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7130 static void __kmp_do_middle_initialize(
void) {
7132 int prev_dflt_team_nth;
7134 if (!__kmp_init_serial) {
7135 __kmp_do_serial_initialize();
7138 KA_TRACE(10, (
"__kmp_middle_initialize: enter\n"));
7142 prev_dflt_team_nth = __kmp_dflt_team_nth;
7144 #if KMP_AFFINITY_SUPPORTED
7147 __kmp_affinity_initialize();
7151 KMP_ASSERT(__kmp_xproc > 0);
7152 if (__kmp_avail_proc == 0) {
7153 __kmp_avail_proc = __kmp_xproc;
7159 while ((j < __kmp_nested_nth.used) && !__kmp_nested_nth.nth[j]) {
7160 __kmp_nested_nth.nth[j] = __kmp_dflt_team_nth = __kmp_dflt_team_nth_ub =
7165 if (__kmp_dflt_team_nth == 0) {
7166 #ifdef KMP_DFLT_NTH_CORES
7168 __kmp_dflt_team_nth = __kmp_ncores;
7169 KA_TRACE(20, (
"__kmp_middle_initialize: setting __kmp_dflt_team_nth = "
7170 "__kmp_ncores (%d)\n",
7171 __kmp_dflt_team_nth));
7174 __kmp_dflt_team_nth = __kmp_avail_proc;
7175 KA_TRACE(20, (
"__kmp_middle_initialize: setting __kmp_dflt_team_nth = "
7176 "__kmp_avail_proc(%d)\n",
7177 __kmp_dflt_team_nth));
7181 if (__kmp_dflt_team_nth < KMP_MIN_NTH) {
7182 __kmp_dflt_team_nth = KMP_MIN_NTH;
7184 if (__kmp_dflt_team_nth > __kmp_sys_max_nth) {
7185 __kmp_dflt_team_nth = __kmp_sys_max_nth;
7188 if (__kmp_nesting_mode > 0)
7189 __kmp_set_nesting_mode_threads();
7193 KMP_DEBUG_ASSERT(__kmp_dflt_team_nth <= __kmp_dflt_team_nth_ub);
7195 if (__kmp_dflt_team_nth != prev_dflt_team_nth) {
7200 for (i = 0; i < __kmp_threads_capacity; i++) {
7201 kmp_info_t *thread = __kmp_threads[i];
7204 if (thread->th.th_current_task->td_icvs.nproc != 0)
7207 set__nproc(__kmp_threads[i], __kmp_dflt_team_nth);
7212 (
"__kmp_middle_initialize: final value for __kmp_dflt_team_nth = %d\n",
7213 __kmp_dflt_team_nth));
7215 #ifdef KMP_ADJUST_BLOCKTIME
7217 if (!__kmp_env_blocktime && (__kmp_avail_proc > 0)) {
7218 KMP_DEBUG_ASSERT(__kmp_avail_proc > 0);
7219 if (__kmp_nth > __kmp_avail_proc) {
7220 __kmp_zero_bt = TRUE;
7226 TCW_SYNC_4(__kmp_init_middle, TRUE);
7228 KA_TRACE(10, (
"__kmp_do_middle_initialize: exit\n"));
7231 void __kmp_middle_initialize(
void) {
7232 if (__kmp_init_middle) {
7235 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
7236 if (__kmp_init_middle) {
7237 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7240 __kmp_do_middle_initialize();
7241 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7244 void __kmp_parallel_initialize(
void) {
7245 int gtid = __kmp_entry_gtid();
7248 if (TCR_4(__kmp_init_parallel))
7250 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
7251 if (TCR_4(__kmp_init_parallel)) {
7252 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7257 if (TCR_4(__kmp_global.g.g_done)) {
7260 (
"__kmp_parallel_initialize: attempt to init while shutting down\n"));
7261 __kmp_infinite_loop();
7267 if (!__kmp_init_middle) {
7268 __kmp_do_middle_initialize();
7270 __kmp_assign_root_init_mask();
7271 __kmp_resume_if_hard_paused();
7274 KA_TRACE(10, (
"__kmp_parallel_initialize: enter\n"));
7275 KMP_ASSERT(KMP_UBER_GTID(gtid));
7277 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
7280 __kmp_store_x87_fpu_control_word(&__kmp_init_x87_fpu_control_word);
7281 __kmp_store_mxcsr(&__kmp_init_mxcsr);
7282 __kmp_init_mxcsr &= KMP_X86_MXCSR_MASK;
7286 #if KMP_HANDLE_SIGNALS
7288 __kmp_install_signals(TRUE);
7292 __kmp_suspend_initialize();
7294 #if defined(USE_LOAD_BALANCE)
7295 if (__kmp_global.g.g_dynamic_mode == dynamic_default) {
7296 __kmp_global.g.g_dynamic_mode = dynamic_load_balance;
7299 if (__kmp_global.g.g_dynamic_mode == dynamic_default) {
7300 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit;
7304 if (__kmp_version) {
7305 __kmp_print_version_2();
7309 TCW_SYNC_4(__kmp_init_parallel, TRUE);
7312 KA_TRACE(10, (
"__kmp_parallel_initialize: exit\n"));
7314 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7317 void __kmp_hidden_helper_initialize() {
7318 if (TCR_4(__kmp_init_hidden_helper))
7322 if (!TCR_4(__kmp_init_parallel))
7323 __kmp_parallel_initialize();
7327 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
7328 if (TCR_4(__kmp_init_hidden_helper)) {
7329 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7334 KMP_ATOMIC_ST_REL(&__kmp_unexecuted_hidden_helper_tasks, 0);
7338 TCW_SYNC_4(__kmp_init_hidden_helper_threads, TRUE);
7341 __kmp_do_initialize_hidden_helper_threads();
7344 __kmp_hidden_helper_threads_initz_wait();
7347 TCW_SYNC_4(__kmp_init_hidden_helper, TRUE);
7349 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
7354 void __kmp_run_before_invoked_task(
int gtid,
int tid, kmp_info_t *this_thr,
7356 kmp_disp_t *dispatch;
7361 this_thr->th.th_local.this_construct = 0;
7362 #if KMP_CACHE_MANAGE
7363 KMP_CACHE_PREFETCH(&this_thr->th.th_bar[bs_forkjoin_barrier].bb.b_arrived);
7365 dispatch = (kmp_disp_t *)TCR_PTR(this_thr->th.th_dispatch);
7366 KMP_DEBUG_ASSERT(dispatch);
7367 KMP_DEBUG_ASSERT(team->t.t_dispatch);
7371 dispatch->th_disp_index = 0;
7372 dispatch->th_doacross_buf_idx = 0;
7373 if (__kmp_env_consistency_check)
7374 __kmp_push_parallel(gtid, team->t.t_ident);
7379 void __kmp_run_after_invoked_task(
int gtid,
int tid, kmp_info_t *this_thr,
7381 if (__kmp_env_consistency_check)
7382 __kmp_pop_parallel(gtid, team->t.t_ident);
7384 __kmp_finish_implicit_task(this_thr);
7387 int __kmp_invoke_task_func(
int gtid) {
7389 int tid = __kmp_tid_from_gtid(gtid);
7390 kmp_info_t *this_thr = __kmp_threads[gtid];
7391 kmp_team_t *team = this_thr->th.th_team;
7393 __kmp_run_before_invoked_task(gtid, tid, this_thr, team);
7395 if (__itt_stack_caller_create_ptr) {
7397 if (team->t.t_stack_id != NULL) {
7398 __kmp_itt_stack_callee_enter((__itt_caller)team->t.t_stack_id);
7400 KMP_DEBUG_ASSERT(team->t.t_parent->t.t_stack_id != NULL);
7401 __kmp_itt_stack_callee_enter(
7402 (__itt_caller)team->t.t_parent->t.t_stack_id);
7406 #if INCLUDE_SSC_MARKS
7407 SSC_MARK_INVOKING();
7412 void **exit_frame_p;
7413 ompt_data_t *my_task_data;
7414 ompt_data_t *my_parallel_data;
7417 if (ompt_enabled.enabled) {
7418 exit_frame_p = &(team->t.t_implicit_task_taskdata[tid]
7419 .ompt_task_info.frame.exit_frame.ptr);
7421 exit_frame_p = &dummy;
7425 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data);
7426 my_parallel_data = &(team->t.ompt_team_info.parallel_data);
7427 if (ompt_enabled.ompt_callback_implicit_task) {
7428 ompt_team_size = team->t.t_nproc;
7429 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
7430 ompt_scope_begin, my_parallel_data, my_task_data, ompt_team_size,
7431 __kmp_tid_from_gtid(gtid), ompt_task_implicit);
7432 OMPT_CUR_TASK_INFO(this_thr)->thread_num = __kmp_tid_from_gtid(gtid);
7436 #if KMP_STATS_ENABLED
7438 if (previous_state == stats_state_e::TEAMS_REGION) {
7439 KMP_PUSH_PARTITIONED_TIMER(OMP_teams);
7441 KMP_PUSH_PARTITIONED_TIMER(OMP_parallel);
7443 KMP_SET_THREAD_STATE(IMPLICIT_TASK);
7446 rc = __kmp_invoke_microtask((microtask_t)TCR_SYNC_PTR(team->t.t_pkfn), gtid,
7447 tid, (
int)team->t.t_argc, (
void **)team->t.t_argv
7454 *exit_frame_p = NULL;
7455 this_thr->th.ompt_thread_info.parallel_flags |= ompt_parallel_team;
7458 #if KMP_STATS_ENABLED
7459 if (previous_state == stats_state_e::TEAMS_REGION) {
7460 KMP_SET_THREAD_STATE(previous_state);
7462 KMP_POP_PARTITIONED_TIMER();
7466 if (__itt_stack_caller_create_ptr) {
7468 if (team->t.t_stack_id != NULL) {
7469 __kmp_itt_stack_callee_leave((__itt_caller)team->t.t_stack_id);
7471 KMP_DEBUG_ASSERT(team->t.t_parent->t.t_stack_id != NULL);
7472 __kmp_itt_stack_callee_leave(
7473 (__itt_caller)team->t.t_parent->t.t_stack_id);
7477 __kmp_run_after_invoked_task(gtid, tid, this_thr, team);
7482 void __kmp_teams_master(
int gtid) {
7484 kmp_info_t *thr = __kmp_threads[gtid];
7485 kmp_team_t *team = thr->th.th_team;
7486 ident_t *loc = team->t.t_ident;
7487 thr->th.th_set_nproc = thr->th.th_teams_size.nth;
7488 KMP_DEBUG_ASSERT(thr->th.th_teams_microtask);
7489 KMP_DEBUG_ASSERT(thr->th.th_set_nproc);
7490 KA_TRACE(20, (
"__kmp_teams_master: T#%d, Tid %d, microtask %p\n", gtid,
7491 __kmp_tid_from_gtid(gtid), thr->th.th_teams_microtask));
7494 kmp_cg_root_t *tmp = (kmp_cg_root_t *)__kmp_allocate(
sizeof(kmp_cg_root_t));
7497 tmp->cg_thread_limit = thr->th.th_current_task->td_icvs.thread_limit;
7498 tmp->cg_nthreads = 1;
7499 KA_TRACE(100, (
"__kmp_teams_master: Thread %p created node %p and init"
7500 " cg_nthreads to 1\n",
7502 tmp->up = thr->th.th_cg_roots;
7503 thr->th.th_cg_roots = tmp;
7507 #if INCLUDE_SSC_MARKS
7510 __kmp_fork_call(loc, gtid, fork_context_intel, team->t.t_argc,
7511 (microtask_t)thr->th.th_teams_microtask,
7512 VOLATILE_CAST(launch_t) __kmp_invoke_task_func, NULL);
7513 #if INCLUDE_SSC_MARKS
7517 if (thr->th.th_team_nproc < thr->th.th_teams_size.nth)
7518 thr->th.th_teams_size.nth = thr->th.th_team_nproc;
7521 __kmp_join_call(loc, gtid
7530 int __kmp_invoke_teams_master(
int gtid) {
7531 kmp_info_t *this_thr = __kmp_threads[gtid];
7532 kmp_team_t *team = this_thr->th.th_team;
7534 if (!__kmp_threads[gtid]->th.th_team->t.t_serialized)
7535 KMP_DEBUG_ASSERT((
void *)__kmp_threads[gtid]->th.th_team->t.t_pkfn ==
7536 (
void *)__kmp_teams_master);
7538 __kmp_run_before_invoked_task(gtid, 0, this_thr, team);
7540 int tid = __kmp_tid_from_gtid(gtid);
7541 ompt_data_t *task_data =
7542 &team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data;
7543 ompt_data_t *parallel_data = &team->t.ompt_team_info.parallel_data;
7544 if (ompt_enabled.ompt_callback_implicit_task) {
7545 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
7546 ompt_scope_begin, parallel_data, task_data, team->t.t_nproc, tid,
7548 OMPT_CUR_TASK_INFO(this_thr)->thread_num = tid;
7551 __kmp_teams_master(gtid);
7553 this_thr->th.ompt_thread_info.parallel_flags |= ompt_parallel_league;
7555 __kmp_run_after_invoked_task(gtid, 0, this_thr, team);
7564 void __kmp_push_num_threads(
ident_t *
id,
int gtid,
int num_threads) {
7565 kmp_info_t *thr = __kmp_threads[gtid];
7567 if (num_threads > 0)
7568 thr->th.th_set_nproc = num_threads;
7571 static void __kmp_push_thread_limit(kmp_info_t *thr,
int num_teams,
7573 KMP_DEBUG_ASSERT(thr);
7575 if (!TCR_4(__kmp_init_middle))
7576 __kmp_middle_initialize();
7577 __kmp_assign_root_init_mask();
7578 KMP_DEBUG_ASSERT(__kmp_avail_proc);
7579 KMP_DEBUG_ASSERT(__kmp_dflt_team_nth);
7581 if (num_threads == 0) {
7582 if (__kmp_teams_thread_limit > 0) {
7583 num_threads = __kmp_teams_thread_limit;
7585 num_threads = __kmp_avail_proc / num_teams;
7590 if (num_threads > __kmp_dflt_team_nth) {
7591 num_threads = __kmp_dflt_team_nth;
7593 if (num_threads > thr->th.th_current_task->td_icvs.thread_limit) {
7594 num_threads = thr->th.th_current_task->td_icvs.thread_limit;
7596 if (num_teams * num_threads > __kmp_teams_max_nth) {
7597 num_threads = __kmp_teams_max_nth / num_teams;
7599 if (num_threads == 0) {
7605 thr->th.th_current_task->td_icvs.thread_limit = num_threads;
7607 if (num_threads > __kmp_dflt_team_nth) {
7608 num_threads = __kmp_dflt_team_nth;
7610 if (num_teams * num_threads > __kmp_teams_max_nth) {
7611 int new_threads = __kmp_teams_max_nth / num_teams;
7612 if (new_threads == 0) {
7615 if (new_threads != num_threads) {
7616 if (!__kmp_reserve_warn) {
7617 __kmp_reserve_warn = 1;
7618 __kmp_msg(kmp_ms_warning,
7619 KMP_MSG(CantFormThrTeam, num_threads, new_threads),
7620 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
7623 num_threads = new_threads;
7626 thr->th.th_teams_size.nth = num_threads;
7631 void __kmp_push_num_teams(
ident_t *
id,
int gtid,
int num_teams,
7633 kmp_info_t *thr = __kmp_threads[gtid];
7634 KMP_DEBUG_ASSERT(num_teams >= 0);
7635 KMP_DEBUG_ASSERT(num_threads >= 0);
7637 if (num_teams == 0) {
7638 if (__kmp_nteams > 0) {
7639 num_teams = __kmp_nteams;
7644 if (num_teams > __kmp_teams_max_nth) {
7645 if (!__kmp_reserve_warn) {
7646 __kmp_reserve_warn = 1;
7647 __kmp_msg(kmp_ms_warning,
7648 KMP_MSG(CantFormThrTeam, num_teams, __kmp_teams_max_nth),
7649 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
7651 num_teams = __kmp_teams_max_nth;
7655 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams;
7657 __kmp_push_thread_limit(thr, num_teams, num_threads);
7662 void __kmp_push_num_teams_51(
ident_t *
id,
int gtid,
int num_teams_lb,
7663 int num_teams_ub,
int num_threads) {
7664 kmp_info_t *thr = __kmp_threads[gtid];
7665 KMP_DEBUG_ASSERT(num_teams_lb >= 0 && num_teams_ub >= 0);
7666 KMP_DEBUG_ASSERT(num_teams_ub >= num_teams_lb);
7667 KMP_DEBUG_ASSERT(num_threads >= 0);
7669 if (num_teams_lb > num_teams_ub) {
7670 __kmp_fatal(KMP_MSG(FailedToCreateTeam, num_teams_lb, num_teams_ub),
7671 KMP_HNT(SetNewBound, __kmp_teams_max_nth), __kmp_msg_null);
7676 if (num_teams_lb == 0 && num_teams_ub > 0)
7677 num_teams_lb = num_teams_ub;
7679 if (num_teams_lb == 0 && num_teams_ub == 0) {
7680 num_teams = (__kmp_nteams > 0) ? __kmp_nteams : num_teams;
7681 if (num_teams > __kmp_teams_max_nth) {
7682 if (!__kmp_reserve_warn) {
7683 __kmp_reserve_warn = 1;
7684 __kmp_msg(kmp_ms_warning,
7685 KMP_MSG(CantFormThrTeam, num_teams, __kmp_teams_max_nth),
7686 KMP_HNT(Unset_ALL_THREADS), __kmp_msg_null);
7688 num_teams = __kmp_teams_max_nth;
7690 }
else if (num_teams_lb == num_teams_ub) {
7691 num_teams = num_teams_ub;
7693 if (num_threads == 0) {
7694 if (num_teams_ub > __kmp_teams_max_nth) {
7695 num_teams = num_teams_lb;
7697 num_teams = num_teams_ub;
7700 num_teams = (num_threads > __kmp_teams_max_nth)
7702 : __kmp_teams_max_nth / num_threads;
7703 if (num_teams < num_teams_lb) {
7704 num_teams = num_teams_lb;
7705 }
else if (num_teams > num_teams_ub) {
7706 num_teams = num_teams_ub;
7712 thr->th.th_set_nproc = thr->th.th_teams_size.nteams = num_teams;
7714 __kmp_push_thread_limit(thr, num_teams, num_threads);
7718 void __kmp_push_proc_bind(
ident_t *
id,
int gtid, kmp_proc_bind_t proc_bind) {
7719 kmp_info_t *thr = __kmp_threads[gtid];
7720 thr->th.th_set_proc_bind = proc_bind;
7725 void __kmp_internal_fork(
ident_t *
id,
int gtid, kmp_team_t *team) {
7726 kmp_info_t *this_thr = __kmp_threads[gtid];
7732 KMP_DEBUG_ASSERT(team);
7733 KMP_DEBUG_ASSERT(this_thr->th.th_team == team);
7734 KMP_ASSERT(KMP_MASTER_GTID(gtid));
7737 team->t.t_construct = 0;
7738 team->t.t_ordered.dt.t_value =
7742 KMP_DEBUG_ASSERT(team->t.t_disp_buffer);
7743 if (team->t.t_max_nproc > 1) {
7745 for (i = 0; i < __kmp_dispatch_num_buffers; ++i) {
7746 team->t.t_disp_buffer[i].buffer_index = i;
7747 team->t.t_disp_buffer[i].doacross_buf_idx = i;
7750 team->t.t_disp_buffer[0].buffer_index = 0;
7751 team->t.t_disp_buffer[0].doacross_buf_idx = 0;
7755 KMP_ASSERT(this_thr->th.th_team == team);
7758 for (f = 0; f < team->t.t_nproc; f++) {
7759 KMP_DEBUG_ASSERT(team->t.t_threads[f] &&
7760 team->t.t_threads[f]->th.th_team_nproc == team->t.t_nproc);
7765 __kmp_fork_barrier(gtid, 0);
7768 void __kmp_internal_join(
ident_t *
id,
int gtid, kmp_team_t *team) {
7769 kmp_info_t *this_thr = __kmp_threads[gtid];
7771 KMP_DEBUG_ASSERT(team);
7772 KMP_DEBUG_ASSERT(this_thr->th.th_team == team);
7773 KMP_ASSERT(KMP_MASTER_GTID(gtid));
7779 if (__kmp_threads[gtid] &&
7780 __kmp_threads[gtid]->th.th_team_nproc != team->t.t_nproc) {
7781 __kmp_printf(
"GTID: %d, __kmp_threads[%d]=%p\n", gtid, gtid,
7782 __kmp_threads[gtid]);
7783 __kmp_printf(
"__kmp_threads[%d]->th.th_team_nproc=%d, TEAM: %p, "
7784 "team->t.t_nproc=%d\n",
7785 gtid, __kmp_threads[gtid]->th.th_team_nproc, team,
7787 __kmp_print_structure();
7789 KMP_DEBUG_ASSERT(__kmp_threads[gtid] &&
7790 __kmp_threads[gtid]->th.th_team_nproc == team->t.t_nproc);
7793 __kmp_join_barrier(gtid);
7795 if (ompt_enabled.enabled &&
7796 this_thr->th.ompt_thread_info.state == ompt_state_wait_barrier_implicit) {
7797 int ds_tid = this_thr->th.th_info.ds.ds_tid;
7798 ompt_data_t *task_data = OMPT_CUR_TASK_DATA(this_thr);
7799 this_thr->th.ompt_thread_info.state = ompt_state_overhead;
7801 void *codeptr = NULL;
7802 if (KMP_MASTER_TID(ds_tid) &&
7803 (ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait) ||
7804 ompt_callbacks.ompt_callback(ompt_callback_sync_region)))
7805 codeptr = OMPT_CUR_TEAM_INFO(this_thr)->master_return_address;
7807 if (ompt_enabled.ompt_callback_sync_region_wait) {
7808 ompt_callbacks.ompt_callback(ompt_callback_sync_region_wait)(
7809 ompt_sync_region_barrier_implicit, ompt_scope_end, NULL, task_data,
7812 if (ompt_enabled.ompt_callback_sync_region) {
7813 ompt_callbacks.ompt_callback(ompt_callback_sync_region)(
7814 ompt_sync_region_barrier_implicit, ompt_scope_end, NULL, task_data,
7818 if (!KMP_MASTER_TID(ds_tid) && ompt_enabled.ompt_callback_implicit_task) {
7819 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)(
7820 ompt_scope_end, NULL, task_data, 0, ds_tid,
7821 ompt_task_implicit);
7827 KMP_ASSERT(this_thr->th.th_team == team);
7832 #ifdef USE_LOAD_BALANCE
7836 static int __kmp_active_hot_team_nproc(kmp_root_t *root) {
7839 kmp_team_t *hot_team;
7841 if (root->r.r_active) {
7844 hot_team = root->r.r_hot_team;
7845 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME) {
7846 return hot_team->t.t_nproc - 1;
7851 for (i = 1; i < hot_team->t.t_nproc; i++) {
7852 if (hot_team->t.t_threads[i]->th.th_active) {
7861 static int __kmp_load_balance_nproc(kmp_root_t *root,
int set_nproc) {
7864 int hot_team_active;
7865 int team_curr_active;
7868 KB_TRACE(20, (
"__kmp_load_balance_nproc: called root:%p set_nproc:%d\n", root,
7870 KMP_DEBUG_ASSERT(root);
7871 KMP_DEBUG_ASSERT(root->r.r_root_team->t.t_threads[0]
7872 ->th.th_current_task->td_icvs.dynamic == TRUE);
7873 KMP_DEBUG_ASSERT(set_nproc > 1);
7875 if (set_nproc == 1) {
7876 KB_TRACE(20, (
"__kmp_load_balance_nproc: serial execution.\n"));
7885 pool_active = __kmp_thread_pool_active_nth;
7886 hot_team_active = __kmp_active_hot_team_nproc(root);
7887 team_curr_active = pool_active + hot_team_active + 1;
7890 system_active = __kmp_get_load_balance(__kmp_avail_proc + team_curr_active);
7891 KB_TRACE(30, (
"__kmp_load_balance_nproc: system active = %d pool active = %d "
7892 "hot team active = %d\n",
7893 system_active, pool_active, hot_team_active));
7895 if (system_active < 0) {
7899 __kmp_global.g.g_dynamic_mode = dynamic_thread_limit;
7900 KMP_WARNING(CantLoadBalUsing,
"KMP_DYNAMIC_MODE=thread limit");
7903 retval = __kmp_avail_proc - __kmp_nth +
7904 (root->r.r_active ? 1 : root->r.r_hot_team->t.t_nproc);
7905 if (retval > set_nproc) {
7908 if (retval < KMP_MIN_NTH) {
7909 retval = KMP_MIN_NTH;
7912 KB_TRACE(20, (
"__kmp_load_balance_nproc: thread limit exit. retval:%d\n",
7920 if (system_active < team_curr_active) {
7921 system_active = team_curr_active;
7923 retval = __kmp_avail_proc - system_active + team_curr_active;
7924 if (retval > set_nproc) {
7927 if (retval < KMP_MIN_NTH) {
7928 retval = KMP_MIN_NTH;
7931 KB_TRACE(20, (
"__kmp_load_balance_nproc: exit. retval:%d\n", retval));
7940 void __kmp_cleanup(
void) {
7943 KA_TRACE(10, (
"__kmp_cleanup: enter\n"));
7945 if (TCR_4(__kmp_init_parallel)) {
7946 #if KMP_HANDLE_SIGNALS
7947 __kmp_remove_signals();
7949 TCW_4(__kmp_init_parallel, FALSE);
7952 if (TCR_4(__kmp_init_middle)) {
7953 #if KMP_AFFINITY_SUPPORTED
7954 __kmp_affinity_uninitialize();
7956 __kmp_cleanup_hierarchy();
7957 TCW_4(__kmp_init_middle, FALSE);
7960 KA_TRACE(10, (
"__kmp_cleanup: go serial cleanup\n"));
7962 if (__kmp_init_serial) {
7963 __kmp_runtime_destroy();
7964 __kmp_init_serial = FALSE;
7967 __kmp_cleanup_threadprivate_caches();
7969 for (f = 0; f < __kmp_threads_capacity; f++) {
7970 if (__kmp_root[f] != NULL) {
7971 __kmp_free(__kmp_root[f]);
7972 __kmp_root[f] = NULL;
7975 __kmp_free(__kmp_threads);
7978 __kmp_threads = NULL;
7980 __kmp_threads_capacity = 0;
7982 #if KMP_USE_DYNAMIC_LOCK
7983 __kmp_cleanup_indirect_user_locks();
7985 __kmp_cleanup_user_locks();
7989 __kmp_free(ompd_env_block);
7990 ompd_env_block = NULL;
7991 ompd_env_block_size = 0;
7995 #if KMP_AFFINITY_SUPPORTED
7996 KMP_INTERNAL_FREE(CCAST(
char *, __kmp_cpuinfo_file));
7997 __kmp_cpuinfo_file = NULL;
8000 #if KMP_USE_ADAPTIVE_LOCKS
8001 #if KMP_DEBUG_ADAPTIVE_LOCKS
8002 __kmp_print_speculative_stats();
8005 KMP_INTERNAL_FREE(__kmp_nested_nth.nth);
8006 __kmp_nested_nth.nth = NULL;
8007 __kmp_nested_nth.size = 0;
8008 __kmp_nested_nth.used = 0;
8009 KMP_INTERNAL_FREE(__kmp_nested_proc_bind.bind_types);
8010 __kmp_nested_proc_bind.bind_types = NULL;
8011 __kmp_nested_proc_bind.size = 0;
8012 __kmp_nested_proc_bind.used = 0;
8013 if (__kmp_affinity_format) {
8014 KMP_INTERNAL_FREE(__kmp_affinity_format);
8015 __kmp_affinity_format = NULL;
8018 __kmp_i18n_catclose();
8020 #if KMP_USE_HIER_SCHED
8021 __kmp_hier_scheds.deallocate();
8024 #if KMP_STATS_ENABLED
8028 KA_TRACE(10, (
"__kmp_cleanup: exit\n"));
8033 int __kmp_ignore_mppbeg(
void) {
8036 if ((env = getenv(
"KMP_IGNORE_MPPBEG")) != NULL) {
8037 if (__kmp_str_match_false(env))
8044 int __kmp_ignore_mppend(
void) {
8047 if ((env = getenv(
"KMP_IGNORE_MPPEND")) != NULL) {
8048 if (__kmp_str_match_false(env))
8055 void __kmp_internal_begin(
void) {
8061 gtid = __kmp_entry_gtid();
8062 root = __kmp_threads[gtid]->th.th_root;
8063 KMP_ASSERT(KMP_UBER_GTID(gtid));
8065 if (root->r.r_begin)
8067 __kmp_acquire_lock(&root->r.r_begin_lock, gtid);
8068 if (root->r.r_begin) {
8069 __kmp_release_lock(&root->r.r_begin_lock, gtid);
8073 root->r.r_begin = TRUE;
8075 __kmp_release_lock(&root->r.r_begin_lock, gtid);
8080 void __kmp_user_set_library(
enum library_type arg) {
8087 gtid = __kmp_entry_gtid();
8088 thread = __kmp_threads[gtid];
8090 root = thread->th.th_root;
8092 KA_TRACE(20, (
"__kmp_user_set_library: enter T#%d, arg: %d, %d\n", gtid, arg,
8094 if (root->r.r_in_parallel) {
8096 KMP_WARNING(SetLibraryIncorrectCall);
8101 case library_serial:
8102 thread->th.th_set_nproc = 0;
8103 set__nproc(thread, 1);
8105 case library_turnaround:
8106 thread->th.th_set_nproc = 0;
8107 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth
8108 : __kmp_dflt_team_nth_ub);
8110 case library_throughput:
8111 thread->th.th_set_nproc = 0;
8112 set__nproc(thread, __kmp_dflt_team_nth ? __kmp_dflt_team_nth
8113 : __kmp_dflt_team_nth_ub);
8116 KMP_FATAL(UnknownLibraryType, arg);
8119 __kmp_aux_set_library(arg);
8122 void __kmp_aux_set_stacksize(
size_t arg) {
8123 if (!__kmp_init_serial)
8124 __kmp_serial_initialize();
8127 if (arg & (0x1000 - 1)) {
8128 arg &= ~(0x1000 - 1);
8133 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
8136 if (!TCR_4(__kmp_init_parallel)) {
8139 if (value < __kmp_sys_min_stksize)
8140 value = __kmp_sys_min_stksize;
8141 else if (value > KMP_MAX_STKSIZE)
8142 value = KMP_MAX_STKSIZE;
8144 __kmp_stksize = value;
8146 __kmp_env_stksize = TRUE;
8149 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
8154 void __kmp_aux_set_library(
enum library_type arg) {
8155 __kmp_library = arg;
8157 switch (__kmp_library) {
8158 case library_serial: {
8159 KMP_INFORM(LibraryIsSerial);
8161 case library_turnaround:
8162 if (__kmp_use_yield == 1 && !__kmp_use_yield_exp_set)
8163 __kmp_use_yield = 2;
8165 case library_throughput:
8166 if (__kmp_dflt_blocktime == KMP_MAX_BLOCKTIME)
8167 __kmp_dflt_blocktime = 200;
8170 KMP_FATAL(UnknownLibraryType, arg);
8176 static kmp_team_t *__kmp_aux_get_team_info(
int &teams_serialized) {
8177 kmp_info_t *thr = __kmp_entry_thread();
8178 teams_serialized = 0;
8179 if (thr->th.th_teams_microtask) {
8180 kmp_team_t *team = thr->th.th_team;
8181 int tlevel = thr->th.th_teams_level;
8182 int ii = team->t.t_level;
8183 teams_serialized = team->t.t_serialized;
8184 int level = tlevel + 1;
8185 KMP_DEBUG_ASSERT(ii >= tlevel);
8186 while (ii > level) {
8187 for (teams_serialized = team->t.t_serialized;
8188 (teams_serialized > 0) && (ii > level); teams_serialized--, ii--) {
8190 if (team->t.t_serialized && (!teams_serialized)) {
8191 team = team->t.t_parent;
8195 team = team->t.t_parent;
8204 int __kmp_aux_get_team_num() {
8206 kmp_team_t *team = __kmp_aux_get_team_info(serialized);
8208 if (serialized > 1) {
8211 return team->t.t_master_tid;
8217 int __kmp_aux_get_num_teams() {
8219 kmp_team_t *team = __kmp_aux_get_team_info(serialized);
8221 if (serialized > 1) {
8224 return team->t.t_parent->t.t_nproc;
8263 typedef struct kmp_affinity_format_field_t {
8265 const char *long_name;
8268 } kmp_affinity_format_field_t;
8270 static const kmp_affinity_format_field_t __kmp_affinity_format_table[] = {
8271 #if KMP_AFFINITY_SUPPORTED
8272 {
'A',
"thread_affinity",
's'},
8274 {
't',
"team_num",
'd'},
8275 {
'T',
"num_teams",
'd'},
8276 {
'L',
"nesting_level",
'd'},
8277 {
'n',
"thread_num",
'd'},
8278 {
'N',
"num_threads",
'd'},
8279 {
'a',
"ancestor_tnum",
'd'},
8281 {
'P',
"process_id",
'd'},
8282 {
'i',
"native_thread_id",
'd'}};
8285 static int __kmp_aux_capture_affinity_field(
int gtid,
const kmp_info_t *th,
8287 kmp_str_buf_t *field_buffer) {
8288 int rc, format_index, field_value;
8289 const char *width_left, *width_right;
8290 bool pad_zeros, right_justify, parse_long_name, found_valid_name;
8291 static const int FORMAT_SIZE = 20;
8292 char format[FORMAT_SIZE] = {0};
8293 char absolute_short_name = 0;
8295 KMP_DEBUG_ASSERT(gtid >= 0);
8296 KMP_DEBUG_ASSERT(th);
8297 KMP_DEBUG_ASSERT(**ptr ==
'%');
8298 KMP_DEBUG_ASSERT(field_buffer);
8300 __kmp_str_buf_clear(field_buffer);
8307 __kmp_str_buf_cat(field_buffer,
"%", 1);
8318 right_justify =
false;
8320 right_justify =
true;
8324 width_left = width_right = NULL;
8325 if (**ptr >=
'0' && **ptr <=
'9') {
8333 format[format_index++] =
'%';
8335 format[format_index++] =
'-';
8337 format[format_index++] =
'0';
8338 if (width_left && width_right) {
8342 while (i < 8 && width_left < width_right) {
8343 format[format_index++] = *width_left;
8351 found_valid_name =
false;
8352 parse_long_name = (**ptr ==
'{');
8353 if (parse_long_name)
8355 for (
size_t i = 0; i <
sizeof(__kmp_affinity_format_table) /
8356 sizeof(__kmp_affinity_format_table[0]);
8358 char short_name = __kmp_affinity_format_table[i].short_name;
8359 const char *long_name = __kmp_affinity_format_table[i].long_name;
8360 char field_format = __kmp_affinity_format_table[i].field_format;
8361 if (parse_long_name) {
8362 size_t length = KMP_STRLEN(long_name);
8363 if (strncmp(*ptr, long_name, length) == 0) {
8364 found_valid_name =
true;
8367 }
else if (**ptr == short_name) {
8368 found_valid_name =
true;
8371 if (found_valid_name) {
8372 format[format_index++] = field_format;
8373 format[format_index++] =
'\0';
8374 absolute_short_name = short_name;
8378 if (parse_long_name) {
8380 absolute_short_name = 0;
8388 switch (absolute_short_name) {
8390 rc = __kmp_str_buf_print(field_buffer, format, __kmp_aux_get_team_num());
8393 rc = __kmp_str_buf_print(field_buffer, format, __kmp_aux_get_num_teams());
8396 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_level);
8399 rc = __kmp_str_buf_print(field_buffer, format, __kmp_tid_from_gtid(gtid));
8402 static const int BUFFER_SIZE = 256;
8403 char buf[BUFFER_SIZE];
8404 __kmp_expand_host_name(buf, BUFFER_SIZE);
8405 rc = __kmp_str_buf_print(field_buffer, format, buf);
8408 rc = __kmp_str_buf_print(field_buffer, format, getpid());
8411 rc = __kmp_str_buf_print(field_buffer, format, __kmp_gettid());
8414 rc = __kmp_str_buf_print(field_buffer, format, th->th.th_team->t.t_nproc);
8418 __kmp_get_ancestor_thread_num(gtid, th->th.th_team->t.t_level - 1);
8419 rc = __kmp_str_buf_print(field_buffer, format, field_value);
8421 #if KMP_AFFINITY_SUPPORTED
8424 __kmp_str_buf_init(&buf);
8425 __kmp_affinity_str_buf_mask(&buf, th->th.th_affin_mask);
8426 rc = __kmp_str_buf_print(field_buffer, format, buf.str);
8427 __kmp_str_buf_free(&buf);
8433 rc = __kmp_str_buf_print(field_buffer,
"%s",
"undefined");
8435 if (parse_long_name) {
8444 KMP_ASSERT(format_index <= FORMAT_SIZE);
8454 size_t __kmp_aux_capture_affinity(
int gtid,
const char *format,
8455 kmp_str_buf_t *buffer) {
8456 const char *parse_ptr;
8458 const kmp_info_t *th;
8459 kmp_str_buf_t field;
8461 KMP_DEBUG_ASSERT(buffer);
8462 KMP_DEBUG_ASSERT(gtid >= 0);
8464 __kmp_str_buf_init(&field);
8465 __kmp_str_buf_clear(buffer);
8467 th = __kmp_threads[gtid];
8473 if (parse_ptr == NULL || *parse_ptr ==
'\0') {
8474 parse_ptr = __kmp_affinity_format;
8476 KMP_DEBUG_ASSERT(parse_ptr);
8478 while (*parse_ptr !=
'\0') {
8480 if (*parse_ptr ==
'%') {
8482 int rc = __kmp_aux_capture_affinity_field(gtid, th, &parse_ptr, &field);
8483 __kmp_str_buf_catbuf(buffer, &field);
8487 __kmp_str_buf_cat(buffer, parse_ptr, 1);
8492 __kmp_str_buf_free(&field);
8497 void __kmp_aux_display_affinity(
int gtid,
const char *format) {
8499 __kmp_str_buf_init(&buf);
8500 __kmp_aux_capture_affinity(gtid, format, &buf);
8501 __kmp_fprintf(kmp_out,
"%s" KMP_END_OF_LINE, buf.str);
8502 __kmp_str_buf_free(&buf);
8507 void __kmp_aux_set_blocktime(
int arg, kmp_info_t *thread,
int tid) {
8508 int blocktime = arg;
8514 __kmp_save_internal_controls(thread);
8517 if (blocktime < KMP_MIN_BLOCKTIME)
8518 blocktime = KMP_MIN_BLOCKTIME;
8519 else if (blocktime > KMP_MAX_BLOCKTIME)
8520 blocktime = KMP_MAX_BLOCKTIME;
8522 set__blocktime_team(thread->th.th_team, tid, blocktime);
8523 set__blocktime_team(thread->th.th_serial_team, 0, blocktime);
8527 bt_intervals = KMP_INTERVALS_FROM_BLOCKTIME(blocktime, __kmp_monitor_wakeups);
8529 set__bt_intervals_team(thread->th.th_team, tid, bt_intervals);
8530 set__bt_intervals_team(thread->th.th_serial_team, 0, bt_intervals);
8536 set__bt_set_team(thread->th.th_team, tid, bt_set);
8537 set__bt_set_team(thread->th.th_serial_team, 0, bt_set);
8539 KF_TRACE(10, (
"kmp_set_blocktime: T#%d(%d:%d), blocktime=%d, "
8540 "bt_intervals=%d, monitor_updates=%d\n",
8541 __kmp_gtid_from_tid(tid, thread->th.th_team),
8542 thread->th.th_team->t.t_id, tid, blocktime, bt_intervals,
8543 __kmp_monitor_wakeups));
8545 KF_TRACE(10, (
"kmp_set_blocktime: T#%d(%d:%d), blocktime=%d\n",
8546 __kmp_gtid_from_tid(tid, thread->th.th_team),
8547 thread->th.th_team->t.t_id, tid, blocktime));
8551 void __kmp_aux_set_defaults(
char const *str,
size_t len) {
8552 if (!__kmp_init_serial) {
8553 __kmp_serial_initialize();
8555 __kmp_env_initialize(str);
8557 if (__kmp_settings || __kmp_display_env || __kmp_display_env_verbose) {
8565 PACKED_REDUCTION_METHOD_T
8566 __kmp_determine_reduction_method(
8567 ident_t *loc, kmp_int32 global_tid, kmp_int32 num_vars,
size_t reduce_size,
8568 void *reduce_data,
void (*reduce_func)(
void *lhs_data,
void *rhs_data),
8569 kmp_critical_name *lck) {
8580 PACKED_REDUCTION_METHOD_T retval;
8584 KMP_DEBUG_ASSERT(loc);
8585 KMP_DEBUG_ASSERT(lck);
8587 #define FAST_REDUCTION_ATOMIC_METHOD_GENERATED \
8588 ((loc->flags & (KMP_IDENT_ATOMIC_REDUCE)) == (KMP_IDENT_ATOMIC_REDUCE))
8589 #define FAST_REDUCTION_TREE_METHOD_GENERATED ((reduce_data) && (reduce_func))
8591 retval = critical_reduce_block;
8594 team_size = __kmp_get_team_num_threads(global_tid);
8595 if (team_size == 1) {
8597 retval = empty_reduce_block;
8601 int atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;
8603 #if KMP_ARCH_X86_64 || KMP_ARCH_PPC64 || KMP_ARCH_AARCH64 || \
8604 KMP_ARCH_MIPS64 || KMP_ARCH_RISCV64
8606 #if KMP_OS_LINUX || KMP_OS_DRAGONFLY || KMP_OS_FREEBSD || KMP_OS_NETBSD || \
8607 KMP_OS_OPENBSD || KMP_OS_WINDOWS || KMP_OS_DARWIN || KMP_OS_HURD
8609 int teamsize_cutoff = 4;
8611 #if KMP_MIC_SUPPORTED
8612 if (__kmp_mic_type != non_mic) {
8613 teamsize_cutoff = 8;
8616 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
8617 if (tree_available) {
8618 if (team_size <= teamsize_cutoff) {
8619 if (atomic_available) {
8620 retval = atomic_reduce_block;
8623 retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER;
8625 }
else if (atomic_available) {
8626 retval = atomic_reduce_block;
8629 #error "Unknown or unsupported OS"
8633 #elif KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_AARCH || KMP_ARCH_MIPS
8635 #if KMP_OS_LINUX || KMP_OS_FREEBSD || KMP_OS_WINDOWS || KMP_OS_HURD
8639 if (atomic_available) {
8640 if (num_vars <= 2) {
8641 retval = atomic_reduce_block;
8647 int tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
8648 if (atomic_available && (num_vars <= 3)) {
8649 retval = atomic_reduce_block;
8650 }
else if (tree_available) {
8651 if ((reduce_size > (9 *
sizeof(kmp_real64))) &&
8652 (reduce_size < (2000 *
sizeof(kmp_real64)))) {
8653 retval = TREE_REDUCE_BLOCK_WITH_PLAIN_BARRIER;
8658 #error "Unknown or unsupported OS"
8662 #error "Unknown or unsupported architecture"
8670 if (__kmp_force_reduction_method != reduction_method_not_defined &&
8673 PACKED_REDUCTION_METHOD_T forced_retval = critical_reduce_block;
8675 int atomic_available, tree_available;
8677 switch ((forced_retval = __kmp_force_reduction_method)) {
8678 case critical_reduce_block:
8682 case atomic_reduce_block:
8683 atomic_available = FAST_REDUCTION_ATOMIC_METHOD_GENERATED;
8684 if (!atomic_available) {
8685 KMP_WARNING(RedMethodNotSupported,
"atomic");
8686 forced_retval = critical_reduce_block;
8690 case tree_reduce_block:
8691 tree_available = FAST_REDUCTION_TREE_METHOD_GENERATED;
8692 if (!tree_available) {
8693 KMP_WARNING(RedMethodNotSupported,
"tree");
8694 forced_retval = critical_reduce_block;
8696 #if KMP_FAST_REDUCTION_BARRIER
8697 forced_retval = TREE_REDUCE_BLOCK_WITH_REDUCTION_BARRIER;
8706 retval = forced_retval;
8709 KA_TRACE(10, (
"reduction method selected=%08x\n", retval));
8711 #undef FAST_REDUCTION_TREE_METHOD_GENERATED
8712 #undef FAST_REDUCTION_ATOMIC_METHOD_GENERATED
8717 kmp_int32 __kmp_get_reduce_method(
void) {
8718 return ((__kmp_entry_thread()->th.th_local.packed_reduction_method) >> 8);
8723 void __kmp_soft_pause() { __kmp_pause_status = kmp_soft_paused; }
8727 void __kmp_hard_pause() {
8728 __kmp_pause_status = kmp_hard_paused;
8729 __kmp_internal_end_thread(-1);
8733 void __kmp_resume_if_soft_paused() {
8734 if (__kmp_pause_status == kmp_soft_paused) {
8735 __kmp_pause_status = kmp_not_paused;
8737 for (
int gtid = 1; gtid < __kmp_threads_capacity; ++gtid) {
8738 kmp_info_t *thread = __kmp_threads[gtid];
8740 kmp_flag_64<> fl(&thread->th.th_bar[bs_forkjoin_barrier].bb.b_go,
8742 if (fl.is_sleeping())
8744 else if (__kmp_try_suspend_mx(thread)) {
8745 __kmp_unlock_suspend_mx(thread);
8748 if (fl.is_sleeping()) {
8751 }
else if (__kmp_try_suspend_mx(thread)) {
8752 __kmp_unlock_suspend_mx(thread);
8764 int __kmp_pause_resource(kmp_pause_status_t level) {
8765 if (level == kmp_not_paused) {
8766 if (__kmp_pause_status == kmp_not_paused) {
8770 KMP_DEBUG_ASSERT(__kmp_pause_status == kmp_soft_paused ||
8771 __kmp_pause_status == kmp_hard_paused);
8772 __kmp_pause_status = kmp_not_paused;
8775 }
else if (level == kmp_soft_paused) {
8776 if (__kmp_pause_status != kmp_not_paused) {
8783 }
else if (level == kmp_hard_paused) {
8784 if (__kmp_pause_status != kmp_not_paused) {
8797 void __kmp_omp_display_env(
int verbose) {
8798 __kmp_acquire_bootstrap_lock(&__kmp_initz_lock);
8799 if (__kmp_init_serial == 0)
8800 __kmp_do_serial_initialize();
8801 __kmp_display_env_impl(!verbose, verbose);
8802 __kmp_release_bootstrap_lock(&__kmp_initz_lock);
8806 void __kmp_resize_dist_barrier(kmp_team_t *team,
int old_nthreads,
8808 KMP_DEBUG_ASSERT(__kmp_barrier_release_pattern[bs_forkjoin_barrier] ==
8810 kmp_info_t **other_threads = team->t.t_threads;
8814 for (
int f = 1; f < old_nthreads; ++f) {
8815 KMP_DEBUG_ASSERT(other_threads[f] != NULL);
8817 if (team->t.t_threads[f]->th.th_used_in_team.load() == 0) {
8823 if (team->t.t_threads[f]->th.th_used_in_team.load() == 3) {
8824 while (team->t.t_threads[f]->th.th_used_in_team.load() == 3)
8828 KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 1);
8830 team->t.t_threads[f]->th.th_used_in_team.store(2);
8831 KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 2);
8834 kmp_uint64 new_value;
8835 new_value = team->t.b->go_release();
8842 int count = old_nthreads - 1;
8844 count = old_nthreads - 1;
8845 for (
int f = 1; f < old_nthreads; ++f) {
8846 my_go_index = f / team->t.b->threads_per_go;
8847 if (other_threads[f]->th.th_used_in_team.load() != 0) {
8848 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
8849 kmp_atomic_flag_64<> *flag = (kmp_atomic_flag_64<> *)CCAST(
8850 void *, other_threads[f]->th.th_sleep_loc);
8851 __kmp_atomic_resume_64(other_threads[f]->th.th_info.ds.ds_gtid, flag);
8854 KMP_DEBUG_ASSERT(team->t.t_threads[f]->th.th_used_in_team.load() == 0);
8860 team->t.b->update_num_threads(new_nthreads);
8861 team->t.b->go_reset();
8864 void __kmp_add_threads_to_team(kmp_team_t *team,
int new_nthreads) {
8866 KMP_DEBUG_ASSERT(team);
8872 for (
int f = 1; f < new_nthreads; ++f) {
8873 KMP_DEBUG_ASSERT(team->t.t_threads[f]);
8874 KMP_COMPARE_AND_STORE_ACQ32(&(team->t.t_threads[f]->th.th_used_in_team), 0,
8876 if (__kmp_dflt_blocktime != KMP_MAX_BLOCKTIME) {
8877 __kmp_resume_32(team->t.t_threads[f]->th.th_info.ds.ds_gtid,
8878 (kmp_flag_32<false, false> *)NULL);
8884 int count = new_nthreads - 1;
8886 count = new_nthreads - 1;
8887 for (
int f = 1; f < new_nthreads; ++f) {
8888 if (team->t.t_threads[f]->th.th_used_in_team.load() == 1) {
8896 kmp_info_t **__kmp_hidden_helper_threads;
8897 kmp_info_t *__kmp_hidden_helper_main_thread;
8898 kmp_int32 __kmp_hidden_helper_threads_num = 8;
8899 std::atomic<kmp_int32> __kmp_unexecuted_hidden_helper_tasks;
8901 kmp_int32 __kmp_enable_hidden_helper = TRUE;
8903 kmp_int32 __kmp_enable_hidden_helper = FALSE;
8907 std::atomic<kmp_int32> __kmp_hit_hidden_helper_threads_num;
8909 void __kmp_hidden_helper_wrapper_fn(
int *gtid,
int *, ...) {
8914 KMP_ATOMIC_INC(&__kmp_hit_hidden_helper_threads_num);
8915 while (KMP_ATOMIC_LD_ACQ(&__kmp_hit_hidden_helper_threads_num) !=
8916 __kmp_hidden_helper_threads_num)
8922 TCW_4(__kmp_init_hidden_helper_threads, FALSE);
8923 __kmp_hidden_helper_initz_release();
8924 __kmp_hidden_helper_main_thread_wait();
8926 for (
int i = 1; i < __kmp_hit_hidden_helper_threads_num; ++i) {
8927 __kmp_hidden_helper_worker_thread_signal();
8933 void __kmp_hidden_helper_threads_initz_routine() {
8935 const int gtid = __kmp_register_root(TRUE);
8936 __kmp_hidden_helper_main_thread = __kmp_threads[gtid];
8937 __kmp_hidden_helper_threads = &__kmp_threads[gtid];
8938 __kmp_hidden_helper_main_thread->th.th_set_nproc =
8939 __kmp_hidden_helper_threads_num;
8941 KMP_ATOMIC_ST_REL(&__kmp_hit_hidden_helper_threads_num, 0);
8946 TCW_SYNC_4(__kmp_init_hidden_helper, FALSE);
8948 __kmp_hidden_helper_threads_deinitz_release();
8968 void __kmp_init_nesting_mode() {
8969 int levels = KMP_HW_LAST;
8970 __kmp_nesting_mode_nlevels = levels;
8971 __kmp_nesting_nth_level = (
int *)KMP_INTERNAL_MALLOC(levels *
sizeof(
int));
8972 for (
int i = 0; i < levels; ++i)
8973 __kmp_nesting_nth_level[i] = 0;
8974 if (__kmp_nested_nth.size < levels) {
8975 __kmp_nested_nth.nth =
8976 (
int *)KMP_INTERNAL_REALLOC(__kmp_nested_nth.nth, levels *
sizeof(
int));
8977 __kmp_nested_nth.size = levels;
8982 void __kmp_set_nesting_mode_threads() {
8983 kmp_info_t *thread = __kmp_threads[__kmp_entry_gtid()];
8985 if (__kmp_nesting_mode == 1)
8986 __kmp_nesting_mode_nlevels = KMP_MAX_ACTIVE_LEVELS_LIMIT;
8987 else if (__kmp_nesting_mode > 1)
8988 __kmp_nesting_mode_nlevels = __kmp_nesting_mode;
8990 if (__kmp_topology) {
8992 for (loc = 0, hw_level = 0; hw_level < __kmp_topology->get_depth() &&
8993 loc < __kmp_nesting_mode_nlevels;
8994 loc++, hw_level++) {
8995 __kmp_nesting_nth_level[loc] = __kmp_topology->get_ratio(hw_level);
8996 if (__kmp_nesting_nth_level[loc] == 1)
9000 if (__kmp_nesting_mode > 1 && loc > 1) {
9001 int core_level = __kmp_topology->get_level(KMP_HW_CORE);
9002 int num_cores = __kmp_topology->get_count(core_level);
9003 int upper_levels = 1;
9004 for (
int level = 0; level < loc - 1; ++level)
9005 upper_levels *= __kmp_nesting_nth_level[level];
9006 if (upper_levels * __kmp_nesting_nth_level[loc - 1] < num_cores)
9007 __kmp_nesting_nth_level[loc - 1] =
9008 num_cores / __kmp_nesting_nth_level[loc - 2];
9010 __kmp_nesting_mode_nlevels = loc;
9011 __kmp_nested_nth.used = __kmp_nesting_mode_nlevels;
9013 if (__kmp_avail_proc >= 4) {
9014 __kmp_nesting_nth_level[0] = __kmp_avail_proc / 2;
9015 __kmp_nesting_nth_level[1] = 2;
9016 __kmp_nesting_mode_nlevels = 2;
9018 __kmp_nesting_nth_level[0] = __kmp_avail_proc;
9019 __kmp_nesting_mode_nlevels = 1;
9021 __kmp_nested_nth.used = __kmp_nesting_mode_nlevels;
9023 for (
int i = 0; i < __kmp_nesting_mode_nlevels; ++i) {
9024 __kmp_nested_nth.nth[i] = __kmp_nesting_nth_level[i];
9026 set__nproc(thread, __kmp_nesting_nth_level[0]);
9027 if (__kmp_nesting_mode > 1 && __kmp_nesting_mode_nlevels > __kmp_nesting_mode)
9028 __kmp_nesting_mode_nlevels = __kmp_nesting_mode;
9029 if (get__max_active_levels(thread) > 1) {
9031 __kmp_nesting_mode_nlevels = get__max_active_levels(thread);
9033 if (__kmp_nesting_mode == 1)
9034 set__max_active_levels(thread, __kmp_nesting_mode_nlevels);
KMP_EXPORT void __kmpc_serialized_parallel(ident_t *, kmp_int32 global_tid)
KMP_EXPORT void __kmpc_fork_call(ident_t *, kmp_int32 nargs, kmpc_micro microtask,...)
KMP_EXPORT void __kmpc_end_serialized_parallel(ident_t *, kmp_int32 global_tid)
#define KMP_INIT_PARTITIONED_TIMERS(name)
Initializes the partitioned timers to begin with name.
#define KMP_COUNT_VALUE(name, value)
Adds value to specified timer (name).
stats_state_e
the states which a thread can be in
KMP_EXPORT kmp_int32 __kmpc_master(ident_t *, kmp_int32 global_tid)