mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-01-31 03:27:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions
cosmopolitan/third_party/openmp/kmp_sched.cpp
Justine Tunney 5f8e9f14c1
Add OpenMP support
2024-01-28 22:39:02 -08:00

1092 lines
40 KiB
C++
Raw Blame History

/*
* kmp_sched.cpp -- static scheduling -- iteration initialization
*/
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
/* Static scheduling initialization.
NOTE: team->t.t_nproc is a constant inside of any dispatch loop, however
it may change values between parallel regions. __kmp_max_nth
is the largest value __kmp_nth may take, 1 is the smallest. */
#include "kmp.h"
#include "kmp_error.h"
#include "kmp_i18n.h"
#include "kmp_itt.h"
#include "kmp_stats.h"
#include "kmp_str.h"
#if OMPT_SUPPORT
#include "ompt-specific.h"
#endif
#ifdef KMP_DEBUG
//-------------------------------------------------------------------------
// template for debug prints specification ( d, u, lld, llu )
char const *traits_t<int>::spec = "d";
char const *traits_t<unsigned int>::spec = "u";
char const *traits_t<long long>::spec = "lld";
char const *traits_t<unsigned long long>::spec = "llu";
char const *traits_t<long>::spec = "ld";
//-------------------------------------------------------------------------
#endif
#if KMP_STATS_ENABLED
#define KMP_STATS_LOOP_END(stat) \
{ \
kmp_int64 t; \
kmp_int64 u = (kmp_int64)(*pupper); \
kmp_int64 l = (kmp_int64)(*plower); \
kmp_int64 i = (kmp_int64)incr; \
if (i == 1) { \
t = u - l + 1; \
} else if (i == -1) { \
t = l - u + 1; \
} else if (i > 0) { \
t = (u - l) / i + 1; \
} else { \
t = (l - u) / (-i) + 1; \
} \
KMP_COUNT_VALUE(stat, t); \
KMP_POP_PARTITIONED_TIMER(); \
}
#else
#define KMP_STATS_LOOP_END(stat) /* Nothing */
#endif
#if USE_ITT_BUILD || defined KMP_DEBUG
static ident_t loc_stub = {0, KMP_IDENT_KMPC, 0, 0, ";unknown;unknown;0;0;;"};
static inline void check_loc(ident_t *&loc) {
if (loc == NULL)
loc = &loc_stub; // may need to report location info to ittnotify
}
#endif
template <typename T>
static void __kmp_for_static_init(ident_t *loc, kmp_int32 global_tid,
kmp_int32 schedtype, kmp_int32 *plastiter,
T *plower, T *pupper,
typename traits_t<T>::signed_t *pstride,
typename traits_t<T>::signed_t incr,
typename traits_t<T>::signed_t chunk
#if OMPT_SUPPORT && OMPT_OPTIONAL
,
void *codeptr
#endif
) {
KMP_COUNT_BLOCK(OMP_LOOP_STATIC);
KMP_PUSH_PARTITIONED_TIMER(OMP_loop_static);
KMP_PUSH_PARTITIONED_TIMER(OMP_loop_static_scheduling);
// Clear monotonic/nonmonotonic bits (ignore it)
schedtype = SCHEDULE_WITHOUT_MODIFIERS(schedtype);
typedef typename traits_t<T>::unsigned_t UT;
typedef typename traits_t<T>::signed_t ST;
/* this all has to be changed back to TID and such.. */
kmp_int32 gtid = global_tid;
kmp_uint32 tid;
kmp_uint32 nth;
UT trip_count;
kmp_team_t *team;
__kmp_assert_valid_gtid(gtid);
kmp_info_t *th = __kmp_threads[gtid];
#if OMPT_SUPPORT && OMPT_OPTIONAL
ompt_team_info_t *team_info = NULL;
ompt_task_info_t *task_info = NULL;
ompt_work_t ompt_work_type = ompt_work_loop;
static kmp_int8 warn = 0;
if (ompt_enabled.ompt_callback_work || ompt_enabled.ompt_callback_dispatch) {
// Only fully initialize variables needed by OMPT if OMPT is enabled.
team_info = __ompt_get_teaminfo(0, NULL);
task_info = __ompt_get_task_info_object(0);
// Determine workshare type
if (loc != NULL) {
if ((loc->flags & KMP_IDENT_WORK_LOOP) != 0) {
ompt_work_type = ompt_work_loop;
} else if ((loc->flags & KMP_IDENT_WORK_SECTIONS) != 0) {
ompt_work_type = ompt_work_sections;
} else if ((loc->flags & KMP_IDENT_WORK_DISTRIBUTE) != 0) {
ompt_work_type = ompt_work_distribute;
} else {
kmp_int8 bool_res =
KMP_COMPARE_AND_STORE_ACQ8(&warn, (kmp_int8)0, (kmp_int8)1);
if (bool_res)
KMP_WARNING(OmptOutdatedWorkshare);
}
KMP_DEBUG_ASSERT(ompt_work_type);
}
}
#endif
KMP_DEBUG_ASSERT(plastiter && plower && pupper && pstride);
KE_TRACE(10, ("__kmpc_for_static_init called (%d)\n", global_tid));
#ifdef KMP_DEBUG
{
char *buff;
// create format specifiers before the debug output
buff = __kmp_str_format(
"__kmpc_for_static_init: T#%%d sched=%%d liter=%%d iter=(%%%s,"
" %%%s, %%%s) incr=%%%s chunk=%%%s signed?<%s>\n",
traits_t<T>::spec, traits_t<T>::spec, traits_t<ST>::spec,
traits_t<ST>::spec, traits_t<ST>::spec, traits_t<T>::spec);
KD_TRACE(100, (buff, global_tid, schedtype, *plastiter, *plower, *pupper,
*pstride, incr, chunk));
__kmp_str_free(&buff);
}
#endif
if (__kmp_env_consistency_check) {
__kmp_push_workshare(global_tid, ct_pdo, loc);
if (incr == 0) {
__kmp_error_construct(kmp_i18n_msg_CnsLoopIncrZeroProhibited, ct_pdo,
loc);
}
}
/* special handling for zero-trip loops */
if (incr > 0 ? (*pupper < *plower) : (*plower < *pupper)) {
if (plastiter != NULL)
*plastiter = FALSE;
/* leave pupper and plower set to entire iteration space */
*pstride = incr; /* value should never be used */
// *plower = *pupper - incr;
// let compiler bypass the illegal loop (like for(i=1;i<10;i--))
// THE LINE COMMENTED ABOVE CAUSED shape2F/h_tests_1.f TO HAVE A FAILURE
// ON A ZERO-TRIP LOOP (lower=1, upper=0,stride=1) - JPH June 23, 2009.
#ifdef KMP_DEBUG
{
char *buff;
// create format specifiers before the debug output
buff = __kmp_str_format("__kmpc_for_static_init:(ZERO TRIP) liter=%%d "
"lower=%%%s upper=%%%s stride = %%%s "
"signed?<%s>, loc = %%s\n",
traits_t<T>::spec, traits_t<T>::spec,
traits_t<ST>::spec, traits_t<T>::spec);
check_loc(loc);
KD_TRACE(100,
(buff, *plastiter, *plower, *pupper, *pstride, loc->psource));
__kmp_str_free(&buff);
}
#endif
KE_TRACE(10, ("__kmpc_for_static_init: T#%d return\n", global_tid));
#if OMPT_SUPPORT && OMPT_OPTIONAL
if (ompt_enabled.ompt_callback_work) {
ompt_callbacks.ompt_callback(ompt_callback_work)(
ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),
&(task_info->task_data), 0, codeptr);
}
#endif
KMP_STATS_LOOP_END(OMP_loop_static_iterations);
return;
}
// Although there are schedule enumerations above kmp_ord_upper which are not
// schedules for "distribute", the only ones which are useful are dynamic, so
// cannot be seen here, since this codepath is only executed for static
// schedules.
if (schedtype > kmp_ord_upper) {
// we are in DISTRIBUTE construct
schedtype += kmp_sch_static -
kmp_distribute_static; // AC: convert to usual schedule type
if (th->th.th_team->t.t_serialized > 1) {
tid = 0;
team = th->th.th_team;
} else {
tid = th->th.th_team->t.t_master_tid;
team = th->th.th_team->t.t_parent;
}
} else {
tid = __kmp_tid_from_gtid(global_tid);
team = th->th.th_team;
}
/* determine if "for" loop is an active worksharing construct */
if (team->t.t_serialized) {
/* serialized parallel, each thread executes whole iteration space */
if (plastiter != NULL)
*plastiter = TRUE;
/* leave pupper and plower set to entire iteration space */
*pstride =
(incr > 0) ? (*pupper - *plower + 1) : (-(*plower - *pupper + 1));
#ifdef KMP_DEBUG
{
char *buff;
// create format specifiers before the debug output
buff = __kmp_str_format("__kmpc_for_static_init: (serial) liter=%%d "
"lower=%%%s upper=%%%s stride = %%%s\n",
traits_t<T>::spec, traits_t<T>::spec,
traits_t<ST>::spec);
KD_TRACE(100, (buff, *plastiter, *plower, *pupper, *pstride));
__kmp_str_free(&buff);
}
#endif
KE_TRACE(10, ("__kmpc_for_static_init: T#%d return\n", global_tid));
#if OMPT_SUPPORT && OMPT_OPTIONAL
if (ompt_enabled.ompt_callback_work) {
ompt_callbacks.ompt_callback(ompt_callback_work)(
ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),
&(task_info->task_data), *pstride, codeptr);
}
#endif
KMP_STATS_LOOP_END(OMP_loop_static_iterations);
return;
}
nth = team->t.t_nproc;
if (nth == 1) {
if (plastiter != NULL)
*plastiter = TRUE;
*pstride =
(incr > 0) ? (*pupper - *plower + 1) : (-(*plower - *pupper + 1));
#ifdef KMP_DEBUG
{
char *buff;
// create format specifiers before the debug output
buff = __kmp_str_format("__kmpc_for_static_init: (serial) liter=%%d "
"lower=%%%s upper=%%%s stride = %%%s\n",
traits_t<T>::spec, traits_t<T>::spec,
traits_t<ST>::spec);
KD_TRACE(100, (buff, *plastiter, *plower, *pupper, *pstride));
__kmp_str_free(&buff);
}
#endif
KE_TRACE(10, ("__kmpc_for_static_init: T#%d return\n", global_tid));
#if OMPT_SUPPORT && OMPT_OPTIONAL
if (ompt_enabled.ompt_callback_work) {
ompt_callbacks.ompt_callback(ompt_callback_work)(
ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),
&(task_info->task_data), *pstride, codeptr);
}
#endif
KMP_STATS_LOOP_END(OMP_loop_static_iterations);
return;
}
/* compute trip count */
if (incr == 1) {
trip_count = *pupper - *plower + 1;
} else if (incr == -1) {
trip_count = *plower - *pupper + 1;
} else if (incr > 0) {
// upper-lower can exceed the limit of signed type
trip_count = (UT)(*pupper - *plower) / incr + 1;
} else {
trip_count = (UT)(*plower - *pupper) / (-incr) + 1;
}
#if KMP_STATS_ENABLED
if (KMP_MASTER_GTID(gtid)) {
KMP_COUNT_VALUE(OMP_loop_static_total_iterations, trip_count);
}
#endif
if (__kmp_env_consistency_check) {
/* tripcount overflow? */
if (trip_count == 0 && *pupper != *plower) {
__kmp_error_construct(kmp_i18n_msg_CnsIterationRangeTooLarge, ct_pdo,
loc);
}
}
/* compute remaining parameters */
switch (schedtype) {
case kmp_sch_static: {
if (trip_count < nth) {
KMP_DEBUG_ASSERT(
__kmp_static == kmp_sch_static_greedy ||
__kmp_static ==
kmp_sch_static_balanced); // Unknown static scheduling type.
if (tid < trip_count) {
*pupper = *plower = *plower + tid * incr;
} else {
// set bounds so non-active threads execute no iterations
*plower = *pupper + (incr > 0 ? 1 : -1);
}
if (plastiter != NULL)
*plastiter = (tid == trip_count - 1);
} else {
if (__kmp_static == kmp_sch_static_balanced) {
UT small_chunk = trip_count / nth;
UT extras = trip_count % nth;
*plower += incr * (tid * small_chunk + (tid < extras ? tid : extras));
*pupper = *plower + small_chunk * incr - (tid < extras ? 0 : incr);
if (plastiter != NULL)
*plastiter = (tid == nth - 1);
} else {
T big_chunk_inc_count =
(trip_count / nth + ((trip_count % nth) ? 1 : 0)) * incr;
T old_upper = *pupper;
KMP_DEBUG_ASSERT(__kmp_static == kmp_sch_static_greedy);
// Unknown static scheduling type.
*plower += tid * big_chunk_inc_count;
*pupper = *plower + big_chunk_inc_count - incr;
if (incr > 0) {
if (*pupper < *plower)
*pupper = traits_t<T>::max_value;
if (plastiter != NULL)
*plastiter = *plower <= old_upper && *pupper > old_upper - incr;
if (*pupper > old_upper)
*pupper = old_upper; // tracker C73258
} else {
if (*pupper > *plower)
*pupper = traits_t<T>::min_value;
if (plastiter != NULL)
*plastiter = *plower >= old_upper && *pupper < old_upper - incr;
if (*pupper < old_upper)
*pupper = old_upper; // tracker C73258
}
}
}
*pstride = trip_count;
break;
}
case kmp_sch_static_chunked: {
ST span;
UT nchunks;
if (chunk < 1)
chunk = 1;
else if ((UT)chunk > trip_count)
chunk = trip_count;
nchunks = (trip_count) / (UT)chunk + (trip_count % (UT)chunk ? 1 : 0);
span = chunk * incr;
if (nchunks < nth) {
*pstride = span * nchunks;
if (tid < nchunks) {
*plower = *plower + (span * tid);
*pupper = *plower + span - incr;
} else {
*plower = *pupper + (incr > 0 ? 1 : -1);
}
} else {
*pstride = span * nth;
*plower = *plower + (span * tid);
*pupper = *plower + span - incr;
}
if (plastiter != NULL)
*plastiter = (tid == (nchunks - 1) % nth);
break;
}
case kmp_sch_static_balanced_chunked: {
T old_upper = *pupper;
// round up to make sure the chunk is enough to cover all iterations
UT span = (trip_count + nth - 1) / nth;
// perform chunk adjustment
chunk = (span + chunk - 1) & ~(chunk - 1);
span = chunk * incr;
*plower = *plower + (span * tid);
*pupper = *plower + span - incr;
if (incr > 0) {
if (*pupper > old_upper)
*pupper = old_upper;
} else if (*pupper < old_upper)
*pupper = old_upper;
if (plastiter != NULL)
*plastiter = (tid == ((trip_count - 1) / (UT)chunk));
break;
}
default:
KMP_ASSERT2(0, "__kmpc_for_static_init: unknown scheduling type");
break;
}
#if USE_ITT_BUILD
// Report loop metadata
if (KMP_MASTER_TID(tid) && __itt_metadata_add_ptr &&
__kmp_forkjoin_frames_mode == 3 && th->th.th_teams_microtask == NULL &&
team->t.t_active_level == 1) {
kmp_uint64 cur_chunk = chunk;
check_loc(loc);
// Calculate chunk in case it was not specified; it is specified for
// kmp_sch_static_chunked
if (schedtype == kmp_sch_static) {
cur_chunk = trip_count / nth + ((trip_count % nth) ? 1 : 0);
}
// 0 - "static" schedule
__kmp_itt_metadata_loop(loc, 0, trip_count, cur_chunk);
}
#endif
#ifdef KMP_DEBUG
{
char *buff;
// create format specifiers before the debug output
buff = __kmp_str_format("__kmpc_for_static_init: liter=%%d lower=%%%s "
"upper=%%%s stride = %%%s signed?<%s>\n",
traits_t<T>::spec, traits_t<T>::spec,
traits_t<ST>::spec, traits_t<T>::spec);
KD_TRACE(100, (buff, *plastiter, *plower, *pupper, *pstride));
__kmp_str_free(&buff);
}
#endif
KE_TRACE(10, ("__kmpc_for_static_init: T#%d return\n", global_tid));
#if OMPT_SUPPORT && OMPT_OPTIONAL
if (ompt_enabled.ompt_callback_work) {
ompt_callbacks.ompt_callback(ompt_callback_work)(
ompt_work_type, ompt_scope_begin, &(team_info->parallel_data),
&(task_info->task_data), trip_count, codeptr);
}
if (ompt_enabled.ompt_callback_dispatch) {
ompt_dispatch_t dispatch_type;
ompt_data_t instance = ompt_data_none;
ompt_dispatch_chunk_t dispatch_chunk;
if (ompt_work_type == ompt_work_sections) {
dispatch_type = ompt_dispatch_section;
instance.ptr = codeptr;
} else {
OMPT_GET_DISPATCH_CHUNK(dispatch_chunk, *plower, *pupper, incr);
dispatch_type = (ompt_work_type == ompt_work_distribute)
? ompt_dispatch_distribute_chunk
: ompt_dispatch_ws_loop_chunk;
instance.ptr = &dispatch_chunk;
}
ompt_callbacks.ompt_callback(ompt_callback_dispatch)(
&(team_info->parallel_data), &(task_info->task_data), dispatch_type,
instance);
}
#endif
KMP_STATS_LOOP_END(OMP_loop_static_iterations);
return;
}
template <typename T>
static void __kmp_dist_for_static_init(ident_t *loc, kmp_int32 gtid,
kmp_int32 schedule, kmp_int32 *plastiter,
T *plower, T *pupper, T *pupperDist,
typename traits_t<T>::signed_t *pstride,
typename traits_t<T>::signed_t incr,
typename traits_t<T>::signed_t chunk
#if OMPT_SUPPORT && OMPT_OPTIONAL
,
void *codeptr
#endif
) {
KMP_COUNT_BLOCK(OMP_DISTRIBUTE);
KMP_PUSH_PARTITIONED_TIMER(OMP_distribute);
KMP_PUSH_PARTITIONED_TIMER(OMP_distribute_scheduling);
typedef typename traits_t<T>::unsigned_t UT;
typedef typename traits_t<T>::signed_t ST;
kmp_uint32 tid;
kmp_uint32 nth;
kmp_uint32 team_id;
kmp_uint32 nteams;
UT trip_count;
kmp_team_t *team;
kmp_info_t *th;
KMP_DEBUG_ASSERT(plastiter && plower && pupper && pupperDist && pstride);
KE_TRACE(10, ("__kmpc_dist_for_static_init called (%d)\n", gtid));
__kmp_assert_valid_gtid(gtid);
#ifdef KMP_DEBUG
{
char *buff;
// create format specifiers before the debug output
buff = __kmp_str_format(
"__kmpc_dist_for_static_init: T#%%d schedLoop=%%d liter=%%d "
"iter=(%%%s, %%%s, %%%s) chunk=%%%s signed?<%s>\n",
traits_t<T>::spec, traits_t<T>::spec, traits_t<ST>::spec,
traits_t<ST>::spec, traits_t<T>::spec);
KD_TRACE(100,
(buff, gtid, schedule, *plastiter, *plower, *pupper, incr, chunk));
__kmp_str_free(&buff);
}
#endif
if (__kmp_env_consistency_check) {
__kmp_push_workshare(gtid, ct_pdo, loc);
if (incr == 0) {
__kmp_error_construct(kmp_i18n_msg_CnsLoopIncrZeroProhibited, ct_pdo,
loc);
}
if (incr > 0 ? (*pupper < *plower) : (*plower < *pupper)) {
// The loop is illegal.
// Some zero-trip loops maintained by compiler, e.g.:
// for(i=10;i<0;++i) // lower >= upper - run-time check
// for(i=0;i>10;--i) // lower <= upper - run-time check
// for(i=0;i>10;++i) // incr > 0 - compile-time check
// for(i=10;i<0;--i) // incr < 0 - compile-time check
// Compiler does not check the following illegal loops:
// for(i=0;i<10;i+=incr) // where incr<0
// for(i=10;i>0;i-=incr) // where incr<0
__kmp_error_construct(kmp_i18n_msg_CnsLoopIncrIllegal, ct_pdo, loc);
}
}
tid = __kmp_tid_from_gtid(gtid);
th = __kmp_threads[gtid];
nth = th->th.th_team_nproc;
team = th->th.th_team;
KMP_DEBUG_ASSERT(th->th.th_teams_microtask); // we are in the teams construct
nteams = th->th.th_teams_size.nteams;
team_id = team->t.t_master_tid;
KMP_DEBUG_ASSERT(nteams == (kmp_uint32)team->t.t_parent->t.t_nproc);
// compute global trip count
if (incr == 1) {
trip_count = *pupper - *plower + 1;
} else if (incr == -1) {
trip_count = *plower - *pupper + 1;
} else if (incr > 0) {
// upper-lower can exceed the limit of signed type
trip_count = (UT)(*pupper - *plower) / incr + 1;
} else {
trip_count = (UT)(*plower - *pupper) / (-incr) + 1;
}
*pstride = *pupper - *plower; // just in case (can be unused)
if (trip_count <= nteams) {
KMP_DEBUG_ASSERT(
__kmp_static == kmp_sch_static_greedy ||
__kmp_static ==
kmp_sch_static_balanced); // Unknown static scheduling type.
// only primary threads of some teams get single iteration, other threads
// get nothing
if (team_id < trip_count && tid == 0) {
*pupper = *pupperDist = *plower = *plower + team_id * incr;
} else {
*pupperDist = *pupper;
*plower = *pupper + incr; // compiler should skip loop body
}
if (plastiter != NULL)
*plastiter = (tid == 0 && team_id == trip_count - 1);
} else {
// Get the team's chunk first (each team gets at most one chunk)
if (__kmp_static == kmp_sch_static_balanced) {
UT chunkD = trip_count / nteams;
UT extras = trip_count % nteams;
*plower +=
incr * (team_id * chunkD + (team_id < extras ? team_id : extras));
*pupperDist = *plower + chunkD * incr - (team_id < extras ? 0 : incr);
if (plastiter != NULL)
*plastiter = (team_id == nteams - 1);
} else {
T chunk_inc_count =
(trip_count / nteams + ((trip_count % nteams) ? 1 : 0)) * incr;
T upper = *pupper;
KMP_DEBUG_ASSERT(__kmp_static == kmp_sch_static_greedy);
// Unknown static scheduling type.
*plower += team_id * chunk_inc_count;
*pupperDist = *plower + chunk_inc_count - incr;
// Check/correct bounds if needed
if (incr > 0) {
if (*pupperDist < *plower)
*pupperDist = traits_t<T>::max_value;
if (plastiter != NULL)
*plastiter = *plower <= upper && *pupperDist > upper - incr;
if (*pupperDist > upper)
*pupperDist = upper; // tracker C73258
if (*plower > *pupperDist) {
*pupper = *pupperDist; // no iterations available for the team
goto end;
}
} else {
if (*pupperDist > *plower)
*pupperDist = traits_t<T>::min_value;
if (plastiter != NULL)
*plastiter = *plower >= upper && *pupperDist < upper - incr;
if (*pupperDist < upper)
*pupperDist = upper; // tracker C73258
if (*plower < *pupperDist) {
*pupper = *pupperDist; // no iterations available for the team
goto end;
}
}
}
// Get the parallel loop chunk now (for thread)
// compute trip count for team's chunk
if (incr == 1) {
trip_count = *pupperDist - *plower + 1;
} else if (incr == -1) {
trip_count = *plower - *pupperDist + 1;
} else if (incr > 1) {
// upper-lower can exceed the limit of signed type
trip_count = (UT)(*pupperDist - *plower) / incr + 1;
} else {
trip_count = (UT)(*plower - *pupperDist) / (-incr) + 1;
}
KMP_DEBUG_ASSERT(trip_count);
switch (schedule) {
case kmp_sch_static: {
if (trip_count <= nth) {
KMP_DEBUG_ASSERT(
__kmp_static == kmp_sch_static_greedy ||
__kmp_static ==
kmp_sch_static_balanced); // Unknown static scheduling type.
if (tid < trip_count)
*pupper = *plower = *plower + tid * incr;
else
*plower = *pupper + incr; // no iterations available
if (plastiter != NULL)
if (*plastiter != 0 && !(tid == trip_count - 1))
*plastiter = 0;
} else {
if (__kmp_static == kmp_sch_static_balanced) {
UT chunkL = trip_count / nth;
UT extras = trip_count % nth;
*plower += incr * (tid * chunkL + (tid < extras ? tid : extras));
*pupper = *plower + chunkL * incr - (tid < extras ? 0 : incr);
if (plastiter != NULL)
if (*plastiter != 0 && !(tid == nth - 1))
*plastiter = 0;
} else {
T chunk_inc_count =
(trip_count / nth + ((trip_count % nth) ? 1 : 0)) * incr;
T upper = *pupperDist;
KMP_DEBUG_ASSERT(__kmp_static == kmp_sch_static_greedy);
// Unknown static scheduling type.
*plower += tid * chunk_inc_count;
*pupper = *plower + chunk_inc_count - incr;
if (incr > 0) {
if (*pupper < *plower)
*pupper = traits_t<T>::max_value;
if (plastiter != NULL)
if (*plastiter != 0 &&
!(*plower <= upper && *pupper > upper - incr))
*plastiter = 0;
if (*pupper > upper)
*pupper = upper; // tracker C73258
} else {
if (*pupper > *plower)
*pupper = traits_t<T>::min_value;
if (plastiter != NULL)
if (*plastiter != 0 &&
!(*plower >= upper && *pupper < upper - incr))
*plastiter = 0;
if (*pupper < upper)
*pupper = upper; // tracker C73258
}
}
}
break;
}
case kmp_sch_static_chunked: {
ST span;
if (chunk < 1)
chunk = 1;
span = chunk * incr;
*pstride = span * nth;
*plower = *plower + (span * tid);
*pupper = *plower + span - incr;
if (plastiter != NULL)
if (*plastiter != 0 && !(tid == ((trip_count - 1) / (UT)chunk) % nth))
*plastiter = 0;
break;
}
default:
KMP_ASSERT2(0,
"__kmpc_dist_for_static_init: unknown loop scheduling type");
break;
}
}
end:;
#ifdef KMP_DEBUG
{
char *buff;
// create format specifiers before the debug output
buff = __kmp_str_format(
"__kmpc_dist_for_static_init: last=%%d lo=%%%s up=%%%s upDist=%%%s "
"stride=%%%s signed?<%s>\n",
traits_t<T>::spec, traits_t<T>::spec, traits_t<T>::spec,
traits_t<ST>::spec, traits_t<T>::spec);
KD_TRACE(100, (buff, *plastiter, *plower, *pupper, *pupperDist, *pstride));
__kmp_str_free(&buff);
}
#endif
KE_TRACE(10, ("__kmpc_dist_for_static_init: T#%d return\n", gtid));
#if OMPT_SUPPORT && OMPT_OPTIONAL
if (ompt_enabled.ompt_callback_work || ompt_enabled.ompt_callback_dispatch) {
ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL);
ompt_task_info_t *task_info = __ompt_get_task_info_object(0);
if (ompt_enabled.ompt_callback_work) {
ompt_callbacks.ompt_callback(ompt_callback_work)(
ompt_work_distribute, ompt_scope_begin, &(team_info->parallel_data),
&(task_info->task_data), 0, codeptr);
}
if (ompt_enabled.ompt_callback_dispatch) {
ompt_data_t instance = ompt_data_none;
ompt_dispatch_chunk_t dispatch_chunk;
OMPT_GET_DISPATCH_CHUNK(dispatch_chunk, *plower, *pupperDist, incr);
instance.ptr = &dispatch_chunk;
ompt_callbacks.ompt_callback(ompt_callback_dispatch)(
&(team_info->parallel_data), &(task_info->task_data),
ompt_dispatch_distribute_chunk, instance);
}
}
#endif // OMPT_SUPPORT && OMPT_OPTIONAL
KMP_STATS_LOOP_END(OMP_distribute_iterations);
return;
}
template <typename T>
static void __kmp_team_static_init(ident_t *loc, kmp_int32 gtid,
kmp_int32 *p_last, T *p_lb, T *p_ub,
typename traits_t<T>::signed_t *p_st,
typename traits_t<T>::signed_t incr,
typename traits_t<T>::signed_t chunk) {
// The routine returns the first chunk distributed to the team and
// stride for next chunks calculation.
// Last iteration flag set for the team that will execute
// the last iteration of the loop.
// The routine is called for dist_schedule(static,chunk) only.
typedef typename traits_t<T>::unsigned_t UT;
typedef typename traits_t<T>::signed_t ST;
kmp_uint32 team_id;
kmp_uint32 nteams;
UT trip_count;
T lower;
T upper;
ST span;
kmp_team_t *team;
kmp_info_t *th;
KMP_DEBUG_ASSERT(p_last && p_lb && p_ub && p_st);
KE_TRACE(10, ("__kmp_team_static_init called (%d)\n", gtid));
__kmp_assert_valid_gtid(gtid);
#ifdef KMP_DEBUG
{
char *buff;
// create format specifiers before the debug output
buff = __kmp_str_format("__kmp_team_static_init enter: T#%%d liter=%%d "
"iter=(%%%s, %%%s, %%%s) chunk %%%s; signed?<%s>\n",
traits_t<T>::spec, traits_t<T>::spec,
traits_t<ST>::spec, traits_t<ST>::spec,
traits_t<T>::spec);
KD_TRACE(100, (buff, gtid, *p_last, *p_lb, *p_ub, *p_st, chunk));
__kmp_str_free(&buff);
}
#endif
lower = *p_lb;
upper = *p_ub;
if (__kmp_env_consistency_check) {
if (incr == 0) {
__kmp_error_construct(kmp_i18n_msg_CnsLoopIncrZeroProhibited, ct_pdo,
loc);
}
if (incr > 0 ? (upper < lower) : (lower < upper)) {
// The loop is illegal.
// Some zero-trip loops maintained by compiler, e.g.:
// for(i=10;i<0;++i) // lower >= upper - run-time check
// for(i=0;i>10;--i) // lower <= upper - run-time check
// for(i=0;i>10;++i) // incr > 0 - compile-time check
// for(i=10;i<0;--i) // incr < 0 - compile-time check
// Compiler does not check the following illegal loops:
// for(i=0;i<10;i+=incr) // where incr<0
// for(i=10;i>0;i-=incr) // where incr<0
__kmp_error_construct(kmp_i18n_msg_CnsLoopIncrIllegal, ct_pdo, loc);
}
}
th = __kmp_threads[gtid];
team = th->th.th_team;
KMP_DEBUG_ASSERT(th->th.th_teams_microtask); // we are in the teams construct
nteams = th->th.th_teams_size.nteams;
team_id = team->t.t_master_tid;
KMP_DEBUG_ASSERT(nteams == (kmp_uint32)team->t.t_parent->t.t_nproc);
// compute trip count
if (incr == 1) {
trip_count = upper - lower + 1;
} else if (incr == -1) {
trip_count = lower - upper + 1;
} else if (incr > 0) {
// upper-lower can exceed the limit of signed type
trip_count = (UT)(upper - lower) / incr + 1;
} else {
trip_count = (UT)(lower - upper) / (-incr) + 1;
}
if (chunk < 1)
chunk = 1;
span = chunk * incr;
*p_st = span * nteams;
*p_lb = lower + (span * team_id);
*p_ub = *p_lb + span - incr;
if (p_last != NULL)
*p_last = (team_id == ((trip_count - 1) / (UT)chunk) % nteams);
// Correct upper bound if needed
if (incr > 0) {
if (*p_ub < *p_lb) // overflow?
*p_ub = traits_t<T>::max_value;
if (*p_ub > upper)
*p_ub = upper; // tracker C73258
} else { // incr < 0
if (*p_ub > *p_lb)
*p_ub = traits_t<T>::min_value;
if (*p_ub < upper)
*p_ub = upper; // tracker C73258
}
#ifdef KMP_DEBUG
{
char *buff;
// create format specifiers before the debug output
buff =
__kmp_str_format("__kmp_team_static_init exit: T#%%d team%%u liter=%%d "
"iter=(%%%s, %%%s, %%%s) chunk %%%s\n",
traits_t<T>::spec, traits_t<T>::spec,
traits_t<ST>::spec, traits_t<ST>::spec);
KD_TRACE(100, (buff, gtid, team_id, *p_last, *p_lb, *p_ub, *p_st, chunk));
__kmp_str_free(&buff);
}
#endif
}
//------------------------------------------------------------------------------
extern "C" {
/*!
@ingroup WORK_SHARING
@param loc Source code location
@param gtid Global thread id of this thread
@param schedtype Scheduling type
@param plastiter Pointer to the "last iteration" flag
@param plower Pointer to the lower bound
@param pupper Pointer to the upper bound
@param pstride Pointer to the stride
@param incr Loop increment
@param chunk The chunk size
Each of the four functions here are identical apart from the argument types.
The functions compute the upper and lower bounds and stride to be used for the
set of iterations to be executed by the current thread from the statically
scheduled loop that is described by the initial values of the bounds, stride,
increment and chunk size.
@{
*/
void __kmpc_for_static_init_4(ident_t *loc, kmp_int32 gtid, kmp_int32 schedtype,
kmp_int32 *plastiter, kmp_int32 *plower,
kmp_int32 *pupper, kmp_int32 *pstride,
kmp_int32 incr, kmp_int32 chunk) {
__kmp_for_static_init<kmp_int32>(loc, gtid, schedtype, plastiter, plower,
pupper, pstride, incr, chunk
#if OMPT_SUPPORT && OMPT_OPTIONAL
,
OMPT_GET_RETURN_ADDRESS(0)
#endif
);
}
/*!
See @ref __kmpc_for_static_init_4
*/
void __kmpc_for_static_init_4u(ident_t *loc, kmp_int32 gtid,
kmp_int32 schedtype, kmp_int32 *plastiter,
kmp_uint32 *plower, kmp_uint32 *pupper,
kmp_int32 *pstride, kmp_int32 incr,
kmp_int32 chunk) {
__kmp_for_static_init<kmp_uint32>(loc, gtid, schedtype, plastiter, plower,
pupper, pstride, incr, chunk
#if OMPT_SUPPORT && OMPT_OPTIONAL
,
OMPT_GET_RETURN_ADDRESS(0)
#endif
);
}
/*!
See @ref __kmpc_for_static_init_4
*/
void __kmpc_for_static_init_8(ident_t *loc, kmp_int32 gtid, kmp_int32 schedtype,
kmp_int32 *plastiter, kmp_int64 *plower,
kmp_int64 *pupper, kmp_int64 *pstride,
kmp_int64 incr, kmp_int64 chunk) {
__kmp_for_static_init<kmp_int64>(loc, gtid, schedtype, plastiter, plower,
pupper, pstride, incr, chunk
#if OMPT_SUPPORT && OMPT_OPTIONAL
,
OMPT_GET_RETURN_ADDRESS(0)
#endif
);
}
/*!
See @ref __kmpc_for_static_init_4
*/
void __kmpc_for_static_init_8u(ident_t *loc, kmp_int32 gtid,
kmp_int32 schedtype, kmp_int32 *plastiter,
kmp_uint64 *plower, kmp_uint64 *pupper,
kmp_int64 *pstride, kmp_int64 incr,
kmp_int64 chunk) {
__kmp_for_static_init<kmp_uint64>(loc, gtid, schedtype, plastiter, plower,
pupper, pstride, incr, chunk
#if OMPT_SUPPORT && OMPT_OPTIONAL
,
OMPT_GET_RETURN_ADDRESS(0)
#endif
);
}
/*!
@}
*/
#if OMPT_SUPPORT && OMPT_OPTIONAL
#define OMPT_CODEPTR_ARG , OMPT_GET_RETURN_ADDRESS(0)
#else
#define OMPT_CODEPTR_ARG
#endif
/*!
@ingroup WORK_SHARING
@param loc Source code location
@param gtid Global thread id of this thread
@param schedule Scheduling type for the parallel loop
@param plastiter Pointer to the "last iteration" flag
@param plower Pointer to the lower bound
@param pupper Pointer to the upper bound of loop chunk
@param pupperD Pointer to the upper bound of dist_chunk
@param pstride Pointer to the stride for parallel loop
@param incr Loop increment
@param chunk The chunk size for the parallel loop
Each of the four functions here are identical apart from the argument types.
The functions compute the upper and lower bounds and strides to be used for the
set of iterations to be executed by the current thread from the statically
scheduled loop that is described by the initial values of the bounds, strides,
increment and chunks for parallel loop and distribute constructs.
@{
*/
void __kmpc_dist_for_static_init_4(ident_t *loc, kmp_int32 gtid,
kmp_int32 schedule, kmp_int32 *plastiter,
kmp_int32 *plower, kmp_int32 *pupper,
kmp_int32 *pupperD, kmp_int32 *pstride,
kmp_int32 incr, kmp_int32 chunk) {
__kmp_dist_for_static_init<kmp_int32>(loc, gtid, schedule, plastiter, plower,
pupper, pupperD, pstride, incr,
chunk OMPT_CODEPTR_ARG);
}
/*!
See @ref __kmpc_dist_for_static_init_4
*/
void __kmpc_dist_for_static_init_4u(ident_t *loc, kmp_int32 gtid,
kmp_int32 schedule, kmp_int32 *plastiter,
kmp_uint32 *plower, kmp_uint32 *pupper,
kmp_uint32 *pupperD, kmp_int32 *pstride,
kmp_int32 incr, kmp_int32 chunk) {
__kmp_dist_for_static_init<kmp_uint32>(loc, gtid, schedule, plastiter, plower,
pupper, pupperD, pstride, incr,
chunk OMPT_CODEPTR_ARG);
}
/*!
See @ref __kmpc_dist_for_static_init_4
*/
void __kmpc_dist_for_static_init_8(ident_t *loc, kmp_int32 gtid,
kmp_int32 schedule, kmp_int32 *plastiter,
kmp_int64 *plower, kmp_int64 *pupper,
kmp_int64 *pupperD, kmp_int64 *pstride,
kmp_int64 incr, kmp_int64 chunk) {
__kmp_dist_for_static_init<kmp_int64>(loc, gtid, schedule, plastiter, plower,
pupper, pupperD, pstride, incr,
chunk OMPT_CODEPTR_ARG);
}
/*!
See @ref __kmpc_dist_for_static_init_4
*/
void __kmpc_dist_for_static_init_8u(ident_t *loc, kmp_int32 gtid,
kmp_int32 schedule, kmp_int32 *plastiter,
kmp_uint64 *plower, kmp_uint64 *pupper,
kmp_uint64 *pupperD, kmp_int64 *pstride,
kmp_int64 incr, kmp_int64 chunk) {
__kmp_dist_for_static_init<kmp_uint64>(loc, gtid, schedule, plastiter, plower,
pupper, pupperD, pstride, incr,
chunk OMPT_CODEPTR_ARG);
}
/*!
@}
*/
//------------------------------------------------------------------------------
// Auxiliary routines for Distribute Parallel Loop construct implementation
// Transfer call to template< type T >
// __kmp_team_static_init( ident_t *loc, int gtid,
// int *p_last, T *lb, T *ub, ST *st, ST incr, ST chunk )
/*!
@ingroup WORK_SHARING
@{
@param loc Source location
@param gtid Global thread id
@param p_last pointer to last iteration flag
@param p_lb pointer to Lower bound
@param p_ub pointer to Upper bound
@param p_st Step (or increment if you prefer)
@param incr Loop increment
@param chunk The chunk size to block with
The functions compute the upper and lower bounds and stride to be used for the
set of iterations to be executed by the current team from the statically
scheduled loop that is described by the initial values of the bounds, stride,
increment and chunk for the distribute construct as part of composite distribute
parallel loop construct. These functions are all identical apart from the types
of the arguments.
*/
void __kmpc_team_static_init_4(ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last,
kmp_int32 *p_lb, kmp_int32 *p_ub,
kmp_int32 *p_st, kmp_int32 incr,
kmp_int32 chunk) {
KMP_DEBUG_ASSERT(__kmp_init_serial);
__kmp_team_static_init<kmp_int32>(loc, gtid, p_last, p_lb, p_ub, p_st, incr,
chunk);
}
/*!
See @ref __kmpc_team_static_init_4
*/
void __kmpc_team_static_init_4u(ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last,
kmp_uint32 *p_lb, kmp_uint32 *p_ub,
kmp_int32 *p_st, kmp_int32 incr,
kmp_int32 chunk) {
KMP_DEBUG_ASSERT(__kmp_init_serial);
__kmp_team_static_init<kmp_uint32>(loc, gtid, p_last, p_lb, p_ub, p_st, incr,
chunk);
}
/*!
See @ref __kmpc_team_static_init_4
*/
void __kmpc_team_static_init_8(ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last,
kmp_int64 *p_lb, kmp_int64 *p_ub,
kmp_int64 *p_st, kmp_int64 incr,
kmp_int64 chunk) {
KMP_DEBUG_ASSERT(__kmp_init_serial);
__kmp_team_static_init<kmp_int64>(loc, gtid, p_last, p_lb, p_ub, p_st, incr,
chunk);
}
/*!
See @ref __kmpc_team_static_init_4
*/
void __kmpc_team_static_init_8u(ident_t *loc, kmp_int32 gtid, kmp_int32 *p_last,
kmp_uint64 *p_lb, kmp_uint64 *p_ub,
kmp_int64 *p_st, kmp_int64 incr,
kmp_int64 chunk) {
KMP_DEBUG_ASSERT(__kmp_init_serial);
__kmp_team_static_init<kmp_uint64>(loc, gtid, p_last, p_lb, p_ub, p_st, incr,
chunk);
}
/*!
@}
*/
} // extern "C"
Reference in a new issue View git blame Copy permalink