mirrors/cosmopolitan
1
0
Fork 0
mirror of https://github.com/jart/cosmopolitan.git synced 2025-01-31 03:27:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions
cosmopolitan/third_party/openmp/kmp_threadprivate.cpp
Justine Tunney 5f8e9f14c1
Add OpenMP support
2024-01-28 22:39:02 -08:00

798 lines
26 KiB
C++
Raw Blame History

/*
* kmp_threadprivate.cpp -- OpenMP threadprivate support library
*/
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "kmp.h"
#include "kmp_i18n.h"
#include "kmp_itt.h"
#define USE_CHECKS_COMMON
#define KMP_INLINE_SUBR 1
void kmp_threadprivate_insert_private_data(int gtid, void *pc_addr,
void *data_addr, size_t pc_size);
struct private_common *kmp_threadprivate_insert(int gtid, void *pc_addr,
void *data_addr,
size_t pc_size);
struct shared_table __kmp_threadprivate_d_table;
static
#ifdef KMP_INLINE_SUBR
__forceinline
#endif
struct private_common *
__kmp_threadprivate_find_task_common(struct common_table *tbl, int gtid,
void *pc_addr)
{
struct private_common *tn;
#ifdef KMP_TASK_COMMON_DEBUG
KC_TRACE(10, ("__kmp_threadprivate_find_task_common: thread#%d, called with "
"address %p\n",
gtid, pc_addr));
dump_list();
#endif
for (tn = tbl->data[KMP_HASH(pc_addr)]; tn; tn = tn->next) {
if (tn->gbl_addr == pc_addr) {
#ifdef KMP_TASK_COMMON_DEBUG
KC_TRACE(10, ("__kmp_threadprivate_find_task_common: thread#%d, found "
"node %p on list\n",
gtid, pc_addr));
#endif
return tn;
}
}
return 0;
}
static
#ifdef KMP_INLINE_SUBR
__forceinline
#endif
struct shared_common *
__kmp_find_shared_task_common(struct shared_table *tbl, int gtid,
void *pc_addr) {
struct shared_common *tn;
for (tn = tbl->data[KMP_HASH(pc_addr)]; tn; tn = tn->next) {
if (tn->gbl_addr == pc_addr) {
#ifdef KMP_TASK_COMMON_DEBUG
KC_TRACE(
10,
("__kmp_find_shared_task_common: thread#%d, found node %p on list\n",
gtid, pc_addr));
#endif
return tn;
}
}
return 0;
}
// Create a template for the data initialized storage. Either the template is
// NULL indicating zero fill, or the template is a copy of the original data.
static struct private_data *__kmp_init_common_data(void *pc_addr,
size_t pc_size) {
struct private_data *d;
size_t i;
char *p;
d = (struct private_data *)__kmp_allocate(sizeof(struct private_data));
/*
d->data = 0; // AC: commented out because __kmp_allocate zeroes the
memory
d->next = 0;
*/
d->size = pc_size;
d->more = 1;
p = (char *)pc_addr;
for (i = pc_size; i > 0; --i) {
if (*p++ != '\0') {
d->data = __kmp_allocate(pc_size);
KMP_MEMCPY(d->data, pc_addr, pc_size);
break;
}
}
return d;
}
// Initialize the data area from the template.
static void __kmp_copy_common_data(void *pc_addr, struct private_data *d) {
char *addr = (char *)pc_addr;
for (size_t offset = 0; d != 0; d = d->next) {
for (int i = d->more; i > 0; --i) {
if (d->data == 0)
memset(&addr[offset], '\0', d->size);
else
KMP_MEMCPY(&addr[offset], d->data, d->size);
offset += d->size;
}
}
}
/* we are called from __kmp_serial_initialize() with __kmp_initz_lock held. */
void __kmp_common_initialize(void) {
if (!TCR_4(__kmp_init_common)) {
int q;
#ifdef KMP_DEBUG
int gtid;
#endif
__kmp_threadpriv_cache_list = NULL;
#ifdef KMP_DEBUG
/* verify the uber masters were initialized */
for (gtid = 0; gtid < __kmp_threads_capacity; gtid++)
if (__kmp_root[gtid]) {
KMP_DEBUG_ASSERT(__kmp_root[gtid]->r.r_uber_thread);
for (q = 0; q < KMP_HASH_TABLE_SIZE; ++q)
KMP_DEBUG_ASSERT(
!__kmp_root[gtid]->r.r_uber_thread->th.th_pri_common->data[q]);
/* __kmp_root[ gitd ]-> r.r_uber_thread ->
* th.th_pri_common -> data[ q ] = 0;*/
}
#endif /* KMP_DEBUG */
for (q = 0; q < KMP_HASH_TABLE_SIZE; ++q)
__kmp_threadprivate_d_table.data[q] = 0;
TCW_4(__kmp_init_common, TRUE);
}
}
/* Call all destructors for threadprivate data belonging to all threads.
Currently unused! */
void __kmp_common_destroy(void) {
if (TCR_4(__kmp_init_common)) {
int q;
TCW_4(__kmp_init_common, FALSE);
for (q = 0; q < KMP_HASH_TABLE_SIZE; ++q) {
int gtid;
struct private_common *tn;
struct shared_common *d_tn;
/* C++ destructors need to be called once per thread before exiting.
Don't call destructors for primary thread though unless we used copy
constructor */
for (d_tn = __kmp_threadprivate_d_table.data[q]; d_tn;
d_tn = d_tn->next) {
if (d_tn->is_vec) {
if (d_tn->dt.dtorv != 0) {
for (gtid = 0; gtid < __kmp_all_nth; ++gtid) {
if (__kmp_threads[gtid]) {
if ((__kmp_foreign_tp) ? (!KMP_INITIAL_GTID(gtid))
: (!KMP_UBER_GTID(gtid))) {
tn = __kmp_threadprivate_find_task_common(
__kmp_threads[gtid]->th.th_pri_common, gtid,
d_tn->gbl_addr);
if (tn) {
(*d_tn->dt.dtorv)(tn->par_addr, d_tn->vec_len);
}
}
}
}
if (d_tn->obj_init != 0) {
(*d_tn->dt.dtorv)(d_tn->obj_init, d_tn->vec_len);
}
}
} else {
if (d_tn->dt.dtor != 0) {
for (gtid = 0; gtid < __kmp_all_nth; ++gtid) {
if (__kmp_threads[gtid]) {
if ((__kmp_foreign_tp) ? (!KMP_INITIAL_GTID(gtid))
: (!KMP_UBER_GTID(gtid))) {
tn = __kmp_threadprivate_find_task_common(
__kmp_threads[gtid]->th.th_pri_common, gtid,
d_tn->gbl_addr);
if (tn) {
(*d_tn->dt.dtor)(tn->par_addr);
}
}
}
}
if (d_tn->obj_init != 0) {
(*d_tn->dt.dtor)(d_tn->obj_init);
}
}
}
}
__kmp_threadprivate_d_table.data[q] = 0;
}
}
}
/* Call all destructors for threadprivate data belonging to this thread */
void __kmp_common_destroy_gtid(int gtid) {
struct private_common *tn;
struct shared_common *d_tn;
if (!TCR_4(__kmp_init_gtid)) {
// This is possible when one of multiple roots initiates early library
// termination in a sequential region while other teams are active, and its
// child threads are about to end.
return;
}
KC_TRACE(10, ("__kmp_common_destroy_gtid: T#%d called\n", gtid));
if ((__kmp_foreign_tp) ? (!KMP_INITIAL_GTID(gtid)) : (!KMP_UBER_GTID(gtid))) {
if (TCR_4(__kmp_init_common)) {
/* Cannot do this here since not all threads have destroyed their data */
/* TCW_4(__kmp_init_common, FALSE); */
for (tn = __kmp_threads[gtid]->th.th_pri_head; tn; tn = tn->link) {
d_tn = __kmp_find_shared_task_common(&__kmp_threadprivate_d_table, gtid,
tn->gbl_addr);
if (d_tn == NULL)
continue;
if (d_tn->is_vec) {
if (d_tn->dt.dtorv != 0) {
(void)(*d_tn->dt.dtorv)(tn->par_addr, d_tn->vec_len);
}
if (d_tn->obj_init != 0) {
(void)(*d_tn->dt.dtorv)(d_tn->obj_init, d_tn->vec_len);
}
} else {
if (d_tn->dt.dtor != 0) {
(void)(*d_tn->dt.dtor)(tn->par_addr);
}
if (d_tn->obj_init != 0) {
(void)(*d_tn->dt.dtor)(d_tn->obj_init);
}
}
}
KC_TRACE(30, ("__kmp_common_destroy_gtid: T#%d threadprivate destructors "
"complete\n",
gtid));
}
}
}
#ifdef KMP_TASK_COMMON_DEBUG
static void dump_list(void) {
int p, q;
for (p = 0; p < __kmp_all_nth; ++p) {
if (!__kmp_threads[p])
continue;
for (q = 0; q < KMP_HASH_TABLE_SIZE; ++q) {
if (__kmp_threads[p]->th.th_pri_common->data[q]) {
struct private_common *tn;
KC_TRACE(10, ("\tdump_list: gtid:%d addresses\n", p));
for (tn = __kmp_threads[p]->th.th_pri_common->data[q]; tn;
tn = tn->next) {
KC_TRACE(10,
("\tdump_list: THREADPRIVATE: Serial %p -> Parallel %p\n",
tn->gbl_addr, tn->par_addr));
}
}
}
}
}
#endif /* KMP_TASK_COMMON_DEBUG */
// NOTE: this routine is to be called only from the serial part of the program.
void kmp_threadprivate_insert_private_data(int gtid, void *pc_addr,
void *data_addr, size_t pc_size) {
struct shared_common **lnk_tn, *d_tn;
KMP_DEBUG_ASSERT(__kmp_threads[gtid] &&
__kmp_threads[gtid]->th.th_root->r.r_active == 0);
d_tn = __kmp_find_shared_task_common(&__kmp_threadprivate_d_table, gtid,
pc_addr);
if (d_tn == 0) {
d_tn = (struct shared_common *)__kmp_allocate(sizeof(struct shared_common));
d_tn->gbl_addr = pc_addr;
d_tn->pod_init = __kmp_init_common_data(data_addr, pc_size);
/*
d_tn->obj_init = 0; // AC: commented out because __kmp_allocate
zeroes the memory
d_tn->ct.ctor = 0;
d_tn->cct.cctor = 0;;
d_tn->dt.dtor = 0;
d_tn->is_vec = FALSE;
d_tn->vec_len = 0L;
*/
d_tn->cmn_size = pc_size;
__kmp_acquire_lock(&__kmp_global_lock, gtid);
lnk_tn = &(__kmp_threadprivate_d_table.data[KMP_HASH(pc_addr)]);
d_tn->next = *lnk_tn;
*lnk_tn = d_tn;
__kmp_release_lock(&__kmp_global_lock, gtid);
}
}
struct private_common *kmp_threadprivate_insert(int gtid, void *pc_addr,
void *data_addr,
size_t pc_size) {
struct private_common *tn, **tt;
struct shared_common *d_tn;
/* +++++++++ START OF CRITICAL SECTION +++++++++ */
__kmp_acquire_lock(&__kmp_global_lock, gtid);
tn = (struct private_common *)__kmp_allocate(sizeof(struct private_common));
tn->gbl_addr = pc_addr;
d_tn = __kmp_find_shared_task_common(
&__kmp_threadprivate_d_table, gtid,
pc_addr); /* Only the MASTER data table exists. */
if (d_tn != 0) {
/* This threadprivate variable has already been seen. */
if (d_tn->pod_init == 0 && d_tn->obj_init == 0) {
d_tn->cmn_size = pc_size;
if (d_tn->is_vec) {
if (d_tn->ct.ctorv != 0) {
/* Construct from scratch so no prototype exists */
d_tn->obj_init = 0;
} else if (d_tn->cct.cctorv != 0) {
/* Now data initialize the prototype since it was previously
* registered */
d_tn->obj_init = (void *)__kmp_allocate(d_tn->cmn_size);
(void)(*d_tn->cct.cctorv)(d_tn->obj_init, pc_addr, d_tn->vec_len);
} else {
d_tn->pod_init = __kmp_init_common_data(data_addr, d_tn->cmn_size);
}
} else {
if (d_tn->ct.ctor != 0) {
/* Construct from scratch so no prototype exists */
d_tn->obj_init = 0;
} else if (d_tn->cct.cctor != 0) {
/* Now data initialize the prototype since it was previously
registered */
d_tn->obj_init = (void *)__kmp_allocate(d_tn->cmn_size);
(void)(*d_tn->cct.cctor)(d_tn->obj_init, pc_addr);
} else {
d_tn->pod_init = __kmp_init_common_data(data_addr, d_tn->cmn_size);
}
}
}
} else {
struct shared_common **lnk_tn;
d_tn = (struct shared_common *)__kmp_allocate(sizeof(struct shared_common));
d_tn->gbl_addr = pc_addr;
d_tn->cmn_size = pc_size;
d_tn->pod_init = __kmp_init_common_data(data_addr, pc_size);
/*
d_tn->obj_init = 0; // AC: commented out because __kmp_allocate
zeroes the memory
d_tn->ct.ctor = 0;
d_tn->cct.cctor = 0;
d_tn->dt.dtor = 0;
d_tn->is_vec = FALSE;
d_tn->vec_len = 0L;
*/
lnk_tn = &(__kmp_threadprivate_d_table.data[KMP_HASH(pc_addr)]);
d_tn->next = *lnk_tn;
*lnk_tn = d_tn;
}
tn->cmn_size = d_tn->cmn_size;
if ((__kmp_foreign_tp) ? (KMP_INITIAL_GTID(gtid)) : (KMP_UBER_GTID(gtid))) {
tn->par_addr = (void *)pc_addr;
} else {
tn->par_addr = (void *)__kmp_allocate(tn->cmn_size);
}
__kmp_release_lock(&__kmp_global_lock, gtid);
/* +++++++++ END OF CRITICAL SECTION +++++++++ */
#ifdef USE_CHECKS_COMMON
if (pc_size > d_tn->cmn_size) {
KC_TRACE(
10, ("__kmp_threadprivate_insert: THREADPRIVATE: %p (%" KMP_UINTPTR_SPEC
" ,%" KMP_UINTPTR_SPEC ")\n",
pc_addr, pc_size, d_tn->cmn_size));
KMP_FATAL(TPCommonBlocksInconsist);
}
#endif /* USE_CHECKS_COMMON */
tt = &(__kmp_threads[gtid]->th.th_pri_common->data[KMP_HASH(pc_addr)]);
#ifdef KMP_TASK_COMMON_DEBUG
if (*tt != 0) {
KC_TRACE(
10,
("__kmp_threadprivate_insert: WARNING! thread#%d: collision on %p\n",
gtid, pc_addr));
}
#endif
tn->next = *tt;
*tt = tn;
#ifdef KMP_TASK_COMMON_DEBUG
KC_TRACE(10,
("__kmp_threadprivate_insert: thread#%d, inserted node %p on list\n",
gtid, pc_addr));
dump_list();
#endif
/* Link the node into a simple list */
tn->link = __kmp_threads[gtid]->th.th_pri_head;
__kmp_threads[gtid]->th.th_pri_head = tn;
if ((__kmp_foreign_tp) ? (KMP_INITIAL_GTID(gtid)) : (KMP_UBER_GTID(gtid)))
return tn;
/* if C++ object with copy constructor, use it;
* else if C++ object with constructor, use it for the non-primary thread
copies only;
* else use pod_init and memcpy
*
* C++ constructors need to be called once for each non-primary thread on
* allocate
* C++ copy constructors need to be called once for each thread on allocate */
/* C++ object with constructors/destructors; don't call constructors for
primary thread though */
if (d_tn->is_vec) {
if (d_tn->ct.ctorv != 0) {
(void)(*d_tn->ct.ctorv)(tn->par_addr, d_tn->vec_len);
} else if (d_tn->cct.cctorv != 0) {
(void)(*d_tn->cct.cctorv)(tn->par_addr, d_tn->obj_init, d_tn->vec_len);
} else if (tn->par_addr != tn->gbl_addr) {
__kmp_copy_common_data(tn->par_addr, d_tn->pod_init);
}
} else {
if (d_tn->ct.ctor != 0) {
(void)(*d_tn->ct.ctor)(tn->par_addr);
} else if (d_tn->cct.cctor != 0) {
(void)(*d_tn->cct.cctor)(tn->par_addr, d_tn->obj_init);
} else if (tn->par_addr != tn->gbl_addr) {
__kmp_copy_common_data(tn->par_addr, d_tn->pod_init);
}
}
/* !BUILD_OPENMP_C
if (tn->par_addr != tn->gbl_addr)
__kmp_copy_common_data( tn->par_addr, d_tn->pod_init ); */
return tn;
}
/* ------------------------------------------------------------------------ */
/* We are currently parallel, and we know the thread id. */
/* ------------------------------------------------------------------------ */
/*!
@ingroup THREADPRIVATE
@param loc source location information
@param data pointer to data being privatized
@param ctor pointer to constructor function for data
@param cctor pointer to copy constructor function for data
@param dtor pointer to destructor function for data
Register constructors and destructors for thread private data.
This function is called when executing in parallel, when we know the thread id.
*/
void __kmpc_threadprivate_register(ident_t *loc, void *data, kmpc_ctor ctor,
kmpc_cctor cctor, kmpc_dtor dtor) {
struct shared_common *d_tn, **lnk_tn;
KC_TRACE(10, ("__kmpc_threadprivate_register: called\n"));
#ifdef USE_CHECKS_COMMON
/* copy constructor must be zero for current code gen (Nov 2002 - jph) */
KMP_ASSERT(cctor == 0);
#endif /* USE_CHECKS_COMMON */
/* Only the global data table exists. */
d_tn = __kmp_find_shared_task_common(&__kmp_threadprivate_d_table, -1, data);
if (d_tn == 0) {
d_tn = (struct shared_common *)__kmp_allocate(sizeof(struct shared_common));
d_tn->gbl_addr = data;
d_tn->ct.ctor = ctor;
d_tn->cct.cctor = cctor;
d_tn->dt.dtor = dtor;
/*
d_tn->is_vec = FALSE; // AC: commented out because __kmp_allocate
zeroes the memory
d_tn->vec_len = 0L;
d_tn->obj_init = 0;
d_tn->pod_init = 0;
*/
lnk_tn = &(__kmp_threadprivate_d_table.data[KMP_HASH(data)]);
d_tn->next = *lnk_tn;
*lnk_tn = d_tn;
}
}
void *__kmpc_threadprivate(ident_t *loc, kmp_int32 global_tid, void *data,
size_t size) {
void *ret;
struct private_common *tn;
KC_TRACE(10, ("__kmpc_threadprivate: T#%d called\n", global_tid));
#ifdef USE_CHECKS_COMMON
if (!__kmp_init_serial)
KMP_FATAL(RTLNotInitialized);
#endif /* USE_CHECKS_COMMON */
if (!__kmp_threads[global_tid]->th.th_root->r.r_active && !__kmp_foreign_tp) {
/* The parallel address will NEVER overlap with the data_address */
/* dkp: 3rd arg to kmp_threadprivate_insert_private_data() is the
* data_address; use data_address = data */
KC_TRACE(20, ("__kmpc_threadprivate: T#%d inserting private data\n",
global_tid));
kmp_threadprivate_insert_private_data(global_tid, data, data, size);
ret = data;
} else {
KC_TRACE(
50,
("__kmpc_threadprivate: T#%d try to find private data at address %p\n",
global_tid, data));
tn = __kmp_threadprivate_find_task_common(
__kmp_threads[global_tid]->th.th_pri_common, global_tid, data);
if (tn) {
KC_TRACE(20, ("__kmpc_threadprivate: T#%d found data\n", global_tid));
#ifdef USE_CHECKS_COMMON
if ((size_t)size > tn->cmn_size) {
KC_TRACE(10, ("THREADPRIVATE: %p (%" KMP_UINTPTR_SPEC
" ,%" KMP_UINTPTR_SPEC ")\n",
data, size, tn->cmn_size));
KMP_FATAL(TPCommonBlocksInconsist);
}
#endif /* USE_CHECKS_COMMON */
} else {
/* The parallel address will NEVER overlap with the data_address */
/* dkp: 3rd arg to kmp_threadprivate_insert() is the data_address; use
* data_address = data */
KC_TRACE(20, ("__kmpc_threadprivate: T#%d inserting data\n", global_tid));
tn = kmp_threadprivate_insert(global_tid, data, data, size);
}
ret = tn->par_addr;
}
KC_TRACE(10, ("__kmpc_threadprivate: T#%d exiting; return value = %p\n",
global_tid, ret));
return ret;
}
static kmp_cached_addr_t *__kmp_find_cache(void *data) {
kmp_cached_addr_t *ptr = __kmp_threadpriv_cache_list;
while (ptr && ptr->data != data)
ptr = ptr->next;
return ptr;
}
/*!
@ingroup THREADPRIVATE
@param loc source location information
@param global_tid global thread number
@param data pointer to data to privatize
@param size size of data to privatize
@param cache pointer to cache
@return pointer to private storage
Allocate private storage for threadprivate data.
*/
void *
__kmpc_threadprivate_cached(ident_t *loc,
kmp_int32 global_tid, // gtid.
void *data, // Pointer to original global variable.
size_t size, // Size of original global variable.
void ***cache) {
KC_TRACE(10, ("__kmpc_threadprivate_cached: T#%d called with cache: %p, "
"address: %p, size: %" KMP_SIZE_T_SPEC "\n",
global_tid, *cache, data, size));
if (TCR_PTR(*cache) == 0) {
__kmp_acquire_lock(&__kmp_global_lock, global_tid);
if (TCR_PTR(*cache) == 0) {
__kmp_acquire_bootstrap_lock(&__kmp_tp_cached_lock);
// Compiler often passes in NULL cache, even if it's already been created
void **my_cache;
kmp_cached_addr_t *tp_cache_addr;
// Look for an existing cache
tp_cache_addr = __kmp_find_cache(data);
if (!tp_cache_addr) { // Cache was never created; do it now
__kmp_tp_cached = 1;
KMP_ITT_IGNORE(my_cache = (void **)__kmp_allocate(
sizeof(void *) * __kmp_tp_capacity +
sizeof(kmp_cached_addr_t)););
// No need to zero the allocated memory; __kmp_allocate does that.
KC_TRACE(50, ("__kmpc_threadprivate_cached: T#%d allocated cache at "
"address %p\n",
global_tid, my_cache));
/* TODO: free all this memory in __kmp_common_destroy using
* __kmp_threadpriv_cache_list */
/* Add address of mycache to linked list for cleanup later */
tp_cache_addr = (kmp_cached_addr_t *)&my_cache[__kmp_tp_capacity];
tp_cache_addr->addr = my_cache;
tp_cache_addr->data = data;
tp_cache_addr->compiler_cache = cache;
tp_cache_addr->next = __kmp_threadpriv_cache_list;
__kmp_threadpriv_cache_list = tp_cache_addr;
} else { // A cache was already created; use it
my_cache = tp_cache_addr->addr;
tp_cache_addr->compiler_cache = cache;
}
KMP_MB();
TCW_PTR(*cache, my_cache);
__kmp_release_bootstrap_lock(&__kmp_tp_cached_lock);
KMP_MB();
}
__kmp_release_lock(&__kmp_global_lock, global_tid);
}
void *ret;
if ((ret = TCR_PTR((*cache)[global_tid])) == 0) {
ret = __kmpc_threadprivate(loc, global_tid, data, (size_t)size);
TCW_PTR((*cache)[global_tid], ret);
}
KC_TRACE(10,
("__kmpc_threadprivate_cached: T#%d exiting; return value = %p\n",
global_tid, ret));
return ret;
}
// This function should only be called when both __kmp_tp_cached_lock and
// kmp_forkjoin_lock are held.
void __kmp_threadprivate_resize_cache(int newCapacity) {
KC_TRACE(10, ("__kmp_threadprivate_resize_cache: called with size: %d\n",
newCapacity));
kmp_cached_addr_t *ptr = __kmp_threadpriv_cache_list;
while (ptr) {
if (ptr->data) { // this location has an active cache; resize it
void **my_cache;
KMP_ITT_IGNORE(my_cache =
(void **)__kmp_allocate(sizeof(void *) * newCapacity +
sizeof(kmp_cached_addr_t)););
// No need to zero the allocated memory; __kmp_allocate does that.
KC_TRACE(50, ("__kmp_threadprivate_resize_cache: allocated cache at %p\n",
my_cache));
// Now copy old cache into new cache
void **old_cache = ptr->addr;
for (int i = 0; i < __kmp_tp_capacity; ++i) {
my_cache[i] = old_cache[i];
}
// Add address of new my_cache to linked list for cleanup later
kmp_cached_addr_t *tp_cache_addr;
tp_cache_addr = (kmp_cached_addr_t *)&my_cache[newCapacity];
tp_cache_addr->addr = my_cache;
tp_cache_addr->data = ptr->data;
tp_cache_addr->compiler_cache = ptr->compiler_cache;
tp_cache_addr->next = __kmp_threadpriv_cache_list;
__kmp_threadpriv_cache_list = tp_cache_addr;
// Copy new cache to compiler's location: We can copy directly
// to (*compiler_cache) if compiler guarantees it will keep
// using the same location for the cache. This is not yet true
// for some compilers, in which case we have to check if
// compiler_cache is still pointing at old cache, and if so, we
// can point it at the new cache with an atomic compare&swap
// operation. (Old method will always work, but we should shift
// to new method (commented line below) when Intel and Clang
// compilers use new method.)
(void)KMP_COMPARE_AND_STORE_PTR(tp_cache_addr->compiler_cache, old_cache,
my_cache);
// TCW_PTR(*(tp_cache_addr->compiler_cache), my_cache);
// If the store doesn't happen here, the compiler's old behavior will
// inevitably call __kmpc_threadprivate_cache with a new location for the
// cache, and that function will store the resized cache there at that
// point.
// Nullify old cache's data pointer so we skip it next time
ptr->data = NULL;
}
ptr = ptr->next;
}
// After all caches are resized, update __kmp_tp_capacity to the new size
*(volatile int *)&__kmp_tp_capacity = newCapacity;
}
/*!
@ingroup THREADPRIVATE
@param loc source location information
@param data pointer to data being privatized
@param ctor pointer to constructor function for data
@param cctor pointer to copy constructor function for data
@param dtor pointer to destructor function for data
@param vector_length length of the vector (bytes or elements?)
Register vector constructors and destructors for thread private data.
*/
void __kmpc_threadprivate_register_vec(ident_t *loc, void *data,
kmpc_ctor_vec ctor, kmpc_cctor_vec cctor,
kmpc_dtor_vec dtor,
size_t vector_length) {
struct shared_common *d_tn, **lnk_tn;
KC_TRACE(10, ("__kmpc_threadprivate_register_vec: called\n"));
#ifdef USE_CHECKS_COMMON
/* copy constructor must be zero for current code gen (Nov 2002 - jph) */
KMP_ASSERT(cctor == 0);
#endif /* USE_CHECKS_COMMON */
d_tn = __kmp_find_shared_task_common(
&__kmp_threadprivate_d_table, -1,
data); /* Only the global data table exists. */
if (d_tn == 0) {
d_tn = (struct shared_common *)__kmp_allocate(sizeof(struct shared_common));
d_tn->gbl_addr = data;
d_tn->ct.ctorv = ctor;
d_tn->cct.cctorv = cctor;
d_tn->dt.dtorv = dtor;
d_tn->is_vec = TRUE;
d_tn->vec_len = (size_t)vector_length;
// d_tn->obj_init = 0; // AC: __kmp_allocate zeroes the memory
// d_tn->pod_init = 0;
lnk_tn = &(__kmp_threadprivate_d_table.data[KMP_HASH(data)]);
d_tn->next = *lnk_tn;
*lnk_tn = d_tn;
}
}
void __kmp_cleanup_threadprivate_caches() {
kmp_cached_addr_t *ptr = __kmp_threadpriv_cache_list;
while (ptr) {
void **cache = ptr->addr;
__kmp_threadpriv_cache_list = ptr->next;
if (*ptr->compiler_cache)
*ptr->compiler_cache = NULL;
ptr->compiler_cache = NULL;
ptr->data = NULL;
ptr->addr = NULL;
ptr->next = NULL;
// Threadprivate data pointed at by cache entries are destroyed at end of
// __kmp_launch_thread with __kmp_common_destroy_gtid.
__kmp_free(cache); // implicitly frees ptr too
ptr = __kmp_threadpriv_cache_list;
}
}
Reference in a new issue View git blame Copy permalink