Add OpenMP support

third_party/openmp/kmp_itt.inc

@@ -0,0 +1,978 @@
+#if USE_ITT_BUILD
+/*
+ * kmp_itt.inl -- Inline functions of ITT Notify.
+ */
+
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Inline function definitions. This file should be included into kmp_itt.h file
+// for production build (to let compiler inline functions) or into kmp_itt.c
+// file for debug build (to reduce the number of files to recompile and save
+// build time).
+
+#include "kmp.h"
+#include "kmp_str.h"
+
+#if KMP_ITT_DEBUG
+extern kmp_bootstrap_lock_t __kmp_itt_debug_lock;
+#define KMP_ITT_DEBUG_LOCK()                                                   \
+  { __kmp_acquire_bootstrap_lock(&__kmp_itt_debug_lock); }
+#define KMP_ITT_DEBUG_PRINT(...)                                               \
+  {                                                                            \
+    fprintf(stderr, "#%02d: ", __kmp_get_gtid());                              \
+    fprintf(stderr, __VA_ARGS__);                                              \
+    fflush(stderr);                                                            \
+    __kmp_release_bootstrap_lock(&__kmp_itt_debug_lock);                       \
+  }
+#else
+#define KMP_ITT_DEBUG_LOCK()
+#define KMP_ITT_DEBUG_PRINT(...)
+#endif // KMP_ITT_DEBUG
+
+// Ensure that the functions are static if they're supposed to be being inlined.
+// Otherwise they cannot be used in more than one file, since there will be
+// multiple definitions.
+#if KMP_DEBUG
+#define LINKAGE
+#else
+#define LINKAGE static inline
+#endif
+
+// ZCA interface used by Intel(R) Inspector. Intel(R) Parallel Amplifier uses
+// this API to support user-defined synchronization primitives, but does not use
+// ZCA; it would be safe to turn this off until wider support becomes available.
+#if USE_ITT_ZCA
+#ifdef __INTEL_COMPILER
+#if __INTEL_COMPILER >= 1200
+#undef __itt_sync_acquired
+#undef __itt_sync_releasing
+#define __itt_sync_acquired(addr)                                              \
+  __notify_zc_intrinsic((char *)"sync_acquired", addr)
+#define __itt_sync_releasing(addr)                                             \
+  __notify_intrinsic((char *)"sync_releasing", addr)
+#endif
+#endif
+#endif
+
+static kmp_bootstrap_lock_t metadata_lock =
+    KMP_BOOTSTRAP_LOCK_INITIALIZER(metadata_lock);
+
+#if USE_ITT_NOTIFY
+LINKAGE size_t __kmp_itthash_hash(kmp_intptr_t addr, size_t hsize) {
+  return ((addr >> 6) ^ (addr >> 2)) % hsize;
+}
+LINKAGE kmp_itthash_entry *__kmp_itthash_find(kmp_info_t *thread,
+                                              kmp_itthash_t *h, ident_t *loc,
+                                              int team_size) {
+  kmp_itthash_entry_t *entry;
+  size_t bucket = __kmp_itthash_hash((kmp_intptr_t)loc, KMP_MAX_FRAME_DOMAINS);
+  for (entry = h->buckets[bucket]; entry; entry = entry->next_in_bucket)
+    if (entry->loc == loc && entry->team_size == team_size)
+      break;
+
+  if (entry == NULL) {
+    // two foreign threads could report frames concurrently
+    int cnt = KMP_TEST_THEN_INC32(&h->count);
+    if (cnt >= KMP_MAX_FRAME_DOMAINS) {
+      KMP_TEST_THEN_DEC32(&h->count); // revert the count
+      return entry; // too many entries
+    }
+    // create new entry
+    entry = (kmp_itthash_entry_t *)__kmp_thread_malloc(
+        thread, sizeof(kmp_itthash_entry_t));
+    entry->loc = loc;
+    entry->team_size = team_size;
+    entry->d = NULL;
+    entry->next_in_bucket = h->buckets[bucket];
+    while (!KMP_COMPARE_AND_STORE_PTR(&h->buckets[bucket],
+                                      entry->next_in_bucket, entry)) {
+      KMP_CPU_PAUSE();
+      entry->next_in_bucket = h->buckets[bucket];
+    }
+  }
+#if KMP_DEBUG
+  else {
+    // check the contents of the location info is unique
+    KMP_DEBUG_ASSERT(loc->psource == entry->loc->psource);
+  }
+#endif
+  return entry;
+}
+#endif
+
+/* Parallel region reporting.
+ * __kmp_itt_region_forking should be called by primary thread of a team.
+   Exact moment of call does not matter, but it should be completed before any
+   thread of this team calls __kmp_itt_region_starting.
+ * __kmp_itt_region_starting should be called by each thread of a team just
+   before entering parallel region body.
+ * __kmp_itt_region_finished should be called by each thread of a team right
+   after returning from parallel region body.
+ * __kmp_itt_region_joined should be called by primary thread of a team, after
+   all threads called __kmp_itt_region_finished.
+
+ Note: Thread waiting at join barrier (after __kmp_itt_region_finished) can
+ execute some more user code -- such a thread can execute tasks.
+
+ Note: The overhead of logging region_starting and region_finished in each
+ thread is too large, so these calls are not used. */
+
+LINKAGE void __kmp_itt_region_forking(int gtid, int team_size, int barriers) {
+#if USE_ITT_NOTIFY
+  kmp_team_t *team = __kmp_team_from_gtid(gtid);
+  if (team->t.t_active_level > 1) {
+    // The frame notifications are only supported for the outermost teams.
+    return;
+  }
+  kmp_info_t *th = __kmp_thread_from_gtid(gtid);
+  ident_t *loc = th->th.th_ident;
+  if (!loc) {
+    // no sense to report a region without location info
+    return;
+  }
+  kmp_itthash_entry *e;
+  e = __kmp_itthash_find(th, &__kmp_itt_region_domains, loc, team_size);
+  if (e == NULL)
+    return; // too many entries in the hash
+  if (e->d == NULL) {
+    // Transform compiler-generated region location into the format
+    // that the tools more or less standardized on:
+    //   "<func>$omp$parallel@[file:]<line>[:<col>]"
+    char *buff = NULL;
+    kmp_str_loc_t str_loc =
+        __kmp_str_loc_init(loc->psource, /* init_fname */ false);
+    buff = __kmp_str_format("%s$omp$parallel:%d@%s:%d:%d", str_loc.func,
+                            team_size, str_loc.file, str_loc.line, str_loc.col);
+
+    __itt_suppress_push(__itt_suppress_memory_errors);
+    e->d = __itt_domain_create(buff);
+    KMP_ASSERT(e->d != NULL);
+    __itt_suppress_pop();
+
+    __kmp_str_free(&buff);
+    if (barriers) {
+      kmp_itthash_entry *e;
+      e = __kmp_itthash_find(th, &__kmp_itt_barrier_domains, loc, 0);
+      if (e != NULL) {
+        KMP_DEBUG_ASSERT(e->d == NULL);
+        char *buff = NULL;
+        buff = __kmp_str_format("%s$omp$barrier@%s:%d", str_loc.func,
+                                str_loc.file, str_loc.line);
+        __itt_suppress_push(__itt_suppress_memory_errors);
+        e->d = __itt_domain_create(buff);
+        KMP_ASSERT(e->d != NULL);
+        __itt_suppress_pop();
+        __kmp_str_free(&buff);
+      }
+    }
+    __kmp_str_loc_free(&str_loc);
+  }
+  __itt_frame_begin_v3(e->d, NULL);
+  KMP_ITT_DEBUG_LOCK();
+  KMP_ITT_DEBUG_PRINT("[frm beg] gtid=%d, domain=%p, loc:%p\n", gtid, e->d,
+                      loc);
+#endif
+} // __kmp_itt_region_forking
+
+// -----------------------------------------------------------------------------
+LINKAGE void __kmp_itt_frame_submit(int gtid, __itt_timestamp begin,
+                                    __itt_timestamp end, int imbalance,
+                                    ident_t *loc, int team_size, int region) {
+#if USE_ITT_NOTIFY
+  if (!loc) {
+    // no sense to report a region without location info
+    return;
+  }
+  kmp_info_t *th = __kmp_thread_from_gtid(gtid);
+  if (region) {
+    kmp_team_t *team = __kmp_team_from_gtid(gtid);
+    int serialized = (region == 2 ? 1 : 0);
+    if (team->t.t_active_level + serialized > 1) {
+      // The frame notifications are only supported for the outermost teams.
+      return;
+    }
+    // Check region domain has not been created before.
+    kmp_itthash_entry *e;
+    e = __kmp_itthash_find(th, &__kmp_itt_region_domains, loc, team_size);
+    if (e == NULL)
+      return; // too many entries in the hash
+    if (e->d == NULL) { // new entry, need to calculate domain
+      // Transform compiler-generated region location into the format
+      // that the tools more or less standardized on:
+      //   "<func>$omp$parallel:team_size@[file:]<line>[:<col>]"
+      char *buff = NULL;
+      kmp_str_loc_t str_loc =
+          __kmp_str_loc_init(loc->psource, /* init_fname */ false);
+      buff =
+          __kmp_str_format("%s$omp$parallel:%d@%s:%d:%d", str_loc.func,
+                           team_size, str_loc.file, str_loc.line, str_loc.col);
+      __itt_suppress_push(__itt_suppress_memory_errors);
+      e->d = __itt_domain_create(buff);
+      KMP_ASSERT(e->d != NULL);
+      __itt_suppress_pop();
+
+      __kmp_str_free(&buff);
+      __kmp_str_loc_free(&str_loc);
+    }
+    __itt_frame_submit_v3(e->d, NULL, begin, end);
+    KMP_ITT_DEBUG_LOCK();
+    KMP_ITT_DEBUG_PRINT(
+        "[reg sub] gtid=%d, domain=%p, region:%d, loc:%p, beg:%llu, end:%llu\n",
+        gtid, e->d, region, loc, begin, end);
+    return;
+  } else { // called for barrier reporting
+    kmp_itthash_entry *e;
+    e = __kmp_itthash_find(th, &__kmp_itt_barrier_domains, loc, 0);
+    if (e == NULL)
+      return; // too many entries in the hash
+    if (e->d == NULL) { // new entry, need to calculate domain
+      // Transform compiler-generated region location into the format
+      // that the tools more or less standardized on:
+      //   "<func>$omp$frame@[file:]<line>[:<col>]"
+      kmp_str_loc_t str_loc =
+          __kmp_str_loc_init(loc->psource, /* init_fname */ false);
+      char *buff = NULL;
+      if (imbalance) {
+        buff =
+            __kmp_str_format("%s$omp$barrier-imbalance:%d@%s:%d", str_loc.func,
+                             team_size, str_loc.file, str_loc.line);
+      } else {
+        buff = __kmp_str_format("%s$omp$barrier@%s:%d", str_loc.func,
+                                str_loc.file, str_loc.line);
+      }
+      __itt_suppress_push(__itt_suppress_memory_errors);
+      e->d = __itt_domain_create(buff);
+      KMP_ASSERT(e->d != NULL);
+      __itt_suppress_pop();
+      __kmp_str_free(&buff);
+      __kmp_str_loc_free(&str_loc);
+    }
+    __itt_frame_submit_v3(e->d, NULL, begin, end);
+    KMP_ITT_DEBUG_LOCK();
+    KMP_ITT_DEBUG_PRINT(
+        "[frm sub] gtid=%d, domain=%p, loc:%p, beg:%llu, end:%llu\n", gtid,
+        e->d, loc, begin, end);
+  }
+#endif
+} // __kmp_itt_frame_submit
+
+// -----------------------------------------------------------------------------
+LINKAGE void __kmp_itt_metadata_imbalance(int gtid, kmp_uint64 begin,
+                                          kmp_uint64 end, kmp_uint64 imbalance,
+                                          kmp_uint64 reduction) {
+#if USE_ITT_NOTIFY
+  if (metadata_domain == NULL) {
+    __kmp_acquire_bootstrap_lock(&metadata_lock);
+    if (metadata_domain == NULL) {
+      __itt_suppress_push(__itt_suppress_memory_errors);
+      metadata_domain = __itt_domain_create("OMP Metadata");
+      string_handle_imbl = __itt_string_handle_create("omp_metadata_imbalance");
+      string_handle_loop = __itt_string_handle_create("omp_metadata_loop");
+      string_handle_sngl = __itt_string_handle_create("omp_metadata_single");
+      __itt_suppress_pop();
+    }
+    __kmp_release_bootstrap_lock(&metadata_lock);
+  }
+
+  kmp_uint64 imbalance_data[4];
+  imbalance_data[0] = begin;
+  imbalance_data[1] = end;
+  imbalance_data[2] = imbalance;
+  imbalance_data[3] = reduction;
+
+  __itt_metadata_add(metadata_domain, __itt_null, string_handle_imbl,
+                     __itt_metadata_u64, 4, imbalance_data);
+#endif
+} // __kmp_itt_metadata_imbalance
+
+// -----------------------------------------------------------------------------
+LINKAGE void __kmp_itt_metadata_loop(ident_t *loc, kmp_uint64 sched_type,
+                                     kmp_uint64 iterations, kmp_uint64 chunk) {
+#if USE_ITT_NOTIFY
+  if (metadata_domain == NULL) {
+    __kmp_acquire_bootstrap_lock(&metadata_lock);
+    if (metadata_domain == NULL) {
+      __itt_suppress_push(__itt_suppress_memory_errors);
+      metadata_domain = __itt_domain_create("OMP Metadata");
+      string_handle_imbl = __itt_string_handle_create("omp_metadata_imbalance");
+      string_handle_loop = __itt_string_handle_create("omp_metadata_loop");
+      string_handle_sngl = __itt_string_handle_create("omp_metadata_single");
+      __itt_suppress_pop();
+    }
+    __kmp_release_bootstrap_lock(&metadata_lock);
+  }
+
+  // Parse line and column from psource string: ";file;func;line;col;;"
+  KMP_DEBUG_ASSERT(loc->psource);
+  kmp_uint64 loop_data[5];
+  int line, col;
+  __kmp_str_loc_numbers(loc->psource, &line, &col);
+  loop_data[0] = line;
+  loop_data[1] = col;
+  loop_data[2] = sched_type;
+  loop_data[3] = iterations;
+  loop_data[4] = chunk;
+
+  __itt_metadata_add(metadata_domain, __itt_null, string_handle_loop,
+                     __itt_metadata_u64, 5, loop_data);
+#endif
+} // __kmp_itt_metadata_loop
+
+// -----------------------------------------------------------------------------
+LINKAGE void __kmp_itt_metadata_single(ident_t *loc) {
+#if USE_ITT_NOTIFY
+  if (metadata_domain == NULL) {
+    __kmp_acquire_bootstrap_lock(&metadata_lock);
+    if (metadata_domain == NULL) {
+      __itt_suppress_push(__itt_suppress_memory_errors);
+      metadata_domain = __itt_domain_create("OMP Metadata");
+      string_handle_imbl = __itt_string_handle_create("omp_metadata_imbalance");
+      string_handle_loop = __itt_string_handle_create("omp_metadata_loop");
+      string_handle_sngl = __itt_string_handle_create("omp_metadata_single");
+      __itt_suppress_pop();
+    }
+    __kmp_release_bootstrap_lock(&metadata_lock);
+  }
+
+  int line, col;
+  __kmp_str_loc_numbers(loc->psource, &line, &col);
+  kmp_uint64 single_data[2];
+  single_data[0] = line;
+  single_data[1] = col;
+
+  __itt_metadata_add(metadata_domain, __itt_null, string_handle_sngl,
+                     __itt_metadata_u64, 2, single_data);
+#endif
+} // __kmp_itt_metadata_single
+
+// -----------------------------------------------------------------------------
+LINKAGE void __kmp_itt_region_starting(int gtid) {
+#if USE_ITT_NOTIFY
+#endif
+} // __kmp_itt_region_starting
+
+// -----------------------------------------------------------------------------
+LINKAGE void __kmp_itt_region_finished(int gtid) {
+#if USE_ITT_NOTIFY
+#endif
+} // __kmp_itt_region_finished
+
+// ----------------------------------------------------------------------------
+LINKAGE void __kmp_itt_region_joined(int gtid) {
+#if USE_ITT_NOTIFY
+  kmp_team_t *team = __kmp_team_from_gtid(gtid);
+  if (team->t.t_active_level > 1) {
+    // The frame notifications are only supported for the outermost teams.
+    return;
+  }
+  kmp_info_t *th = __kmp_thread_from_gtid(gtid);
+  ident_t *loc = th->th.th_ident;
+  if (loc) {
+    kmp_itthash_entry *e = __kmp_itthash_find(th, &__kmp_itt_region_domains,
+                                              loc, th->th.th_team_nproc);
+    if (e == NULL)
+      return; // too many entries in the hash
+    KMP_DEBUG_ASSERT(e->d);
+    KMP_ITT_DEBUG_LOCK();
+    __itt_frame_end_v3(e->d, NULL);
+    KMP_ITT_DEBUG_PRINT("[frm end] gtid=%d, domain=%p, loc:%p\n", gtid, e->d,
+                        loc);
+  }
+#endif
+} // __kmp_itt_region_joined
+
+/* Barriers reporting.
+
+   A barrier consists of two phases:
+   1. Gather -- primary thread waits for all worker threads to arrive; each
+      worker thread registers arrival and goes further.
+   2. Release -- each worker thread waits until primary thread lets it go;
+      primary thread lets worker threads go.
+
+   Function should be called by each thread:
+   * __kmp_itt_barrier_starting() -- before arriving to the gather phase.
+   * __kmp_itt_barrier_middle()   -- between gather and release phases.
+   * __kmp_itt_barrier_finished() -- after release phase.
+
+   Note: Call __kmp_itt_barrier_object() before call to
+   __kmp_itt_barrier_starting() and save result in local variable.
+   __kmp_itt_barrier_object(), being called too late (e. g. after gather phase)
+   would return itt sync object for the next barrier!
+
+   ITT need an address (void *) to be specified as a sync object. OpenMP RTL
+   does not have barrier object or barrier data structure. Barrier is just a
+   counter in team and thread structures. We could use an address of team
+   structure as a barrier sync object, but ITT wants different objects for
+   different barriers (even whithin the same team). So let us use team address
+   as barrier sync object for the first barrier, then increase it by one for the
+   next barrier, and so on (but wrap it not to use addresses outside of team
+   structure). */
+
+void *__kmp_itt_barrier_object(int gtid, int bt, int set_name,
+                               int delta // 0 (current barrier) is default
+                               // value; specify -1 to get previous
+                               // barrier.
+                               ) {
+  void *object = NULL;
+#if USE_ITT_NOTIFY
+  kmp_info_t *thr = __kmp_thread_from_gtid(gtid);
+  kmp_team_t *team = thr->th.th_team;
+
+  // NOTE: If the function is called from __kmp_fork_barrier, team pointer can
+  // be NULL. This "if" helps to avoid crash. However, this is not complete
+  // solution, and reporting fork/join barriers to ITT should be revisited.
+
+  if (team != NULL) {
+    // Primary thread increases b_arrived by KMP_BARRIER_STATE_BUMP each time.
+    // Divide b_arrived by KMP_BARRIER_STATE_BUMP to get plain barrier counter.
+    kmp_uint64 counter =
+        team->t.t_bar[bt].b_arrived / KMP_BARRIER_STATE_BUMP + delta;
+    // Now form the barrier id. Encode barrier type (bt) in barrier id too, so
+    // barriers of different types do not have the same ids.
+    KMP_BUILD_ASSERT(sizeof(kmp_team_t) >= bs_last_barrier);
+    // This condition is a must (we would have zero divide otherwise).
+    KMP_BUILD_ASSERT(sizeof(kmp_team_t) >= 2 * bs_last_barrier);
+    // More strong condition: make sure we have room at least for two
+    // different ids (for each barrier type).
+    object = reinterpret_cast<void *>(
+        (kmp_uintptr_t)(team) +
+        (kmp_uintptr_t)counter % (sizeof(kmp_team_t) / bs_last_barrier) *
+            bs_last_barrier +
+        bt);
+    KMP_ITT_DEBUG_LOCK();
+    KMP_ITT_DEBUG_PRINT("[bar obj] type=%d, counter=%lld, object=%p\n", bt,
+                        counter, object);
+
+    if (set_name) {
+      ident_t const *loc = NULL;
+      char const *src = NULL;
+      char const *type = "OMP Barrier";
+      switch (bt) {
+      case bs_plain_barrier: {
+        // For plain barrier compiler calls __kmpc_barrier() function, which
+        // saves location in thr->th.th_ident.
+        loc = thr->th.th_ident;
+        // Get the barrier type from flags provided by compiler.
+        kmp_int32 expl = 0;
+        kmp_uint32 impl = 0;
+        if (loc != NULL) {
+          src = loc->psource;
+          expl = (loc->flags & KMP_IDENT_BARRIER_EXPL) != 0;
+          impl = (loc->flags & KMP_IDENT_BARRIER_IMPL) != 0;
+        }
+        if (impl) {
+          switch (loc->flags & KMP_IDENT_BARRIER_IMPL_MASK) {
+          case KMP_IDENT_BARRIER_IMPL_FOR: {
+            type = "OMP For Barrier";
+          } break;
+          case KMP_IDENT_BARRIER_IMPL_SECTIONS: {
+            type = "OMP Sections Barrier";
+          } break;
+          case KMP_IDENT_BARRIER_IMPL_SINGLE: {
+            type = "OMP Single Barrier";
+          } break;
+          case KMP_IDENT_BARRIER_IMPL_WORKSHARE: {
+            type = "OMP Workshare Barrier";
+          } break;
+          default: {
+            type = "OMP Implicit Barrier";
+            KMP_DEBUG_ASSERT(0);
+          }
+          }
+        } else if (expl) {
+          type = "OMP Explicit Barrier";
+        }
+      } break;
+      case bs_forkjoin_barrier: {
+        // In case of fork/join barrier we can read thr->th.th_ident, because it
+        // contains location of last passed construct (while join barrier is not
+        // such one). Use th_ident of primary thread instead --
+        // __kmp_join_call() called by the primary thread saves location.
+        //
+        // AC: cannot read from primary thread because __kmp_join_call may not
+        //    be called yet, so we read the location from team. This is the
+        //    same location. Team is valid on entry to join barrier where this
+        //    happens.
+        loc = team->t.t_ident;
+        if (loc != NULL) {
+          src = loc->psource;
+        }
+        type = "OMP Join Barrier";
+      } break;
+      }
+      KMP_ITT_DEBUG_LOCK();
+      __itt_sync_create(object, type, src, __itt_attr_barrier);
+      KMP_ITT_DEBUG_PRINT(
+          "[bar sta] scre( %p, \"%s\", \"%s\", __itt_attr_barrier )\n", object,
+          type, src);
+    }
+  }
+#endif
+  return object;
+} // __kmp_itt_barrier_object
+
+// -----------------------------------------------------------------------------
+void __kmp_itt_barrier_starting(int gtid, void *object) {
+#if USE_ITT_NOTIFY
+  if (!KMP_MASTER_GTID(gtid)) {
+    KMP_ITT_DEBUG_LOCK();
+    __itt_sync_releasing(object);
+    KMP_ITT_DEBUG_PRINT("[bar sta] srel( %p )\n", object);
+  }
+  KMP_ITT_DEBUG_LOCK();
+  __itt_sync_prepare(object);
+  KMP_ITT_DEBUG_PRINT("[bar sta] spre( %p )\n", object);
+#endif
+} // __kmp_itt_barrier_starting
+
+// -----------------------------------------------------------------------------
+void __kmp_itt_barrier_middle(int gtid, void *object) {
+#if USE_ITT_NOTIFY
+  if (KMP_MASTER_GTID(gtid)) {
+    KMP_ITT_DEBUG_LOCK();
+    __itt_sync_acquired(object);
+    KMP_ITT_DEBUG_PRINT("[bar mid] sacq( %p )\n", object);
+    KMP_ITT_DEBUG_LOCK();
+    __itt_sync_releasing(object);
+    KMP_ITT_DEBUG_PRINT("[bar mid] srel( %p )\n", object);
+  } else {
+  }
+#endif
+} // __kmp_itt_barrier_middle
+
+// -----------------------------------------------------------------------------
+void __kmp_itt_barrier_finished(int gtid, void *object) {
+#if USE_ITT_NOTIFY
+  if (KMP_MASTER_GTID(gtid)) {
+  } else {
+    KMP_ITT_DEBUG_LOCK();
+    __itt_sync_acquired(object);
+    KMP_ITT_DEBUG_PRINT("[bar end] sacq( %p )\n", object);
+  }
+#endif
+} // __kmp_itt_barrier_finished
+
+/* Taskwait reporting.
+   ITT need an address (void *) to be specified as a sync object. OpenMP RTL
+   does not have taskwait structure, so we need to construct something. */
+
+void *__kmp_itt_taskwait_object(int gtid) {
+  void *object = NULL;
+#if USE_ITT_NOTIFY
+  if (UNLIKELY(__itt_sync_create_ptr)) {
+    kmp_info_t *thread = __kmp_thread_from_gtid(gtid);
+    kmp_taskdata_t *taskdata = thread->th.th_current_task;
+    object = reinterpret_cast<void *>(kmp_uintptr_t(taskdata) +
+                                      taskdata->td_taskwait_counter %
+                                          sizeof(kmp_taskdata_t));
+  }
+#endif
+  return object;
+} // __kmp_itt_taskwait_object
+
+void __kmp_itt_taskwait_starting(int gtid, void *object) {
+#if USE_ITT_NOTIFY
+  kmp_info_t *thread = __kmp_thread_from_gtid(gtid);
+  kmp_taskdata_t *taskdata = thread->th.th_current_task;
+  ident_t const *loc = taskdata->td_taskwait_ident;
+  char const *src = (loc == NULL ? NULL : loc->psource);
+  KMP_ITT_DEBUG_LOCK();
+  __itt_sync_create(object, "OMP Taskwait", src, 0);
+  KMP_ITT_DEBUG_PRINT("[twa sta] scre( %p, \"OMP Taskwait\", \"%s\", 0 )\n",
+                      object, src);
+  KMP_ITT_DEBUG_LOCK();
+  __itt_sync_prepare(object);
+  KMP_ITT_DEBUG_PRINT("[twa sta] spre( %p )\n", object);
+#endif
+} // __kmp_itt_taskwait_starting
+
+void __kmp_itt_taskwait_finished(int gtid, void *object) {
+#if USE_ITT_NOTIFY
+  KMP_ITT_DEBUG_LOCK();
+  __itt_sync_acquired(object);
+  KMP_ITT_DEBUG_PRINT("[twa end] sacq( %p )\n", object);
+  KMP_ITT_DEBUG_LOCK();
+  __itt_sync_destroy(object);
+  KMP_ITT_DEBUG_PRINT("[twa end] sdes( %p )\n", object);
+#endif
+} // __kmp_itt_taskwait_finished
+
+/* Task reporting.
+   Only those tasks are reported which are executed by a thread spinning at
+   barrier (or taskwait). Synch object passed to the function must be barrier of
+   taskwait the threads waiting at. */
+
+void __kmp_itt_task_starting(
+    void *object // ITT sync object: barrier or taskwait.
+    ) {
+#if USE_ITT_NOTIFY
+  if (UNLIKELY(object != NULL)) {
+    KMP_ITT_DEBUG_LOCK();
+    __itt_sync_cancel(object);
+    KMP_ITT_DEBUG_PRINT("[tsk sta] scan( %p )\n", object);
+  }
+#endif
+} // __kmp_itt_task_starting
+
+// -----------------------------------------------------------------------------
+void __kmp_itt_task_finished(
+    void *object // ITT sync object: barrier or taskwait.
+    ) {
+#if USE_ITT_NOTIFY
+  KMP_ITT_DEBUG_LOCK();
+  __itt_sync_prepare(object);
+  KMP_ITT_DEBUG_PRINT("[tsk end] spre( %p )\n", object);
+#endif
+} // __kmp_itt_task_finished
+
+/* Lock reporting.
+ * __kmp_itt_lock_creating( lock ) should be called *before* the first lock
+   operation (set/unset). It is not a real event shown to the user but just
+   setting a name for synchronization object. `lock' is an address of sync
+   object, the same address should be used in all subsequent calls.
+ * __kmp_itt_lock_acquiring() should be called before setting the lock.
+ * __kmp_itt_lock_acquired() should be called after setting the lock.
+ * __kmp_itt_lock_realeasing() should be called before unsetting the lock.
+ * __kmp_itt_lock_cancelled() should be called after thread cancelled waiting
+   for the lock.
+ * __kmp_itt_lock_destroyed( lock ) should be called after the last lock
+   operation. After __kmp_itt_lock_destroyed() all the references to the same
+   address will be considered as another sync object, not related with the
+   original one.  */
+
+#if KMP_USE_DYNAMIC_LOCK
+// Takes location information directly
+__kmp_inline void ___kmp_itt_lock_init(kmp_user_lock_p lock, char const *type,
+                                       const ident_t *loc) {
+#if USE_ITT_NOTIFY
+  if (__itt_sync_create_ptr) {
+    char const *src = (loc == NULL ? NULL : loc->psource);
+    KMP_ITT_DEBUG_LOCK();
+    __itt_sync_create(lock, type, src, 0);
+    KMP_ITT_DEBUG_PRINT("[lck ini] scre( %p, \"%s\", \"%s\", 0 )\n", lock, type,
+                        src);
+  }
+#endif
+}
+#else // KMP_USE_DYNAMIC_LOCK
+// Internal guts -- common code for locks and critical sections, do not call
+// directly.
+__kmp_inline void ___kmp_itt_lock_init(kmp_user_lock_p lock, char const *type) {
+#if USE_ITT_NOTIFY
+  if (__itt_sync_create_ptr) {
+    ident_t const *loc = NULL;
+    if (__kmp_get_user_lock_location_ != NULL)
+      loc = __kmp_get_user_lock_location_((lock));
+    char const *src = (loc == NULL ? NULL : loc->psource);
+    KMP_ITT_DEBUG_LOCK();
+    __itt_sync_create(lock, type, src, 0);
+    KMP_ITT_DEBUG_PRINT("[lck ini] scre( %p, \"%s\", \"%s\", 0 )\n", lock, type,
+                        src);
+  }
+#endif
+} // ___kmp_itt_lock_init
+#endif // KMP_USE_DYNAMIC_LOCK
+
+// Internal guts -- common code for locks and critical sections, do not call
+// directly.
+__kmp_inline void ___kmp_itt_lock_fini(kmp_user_lock_p lock, char const *type) {
+#if USE_ITT_NOTIFY
+  KMP_ITT_DEBUG_LOCK();
+  __itt_sync_destroy(lock);
+  KMP_ITT_DEBUG_PRINT("[lck dst] sdes( %p )\n", lock);
+#endif
+} // ___kmp_itt_lock_fini
+
+// -----------------------------------------------------------------------------
+#if KMP_USE_DYNAMIC_LOCK
+void __kmp_itt_lock_creating(kmp_user_lock_p lock, const ident_t *loc) {
+  ___kmp_itt_lock_init(lock, "OMP Lock", loc);
+}
+#else
+void __kmp_itt_lock_creating(kmp_user_lock_p lock) {
+  ___kmp_itt_lock_init(lock, "OMP Lock");
+} // __kmp_itt_lock_creating
+#endif
+
+void __kmp_itt_lock_acquiring(kmp_user_lock_p lock) {
+#if KMP_USE_DYNAMIC_LOCK && USE_ITT_NOTIFY
+  // postpone lock object access
+  if (__itt_sync_prepare_ptr) {
+    if (KMP_EXTRACT_D_TAG(lock) == 0) {
+      kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
+      __itt_sync_prepare(ilk->lock);
+    } else {
+      __itt_sync_prepare(lock);
+    }
+  }
+#else
+  __itt_sync_prepare(lock);
+#endif
+} // __kmp_itt_lock_acquiring
+
+void __kmp_itt_lock_acquired(kmp_user_lock_p lock) {
+#if KMP_USE_DYNAMIC_LOCK && USE_ITT_NOTIFY
+  // postpone lock object access
+  if (__itt_sync_acquired_ptr) {
+    if (KMP_EXTRACT_D_TAG(lock) == 0) {
+      kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
+      __itt_sync_acquired(ilk->lock);
+    } else {
+      __itt_sync_acquired(lock);
+    }
+  }
+#else
+  __itt_sync_acquired(lock);
+#endif
+} // __kmp_itt_lock_acquired
+
+void __kmp_itt_lock_releasing(kmp_user_lock_p lock) {
+#if KMP_USE_DYNAMIC_LOCK && USE_ITT_NOTIFY
+  if (__itt_sync_releasing_ptr) {
+    if (KMP_EXTRACT_D_TAG(lock) == 0) {
+      kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
+      __itt_sync_releasing(ilk->lock);
+    } else {
+      __itt_sync_releasing(lock);
+    }
+  }
+#else
+  __itt_sync_releasing(lock);
+#endif
+} // __kmp_itt_lock_releasing
+
+void __kmp_itt_lock_cancelled(kmp_user_lock_p lock) {
+#if KMP_USE_DYNAMIC_LOCK && USE_ITT_NOTIFY
+  if (__itt_sync_cancel_ptr) {
+    if (KMP_EXTRACT_D_TAG(lock) == 0) {
+      kmp_indirect_lock_t *ilk = KMP_LOOKUP_I_LOCK(lock);
+      __itt_sync_cancel(ilk->lock);
+    } else {
+      __itt_sync_cancel(lock);
+    }
+  }
+#else
+  __itt_sync_cancel(lock);
+#endif
+} // __kmp_itt_lock_cancelled
+
+void __kmp_itt_lock_destroyed(kmp_user_lock_p lock) {
+  ___kmp_itt_lock_fini(lock, "OMP Lock");
+} // __kmp_itt_lock_destroyed
+
+/* Critical reporting.
+   Critical sections are treated exactly as locks (but have different object
+   type). */
+#if KMP_USE_DYNAMIC_LOCK
+void __kmp_itt_critical_creating(kmp_user_lock_p lock, const ident_t *loc) {
+  ___kmp_itt_lock_init(lock, "OMP Critical", loc);
+}
+#else
+void __kmp_itt_critical_creating(kmp_user_lock_p lock) {
+  ___kmp_itt_lock_init(lock, "OMP Critical");
+} // __kmp_itt_critical_creating
+#endif
+
+void __kmp_itt_critical_acquiring(kmp_user_lock_p lock) {
+  __itt_sync_prepare(lock);
+} // __kmp_itt_critical_acquiring
+
+void __kmp_itt_critical_acquired(kmp_user_lock_p lock) {
+  __itt_sync_acquired(lock);
+} // __kmp_itt_critical_acquired
+
+void __kmp_itt_critical_releasing(kmp_user_lock_p lock) {
+  __itt_sync_releasing(lock);
+} // __kmp_itt_critical_releasing
+
+void __kmp_itt_critical_destroyed(kmp_user_lock_p lock) {
+  ___kmp_itt_lock_fini(lock, "OMP Critical");
+} // __kmp_itt_critical_destroyed
+
+/* Single reporting. */
+
+void __kmp_itt_single_start(int gtid) {
+#if USE_ITT_NOTIFY
+  if (__itt_mark_create_ptr || KMP_ITT_DEBUG) {
+    kmp_info_t *thr = __kmp_thread_from_gtid((gtid));
+    ident_t *loc = thr->th.th_ident;
+    char const *src = (loc == NULL ? NULL : loc->psource);
+    kmp_str_buf_t name;
+    __kmp_str_buf_init(&name);
+    __kmp_str_buf_print(&name, "OMP Single-%s", src);
+    KMP_ITT_DEBUG_LOCK();
+    thr->th.th_itt_mark_single = __itt_mark_create(name.str);
+    KMP_ITT_DEBUG_PRINT("[sin sta] mcre( \"%s\") -> %d\n", name.str,
+                        thr->th.th_itt_mark_single);
+    __kmp_str_buf_free(&name);
+    KMP_ITT_DEBUG_LOCK();
+    __itt_mark(thr->th.th_itt_mark_single, NULL);
+    KMP_ITT_DEBUG_PRINT("[sin sta] mark( %d, NULL )\n",
+                        thr->th.th_itt_mark_single);
+  }
+#endif
+} // __kmp_itt_single_start
+
+void __kmp_itt_single_end(int gtid) {
+#if USE_ITT_NOTIFY
+  __itt_mark_type mark = __kmp_thread_from_gtid(gtid)->th.th_itt_mark_single;
+  KMP_ITT_DEBUG_LOCK();
+  __itt_mark_off(mark);
+  KMP_ITT_DEBUG_PRINT("[sin end] moff( %d )\n", mark);
+#endif
+} // __kmp_itt_single_end
+
+/* Ordered reporting.
+ * __kmp_itt_ordered_init is called by each thread *before* first using sync
+   object. ITT team would like it to be called once, but it requires extra
+   synchronization.
+ * __kmp_itt_ordered_prep is called when thread is going to enter ordered
+   section (before synchronization).
+ * __kmp_itt_ordered_start is called just before entering user code (after
+   synchronization).
+ * __kmp_itt_ordered_end is called after returning from user code.
+
+ Sync object is th->th.th_dispatch->th_dispatch_sh_current.
+ Events are not generated in case of serialized team. */
+
+void __kmp_itt_ordered_init(int gtid) {
+#if USE_ITT_NOTIFY
+  if (__itt_sync_create_ptr) {
+    kmp_info_t *thr = __kmp_thread_from_gtid(gtid);
+    ident_t const *loc = thr->th.th_ident;
+    char const *src = (loc == NULL ? NULL : loc->psource);
+    __itt_sync_create(thr->th.th_dispatch->th_dispatch_sh_current,
+                      "OMP Ordered", src, 0);
+  }
+#endif
+} // __kmp_itt_ordered_init
+
+void __kmp_itt_ordered_prep(int gtid) {
+#if USE_ITT_NOTIFY
+  if (__itt_sync_create_ptr) {
+    kmp_team_t *t = __kmp_team_from_gtid(gtid);
+    if (!t->t.t_serialized) {
+      kmp_info_t *th = __kmp_thread_from_gtid(gtid);
+      __itt_sync_prepare(th->th.th_dispatch->th_dispatch_sh_current);
+    }
+  }
+#endif
+} // __kmp_itt_ordered_prep
+
+void __kmp_itt_ordered_start(int gtid) {
+#if USE_ITT_NOTIFY
+  if (__itt_sync_create_ptr) {
+    kmp_team_t *t = __kmp_team_from_gtid(gtid);
+    if (!t->t.t_serialized) {
+      kmp_info_t *th = __kmp_thread_from_gtid(gtid);
+      __itt_sync_acquired(th->th.th_dispatch->th_dispatch_sh_current);
+    }
+  }
+#endif
+} // __kmp_itt_ordered_start
+
+void __kmp_itt_ordered_end(int gtid) {
+#if USE_ITT_NOTIFY
+  if (__itt_sync_create_ptr) {
+    kmp_team_t *t = __kmp_team_from_gtid(gtid);
+    if (!t->t.t_serialized) {
+      kmp_info_t *th = __kmp_thread_from_gtid(gtid);
+      __itt_sync_releasing(th->th.th_dispatch->th_dispatch_sh_current);
+    }
+  }
+#endif
+} // __kmp_itt_ordered_end
+
+/* Threads reporting. */
+
+void __kmp_itt_thread_ignore() {
+  __itt_thr_ignore();
+} // __kmp_itt_thread_ignore
+
+void __kmp_itt_thread_name(int gtid) {
+#if USE_ITT_NOTIFY
+  if (__itt_thr_name_set_ptr) {
+    kmp_str_buf_t name;
+    __kmp_str_buf_init(&name);
+    if (KMP_MASTER_GTID(gtid)) {
+      __kmp_str_buf_print(&name, "OMP Primary Thread #%d", gtid);
+    } else {
+      __kmp_str_buf_print(&name, "OMP Worker Thread #%d", gtid);
+    }
+    KMP_ITT_DEBUG_LOCK();
+    __itt_thr_name_set(name.str, name.used);
+    KMP_ITT_DEBUG_PRINT("[thr nam] name( \"%s\")\n", name.str);
+    __kmp_str_buf_free(&name);
+  }
+#endif
+} // __kmp_itt_thread_name
+
+/* System object reporting.
+   ITT catches operations with system sync objects (like Windows* OS on IA-32
+   architecture API critical sections and events). We only need to specify
+   name ("OMP Scheduler") for the object to let ITT know it is an object used
+   by OpenMP RTL for internal purposes. */
+
+void __kmp_itt_system_object_created(void *object, char const *name) {
+#if USE_ITT_NOTIFY
+  KMP_ITT_DEBUG_LOCK();
+  __itt_sync_create(object, "OMP Scheduler", name, 0);
+  KMP_ITT_DEBUG_PRINT("[sys obj] scre( %p, \"OMP Scheduler\", \"%s\", 0 )\n",
+                      object, name);
+#endif
+} // __kmp_itt_system_object_created
+
+/* Stack stitching api.
+   Primary thread calls "create" and put the stitching id into team structure.
+   Workers read the stitching id and call "enter" / "leave" api.
+   Primary thread calls "destroy" at the end of the parallel region. */
+
+__itt_caller __kmp_itt_stack_caller_create() {
+#if USE_ITT_NOTIFY
+  if (!__itt_stack_caller_create_ptr)
+    return NULL;
+  KMP_ITT_DEBUG_LOCK();
+  __itt_caller id = __itt_stack_caller_create();
+  KMP_ITT_DEBUG_PRINT("[stk cre] %p\n", id);
+  return id;
+#endif
+  return NULL;
+}
+
+void __kmp_itt_stack_caller_destroy(__itt_caller id) {
+#if USE_ITT_NOTIFY
+  if (__itt_stack_caller_destroy_ptr) {
+    KMP_ITT_DEBUG_LOCK();
+    __itt_stack_caller_destroy(id);
+    KMP_ITT_DEBUG_PRINT("[stk des] %p\n", id);
+  }
+#endif
+}
+
+void __kmp_itt_stack_callee_enter(__itt_caller id) {
+#if USE_ITT_NOTIFY
+  if (__itt_stack_callee_enter_ptr) {
+    KMP_ITT_DEBUG_LOCK();
+    __itt_stack_callee_enter(id);
+    KMP_ITT_DEBUG_PRINT("[stk ent] %p\n", id);
+  }
+#endif
+}
+
+void __kmp_itt_stack_callee_leave(__itt_caller id) {
+#if USE_ITT_NOTIFY
+  if (__itt_stack_callee_leave_ptr) {
+    KMP_ITT_DEBUG_LOCK();
+    __itt_stack_callee_leave(id);
+    KMP_ITT_DEBUG_PRINT("[stk lea] %p\n", id);
+  }
+#endif
+}
+
+#endif /* USE_ITT_BUILD */