21 #include "kmp_wait_release.h"
22 #include "kmp_wrapper_getpid.h"
25 #include <sys/syscall.h>
43 void __kmp_validate_locks(
void) {
48 x = ~((kmp_uint32)0) - 2;
51 for (i = 0; i < 8; ++i, ++x, ++y) {
52 kmp_uint32 z = (x - y);
56 KMP_ASSERT(offsetof(kmp_base_queuing_lock, tail_id) % 8 == 0);
70 static kmp_int32 __kmp_get_tas_lock_owner(kmp_tas_lock_t *lck) {
71 return KMP_LOCK_STRIP(KMP_ATOMIC_LD_RLX(&lck->lk.poll)) - 1;
74 static inline bool __kmp_is_tas_lock_nestable(kmp_tas_lock_t *lck) {
75 return lck->lk.depth_locked != -1;
78 __forceinline
static int
79 __kmp_acquire_tas_lock_timed_template(kmp_tas_lock_t *lck, kmp_int32 gtid) {
82 #ifdef USE_LOCK_PROFILE
83 kmp_uint32 curr = KMP_LOCK_STRIP(lck->lk.poll);
84 if ((curr != 0) && (curr != gtid + 1))
85 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
89 kmp_int32 tas_free = KMP_LOCK_FREE(tas);
90 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas);
92 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == tas_free &&
93 __kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy)) {
94 KMP_FSYNC_ACQUIRED(lck);
95 return KMP_LOCK_ACQUIRED_FIRST;
99 KMP_FSYNC_PREPARE(lck);
100 KMP_INIT_YIELD(spins);
101 kmp_backoff_t backoff = __kmp_spin_backoff_params;
103 __kmp_spin_backoff(&backoff);
104 KMP_YIELD_OVERSUB_ELSE_SPIN(spins);
105 }
while (KMP_ATOMIC_LD_RLX(&lck->lk.poll) != tas_free ||
106 !__kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy));
107 KMP_FSYNC_ACQUIRED(lck);
108 return KMP_LOCK_ACQUIRED_FIRST;
111 int __kmp_acquire_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
112 int retval = __kmp_acquire_tas_lock_timed_template(lck, gtid);
116 static int __kmp_acquire_tas_lock_with_checks(kmp_tas_lock_t *lck,
118 char const *
const func =
"omp_set_lock";
119 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
120 __kmp_is_tas_lock_nestable(lck)) {
121 KMP_FATAL(LockNestableUsedAsSimple, func);
123 if ((gtid >= 0) && (__kmp_get_tas_lock_owner(lck) == gtid)) {
124 KMP_FATAL(LockIsAlreadyOwned, func);
126 return __kmp_acquire_tas_lock(lck, gtid);
129 int __kmp_test_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
130 kmp_int32 tas_free = KMP_LOCK_FREE(tas);
131 kmp_int32 tas_busy = KMP_LOCK_BUSY(gtid + 1, tas);
132 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == tas_free &&
133 __kmp_atomic_compare_store_acq(&lck->lk.poll, tas_free, tas_busy)) {
134 KMP_FSYNC_ACQUIRED(lck);
140 static int __kmp_test_tas_lock_with_checks(kmp_tas_lock_t *lck,
142 char const *
const func =
"omp_test_lock";
143 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
144 __kmp_is_tas_lock_nestable(lck)) {
145 KMP_FATAL(LockNestableUsedAsSimple, func);
147 return __kmp_test_tas_lock(lck, gtid);
150 int __kmp_release_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
153 KMP_FSYNC_RELEASING(lck);
154 KMP_ATOMIC_ST_REL(&lck->lk.poll, KMP_LOCK_FREE(tas));
158 return KMP_LOCK_RELEASED;
161 static int __kmp_release_tas_lock_with_checks(kmp_tas_lock_t *lck,
163 char const *
const func =
"omp_unset_lock";
165 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
166 __kmp_is_tas_lock_nestable(lck)) {
167 KMP_FATAL(LockNestableUsedAsSimple, func);
169 if (__kmp_get_tas_lock_owner(lck) == -1) {
170 KMP_FATAL(LockUnsettingFree, func);
172 if ((gtid >= 0) && (__kmp_get_tas_lock_owner(lck) >= 0) &&
173 (__kmp_get_tas_lock_owner(lck) != gtid)) {
174 KMP_FATAL(LockUnsettingSetByAnother, func);
176 return __kmp_release_tas_lock(lck, gtid);
179 void __kmp_init_tas_lock(kmp_tas_lock_t *lck) {
180 lck->lk.poll = KMP_LOCK_FREE(tas);
183 void __kmp_destroy_tas_lock(kmp_tas_lock_t *lck) { lck->lk.poll = 0; }
185 static void __kmp_destroy_tas_lock_with_checks(kmp_tas_lock_t *lck) {
186 char const *
const func =
"omp_destroy_lock";
187 if ((
sizeof(kmp_tas_lock_t) <= OMP_LOCK_T_SIZE) &&
188 __kmp_is_tas_lock_nestable(lck)) {
189 KMP_FATAL(LockNestableUsedAsSimple, func);
191 if (__kmp_get_tas_lock_owner(lck) != -1) {
192 KMP_FATAL(LockStillOwned, func);
194 __kmp_destroy_tas_lock(lck);
199 int __kmp_acquire_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
200 KMP_DEBUG_ASSERT(gtid >= 0);
202 if (__kmp_get_tas_lock_owner(lck) == gtid) {
203 lck->lk.depth_locked += 1;
204 return KMP_LOCK_ACQUIRED_NEXT;
206 __kmp_acquire_tas_lock_timed_template(lck, gtid);
207 lck->lk.depth_locked = 1;
208 return KMP_LOCK_ACQUIRED_FIRST;
212 static int __kmp_acquire_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
214 char const *
const func =
"omp_set_nest_lock";
215 if (!__kmp_is_tas_lock_nestable(lck)) {
216 KMP_FATAL(LockSimpleUsedAsNestable, func);
218 return __kmp_acquire_nested_tas_lock(lck, gtid);
221 int __kmp_test_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
224 KMP_DEBUG_ASSERT(gtid >= 0);
226 if (__kmp_get_tas_lock_owner(lck) == gtid) {
227 retval = ++lck->lk.depth_locked;
228 }
else if (!__kmp_test_tas_lock(lck, gtid)) {
232 retval = lck->lk.depth_locked = 1;
237 static int __kmp_test_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
239 char const *
const func =
"omp_test_nest_lock";
240 if (!__kmp_is_tas_lock_nestable(lck)) {
241 KMP_FATAL(LockSimpleUsedAsNestable, func);
243 return __kmp_test_nested_tas_lock(lck, gtid);
246 int __kmp_release_nested_tas_lock(kmp_tas_lock_t *lck, kmp_int32 gtid) {
247 KMP_DEBUG_ASSERT(gtid >= 0);
250 if (--(lck->lk.depth_locked) == 0) {
251 __kmp_release_tas_lock(lck, gtid);
252 return KMP_LOCK_RELEASED;
254 return KMP_LOCK_STILL_HELD;
257 static int __kmp_release_nested_tas_lock_with_checks(kmp_tas_lock_t *lck,
259 char const *
const func =
"omp_unset_nest_lock";
261 if (!__kmp_is_tas_lock_nestable(lck)) {
262 KMP_FATAL(LockSimpleUsedAsNestable, func);
264 if (__kmp_get_tas_lock_owner(lck) == -1) {
265 KMP_FATAL(LockUnsettingFree, func);
267 if (__kmp_get_tas_lock_owner(lck) != gtid) {
268 KMP_FATAL(LockUnsettingSetByAnother, func);
270 return __kmp_release_nested_tas_lock(lck, gtid);
273 void __kmp_init_nested_tas_lock(kmp_tas_lock_t *lck) {
274 __kmp_init_tas_lock(lck);
275 lck->lk.depth_locked = 0;
278 void __kmp_destroy_nested_tas_lock(kmp_tas_lock_t *lck) {
279 __kmp_destroy_tas_lock(lck);
280 lck->lk.depth_locked = 0;
283 static void __kmp_destroy_nested_tas_lock_with_checks(kmp_tas_lock_t *lck) {
284 char const *
const func =
"omp_destroy_nest_lock";
285 if (!__kmp_is_tas_lock_nestable(lck)) {
286 KMP_FATAL(LockSimpleUsedAsNestable, func);
288 if (__kmp_get_tas_lock_owner(lck) != -1) {
289 KMP_FATAL(LockStillOwned, func);
291 __kmp_destroy_nested_tas_lock(lck);
304 static kmp_int32 __kmp_get_futex_lock_owner(kmp_futex_lock_t *lck) {
305 return KMP_LOCK_STRIP((TCR_4(lck->lk.poll) >> 1)) - 1;
308 static inline bool __kmp_is_futex_lock_nestable(kmp_futex_lock_t *lck) {
309 return lck->lk.depth_locked != -1;
312 __forceinline
static int
313 __kmp_acquire_futex_lock_timed_template(kmp_futex_lock_t *lck, kmp_int32 gtid) {
314 kmp_int32 gtid_code = (gtid + 1) << 1;
318 #ifdef USE_LOCK_PROFILE
319 kmp_uint32 curr = KMP_LOCK_STRIP(TCR_4(lck->lk.poll));
320 if ((curr != 0) && (curr != gtid_code))
321 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
325 KMP_FSYNC_PREPARE(lck);
326 KA_TRACE(1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d entering\n",
327 lck, lck->lk.poll, gtid));
331 while ((poll_val = KMP_COMPARE_AND_STORE_RET32(
332 &(lck->lk.poll), KMP_LOCK_FREE(futex),
333 KMP_LOCK_BUSY(gtid_code, futex))) != KMP_LOCK_FREE(futex)) {
335 kmp_int32 cond = KMP_LOCK_STRIP(poll_val) & 1;
338 (
"__kmp_acquire_futex_lock: lck:%p, T#%d poll_val = 0x%x cond = 0x%x\n",
339 lck, gtid, poll_val, cond));
350 if (!KMP_COMPARE_AND_STORE_REL32(&(lck->lk.poll), poll_val,
351 poll_val | KMP_LOCK_BUSY(1, futex))) {
354 (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d can't set bit 0\n",
355 lck, lck->lk.poll, gtid));
358 poll_val |= KMP_LOCK_BUSY(1, futex);
361 (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d bit 0 set\n", lck,
362 lck->lk.poll, gtid));
367 (
"__kmp_acquire_futex_lock: lck:%p, T#%d before futex_wait(0x%x)\n",
368 lck, gtid, poll_val));
371 if ((rc = syscall(__NR_futex, &(lck->lk.poll), FUTEX_WAIT, poll_val, NULL,
373 KA_TRACE(1000, (
"__kmp_acquire_futex_lock: lck:%p, T#%d futex_wait(0x%x) "
374 "failed (rc=%ld errno=%d)\n",
375 lck, gtid, poll_val, rc, errno));
380 (
"__kmp_acquire_futex_lock: lck:%p, T#%d after futex_wait(0x%x)\n",
381 lck, gtid, poll_val));
388 KMP_FSYNC_ACQUIRED(lck);
389 KA_TRACE(1000, (
"__kmp_acquire_futex_lock: lck:%p(0x%x), T#%d exiting\n", lck,
390 lck->lk.poll, gtid));
391 return KMP_LOCK_ACQUIRED_FIRST;
394 int __kmp_acquire_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
395 int retval = __kmp_acquire_futex_lock_timed_template(lck, gtid);
399 static int __kmp_acquire_futex_lock_with_checks(kmp_futex_lock_t *lck,
401 char const *
const func =
"omp_set_lock";
402 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
403 __kmp_is_futex_lock_nestable(lck)) {
404 KMP_FATAL(LockNestableUsedAsSimple, func);
406 if ((gtid >= 0) && (__kmp_get_futex_lock_owner(lck) == gtid)) {
407 KMP_FATAL(LockIsAlreadyOwned, func);
409 return __kmp_acquire_futex_lock(lck, gtid);
412 int __kmp_test_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
413 if (KMP_COMPARE_AND_STORE_ACQ32(&(lck->lk.poll), KMP_LOCK_FREE(futex),
414 KMP_LOCK_BUSY((gtid + 1) << 1, futex))) {
415 KMP_FSYNC_ACQUIRED(lck);
421 static int __kmp_test_futex_lock_with_checks(kmp_futex_lock_t *lck,
423 char const *
const func =
"omp_test_lock";
424 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
425 __kmp_is_futex_lock_nestable(lck)) {
426 KMP_FATAL(LockNestableUsedAsSimple, func);
428 return __kmp_test_futex_lock(lck, gtid);
431 int __kmp_release_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
434 KA_TRACE(1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d entering\n",
435 lck, lck->lk.poll, gtid));
437 KMP_FSYNC_RELEASING(lck);
439 kmp_int32 poll_val = KMP_XCHG_FIXED32(&(lck->lk.poll), KMP_LOCK_FREE(futex));
442 (
"__kmp_release_futex_lock: lck:%p, T#%d released poll_val = 0x%x\n",
443 lck, gtid, poll_val));
445 if (KMP_LOCK_STRIP(poll_val) & 1) {
447 (
"__kmp_release_futex_lock: lck:%p, T#%d futex_wake 1 thread\n",
449 syscall(__NR_futex, &(lck->lk.poll), FUTEX_WAKE, KMP_LOCK_BUSY(1, futex),
455 KA_TRACE(1000, (
"__kmp_release_futex_lock: lck:%p(0x%x), T#%d exiting\n", lck,
456 lck->lk.poll, gtid));
459 return KMP_LOCK_RELEASED;
462 static int __kmp_release_futex_lock_with_checks(kmp_futex_lock_t *lck,
464 char const *
const func =
"omp_unset_lock";
466 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
467 __kmp_is_futex_lock_nestable(lck)) {
468 KMP_FATAL(LockNestableUsedAsSimple, func);
470 if (__kmp_get_futex_lock_owner(lck) == -1) {
471 KMP_FATAL(LockUnsettingFree, func);
473 if ((gtid >= 0) && (__kmp_get_futex_lock_owner(lck) >= 0) &&
474 (__kmp_get_futex_lock_owner(lck) != gtid)) {
475 KMP_FATAL(LockUnsettingSetByAnother, func);
477 return __kmp_release_futex_lock(lck, gtid);
480 void __kmp_init_futex_lock(kmp_futex_lock_t *lck) {
481 TCW_4(lck->lk.poll, KMP_LOCK_FREE(futex));
484 void __kmp_destroy_futex_lock(kmp_futex_lock_t *lck) { lck->lk.poll = 0; }
486 static void __kmp_destroy_futex_lock_with_checks(kmp_futex_lock_t *lck) {
487 char const *
const func =
"omp_destroy_lock";
488 if ((
sizeof(kmp_futex_lock_t) <= OMP_LOCK_T_SIZE) &&
489 __kmp_is_futex_lock_nestable(lck)) {
490 KMP_FATAL(LockNestableUsedAsSimple, func);
492 if (__kmp_get_futex_lock_owner(lck) != -1) {
493 KMP_FATAL(LockStillOwned, func);
495 __kmp_destroy_futex_lock(lck);
500 int __kmp_acquire_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
501 KMP_DEBUG_ASSERT(gtid >= 0);
503 if (__kmp_get_futex_lock_owner(lck) == gtid) {
504 lck->lk.depth_locked += 1;
505 return KMP_LOCK_ACQUIRED_NEXT;
507 __kmp_acquire_futex_lock_timed_template(lck, gtid);
508 lck->lk.depth_locked = 1;
509 return KMP_LOCK_ACQUIRED_FIRST;
513 static int __kmp_acquire_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
515 char const *
const func =
"omp_set_nest_lock";
516 if (!__kmp_is_futex_lock_nestable(lck)) {
517 KMP_FATAL(LockSimpleUsedAsNestable, func);
519 return __kmp_acquire_nested_futex_lock(lck, gtid);
522 int __kmp_test_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
525 KMP_DEBUG_ASSERT(gtid >= 0);
527 if (__kmp_get_futex_lock_owner(lck) == gtid) {
528 retval = ++lck->lk.depth_locked;
529 }
else if (!__kmp_test_futex_lock(lck, gtid)) {
533 retval = lck->lk.depth_locked = 1;
538 static int __kmp_test_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
540 char const *
const func =
"omp_test_nest_lock";
541 if (!__kmp_is_futex_lock_nestable(lck)) {
542 KMP_FATAL(LockSimpleUsedAsNestable, func);
544 return __kmp_test_nested_futex_lock(lck, gtid);
547 int __kmp_release_nested_futex_lock(kmp_futex_lock_t *lck, kmp_int32 gtid) {
548 KMP_DEBUG_ASSERT(gtid >= 0);
551 if (--(lck->lk.depth_locked) == 0) {
552 __kmp_release_futex_lock(lck, gtid);
553 return KMP_LOCK_RELEASED;
555 return KMP_LOCK_STILL_HELD;
558 static int __kmp_release_nested_futex_lock_with_checks(kmp_futex_lock_t *lck,
560 char const *
const func =
"omp_unset_nest_lock";
562 if (!__kmp_is_futex_lock_nestable(lck)) {
563 KMP_FATAL(LockSimpleUsedAsNestable, func);
565 if (__kmp_get_futex_lock_owner(lck) == -1) {
566 KMP_FATAL(LockUnsettingFree, func);
568 if (__kmp_get_futex_lock_owner(lck) != gtid) {
569 KMP_FATAL(LockUnsettingSetByAnother, func);
571 return __kmp_release_nested_futex_lock(lck, gtid);
574 void __kmp_init_nested_futex_lock(kmp_futex_lock_t *lck) {
575 __kmp_init_futex_lock(lck);
576 lck->lk.depth_locked = 0;
579 void __kmp_destroy_nested_futex_lock(kmp_futex_lock_t *lck) {
580 __kmp_destroy_futex_lock(lck);
581 lck->lk.depth_locked = 0;
584 static void __kmp_destroy_nested_futex_lock_with_checks(kmp_futex_lock_t *lck) {
585 char const *
const func =
"omp_destroy_nest_lock";
586 if (!__kmp_is_futex_lock_nestable(lck)) {
587 KMP_FATAL(LockSimpleUsedAsNestable, func);
589 if (__kmp_get_futex_lock_owner(lck) != -1) {
590 KMP_FATAL(LockStillOwned, func);
592 __kmp_destroy_nested_futex_lock(lck);
600 static kmp_int32 __kmp_get_ticket_lock_owner(kmp_ticket_lock_t *lck) {
601 return std::atomic_load_explicit(&lck->lk.owner_id,
602 std::memory_order_relaxed) -
606 static inline bool __kmp_is_ticket_lock_nestable(kmp_ticket_lock_t *lck) {
607 return std::atomic_load_explicit(&lck->lk.depth_locked,
608 std::memory_order_relaxed) != -1;
611 static kmp_uint32 __kmp_bakery_check(
void *now_serving, kmp_uint32 my_ticket) {
612 return std::atomic_load_explicit((std::atomic<unsigned> *)now_serving,
613 std::memory_order_acquire) == my_ticket;
616 __forceinline
static int
617 __kmp_acquire_ticket_lock_timed_template(kmp_ticket_lock_t *lck,
619 kmp_uint32 my_ticket = std::atomic_fetch_add_explicit(
620 &lck->lk.next_ticket, 1U, std::memory_order_relaxed);
622 #ifdef USE_LOCK_PROFILE
623 if (std::atomic_load_explicit(&lck->lk.now_serving,
624 std::memory_order_relaxed) != my_ticket)
625 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
629 if (std::atomic_load_explicit(&lck->lk.now_serving,
630 std::memory_order_acquire) == my_ticket) {
631 return KMP_LOCK_ACQUIRED_FIRST;
633 KMP_WAIT_PTR(&lck->lk.now_serving, my_ticket, __kmp_bakery_check, lck);
634 return KMP_LOCK_ACQUIRED_FIRST;
637 int __kmp_acquire_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
638 int retval = __kmp_acquire_ticket_lock_timed_template(lck, gtid);
642 static int __kmp_acquire_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
644 char const *
const func =
"omp_set_lock";
646 if (!std::atomic_load_explicit(&lck->lk.initialized,
647 std::memory_order_relaxed)) {
648 KMP_FATAL(LockIsUninitialized, func);
650 if (lck->lk.self != lck) {
651 KMP_FATAL(LockIsUninitialized, func);
653 if (__kmp_is_ticket_lock_nestable(lck)) {
654 KMP_FATAL(LockNestableUsedAsSimple, func);
656 if ((gtid >= 0) && (__kmp_get_ticket_lock_owner(lck) == gtid)) {
657 KMP_FATAL(LockIsAlreadyOwned, func);
660 __kmp_acquire_ticket_lock(lck, gtid);
662 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
663 std::memory_order_relaxed);
664 return KMP_LOCK_ACQUIRED_FIRST;
667 int __kmp_test_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
668 kmp_uint32 my_ticket = std::atomic_load_explicit(&lck->lk.next_ticket,
669 std::memory_order_relaxed);
671 if (std::atomic_load_explicit(&lck->lk.now_serving,
672 std::memory_order_relaxed) == my_ticket) {
673 kmp_uint32 next_ticket = my_ticket + 1;
674 if (std::atomic_compare_exchange_strong_explicit(
675 &lck->lk.next_ticket, &my_ticket, next_ticket,
676 std::memory_order_acquire, std::memory_order_acquire)) {
683 static int __kmp_test_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
685 char const *
const func =
"omp_test_lock";
687 if (!std::atomic_load_explicit(&lck->lk.initialized,
688 std::memory_order_relaxed)) {
689 KMP_FATAL(LockIsUninitialized, func);
691 if (lck->lk.self != lck) {
692 KMP_FATAL(LockIsUninitialized, func);
694 if (__kmp_is_ticket_lock_nestable(lck)) {
695 KMP_FATAL(LockNestableUsedAsSimple, func);
698 int retval = __kmp_test_ticket_lock(lck, gtid);
701 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
702 std::memory_order_relaxed);
707 int __kmp_release_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
708 kmp_uint32 distance = std::atomic_load_explicit(&lck->lk.next_ticket,
709 std::memory_order_relaxed) -
710 std::atomic_load_explicit(&lck->lk.now_serving,
711 std::memory_order_relaxed);
713 std::atomic_fetch_add_explicit(&lck->lk.now_serving, 1U,
714 std::memory_order_release);
717 (kmp_uint32)(__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc));
718 return KMP_LOCK_RELEASED;
721 static int __kmp_release_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
723 char const *
const func =
"omp_unset_lock";
725 if (!std::atomic_load_explicit(&lck->lk.initialized,
726 std::memory_order_relaxed)) {
727 KMP_FATAL(LockIsUninitialized, func);
729 if (lck->lk.self != lck) {
730 KMP_FATAL(LockIsUninitialized, func);
732 if (__kmp_is_ticket_lock_nestable(lck)) {
733 KMP_FATAL(LockNestableUsedAsSimple, func);
735 if (__kmp_get_ticket_lock_owner(lck) == -1) {
736 KMP_FATAL(LockUnsettingFree, func);
738 if ((gtid >= 0) && (__kmp_get_ticket_lock_owner(lck) >= 0) &&
739 (__kmp_get_ticket_lock_owner(lck) != gtid)) {
740 KMP_FATAL(LockUnsettingSetByAnother, func);
742 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
743 return __kmp_release_ticket_lock(lck, gtid);
746 void __kmp_init_ticket_lock(kmp_ticket_lock_t *lck) {
747 lck->lk.location = NULL;
749 std::atomic_store_explicit(&lck->lk.next_ticket, 0U,
750 std::memory_order_relaxed);
751 std::atomic_store_explicit(&lck->lk.now_serving, 0U,
752 std::memory_order_relaxed);
753 std::atomic_store_explicit(
754 &lck->lk.owner_id, 0,
755 std::memory_order_relaxed);
756 std::atomic_store_explicit(
757 &lck->lk.depth_locked, -1,
758 std::memory_order_relaxed);
759 std::atomic_store_explicit(&lck->lk.initialized,
true,
760 std::memory_order_release);
763 void __kmp_destroy_ticket_lock(kmp_ticket_lock_t *lck) {
764 std::atomic_store_explicit(&lck->lk.initialized,
false,
765 std::memory_order_release);
767 lck->lk.location = NULL;
768 std::atomic_store_explicit(&lck->lk.next_ticket, 0U,
769 std::memory_order_relaxed);
770 std::atomic_store_explicit(&lck->lk.now_serving, 0U,
771 std::memory_order_relaxed);
772 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
773 std::atomic_store_explicit(&lck->lk.depth_locked, -1,
774 std::memory_order_relaxed);
777 static void __kmp_destroy_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
778 char const *
const func =
"omp_destroy_lock";
780 if (!std::atomic_load_explicit(&lck->lk.initialized,
781 std::memory_order_relaxed)) {
782 KMP_FATAL(LockIsUninitialized, func);
784 if (lck->lk.self != lck) {
785 KMP_FATAL(LockIsUninitialized, func);
787 if (__kmp_is_ticket_lock_nestable(lck)) {
788 KMP_FATAL(LockNestableUsedAsSimple, func);
790 if (__kmp_get_ticket_lock_owner(lck) != -1) {
791 KMP_FATAL(LockStillOwned, func);
793 __kmp_destroy_ticket_lock(lck);
798 int __kmp_acquire_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
799 KMP_DEBUG_ASSERT(gtid >= 0);
801 if (__kmp_get_ticket_lock_owner(lck) == gtid) {
802 std::atomic_fetch_add_explicit(&lck->lk.depth_locked, 1,
803 std::memory_order_relaxed);
804 return KMP_LOCK_ACQUIRED_NEXT;
806 __kmp_acquire_ticket_lock_timed_template(lck, gtid);
807 std::atomic_store_explicit(&lck->lk.depth_locked, 1,
808 std::memory_order_relaxed);
809 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
810 std::memory_order_relaxed);
811 return KMP_LOCK_ACQUIRED_FIRST;
815 static int __kmp_acquire_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
817 char const *
const func =
"omp_set_nest_lock";
819 if (!std::atomic_load_explicit(&lck->lk.initialized,
820 std::memory_order_relaxed)) {
821 KMP_FATAL(LockIsUninitialized, func);
823 if (lck->lk.self != lck) {
824 KMP_FATAL(LockIsUninitialized, func);
826 if (!__kmp_is_ticket_lock_nestable(lck)) {
827 KMP_FATAL(LockSimpleUsedAsNestable, func);
829 return __kmp_acquire_nested_ticket_lock(lck, gtid);
832 int __kmp_test_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
835 KMP_DEBUG_ASSERT(gtid >= 0);
837 if (__kmp_get_ticket_lock_owner(lck) == gtid) {
838 retval = std::atomic_fetch_add_explicit(&lck->lk.depth_locked, 1,
839 std::memory_order_relaxed) +
841 }
else if (!__kmp_test_ticket_lock(lck, gtid)) {
844 std::atomic_store_explicit(&lck->lk.depth_locked, 1,
845 std::memory_order_relaxed);
846 std::atomic_store_explicit(&lck->lk.owner_id, gtid + 1,
847 std::memory_order_relaxed);
853 static int __kmp_test_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
855 char const *
const func =
"omp_test_nest_lock";
857 if (!std::atomic_load_explicit(&lck->lk.initialized,
858 std::memory_order_relaxed)) {
859 KMP_FATAL(LockIsUninitialized, func);
861 if (lck->lk.self != lck) {
862 KMP_FATAL(LockIsUninitialized, func);
864 if (!__kmp_is_ticket_lock_nestable(lck)) {
865 KMP_FATAL(LockSimpleUsedAsNestable, func);
867 return __kmp_test_nested_ticket_lock(lck, gtid);
870 int __kmp_release_nested_ticket_lock(kmp_ticket_lock_t *lck, kmp_int32 gtid) {
871 KMP_DEBUG_ASSERT(gtid >= 0);
873 if ((std::atomic_fetch_add_explicit(&lck->lk.depth_locked, -1,
874 std::memory_order_relaxed) -
876 std::atomic_store_explicit(&lck->lk.owner_id, 0, std::memory_order_relaxed);
877 __kmp_release_ticket_lock(lck, gtid);
878 return KMP_LOCK_RELEASED;
880 return KMP_LOCK_STILL_HELD;
883 static int __kmp_release_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck,
885 char const *
const func =
"omp_unset_nest_lock";
887 if (!std::atomic_load_explicit(&lck->lk.initialized,
888 std::memory_order_relaxed)) {
889 KMP_FATAL(LockIsUninitialized, func);
891 if (lck->lk.self != lck) {
892 KMP_FATAL(LockIsUninitialized, func);
894 if (!__kmp_is_ticket_lock_nestable(lck)) {
895 KMP_FATAL(LockSimpleUsedAsNestable, func);
897 if (__kmp_get_ticket_lock_owner(lck) == -1) {
898 KMP_FATAL(LockUnsettingFree, func);
900 if (__kmp_get_ticket_lock_owner(lck) != gtid) {
901 KMP_FATAL(LockUnsettingSetByAnother, func);
903 return __kmp_release_nested_ticket_lock(lck, gtid);
906 void __kmp_init_nested_ticket_lock(kmp_ticket_lock_t *lck) {
907 __kmp_init_ticket_lock(lck);
908 std::atomic_store_explicit(&lck->lk.depth_locked, 0,
909 std::memory_order_relaxed);
913 void __kmp_destroy_nested_ticket_lock(kmp_ticket_lock_t *lck) {
914 __kmp_destroy_ticket_lock(lck);
915 std::atomic_store_explicit(&lck->lk.depth_locked, 0,
916 std::memory_order_relaxed);
920 __kmp_destroy_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
921 char const *
const func =
"omp_destroy_nest_lock";
923 if (!std::atomic_load_explicit(&lck->lk.initialized,
924 std::memory_order_relaxed)) {
925 KMP_FATAL(LockIsUninitialized, func);
927 if (lck->lk.self != lck) {
928 KMP_FATAL(LockIsUninitialized, func);
930 if (!__kmp_is_ticket_lock_nestable(lck)) {
931 KMP_FATAL(LockSimpleUsedAsNestable, func);
933 if (__kmp_get_ticket_lock_owner(lck) != -1) {
934 KMP_FATAL(LockStillOwned, func);
936 __kmp_destroy_nested_ticket_lock(lck);
941 static const ident_t *__kmp_get_ticket_lock_location(kmp_ticket_lock_t *lck) {
942 return lck->lk.location;
945 static void __kmp_set_ticket_lock_location(kmp_ticket_lock_t *lck,
947 lck->lk.location = loc;
950 static kmp_lock_flags_t __kmp_get_ticket_lock_flags(kmp_ticket_lock_t *lck) {
951 return lck->lk.flags;
954 static void __kmp_set_ticket_lock_flags(kmp_ticket_lock_t *lck,
955 kmp_lock_flags_t flags) {
956 lck->lk.flags = flags;
1014 #ifdef DEBUG_QUEUING_LOCKS
1017 #define TRACE_BUF_ELE 1024
1018 static char traces[TRACE_BUF_ELE][128] = {0};
1020 #define TRACE_LOCK(X, Y) \
1021 KMP_SNPRINTF(traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s\n", X, Y);
1022 #define TRACE_LOCK_T(X, Y, Z) \
1023 KMP_SNPRINTF(traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s%d\n", X, Y, Z);
1024 #define TRACE_LOCK_HT(X, Y, Z, Q) \
1025 KMP_SNPRINTF(traces[tc++ % TRACE_BUF_ELE], 128, "t%d at %s %d,%d\n", X, Y, \
1028 static void __kmp_dump_queuing_lock(kmp_info_t *this_thr, kmp_int32 gtid,
1029 kmp_queuing_lock_t *lck, kmp_int32 head_id,
1030 kmp_int32 tail_id) {
1033 __kmp_printf_no_lock(
"\n__kmp_dump_queuing_lock: TRACE BEGINS HERE! \n");
1035 i = tc % TRACE_BUF_ELE;
1036 __kmp_printf_no_lock(
"%s\n", traces[i]);
1037 i = (i + 1) % TRACE_BUF_ELE;
1038 while (i != (tc % TRACE_BUF_ELE)) {
1039 __kmp_printf_no_lock(
"%s", traces[i]);
1040 i = (i + 1) % TRACE_BUF_ELE;
1042 __kmp_printf_no_lock(
"\n");
1044 __kmp_printf_no_lock(
"\n__kmp_dump_queuing_lock: gtid+1:%d, spin_here:%d, "
1045 "next_wait:%d, head_id:%d, tail_id:%d\n",
1046 gtid + 1, this_thr->th.th_spin_here,
1047 this_thr->th.th_next_waiting, head_id, tail_id);
1049 __kmp_printf_no_lock(
"\t\thead: %d ", lck->lk.head_id);
1051 if (lck->lk.head_id >= 1) {
1052 t = __kmp_threads[lck->lk.head_id - 1]->th.th_next_waiting;
1054 __kmp_printf_no_lock(
"-> %d ", t);
1055 t = __kmp_threads[t - 1]->th.th_next_waiting;
1058 __kmp_printf_no_lock(
"; tail: %d ", lck->lk.tail_id);
1059 __kmp_printf_no_lock(
"\n\n");
1064 static kmp_int32 __kmp_get_queuing_lock_owner(kmp_queuing_lock_t *lck) {
1065 return TCR_4(lck->lk.owner_id) - 1;
1068 static inline bool __kmp_is_queuing_lock_nestable(kmp_queuing_lock_t *lck) {
1069 return lck->lk.depth_locked != -1;
1073 template <
bool takeTime>
1076 __forceinline
static int
1077 __kmp_acquire_queuing_lock_timed_template(kmp_queuing_lock_t *lck,
1079 kmp_info_t *this_thr = __kmp_thread_from_gtid(gtid);
1080 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1081 volatile kmp_int32 *tail_id_p = &lck->lk.tail_id;
1082 volatile kmp_uint32 *spin_here_p;
1085 ompt_state_t prev_state = ompt_state_undefined;
1089 (
"__kmp_acquire_queuing_lock: lck:%p, T#%d entering\n", lck, gtid));
1091 KMP_FSYNC_PREPARE(lck);
1092 KMP_DEBUG_ASSERT(this_thr != NULL);
1093 spin_here_p = &this_thr->th.th_spin_here;
1095 #ifdef DEBUG_QUEUING_LOCKS
1096 TRACE_LOCK(gtid + 1,
"acq ent");
1098 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1099 if (this_thr->th.th_next_waiting != 0)
1100 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1102 KMP_DEBUG_ASSERT(!*spin_here_p);
1103 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
1112 *spin_here_p = TRUE;
1124 #ifdef DEBUG_QUEUING_LOCKS
1126 TRACE_LOCK_HT(gtid + 1,
"acq read: ", head, tail);
1134 enqueued = KMP_COMPARE_AND_STORE_ACQ64((
volatile kmp_int64 *)tail_id_p,
1136 KMP_PACK_64(gtid + 1, gtid + 1));
1137 #ifdef DEBUG_QUEUING_LOCKS
1139 TRACE_LOCK(gtid + 1,
"acq enq: (-1,0)->(tid,tid)");
1145 KMP_DEBUG_ASSERT(tail != gtid + 1);
1147 #ifdef DEBUG_QUEUING_LOCKS
1148 TRACE_LOCK_HT(gtid + 1,
"acq read: ", head, tail);
1155 enqueued = KMP_COMPARE_AND_STORE_ACQ32(tail_id_p, tail, gtid + 1);
1157 #ifdef DEBUG_QUEUING_LOCKS
1159 TRACE_LOCK(gtid + 1,
"acq enq: (h,t)->(h,tid)");
1166 kmp_int32 grabbed_lock;
1168 #ifdef DEBUG_QUEUING_LOCKS
1170 TRACE_LOCK_HT(gtid + 1,
"acq read: ", head, tail);
1176 grabbed_lock = KMP_COMPARE_AND_STORE_ACQ32(head_id_p, 0, -1);
1180 *spin_here_p = FALSE;
1184 (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: no queuing\n",
1186 #ifdef DEBUG_QUEUING_LOCKS
1187 TRACE_LOCK_HT(gtid + 1,
"acq exit: ", head, 0);
1191 if (ompt_enabled.enabled && prev_state != ompt_state_undefined) {
1193 this_thr->th.ompt_thread_info.state = prev_state;
1194 this_thr->th.ompt_thread_info.wait_id = 0;
1198 KMP_FSYNC_ACQUIRED(lck);
1199 return KMP_LOCK_ACQUIRED_FIRST;
1206 if (ompt_enabled.enabled && prev_state == ompt_state_undefined) {
1208 prev_state = this_thr->th.ompt_thread_info.state;
1209 this_thr->th.ompt_thread_info.wait_id = (uint64_t)lck;
1210 this_thr->th.ompt_thread_info.state = ompt_state_wait_lock;
1216 kmp_info_t *tail_thr = __kmp_thread_from_gtid(tail - 1);
1217 KMP_ASSERT(tail_thr != NULL);
1218 tail_thr->th.th_next_waiting = gtid + 1;
1222 (
"__kmp_acquire_queuing_lock: lck:%p, T#%d waiting for lock\n",
1227 KMP_WAIT(spin_here_p, FALSE, KMP_EQ, lck);
1232 #ifdef DEBUG_QUEUING_LOCKS
1233 TRACE_LOCK(gtid + 1,
"acq spin");
1235 if (this_thr->th.th_next_waiting != 0)
1236 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1238 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
1239 KA_TRACE(1000, (
"__kmp_acquire_queuing_lock: lck:%p, T#%d exiting: after "
1240 "waiting on queue\n",
1243 #ifdef DEBUG_QUEUING_LOCKS
1244 TRACE_LOCK(gtid + 1,
"acq exit 2");
1249 this_thr->th.ompt_thread_info.state = prev_state;
1250 this_thr->th.ompt_thread_info.wait_id = 0;
1254 return KMP_LOCK_ACQUIRED_FIRST;
1260 KMP_YIELD_OVERSUB();
1262 #ifdef DEBUG_QUEUING_LOCKS
1263 TRACE_LOCK(gtid + 1,
"acq retry");
1266 KMP_ASSERT2(0,
"should not get here");
1267 return KMP_LOCK_ACQUIRED_FIRST;
1270 int __kmp_acquire_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1271 KMP_DEBUG_ASSERT(gtid >= 0);
1273 int retval = __kmp_acquire_queuing_lock_timed_template<false>(lck, gtid);
1277 static int __kmp_acquire_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1279 char const *
const func =
"omp_set_lock";
1280 if (lck->lk.initialized != lck) {
1281 KMP_FATAL(LockIsUninitialized, func);
1283 if (__kmp_is_queuing_lock_nestable(lck)) {
1284 KMP_FATAL(LockNestableUsedAsSimple, func);
1286 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1287 KMP_FATAL(LockIsAlreadyOwned, func);
1290 __kmp_acquire_queuing_lock(lck, gtid);
1292 lck->lk.owner_id = gtid + 1;
1293 return KMP_LOCK_ACQUIRED_FIRST;
1296 int __kmp_test_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1297 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1300 kmp_info_t *this_thr;
1303 KA_TRACE(1000, (
"__kmp_test_queuing_lock: T#%d entering\n", gtid));
1304 KMP_DEBUG_ASSERT(gtid >= 0);
1306 this_thr = __kmp_thread_from_gtid(gtid);
1307 KMP_DEBUG_ASSERT(this_thr != NULL);
1308 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
1315 if (KMP_COMPARE_AND_STORE_ACQ32(head_id_p, 0, -1)) {
1317 (
"__kmp_test_queuing_lock: T#%d exiting: holding lock\n", gtid));
1318 KMP_FSYNC_ACQUIRED(lck);
1324 (
"__kmp_test_queuing_lock: T#%d exiting: without lock\n", gtid));
1328 static int __kmp_test_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1330 char const *
const func =
"omp_test_lock";
1331 if (lck->lk.initialized != lck) {
1332 KMP_FATAL(LockIsUninitialized, func);
1334 if (__kmp_is_queuing_lock_nestable(lck)) {
1335 KMP_FATAL(LockNestableUsedAsSimple, func);
1338 int retval = __kmp_test_queuing_lock(lck, gtid);
1341 lck->lk.owner_id = gtid + 1;
1346 int __kmp_release_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1347 volatile kmp_int32 *head_id_p = &lck->lk.head_id;
1348 volatile kmp_int32 *tail_id_p = &lck->lk.tail_id;
1351 (
"__kmp_release_queuing_lock: lck:%p, T#%d entering\n", lck, gtid));
1352 KMP_DEBUG_ASSERT(gtid >= 0);
1353 #if KMP_DEBUG || DEBUG_QUEUING_LOCKS
1354 kmp_info_t *this_thr = __kmp_thread_from_gtid(gtid);
1356 KMP_DEBUG_ASSERT(this_thr != NULL);
1357 #ifdef DEBUG_QUEUING_LOCKS
1358 TRACE_LOCK(gtid + 1,
"rel ent");
1360 if (this_thr->th.th_spin_here)
1361 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1362 if (this_thr->th.th_next_waiting != 0)
1363 __kmp_dump_queuing_lock(this_thr, gtid, lck, *head_id_p, *tail_id_p);
1365 KMP_DEBUG_ASSERT(!this_thr->th.th_spin_here);
1366 KMP_DEBUG_ASSERT(this_thr->th.th_next_waiting == 0);
1368 KMP_FSYNC_RELEASING(lck);
1377 #ifdef DEBUG_QUEUING_LOCKS
1379 TRACE_LOCK_HT(gtid + 1,
"rel read: ", head, tail);
1381 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1383 KMP_DEBUG_ASSERT(head !=
1388 if (KMP_COMPARE_AND_STORE_REL32(head_id_p, -1, 0)) {
1391 (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: queue empty\n",
1393 #ifdef DEBUG_QUEUING_LOCKS
1394 TRACE_LOCK_HT(gtid + 1,
"rel exit: ", 0, 0);
1400 return KMP_LOCK_RELEASED;
1407 #ifdef DEBUG_QUEUING_LOCKS
1409 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1411 KMP_DEBUG_ASSERT(head > 0);
1414 dequeued = KMP_COMPARE_AND_STORE_REL64(
1415 RCAST(
volatile kmp_int64 *, tail_id_p), KMP_PACK_64(head, head),
1416 KMP_PACK_64(-1, 0));
1417 #ifdef DEBUG_QUEUING_LOCKS
1418 TRACE_LOCK(gtid + 1,
"rel deq: (h,h)->(-1,0)");
1422 volatile kmp_int32 *waiting_id_p;
1423 kmp_info_t *head_thr = __kmp_thread_from_gtid(head - 1);
1424 KMP_DEBUG_ASSERT(head_thr != NULL);
1425 waiting_id_p = &head_thr->th.th_next_waiting;
1428 #ifdef DEBUG_QUEUING_LOCKS
1429 if (head <= 0 || tail <= 0)
1430 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1432 KMP_DEBUG_ASSERT(head > 0 && tail > 0);
1439 KMP_WAIT((
volatile kmp_uint32 *)waiting_id_p, 0, KMP_NEQ, NULL);
1440 #ifdef DEBUG_QUEUING_LOCKS
1441 TRACE_LOCK(gtid + 1,
"rel deq: (h,t)->(h',t)");
1448 kmp_info_t *head_thr = __kmp_thread_from_gtid(head - 1);
1449 KMP_DEBUG_ASSERT(head_thr != NULL);
1452 #ifdef DEBUG_QUEUING_LOCKS
1453 if (head <= 0 || tail <= 0)
1454 __kmp_dump_queuing_lock(this_thr, gtid, lck, head, tail);
1456 KMP_DEBUG_ASSERT(head > 0 && tail > 0);
1460 head_thr->th.th_next_waiting = 0;
1461 #ifdef DEBUG_QUEUING_LOCKS
1462 TRACE_LOCK_T(gtid + 1,
"rel nw=0 for t=", head);
1467 head_thr->th.th_spin_here = FALSE;
1469 KA_TRACE(1000, (
"__kmp_release_queuing_lock: lck:%p, T#%d exiting: after "
1472 #ifdef DEBUG_QUEUING_LOCKS
1473 TRACE_LOCK(gtid + 1,
"rel exit 2");
1475 return KMP_LOCK_RELEASED;
1480 #ifdef DEBUG_QUEUING_LOCKS
1481 TRACE_LOCK(gtid + 1,
"rel retry");
1485 KMP_ASSERT2(0,
"should not get here");
1486 return KMP_LOCK_RELEASED;
1489 static int __kmp_release_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1491 char const *
const func =
"omp_unset_lock";
1493 if (lck->lk.initialized != lck) {
1494 KMP_FATAL(LockIsUninitialized, func);
1496 if (__kmp_is_queuing_lock_nestable(lck)) {
1497 KMP_FATAL(LockNestableUsedAsSimple, func);
1499 if (__kmp_get_queuing_lock_owner(lck) == -1) {
1500 KMP_FATAL(LockUnsettingFree, func);
1502 if (__kmp_get_queuing_lock_owner(lck) != gtid) {
1503 KMP_FATAL(LockUnsettingSetByAnother, func);
1505 lck->lk.owner_id = 0;
1506 return __kmp_release_queuing_lock(lck, gtid);
1509 void __kmp_init_queuing_lock(kmp_queuing_lock_t *lck) {
1510 lck->lk.location = NULL;
1511 lck->lk.head_id = 0;
1512 lck->lk.tail_id = 0;
1513 lck->lk.next_ticket = 0;
1514 lck->lk.now_serving = 0;
1515 lck->lk.owner_id = 0;
1516 lck->lk.depth_locked = -1;
1517 lck->lk.initialized = lck;
1519 KA_TRACE(1000, (
"__kmp_init_queuing_lock: lock %p initialized\n", lck));
1522 void __kmp_destroy_queuing_lock(kmp_queuing_lock_t *lck) {
1523 lck->lk.initialized = NULL;
1524 lck->lk.location = NULL;
1525 lck->lk.head_id = 0;
1526 lck->lk.tail_id = 0;
1527 lck->lk.next_ticket = 0;
1528 lck->lk.now_serving = 0;
1529 lck->lk.owner_id = 0;
1530 lck->lk.depth_locked = -1;
1533 static void __kmp_destroy_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
1534 char const *
const func =
"omp_destroy_lock";
1535 if (lck->lk.initialized != lck) {
1536 KMP_FATAL(LockIsUninitialized, func);
1538 if (__kmp_is_queuing_lock_nestable(lck)) {
1539 KMP_FATAL(LockNestableUsedAsSimple, func);
1541 if (__kmp_get_queuing_lock_owner(lck) != -1) {
1542 KMP_FATAL(LockStillOwned, func);
1544 __kmp_destroy_queuing_lock(lck);
1549 int __kmp_acquire_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1550 KMP_DEBUG_ASSERT(gtid >= 0);
1552 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1553 lck->lk.depth_locked += 1;
1554 return KMP_LOCK_ACQUIRED_NEXT;
1556 __kmp_acquire_queuing_lock_timed_template<false>(lck, gtid);
1558 lck->lk.depth_locked = 1;
1560 lck->lk.owner_id = gtid + 1;
1561 return KMP_LOCK_ACQUIRED_FIRST;
1566 __kmp_acquire_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1568 char const *
const func =
"omp_set_nest_lock";
1569 if (lck->lk.initialized != lck) {
1570 KMP_FATAL(LockIsUninitialized, func);
1572 if (!__kmp_is_queuing_lock_nestable(lck)) {
1573 KMP_FATAL(LockSimpleUsedAsNestable, func);
1575 return __kmp_acquire_nested_queuing_lock(lck, gtid);
1578 int __kmp_test_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1581 KMP_DEBUG_ASSERT(gtid >= 0);
1583 if (__kmp_get_queuing_lock_owner(lck) == gtid) {
1584 retval = ++lck->lk.depth_locked;
1585 }
else if (!__kmp_test_queuing_lock(lck, gtid)) {
1589 retval = lck->lk.depth_locked = 1;
1591 lck->lk.owner_id = gtid + 1;
1596 static int __kmp_test_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1598 char const *
const func =
"omp_test_nest_lock";
1599 if (lck->lk.initialized != lck) {
1600 KMP_FATAL(LockIsUninitialized, func);
1602 if (!__kmp_is_queuing_lock_nestable(lck)) {
1603 KMP_FATAL(LockSimpleUsedAsNestable, func);
1605 return __kmp_test_nested_queuing_lock(lck, gtid);
1608 int __kmp_release_nested_queuing_lock(kmp_queuing_lock_t *lck, kmp_int32 gtid) {
1609 KMP_DEBUG_ASSERT(gtid >= 0);
1612 if (--(lck->lk.depth_locked) == 0) {
1614 lck->lk.owner_id = 0;
1615 __kmp_release_queuing_lock(lck, gtid);
1616 return KMP_LOCK_RELEASED;
1618 return KMP_LOCK_STILL_HELD;
1622 __kmp_release_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
1624 char const *
const func =
"omp_unset_nest_lock";
1626 if (lck->lk.initialized != lck) {
1627 KMP_FATAL(LockIsUninitialized, func);
1629 if (!__kmp_is_queuing_lock_nestable(lck)) {
1630 KMP_FATAL(LockSimpleUsedAsNestable, func);
1632 if (__kmp_get_queuing_lock_owner(lck) == -1) {
1633 KMP_FATAL(LockUnsettingFree, func);
1635 if (__kmp_get_queuing_lock_owner(lck) != gtid) {
1636 KMP_FATAL(LockUnsettingSetByAnother, func);
1638 return __kmp_release_nested_queuing_lock(lck, gtid);
1641 void __kmp_init_nested_queuing_lock(kmp_queuing_lock_t *lck) {
1642 __kmp_init_queuing_lock(lck);
1643 lck->lk.depth_locked = 0;
1646 void __kmp_destroy_nested_queuing_lock(kmp_queuing_lock_t *lck) {
1647 __kmp_destroy_queuing_lock(lck);
1648 lck->lk.depth_locked = 0;
1652 __kmp_destroy_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
1653 char const *
const func =
"omp_destroy_nest_lock";
1654 if (lck->lk.initialized != lck) {
1655 KMP_FATAL(LockIsUninitialized, func);
1657 if (!__kmp_is_queuing_lock_nestable(lck)) {
1658 KMP_FATAL(LockSimpleUsedAsNestable, func);
1660 if (__kmp_get_queuing_lock_owner(lck) != -1) {
1661 KMP_FATAL(LockStillOwned, func);
1663 __kmp_destroy_nested_queuing_lock(lck);
1668 static const ident_t *__kmp_get_queuing_lock_location(kmp_queuing_lock_t *lck) {
1669 return lck->lk.location;
1672 static void __kmp_set_queuing_lock_location(kmp_queuing_lock_t *lck,
1674 lck->lk.location = loc;
1677 static kmp_lock_flags_t __kmp_get_queuing_lock_flags(kmp_queuing_lock_t *lck) {
1678 return lck->lk.flags;
1681 static void __kmp_set_queuing_lock_flags(kmp_queuing_lock_t *lck,
1682 kmp_lock_flags_t flags) {
1683 lck->lk.flags = flags;
1686 #if KMP_USE_ADAPTIVE_LOCKS
1690 #if KMP_HAVE_RTM_INTRINSICS
1691 #include <immintrin.h>
1692 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT)
1697 #define _XBEGIN_STARTED (~0u)
1698 #define _XABORT_EXPLICIT (1 << 0)
1699 #define _XABORT_RETRY (1 << 1)
1700 #define _XABORT_CONFLICT (1 << 2)
1701 #define _XABORT_CAPACITY (1 << 3)
1702 #define _XABORT_DEBUG (1 << 4)
1703 #define _XABORT_NESTED (1 << 5)
1704 #define _XABORT_CODE(x) ((unsigned char)(((x) >> 24) & 0xFF))
1707 #define SOFT_ABORT_MASK (_XABORT_RETRY | _XABORT_CONFLICT | _XABORT_EXPLICIT)
1709 #define STRINGIZE_INTERNAL(arg) #arg
1710 #define STRINGIZE(arg) STRINGIZE_INTERNAL(arg)
1716 static __inline
int _xbegin() {
1753 __asm__
volatile(
"1: .byte 0xC7; .byte 0xF8;\n"
1756 "1: movl %%eax,%0\n"
1758 :
"+r"(res)::
"memory",
"%eax");
1764 static __inline
void _xend() {
1772 __asm__
volatile(
".byte 0x0f; .byte 0x01; .byte 0xd5" :::
"memory");
1781 #define _xabort(ARG) _asm _emit 0xc6 _asm _emit 0xf8 _asm _emit ARG
1783 #define _xabort(ARG) \
1784 __asm__ volatile(".byte 0xC6; .byte 0xF8; .byte " STRINGIZE(ARG):::"memory");
1790 #if KMP_DEBUG_ADAPTIVE_LOCKS
1795 static kmp_adaptive_lock_statistics_t destroyedStats;
1798 static kmp_adaptive_lock_info_t liveLocks;
1801 static kmp_bootstrap_lock_t chain_lock =
1802 KMP_BOOTSTRAP_LOCK_INITIALIZER(chain_lock);
1805 void __kmp_init_speculative_stats() {
1806 kmp_adaptive_lock_info_t *lck = &liveLocks;
1808 memset(CCAST(kmp_adaptive_lock_statistics_t *, &(lck->stats)), 0,
1809 sizeof(lck->stats));
1810 lck->stats.next = lck;
1811 lck->stats.prev = lck;
1813 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1814 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1816 __kmp_init_bootstrap_lock(&chain_lock);
1820 static void __kmp_remember_lock(kmp_adaptive_lock_info_t *lck) {
1821 __kmp_acquire_bootstrap_lock(&chain_lock);
1823 lck->stats.next = liveLocks.stats.next;
1824 lck->stats.prev = &liveLocks;
1826 liveLocks.stats.next = lck;
1827 lck->stats.next->stats.prev = lck;
1829 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1830 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1832 __kmp_release_bootstrap_lock(&chain_lock);
1835 static void __kmp_forget_lock(kmp_adaptive_lock_info_t *lck) {
1836 KMP_ASSERT(lck->stats.next->stats.prev == lck);
1837 KMP_ASSERT(lck->stats.prev->stats.next == lck);
1839 kmp_adaptive_lock_info_t *n = lck->stats.next;
1840 kmp_adaptive_lock_info_t *p = lck->stats.prev;
1846 static void __kmp_zero_speculative_stats(kmp_adaptive_lock_info_t *lck) {
1847 memset(CCAST(kmp_adaptive_lock_statistics_t *, &lck->stats), 0,
1848 sizeof(lck->stats));
1849 __kmp_remember_lock(lck);
1852 static void __kmp_add_stats(kmp_adaptive_lock_statistics_t *t,
1853 kmp_adaptive_lock_info_t *lck) {
1854 kmp_adaptive_lock_statistics_t
volatile *s = &lck->stats;
1856 t->nonSpeculativeAcquireAttempts += lck->acquire_attempts;
1857 t->successfulSpeculations += s->successfulSpeculations;
1858 t->hardFailedSpeculations += s->hardFailedSpeculations;
1859 t->softFailedSpeculations += s->softFailedSpeculations;
1860 t->nonSpeculativeAcquires += s->nonSpeculativeAcquires;
1861 t->lemmingYields += s->lemmingYields;
1864 static void __kmp_accumulate_speculative_stats(kmp_adaptive_lock_info_t *lck) {
1865 __kmp_acquire_bootstrap_lock(&chain_lock);
1867 __kmp_add_stats(&destroyedStats, lck);
1868 __kmp_forget_lock(lck);
1870 __kmp_release_bootstrap_lock(&chain_lock);
1873 static float percent(kmp_uint32 count, kmp_uint32 total) {
1874 return (total == 0) ? 0.0 : (100.0 * count) / total;
1877 void __kmp_print_speculative_stats() {
1878 kmp_adaptive_lock_statistics_t total = destroyedStats;
1879 kmp_adaptive_lock_info_t *lck;
1881 for (lck = liveLocks.stats.next; lck != &liveLocks; lck = lck->stats.next) {
1882 __kmp_add_stats(&total, lck);
1884 kmp_adaptive_lock_statistics_t *t = &total;
1885 kmp_uint32 totalSections =
1886 t->nonSpeculativeAcquires + t->successfulSpeculations;
1887 kmp_uint32 totalSpeculations = t->successfulSpeculations +
1888 t->hardFailedSpeculations +
1889 t->softFailedSpeculations;
1890 if (totalSections <= 0)
1894 if (strcmp(__kmp_speculative_statsfile,
"-") == 0) {
1897 size_t buffLen = KMP_STRLEN(__kmp_speculative_statsfile) + 20;
1898 char buffer[buffLen];
1899 KMP_SNPRINTF(&buffer[0], buffLen, __kmp_speculative_statsfile,
1900 (kmp_int32)getpid());
1901 statsFile.
open(buffer,
"w");
1904 fprintf(statsFile,
"Speculative lock statistics (all approximate!)\n");
1906 " Lock parameters: \n"
1907 " max_soft_retries : %10d\n"
1908 " max_badness : %10d\n",
1909 __kmp_adaptive_backoff_params.max_soft_retries,
1910 __kmp_adaptive_backoff_params.max_badness);
1911 fprintf(statsFile,
" Non-speculative acquire attempts : %10d\n",
1912 t->nonSpeculativeAcquireAttempts);
1913 fprintf(statsFile,
" Total critical sections : %10d\n",
1915 fprintf(statsFile,
" Successful speculations : %10d (%5.1f%%)\n",
1916 t->successfulSpeculations,
1917 percent(t->successfulSpeculations, totalSections));
1918 fprintf(statsFile,
" Non-speculative acquires : %10d (%5.1f%%)\n",
1919 t->nonSpeculativeAcquires,
1920 percent(t->nonSpeculativeAcquires, totalSections));
1921 fprintf(statsFile,
" Lemming yields : %10d\n\n",
1924 fprintf(statsFile,
" Speculative acquire attempts : %10d\n",
1926 fprintf(statsFile,
" Successes : %10d (%5.1f%%)\n",
1927 t->successfulSpeculations,
1928 percent(t->successfulSpeculations, totalSpeculations));
1929 fprintf(statsFile,
" Soft failures : %10d (%5.1f%%)\n",
1930 t->softFailedSpeculations,
1931 percent(t->softFailedSpeculations, totalSpeculations));
1932 fprintf(statsFile,
" Hard failures : %10d (%5.1f%%)\n",
1933 t->hardFailedSpeculations,
1934 percent(t->hardFailedSpeculations, totalSpeculations));
1937 #define KMP_INC_STAT(lck, stat) (lck->lk.adaptive.stats.stat++)
1939 #define KMP_INC_STAT(lck, stat)
1943 static inline bool __kmp_is_unlocked_queuing_lock(kmp_queuing_lock_t *lck) {
1946 bool res = lck->lk.head_id == 0;
1950 #if KMP_COMPILER_ICC
1953 __sync_synchronize();
1960 static __inline
void
1961 __kmp_update_badness_after_success(kmp_adaptive_lock_t *lck) {
1963 lck->lk.adaptive.badness = 0;
1964 KMP_INC_STAT(lck, successfulSpeculations);
1968 static __inline
void __kmp_step_badness(kmp_adaptive_lock_t *lck) {
1969 kmp_uint32 newBadness = (lck->lk.adaptive.badness << 1) | 1;
1970 if (newBadness > lck->lk.adaptive.max_badness) {
1973 lck->lk.adaptive.badness = newBadness;
1978 KMP_ATTRIBUTE_TARGET_RTM
1979 static __inline
int __kmp_should_speculate(kmp_adaptive_lock_t *lck,
1981 kmp_uint32 badness = lck->lk.adaptive.badness;
1982 kmp_uint32 attempts = lck->lk.adaptive.acquire_attempts;
1983 int res = (attempts & badness) == 0;
1989 KMP_ATTRIBUTE_TARGET_RTM
1990 static int __kmp_test_adaptive_lock_only(kmp_adaptive_lock_t *lck,
1992 int retries = lck->lk.adaptive.max_soft_retries;
1999 kmp_uint32 status = _xbegin();
2004 if (status == _XBEGIN_STARTED) {
2009 if (!__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2013 KMP_ASSERT2(0,
"should not get here");
2018 if (status & SOFT_ABORT_MASK) {
2019 KMP_INC_STAT(lck, softFailedSpeculations);
2022 KMP_INC_STAT(lck, hardFailedSpeculations);
2027 }
while (retries--);
2031 __kmp_step_badness(lck);
2038 static int __kmp_test_adaptive_lock(kmp_adaptive_lock_t *lck, kmp_int32 gtid) {
2040 if (__kmp_should_speculate(lck, gtid) &&
2041 __kmp_test_adaptive_lock_only(lck, gtid))
2046 lck->lk.adaptive.acquire_attempts++;
2049 if (__kmp_test_queuing_lock(GET_QLK_PTR(lck), gtid)) {
2050 KMP_INC_STAT(lck, nonSpeculativeAcquires);
2057 static int __kmp_test_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2059 char const *
const func =
"omp_test_lock";
2060 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2061 KMP_FATAL(LockIsUninitialized, func);
2064 int retval = __kmp_test_adaptive_lock(lck, gtid);
2067 lck->lk.qlk.owner_id = gtid + 1;
2083 static void __kmp_acquire_adaptive_lock(kmp_adaptive_lock_t *lck,
2085 if (__kmp_should_speculate(lck, gtid)) {
2086 if (__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2087 if (__kmp_test_adaptive_lock_only(lck, gtid))
2096 while (!__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(lck))) {
2097 KMP_INC_STAT(lck, lemmingYields);
2101 if (__kmp_test_adaptive_lock_only(lck, gtid))
2108 lck->lk.adaptive.acquire_attempts++;
2110 __kmp_acquire_queuing_lock_timed_template<FALSE>(GET_QLK_PTR(lck), gtid);
2112 KMP_INC_STAT(lck, nonSpeculativeAcquires);
2115 static void __kmp_acquire_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2117 char const *
const func =
"omp_set_lock";
2118 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2119 KMP_FATAL(LockIsUninitialized, func);
2121 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) == gtid) {
2122 KMP_FATAL(LockIsAlreadyOwned, func);
2125 __kmp_acquire_adaptive_lock(lck, gtid);
2127 lck->lk.qlk.owner_id = gtid + 1;
2130 KMP_ATTRIBUTE_TARGET_RTM
2131 static int __kmp_release_adaptive_lock(kmp_adaptive_lock_t *lck,
2133 if (__kmp_is_unlocked_queuing_lock(GET_QLK_PTR(
2138 __kmp_update_badness_after_success(lck);
2141 __kmp_release_queuing_lock(GET_QLK_PTR(lck), gtid);
2143 return KMP_LOCK_RELEASED;
2146 static int __kmp_release_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck,
2148 char const *
const func =
"omp_unset_lock";
2150 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2151 KMP_FATAL(LockIsUninitialized, func);
2153 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) == -1) {
2154 KMP_FATAL(LockUnsettingFree, func);
2156 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) != gtid) {
2157 KMP_FATAL(LockUnsettingSetByAnother, func);
2159 lck->lk.qlk.owner_id = 0;
2160 __kmp_release_adaptive_lock(lck, gtid);
2161 return KMP_LOCK_RELEASED;
2164 static void __kmp_init_adaptive_lock(kmp_adaptive_lock_t *lck) {
2165 __kmp_init_queuing_lock(GET_QLK_PTR(lck));
2166 lck->lk.adaptive.badness = 0;
2167 lck->lk.adaptive.acquire_attempts = 0;
2168 lck->lk.adaptive.max_soft_retries =
2169 __kmp_adaptive_backoff_params.max_soft_retries;
2170 lck->lk.adaptive.max_badness = __kmp_adaptive_backoff_params.max_badness;
2171 #if KMP_DEBUG_ADAPTIVE_LOCKS
2172 __kmp_zero_speculative_stats(&lck->lk.adaptive);
2174 KA_TRACE(1000, (
"__kmp_init_adaptive_lock: lock %p initialized\n", lck));
2177 static void __kmp_destroy_adaptive_lock(kmp_adaptive_lock_t *lck) {
2178 #if KMP_DEBUG_ADAPTIVE_LOCKS
2179 __kmp_accumulate_speculative_stats(&lck->lk.adaptive);
2181 __kmp_destroy_queuing_lock(GET_QLK_PTR(lck));
2185 static void __kmp_destroy_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck) {
2186 char const *
const func =
"omp_destroy_lock";
2187 if (lck->lk.qlk.initialized != GET_QLK_PTR(lck)) {
2188 KMP_FATAL(LockIsUninitialized, func);
2190 if (__kmp_get_queuing_lock_owner(GET_QLK_PTR(lck)) != -1) {
2191 KMP_FATAL(LockStillOwned, func);
2193 __kmp_destroy_adaptive_lock(lck);
2202 static kmp_int32 __kmp_get_drdpa_lock_owner(kmp_drdpa_lock_t *lck) {
2203 return lck->lk.owner_id - 1;
2206 static inline bool __kmp_is_drdpa_lock_nestable(kmp_drdpa_lock_t *lck) {
2207 return lck->lk.depth_locked != -1;
2210 __forceinline
static int
2211 __kmp_acquire_drdpa_lock_timed_template(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2212 kmp_uint64 ticket = KMP_ATOMIC_INC(&lck->lk.next_ticket);
2213 kmp_uint64 mask = lck->lk.mask;
2214 std::atomic<kmp_uint64> *polls = lck->lk.polls;
2216 #ifdef USE_LOCK_PROFILE
2217 if (polls[ticket & mask] != ticket)
2218 __kmp_printf(
"LOCK CONTENTION: %p\n", lck);
2230 KMP_FSYNC_PREPARE(lck);
2231 KMP_INIT_YIELD(spins);
2232 while (polls[ticket & mask] < ticket) {
2233 KMP_YIELD_OVERSUB_ELSE_SPIN(spins);
2241 mask = lck->lk.mask;
2242 polls = lck->lk.polls;
2246 KMP_FSYNC_ACQUIRED(lck);
2247 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld acquired lock %p\n",
2249 lck->lk.now_serving = ticket;
2256 if ((lck->lk.old_polls != NULL) && (ticket >= lck->lk.cleanup_ticket)) {
2257 __kmp_free(lck->lk.old_polls);
2258 lck->lk.old_polls = NULL;
2259 lck->lk.cleanup_ticket = 0;
2265 if (lck->lk.old_polls == NULL) {
2266 bool reconfigure =
false;
2267 std::atomic<kmp_uint64> *old_polls = polls;
2268 kmp_uint32 num_polls = TCR_4(lck->lk.num_polls);
2270 if (TCR_4(__kmp_nth) >
2271 (__kmp_avail_proc ? __kmp_avail_proc : __kmp_xproc)) {
2274 if (num_polls > 1) {
2276 num_polls = TCR_4(lck->lk.num_polls);
2279 polls = (std::atomic<kmp_uint64> *)__kmp_allocate(num_polls *
2287 kmp_uint64 num_waiting = TCR_8(lck->lk.next_ticket) - ticket - 1;
2288 if (num_waiting > num_polls) {
2289 kmp_uint32 old_num_polls = num_polls;
2292 mask = (mask << 1) | 1;
2294 }
while (num_polls <= num_waiting);
2300 polls = (std::atomic<kmp_uint64> *)__kmp_allocate(num_polls *
2303 for (i = 0; i < old_num_polls; i++) {
2304 polls[i].store(old_polls[i]);
2319 KA_TRACE(1000, (
"__kmp_acquire_drdpa_lock: ticket #%lld reconfiguring "
2320 "lock %p to %d polls\n",
2321 ticket, lck, num_polls));
2323 lck->lk.old_polls = old_polls;
2324 lck->lk.polls = polls;
2328 lck->lk.num_polls = num_polls;
2329 lck->lk.mask = mask;
2337 lck->lk.cleanup_ticket = lck->lk.next_ticket;
2340 return KMP_LOCK_ACQUIRED_FIRST;
2343 int __kmp_acquire_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2344 int retval = __kmp_acquire_drdpa_lock_timed_template(lck, gtid);
2348 static int __kmp_acquire_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2350 char const *
const func =
"omp_set_lock";
2351 if (lck->lk.initialized != lck) {
2352 KMP_FATAL(LockIsUninitialized, func);
2354 if (__kmp_is_drdpa_lock_nestable(lck)) {
2355 KMP_FATAL(LockNestableUsedAsSimple, func);
2357 if ((gtid >= 0) && (__kmp_get_drdpa_lock_owner(lck) == gtid)) {
2358 KMP_FATAL(LockIsAlreadyOwned, func);
2361 __kmp_acquire_drdpa_lock(lck, gtid);
2363 lck->lk.owner_id = gtid + 1;
2364 return KMP_LOCK_ACQUIRED_FIRST;
2367 int __kmp_test_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2370 kmp_uint64 ticket = lck->lk.next_ticket;
2371 std::atomic<kmp_uint64> *polls = lck->lk.polls;
2372 kmp_uint64 mask = lck->lk.mask;
2373 if (polls[ticket & mask] == ticket) {
2374 kmp_uint64 next_ticket = ticket + 1;
2375 if (__kmp_atomic_compare_store_acq(&lck->lk.next_ticket, ticket,
2377 KMP_FSYNC_ACQUIRED(lck);
2378 KA_TRACE(1000, (
"__kmp_test_drdpa_lock: ticket #%lld acquired lock %p\n",
2380 lck->lk.now_serving = ticket;
2394 static int __kmp_test_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2396 char const *
const func =
"omp_test_lock";
2397 if (lck->lk.initialized != lck) {
2398 KMP_FATAL(LockIsUninitialized, func);
2400 if (__kmp_is_drdpa_lock_nestable(lck)) {
2401 KMP_FATAL(LockNestableUsedAsSimple, func);
2404 int retval = __kmp_test_drdpa_lock(lck, gtid);
2407 lck->lk.owner_id = gtid + 1;
2412 int __kmp_release_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2415 kmp_uint64 ticket = lck->lk.now_serving + 1;
2416 std::atomic<kmp_uint64> *polls = lck->lk.polls;
2417 kmp_uint64 mask = lck->lk.mask;
2418 KA_TRACE(1000, (
"__kmp_release_drdpa_lock: ticket #%lld released lock %p\n",
2420 KMP_FSYNC_RELEASING(lck);
2421 polls[ticket & mask] = ticket;
2422 return KMP_LOCK_RELEASED;
2425 static int __kmp_release_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2427 char const *
const func =
"omp_unset_lock";
2429 if (lck->lk.initialized != lck) {
2430 KMP_FATAL(LockIsUninitialized, func);
2432 if (__kmp_is_drdpa_lock_nestable(lck)) {
2433 KMP_FATAL(LockNestableUsedAsSimple, func);
2435 if (__kmp_get_drdpa_lock_owner(lck) == -1) {
2436 KMP_FATAL(LockUnsettingFree, func);
2438 if ((gtid >= 0) && (__kmp_get_drdpa_lock_owner(lck) >= 0) &&
2439 (__kmp_get_drdpa_lock_owner(lck) != gtid)) {
2440 KMP_FATAL(LockUnsettingSetByAnother, func);
2442 lck->lk.owner_id = 0;
2443 return __kmp_release_drdpa_lock(lck, gtid);
2446 void __kmp_init_drdpa_lock(kmp_drdpa_lock_t *lck) {
2447 lck->lk.location = NULL;
2449 lck->lk.num_polls = 1;
2450 lck->lk.polls = (std::atomic<kmp_uint64> *)__kmp_allocate(
2451 lck->lk.num_polls *
sizeof(*(lck->lk.polls)));
2452 lck->lk.cleanup_ticket = 0;
2453 lck->lk.old_polls = NULL;
2454 lck->lk.next_ticket = 0;
2455 lck->lk.now_serving = 0;
2456 lck->lk.owner_id = 0;
2457 lck->lk.depth_locked = -1;
2458 lck->lk.initialized = lck;
2460 KA_TRACE(1000, (
"__kmp_init_drdpa_lock: lock %p initialized\n", lck));
2463 void __kmp_destroy_drdpa_lock(kmp_drdpa_lock_t *lck) {
2464 lck->lk.initialized = NULL;
2465 lck->lk.location = NULL;
2466 if (lck->lk.polls.load() != NULL) {
2467 __kmp_free(lck->lk.polls.load());
2468 lck->lk.polls = NULL;
2470 if (lck->lk.old_polls != NULL) {
2471 __kmp_free(lck->lk.old_polls);
2472 lck->lk.old_polls = NULL;
2475 lck->lk.num_polls = 0;
2476 lck->lk.cleanup_ticket = 0;
2477 lck->lk.next_ticket = 0;
2478 lck->lk.now_serving = 0;
2479 lck->lk.owner_id = 0;
2480 lck->lk.depth_locked = -1;
2483 static void __kmp_destroy_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
2484 char const *
const func =
"omp_destroy_lock";
2485 if (lck->lk.initialized != lck) {
2486 KMP_FATAL(LockIsUninitialized, func);
2488 if (__kmp_is_drdpa_lock_nestable(lck)) {
2489 KMP_FATAL(LockNestableUsedAsSimple, func);
2491 if (__kmp_get_drdpa_lock_owner(lck) != -1) {
2492 KMP_FATAL(LockStillOwned, func);
2494 __kmp_destroy_drdpa_lock(lck);
2499 int __kmp_acquire_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2500 KMP_DEBUG_ASSERT(gtid >= 0);
2502 if (__kmp_get_drdpa_lock_owner(lck) == gtid) {
2503 lck->lk.depth_locked += 1;
2504 return KMP_LOCK_ACQUIRED_NEXT;
2506 __kmp_acquire_drdpa_lock_timed_template(lck, gtid);
2508 lck->lk.depth_locked = 1;
2510 lck->lk.owner_id = gtid + 1;
2511 return KMP_LOCK_ACQUIRED_FIRST;
2515 static void __kmp_acquire_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2517 char const *
const func =
"omp_set_nest_lock";
2518 if (lck->lk.initialized != lck) {
2519 KMP_FATAL(LockIsUninitialized, func);
2521 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2522 KMP_FATAL(LockSimpleUsedAsNestable, func);
2524 __kmp_acquire_nested_drdpa_lock(lck, gtid);
2527 int __kmp_test_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2530 KMP_DEBUG_ASSERT(gtid >= 0);
2532 if (__kmp_get_drdpa_lock_owner(lck) == gtid) {
2533 retval = ++lck->lk.depth_locked;
2534 }
else if (!__kmp_test_drdpa_lock(lck, gtid)) {
2538 retval = lck->lk.depth_locked = 1;
2540 lck->lk.owner_id = gtid + 1;
2545 static int __kmp_test_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2547 char const *
const func =
"omp_test_nest_lock";
2548 if (lck->lk.initialized != lck) {
2549 KMP_FATAL(LockIsUninitialized, func);
2551 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2552 KMP_FATAL(LockSimpleUsedAsNestable, func);
2554 return __kmp_test_nested_drdpa_lock(lck, gtid);
2557 int __kmp_release_nested_drdpa_lock(kmp_drdpa_lock_t *lck, kmp_int32 gtid) {
2558 KMP_DEBUG_ASSERT(gtid >= 0);
2561 if (--(lck->lk.depth_locked) == 0) {
2563 lck->lk.owner_id = 0;
2564 __kmp_release_drdpa_lock(lck, gtid);
2565 return KMP_LOCK_RELEASED;
2567 return KMP_LOCK_STILL_HELD;
2570 static int __kmp_release_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck,
2572 char const *
const func =
"omp_unset_nest_lock";
2574 if (lck->lk.initialized != lck) {
2575 KMP_FATAL(LockIsUninitialized, func);
2577 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2578 KMP_FATAL(LockSimpleUsedAsNestable, func);
2580 if (__kmp_get_drdpa_lock_owner(lck) == -1) {
2581 KMP_FATAL(LockUnsettingFree, func);
2583 if (__kmp_get_drdpa_lock_owner(lck) != gtid) {
2584 KMP_FATAL(LockUnsettingSetByAnother, func);
2586 return __kmp_release_nested_drdpa_lock(lck, gtid);
2589 void __kmp_init_nested_drdpa_lock(kmp_drdpa_lock_t *lck) {
2590 __kmp_init_drdpa_lock(lck);
2591 lck->lk.depth_locked = 0;
2594 void __kmp_destroy_nested_drdpa_lock(kmp_drdpa_lock_t *lck) {
2595 __kmp_destroy_drdpa_lock(lck);
2596 lck->lk.depth_locked = 0;
2599 static void __kmp_destroy_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
2600 char const *
const func =
"omp_destroy_nest_lock";
2601 if (lck->lk.initialized != lck) {
2602 KMP_FATAL(LockIsUninitialized, func);
2604 if (!__kmp_is_drdpa_lock_nestable(lck)) {
2605 KMP_FATAL(LockSimpleUsedAsNestable, func);
2607 if (__kmp_get_drdpa_lock_owner(lck) != -1) {
2608 KMP_FATAL(LockStillOwned, func);
2610 __kmp_destroy_nested_drdpa_lock(lck);
2615 static const ident_t *__kmp_get_drdpa_lock_location(kmp_drdpa_lock_t *lck) {
2616 return lck->lk.location;
2619 static void __kmp_set_drdpa_lock_location(kmp_drdpa_lock_t *lck,
2621 lck->lk.location = loc;
2624 static kmp_lock_flags_t __kmp_get_drdpa_lock_flags(kmp_drdpa_lock_t *lck) {
2625 return lck->lk.flags;
2628 static void __kmp_set_drdpa_lock_flags(kmp_drdpa_lock_t *lck,
2629 kmp_lock_flags_t flags) {
2630 lck->lk.flags = flags;
2634 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
2635 #define __kmp_tsc() __kmp_hardware_timestamp()
2637 kmp_backoff_t __kmp_spin_backoff_params = {1, 4096, 100};
2640 extern kmp_uint64 __kmp_now_nsec();
2641 kmp_backoff_t __kmp_spin_backoff_params = {1, 256, 100};
2642 #define __kmp_tsc() __kmp_now_nsec()
2652 static inline bool before(kmp_uint64 a, kmp_uint64 b) {
2653 return ((kmp_int64)b - (kmp_int64)a) > 0;
2657 void __kmp_spin_backoff(kmp_backoff_t *boff) {
2660 for (i = boff->step; i > 0; i--) {
2661 kmp_uint64 goal = __kmp_tsc() + boff->min_tick;
2664 }
while (before(__kmp_tsc(), goal));
2666 boff->step = (boff->step << 1 | 1) & (boff->max_backoff - 1);
2669 #if KMP_USE_DYNAMIC_LOCK
2673 static void __kmp_init_direct_lock(kmp_dyna_lock_t *lck,
2674 kmp_dyna_lockseq_t seq) {
2675 TCW_4(*lck, KMP_GET_D_TAG(seq));
2678 (
"__kmp_init_direct_lock: initialized direct lock with type#%d\n", seq));
2684 #define HLE_ACQUIRE ".byte 0xf2;"
2685 #define HLE_RELEASE ".byte 0xf3;"
2687 static inline kmp_uint32 swap4(kmp_uint32
volatile *p, kmp_uint32 v) {
2688 __asm__
volatile(HLE_ACQUIRE
"xchg %1,%0" :
"+r"(v),
"+m"(*p) : :
"memory");
2692 static void __kmp_destroy_hle_lock(kmp_dyna_lock_t *lck) { TCW_4(*lck, 0); }
2694 static void __kmp_destroy_hle_lock_with_checks(kmp_dyna_lock_t *lck) {
2698 static void __kmp_acquire_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2700 if (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle)) {
2703 while (*(kmp_uint32
volatile *)lck != KMP_LOCK_FREE(hle)) {
2704 for (
int i = delay; i != 0; --i)
2706 delay = ((delay << 1) | 1) & 7;
2708 }
while (swap4(lck, KMP_LOCK_BUSY(1, hle)) != KMP_LOCK_FREE(hle));
2712 static void __kmp_acquire_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2714 __kmp_acquire_hle_lock(lck, gtid);
2717 static int __kmp_release_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2718 __asm__
volatile(HLE_RELEASE
"movl %1,%0"
2720 :
"r"(KMP_LOCK_FREE(hle))
2722 return KMP_LOCK_RELEASED;
2725 static int __kmp_release_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2727 return __kmp_release_hle_lock(lck, gtid);
2730 static int __kmp_test_hle_lock(kmp_dyna_lock_t *lck, kmp_int32 gtid) {
2731 return swap4(lck, KMP_LOCK_BUSY(1, hle)) == KMP_LOCK_FREE(hle);
2734 static int __kmp_test_hle_lock_with_checks(kmp_dyna_lock_t *lck,
2736 return __kmp_test_hle_lock(lck, gtid);
2739 static void __kmp_init_rtm_queuing_lock(kmp_queuing_lock_t *lck) {
2740 __kmp_init_queuing_lock(lck);
2743 static void __kmp_destroy_rtm_queuing_lock(kmp_queuing_lock_t *lck) {
2744 __kmp_destroy_queuing_lock(lck);
2748 __kmp_destroy_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
2749 __kmp_destroy_queuing_lock_with_checks(lck);
2752 KMP_ATTRIBUTE_TARGET_RTM
2753 static void __kmp_acquire_rtm_queuing_lock(kmp_queuing_lock_t *lck,
2755 unsigned retries = 3, status;
2758 if (status == _XBEGIN_STARTED) {
2759 if (__kmp_is_unlocked_queuing_lock(lck))
2763 if ((status & _XABORT_EXPLICIT) && _XABORT_CODE(status) == 0xff) {
2765 while (!__kmp_is_unlocked_queuing_lock(lck)) {
2768 }
else if (!(status & _XABORT_RETRY))
2770 }
while (retries--);
2773 __kmp_acquire_queuing_lock(lck, gtid);
2776 static void __kmp_acquire_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
2778 __kmp_acquire_rtm_queuing_lock(lck, gtid);
2781 KMP_ATTRIBUTE_TARGET_RTM
2782 static int __kmp_release_rtm_queuing_lock(kmp_queuing_lock_t *lck,
2784 if (__kmp_is_unlocked_queuing_lock(lck)) {
2789 __kmp_release_queuing_lock(lck, gtid);
2791 return KMP_LOCK_RELEASED;
2794 static int __kmp_release_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
2796 return __kmp_release_rtm_queuing_lock(lck, gtid);
2799 KMP_ATTRIBUTE_TARGET_RTM
2800 static int __kmp_test_rtm_queuing_lock(kmp_queuing_lock_t *lck,
2802 unsigned retries = 3, status;
2805 if (status == _XBEGIN_STARTED && __kmp_is_unlocked_queuing_lock(lck)) {
2808 if (!(status & _XABORT_RETRY))
2810 }
while (retries--);
2812 return __kmp_test_queuing_lock(lck, gtid);
2815 static int __kmp_test_rtm_queuing_lock_with_checks(kmp_queuing_lock_t *lck,
2817 return __kmp_test_rtm_queuing_lock(lck, gtid);
2821 typedef kmp_tas_lock_t kmp_rtm_spin_lock_t;
2823 static void __kmp_destroy_rtm_spin_lock(kmp_rtm_spin_lock_t *lck) {
2824 KMP_ATOMIC_ST_REL(&lck->lk.poll, 0);
2827 static void __kmp_destroy_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck) {
2828 __kmp_destroy_rtm_spin_lock(lck);
2831 KMP_ATTRIBUTE_TARGET_RTM
2832 static int __kmp_acquire_rtm_spin_lock(kmp_rtm_spin_lock_t *lck,
2834 unsigned retries = 3, status;
2835 kmp_int32 lock_free = KMP_LOCK_FREE(rtm_spin);
2836 kmp_int32 lock_busy = KMP_LOCK_BUSY(1, rtm_spin);
2839 if (status == _XBEGIN_STARTED) {
2840 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == lock_free)
2841 return KMP_LOCK_ACQUIRED_FIRST;
2844 if ((status & _XABORT_EXPLICIT) && _XABORT_CODE(status) == 0xff) {
2846 while (KMP_ATOMIC_LD_RLX(&lck->lk.poll) != lock_free) {
2849 }
else if (!(status & _XABORT_RETRY))
2851 }
while (retries--);
2854 KMP_FSYNC_PREPARE(lck);
2855 kmp_backoff_t backoff = __kmp_spin_backoff_params;
2856 while (KMP_ATOMIC_LD_RLX(&lck->lk.poll) != lock_free ||
2857 !__kmp_atomic_compare_store_acq(&lck->lk.poll, lock_free, lock_busy)) {
2858 __kmp_spin_backoff(&backoff);
2860 KMP_FSYNC_ACQUIRED(lck);
2861 return KMP_LOCK_ACQUIRED_FIRST;
2864 static int __kmp_acquire_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck,
2866 return __kmp_acquire_rtm_spin_lock(lck, gtid);
2869 KMP_ATTRIBUTE_TARGET_RTM
2870 static int __kmp_release_rtm_spin_lock(kmp_rtm_spin_lock_t *lck,
2872 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == KMP_LOCK_FREE(rtm_spin)) {
2877 KMP_FSYNC_RELEASING(lck);
2878 KMP_ATOMIC_ST_REL(&lck->lk.poll, KMP_LOCK_FREE(rtm_spin));
2880 return KMP_LOCK_RELEASED;
2883 static int __kmp_release_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck,
2885 return __kmp_release_rtm_spin_lock(lck, gtid);
2888 KMP_ATTRIBUTE_TARGET_RTM
2889 static int __kmp_test_rtm_spin_lock(kmp_rtm_spin_lock_t *lck, kmp_int32 gtid) {
2890 unsigned retries = 3, status;
2891 kmp_int32 lock_free = KMP_LOCK_FREE(rtm_spin);
2892 kmp_int32 lock_busy = KMP_LOCK_BUSY(1, rtm_spin);
2895 if (status == _XBEGIN_STARTED &&
2896 KMP_ATOMIC_LD_RLX(&lck->lk.poll) == lock_free) {
2899 if (!(status & _XABORT_RETRY))
2901 }
while (retries--);
2903 if (KMP_ATOMIC_LD_RLX(&lck->lk.poll) == lock_free &&
2904 __kmp_atomic_compare_store_acq(&lck->lk.poll, lock_free, lock_busy)) {
2905 KMP_FSYNC_ACQUIRED(lck);
2911 static int __kmp_test_rtm_spin_lock_with_checks(kmp_rtm_spin_lock_t *lck,
2913 return __kmp_test_rtm_spin_lock(lck, gtid);
2919 static void __kmp_init_indirect_lock(kmp_dyna_lock_t *l,
2920 kmp_dyna_lockseq_t tag);
2921 static void __kmp_destroy_indirect_lock(kmp_dyna_lock_t *lock);
2922 static int __kmp_set_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32);
2923 static int __kmp_unset_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32);
2924 static int __kmp_test_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32);
2925 static int __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
2927 static int __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
2929 static int __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
2933 #define KMP_FOREACH_LOCK_KIND(m, a) m(ticket, a) m(queuing, a) m(drdpa, a)
2935 #define expand1(lk, op) \
2936 static void __kmp_##op##_##lk##_##lock(kmp_user_lock_p lock) { \
2937 __kmp_##op##_##lk##_##lock(&lock->lk); \
2939 #define expand2(lk, op) \
2940 static int __kmp_##op##_##lk##_##lock(kmp_user_lock_p lock, \
2942 return __kmp_##op##_##lk##_##lock(&lock->lk, gtid); \
2944 #define expand3(lk, op) \
2945 static void __kmp_set_##lk##_##lock_flags(kmp_user_lock_p lock, \
2946 kmp_lock_flags_t flags) { \
2947 __kmp_set_##lk##_lock_flags(&lock->lk, flags); \
2949 #define expand4(lk, op) \
2950 static void __kmp_set_##lk##_##lock_location(kmp_user_lock_p lock, \
2951 const ident_t *loc) { \
2952 __kmp_set_##lk##_lock_location(&lock->lk, loc); \
2955 KMP_FOREACH_LOCK_KIND(expand1, init)
2956 KMP_FOREACH_LOCK_KIND(expand1, init_nested)
2957 KMP_FOREACH_LOCK_KIND(expand1, destroy)
2958 KMP_FOREACH_LOCK_KIND(expand1, destroy_nested)
2959 KMP_FOREACH_LOCK_KIND(expand2, acquire)
2960 KMP_FOREACH_LOCK_KIND(expand2, acquire_nested)
2961 KMP_FOREACH_LOCK_KIND(expand2, release)
2962 KMP_FOREACH_LOCK_KIND(expand2, release_nested)
2963 KMP_FOREACH_LOCK_KIND(expand2, test)
2964 KMP_FOREACH_LOCK_KIND(expand2, test_nested)
2965 KMP_FOREACH_LOCK_KIND(expand3, )
2966 KMP_FOREACH_LOCK_KIND(expand4, )
2977 #define expand(l, op) 0, __kmp_init_direct_lock,
2978 void (*__kmp_direct_init[])(kmp_dyna_lock_t *, kmp_dyna_lockseq_t) = {
2979 __kmp_init_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, init)};
2983 #define expand(l, op) 0, (void (*)(kmp_dyna_lock_t *))__kmp_##op##_##l##_lock,
2984 static void (*direct_destroy[])(kmp_dyna_lock_t *) = {
2985 __kmp_destroy_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, destroy)};
2987 #define expand(l, op) \
2988 0, (void (*)(kmp_dyna_lock_t *))__kmp_destroy_##l##_lock_with_checks,
2989 static void (*direct_destroy_check[])(kmp_dyna_lock_t *) = {
2990 __kmp_destroy_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, destroy)};
2994 #define expand(l, op) \
2995 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock,
2996 static int (*direct_set[])(kmp_dyna_lock_t *, kmp_int32) = {
2997 __kmp_set_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, acquire)};
2999 #define expand(l, op) \
3000 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock_with_checks,
3001 static int (*direct_set_check[])(kmp_dyna_lock_t *, kmp_int32) = {
3002 __kmp_set_indirect_lock_with_checks, 0,
3003 KMP_FOREACH_D_LOCK(expand, acquire)};
3007 #define expand(l, op) \
3008 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock,
3009 static int (*direct_unset[])(kmp_dyna_lock_t *, kmp_int32) = {
3010 __kmp_unset_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, release)};
3011 static int (*direct_test[])(kmp_dyna_lock_t *, kmp_int32) = {
3012 __kmp_test_indirect_lock, 0, KMP_FOREACH_D_LOCK(expand, test)};
3014 #define expand(l, op) \
3015 0, (int (*)(kmp_dyna_lock_t *, kmp_int32))__kmp_##op##_##l##_lock_with_checks,
3016 static int (*direct_unset_check[])(kmp_dyna_lock_t *, kmp_int32) = {
3017 __kmp_unset_indirect_lock_with_checks, 0,
3018 KMP_FOREACH_D_LOCK(expand, release)};
3019 static int (*direct_test_check[])(kmp_dyna_lock_t *, kmp_int32) = {
3020 __kmp_test_indirect_lock_with_checks, 0, KMP_FOREACH_D_LOCK(expand, test)};
3024 void (**__kmp_direct_destroy)(kmp_dyna_lock_t *) = 0;
3025 int (**__kmp_direct_set)(kmp_dyna_lock_t *, kmp_int32) = 0;
3026 int (**__kmp_direct_unset)(kmp_dyna_lock_t *, kmp_int32) = 0;
3027 int (**__kmp_direct_test)(kmp_dyna_lock_t *, kmp_int32) = 0;
3030 #define expand(l, op) (void (*)(kmp_user_lock_p)) __kmp_##op##_##l##_##lock,
3031 void (*__kmp_indirect_init[])(kmp_user_lock_p) = {
3032 KMP_FOREACH_I_LOCK(expand, init)};
3035 #define expand(l, op) (void (*)(kmp_user_lock_p)) __kmp_##op##_##l##_##lock,
3036 static void (*indirect_destroy[])(kmp_user_lock_p) = {
3037 KMP_FOREACH_I_LOCK(expand, destroy)};
3039 #define expand(l, op) \
3040 (void (*)(kmp_user_lock_p)) __kmp_##op##_##l##_##lock_with_checks,
3041 static void (*indirect_destroy_check[])(kmp_user_lock_p) = {
3042 KMP_FOREACH_I_LOCK(expand, destroy)};
3046 #define expand(l, op) \
3047 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock,
3048 static int (*indirect_set[])(kmp_user_lock_p,
3049 kmp_int32) = {KMP_FOREACH_I_LOCK(expand, acquire)};
3051 #define expand(l, op) \
3052 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock_with_checks,
3053 static int (*indirect_set_check[])(kmp_user_lock_p, kmp_int32) = {
3054 KMP_FOREACH_I_LOCK(expand, acquire)};
3058 #define expand(l, op) \
3059 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock,
3060 static int (*indirect_unset[])(kmp_user_lock_p, kmp_int32) = {
3061 KMP_FOREACH_I_LOCK(expand, release)};
3062 static int (*indirect_test[])(kmp_user_lock_p,
3063 kmp_int32) = {KMP_FOREACH_I_LOCK(expand, test)};
3065 #define expand(l, op) \
3066 (int (*)(kmp_user_lock_p, kmp_int32)) __kmp_##op##_##l##_##lock_with_checks,
3067 static int (*indirect_unset_check[])(kmp_user_lock_p, kmp_int32) = {
3068 KMP_FOREACH_I_LOCK(expand, release)};
3069 static int (*indirect_test_check[])(kmp_user_lock_p, kmp_int32) = {
3070 KMP_FOREACH_I_LOCK(expand, test)};
3074 void (**__kmp_indirect_destroy)(kmp_user_lock_p) = 0;
3075 int (**__kmp_indirect_set)(kmp_user_lock_p, kmp_int32) = 0;
3076 int (**__kmp_indirect_unset)(kmp_user_lock_p, kmp_int32) = 0;
3077 int (**__kmp_indirect_test)(kmp_user_lock_p, kmp_int32) = 0;
3080 kmp_indirect_lock_table_t __kmp_i_lock_table;
3083 static kmp_uint32 __kmp_indirect_lock_size[KMP_NUM_I_LOCKS] = {0};
3086 void (*__kmp_indirect_set_location[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
3088 void (*__kmp_indirect_set_flags[KMP_NUM_I_LOCKS])(kmp_user_lock_p,
3089 kmp_lock_flags_t) = {0};
3090 const ident_t *(*__kmp_indirect_get_location[KMP_NUM_I_LOCKS])(
3091 kmp_user_lock_p) = {0};
3092 kmp_lock_flags_t (*__kmp_indirect_get_flags[KMP_NUM_I_LOCKS])(
3093 kmp_user_lock_p) = {0};
3096 static kmp_indirect_lock_t *__kmp_indirect_lock_pool[KMP_NUM_I_LOCKS] = {0};
3103 kmp_indirect_lock_t *__kmp_allocate_indirect_lock(
void **user_lock,
3105 kmp_indirect_locktag_t tag) {
3106 kmp_indirect_lock_t *lck;
3107 kmp_lock_index_t idx, table_idx;
3109 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3111 if (__kmp_indirect_lock_pool[tag] != NULL) {
3113 lck = __kmp_indirect_lock_pool[tag];
3114 if (OMP_LOCK_T_SIZE <
sizeof(
void *))
3115 idx = lck->lock->pool.index;
3116 __kmp_indirect_lock_pool[tag] = (kmp_indirect_lock_t *)lck->lock->pool.next;
3117 KA_TRACE(20, (
"__kmp_allocate_indirect_lock: reusing an existing lock %p\n",
3120 kmp_uint32 row, col;
3121 kmp_indirect_lock_table_t *lock_table = &__kmp_i_lock_table;
3125 table_idx = lock_table->next;
3126 idx += lock_table->next;
3127 if (table_idx < lock_table->nrow_ptrs * KMP_I_LOCK_CHUNK) {
3128 row = table_idx / KMP_I_LOCK_CHUNK;
3129 col = table_idx % KMP_I_LOCK_CHUNK;
3131 if (!lock_table->table[row]) {
3132 lock_table->table[row] = (kmp_indirect_lock_t *)__kmp_allocate(
3133 sizeof(kmp_indirect_lock_t) * KMP_I_LOCK_CHUNK);
3138 if (!lock_table->next_table) {
3139 kmp_indirect_lock_table_t *next_table =
3140 (kmp_indirect_lock_table_t *)__kmp_allocate(
3141 sizeof(kmp_indirect_lock_table_t));
3142 next_table->table = (kmp_indirect_lock_t **)__kmp_allocate(
3143 sizeof(kmp_indirect_lock_t *) * 2 * lock_table->nrow_ptrs);
3144 next_table->nrow_ptrs = 2 * lock_table->nrow_ptrs;
3145 next_table->next = 0;
3146 next_table->next_table =
nullptr;
3147 lock_table->next_table = next_table;
3149 lock_table = lock_table->next_table;
3150 KMP_ASSERT(lock_table);
3154 lck = &lock_table->table[row][col];
3156 lck->lock = (kmp_user_lock_p)__kmp_allocate(__kmp_indirect_lock_size[tag]);
3158 (
"__kmp_allocate_indirect_lock: allocated a new lock %p\n", lck));
3161 __kmp_release_lock(&__kmp_global_lock, gtid);
3165 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3166 *((kmp_lock_index_t *)user_lock) = idx
3169 *((kmp_indirect_lock_t **)user_lock) = lck;
3176 static __forceinline kmp_indirect_lock_t *
3177 __kmp_lookup_indirect_lock(
void **user_lock,
const char *func) {
3178 if (__kmp_env_consistency_check) {
3179 kmp_indirect_lock_t *lck = NULL;
3180 if (user_lock == NULL) {
3181 KMP_FATAL(LockIsUninitialized, func);
3183 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3184 kmp_lock_index_t idx = KMP_EXTRACT_I_INDEX(user_lock);
3185 lck = __kmp_get_i_lock(idx);
3187 lck = *((kmp_indirect_lock_t **)user_lock);
3190 KMP_FATAL(LockIsUninitialized, func);
3194 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3195 return __kmp_get_i_lock(KMP_EXTRACT_I_INDEX(user_lock));
3197 return *((kmp_indirect_lock_t **)user_lock);
3202 static void __kmp_init_indirect_lock(kmp_dyna_lock_t *lock,
3203 kmp_dyna_lockseq_t seq) {
3204 #if KMP_USE_ADAPTIVE_LOCKS
3205 if (seq == lockseq_adaptive && !__kmp_cpuinfo.rtm) {
3206 KMP_WARNING(AdaptiveNotSupported,
"kmp_lockseq_t",
"adaptive");
3207 seq = lockseq_queuing;
3211 if (seq == lockseq_rtm_queuing && !__kmp_cpuinfo.rtm) {
3212 seq = lockseq_queuing;
3215 kmp_indirect_locktag_t tag = KMP_GET_I_TAG(seq);
3216 kmp_indirect_lock_t *l =
3217 __kmp_allocate_indirect_lock((
void **)lock, __kmp_entry_gtid(), tag);
3218 KMP_I_LOCK_FUNC(l, init)(l->lock);
3220 20, (
"__kmp_init_indirect_lock: initialized indirect lock with type#%d\n",
3224 static void __kmp_destroy_indirect_lock(kmp_dyna_lock_t *lock) {
3225 kmp_uint32 gtid = __kmp_entry_gtid();
3226 kmp_indirect_lock_t *l =
3227 __kmp_lookup_indirect_lock((
void **)lock,
"omp_destroy_lock");
3228 KMP_I_LOCK_FUNC(l, destroy)(l->lock);
3229 kmp_indirect_locktag_t tag = l->type;
3231 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3234 l->lock->pool.next = (kmp_user_lock_p)__kmp_indirect_lock_pool[tag];
3235 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3236 l->lock->pool.index = KMP_EXTRACT_I_INDEX(lock);
3238 __kmp_indirect_lock_pool[tag] = l;
3240 __kmp_release_lock(&__kmp_global_lock, gtid);
3243 static int __kmp_set_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) {
3244 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3245 return KMP_I_LOCK_FUNC(l, set)(l->lock, gtid);
3248 static int __kmp_unset_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) {
3249 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3250 return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3253 static int __kmp_test_indirect_lock(kmp_dyna_lock_t *lock, kmp_int32 gtid) {
3254 kmp_indirect_lock_t *l = KMP_LOOKUP_I_LOCK(lock);
3255 return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid);
3258 static int __kmp_set_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
3260 kmp_indirect_lock_t *l =
3261 __kmp_lookup_indirect_lock((
void **)lock,
"omp_set_lock");
3262 return KMP_I_LOCK_FUNC(l, set)(l->lock, gtid);
3265 static int __kmp_unset_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
3267 kmp_indirect_lock_t *l =
3268 __kmp_lookup_indirect_lock((
void **)lock,
"omp_unset_lock");
3269 return KMP_I_LOCK_FUNC(l, unset)(l->lock, gtid);
3272 static int __kmp_test_indirect_lock_with_checks(kmp_dyna_lock_t *lock,
3274 kmp_indirect_lock_t *l =
3275 __kmp_lookup_indirect_lock((
void **)lock,
"omp_test_lock");
3276 return KMP_I_LOCK_FUNC(l, test)(l->lock, gtid);
3279 kmp_dyna_lockseq_t __kmp_user_lock_seq = lockseq_queuing;
3282 kmp_int32 __kmp_get_user_lock_owner(kmp_user_lock_p lck, kmp_uint32 seq) {
3285 case lockseq_nested_tas:
3286 return __kmp_get_tas_lock_owner((kmp_tas_lock_t *)lck);
3289 case lockseq_nested_futex:
3290 return __kmp_get_futex_lock_owner((kmp_futex_lock_t *)lck);
3292 case lockseq_ticket:
3293 case lockseq_nested_ticket:
3294 return __kmp_get_ticket_lock_owner((kmp_ticket_lock_t *)lck);
3295 case lockseq_queuing:
3296 case lockseq_nested_queuing:
3297 #if KMP_USE_ADAPTIVE_LOCKS
3298 case lockseq_adaptive:
3300 return __kmp_get_queuing_lock_owner((kmp_queuing_lock_t *)lck);
3302 case lockseq_nested_drdpa:
3303 return __kmp_get_drdpa_lock_owner((kmp_drdpa_lock_t *)lck);
3310 void __kmp_init_dynamic_user_locks() {
3312 if (__kmp_env_consistency_check) {
3313 __kmp_direct_set = direct_set_check;
3314 __kmp_direct_unset = direct_unset_check;
3315 __kmp_direct_test = direct_test_check;
3316 __kmp_direct_destroy = direct_destroy_check;
3317 __kmp_indirect_set = indirect_set_check;
3318 __kmp_indirect_unset = indirect_unset_check;
3319 __kmp_indirect_test = indirect_test_check;
3320 __kmp_indirect_destroy = indirect_destroy_check;
3322 __kmp_direct_set = direct_set;
3323 __kmp_direct_unset = direct_unset;
3324 __kmp_direct_test = direct_test;
3325 __kmp_direct_destroy = direct_destroy;
3326 __kmp_indirect_set = indirect_set;
3327 __kmp_indirect_unset = indirect_unset;
3328 __kmp_indirect_test = indirect_test;
3329 __kmp_indirect_destroy = indirect_destroy;
3334 if (__kmp_init_user_locks)
3338 __kmp_i_lock_table.nrow_ptrs = KMP_I_LOCK_TABLE_INIT_NROW_PTRS;
3339 __kmp_i_lock_table.table = (kmp_indirect_lock_t **)__kmp_allocate(
3340 sizeof(kmp_indirect_lock_t *) * KMP_I_LOCK_TABLE_INIT_NROW_PTRS);
3341 *(__kmp_i_lock_table.table) = (kmp_indirect_lock_t *)__kmp_allocate(
3342 KMP_I_LOCK_CHUNK *
sizeof(kmp_indirect_lock_t));
3343 __kmp_i_lock_table.next = 0;
3344 __kmp_i_lock_table.next_table =
nullptr;
3347 __kmp_indirect_lock_size[locktag_ticket] =
sizeof(kmp_ticket_lock_t);
3348 __kmp_indirect_lock_size[locktag_queuing] =
sizeof(kmp_queuing_lock_t);
3349 #if KMP_USE_ADAPTIVE_LOCKS
3350 __kmp_indirect_lock_size[locktag_adaptive] =
sizeof(kmp_adaptive_lock_t);
3352 __kmp_indirect_lock_size[locktag_drdpa] =
sizeof(kmp_drdpa_lock_t);
3354 __kmp_indirect_lock_size[locktag_rtm_queuing] =
sizeof(kmp_queuing_lock_t);
3356 __kmp_indirect_lock_size[locktag_nested_tas] =
sizeof(kmp_tas_lock_t);
3358 __kmp_indirect_lock_size[locktag_nested_futex] =
sizeof(kmp_futex_lock_t);
3360 __kmp_indirect_lock_size[locktag_nested_ticket] =
sizeof(kmp_ticket_lock_t);
3361 __kmp_indirect_lock_size[locktag_nested_queuing] =
sizeof(kmp_queuing_lock_t);
3362 __kmp_indirect_lock_size[locktag_nested_drdpa] =
sizeof(kmp_drdpa_lock_t);
3365 #define fill_jumps(table, expand, sep) \
3367 table[locktag##sep##ticket] = expand(ticket); \
3368 table[locktag##sep##queuing] = expand(queuing); \
3369 table[locktag##sep##drdpa] = expand(drdpa); \
3372 #if KMP_USE_ADAPTIVE_LOCKS
3373 #define fill_table(table, expand) \
3375 fill_jumps(table, expand, _); \
3376 table[locktag_adaptive] = expand(queuing); \
3377 fill_jumps(table, expand, _nested_); \
3380 #define fill_table(table, expand) \
3382 fill_jumps(table, expand, _); \
3383 fill_jumps(table, expand, _nested_); \
3388 (void (*)(kmp_user_lock_p, const ident_t *)) __kmp_set_##l##_lock_location
3389 fill_table(__kmp_indirect_set_location, expand);
3392 (void (*)(kmp_user_lock_p, kmp_lock_flags_t)) __kmp_set_##l##_lock_flags
3393 fill_table(__kmp_indirect_set_flags, expand);
3396 (const ident_t *(*)(kmp_user_lock_p)) __kmp_get_##l##_lock_location
3397 fill_table(__kmp_indirect_get_location, expand);
3400 (kmp_lock_flags_t(*)(kmp_user_lock_p)) __kmp_get_##l##_lock_flags
3401 fill_table(__kmp_indirect_get_flags, expand);
3404 __kmp_init_user_locks = TRUE;
3408 void __kmp_cleanup_indirect_user_locks() {
3413 for (k = 0; k < KMP_NUM_I_LOCKS; ++k) {
3414 kmp_indirect_lock_t *l = __kmp_indirect_lock_pool[k];
3416 kmp_indirect_lock_t *ll = l;
3417 l = (kmp_indirect_lock_t *)l->lock->pool.next;
3418 KA_TRACE(20, (
"__kmp_cleanup_indirect_user_locks: freeing %p from pool\n",
3420 __kmp_free(ll->lock);
3423 __kmp_indirect_lock_pool[k] = NULL;
3426 kmp_indirect_lock_table_t *ptr = &__kmp_i_lock_table;
3428 for (kmp_uint32 row = 0; row < ptr->nrow_ptrs; ++row) {
3429 if (!ptr->table[row])
3431 for (kmp_uint32 col = 0; col < KMP_I_LOCK_CHUNK; ++col) {
3432 kmp_indirect_lock_t *l = &ptr->table[row][col];
3435 KMP_I_LOCK_FUNC(l, destroy)(l->lock);
3436 KA_TRACE(20, (
"__kmp_cleanup_indirect_user_locks: destroy/freeing %p "
3439 __kmp_free(l->lock);
3442 __kmp_free(ptr->table[row]);
3444 kmp_indirect_lock_table_t *next_table = ptr->next_table;
3445 if (ptr != &__kmp_i_lock_table)
3450 __kmp_init_user_locks = FALSE;
3453 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3454 int __kmp_num_locks_in_block = 1;
3458 static void __kmp_init_tas_lock_with_checks(kmp_tas_lock_t *lck) {
3459 __kmp_init_tas_lock(lck);
3462 static void __kmp_init_nested_tas_lock_with_checks(kmp_tas_lock_t *lck) {
3463 __kmp_init_nested_tas_lock(lck);
3467 static void __kmp_init_futex_lock_with_checks(kmp_futex_lock_t *lck) {
3468 __kmp_init_futex_lock(lck);
3471 static void __kmp_init_nested_futex_lock_with_checks(kmp_futex_lock_t *lck) {
3472 __kmp_init_nested_futex_lock(lck);
3476 static int __kmp_is_ticket_lock_initialized(kmp_ticket_lock_t *lck) {
3477 return lck == lck->lk.self;
3480 static void __kmp_init_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
3481 __kmp_init_ticket_lock(lck);
3484 static void __kmp_init_nested_ticket_lock_with_checks(kmp_ticket_lock_t *lck) {
3485 __kmp_init_nested_ticket_lock(lck);
3488 static int __kmp_is_queuing_lock_initialized(kmp_queuing_lock_t *lck) {
3489 return lck == lck->lk.initialized;
3492 static void __kmp_init_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
3493 __kmp_init_queuing_lock(lck);
3497 __kmp_init_nested_queuing_lock_with_checks(kmp_queuing_lock_t *lck) {
3498 __kmp_init_nested_queuing_lock(lck);
3501 #if KMP_USE_ADAPTIVE_LOCKS
3502 static void __kmp_init_adaptive_lock_with_checks(kmp_adaptive_lock_t *lck) {
3503 __kmp_init_adaptive_lock(lck);
3507 static int __kmp_is_drdpa_lock_initialized(kmp_drdpa_lock_t *lck) {
3508 return lck == lck->lk.initialized;
3511 static void __kmp_init_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
3512 __kmp_init_drdpa_lock(lck);
3515 static void __kmp_init_nested_drdpa_lock_with_checks(kmp_drdpa_lock_t *lck) {
3516 __kmp_init_nested_drdpa_lock(lck);
3523 enum kmp_lock_kind __kmp_user_lock_kind = lk_default;
3525 size_t __kmp_base_user_lock_size = 0;
3526 size_t __kmp_user_lock_size = 0;
3528 kmp_int32 (*__kmp_get_user_lock_owner_)(kmp_user_lock_p lck) = NULL;
3529 int (*__kmp_acquire_user_lock_with_checks_)(kmp_user_lock_p lck,
3530 kmp_int32 gtid) = NULL;
3532 int (*__kmp_test_user_lock_with_checks_)(kmp_user_lock_p lck,
3533 kmp_int32 gtid) = NULL;
3534 int (*__kmp_release_user_lock_with_checks_)(kmp_user_lock_p lck,
3535 kmp_int32 gtid) = NULL;
3536 void (*__kmp_init_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3537 void (*__kmp_destroy_user_lock_)(kmp_user_lock_p lck) = NULL;
3538 void (*__kmp_destroy_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3539 int (*__kmp_acquire_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3540 kmp_int32 gtid) = NULL;
3542 int (*__kmp_test_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3543 kmp_int32 gtid) = NULL;
3544 int (*__kmp_release_nested_user_lock_with_checks_)(kmp_user_lock_p lck,
3545 kmp_int32 gtid) = NULL;
3546 void (*__kmp_init_nested_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3547 void (*__kmp_destroy_nested_user_lock_with_checks_)(kmp_user_lock_p lck) = NULL;
3549 int (*__kmp_is_user_lock_initialized_)(kmp_user_lock_p lck) = NULL;
3550 const ident_t *(*__kmp_get_user_lock_location_)(kmp_user_lock_p lck) = NULL;
3551 void (*__kmp_set_user_lock_location_)(kmp_user_lock_p lck,
3553 kmp_lock_flags_t (*__kmp_get_user_lock_flags_)(kmp_user_lock_p lck) = NULL;
3554 void (*__kmp_set_user_lock_flags_)(kmp_user_lock_p lck,
3555 kmp_lock_flags_t flags) = NULL;
3557 void __kmp_set_user_lock_vptrs(kmp_lock_kind_t user_lock_kind) {
3558 switch (user_lock_kind) {
3564 __kmp_base_user_lock_size =
sizeof(kmp_base_tas_lock_t);
3565 __kmp_user_lock_size =
sizeof(kmp_tas_lock_t);
3567 __kmp_get_user_lock_owner_ =
3568 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_tas_lock_owner);
3570 if (__kmp_env_consistency_check) {
3571 KMP_BIND_USER_LOCK_WITH_CHECKS(tas);
3572 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(tas);
3574 KMP_BIND_USER_LOCK(tas);
3575 KMP_BIND_NESTED_USER_LOCK(tas);
3578 __kmp_destroy_user_lock_ =
3579 (void (*)(kmp_user_lock_p))(&__kmp_destroy_tas_lock);
3581 __kmp_is_user_lock_initialized_ = (int (*)(kmp_user_lock_p))NULL;
3583 __kmp_get_user_lock_location_ = (
const ident_t *(*)(kmp_user_lock_p))NULL;
3585 __kmp_set_user_lock_location_ =
3586 (void (*)(kmp_user_lock_p,
const ident_t *))NULL;
3588 __kmp_get_user_lock_flags_ = (kmp_lock_flags_t(*)(kmp_user_lock_p))NULL;
3590 __kmp_set_user_lock_flags_ =
3591 (void (*)(kmp_user_lock_p, kmp_lock_flags_t))NULL;
3597 __kmp_base_user_lock_size =
sizeof(kmp_base_futex_lock_t);
3598 __kmp_user_lock_size =
sizeof(kmp_futex_lock_t);
3600 __kmp_get_user_lock_owner_ =
3601 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_futex_lock_owner);
3603 if (__kmp_env_consistency_check) {
3604 KMP_BIND_USER_LOCK_WITH_CHECKS(futex);
3605 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(futex);
3607 KMP_BIND_USER_LOCK(futex);
3608 KMP_BIND_NESTED_USER_LOCK(futex);
3611 __kmp_destroy_user_lock_ =
3612 (void (*)(kmp_user_lock_p))(&__kmp_destroy_futex_lock);
3614 __kmp_is_user_lock_initialized_ = (int (*)(kmp_user_lock_p))NULL;
3616 __kmp_get_user_lock_location_ = (
const ident_t *(*)(kmp_user_lock_p))NULL;
3618 __kmp_set_user_lock_location_ =
3619 (void (*)(kmp_user_lock_p,
const ident_t *))NULL;
3621 __kmp_get_user_lock_flags_ = (kmp_lock_flags_t(*)(kmp_user_lock_p))NULL;
3623 __kmp_set_user_lock_flags_ =
3624 (void (*)(kmp_user_lock_p, kmp_lock_flags_t))NULL;
3630 __kmp_base_user_lock_size =
sizeof(kmp_base_ticket_lock_t);
3631 __kmp_user_lock_size =
sizeof(kmp_ticket_lock_t);
3633 __kmp_get_user_lock_owner_ =
3634 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_ticket_lock_owner);
3636 if (__kmp_env_consistency_check) {
3637 KMP_BIND_USER_LOCK_WITH_CHECKS(ticket);
3638 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(ticket);
3640 KMP_BIND_USER_LOCK(ticket);
3641 KMP_BIND_NESTED_USER_LOCK(ticket);
3644 __kmp_destroy_user_lock_ =
3645 (void (*)(kmp_user_lock_p))(&__kmp_destroy_ticket_lock);
3647 __kmp_is_user_lock_initialized_ =
3648 (int (*)(kmp_user_lock_p))(&__kmp_is_ticket_lock_initialized);
3650 __kmp_get_user_lock_location_ =
3651 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_ticket_lock_location);
3653 __kmp_set_user_lock_location_ = (void (*)(
3654 kmp_user_lock_p,
const ident_t *))(&__kmp_set_ticket_lock_location);
3656 __kmp_get_user_lock_flags_ =
3657 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_ticket_lock_flags);
3659 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3660 &__kmp_set_ticket_lock_flags);
3664 __kmp_base_user_lock_size =
sizeof(kmp_base_queuing_lock_t);
3665 __kmp_user_lock_size =
sizeof(kmp_queuing_lock_t);
3667 __kmp_get_user_lock_owner_ =
3668 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_owner);
3670 if (__kmp_env_consistency_check) {
3671 KMP_BIND_USER_LOCK_WITH_CHECKS(queuing);
3672 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(queuing);
3674 KMP_BIND_USER_LOCK(queuing);
3675 KMP_BIND_NESTED_USER_LOCK(queuing);
3678 __kmp_destroy_user_lock_ =
3679 (void (*)(kmp_user_lock_p))(&__kmp_destroy_queuing_lock);
3681 __kmp_is_user_lock_initialized_ =
3682 (int (*)(kmp_user_lock_p))(&__kmp_is_queuing_lock_initialized);
3684 __kmp_get_user_lock_location_ =
3685 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_location);
3687 __kmp_set_user_lock_location_ = (void (*)(
3688 kmp_user_lock_p,
const ident_t *))(&__kmp_set_queuing_lock_location);
3690 __kmp_get_user_lock_flags_ =
3691 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_flags);
3693 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3694 &__kmp_set_queuing_lock_flags);
3697 #if KMP_USE_ADAPTIVE_LOCKS
3699 __kmp_base_user_lock_size =
sizeof(kmp_base_adaptive_lock_t);
3700 __kmp_user_lock_size =
sizeof(kmp_adaptive_lock_t);
3702 __kmp_get_user_lock_owner_ =
3703 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_owner);
3705 if (__kmp_env_consistency_check) {
3706 KMP_BIND_USER_LOCK_WITH_CHECKS(adaptive);
3708 KMP_BIND_USER_LOCK(adaptive);
3711 __kmp_destroy_user_lock_ =
3712 (void (*)(kmp_user_lock_p))(&__kmp_destroy_adaptive_lock);
3714 __kmp_is_user_lock_initialized_ =
3715 (int (*)(kmp_user_lock_p))(&__kmp_is_queuing_lock_initialized);
3717 __kmp_get_user_lock_location_ =
3718 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_location);
3720 __kmp_set_user_lock_location_ = (void (*)(
3721 kmp_user_lock_p,
const ident_t *))(&__kmp_set_queuing_lock_location);
3723 __kmp_get_user_lock_flags_ =
3724 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_queuing_lock_flags);
3726 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3727 &__kmp_set_queuing_lock_flags);
3733 __kmp_base_user_lock_size =
sizeof(kmp_base_drdpa_lock_t);
3734 __kmp_user_lock_size =
sizeof(kmp_drdpa_lock_t);
3736 __kmp_get_user_lock_owner_ =
3737 (kmp_int32(*)(kmp_user_lock_p))(&__kmp_get_drdpa_lock_owner);
3739 if (__kmp_env_consistency_check) {
3740 KMP_BIND_USER_LOCK_WITH_CHECKS(drdpa);
3741 KMP_BIND_NESTED_USER_LOCK_WITH_CHECKS(drdpa);
3743 KMP_BIND_USER_LOCK(drdpa);
3744 KMP_BIND_NESTED_USER_LOCK(drdpa);
3747 __kmp_destroy_user_lock_ =
3748 (void (*)(kmp_user_lock_p))(&__kmp_destroy_drdpa_lock);
3750 __kmp_is_user_lock_initialized_ =
3751 (int (*)(kmp_user_lock_p))(&__kmp_is_drdpa_lock_initialized);
3753 __kmp_get_user_lock_location_ =
3754 (
const ident_t *(*)(kmp_user_lock_p))(&__kmp_get_drdpa_lock_location);
3756 __kmp_set_user_lock_location_ = (void (*)(
3757 kmp_user_lock_p,
const ident_t *))(&__kmp_set_drdpa_lock_location);
3759 __kmp_get_user_lock_flags_ =
3760 (kmp_lock_flags_t(*)(kmp_user_lock_p))(&__kmp_get_drdpa_lock_flags);
3762 __kmp_set_user_lock_flags_ = (void (*)(kmp_user_lock_p, kmp_lock_flags_t))(
3763 &__kmp_set_drdpa_lock_flags);
3771 kmp_lock_table_t __kmp_user_lock_table = {1, 0, NULL};
3772 kmp_user_lock_p __kmp_lock_pool = NULL;
3775 kmp_block_of_locks *__kmp_lock_blocks = NULL;
3776 int __kmp_num_locks_in_block = 1;
3778 static kmp_lock_index_t __kmp_lock_table_insert(kmp_user_lock_p lck) {
3780 kmp_lock_index_t index;
3781 if (__kmp_user_lock_table.used >= __kmp_user_lock_table.allocated) {
3782 kmp_lock_index_t size;
3783 kmp_user_lock_p *table;
3785 if (__kmp_user_lock_table.allocated == 0) {
3788 size = __kmp_user_lock_table.allocated * 2;
3790 table = (kmp_user_lock_p *)__kmp_allocate(
sizeof(kmp_user_lock_p) * size);
3791 KMP_MEMCPY(table + 1, __kmp_user_lock_table.table + 1,
3792 sizeof(kmp_user_lock_p) * (__kmp_user_lock_table.used - 1));
3793 table[0] = (kmp_user_lock_p)__kmp_user_lock_table.table;
3798 __kmp_user_lock_table.table = table;
3799 __kmp_user_lock_table.allocated = size;
3801 KMP_DEBUG_ASSERT(__kmp_user_lock_table.used <
3802 __kmp_user_lock_table.allocated);
3803 index = __kmp_user_lock_table.used;
3804 __kmp_user_lock_table.table[index] = lck;
3805 ++__kmp_user_lock_table.used;
3809 static kmp_user_lock_p __kmp_lock_block_allocate() {
3811 static int last_index = 0;
3812 if ((last_index >= __kmp_num_locks_in_block) || (__kmp_lock_blocks == NULL)) {
3816 KMP_DEBUG_ASSERT(__kmp_user_lock_size > 0);
3817 size_t space_for_locks = __kmp_user_lock_size * __kmp_num_locks_in_block;
3819 (
char *)__kmp_allocate(space_for_locks +
sizeof(kmp_block_of_locks));
3821 kmp_block_of_locks *new_block =
3822 (kmp_block_of_locks *)(&buffer[space_for_locks]);
3823 new_block->next_block = __kmp_lock_blocks;
3824 new_block->locks = (
void *)buffer;
3827 __kmp_lock_blocks = new_block;
3829 kmp_user_lock_p ret = (kmp_user_lock_p)(&(
3830 ((
char *)(__kmp_lock_blocks->locks))[last_index * __kmp_user_lock_size]));
3837 kmp_user_lock_p __kmp_user_lock_allocate(
void **user_lock, kmp_int32 gtid,
3838 kmp_lock_flags_t flags) {
3839 kmp_user_lock_p lck;
3840 kmp_lock_index_t index;
3841 KMP_DEBUG_ASSERT(user_lock);
3843 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3845 if (__kmp_lock_pool == NULL) {
3848 if (__kmp_num_locks_in_block <= 1) {
3849 lck = (kmp_user_lock_p)__kmp_allocate(__kmp_user_lock_size);
3851 lck = __kmp_lock_block_allocate();
3856 index = __kmp_lock_table_insert(lck);
3859 lck = __kmp_lock_pool;
3860 index = __kmp_lock_pool->pool.index;
3861 __kmp_lock_pool = __kmp_lock_pool->pool.next;
3866 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3867 *((kmp_lock_index_t *)user_lock) = index;
3869 *((kmp_user_lock_p *)user_lock) = lck;
3873 __kmp_set_user_lock_flags(lck, flags);
3875 __kmp_release_lock(&__kmp_global_lock, gtid);
3881 void __kmp_user_lock_free(
void **user_lock, kmp_int32 gtid,
3882 kmp_user_lock_p lck) {
3883 KMP_DEBUG_ASSERT(user_lock != NULL);
3884 KMP_DEBUG_ASSERT(lck != NULL);
3886 __kmp_acquire_lock(&__kmp_global_lock, gtid);
3888 lck->pool.next = __kmp_lock_pool;
3889 __kmp_lock_pool = lck;
3890 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3891 kmp_lock_index_t index = *((kmp_lock_index_t *)user_lock);
3892 KMP_DEBUG_ASSERT(0 < index && index <= __kmp_user_lock_table.used);
3893 lck->pool.index = index;
3896 __kmp_release_lock(&__kmp_global_lock, gtid);
3899 kmp_user_lock_p __kmp_lookup_user_lock(
void **user_lock,
char const *func) {
3900 kmp_user_lock_p lck = NULL;
3902 if (__kmp_env_consistency_check) {
3903 if (user_lock == NULL) {
3904 KMP_FATAL(LockIsUninitialized, func);
3908 if (OMP_LOCK_T_SIZE <
sizeof(
void *)) {
3909 kmp_lock_index_t index = *((kmp_lock_index_t *)user_lock);
3910 if (__kmp_env_consistency_check) {
3911 if (!(0 < index && index < __kmp_user_lock_table.used)) {
3912 KMP_FATAL(LockIsUninitialized, func);
3915 KMP_DEBUG_ASSERT(0 < index && index < __kmp_user_lock_table.used);
3916 KMP_DEBUG_ASSERT(__kmp_user_lock_size > 0);
3917 lck = __kmp_user_lock_table.table[index];
3919 lck = *((kmp_user_lock_p *)user_lock);
3922 if (__kmp_env_consistency_check) {
3924 KMP_FATAL(LockIsUninitialized, func);
3931 void __kmp_cleanup_user_locks(
void) {
3934 __kmp_lock_pool = NULL;
3936 #define IS_CRITICAL(lck) \
3937 ((__kmp_get_user_lock_flags_ != NULL) && \
3938 ((*__kmp_get_user_lock_flags_)(lck)&kmp_lf_critical_section))
3963 while (__kmp_user_lock_table.used > 1) {
3968 kmp_user_lock_p lck =
3969 __kmp_user_lock_table.table[--__kmp_user_lock_table.used];
3971 if ((__kmp_is_user_lock_initialized_ != NULL) &&
3972 (*__kmp_is_user_lock_initialized_)(lck)) {
3976 if (__kmp_env_consistency_check && (!IS_CRITICAL(lck)) &&
3977 ((loc = __kmp_get_user_lock_location(lck)) != NULL) &&
3979 kmp_str_loc_t str_loc = __kmp_str_loc_init(loc->
psource,
false);
3980 KMP_WARNING(CnsLockNotDestroyed, str_loc.file, str_loc.line);
3981 __kmp_str_loc_free(&str_loc);
3985 if (IS_CRITICAL(lck)) {
3988 (
"__kmp_cleanup_user_locks: free critical section lock %p (%p)\n",
3989 lck, *(
void **)lck));
3991 KA_TRACE(20, (
"__kmp_cleanup_user_locks: free lock %p (%p)\n", lck,
3997 __kmp_destroy_user_lock(lck);
4001 if (__kmp_lock_blocks == NULL) {
4009 kmp_user_lock_p *table_ptr = __kmp_user_lock_table.table;
4010 __kmp_user_lock_table.table = NULL;
4011 __kmp_user_lock_table.allocated = 0;
4013 while (table_ptr != NULL) {
4016 kmp_user_lock_p *next = (kmp_user_lock_p *)(table_ptr[0]);
4017 __kmp_free(table_ptr);
4022 kmp_block_of_locks_t *block_ptr = __kmp_lock_blocks;
4023 __kmp_lock_blocks = NULL;
4025 while (block_ptr != NULL) {
4026 kmp_block_of_locks_t *next = block_ptr->next_block;
4027 __kmp_free(block_ptr->locks);
4032 TCW_4(__kmp_init_user_locks, FALSE);
void open(const char *filename, const char *mode, const char *env_var=nullptr)