14 #include "kmp_atomic.h"
17 typedef unsigned char uchar;
18 typedef unsigned short ushort;
566 #ifndef KMP_GOMP_COMPAT
567 int __kmp_atomic_mode = 1;
569 int __kmp_atomic_mode = 2;
575 kmp_atomic_lock_t __kmp_atomic_lock;
577 kmp_atomic_lock_t __kmp_atomic_lock_1i;
579 kmp_atomic_lock_t __kmp_atomic_lock_2i;
581 kmp_atomic_lock_t __kmp_atomic_lock_4i;
583 kmp_atomic_lock_t __kmp_atomic_lock_4r;
585 kmp_atomic_lock_t __kmp_atomic_lock_8i;
587 kmp_atomic_lock_t __kmp_atomic_lock_8r;
589 kmp_atomic_lock_t __kmp_atomic_lock_8c;
591 kmp_atomic_lock_t __kmp_atomic_lock_10r;
593 kmp_atomic_lock_t __kmp_atomic_lock_16r;
595 kmp_atomic_lock_t __kmp_atomic_lock_16c;
597 kmp_atomic_lock_t __kmp_atomic_lock_20c;
599 kmp_atomic_lock_t __kmp_atomic_lock_32c;
606 #define KMP_ATOMIC_VOLATILE volatile
608 #if (KMP_ARCH_X86) && KMP_HAVE_QUAD
610 static inline void operator+=(Quad_a4_t &lhs, Quad_a4_t &rhs) {
613 static inline void operator-=(Quad_a4_t &lhs, Quad_a4_t &rhs) {
616 static inline void operator*=(Quad_a4_t &lhs, Quad_a4_t &rhs) {
619 static inline void operator/=(Quad_a4_t &lhs, Quad_a4_t &rhs) {
622 static inline bool operator<(Quad_a4_t &lhs, Quad_a4_t &rhs) {
623 return lhs.q < rhs.q;
625 static inline bool operator>(Quad_a4_t &lhs, Quad_a4_t &rhs) {
626 return lhs.q > rhs.q;
629 static inline void operator+=(Quad_a16_t &lhs, Quad_a16_t &rhs) {
632 static inline void operator-=(Quad_a16_t &lhs, Quad_a16_t &rhs) {
635 static inline void operator*=(Quad_a16_t &lhs, Quad_a16_t &rhs) {
638 static inline void operator/=(Quad_a16_t &lhs, Quad_a16_t &rhs) {
641 static inline bool operator<(Quad_a16_t &lhs, Quad_a16_t &rhs) {
642 return lhs.q < rhs.q;
644 static inline bool operator>(Quad_a16_t &lhs, Quad_a16_t &rhs) {
645 return lhs.q > rhs.q;
648 static inline void operator+=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) {
651 static inline void operator-=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) {
654 static inline void operator*=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) {
657 static inline void operator/=(kmp_cmplx128_a4_t &lhs, kmp_cmplx128_a4_t &rhs) {
661 static inline void operator+=(kmp_cmplx128_a16_t &lhs,
662 kmp_cmplx128_a16_t &rhs) {
665 static inline void operator-=(kmp_cmplx128_a16_t &lhs,
666 kmp_cmplx128_a16_t &rhs) {
669 static inline void operator*=(kmp_cmplx128_a16_t &lhs,
670 kmp_cmplx128_a16_t &rhs) {
673 static inline void operator/=(kmp_cmplx128_a16_t &lhs,
674 kmp_cmplx128_a16_t &rhs) {
685 #define KMP_CHECK_GTID \
686 if (gtid == KMP_GTID_UNKNOWN) { \
687 gtid = __kmp_entry_gtid(); \
688 } // check and get gtid when needed
695 #define ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
696 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \
697 TYPE *lhs, TYPE rhs) { \
698 KMP_DEBUG_ASSERT(__kmp_init_serial); \
699 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
703 #define ATOMIC_LOCK0 __kmp_atomic_lock // all types, for Gnu compat
704 #define ATOMIC_LOCK1i __kmp_atomic_lock_1i // char
705 #define ATOMIC_LOCK2i __kmp_atomic_lock_2i // short
706 #define ATOMIC_LOCK4i __kmp_atomic_lock_4i // long int
707 #define ATOMIC_LOCK4r __kmp_atomic_lock_4r // float
708 #define ATOMIC_LOCK8i __kmp_atomic_lock_8i // long long int
709 #define ATOMIC_LOCK8r __kmp_atomic_lock_8r // double
710 #define ATOMIC_LOCK8c __kmp_atomic_lock_8c // float complex
711 #define ATOMIC_LOCK10r __kmp_atomic_lock_10r // long double
712 #define ATOMIC_LOCK16r __kmp_atomic_lock_16r // _Quad
713 #define ATOMIC_LOCK16c __kmp_atomic_lock_16c // double complex
714 #define ATOMIC_LOCK20c __kmp_atomic_lock_20c // long double complex
715 #define ATOMIC_LOCK32c __kmp_atomic_lock_32c // _Quad complex
723 #define OP_CRITICAL(OP, LCK_ID) \
724 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
728 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
752 #ifdef KMP_GOMP_COMPAT
753 #define OP_GOMP_CRITICAL(OP, FLAG) \
754 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
756 OP_CRITICAL(OP, 0); \
760 #define OP_GOMP_CRITICAL(OP, FLAG)
764 #define KMP_DO_PAUSE _mm_delay_32(1)
766 #define KMP_DO_PAUSE KMP_CPU_PAUSE()
774 #define OP_CMPXCHG(TYPE, BITS, OP) \
776 TYPE old_value, new_value; \
777 old_value = *(TYPE volatile *)lhs; \
778 new_value = old_value OP rhs; \
779 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
780 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
781 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
784 old_value = *(TYPE volatile *)lhs; \
785 new_value = old_value OP rhs; \
796 #define OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \
800 kmp_int##BITS *vvv; \
802 struct _sss old_value, new_value; \
803 old_value.vvv = (kmp_int##BITS *)&old_value.cmp; \
804 new_value.vvv = (kmp_int##BITS *)&new_value.cmp; \
805 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \
806 new_value.cmp = old_value.cmp OP rhs; \
807 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
808 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \
809 *VOLATILE_CAST(kmp_int##BITS *) new_value.vvv)) { \
812 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \
813 new_value.cmp = old_value.cmp OP rhs; \
817 #endif // USE_CMPXCHG_FIX
819 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
823 #define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
825 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
826 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
828 KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \
831 #define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
833 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
834 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
835 OP_CMPXCHG(TYPE, BITS, OP) \
840 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \
842 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
843 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
844 OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \
852 #define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
854 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
855 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
856 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
858 KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \
861 OP_CRITICAL(OP## =, LCK_ID) \
865 #define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
867 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
868 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
869 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
870 OP_CMPXCHG(TYPE, BITS, OP) \
873 OP_CRITICAL(OP## =, LCK_ID) \
879 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \
881 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
882 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
883 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
884 OP_CMPXCHG(TYPE, BITS, OP) \
887 OP_CRITICAL(OP## =, LCK_ID) \
891 #endif // USE_CMPXCHG_FIX
895 ATOMIC_FIXED_ADD(fixed4, add, kmp_int32, 32, +, 4i, 3,
897 ATOMIC_FIXED_ADD(fixed4, sub, kmp_int32, 32, -, 4i, 3,
900 ATOMIC_CMPXCHG(float4, add, kmp_real32, 32, +, 4r, 3,
902 ATOMIC_CMPXCHG(float4, sub, kmp_real32, 32, -, 4r, 3,
906 ATOMIC_FIXED_ADD(fixed8, add, kmp_int64, 64, +, 8i, 7,
908 ATOMIC_FIXED_ADD(fixed8, sub, kmp_int64, 64, -, 8i, 7,
911 ATOMIC_CMPXCHG(float8, add, kmp_real64, 64, +, 8r, 7,
913 ATOMIC_CMPXCHG(float8, sub, kmp_real64, 64, -, 8r, 7,
931 ATOMIC_CMPXCHG(fixed1, add, kmp_int8, 8, +, 1i, 0,
933 ATOMIC_CMPXCHG(fixed1, andb, kmp_int8, 8, &, 1i, 0,
935 ATOMIC_CMPXCHG(fixed1, div, kmp_int8, 8, /, 1i, 0,
937 ATOMIC_CMPXCHG(fixed1u, div, kmp_uint8, 8, /, 1i, 0,
939 ATOMIC_CMPXCHG(fixed1, mul, kmp_int8, 8, *, 1i, 0,
941 ATOMIC_CMPXCHG(fixed1, orb, kmp_int8, 8, |, 1i, 0,
943 ATOMIC_CMPXCHG(fixed1, shl, kmp_int8, 8, <<, 1i, 0,
945 ATOMIC_CMPXCHG(fixed1, shr, kmp_int8, 8, >>, 1i, 0,
947 ATOMIC_CMPXCHG(fixed1u, shr, kmp_uint8, 8, >>, 1i, 0,
949 ATOMIC_CMPXCHG(fixed1, sub, kmp_int8, 8, -, 1i, 0,
951 ATOMIC_CMPXCHG(fixed1, xor, kmp_int8, 8, ^, 1i, 0,
953 ATOMIC_CMPXCHG(fixed2, add, kmp_int16, 16, +, 2i, 1,
955 ATOMIC_CMPXCHG(fixed2, andb, kmp_int16, 16, &, 2i, 1,
957 ATOMIC_CMPXCHG(fixed2, div, kmp_int16, 16, /, 2i, 1,
959 ATOMIC_CMPXCHG(fixed2u, div, kmp_uint16, 16, /, 2i, 1,
961 ATOMIC_CMPXCHG(fixed2, mul, kmp_int16, 16, *, 2i, 1,
963 ATOMIC_CMPXCHG(fixed2, orb, kmp_int16, 16, |, 2i, 1,
965 ATOMIC_CMPXCHG(fixed2, shl, kmp_int16, 16, <<, 2i, 1,
967 ATOMIC_CMPXCHG(fixed2, shr, kmp_int16, 16, >>, 2i, 1,
969 ATOMIC_CMPXCHG(fixed2u, shr, kmp_uint16, 16, >>, 2i, 1,
971 ATOMIC_CMPXCHG(fixed2, sub, kmp_int16, 16, -, 2i, 1,
973 ATOMIC_CMPXCHG(fixed2, xor, kmp_int16, 16, ^, 2i, 1,
975 ATOMIC_CMPXCHG(fixed4, andb, kmp_int32, 32, &, 4i, 3,
977 ATOMIC_CMPXCHG(fixed4, div, kmp_int32, 32, /, 4i, 3,
979 ATOMIC_CMPXCHG(fixed4u, div, kmp_uint32, 32, /, 4i, 3,
981 ATOMIC_CMPXCHG(fixed4, mul, kmp_int32, 32, *, 4i, 3,
983 ATOMIC_CMPXCHG(fixed4, orb, kmp_int32, 32, |, 4i, 3,
985 ATOMIC_CMPXCHG(fixed4, shl, kmp_int32, 32, <<, 4i, 3,
987 ATOMIC_CMPXCHG(fixed4, shr, kmp_int32, 32, >>, 4i, 3,
989 ATOMIC_CMPXCHG(fixed4u, shr, kmp_uint32, 32, >>, 4i, 3,
991 ATOMIC_CMPXCHG(fixed4, xor, kmp_int32, 32, ^, 4i, 3,
993 ATOMIC_CMPXCHG(fixed8, andb, kmp_int64, 64, &, 8i, 7,
995 ATOMIC_CMPXCHG(fixed8, div, kmp_int64, 64, /, 8i, 7,
997 ATOMIC_CMPXCHG(fixed8u, div, kmp_uint64, 64, /, 8i, 7,
999 ATOMIC_CMPXCHG(fixed8, mul, kmp_int64, 64, *, 8i, 7,
1001 ATOMIC_CMPXCHG(fixed8, orb, kmp_int64, 64, |, 8i, 7,
1003 ATOMIC_CMPXCHG(fixed8, shl, kmp_int64, 64, <<, 8i, 7,
1005 ATOMIC_CMPXCHG(fixed8, shr, kmp_int64, 64, >>, 8i, 7,
1007 ATOMIC_CMPXCHG(fixed8u, shr, kmp_uint64, 64, >>, 8i, 7,
1009 ATOMIC_CMPXCHG(fixed8, xor, kmp_int64, 64, ^, 8i, 7,
1011 ATOMIC_CMPXCHG(float4, div, kmp_real32, 32, /, 4r, 3,
1013 ATOMIC_CMPXCHG(float4, mul, kmp_real32, 32, *, 4r, 3,
1015 ATOMIC_CMPXCHG(float8, div, kmp_real64, 64, /, 8r, 7,
1017 ATOMIC_CMPXCHG(float8, mul, kmp_real64, 64, *, 8r, 7,
1027 #define ATOMIC_CRIT_L(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1028 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1029 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \
1030 OP_CRITICAL(= *lhs OP, LCK_ID) \
1033 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1037 #define ATOMIC_CMPX_L(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG) \
1038 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1039 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \
1040 OP_CMPXCHG(TYPE, BITS, OP) \
1046 #define ATOMIC_CMPX_L(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG) \
1047 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1048 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \
1049 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1050 OP_CMPXCHG(TYPE, BITS, OP) \
1053 OP_CRITICAL(= *lhs OP, LCK_ID) \
1058 ATOMIC_CMPX_L(fixed1, andl,
char, 8, &&, 1i, 0,
1060 ATOMIC_CMPX_L(fixed1, orl,
char, 8, ||, 1i, 0,
1062 ATOMIC_CMPX_L(fixed2, andl,
short, 16, &&, 2i, 1,
1064 ATOMIC_CMPX_L(fixed2, orl,
short, 16, ||, 2i, 1,
1066 ATOMIC_CMPX_L(fixed4, andl, kmp_int32, 32, &&, 4i, 3,
1068 ATOMIC_CMPX_L(fixed4, orl, kmp_int32, 32, ||, 4i, 3,
1070 ATOMIC_CMPX_L(fixed8, andl, kmp_int64, 64, &&, 8i, 7,
1072 ATOMIC_CMPX_L(fixed8, orl, kmp_int64, 64, ||, 8i, 7,
1084 #define MIN_MAX_CRITSECT(OP, LCK_ID) \
1085 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
1087 if (*lhs OP rhs) { \
1090 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
1093 #ifdef KMP_GOMP_COMPAT
1094 #define GOMP_MIN_MAX_CRITSECT(OP, FLAG) \
1095 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
1097 MIN_MAX_CRITSECT(OP, 0); \
1101 #define GOMP_MIN_MAX_CRITSECT(OP, FLAG)
1105 #define MIN_MAX_CMPXCHG(TYPE, BITS, OP) \
1107 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1110 old_value = temp_val; \
1111 while (old_value OP rhs && \
1112 !KMP_COMPARE_AND_STORE_ACQ##BITS( \
1113 (kmp_int##BITS *)lhs, \
1114 *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
1115 *VOLATILE_CAST(kmp_int##BITS *) & rhs)) { \
1118 old_value = temp_val; \
1124 #define MIN_MAX_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1125 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1126 if (*lhs OP rhs) { \
1127 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \
1128 MIN_MAX_CRITSECT(OP, LCK_ID) \
1132 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1136 #define MIN_MAX_COMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1138 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1139 if (*lhs OP rhs) { \
1140 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \
1141 MIN_MAX_CMPXCHG(TYPE, BITS, OP) \
1148 #define MIN_MAX_COMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1150 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1151 if (*lhs OP rhs) { \
1152 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \
1153 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1154 MIN_MAX_CMPXCHG(TYPE, BITS, OP) \
1157 MIN_MAX_CRITSECT(OP, LCK_ID) \
1163 MIN_MAX_COMPXCHG(fixed1, max,
char, 8, <, 1i, 0,
1165 MIN_MAX_COMPXCHG(fixed1, min,
char, 8, >, 1i, 0,
1167 MIN_MAX_COMPXCHG(fixed2, max,
short, 16, <, 2i, 1,
1169 MIN_MAX_COMPXCHG(fixed2, min,
short, 16, >, 2i, 1,
1171 MIN_MAX_COMPXCHG(fixed4, max, kmp_int32, 32, <, 4i, 3,
1173 MIN_MAX_COMPXCHG(fixed4, min, kmp_int32, 32, >, 4i, 3,
1175 MIN_MAX_COMPXCHG(fixed8, max, kmp_int64, 64, <, 8i, 7,
1177 MIN_MAX_COMPXCHG(fixed8, min, kmp_int64, 64, >, 8i, 7,
1179 MIN_MAX_COMPXCHG(float4, max, kmp_real32, 32, <, 4r, 3,
1181 MIN_MAX_COMPXCHG(float4, min, kmp_real32, 32, >, 4r, 3,
1183 MIN_MAX_COMPXCHG(float8, max, kmp_real64, 64, <, 8r, 7,
1185 MIN_MAX_COMPXCHG(float8, min, kmp_real64, 64, >, 8r, 7,
1188 MIN_MAX_CRITICAL(float16, max, QUAD_LEGACY, <, 16r,
1190 MIN_MAX_CRITICAL(float16, min, QUAD_LEGACY, >, 16r,
1193 MIN_MAX_CRITICAL(float16, max_a16, Quad_a16_t, <, 16r,
1195 MIN_MAX_CRITICAL(float16, min_a16, Quad_a16_t, >, 16r,
1202 #define ATOMIC_CRIT_EQV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1203 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1204 OP_GOMP_CRITICAL(^= ~, GOMP_FLAG) \
1205 OP_CRITICAL(^= ~, LCK_ID) \
1209 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1212 #define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1214 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1215 OP_GOMP_CRITICAL(^= ~, GOMP_FLAG) \
1216 OP_CMPXCHG(TYPE, BITS, OP) \
1222 #define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1224 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1225 OP_GOMP_CRITICAL(^= ~, GOMP_FLAG) \
1226 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1227 OP_CMPXCHG(TYPE, BITS, OP) \
1230 OP_CRITICAL(^= ~, LCK_ID) \
1235 ATOMIC_CMPXCHG(fixed1, neqv, kmp_int8, 8, ^, 1i, 0,
1237 ATOMIC_CMPXCHG(fixed2, neqv, kmp_int16, 16, ^, 2i, 1,
1239 ATOMIC_CMPXCHG(fixed4, neqv, kmp_int32, 32, ^, 4i, 3,
1241 ATOMIC_CMPXCHG(fixed8, neqv, kmp_int64, 64, ^, 8i, 7,
1243 ATOMIC_CMPX_EQV(fixed1, eqv, kmp_int8, 8, ^~, 1i, 0,
1245 ATOMIC_CMPX_EQV(fixed2, eqv, kmp_int16, 16, ^~, 2i, 1,
1247 ATOMIC_CMPX_EQV(fixed4, eqv, kmp_int32, 32, ^~, 4i, 3,
1249 ATOMIC_CMPX_EQV(fixed8, eqv, kmp_int64, 64, ^~, 8i, 7,
1258 #define ATOMIC_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1259 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1260 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
1261 OP_CRITICAL(OP## =, LCK_ID) \
1266 ATOMIC_CRITICAL(float10, add,
long double, +, 10r,
1268 ATOMIC_CRITICAL(float10, sub,
long double, -, 10r,
1270 ATOMIC_CRITICAL(float10, mul,
long double, *, 10r,
1272 ATOMIC_CRITICAL(float10, div,
long double, /, 10r,
1276 ATOMIC_CRITICAL(float16, add, QUAD_LEGACY, +, 16r,
1278 ATOMIC_CRITICAL(float16, sub, QUAD_LEGACY, -, 16r,
1280 ATOMIC_CRITICAL(float16, mul, QUAD_LEGACY, *, 16r,
1282 ATOMIC_CRITICAL(float16, div, QUAD_LEGACY, /, 16r,
1285 ATOMIC_CRITICAL(float16, add_a16, Quad_a16_t, +, 16r,
1287 ATOMIC_CRITICAL(float16, sub_a16, Quad_a16_t, -, 16r,
1289 ATOMIC_CRITICAL(float16, mul_a16, Quad_a16_t, *, 16r,
1291 ATOMIC_CRITICAL(float16, div_a16, Quad_a16_t, /, 16r,
1299 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, add, kmp_cmplx32, 64, +, 8c, 7,
1301 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, sub, kmp_cmplx32, 64, -, 8c, 7,
1303 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, mul, kmp_cmplx32, 64, *, 8c, 7,
1305 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, div, kmp_cmplx32, 64, /, 8c, 7,
1309 ATOMIC_CRITICAL(cmplx4, add, kmp_cmplx32, +, 8c, 1)
1310 ATOMIC_CRITICAL(cmplx4, sub, kmp_cmplx32, -, 8c, 1)
1311 ATOMIC_CRITICAL(cmplx4, mul, kmp_cmplx32, *, 8c, 1)
1312 ATOMIC_CRITICAL(cmplx4, div, kmp_cmplx32, /, 8c, 1)
1313 #endif // USE_CMPXCHG_FIX
1315 ATOMIC_CRITICAL(cmplx8, add, kmp_cmplx64, +, 16c, 1)
1316 ATOMIC_CRITICAL(cmplx8, sub, kmp_cmplx64, -, 16c, 1)
1317 ATOMIC_CRITICAL(cmplx8, mul, kmp_cmplx64, *, 16c, 1)
1318 ATOMIC_CRITICAL(cmplx8, div, kmp_cmplx64, /, 16c, 1)
1319 ATOMIC_CRITICAL(cmplx10, add, kmp_cmplx80, +, 20c,
1321 ATOMIC_CRITICAL(cmplx10, sub, kmp_cmplx80, -, 20c,
1323 ATOMIC_CRITICAL(cmplx10, mul, kmp_cmplx80, *, 20c,
1325 ATOMIC_CRITICAL(cmplx10, div, kmp_cmplx80, /, 20c,
1328 ATOMIC_CRITICAL(cmplx16, add, CPLX128_LEG, +, 32c,
1330 ATOMIC_CRITICAL(cmplx16, sub, CPLX128_LEG, -, 32c,
1332 ATOMIC_CRITICAL(cmplx16, mul, CPLX128_LEG, *, 32c,
1334 ATOMIC_CRITICAL(cmplx16, div, CPLX128_LEG, /, 32c,
1337 ATOMIC_CRITICAL(cmplx16, add_a16, kmp_cmplx128_a16_t, +, 32c,
1339 ATOMIC_CRITICAL(cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c,
1341 ATOMIC_CRITICAL(cmplx16, mul_a16, kmp_cmplx128_a16_t, *, 32c,
1343 ATOMIC_CRITICAL(cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c,
1352 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1360 #define OP_CRITICAL_REV(OP, LCK_ID) \
1361 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
1363 (*lhs) = (rhs)OP(*lhs); \
1365 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
1367 #ifdef KMP_GOMP_COMPAT
1368 #define OP_GOMP_CRITICAL_REV(OP, FLAG) \
1369 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
1371 OP_CRITICAL_REV(OP, 0); \
1375 #define OP_GOMP_CRITICAL_REV(OP, FLAG)
1383 #define ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
1384 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_rev(ident_t *id_ref, int gtid, \
1385 TYPE *lhs, TYPE rhs) { \
1386 KMP_DEBUG_ASSERT(__kmp_init_serial); \
1387 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_rev: T#%d\n", gtid));
1396 #define OP_CMPXCHG_REV(TYPE, BITS, OP) \
1398 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1399 TYPE old_value, new_value; \
1401 old_value = temp_val; \
1402 new_value = rhs OP old_value; \
1403 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
1404 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
1405 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
1409 old_value = temp_val; \
1410 new_value = rhs OP old_value; \
1415 #define ATOMIC_CMPXCHG_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, GOMP_FLAG) \
1416 ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \
1417 OP_GOMP_CRITICAL_REV(OP, GOMP_FLAG) \
1418 OP_CMPXCHG_REV(TYPE, BITS, OP) \
1435 ATOMIC_CMPXCHG_REV(fixed1, div, kmp_int8, 8, /, 1i,
1437 ATOMIC_CMPXCHG_REV(fixed1u, div, kmp_uint8, 8, /, 1i,
1439 ATOMIC_CMPXCHG_REV(fixed1, shl, kmp_int8, 8, <<, 1i,
1441 ATOMIC_CMPXCHG_REV(fixed1, shr, kmp_int8, 8, >>, 1i,
1443 ATOMIC_CMPXCHG_REV(fixed1u, shr, kmp_uint8, 8, >>, 1i,
1445 ATOMIC_CMPXCHG_REV(fixed1, sub, kmp_int8, 8, -, 1i,
1448 ATOMIC_CMPXCHG_REV(fixed2, div, kmp_int16, 16, /, 2i,
1450 ATOMIC_CMPXCHG_REV(fixed2u, div, kmp_uint16, 16, /, 2i,
1452 ATOMIC_CMPXCHG_REV(fixed2, shl, kmp_int16, 16, <<, 2i,
1454 ATOMIC_CMPXCHG_REV(fixed2, shr, kmp_int16, 16, >>, 2i,
1456 ATOMIC_CMPXCHG_REV(fixed2u, shr, kmp_uint16, 16, >>, 2i,
1458 ATOMIC_CMPXCHG_REV(fixed2, sub, kmp_int16, 16, -, 2i,
1461 ATOMIC_CMPXCHG_REV(fixed4, div, kmp_int32, 32, /, 4i,
1463 ATOMIC_CMPXCHG_REV(fixed4u, div, kmp_uint32, 32, /, 4i,
1465 ATOMIC_CMPXCHG_REV(fixed4, shl, kmp_int32, 32, <<, 4i,
1467 ATOMIC_CMPXCHG_REV(fixed4, shr, kmp_int32, 32, >>, 4i,
1469 ATOMIC_CMPXCHG_REV(fixed4u, shr, kmp_uint32, 32, >>, 4i,
1471 ATOMIC_CMPXCHG_REV(fixed4, sub, kmp_int32, 32, -, 4i,
1474 ATOMIC_CMPXCHG_REV(fixed8, div, kmp_int64, 64, /, 8i,
1476 ATOMIC_CMPXCHG_REV(fixed8u, div, kmp_uint64, 64, /, 8i,
1478 ATOMIC_CMPXCHG_REV(fixed8, shl, kmp_int64, 64, <<, 8i,
1480 ATOMIC_CMPXCHG_REV(fixed8, shr, kmp_int64, 64, >>, 8i,
1482 ATOMIC_CMPXCHG_REV(fixed8u, shr, kmp_uint64, 64, >>, 8i,
1484 ATOMIC_CMPXCHG_REV(fixed8, sub, kmp_int64, 64, -, 8i,
1487 ATOMIC_CMPXCHG_REV(float4, div, kmp_real32, 32, /, 4r,
1489 ATOMIC_CMPXCHG_REV(float4, sub, kmp_real32, 32, -, 4r,
1492 ATOMIC_CMPXCHG_REV(float8, div, kmp_real64, 64, /, 8r,
1494 ATOMIC_CMPXCHG_REV(float8, sub, kmp_real64, 64, -, 8r,
1504 #define ATOMIC_CRITICAL_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1505 ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \
1506 OP_GOMP_CRITICAL_REV(OP, GOMP_FLAG) \
1507 OP_CRITICAL_REV(OP, LCK_ID) \
1512 ATOMIC_CRITICAL_REV(float10, sub,
long double, -, 10r,
1514 ATOMIC_CRITICAL_REV(float10, div,
long double, /, 10r,
1518 ATOMIC_CRITICAL_REV(float16, sub, QUAD_LEGACY, -, 16r,
1520 ATOMIC_CRITICAL_REV(float16, div, QUAD_LEGACY, /, 16r,
1523 ATOMIC_CRITICAL_REV(float16, sub_a16, Quad_a16_t, -, 16r,
1525 ATOMIC_CRITICAL_REV(float16, div_a16, Quad_a16_t, /, 16r,
1531 ATOMIC_CRITICAL_REV(cmplx4, sub, kmp_cmplx32, -, 8c,
1533 ATOMIC_CRITICAL_REV(cmplx4, div, kmp_cmplx32, /, 8c,
1535 ATOMIC_CRITICAL_REV(cmplx8, sub, kmp_cmplx64, -, 16c,
1537 ATOMIC_CRITICAL_REV(cmplx8, div, kmp_cmplx64, /, 16c,
1539 ATOMIC_CRITICAL_REV(cmplx10, sub, kmp_cmplx80, -, 20c,
1541 ATOMIC_CRITICAL_REV(cmplx10, div, kmp_cmplx80, /, 20c,
1544 ATOMIC_CRITICAL_REV(cmplx16, sub, CPLX128_LEG, -, 32c,
1546 ATOMIC_CRITICAL_REV(cmplx16, div, CPLX128_LEG, /, 32c,
1549 ATOMIC_CRITICAL_REV(cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c,
1551 ATOMIC_CRITICAL_REV(cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c,
1556 #endif // KMP_ARCH_X86 || KMP_ARCH_X86_64
1559 #endif // OMP_40_ENABLED
1572 #define ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1573 void __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( \
1574 ident_t *id_ref, int gtid, TYPE *lhs, RTYPE rhs) { \
1575 KMP_DEBUG_ASSERT(__kmp_init_serial); \
1577 ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \
1581 #define ATOMIC_CRITICAL_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, LCK_ID, \
1583 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1584 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
1585 OP_CRITICAL(OP## =, LCK_ID) \
1589 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1592 #define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1593 LCK_ID, MASK, GOMP_FLAG) \
1594 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1595 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
1596 OP_CMPXCHG(TYPE, BITS, OP) \
1602 #define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1603 LCK_ID, MASK, GOMP_FLAG) \
1604 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1605 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
1606 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1607 OP_CMPXCHG(TYPE, BITS, OP) \
1610 OP_CRITICAL(OP## =, LCK_ID) \
1616 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1618 #define ATOMIC_CMPXCHG_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \
1619 RTYPE, LCK_ID, MASK, GOMP_FLAG) \
1620 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1621 OP_GOMP_CRITICAL_REV(OP, GOMP_FLAG) \
1622 OP_CMPXCHG_REV(TYPE, BITS, OP) \
1624 #define ATOMIC_CRITICAL_REV_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \
1625 LCK_ID, GOMP_FLAG) \
1626 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1627 OP_GOMP_CRITICAL_REV(OP, GOMP_FLAG) \
1628 OP_CRITICAL_REV(OP, LCK_ID) \
1633 ATOMIC_CMPXCHG_MIX(fixed1,
char, mul, 8, *, float8, kmp_real64, 1i, 0,
1635 ATOMIC_CMPXCHG_MIX(fixed1,
char, div, 8, /, float8, kmp_real64, 1i, 0,
1637 ATOMIC_CMPXCHG_MIX(fixed2,
short, mul, 16, *, float8, kmp_real64, 2i, 1,
1639 ATOMIC_CMPXCHG_MIX(fixed2,
short, div, 16, /, float8, kmp_real64, 2i, 1,
1641 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, mul, 32, *, float8, kmp_real64, 4i, 3,
1643 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, div, 32, /, float8, kmp_real64, 4i, 3,
1645 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, mul, 64, *, float8, kmp_real64, 8i, 7,
1647 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, div, 64, /, float8, kmp_real64, 8i, 7,
1649 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, add, 32, +, float8, kmp_real64, 4r, 3,
1651 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, sub, 32, -, float8, kmp_real64, 4r, 3,
1653 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, mul, 32, *, float8, kmp_real64, 4r, 3,
1655 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, div, 32, /, float8, kmp_real64, 4r, 3,
1661 ATOMIC_CMPXCHG_MIX(fixed1,
char, add, 8, +, fp, _Quad, 1i, 0,
1663 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, add, 8, +, fp, _Quad, 1i, 0,
1665 ATOMIC_CMPXCHG_MIX(fixed1,
char, sub, 8, -, fp, _Quad, 1i, 0,
1667 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, sub, 8, -, fp, _Quad, 1i, 0,
1669 ATOMIC_CMPXCHG_MIX(fixed1,
char, mul, 8, *, fp, _Quad, 1i, 0,
1671 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, mul, 8, *, fp, _Quad, 1i, 0,
1673 ATOMIC_CMPXCHG_MIX(fixed1,
char, div, 8, /, fp, _Quad, 1i, 0,
1675 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, div, 8, /, fp, _Quad, 1i, 0,
1678 ATOMIC_CMPXCHG_MIX(fixed2,
short, add, 16, +, fp, _Quad, 2i, 1,
1680 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, add, 16, +, fp, _Quad, 2i, 1,
1682 ATOMIC_CMPXCHG_MIX(fixed2,
short, sub, 16, -, fp, _Quad, 2i, 1,
1684 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, sub, 16, -, fp, _Quad, 2i, 1,
1686 ATOMIC_CMPXCHG_MIX(fixed2,
short, mul, 16, *, fp, _Quad, 2i, 1,
1688 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, mul, 16, *, fp, _Quad, 2i, 1,
1690 ATOMIC_CMPXCHG_MIX(fixed2,
short, div, 16, /, fp, _Quad, 2i, 1,
1692 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, div, 16, /, fp, _Quad, 2i, 1,
1695 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, add, 32, +, fp, _Quad, 4i, 3,
1697 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, add, 32, +, fp, _Quad, 4i, 3,
1699 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, sub, 32, -, fp, _Quad, 4i, 3,
1701 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, sub, 32, -, fp, _Quad, 4i, 3,
1703 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, mul, 32, *, fp, _Quad, 4i, 3,
1705 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, mul, 32, *, fp, _Quad, 4i, 3,
1707 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, div, 32, /, fp, _Quad, 4i, 3,
1709 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, div, 32, /, fp, _Quad, 4i, 3,
1712 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, add, 64, +, fp, _Quad, 8i, 7,
1714 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, add, 64, +, fp, _Quad, 8i, 7,
1716 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, sub, 64, -, fp, _Quad, 8i, 7,
1718 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, sub, 64, -, fp, _Quad, 8i, 7,
1720 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, mul, 64, *, fp, _Quad, 8i, 7,
1722 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, mul, 64, *, fp, _Quad, 8i, 7,
1724 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, div, 64, /, fp, _Quad, 8i, 7,
1726 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, div, 64, /, fp, _Quad, 8i, 7,
1729 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, add, 32, +, fp, _Quad, 4r, 3,
1731 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, sub, 32, -, fp, _Quad, 4r, 3,
1733 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, mul, 32, *, fp, _Quad, 4r, 3,
1735 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, div, 32, /, fp, _Quad, 4r, 3,
1738 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, add, 64, +, fp, _Quad, 8r, 7,
1740 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, sub, 64, -, fp, _Quad, 8r, 7,
1742 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, mul, 64, *, fp, _Quad, 8r, 7,
1744 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, div, 64, /, fp, _Quad, 8r, 7,
1747 ATOMIC_CRITICAL_FP(float10,
long double, add, +, fp, _Quad, 10r,
1749 ATOMIC_CRITICAL_FP(float10,
long double, sub, -, fp, _Quad, 10r,
1751 ATOMIC_CRITICAL_FP(float10,
long double, mul, *, fp, _Quad, 10r,
1753 ATOMIC_CRITICAL_FP(float10,
long double, div, /, fp, _Quad, 10r,
1756 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1758 ATOMIC_CMPXCHG_REV_MIX(fixed1,
char, sub_rev, 8, -, fp, _Quad, 1i, 0,
1760 ATOMIC_CMPXCHG_REV_MIX(fixed1u, uchar, sub_rev, 8, -, fp, _Quad, 1i, 0,
1762 ATOMIC_CMPXCHG_REV_MIX(fixed1,
char, div_rev, 8, /, fp, _Quad, 1i, 0,
1764 ATOMIC_CMPXCHG_REV_MIX(fixed1u, uchar, div_rev, 8, /, fp, _Quad, 1i, 0,
1767 ATOMIC_CMPXCHG_REV_MIX(fixed2,
short, sub_rev, 16, -, fp, _Quad, 2i, 1,
1769 ATOMIC_CMPXCHG_REV_MIX(fixed2u, ushort, sub_rev, 16, -, fp, _Quad, 2i, 1,
1771 ATOMIC_CMPXCHG_REV_MIX(fixed2,
short, div_rev, 16, /, fp, _Quad, 2i, 1,
1773 ATOMIC_CMPXCHG_REV_MIX(fixed2u, ushort, div_rev, 16, /, fp, _Quad, 2i, 1,
1776 ATOMIC_CMPXCHG_REV_MIX(fixed4, kmp_int32, sub_rev, 32, -, fp, _Quad, 4i, 3,
1778 ATOMIC_CMPXCHG_REV_MIX(fixed4u, kmp_uint32, sub_rev, 32, -, fp, _Quad, 4i, 3,
1780 ATOMIC_CMPXCHG_REV_MIX(fixed4, kmp_int32, div_rev, 32, /, fp, _Quad, 4i, 3,
1782 ATOMIC_CMPXCHG_REV_MIX(fixed4u, kmp_uint32, div_rev, 32, /, fp, _Quad, 4i, 3,
1785 ATOMIC_CMPXCHG_REV_MIX(fixed8, kmp_int64, sub_rev, 64, -, fp, _Quad, 8i, 7,
1787 ATOMIC_CMPXCHG_REV_MIX(fixed8u, kmp_uint64, sub_rev, 64, -, fp, _Quad, 8i, 7,
1789 ATOMIC_CMPXCHG_REV_MIX(fixed8, kmp_int64, div_rev, 64, /, fp, _Quad, 8i, 7,
1791 ATOMIC_CMPXCHG_REV_MIX(fixed8u, kmp_uint64, div_rev, 64, /, fp, _Quad, 8i, 7,
1794 ATOMIC_CMPXCHG_REV_MIX(float4, kmp_real32, sub_rev, 32, -, fp, _Quad, 4r, 3,
1796 ATOMIC_CMPXCHG_REV_MIX(float4, kmp_real32, div_rev, 32, /, fp, _Quad, 4r, 3,
1799 ATOMIC_CMPXCHG_REV_MIX(float8, kmp_real64, sub_rev, 64, -, fp, _Quad, 8r, 7,
1801 ATOMIC_CMPXCHG_REV_MIX(float8, kmp_real64, div_rev, 64, /, fp, _Quad, 8r, 7,
1804 ATOMIC_CRITICAL_REV_FP(float10,
long double, sub_rev, -, fp, _Quad, 10r,
1806 ATOMIC_CRITICAL_REV_FP(float10,
long double, div_rev, /, fp, _Quad, 10r,
1812 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1817 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1818 LCK_ID, MASK, GOMP_FLAG) \
1819 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1820 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
1821 OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \
1825 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1826 LCK_ID, MASK, GOMP_FLAG) \
1827 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1828 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
1829 OP_CMPXCHG(TYPE, BITS, OP) \
1831 #endif // USE_CMPXCHG_FIX
1835 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1836 LCK_ID, MASK, GOMP_FLAG) \
1837 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1838 OP_GOMP_CRITICAL(OP## =, GOMP_FLAG) \
1839 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1840 OP_CMPXCHG(TYPE, BITS, OP) \
1843 OP_CRITICAL(OP## =, LCK_ID) \
1848 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, add, 64, +, cmplx8, kmp_cmplx64, 8c,
1850 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, sub, 64, -, cmplx8, kmp_cmplx64, 8c,
1852 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, mul, 64, *, cmplx8, kmp_cmplx64, 8c,
1854 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, div, 64, /, cmplx8, kmp_cmplx64, 8c,
1858 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1869 #define ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
1870 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \
1872 KMP_DEBUG_ASSERT(__kmp_init_serial); \
1873 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
1884 #define OP_CMPXCHG_READ(TYPE, BITS, OP) \
1886 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1889 kmp_int##BITS i_val; \
1891 union f_i_union old_value; \
1893 old_value.f_val = temp_val; \
1894 old_value.i_val = KMP_COMPARE_AND_STORE_RET##BITS( \
1895 (kmp_int##BITS *)loc, \
1896 *VOLATILE_CAST(kmp_int##BITS *) & old_value.i_val, \
1897 *VOLATILE_CAST(kmp_int##BITS *) & old_value.i_val); \
1898 new_value = old_value.f_val; \
1908 #define OP_CRITICAL_READ(OP, LCK_ID) \
1909 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
1911 new_value = (*loc); \
1913 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
1916 #ifdef KMP_GOMP_COMPAT
1917 #define OP_GOMP_CRITICAL_READ(OP, FLAG) \
1918 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
1920 OP_CRITICAL_READ(OP, 0); \
1924 #define OP_GOMP_CRITICAL_READ(OP, FLAG)
1928 #define ATOMIC_FIXED_READ(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
1929 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \
1931 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \
1932 new_value = KMP_TEST_THEN_ADD##BITS(loc, OP 0); \
1936 #define ATOMIC_CMPXCHG_READ(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
1937 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \
1939 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \
1940 OP_CMPXCHG_READ(TYPE, BITS, OP) \
1948 #define ATOMIC_CRITICAL_READ(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1949 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \
1951 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \
1952 OP_CRITICAL_READ(OP, LCK_ID) \
1960 #if (KMP_OS_WINDOWS)
1962 #define OP_CRITICAL_READ_WRK(OP, LCK_ID) \
1963 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
1967 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
1969 #ifdef KMP_GOMP_COMPAT
1970 #define OP_GOMP_CRITICAL_READ_WRK(OP, FLAG) \
1971 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
1973 OP_CRITICAL_READ_WRK(OP, 0); \
1976 #define OP_GOMP_CRITICAL_READ_WRK(OP, FLAG)
1979 #define ATOMIC_BEGIN_READ_WRK(TYPE_ID, OP_ID, TYPE) \
1980 void __kmpc_atomic_##TYPE_ID##_##OP_ID(TYPE *out, ident_t *id_ref, int gtid, \
1982 KMP_DEBUG_ASSERT(__kmp_init_serial); \
1983 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
1986 #define ATOMIC_CRITICAL_READ_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1987 ATOMIC_BEGIN_READ_WRK(TYPE_ID, OP_ID, TYPE) \
1988 OP_GOMP_CRITICAL_READ_WRK(OP## =, GOMP_FLAG) \
1989 OP_CRITICAL_READ_WRK(OP, LCK_ID) \
1992 #endif // KMP_OS_WINDOWS
1996 ATOMIC_FIXED_READ(fixed4, rd, kmp_int32, 32, +, 0)
1997 ATOMIC_FIXED_READ(fixed8, rd, kmp_int64, 64, +,
1999 ATOMIC_CMPXCHG_READ(float4, rd, kmp_real32, 32, +,
2001 ATOMIC_CMPXCHG_READ(float8, rd, kmp_real64, 64, +,
2005 ATOMIC_CMPXCHG_READ(fixed1, rd, kmp_int8, 8, +,
2007 ATOMIC_CMPXCHG_READ(fixed2, rd, kmp_int16, 16, +,
2010 ATOMIC_CRITICAL_READ(float10, rd,
long double, +, 10r,
2013 ATOMIC_CRITICAL_READ(float16, rd, QUAD_LEGACY, +, 16r,
2015 #endif // KMP_HAVE_QUAD
2018 #if (KMP_OS_WINDOWS)
2019 ATOMIC_CRITICAL_READ_WRK(cmplx4, rd, kmp_cmplx32, +, 8c,
2022 ATOMIC_CRITICAL_READ(cmplx4, rd, kmp_cmplx32, +, 8c,
2025 ATOMIC_CRITICAL_READ(cmplx8, rd, kmp_cmplx64, +, 16c,
2027 ATOMIC_CRITICAL_READ(cmplx10, rd, kmp_cmplx80, +, 20c,
2030 ATOMIC_CRITICAL_READ(cmplx16, rd, CPLX128_LEG, +, 32c,
2033 ATOMIC_CRITICAL_READ(float16, a16_rd, Quad_a16_t, +, 16r,
2035 ATOMIC_CRITICAL_READ(cmplx16, a16_rd, kmp_cmplx128_a16_t, +, 32c,
2043 #define ATOMIC_XCHG_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2044 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
2045 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \
2046 KMP_XCHG_FIXED##BITS(lhs, rhs); \
2049 #define ATOMIC_XCHG_FLOAT_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2050 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
2051 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \
2052 KMP_XCHG_REAL##BITS(lhs, rhs); \
2062 #define OP_CMPXCHG_WR(TYPE, BITS, OP) \
2064 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2065 TYPE old_value, new_value; \
2067 old_value = temp_val; \
2069 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
2070 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
2071 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
2075 old_value = temp_val; \
2081 #define ATOMIC_CMPXCHG_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2082 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
2083 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \
2084 OP_CMPXCHG_WR(TYPE, BITS, OP) \
2093 #define ATOMIC_CRITICAL_WR(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2094 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
2095 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \
2096 OP_CRITICAL(OP, LCK_ID) \
2100 ATOMIC_XCHG_WR(fixed1, wr, kmp_int8, 8, =,
2102 ATOMIC_XCHG_WR(fixed2, wr, kmp_int16, 16, =,
2104 ATOMIC_XCHG_WR(fixed4, wr, kmp_int32, 32, =,
2107 ATOMIC_CMPXCHG_WR(fixed8, wr, kmp_int64, 64, =,
2110 ATOMIC_XCHG_WR(fixed8, wr, kmp_int64, 64, =,
2114 ATOMIC_XCHG_FLOAT_WR(float4, wr, kmp_real32, 32, =,
2117 ATOMIC_CMPXCHG_WR(float8, wr, kmp_real64, 64, =,
2120 ATOMIC_XCHG_FLOAT_WR(float8, wr, kmp_real64, 64, =,
2124 ATOMIC_CRITICAL_WR(float10, wr,
long double, =, 10r,
2127 ATOMIC_CRITICAL_WR(float16, wr, QUAD_LEGACY, =, 16r,
2130 ATOMIC_CRITICAL_WR(cmplx4, wr, kmp_cmplx32, =, 8c, 1)
2131 ATOMIC_CRITICAL_WR(cmplx8, wr, kmp_cmplx64, =, 16c,
2133 ATOMIC_CRITICAL_WR(cmplx10, wr, kmp_cmplx80, =, 20c,
2136 ATOMIC_CRITICAL_WR(cmplx16, wr, CPLX128_LEG, =, 32c,
2139 ATOMIC_CRITICAL_WR(float16, a16_wr, Quad_a16_t, =, 16r,
2141 ATOMIC_CRITICAL_WR(cmplx16, a16_wr, kmp_cmplx128_a16_t, =, 32c,
2154 #define ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
2155 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \
2156 TYPE *lhs, TYPE rhs, int flag) { \
2157 KMP_DEBUG_ASSERT(__kmp_init_serial); \
2158 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
2166 #define OP_CRITICAL_CPT(OP, LCK_ID) \
2167 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2171 new_value = (*lhs); \
2173 new_value = (*lhs); \
2177 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2181 #ifdef KMP_GOMP_COMPAT
2182 #define OP_GOMP_CRITICAL_CPT(OP, FLAG) \
2183 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2185 OP_CRITICAL_CPT(OP## =, 0); \
2188 #define OP_GOMP_CRITICAL_CPT(OP, FLAG)
2198 #define OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2200 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2201 TYPE old_value, new_value; \
2203 old_value = temp_val; \
2204 new_value = old_value OP rhs; \
2205 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
2206 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
2207 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
2211 old_value = temp_val; \
2212 new_value = old_value OP rhs; \
2221 #define ATOMIC_CMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2222 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2224 OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) \
2225 OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2229 #define ATOMIC_FIXED_ADD_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2230 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2231 TYPE old_value, new_value; \
2232 OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) \
2234 old_value = KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \
2236 return old_value OP rhs; \
2242 ATOMIC_FIXED_ADD_CPT(fixed4, add_cpt, kmp_int32, 32, +,
2244 ATOMIC_FIXED_ADD_CPT(fixed4, sub_cpt, kmp_int32, 32, -,
2246 ATOMIC_FIXED_ADD_CPT(fixed8, add_cpt, kmp_int64, 64, +,
2248 ATOMIC_FIXED_ADD_CPT(fixed8, sub_cpt, kmp_int64, 64, -,
2251 ATOMIC_CMPXCHG_CPT(float4, add_cpt, kmp_real32, 32, +,
2253 ATOMIC_CMPXCHG_CPT(float4, sub_cpt, kmp_real32, 32, -,
2255 ATOMIC_CMPXCHG_CPT(float8, add_cpt, kmp_real64, 64, +,
2257 ATOMIC_CMPXCHG_CPT(float8, sub_cpt, kmp_real64, 64, -,
2272 ATOMIC_CMPXCHG_CPT(fixed1, add_cpt, kmp_int8, 8, +,
2274 ATOMIC_CMPXCHG_CPT(fixed1, andb_cpt, kmp_int8, 8, &,
2276 ATOMIC_CMPXCHG_CPT(fixed1, div_cpt, kmp_int8, 8, /,
2278 ATOMIC_CMPXCHG_CPT(fixed1u, div_cpt, kmp_uint8, 8, /,
2280 ATOMIC_CMPXCHG_CPT(fixed1, mul_cpt, kmp_int8, 8, *,
2282 ATOMIC_CMPXCHG_CPT(fixed1, orb_cpt, kmp_int8, 8, |,
2284 ATOMIC_CMPXCHG_CPT(fixed1, shl_cpt, kmp_int8, 8, <<,
2286 ATOMIC_CMPXCHG_CPT(fixed1, shr_cpt, kmp_int8, 8, >>,
2288 ATOMIC_CMPXCHG_CPT(fixed1u, shr_cpt, kmp_uint8, 8, >>,
2290 ATOMIC_CMPXCHG_CPT(fixed1, sub_cpt, kmp_int8, 8, -,
2292 ATOMIC_CMPXCHG_CPT(fixed1, xor_cpt, kmp_int8, 8, ^,
2294 ATOMIC_CMPXCHG_CPT(fixed2, add_cpt, kmp_int16, 16, +,
2296 ATOMIC_CMPXCHG_CPT(fixed2, andb_cpt, kmp_int16, 16, &,
2298 ATOMIC_CMPXCHG_CPT(fixed2, div_cpt, kmp_int16, 16, /,
2300 ATOMIC_CMPXCHG_CPT(fixed2u, div_cpt, kmp_uint16, 16, /,
2302 ATOMIC_CMPXCHG_CPT(fixed2, mul_cpt, kmp_int16, 16, *,
2304 ATOMIC_CMPXCHG_CPT(fixed2, orb_cpt, kmp_int16, 16, |,
2306 ATOMIC_CMPXCHG_CPT(fixed2, shl_cpt, kmp_int16, 16, <<,
2308 ATOMIC_CMPXCHG_CPT(fixed2, shr_cpt, kmp_int16, 16, >>,
2310 ATOMIC_CMPXCHG_CPT(fixed2u, shr_cpt, kmp_uint16, 16, >>,
2312 ATOMIC_CMPXCHG_CPT(fixed2, sub_cpt, kmp_int16, 16, -,
2314 ATOMIC_CMPXCHG_CPT(fixed2, xor_cpt, kmp_int16, 16, ^,
2316 ATOMIC_CMPXCHG_CPT(fixed4, andb_cpt, kmp_int32, 32, &,
2318 ATOMIC_CMPXCHG_CPT(fixed4, div_cpt, kmp_int32, 32, /,
2320 ATOMIC_CMPXCHG_CPT(fixed4u, div_cpt, kmp_uint32, 32, /,
2322 ATOMIC_CMPXCHG_CPT(fixed4, mul_cpt, kmp_int32, 32, *,
2324 ATOMIC_CMPXCHG_CPT(fixed4, orb_cpt, kmp_int32, 32, |,
2326 ATOMIC_CMPXCHG_CPT(fixed4, shl_cpt, kmp_int32, 32, <<,
2328 ATOMIC_CMPXCHG_CPT(fixed4, shr_cpt, kmp_int32, 32, >>,
2330 ATOMIC_CMPXCHG_CPT(fixed4u, shr_cpt, kmp_uint32, 32, >>,
2332 ATOMIC_CMPXCHG_CPT(fixed4, xor_cpt, kmp_int32, 32, ^,
2334 ATOMIC_CMPXCHG_CPT(fixed8, andb_cpt, kmp_int64, 64, &,
2336 ATOMIC_CMPXCHG_CPT(fixed8, div_cpt, kmp_int64, 64, /,
2338 ATOMIC_CMPXCHG_CPT(fixed8u, div_cpt, kmp_uint64, 64, /,
2340 ATOMIC_CMPXCHG_CPT(fixed8, mul_cpt, kmp_int64, 64, *,
2342 ATOMIC_CMPXCHG_CPT(fixed8, orb_cpt, kmp_int64, 64, |,
2344 ATOMIC_CMPXCHG_CPT(fixed8, shl_cpt, kmp_int64, 64, <<,
2346 ATOMIC_CMPXCHG_CPT(fixed8, shr_cpt, kmp_int64, 64, >>,
2348 ATOMIC_CMPXCHG_CPT(fixed8u, shr_cpt, kmp_uint64, 64, >>,
2350 ATOMIC_CMPXCHG_CPT(fixed8, xor_cpt, kmp_int64, 64, ^,
2352 ATOMIC_CMPXCHG_CPT(float4, div_cpt, kmp_real32, 32, /,
2354 ATOMIC_CMPXCHG_CPT(float4, mul_cpt, kmp_real32, 32, *,
2356 ATOMIC_CMPXCHG_CPT(float8, div_cpt, kmp_real64, 64, /,
2358 ATOMIC_CMPXCHG_CPT(float8, mul_cpt, kmp_real64, 64, *,
2370 #define ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
2371 TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( \
2372 ident_t *id_ref, int gtid, TYPE *lhs, RTYPE rhs, int flag) { \
2373 KMP_DEBUG_ASSERT(__kmp_init_serial); \
2375 ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \
2379 #define ATOMIC_CMPXCHG_CPT_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \
2380 RTYPE, LCK_ID, MASK, GOMP_FLAG) \
2381 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
2383 OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) \
2384 OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2388 #define ATOMIC_CRITICAL_CPT_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \
2389 LCK_ID, GOMP_FLAG) \
2390 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
2392 OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) \
2393 OP_CRITICAL_CPT(OP## =, LCK_ID) \
2396 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, add_cpt, 8, +, fp, _Quad, 1i, 0,
2398 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, add_cpt, 8, +, fp, _Quad, 1i, 0,
2400 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, sub_cpt, 8, -, fp, _Quad, 1i, 0,
2402 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, sub_cpt, 8, -, fp, _Quad, 1i, 0,
2404 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, mul_cpt, 8, *, fp, _Quad, 1i, 0,
2406 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, mul_cpt, 8, *, fp, _Quad, 1i, 0,
2408 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, div_cpt, 8, /, fp, _Quad, 1i, 0,
2410 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, div_cpt, 8, /, fp, _Quad, 1i, 0,
2413 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, add_cpt, 16, +, fp, _Quad, 2i, 1,
2415 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, add_cpt, 16, +, fp, _Quad, 2i, 1,
2417 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, sub_cpt, 16, -, fp, _Quad, 2i, 1,
2419 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, sub_cpt, 16, -, fp, _Quad, 2i, 1,
2421 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, mul_cpt, 16, *, fp, _Quad, 2i, 1,
2423 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, mul_cpt, 16, *, fp, _Quad, 2i, 1,
2425 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, div_cpt, 16, /, fp, _Quad, 2i, 1,
2427 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, div_cpt, 16, /, fp, _Quad, 2i, 1,
2430 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, add_cpt, 32, +, fp, _Quad, 4i, 3,
2432 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, add_cpt, 32, +, fp, _Quad, 4i, 3,
2434 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, sub_cpt, 32, -, fp, _Quad, 4i, 3,
2436 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, sub_cpt, 32, -, fp, _Quad, 4i, 3,
2438 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, mul_cpt, 32, *, fp, _Quad, 4i, 3,
2440 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, mul_cpt, 32, *, fp, _Quad, 4i, 3,
2442 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, div_cpt, 32, /, fp, _Quad, 4i, 3,
2444 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, div_cpt, 32, /, fp, _Quad, 4i, 3,
2447 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, add_cpt, 64, +, fp, _Quad, 8i, 7,
2449 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, add_cpt, 64, +, fp, _Quad, 8i, 7,
2451 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, sub_cpt, 64, -, fp, _Quad, 8i, 7,
2453 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, sub_cpt, 64, -, fp, _Quad, 8i, 7,
2455 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, mul_cpt, 64, *, fp, _Quad, 8i, 7,
2457 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, mul_cpt, 64, *, fp, _Quad, 8i, 7,
2459 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, div_cpt, 64, /, fp, _Quad, 8i, 7,
2461 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, div_cpt, 64, /, fp, _Quad, 8i, 7,
2464 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, add_cpt, 32, +, fp, _Quad, 4r, 3,
2466 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, sub_cpt, 32, -, fp, _Quad, 4r, 3,
2468 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, mul_cpt, 32, *, fp, _Quad, 4r, 3,
2470 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, div_cpt, 32, /, fp, _Quad, 4r, 3,
2473 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, add_cpt, 64, +, fp, _Quad, 8r, 7,
2475 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, sub_cpt, 64, -, fp, _Quad, 8r, 7,
2477 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, mul_cpt, 64, *, fp, _Quad, 8r, 7,
2479 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, div_cpt, 64, /, fp, _Quad, 8r, 7,
2482 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, add_cpt, +, fp, _Quad, 10r,
2484 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, sub_cpt, -, fp, _Quad, 10r,
2486 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, mul_cpt, *, fp, _Quad, 10r,
2488 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, div_cpt, /, fp, _Quad, 10r,
2491 #endif // KMP_HAVE_QUAD
2502 #define OP_CRITICAL_L_CPT(OP, LCK_ID) \
2503 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2508 new_value = (*lhs); \
2510 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
2513 #ifdef KMP_GOMP_COMPAT
2514 #define OP_GOMP_CRITICAL_L_CPT(OP, FLAG) \
2515 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2517 OP_CRITICAL_L_CPT(OP, 0); \
2521 #define OP_GOMP_CRITICAL_L_CPT(OP, FLAG)
2526 #define ATOMIC_CMPX_L_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2527 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2529 OP_GOMP_CRITICAL_L_CPT(= *lhs OP, GOMP_FLAG) \
2530 OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2533 ATOMIC_CMPX_L_CPT(fixed1, andl_cpt,
char, 8, &&,
2535 ATOMIC_CMPX_L_CPT(fixed1, orl_cpt,
char, 8, ||,
2537 ATOMIC_CMPX_L_CPT(fixed2, andl_cpt,
short, 16, &&,
2539 ATOMIC_CMPX_L_CPT(fixed2, orl_cpt,
short, 16, ||,
2541 ATOMIC_CMPX_L_CPT(fixed4, andl_cpt, kmp_int32, 32, &&,
2543 ATOMIC_CMPX_L_CPT(fixed4, orl_cpt, kmp_int32, 32, ||,
2545 ATOMIC_CMPX_L_CPT(fixed8, andl_cpt, kmp_int64, 64, &&,
2547 ATOMIC_CMPX_L_CPT(fixed8, orl_cpt, kmp_int64, 64, ||,
2559 #define MIN_MAX_CRITSECT_CPT(OP, LCK_ID) \
2560 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2562 if (*lhs OP rhs) { \
2568 new_value = old_value; \
2570 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2574 #ifdef KMP_GOMP_COMPAT
2575 #define GOMP_MIN_MAX_CRITSECT_CPT(OP, FLAG) \
2576 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2578 MIN_MAX_CRITSECT_CPT(OP, 0); \
2581 #define GOMP_MIN_MAX_CRITSECT_CPT(OP, FLAG)
2585 #define MIN_MAX_CMPXCHG_CPT(TYPE, BITS, OP) \
2587 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2590 old_value = temp_val; \
2591 while (old_value OP rhs && \
2592 !KMP_COMPARE_AND_STORE_ACQ##BITS( \
2593 (kmp_int##BITS *)lhs, \
2594 *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
2595 *VOLATILE_CAST(kmp_int##BITS *) & rhs)) { \
2598 old_value = temp_val; \
2608 #define MIN_MAX_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2609 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2610 TYPE new_value, old_value; \
2611 if (*lhs OP rhs) { \
2612 GOMP_MIN_MAX_CRITSECT_CPT(OP, GOMP_FLAG) \
2613 MIN_MAX_CRITSECT_CPT(OP, LCK_ID) \
2618 #define MIN_MAX_COMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2619 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2620 TYPE new_value, old_value; \
2621 if (*lhs OP rhs) { \
2622 GOMP_MIN_MAX_CRITSECT_CPT(OP, GOMP_FLAG) \
2623 MIN_MAX_CMPXCHG_CPT(TYPE, BITS, OP) \
2628 MIN_MAX_COMPXCHG_CPT(fixed1, max_cpt,
char, 8, <,
2630 MIN_MAX_COMPXCHG_CPT(fixed1, min_cpt,
char, 8, >,
2632 MIN_MAX_COMPXCHG_CPT(fixed2, max_cpt,
short, 16, <,
2634 MIN_MAX_COMPXCHG_CPT(fixed2, min_cpt,
short, 16, >,
2636 MIN_MAX_COMPXCHG_CPT(fixed4, max_cpt, kmp_int32, 32, <,
2638 MIN_MAX_COMPXCHG_CPT(fixed4, min_cpt, kmp_int32, 32, >,
2640 MIN_MAX_COMPXCHG_CPT(fixed8, max_cpt, kmp_int64, 64, <,
2642 MIN_MAX_COMPXCHG_CPT(fixed8, min_cpt, kmp_int64, 64, >,
2644 MIN_MAX_COMPXCHG_CPT(float4, max_cpt, kmp_real32, 32, <,
2646 MIN_MAX_COMPXCHG_CPT(float4, min_cpt, kmp_real32, 32, >,
2648 MIN_MAX_COMPXCHG_CPT(float8, max_cpt, kmp_real64, 64, <,
2650 MIN_MAX_COMPXCHG_CPT(float8, min_cpt, kmp_real64, 64, >,
2653 MIN_MAX_CRITICAL_CPT(float16, max_cpt, QUAD_LEGACY, <, 16r,
2655 MIN_MAX_CRITICAL_CPT(float16, min_cpt, QUAD_LEGACY, >, 16r,
2658 MIN_MAX_CRITICAL_CPT(float16, max_a16_cpt, Quad_a16_t, <, 16r,
2660 MIN_MAX_CRITICAL_CPT(float16, min_a16_cpt, Quad_a16_t, >, 16r,
2666 #ifdef KMP_GOMP_COMPAT
2667 #define OP_GOMP_CRITICAL_EQV_CPT(OP, FLAG) \
2668 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2670 OP_CRITICAL_CPT(OP, 0); \
2673 #define OP_GOMP_CRITICAL_EQV_CPT(OP, FLAG)
2676 #define ATOMIC_CMPX_EQV_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2677 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2679 OP_GOMP_CRITICAL_EQV_CPT(^= ~, GOMP_FLAG) \
2680 OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2685 ATOMIC_CMPXCHG_CPT(fixed1, neqv_cpt, kmp_int8, 8, ^,
2687 ATOMIC_CMPXCHG_CPT(fixed2, neqv_cpt, kmp_int16, 16, ^,
2689 ATOMIC_CMPXCHG_CPT(fixed4, neqv_cpt, kmp_int32, 32, ^,
2691 ATOMIC_CMPXCHG_CPT(fixed8, neqv_cpt, kmp_int64, 64, ^,
2693 ATOMIC_CMPX_EQV_CPT(fixed1, eqv_cpt, kmp_int8, 8, ^~,
2695 ATOMIC_CMPX_EQV_CPT(fixed2, eqv_cpt, kmp_int16, 16, ^~,
2697 ATOMIC_CMPX_EQV_CPT(fixed4, eqv_cpt, kmp_int32, 32, ^~,
2699 ATOMIC_CMPX_EQV_CPT(fixed8, eqv_cpt, kmp_int64, 64, ^~,
2708 #define ATOMIC_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2709 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2711 OP_GOMP_CRITICAL_CPT(OP, GOMP_FLAG) \
2712 OP_CRITICAL_CPT(OP## =, LCK_ID) \
2718 #define OP_CRITICAL_CPT_WRK(OP, LCK_ID) \
2719 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2729 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2733 #ifdef KMP_GOMP_COMPAT
2734 #define OP_GOMP_CRITICAL_CPT_WRK(OP, FLAG) \
2735 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2737 OP_CRITICAL_CPT_WRK(OP## =, 0); \
2740 #define OP_GOMP_CRITICAL_CPT_WRK(OP, FLAG)
2744 #define ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \
2745 void __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, TYPE *lhs, \
2746 TYPE rhs, TYPE *out, int flag) { \
2747 KMP_DEBUG_ASSERT(__kmp_init_serial); \
2748 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
2751 #define ATOMIC_CRITICAL_CPT_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2752 ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \
2753 OP_GOMP_CRITICAL_CPT_WRK(OP, GOMP_FLAG) \
2754 OP_CRITICAL_CPT_WRK(OP## =, LCK_ID) \
2760 ATOMIC_CRITICAL_CPT(float10, add_cpt,
long double, +, 10r,
2762 ATOMIC_CRITICAL_CPT(float10, sub_cpt,
long double, -, 10r,
2764 ATOMIC_CRITICAL_CPT(float10, mul_cpt,
long double, *, 10r,
2766 ATOMIC_CRITICAL_CPT(float10, div_cpt,
long double, /, 10r,
2770 ATOMIC_CRITICAL_CPT(float16, add_cpt, QUAD_LEGACY, +, 16r,
2772 ATOMIC_CRITICAL_CPT(float16, sub_cpt, QUAD_LEGACY, -, 16r,
2774 ATOMIC_CRITICAL_CPT(float16, mul_cpt, QUAD_LEGACY, *, 16r,
2776 ATOMIC_CRITICAL_CPT(float16, div_cpt, QUAD_LEGACY, /, 16r,
2779 ATOMIC_CRITICAL_CPT(float16, add_a16_cpt, Quad_a16_t, +, 16r,
2781 ATOMIC_CRITICAL_CPT(float16, sub_a16_cpt, Quad_a16_t, -, 16r,
2783 ATOMIC_CRITICAL_CPT(float16, mul_a16_cpt, Quad_a16_t, *, 16r,
2785 ATOMIC_CRITICAL_CPT(float16, div_a16_cpt, Quad_a16_t, /, 16r,
2793 ATOMIC_CRITICAL_CPT_WRK(cmplx4, add_cpt, kmp_cmplx32, +, 8c,
2795 ATOMIC_CRITICAL_CPT_WRK(cmplx4, sub_cpt, kmp_cmplx32, -, 8c,
2797 ATOMIC_CRITICAL_CPT_WRK(cmplx4, mul_cpt, kmp_cmplx32, *, 8c,
2799 ATOMIC_CRITICAL_CPT_WRK(cmplx4, div_cpt, kmp_cmplx32, /, 8c,
2802 ATOMIC_CRITICAL_CPT(cmplx8, add_cpt, kmp_cmplx64, +, 16c,
2804 ATOMIC_CRITICAL_CPT(cmplx8, sub_cpt, kmp_cmplx64, -, 16c,
2806 ATOMIC_CRITICAL_CPT(cmplx8, mul_cpt, kmp_cmplx64, *, 16c,
2808 ATOMIC_CRITICAL_CPT(cmplx8, div_cpt, kmp_cmplx64, /, 16c,
2810 ATOMIC_CRITICAL_CPT(cmplx10, add_cpt, kmp_cmplx80, +, 20c,
2812 ATOMIC_CRITICAL_CPT(cmplx10, sub_cpt, kmp_cmplx80, -, 20c,
2814 ATOMIC_CRITICAL_CPT(cmplx10, mul_cpt, kmp_cmplx80, *, 20c,
2816 ATOMIC_CRITICAL_CPT(cmplx10, div_cpt, kmp_cmplx80, /, 20c,
2819 ATOMIC_CRITICAL_CPT(cmplx16, add_cpt, CPLX128_LEG, +, 32c,
2821 ATOMIC_CRITICAL_CPT(cmplx16, sub_cpt, CPLX128_LEG, -, 32c,
2823 ATOMIC_CRITICAL_CPT(cmplx16, mul_cpt, CPLX128_LEG, *, 32c,
2825 ATOMIC_CRITICAL_CPT(cmplx16, div_cpt, CPLX128_LEG, /, 32c,
2828 ATOMIC_CRITICAL_CPT(cmplx16, add_a16_cpt, kmp_cmplx128_a16_t, +, 32c,
2830 ATOMIC_CRITICAL_CPT(cmplx16, sub_a16_cpt, kmp_cmplx128_a16_t, -, 32c,
2832 ATOMIC_CRITICAL_CPT(cmplx16, mul_a16_cpt, kmp_cmplx128_a16_t, *, 32c,
2834 ATOMIC_CRITICAL_CPT(cmplx16, div_a16_cpt, kmp_cmplx128_a16_t, /, 32c,
2851 #define OP_CRITICAL_CPT_REV(OP, LCK_ID) \
2852 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2856 (*lhs) = (rhs)OP(*lhs); \
2857 new_value = (*lhs); \
2859 new_value = (*lhs); \
2860 (*lhs) = (rhs)OP(*lhs); \
2862 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2866 #ifdef KMP_GOMP_COMPAT
2867 #define OP_GOMP_CRITICAL_CPT_REV(OP, FLAG) \
2868 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2870 OP_CRITICAL_CPT_REV(OP, 0); \
2873 #define OP_GOMP_CRITICAL_CPT_REV(OP, FLAG)
2883 #define OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \
2885 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2886 TYPE old_value, new_value; \
2888 old_value = temp_val; \
2889 new_value = rhs OP old_value; \
2890 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
2891 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
2892 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
2896 old_value = temp_val; \
2897 new_value = rhs OP old_value; \
2906 #define ATOMIC_CMPXCHG_CPT_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2907 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2909 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2910 OP_GOMP_CRITICAL_CPT_REV(OP, GOMP_FLAG) \
2911 OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \
2914 ATOMIC_CMPXCHG_CPT_REV(fixed1, div_cpt_rev, kmp_int8, 8, /,
2916 ATOMIC_CMPXCHG_CPT_REV(fixed1u, div_cpt_rev, kmp_uint8, 8, /,
2918 ATOMIC_CMPXCHG_CPT_REV(fixed1, shl_cpt_rev, kmp_int8, 8, <<,
2920 ATOMIC_CMPXCHG_CPT_REV(fixed1, shr_cpt_rev, kmp_int8, 8, >>,
2922 ATOMIC_CMPXCHG_CPT_REV(fixed1u, shr_cpt_rev, kmp_uint8, 8, >>,
2924 ATOMIC_CMPXCHG_CPT_REV(fixed1, sub_cpt_rev, kmp_int8, 8, -,
2926 ATOMIC_CMPXCHG_CPT_REV(fixed2, div_cpt_rev, kmp_int16, 16, /,
2928 ATOMIC_CMPXCHG_CPT_REV(fixed2u, div_cpt_rev, kmp_uint16, 16, /,
2930 ATOMIC_CMPXCHG_CPT_REV(fixed2, shl_cpt_rev, kmp_int16, 16, <<,
2932 ATOMIC_CMPXCHG_CPT_REV(fixed2, shr_cpt_rev, kmp_int16, 16, >>,
2934 ATOMIC_CMPXCHG_CPT_REV(fixed2u, shr_cpt_rev, kmp_uint16, 16, >>,
2936 ATOMIC_CMPXCHG_CPT_REV(fixed2, sub_cpt_rev, kmp_int16, 16, -,
2938 ATOMIC_CMPXCHG_CPT_REV(fixed4, div_cpt_rev, kmp_int32, 32, /,
2940 ATOMIC_CMPXCHG_CPT_REV(fixed4u, div_cpt_rev, kmp_uint32, 32, /,
2942 ATOMIC_CMPXCHG_CPT_REV(fixed4, shl_cpt_rev, kmp_int32, 32, <<,
2944 ATOMIC_CMPXCHG_CPT_REV(fixed4, shr_cpt_rev, kmp_int32, 32, >>,
2946 ATOMIC_CMPXCHG_CPT_REV(fixed4u, shr_cpt_rev, kmp_uint32, 32, >>,
2948 ATOMIC_CMPXCHG_CPT_REV(fixed4, sub_cpt_rev, kmp_int32, 32, -,
2950 ATOMIC_CMPXCHG_CPT_REV(fixed8, div_cpt_rev, kmp_int64, 64, /,
2952 ATOMIC_CMPXCHG_CPT_REV(fixed8u, div_cpt_rev, kmp_uint64, 64, /,
2954 ATOMIC_CMPXCHG_CPT_REV(fixed8, shl_cpt_rev, kmp_int64, 64, <<,
2956 ATOMIC_CMPXCHG_CPT_REV(fixed8, shr_cpt_rev, kmp_int64, 64, >>,
2958 ATOMIC_CMPXCHG_CPT_REV(fixed8u, shr_cpt_rev, kmp_uint64, 64, >>,
2960 ATOMIC_CMPXCHG_CPT_REV(fixed8, sub_cpt_rev, kmp_int64, 64, -,
2962 ATOMIC_CMPXCHG_CPT_REV(float4, div_cpt_rev, kmp_real32, 32, /,
2964 ATOMIC_CMPXCHG_CPT_REV(float4, sub_cpt_rev, kmp_real32, 32, -,
2966 ATOMIC_CMPXCHG_CPT_REV(float8, div_cpt_rev, kmp_real64, 64, /,
2968 ATOMIC_CMPXCHG_CPT_REV(float8, sub_cpt_rev, kmp_real64, 64, -,
2978 #define ATOMIC_CRITICAL_CPT_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2979 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2981 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2983 OP_GOMP_CRITICAL_CPT_REV(OP, GOMP_FLAG) \
2984 OP_CRITICAL_CPT_REV(OP, LCK_ID) \
2989 ATOMIC_CRITICAL_CPT_REV(float10, sub_cpt_rev,
long double, -, 10r,
2991 ATOMIC_CRITICAL_CPT_REV(float10, div_cpt_rev,
long double, /, 10r,
2995 ATOMIC_CRITICAL_CPT_REV(float16, sub_cpt_rev, QUAD_LEGACY, -, 16r,
2997 ATOMIC_CRITICAL_CPT_REV(float16, div_cpt_rev, QUAD_LEGACY, /, 16r,
3000 ATOMIC_CRITICAL_CPT_REV(float16, sub_a16_cpt_rev, Quad_a16_t, -, 16r,
3002 ATOMIC_CRITICAL_CPT_REV(float16, div_a16_cpt_rev, Quad_a16_t, /, 16r,
3012 #define OP_CRITICAL_CPT_REV_WRK(OP, LCK_ID) \
3013 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3016 (*lhs) = (rhs)OP(*lhs); \
3020 (*lhs) = (rhs)OP(*lhs); \
3023 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3027 #ifdef KMP_GOMP_COMPAT
3028 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP, FLAG) \
3029 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
3031 OP_CRITICAL_CPT_REV_WRK(OP, 0); \
3034 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP, FLAG)
3038 #define ATOMIC_CRITICAL_CPT_REV_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, \
3040 ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \
3041 OP_GOMP_CRITICAL_CPT_REV_WRK(OP, GOMP_FLAG) \
3042 OP_CRITICAL_CPT_REV_WRK(OP, LCK_ID) \
3048 ATOMIC_CRITICAL_CPT_REV_WRK(cmplx4, sub_cpt_rev, kmp_cmplx32, -, 8c,
3050 ATOMIC_CRITICAL_CPT_REV_WRK(cmplx4, div_cpt_rev, kmp_cmplx32, /, 8c,
3053 ATOMIC_CRITICAL_CPT_REV(cmplx8, sub_cpt_rev, kmp_cmplx64, -, 16c,
3055 ATOMIC_CRITICAL_CPT_REV(cmplx8, div_cpt_rev, kmp_cmplx64, /, 16c,
3057 ATOMIC_CRITICAL_CPT_REV(cmplx10, sub_cpt_rev, kmp_cmplx80, -, 20c,
3059 ATOMIC_CRITICAL_CPT_REV(cmplx10, div_cpt_rev, kmp_cmplx80, /, 20c,
3062 ATOMIC_CRITICAL_CPT_REV(cmplx16, sub_cpt_rev, CPLX128_LEG, -, 32c,
3064 ATOMIC_CRITICAL_CPT_REV(cmplx16, div_cpt_rev, CPLX128_LEG, /, 32c,
3067 ATOMIC_CRITICAL_CPT_REV(cmplx16, sub_a16_cpt_rev, kmp_cmplx128_a16_t, -, 32c,
3069 ATOMIC_CRITICAL_CPT_REV(cmplx16, div_a16_cpt_rev, kmp_cmplx128_a16_t, /, 32c,
3083 #define ATOMIC_CMPXCHG_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \
3084 RTYPE, LCK_ID, MASK, GOMP_FLAG) \
3085 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
3087 OP_GOMP_CRITICAL_CPT_REV(OP, GOMP_FLAG) \
3088 OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \
3092 #define ATOMIC_CRITICAL_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \
3093 LCK_ID, GOMP_FLAG) \
3094 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
3096 OP_GOMP_CRITICAL_CPT_REV(OP, GOMP_FLAG) \
3097 OP_CRITICAL_CPT_REV(OP, LCK_ID) \
3100 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1,
char, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0,
3102 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1u, uchar, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0,
3104 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1,
char, div_cpt_rev, 8, /, fp, _Quad, 1i, 0,
3106 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1u, uchar, div_cpt_rev, 8, /, fp, _Quad, 1i, 0,
3109 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2,
short, sub_cpt_rev, 16, -, fp, _Quad, 2i, 1,
3111 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2u, ushort, sub_cpt_rev, 16, -, fp, _Quad, 2i,
3114 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2,
short, div_cpt_rev, 16, /, fp, _Quad, 2i, 1,
3116 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2u, ushort, div_cpt_rev, 16, /, fp, _Quad, 2i,
3120 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4, kmp_int32, sub_cpt_rev, 32, -, fp, _Quad, 4i,
3122 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4u, kmp_uint32, sub_cpt_rev, 32, -, fp, _Quad,
3124 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4, kmp_int32, div_cpt_rev, 32, /, fp, _Quad, 4i,
3126 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4u, kmp_uint32, div_cpt_rev, 32, /, fp, _Quad,
3129 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8, kmp_int64, sub_cpt_rev, 64, -, fp, _Quad, 8i,
3132 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8u, kmp_uint64, sub_cpt_rev, 64, -, fp, _Quad,
3135 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8, kmp_int64, div_cpt_rev, 64, /, fp, _Quad, 8i,
3138 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8u, kmp_uint64, div_cpt_rev, 64, /, fp, _Quad,
3142 ATOMIC_CMPXCHG_CPT_REV_MIX(float4, kmp_real32, sub_cpt_rev, 32, -, fp, _Quad,
3145 ATOMIC_CMPXCHG_CPT_REV_MIX(float4, kmp_real32, div_cpt_rev, 32, /, fp, _Quad,
3149 ATOMIC_CMPXCHG_CPT_REV_MIX(float8, kmp_real64, sub_cpt_rev, 64, -, fp, _Quad,
3152 ATOMIC_CMPXCHG_CPT_REV_MIX(float8, kmp_real64, div_cpt_rev, 64, /, fp, _Quad,
3156 ATOMIC_CRITICAL_CPT_REV_MIX(float10,
long double, sub_cpt_rev, -, fp, _Quad,
3158 ATOMIC_CRITICAL_CPT_REV_MIX(float10,
long double, div_cpt_rev, /, fp, _Quad,
3161 #endif // KMP_HAVE_QUAD
3165 #define ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3166 TYPE __kmpc_atomic_##TYPE_ID##_swp(ident_t *id_ref, int gtid, TYPE *lhs, \
3168 KMP_DEBUG_ASSERT(__kmp_init_serial); \
3169 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid));
3171 #define CRITICAL_SWP(LCK_ID) \
3172 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3174 old_value = (*lhs); \
3177 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3181 #ifdef KMP_GOMP_COMPAT
3182 #define GOMP_CRITICAL_SWP(FLAG) \
3183 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
3188 #define GOMP_CRITICAL_SWP(FLAG)
3191 #define ATOMIC_XCHG_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \
3192 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3194 GOMP_CRITICAL_SWP(GOMP_FLAG) \
3195 old_value = KMP_XCHG_FIXED##BITS(lhs, rhs); \
3199 #define ATOMIC_XCHG_FLOAT_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \
3200 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3202 GOMP_CRITICAL_SWP(GOMP_FLAG) \
3203 old_value = KMP_XCHG_REAL##BITS(lhs, rhs); \
3208 #define CMPXCHG_SWP(TYPE, BITS) \
3210 TYPE KMP_ATOMIC_VOLATILE temp_val; \
3211 TYPE old_value, new_value; \
3213 old_value = temp_val; \
3215 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
3216 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
3217 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
3221 old_value = temp_val; \
3228 #define ATOMIC_CMPXCHG_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \
3229 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3231 GOMP_CRITICAL_SWP(GOMP_FLAG) \
3232 CMPXCHG_SWP(TYPE, BITS) \
3235 ATOMIC_XCHG_SWP(fixed1, kmp_int8, 8, KMP_ARCH_X86)
3236 ATOMIC_XCHG_SWP(fixed2, kmp_int16, 16, KMP_ARCH_X86)
3237 ATOMIC_XCHG_SWP(fixed4, kmp_int32, 32, KMP_ARCH_X86)
3239 ATOMIC_XCHG_FLOAT_SWP(float4, kmp_real32, 32,
3243 ATOMIC_CMPXCHG_SWP(fixed8, kmp_int64, 64,
3245 ATOMIC_CMPXCHG_SWP(float8, kmp_real64, 64,
3248 ATOMIC_XCHG_SWP(fixed8, kmp_int64, 64, KMP_ARCH_X86)
3249 ATOMIC_XCHG_FLOAT_SWP(float8, kmp_real64, 64,
3256 #define ATOMIC_CRITICAL_SWP(TYPE_ID, TYPE, LCK_ID, GOMP_FLAG) \
3257 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3259 GOMP_CRITICAL_SWP(GOMP_FLAG) \
3260 CRITICAL_SWP(LCK_ID) \
3268 #define ATOMIC_BEGIN_SWP_WRK(TYPE_ID, TYPE) \
3269 void __kmpc_atomic_##TYPE_ID##_swp(ident_t *id_ref, int gtid, TYPE *lhs, \
3270 TYPE rhs, TYPE *out) { \
3271 KMP_DEBUG_ASSERT(__kmp_init_serial); \
3272 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid));
3274 #define CRITICAL_SWP_WRK(LCK_ID) \
3275 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3280 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3284 #ifdef KMP_GOMP_COMPAT
3285 #define GOMP_CRITICAL_SWP_WRK(FLAG) \
3286 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
3288 CRITICAL_SWP_WRK(0); \
3291 #define GOMP_CRITICAL_SWP_WRK(FLAG)
3295 #define ATOMIC_CRITICAL_SWP_WRK(TYPE_ID, TYPE, LCK_ID, GOMP_FLAG) \
3296 ATOMIC_BEGIN_SWP_WRK(TYPE_ID, TYPE) \
3298 GOMP_CRITICAL_SWP_WRK(GOMP_FLAG) \
3299 CRITICAL_SWP_WRK(LCK_ID) \
3303 ATOMIC_CRITICAL_SWP(float10,
long double, 10r, 1)
3305 ATOMIC_CRITICAL_SWP(float16, QUAD_LEGACY, 16r, 1)
3308 ATOMIC_CRITICAL_SWP_WRK(cmplx4, kmp_cmplx32, 8c, 1)
3313 ATOMIC_CRITICAL_SWP(cmplx8, kmp_cmplx64, 16c, 1)
3314 ATOMIC_CRITICAL_SWP(cmplx10, kmp_cmplx80, 20c, 1)
3316 ATOMIC_CRITICAL_SWP(cmplx16, CPLX128_LEG, 32c, 1)
3318 ATOMIC_CRITICAL_SWP(float16_a16, Quad_a16_t, 16r,
3320 ATOMIC_CRITICAL_SWP(cmplx16_a16, kmp_cmplx128_a16_t, 32c,
3327 #endif // OMP_40_ENABLED
3329 #endif // KMP_ARCH_X86 || KMP_ARCH_X86_64
3336 void __kmpc_atomic_1(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3337 void (*f)(
void *,
void *,
void *)) {
3338 KMP_DEBUG_ASSERT(__kmp_init_serial);
3341 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3347 kmp_int8 old_value, new_value;
3349 old_value = *(kmp_int8 *)lhs;
3350 (*f)(&new_value, &old_value, rhs);
3353 while (!KMP_COMPARE_AND_STORE_ACQ8((kmp_int8 *)lhs, *(kmp_int8 *)&old_value,
3354 *(kmp_int8 *)&new_value)) {
3357 old_value = *(kmp_int8 *)lhs;
3358 (*f)(&new_value, &old_value, rhs);
3365 #ifdef KMP_GOMP_COMPAT
3366 if (__kmp_atomic_mode == 2) {
3367 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3370 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_1i, gtid);
3372 (*f)(lhs, lhs, rhs);
3374 #ifdef KMP_GOMP_COMPAT
3375 if (__kmp_atomic_mode == 2) {
3376 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3379 __kmp_release_atomic_lock(&__kmp_atomic_lock_1i, gtid);
3383 void __kmpc_atomic_2(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3384 void (*f)(
void *,
void *,
void *)) {
3386 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3388 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
3391 !((kmp_uintptr_t)lhs & 0x1)
3394 kmp_int16 old_value, new_value;
3396 old_value = *(kmp_int16 *)lhs;
3397 (*f)(&new_value, &old_value, rhs);
3400 while (!KMP_COMPARE_AND_STORE_ACQ16(
3401 (kmp_int16 *)lhs, *(kmp_int16 *)&old_value, *(kmp_int16 *)&new_value)) {
3404 old_value = *(kmp_int16 *)lhs;
3405 (*f)(&new_value, &old_value, rhs);
3412 #ifdef KMP_GOMP_COMPAT
3413 if (__kmp_atomic_mode == 2) {
3414 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3417 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_2i, gtid);
3419 (*f)(lhs, lhs, rhs);
3421 #ifdef KMP_GOMP_COMPAT
3422 if (__kmp_atomic_mode == 2) {
3423 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3426 __kmp_release_atomic_lock(&__kmp_atomic_lock_2i, gtid);
3430 void __kmpc_atomic_4(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3431 void (*f)(
void *,
void *,
void *)) {
3432 KMP_DEBUG_ASSERT(__kmp_init_serial);
3437 #
if KMP_ARCH_X86 || KMP_ARCH_X86_64
3440 !((kmp_uintptr_t)lhs & 0x3)
3443 kmp_int32 old_value, new_value;
3445 old_value = *(kmp_int32 *)lhs;
3446 (*f)(&new_value, &old_value, rhs);
3449 while (!KMP_COMPARE_AND_STORE_ACQ32(
3450 (kmp_int32 *)lhs, *(kmp_int32 *)&old_value, *(kmp_int32 *)&new_value)) {
3453 old_value = *(kmp_int32 *)lhs;
3454 (*f)(&new_value, &old_value, rhs);
3462 #ifdef KMP_GOMP_COMPAT
3463 if (__kmp_atomic_mode == 2) {
3464 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3467 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_4i, gtid);
3469 (*f)(lhs, lhs, rhs);
3471 #ifdef KMP_GOMP_COMPAT
3472 if (__kmp_atomic_mode == 2) {
3473 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3476 __kmp_release_atomic_lock(&__kmp_atomic_lock_4i, gtid);
3480 void __kmpc_atomic_8(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3481 void (*f)(
void *,
void *,
void *)) {
3482 KMP_DEBUG_ASSERT(__kmp_init_serial);
3485 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3487 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
3490 !((kmp_uintptr_t)lhs & 0x7)
3493 kmp_int64 old_value, new_value;
3495 old_value = *(kmp_int64 *)lhs;
3496 (*f)(&new_value, &old_value, rhs);
3498 while (!KMP_COMPARE_AND_STORE_ACQ64(
3499 (kmp_int64 *)lhs, *(kmp_int64 *)&old_value, *(kmp_int64 *)&new_value)) {
3502 old_value = *(kmp_int64 *)lhs;
3503 (*f)(&new_value, &old_value, rhs);
3511 #ifdef KMP_GOMP_COMPAT
3512 if (__kmp_atomic_mode == 2) {
3513 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3516 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_8i, gtid);
3518 (*f)(lhs, lhs, rhs);
3520 #ifdef KMP_GOMP_COMPAT
3521 if (__kmp_atomic_mode == 2) {
3522 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3525 __kmp_release_atomic_lock(&__kmp_atomic_lock_8i, gtid);
3529 void __kmpc_atomic_10(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3530 void (*f)(
void *,
void *,
void *)) {
3531 KMP_DEBUG_ASSERT(__kmp_init_serial);
3533 #ifdef KMP_GOMP_COMPAT
3534 if (__kmp_atomic_mode == 2) {
3535 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3538 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_10r, gtid);
3540 (*f)(lhs, lhs, rhs);
3542 #ifdef KMP_GOMP_COMPAT
3543 if (__kmp_atomic_mode == 2) {
3544 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3547 __kmp_release_atomic_lock(&__kmp_atomic_lock_10r, gtid);
3550 void __kmpc_atomic_16(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3551 void (*f)(
void *,
void *,
void *)) {
3552 KMP_DEBUG_ASSERT(__kmp_init_serial);
3554 #ifdef KMP_GOMP_COMPAT
3555 if (__kmp_atomic_mode == 2) {
3556 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3559 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_16c, gtid);
3561 (*f)(lhs, lhs, rhs);
3563 #ifdef KMP_GOMP_COMPAT
3564 if (__kmp_atomic_mode == 2) {
3565 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3568 __kmp_release_atomic_lock(&__kmp_atomic_lock_16c, gtid);
3571 void __kmpc_atomic_20(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3572 void (*f)(
void *,
void *,
void *)) {
3573 KMP_DEBUG_ASSERT(__kmp_init_serial);
3575 #ifdef KMP_GOMP_COMPAT
3576 if (__kmp_atomic_mode == 2) {
3577 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3580 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_20c, gtid);
3582 (*f)(lhs, lhs, rhs);
3584 #ifdef KMP_GOMP_COMPAT
3585 if (__kmp_atomic_mode == 2) {
3586 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3589 __kmp_release_atomic_lock(&__kmp_atomic_lock_20c, gtid);
3592 void __kmpc_atomic_32(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3593 void (*f)(
void *,
void *,
void *)) {
3594 KMP_DEBUG_ASSERT(__kmp_init_serial);
3596 #ifdef KMP_GOMP_COMPAT
3597 if (__kmp_atomic_mode == 2) {
3598 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3601 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_32c, gtid);
3603 (*f)(lhs, lhs, rhs);
3605 #ifdef KMP_GOMP_COMPAT
3606 if (__kmp_atomic_mode == 2) {
3607 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3610 __kmp_release_atomic_lock(&__kmp_atomic_lock_32c, gtid);
3616 void __kmpc_atomic_start(
void) {
3617 int gtid = __kmp_entry_gtid();
3618 KA_TRACE(20, (
"__kmpc_atomic_start: T#%d\n", gtid));
3619 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3622 void __kmpc_atomic_end(
void) {
3623 int gtid = __kmp_get_gtid();
3624 KA_TRACE(20, (
"__kmpc_atomic_end: T#%d\n", gtid));
3625 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);