13 #include "kmp_atomic.h"
16 typedef unsigned char uchar;
17 typedef unsigned short ushort;
565 #ifndef KMP_GOMP_COMPAT
566 int __kmp_atomic_mode = 1;
568 int __kmp_atomic_mode = 2;
574 kmp_atomic_lock_t __kmp_atomic_lock;
576 kmp_atomic_lock_t __kmp_atomic_lock_1i;
578 kmp_atomic_lock_t __kmp_atomic_lock_2i;
580 kmp_atomic_lock_t __kmp_atomic_lock_4i;
582 kmp_atomic_lock_t __kmp_atomic_lock_4r;
584 kmp_atomic_lock_t __kmp_atomic_lock_8i;
586 kmp_atomic_lock_t __kmp_atomic_lock_8r;
588 kmp_atomic_lock_t __kmp_atomic_lock_8c;
590 kmp_atomic_lock_t __kmp_atomic_lock_10r;
592 kmp_atomic_lock_t __kmp_atomic_lock_16r;
594 kmp_atomic_lock_t __kmp_atomic_lock_16c;
596 kmp_atomic_lock_t __kmp_atomic_lock_20c;
598 kmp_atomic_lock_t __kmp_atomic_lock_32c;
605 #define KMP_ATOMIC_VOLATILE volatile
607 #if (KMP_ARCH_X86) && KMP_HAVE_QUAD
609 static inline Quad_a4_t operator+(Quad_a4_t &lhs, Quad_a4_t &rhs) {
610 return lhs.q + rhs.q;
612 static inline Quad_a4_t operator-(Quad_a4_t &lhs, Quad_a4_t &rhs) {
613 return lhs.q - rhs.q;
615 static inline Quad_a4_t operator*(Quad_a4_t &lhs, Quad_a4_t &rhs) {
616 return lhs.q * rhs.q;
618 static inline Quad_a4_t operator/(Quad_a4_t &lhs, Quad_a4_t &rhs) {
619 return lhs.q / rhs.q;
621 static inline bool operator<(Quad_a4_t &lhs, Quad_a4_t &rhs) {
622 return lhs.q < rhs.q;
624 static inline bool operator>(Quad_a4_t &lhs, Quad_a4_t &rhs) {
625 return lhs.q > rhs.q;
628 static inline Quad_a16_t operator+(Quad_a16_t &lhs, Quad_a16_t &rhs) {
629 return lhs.q + rhs.q;
631 static inline Quad_a16_t operator-(Quad_a16_t &lhs, Quad_a16_t &rhs) {
632 return lhs.q - rhs.q;
634 static inline Quad_a16_t operator*(Quad_a16_t &lhs, Quad_a16_t &rhs) {
635 return lhs.q * rhs.q;
637 static inline Quad_a16_t operator/(Quad_a16_t &lhs, Quad_a16_t &rhs) {
638 return lhs.q / rhs.q;
640 static inline bool operator<(Quad_a16_t &lhs, Quad_a16_t &rhs) {
641 return lhs.q < rhs.q;
643 static inline bool operator>(Quad_a16_t &lhs, Quad_a16_t &rhs) {
644 return lhs.q > rhs.q;
647 static inline kmp_cmplx128_a4_t operator+(kmp_cmplx128_a4_t &lhs,
648 kmp_cmplx128_a4_t &rhs) {
649 return lhs.q + rhs.q;
651 static inline kmp_cmplx128_a4_t operator-(kmp_cmplx128_a4_t &lhs,
652 kmp_cmplx128_a4_t &rhs) {
653 return lhs.q - rhs.q;
655 static inline kmp_cmplx128_a4_t operator*(kmp_cmplx128_a4_t &lhs,
656 kmp_cmplx128_a4_t &rhs) {
657 return lhs.q * rhs.q;
659 static inline kmp_cmplx128_a4_t operator/(kmp_cmplx128_a4_t &lhs,
660 kmp_cmplx128_a4_t &rhs) {
661 return lhs.q / rhs.q;
664 static inline kmp_cmplx128_a16_t operator+(kmp_cmplx128_a16_t &lhs,
665 kmp_cmplx128_a16_t &rhs) {
666 return lhs.q + rhs.q;
668 static inline kmp_cmplx128_a16_t operator-(kmp_cmplx128_a16_t &lhs,
669 kmp_cmplx128_a16_t &rhs) {
670 return lhs.q - rhs.q;
672 static inline kmp_cmplx128_a16_t operator*(kmp_cmplx128_a16_t &lhs,
673 kmp_cmplx128_a16_t &rhs) {
674 return lhs.q * rhs.q;
676 static inline kmp_cmplx128_a16_t operator/(kmp_cmplx128_a16_t &lhs,
677 kmp_cmplx128_a16_t &rhs) {
678 return lhs.q / rhs.q;
688 #define KMP_CHECK_GTID \
689 if (gtid == KMP_GTID_UNKNOWN) { \
690 gtid = __kmp_entry_gtid(); \
698 #define ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
699 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \
700 TYPE *lhs, TYPE rhs) { \
701 KMP_DEBUG_ASSERT(__kmp_init_serial); \
702 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
706 #define ATOMIC_LOCK0 __kmp_atomic_lock
707 #define ATOMIC_LOCK1i __kmp_atomic_lock_1i
708 #define ATOMIC_LOCK2i __kmp_atomic_lock_2i
709 #define ATOMIC_LOCK4i __kmp_atomic_lock_4i
710 #define ATOMIC_LOCK4r __kmp_atomic_lock_4r
711 #define ATOMIC_LOCK8i __kmp_atomic_lock_8i
712 #define ATOMIC_LOCK8r __kmp_atomic_lock_8r
713 #define ATOMIC_LOCK8c __kmp_atomic_lock_8c
714 #define ATOMIC_LOCK10r __kmp_atomic_lock_10r
715 #define ATOMIC_LOCK16r __kmp_atomic_lock_16r
716 #define ATOMIC_LOCK16c __kmp_atomic_lock_16c
717 #define ATOMIC_LOCK20c __kmp_atomic_lock_20c
718 #define ATOMIC_LOCK32c __kmp_atomic_lock_32c
726 #define OP_CRITICAL(OP, LCK_ID) \
727 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
731 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
733 #define OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) \
734 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
735 (*lhs) = (TYPE)((*lhs)OP rhs); \
736 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
760 #ifdef KMP_GOMP_COMPAT
761 #define OP_GOMP_CRITICAL(OP, FLAG) \
762 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
764 OP_CRITICAL(OP, 0); \
768 #define OP_UPDATE_GOMP_CRITICAL(TYPE, OP, FLAG) \
769 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
771 OP_UPDATE_CRITICAL(TYPE, OP, 0); \
775 #define OP_GOMP_CRITICAL(OP, FLAG)
776 #define OP_UPDATE_GOMP_CRITICAL(TYPE, OP, FLAG)
780 #define KMP_DO_PAUSE _mm_delay_32(1)
790 #define OP_CMPXCHG(TYPE, BITS, OP) \
792 TYPE old_value, new_value; \
793 old_value = *(TYPE volatile *)lhs; \
794 new_value = (TYPE)(old_value OP rhs); \
795 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
796 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
797 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
800 old_value = *(TYPE volatile *)lhs; \
801 new_value = (TYPE)(old_value OP rhs); \
812 #define OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \
816 kmp_int##BITS *vvv; \
818 struct _sss old_value, new_value; \
819 old_value.vvv = (kmp_int##BITS *)&old_value.cmp; \
820 new_value.vvv = (kmp_int##BITS *)&new_value.cmp; \
821 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \
822 new_value.cmp = (TYPE)(old_value.cmp OP rhs); \
823 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
824 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \
825 *VOLATILE_CAST(kmp_int##BITS *) new_value.vvv)) { \
828 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \
829 new_value.cmp = (TYPE)(old_value.cmp OP rhs); \
835 #if KMP_OS_WINDOWS && KMP_ARCH_AARCH64
839 #define OP_CMPXCHG(TYPE, BITS, OP) \
843 kmp_int##BITS *vvv; \
845 struct _sss old_value, new_value; \
846 old_value.vvv = (kmp_int##BITS *)&old_value.cmp; \
847 new_value.vvv = (kmp_int##BITS *)&new_value.cmp; \
848 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \
849 new_value.cmp = old_value.cmp OP rhs; \
850 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
851 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) old_value.vvv, \
852 *VOLATILE_CAST(kmp_int##BITS *) new_value.vvv)) { \
855 *old_value.vvv = *(volatile kmp_int##BITS *)lhs; \
856 new_value.cmp = old_value.cmp OP rhs; \
860 #undef OP_UPDATE_CRITICAL
861 #define OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) \
862 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
863 (*lhs) = (*lhs)OP rhs; \
864 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
868 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
872 #define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
874 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
875 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
877 KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \
880 #define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
882 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
883 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
884 OP_CMPXCHG(TYPE, BITS, OP) \
889 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \
891 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
892 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
893 OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \
901 #define ATOMIC_FIXED_ADD(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
903 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
904 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
905 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
907 KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \
910 OP_UPDATE_CRITICAL(TYPE, OP, \
915 #define ATOMIC_CMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
917 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
918 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
919 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
920 OP_CMPXCHG(TYPE, BITS, OP) \
923 OP_UPDATE_CRITICAL(TYPE, OP, \
930 #define ATOMIC_CMPXCHG_WORKAROUND(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, \
932 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
933 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
934 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
935 OP_CMPXCHG(TYPE, BITS, OP) \
938 OP_UPDATE_CRITICAL(TYPE, OP, \
947 ATOMIC_FIXED_ADD(fixed4, add, kmp_int32, 32, +, 4i, 3,
949 ATOMIC_FIXED_ADD(fixed4, sub, kmp_int32, 32, -, 4i, 3,
952 ATOMIC_CMPXCHG(float4, add, kmp_real32, 32, +, 4r, 3,
954 ATOMIC_CMPXCHG(float4, sub, kmp_real32, 32, -, 4r, 3,
958 ATOMIC_FIXED_ADD(fixed8, add, kmp_int64, 64, +, 8i, 7,
960 ATOMIC_FIXED_ADD(fixed8, sub, kmp_int64, 64, -, 8i, 7,
963 ATOMIC_CMPXCHG(float8, add, kmp_real64, 64, +, 8r, 7,
965 ATOMIC_CMPXCHG(float8, sub, kmp_real64, 64, -, 8r, 7,
983 ATOMIC_CMPXCHG(fixed1, add, kmp_int8, 8, +, 1i, 0,
985 ATOMIC_CMPXCHG(fixed1, andb, kmp_int8, 8, &, 1i, 0,
987 ATOMIC_CMPXCHG(fixed1, div, kmp_int8, 8, /, 1i, 0,
989 ATOMIC_CMPXCHG(fixed1u, div, kmp_uint8, 8, /, 1i, 0,
991 ATOMIC_CMPXCHG(fixed1, mul, kmp_int8, 8, *, 1i, 0,
993 ATOMIC_CMPXCHG(fixed1, orb, kmp_int8, 8, |, 1i, 0,
995 ATOMIC_CMPXCHG(fixed1, shl, kmp_int8, 8, <<, 1i, 0,
997 ATOMIC_CMPXCHG(fixed1, shr, kmp_int8, 8, >>, 1i, 0,
999 ATOMIC_CMPXCHG(fixed1u, shr, kmp_uint8, 8, >>, 1i, 0,
1001 ATOMIC_CMPXCHG(fixed1, sub, kmp_int8, 8, -, 1i, 0,
1003 ATOMIC_CMPXCHG(fixed1, xor, kmp_int8, 8, ^, 1i, 0,
1005 ATOMIC_CMPXCHG(fixed2, add, kmp_int16, 16, +, 2i, 1,
1007 ATOMIC_CMPXCHG(fixed2, andb, kmp_int16, 16, &, 2i, 1,
1009 ATOMIC_CMPXCHG(fixed2, div, kmp_int16, 16, /, 2i, 1,
1011 ATOMIC_CMPXCHG(fixed2u, div, kmp_uint16, 16, /, 2i, 1,
1013 ATOMIC_CMPXCHG(fixed2, mul, kmp_int16, 16, *, 2i, 1,
1015 ATOMIC_CMPXCHG(fixed2, orb, kmp_int16, 16, |, 2i, 1,
1017 ATOMIC_CMPXCHG(fixed2, shl, kmp_int16, 16, <<, 2i, 1,
1019 ATOMIC_CMPXCHG(fixed2, shr, kmp_int16, 16, >>, 2i, 1,
1021 ATOMIC_CMPXCHG(fixed2u, shr, kmp_uint16, 16, >>, 2i, 1,
1023 ATOMIC_CMPXCHG(fixed2, sub, kmp_int16, 16, -, 2i, 1,
1025 ATOMIC_CMPXCHG(fixed2, xor, kmp_int16, 16, ^, 2i, 1,
1027 ATOMIC_CMPXCHG(fixed4, andb, kmp_int32, 32, &, 4i, 3,
1029 ATOMIC_CMPXCHG(fixed4, div, kmp_int32, 32, /, 4i, 3,
1031 ATOMIC_CMPXCHG(fixed4u, div, kmp_uint32, 32, /, 4i, 3,
1033 ATOMIC_CMPXCHG(fixed4, mul, kmp_int32, 32, *, 4i, 3,
1035 ATOMIC_CMPXCHG(fixed4, orb, kmp_int32, 32, |, 4i, 3,
1037 ATOMIC_CMPXCHG(fixed4, shl, kmp_int32, 32, <<, 4i, 3,
1039 ATOMIC_CMPXCHG(fixed4, shr, kmp_int32, 32, >>, 4i, 3,
1041 ATOMIC_CMPXCHG(fixed4u, shr, kmp_uint32, 32, >>, 4i, 3,
1043 ATOMIC_CMPXCHG(fixed4, xor, kmp_int32, 32, ^, 4i, 3,
1045 ATOMIC_CMPXCHG(fixed8, andb, kmp_int64, 64, &, 8i, 7,
1047 ATOMIC_CMPXCHG(fixed8, div, kmp_int64, 64, /, 8i, 7,
1049 ATOMIC_CMPXCHG(fixed8u, div, kmp_uint64, 64, /, 8i, 7,
1051 ATOMIC_CMPXCHG(fixed8, mul, kmp_int64, 64, *, 8i, 7,
1053 ATOMIC_CMPXCHG(fixed8, orb, kmp_int64, 64, |, 8i, 7,
1055 ATOMIC_CMPXCHG(fixed8, shl, kmp_int64, 64, <<, 8i, 7,
1057 ATOMIC_CMPXCHG(fixed8, shr, kmp_int64, 64, >>, 8i, 7,
1059 ATOMIC_CMPXCHG(fixed8u, shr, kmp_uint64, 64, >>, 8i, 7,
1061 ATOMIC_CMPXCHG(fixed8, xor, kmp_int64, 64, ^, 8i, 7,
1063 ATOMIC_CMPXCHG(float4, div, kmp_real32, 32, /, 4r, 3,
1065 ATOMIC_CMPXCHG(float4, mul, kmp_real32, 32, *, 4r, 3,
1067 ATOMIC_CMPXCHG(float8, div, kmp_real64, 64, /, 8r, 7,
1069 ATOMIC_CMPXCHG(float8, mul, kmp_real64, 64, *, 8r, 7,
1079 #define ATOMIC_CRIT_L(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1080 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1081 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \
1082 OP_CRITICAL(= *lhs OP, LCK_ID) \
1085 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1089 #define ATOMIC_CMPX_L(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG) \
1090 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1091 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \
1092 OP_CMPXCHG(TYPE, BITS, OP) \
1098 #define ATOMIC_CMPX_L(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, GOMP_FLAG) \
1099 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1100 OP_GOMP_CRITICAL(= *lhs OP, GOMP_FLAG) \
1101 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1102 OP_CMPXCHG(TYPE, BITS, OP) \
1105 OP_CRITICAL(= *lhs OP, LCK_ID) \
1110 ATOMIC_CMPX_L(fixed1, andl,
char, 8, &&, 1i, 0,
1112 ATOMIC_CMPX_L(fixed1, orl,
char, 8, ||, 1i, 0,
1114 ATOMIC_CMPX_L(fixed2, andl,
short, 16, &&, 2i, 1,
1116 ATOMIC_CMPX_L(fixed2, orl,
short, 16, ||, 2i, 1,
1118 ATOMIC_CMPX_L(fixed4, andl, kmp_int32, 32, &&, 4i, 3,
1120 ATOMIC_CMPX_L(fixed4, orl, kmp_int32, 32, ||, 4i, 3,
1122 ATOMIC_CMPX_L(fixed8, andl, kmp_int64, 64, &&, 8i, 7,
1124 ATOMIC_CMPX_L(fixed8, orl, kmp_int64, 64, ||, 8i, 7,
1136 #define MIN_MAX_CRITSECT(OP, LCK_ID) \
1137 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
1139 if (*lhs OP rhs) { \
1142 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
1145 #ifdef KMP_GOMP_COMPAT
1146 #define GOMP_MIN_MAX_CRITSECT(OP, FLAG) \
1147 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
1149 MIN_MAX_CRITSECT(OP, 0); \
1153 #define GOMP_MIN_MAX_CRITSECT(OP, FLAG)
1157 #define MIN_MAX_CMPXCHG(TYPE, BITS, OP) \
1159 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1162 old_value = temp_val; \
1163 while (old_value OP rhs && \
1164 !KMP_COMPARE_AND_STORE_ACQ##BITS( \
1165 (kmp_int##BITS *)lhs, \
1166 *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
1167 *VOLATILE_CAST(kmp_int##BITS *) & rhs)) { \
1169 old_value = temp_val; \
1175 #define MIN_MAX_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1176 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1177 if (*lhs OP rhs) { \
1178 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \
1179 MIN_MAX_CRITSECT(OP, LCK_ID) \
1183 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1187 #define MIN_MAX_COMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1189 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1190 if (*lhs OP rhs) { \
1191 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \
1192 MIN_MAX_CMPXCHG(TYPE, BITS, OP) \
1199 #define MIN_MAX_COMPXCHG(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1201 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1202 if (*lhs OP rhs) { \
1203 GOMP_MIN_MAX_CRITSECT(OP, GOMP_FLAG) \
1204 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1205 MIN_MAX_CMPXCHG(TYPE, BITS, OP) \
1208 MIN_MAX_CRITSECT(OP, LCK_ID) \
1214 MIN_MAX_COMPXCHG(fixed1, max,
char, 8, <, 1i, 0,
1216 MIN_MAX_COMPXCHG(fixed1, min,
char, 8, >, 1i, 0,
1218 MIN_MAX_COMPXCHG(fixed2, max,
short, 16, <, 2i, 1,
1220 MIN_MAX_COMPXCHG(fixed2, min,
short, 16, >, 2i, 1,
1222 MIN_MAX_COMPXCHG(fixed4, max, kmp_int32, 32, <, 4i, 3,
1224 MIN_MAX_COMPXCHG(fixed4, min, kmp_int32, 32, >, 4i, 3,
1226 MIN_MAX_COMPXCHG(fixed8, max, kmp_int64, 64, <, 8i, 7,
1228 MIN_MAX_COMPXCHG(fixed8, min, kmp_int64, 64, >, 8i, 7,
1230 MIN_MAX_COMPXCHG(float4, max, kmp_real32, 32, <, 4r, 3,
1232 MIN_MAX_COMPXCHG(float4, min, kmp_real32, 32, >, 4r, 3,
1234 MIN_MAX_COMPXCHG(float8, max, kmp_real64, 64, <, 8r, 7,
1236 MIN_MAX_COMPXCHG(float8, min, kmp_real64, 64, >, 8r, 7,
1238 MIN_MAX_CRITICAL(float10, max,
long double, <, 10r,
1240 MIN_MAX_CRITICAL(float10, min,
long double, >, 10r,
1243 MIN_MAX_CRITICAL(float16, max, QUAD_LEGACY, <, 16r,
1245 MIN_MAX_CRITICAL(float16, min, QUAD_LEGACY, >, 16r,
1248 MIN_MAX_CRITICAL(float16, max_a16, Quad_a16_t, <, 16r,
1250 MIN_MAX_CRITICAL(float16, min_a16, Quad_a16_t, >, 16r,
1257 #define ATOMIC_CRIT_EQV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1258 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1259 OP_GOMP_CRITICAL(^= (TYPE) ~, GOMP_FLAG) \
1260 OP_CRITICAL(^= (TYPE) ~, LCK_ID) \
1264 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1267 #define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1269 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1270 OP_GOMP_CRITICAL(^= (TYPE) ~, GOMP_FLAG) \
1271 OP_CMPXCHG(TYPE, BITS, OP) \
1277 #define ATOMIC_CMPX_EQV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, MASK, \
1279 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1280 OP_GOMP_CRITICAL(^= (TYPE) ~, GOMP_FLAG) \
1281 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1282 OP_CMPXCHG(TYPE, BITS, OP) \
1285 OP_CRITICAL(^= (TYPE) ~, LCK_ID) \
1290 ATOMIC_CMPXCHG(fixed1, neqv, kmp_int8, 8, ^, 1i, 0,
1292 ATOMIC_CMPXCHG(fixed2, neqv, kmp_int16, 16, ^, 2i, 1,
1294 ATOMIC_CMPXCHG(fixed4, neqv, kmp_int32, 32, ^, 4i, 3,
1296 ATOMIC_CMPXCHG(fixed8, neqv, kmp_int64, 64, ^, 8i, 7,
1298 ATOMIC_CMPX_EQV(fixed1, eqv, kmp_int8, 8, ^~, 1i, 0,
1300 ATOMIC_CMPX_EQV(fixed2, eqv, kmp_int16, 16, ^~, 2i, 1,
1302 ATOMIC_CMPX_EQV(fixed4, eqv, kmp_int32, 32, ^~, 4i, 3,
1304 ATOMIC_CMPX_EQV(fixed8, eqv, kmp_int64, 64, ^~, 8i, 7,
1313 #define ATOMIC_CRITICAL(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1314 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
1315 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
1316 OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) \
1321 ATOMIC_CRITICAL(float10, add,
long double, +, 10r,
1323 ATOMIC_CRITICAL(float10, sub,
long double, -, 10r,
1325 ATOMIC_CRITICAL(float10, mul,
long double, *, 10r,
1327 ATOMIC_CRITICAL(float10, div,
long double, /, 10r,
1331 ATOMIC_CRITICAL(float16, add, QUAD_LEGACY, +, 16r,
1333 ATOMIC_CRITICAL(float16, sub, QUAD_LEGACY, -, 16r,
1335 ATOMIC_CRITICAL(float16, mul, QUAD_LEGACY, *, 16r,
1337 ATOMIC_CRITICAL(float16, div, QUAD_LEGACY, /, 16r,
1340 ATOMIC_CRITICAL(float16, add_a16, Quad_a16_t, +, 16r,
1342 ATOMIC_CRITICAL(float16, sub_a16, Quad_a16_t, -, 16r,
1344 ATOMIC_CRITICAL(float16, mul_a16, Quad_a16_t, *, 16r,
1346 ATOMIC_CRITICAL(float16, div_a16, Quad_a16_t, /, 16r,
1354 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, add, kmp_cmplx32, 64, +, 8c, 7,
1356 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, sub, kmp_cmplx32, 64, -, 8c, 7,
1358 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, mul, kmp_cmplx32, 64, *, 8c, 7,
1360 ATOMIC_CMPXCHG_WORKAROUND(cmplx4, div, kmp_cmplx32, 64, /, 8c, 7,
1364 ATOMIC_CRITICAL(cmplx4, add, kmp_cmplx32, +, 8c, 1)
1365 ATOMIC_CRITICAL(cmplx4, sub, kmp_cmplx32, -, 8c, 1)
1366 ATOMIC_CRITICAL(cmplx4, mul, kmp_cmplx32, *, 8c, 1)
1367 ATOMIC_CRITICAL(cmplx4, div, kmp_cmplx32, /, 8c, 1)
1370 ATOMIC_CRITICAL(cmplx8, add, kmp_cmplx64, +, 16c, 1)
1371 ATOMIC_CRITICAL(cmplx8, sub, kmp_cmplx64, -, 16c, 1)
1372 ATOMIC_CRITICAL(cmplx8, mul, kmp_cmplx64, *, 16c, 1)
1373 ATOMIC_CRITICAL(cmplx8, div, kmp_cmplx64, /, 16c, 1)
1374 ATOMIC_CRITICAL(cmplx10, add, kmp_cmplx80, +, 20c,
1376 ATOMIC_CRITICAL(cmplx10, sub, kmp_cmplx80, -, 20c,
1378 ATOMIC_CRITICAL(cmplx10, mul, kmp_cmplx80, *, 20c,
1380 ATOMIC_CRITICAL(cmplx10, div, kmp_cmplx80, /, 20c,
1383 ATOMIC_CRITICAL(cmplx16, add, CPLX128_LEG, +, 32c,
1385 ATOMIC_CRITICAL(cmplx16, sub, CPLX128_LEG, -, 32c,
1387 ATOMIC_CRITICAL(cmplx16, mul, CPLX128_LEG, *, 32c,
1389 ATOMIC_CRITICAL(cmplx16, div, CPLX128_LEG, /, 32c,
1392 ATOMIC_CRITICAL(cmplx16, add_a16, kmp_cmplx128_a16_t, +, 32c,
1394 ATOMIC_CRITICAL(cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c,
1396 ATOMIC_CRITICAL(cmplx16, mul_a16, kmp_cmplx128_a16_t, *, 32c,
1398 ATOMIC_CRITICAL(cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c,
1405 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1413 #define OP_CRITICAL_REV(TYPE, OP, LCK_ID) \
1414 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
1416 (*lhs) = (TYPE)((rhs)OP(*lhs)); \
1418 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
1420 #ifdef KMP_GOMP_COMPAT
1421 #define OP_GOMP_CRITICAL_REV(TYPE, OP, FLAG) \
1422 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
1424 OP_CRITICAL_REV(TYPE, OP, 0); \
1429 #define OP_GOMP_CRITICAL_REV(TYPE, OP, FLAG)
1437 #define ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
1438 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_rev(ident_t *id_ref, int gtid, \
1439 TYPE *lhs, TYPE rhs) { \
1440 KMP_DEBUG_ASSERT(__kmp_init_serial); \
1441 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_rev: T#%d\n", gtid));
1450 #define OP_CMPXCHG_REV(TYPE, BITS, OP) \
1452 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1453 TYPE old_value, new_value; \
1455 old_value = temp_val; \
1456 new_value = (TYPE)(rhs OP old_value); \
1457 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
1458 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
1459 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
1463 old_value = temp_val; \
1464 new_value = (TYPE)(rhs OP old_value); \
1469 #define ATOMIC_CMPXCHG_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, LCK_ID, GOMP_FLAG) \
1470 ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \
1471 OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \
1472 OP_CMPXCHG_REV(TYPE, BITS, OP) \
1489 ATOMIC_CMPXCHG_REV(fixed1, div, kmp_int8, 8, /, 1i,
1491 ATOMIC_CMPXCHG_REV(fixed1u, div, kmp_uint8, 8, /, 1i,
1493 ATOMIC_CMPXCHG_REV(fixed1, shl, kmp_int8, 8, <<, 1i,
1495 ATOMIC_CMPXCHG_REV(fixed1, shr, kmp_int8, 8, >>, 1i,
1497 ATOMIC_CMPXCHG_REV(fixed1u, shr, kmp_uint8, 8, >>, 1i,
1499 ATOMIC_CMPXCHG_REV(fixed1, sub, kmp_int8, 8, -, 1i,
1502 ATOMIC_CMPXCHG_REV(fixed2, div, kmp_int16, 16, /, 2i,
1504 ATOMIC_CMPXCHG_REV(fixed2u, div, kmp_uint16, 16, /, 2i,
1506 ATOMIC_CMPXCHG_REV(fixed2, shl, kmp_int16, 16, <<, 2i,
1508 ATOMIC_CMPXCHG_REV(fixed2, shr, kmp_int16, 16, >>, 2i,
1510 ATOMIC_CMPXCHG_REV(fixed2u, shr, kmp_uint16, 16, >>, 2i,
1512 ATOMIC_CMPXCHG_REV(fixed2, sub, kmp_int16, 16, -, 2i,
1515 ATOMIC_CMPXCHG_REV(fixed4, div, kmp_int32, 32, /, 4i,
1517 ATOMIC_CMPXCHG_REV(fixed4u, div, kmp_uint32, 32, /, 4i,
1519 ATOMIC_CMPXCHG_REV(fixed4, shl, kmp_int32, 32, <<, 4i,
1521 ATOMIC_CMPXCHG_REV(fixed4, shr, kmp_int32, 32, >>, 4i,
1523 ATOMIC_CMPXCHG_REV(fixed4u, shr, kmp_uint32, 32, >>, 4i,
1525 ATOMIC_CMPXCHG_REV(fixed4, sub, kmp_int32, 32, -, 4i,
1528 ATOMIC_CMPXCHG_REV(fixed8, div, kmp_int64, 64, /, 8i,
1530 ATOMIC_CMPXCHG_REV(fixed8u, div, kmp_uint64, 64, /, 8i,
1532 ATOMIC_CMPXCHG_REV(fixed8, shl, kmp_int64, 64, <<, 8i,
1534 ATOMIC_CMPXCHG_REV(fixed8, shr, kmp_int64, 64, >>, 8i,
1536 ATOMIC_CMPXCHG_REV(fixed8u, shr, kmp_uint64, 64, >>, 8i,
1538 ATOMIC_CMPXCHG_REV(fixed8, sub, kmp_int64, 64, -, 8i,
1541 ATOMIC_CMPXCHG_REV(float4, div, kmp_real32, 32, /, 4r,
1543 ATOMIC_CMPXCHG_REV(float4, sub, kmp_real32, 32, -, 4r,
1546 ATOMIC_CMPXCHG_REV(float8, div, kmp_real64, 64, /, 8r,
1548 ATOMIC_CMPXCHG_REV(float8, sub, kmp_real64, 64, -, 8r,
1558 #define ATOMIC_CRITICAL_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
1559 ATOMIC_BEGIN_REV(TYPE_ID, OP_ID, TYPE, void) \
1560 OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \
1561 OP_CRITICAL_REV(TYPE, OP, LCK_ID) \
1566 ATOMIC_CRITICAL_REV(float10, sub,
long double, -, 10r,
1568 ATOMIC_CRITICAL_REV(float10, div,
long double, /, 10r,
1572 ATOMIC_CRITICAL_REV(float16, sub, QUAD_LEGACY, -, 16r,
1574 ATOMIC_CRITICAL_REV(float16, div, QUAD_LEGACY, /, 16r,
1577 ATOMIC_CRITICAL_REV(float16, sub_a16, Quad_a16_t, -, 16r,
1579 ATOMIC_CRITICAL_REV(float16, div_a16, Quad_a16_t, /, 16r,
1585 ATOMIC_CRITICAL_REV(cmplx4, sub, kmp_cmplx32, -, 8c,
1587 ATOMIC_CRITICAL_REV(cmplx4, div, kmp_cmplx32, /, 8c,
1589 ATOMIC_CRITICAL_REV(cmplx8, sub, kmp_cmplx64, -, 16c,
1591 ATOMIC_CRITICAL_REV(cmplx8, div, kmp_cmplx64, /, 16c,
1593 ATOMIC_CRITICAL_REV(cmplx10, sub, kmp_cmplx80, -, 20c,
1595 ATOMIC_CRITICAL_REV(cmplx10, div, kmp_cmplx80, /, 20c,
1598 ATOMIC_CRITICAL_REV(cmplx16, sub, CPLX128_LEG, -, 32c,
1600 ATOMIC_CRITICAL_REV(cmplx16, div, CPLX128_LEG, /, 32c,
1603 ATOMIC_CRITICAL_REV(cmplx16, sub_a16, kmp_cmplx128_a16_t, -, 32c,
1605 ATOMIC_CRITICAL_REV(cmplx16, div_a16, kmp_cmplx128_a16_t, /, 32c,
1624 #define ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1625 void __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( \
1626 ident_t *id_ref, int gtid, TYPE *lhs, RTYPE rhs) { \
1627 KMP_DEBUG_ASSERT(__kmp_init_serial); \
1629 ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \
1633 #define ATOMIC_CRITICAL_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, LCK_ID, \
1635 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1636 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
1637 OP_UPDATE_CRITICAL(TYPE, OP, LCK_ID) \
1641 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1644 #define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1645 LCK_ID, MASK, GOMP_FLAG) \
1646 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1647 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
1648 OP_CMPXCHG(TYPE, BITS, OP) \
1654 #define ATOMIC_CMPXCHG_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1655 LCK_ID, MASK, GOMP_FLAG) \
1656 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1657 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
1658 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1659 OP_CMPXCHG(TYPE, BITS, OP) \
1662 OP_UPDATE_CRITICAL(TYPE, OP, \
1669 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1671 #define ATOMIC_CMPXCHG_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \
1672 RTYPE, LCK_ID, MASK, GOMP_FLAG) \
1673 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1674 OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \
1675 OP_CMPXCHG_REV(TYPE, BITS, OP) \
1677 #define ATOMIC_CRITICAL_REV_FP(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \
1678 LCK_ID, GOMP_FLAG) \
1679 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1680 OP_GOMP_CRITICAL_REV(TYPE, OP, GOMP_FLAG) \
1681 OP_CRITICAL_REV(TYPE, OP, LCK_ID) \
1686 ATOMIC_CMPXCHG_MIX(fixed1,
char, mul, 8, *, float8, kmp_real64, 1i, 0,
1688 ATOMIC_CMPXCHG_MIX(fixed1,
char, div, 8, /, float8, kmp_real64, 1i, 0,
1690 ATOMIC_CMPXCHG_MIX(fixed2,
short, mul, 16, *, float8, kmp_real64, 2i, 1,
1692 ATOMIC_CMPXCHG_MIX(fixed2,
short, div, 16, /, float8, kmp_real64, 2i, 1,
1694 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, mul, 32, *, float8, kmp_real64, 4i, 3,
1696 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, div, 32, /, float8, kmp_real64, 4i, 3,
1698 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, mul, 64, *, float8, kmp_real64, 8i, 7,
1700 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, div, 64, /, float8, kmp_real64, 8i, 7,
1702 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, add, 32, +, float8, kmp_real64, 4r, 3,
1704 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, sub, 32, -, float8, kmp_real64, 4r, 3,
1706 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, mul, 32, *, float8, kmp_real64, 4r, 3,
1708 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, div, 32, /, float8, kmp_real64, 4r, 3,
1714 ATOMIC_CMPXCHG_MIX(fixed1,
char, add, 8, +, fp, _Quad, 1i, 0,
1716 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, add, 8, +, fp, _Quad, 1i, 0,
1718 ATOMIC_CMPXCHG_MIX(fixed1,
char, sub, 8, -, fp, _Quad, 1i, 0,
1720 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, sub, 8, -, fp, _Quad, 1i, 0,
1722 ATOMIC_CMPXCHG_MIX(fixed1,
char, mul, 8, *, fp, _Quad, 1i, 0,
1724 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, mul, 8, *, fp, _Quad, 1i, 0,
1726 ATOMIC_CMPXCHG_MIX(fixed1,
char, div, 8, /, fp, _Quad, 1i, 0,
1728 ATOMIC_CMPXCHG_MIX(fixed1u, uchar, div, 8, /, fp, _Quad, 1i, 0,
1731 ATOMIC_CMPXCHG_MIX(fixed2,
short, add, 16, +, fp, _Quad, 2i, 1,
1733 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, add, 16, +, fp, _Quad, 2i, 1,
1735 ATOMIC_CMPXCHG_MIX(fixed2,
short, sub, 16, -, fp, _Quad, 2i, 1,
1737 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, sub, 16, -, fp, _Quad, 2i, 1,
1739 ATOMIC_CMPXCHG_MIX(fixed2,
short, mul, 16, *, fp, _Quad, 2i, 1,
1741 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, mul, 16, *, fp, _Quad, 2i, 1,
1743 ATOMIC_CMPXCHG_MIX(fixed2,
short, div, 16, /, fp, _Quad, 2i, 1,
1745 ATOMIC_CMPXCHG_MIX(fixed2u, ushort, div, 16, /, fp, _Quad, 2i, 1,
1748 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, add, 32, +, fp, _Quad, 4i, 3,
1750 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, add, 32, +, fp, _Quad, 4i, 3,
1752 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, sub, 32, -, fp, _Quad, 4i, 3,
1754 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, sub, 32, -, fp, _Quad, 4i, 3,
1756 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, mul, 32, *, fp, _Quad, 4i, 3,
1758 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, mul, 32, *, fp, _Quad, 4i, 3,
1760 ATOMIC_CMPXCHG_MIX(fixed4, kmp_int32, div, 32, /, fp, _Quad, 4i, 3,
1762 ATOMIC_CMPXCHG_MIX(fixed4u, kmp_uint32, div, 32, /, fp, _Quad, 4i, 3,
1765 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, add, 64, +, fp, _Quad, 8i, 7,
1767 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, add, 64, +, fp, _Quad, 8i, 7,
1769 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, sub, 64, -, fp, _Quad, 8i, 7,
1771 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, sub, 64, -, fp, _Quad, 8i, 7,
1773 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, mul, 64, *, fp, _Quad, 8i, 7,
1775 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, mul, 64, *, fp, _Quad, 8i, 7,
1777 ATOMIC_CMPXCHG_MIX(fixed8, kmp_int64, div, 64, /, fp, _Quad, 8i, 7,
1779 ATOMIC_CMPXCHG_MIX(fixed8u, kmp_uint64, div, 64, /, fp, _Quad, 8i, 7,
1782 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, add, 32, +, fp, _Quad, 4r, 3,
1784 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, sub, 32, -, fp, _Quad, 4r, 3,
1786 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, mul, 32, *, fp, _Quad, 4r, 3,
1788 ATOMIC_CMPXCHG_MIX(float4, kmp_real32, div, 32, /, fp, _Quad, 4r, 3,
1791 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, add, 64, +, fp, _Quad, 8r, 7,
1793 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, sub, 64, -, fp, _Quad, 8r, 7,
1795 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, mul, 64, *, fp, _Quad, 8r, 7,
1797 ATOMIC_CMPXCHG_MIX(float8, kmp_real64, div, 64, /, fp, _Quad, 8r, 7,
1800 ATOMIC_CRITICAL_FP(float10,
long double, add, +, fp, _Quad, 10r,
1802 ATOMIC_CRITICAL_FP(float10,
long double, sub, -, fp, _Quad, 10r,
1804 ATOMIC_CRITICAL_FP(float10,
long double, mul, *, fp, _Quad, 10r,
1806 ATOMIC_CRITICAL_FP(float10,
long double, div, /, fp, _Quad, 10r,
1809 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1811 ATOMIC_CMPXCHG_REV_MIX(fixed1,
char, sub_rev, 8, -, fp, _Quad, 1i, 0,
1813 ATOMIC_CMPXCHG_REV_MIX(fixed1u, uchar, sub_rev, 8, -, fp, _Quad, 1i, 0,
1815 ATOMIC_CMPXCHG_REV_MIX(fixed1,
char, div_rev, 8, /, fp, _Quad, 1i, 0,
1817 ATOMIC_CMPXCHG_REV_MIX(fixed1u, uchar, div_rev, 8, /, fp, _Quad, 1i, 0,
1820 ATOMIC_CMPXCHG_REV_MIX(fixed2,
short, sub_rev, 16, -, fp, _Quad, 2i, 1,
1822 ATOMIC_CMPXCHG_REV_MIX(fixed2u, ushort, sub_rev, 16, -, fp, _Quad, 2i, 1,
1824 ATOMIC_CMPXCHG_REV_MIX(fixed2,
short, div_rev, 16, /, fp, _Quad, 2i, 1,
1826 ATOMIC_CMPXCHG_REV_MIX(fixed2u, ushort, div_rev, 16, /, fp, _Quad, 2i, 1,
1829 ATOMIC_CMPXCHG_REV_MIX(fixed4, kmp_int32, sub_rev, 32, -, fp, _Quad, 4i, 3,
1831 ATOMIC_CMPXCHG_REV_MIX(fixed4u, kmp_uint32, sub_rev, 32, -, fp, _Quad, 4i, 3,
1833 ATOMIC_CMPXCHG_REV_MIX(fixed4, kmp_int32, div_rev, 32, /, fp, _Quad, 4i, 3,
1835 ATOMIC_CMPXCHG_REV_MIX(fixed4u, kmp_uint32, div_rev, 32, /, fp, _Quad, 4i, 3,
1838 ATOMIC_CMPXCHG_REV_MIX(fixed8, kmp_int64, sub_rev, 64, -, fp, _Quad, 8i, 7,
1840 ATOMIC_CMPXCHG_REV_MIX(fixed8u, kmp_uint64, sub_rev, 64, -, fp, _Quad, 8i, 7,
1842 ATOMIC_CMPXCHG_REV_MIX(fixed8, kmp_int64, div_rev, 64, /, fp, _Quad, 8i, 7,
1844 ATOMIC_CMPXCHG_REV_MIX(fixed8u, kmp_uint64, div_rev, 64, /, fp, _Quad, 8i, 7,
1847 ATOMIC_CMPXCHG_REV_MIX(float4, kmp_real32, sub_rev, 32, -, fp, _Quad, 4r, 3,
1849 ATOMIC_CMPXCHG_REV_MIX(float4, kmp_real32, div_rev, 32, /, fp, _Quad, 4r, 3,
1852 ATOMIC_CMPXCHG_REV_MIX(float8, kmp_real64, sub_rev, 64, -, fp, _Quad, 8r, 7,
1854 ATOMIC_CMPXCHG_REV_MIX(float8, kmp_real64, div_rev, 64, /, fp, _Quad, 8r, 7,
1857 ATOMIC_CRITICAL_REV_FP(float10,
long double, sub_rev, -, fp, _Quad, 10r,
1859 ATOMIC_CRITICAL_REV_FP(float10,
long double, div_rev, /, fp, _Quad, 10r,
1865 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1870 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1871 LCK_ID, MASK, GOMP_FLAG) \
1872 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1873 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
1874 OP_CMPXCHG_WORKAROUND(TYPE, BITS, OP) \
1878 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1879 LCK_ID, MASK, GOMP_FLAG) \
1880 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1881 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
1882 OP_CMPXCHG(TYPE, BITS, OP) \
1888 #define ATOMIC_CMPXCHG_CMPLX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, RTYPE, \
1889 LCK_ID, MASK, GOMP_FLAG) \
1890 ATOMIC_BEGIN_MIX(TYPE_ID, TYPE, OP_ID, RTYPE_ID, RTYPE) \
1891 OP_UPDATE_GOMP_CRITICAL(TYPE, OP, GOMP_FLAG) \
1892 if (!((kmp_uintptr_t)lhs & 0x##MASK)) { \
1893 OP_CMPXCHG(TYPE, BITS, OP) \
1896 OP_UPDATE_CRITICAL(TYPE, OP, \
1902 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, add, 64, +, cmplx8, kmp_cmplx64, 8c,
1904 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, sub, 64, -, cmplx8, kmp_cmplx64, 8c,
1906 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, mul, 64, *, cmplx8, kmp_cmplx64, 8c,
1908 ATOMIC_CMPXCHG_CMPLX(cmplx4, kmp_cmplx32, div, 64, /, cmplx8, kmp_cmplx64, 8c,
1912 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
1923 #define ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
1924 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \
1926 KMP_DEBUG_ASSERT(__kmp_init_serial); \
1927 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
1938 #define OP_CMPXCHG_READ(TYPE, BITS, OP) \
1940 TYPE KMP_ATOMIC_VOLATILE temp_val; \
1943 kmp_int##BITS i_val; \
1945 union f_i_union old_value; \
1947 old_value.f_val = temp_val; \
1948 old_value.i_val = KMP_COMPARE_AND_STORE_RET##BITS( \
1949 (kmp_int##BITS *)loc, \
1950 *VOLATILE_CAST(kmp_int##BITS *) & old_value.i_val, \
1951 *VOLATILE_CAST(kmp_int##BITS *) & old_value.i_val); \
1952 new_value = old_value.f_val; \
1962 #define OP_CRITICAL_READ(OP, LCK_ID) \
1963 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
1965 new_value = (*loc); \
1967 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
1970 #ifdef KMP_GOMP_COMPAT
1971 #define OP_GOMP_CRITICAL_READ(OP, FLAG) \
1972 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
1974 OP_CRITICAL_READ(OP, 0); \
1978 #define OP_GOMP_CRITICAL_READ(OP, FLAG)
1982 #define ATOMIC_FIXED_READ(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
1983 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \
1985 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \
1986 new_value = KMP_TEST_THEN_ADD##BITS(loc, OP 0); \
1990 #define ATOMIC_CMPXCHG_READ(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
1991 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \
1993 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \
1994 OP_CMPXCHG_READ(TYPE, BITS, OP) \
2002 #define ATOMIC_CRITICAL_READ(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2003 ATOMIC_BEGIN_READ(TYPE_ID, OP_ID, TYPE, TYPE) \
2005 OP_GOMP_CRITICAL_READ(OP## =, GOMP_FLAG) \
2006 OP_CRITICAL_READ(OP, LCK_ID) \
2014 #if (KMP_OS_WINDOWS)
2016 #define OP_CRITICAL_READ_WRK(OP, LCK_ID) \
2017 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2021 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
2023 #ifdef KMP_GOMP_COMPAT
2024 #define OP_GOMP_CRITICAL_READ_WRK(OP, FLAG) \
2025 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2027 OP_CRITICAL_READ_WRK(OP, 0); \
2030 #define OP_GOMP_CRITICAL_READ_WRK(OP, FLAG)
2033 #define ATOMIC_BEGIN_READ_WRK(TYPE_ID, OP_ID, TYPE) \
2034 void __kmpc_atomic_##TYPE_ID##_##OP_ID(TYPE *out, ident_t *id_ref, int gtid, \
2036 KMP_DEBUG_ASSERT(__kmp_init_serial); \
2037 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
2040 #define ATOMIC_CRITICAL_READ_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2041 ATOMIC_BEGIN_READ_WRK(TYPE_ID, OP_ID, TYPE) \
2042 OP_GOMP_CRITICAL_READ_WRK(OP## =, GOMP_FLAG) \
2043 OP_CRITICAL_READ_WRK(OP, LCK_ID) \
2050 ATOMIC_FIXED_READ(fixed4, rd, kmp_int32, 32, +, 0)
2051 ATOMIC_FIXED_READ(fixed8, rd, kmp_int64, 64, +,
2053 ATOMIC_CMPXCHG_READ(float4, rd, kmp_real32, 32, +,
2055 ATOMIC_CMPXCHG_READ(float8, rd, kmp_real64, 64, +,
2059 ATOMIC_CMPXCHG_READ(fixed1, rd, kmp_int8, 8, +,
2061 ATOMIC_CMPXCHG_READ(fixed2, rd, kmp_int16, 16, +,
2064 ATOMIC_CRITICAL_READ(float10, rd,
long double, +, 10r,
2067 ATOMIC_CRITICAL_READ(float16, rd, QUAD_LEGACY, +, 16r,
2072 #if (KMP_OS_WINDOWS)
2073 ATOMIC_CRITICAL_READ_WRK(cmplx4, rd, kmp_cmplx32, +, 8c,
2076 ATOMIC_CRITICAL_READ(cmplx4, rd, kmp_cmplx32, +, 8c,
2079 ATOMIC_CRITICAL_READ(cmplx8, rd, kmp_cmplx64, +, 16c,
2081 ATOMIC_CRITICAL_READ(cmplx10, rd, kmp_cmplx80, +, 20c,
2084 ATOMIC_CRITICAL_READ(cmplx16, rd, CPLX128_LEG, +, 32c,
2087 ATOMIC_CRITICAL_READ(float16, a16_rd, Quad_a16_t, +, 16r,
2089 ATOMIC_CRITICAL_READ(cmplx16, a16_rd, kmp_cmplx128_a16_t, +, 32c,
2097 #define ATOMIC_XCHG_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2098 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
2099 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \
2100 KMP_XCHG_FIXED##BITS(lhs, rhs); \
2103 #define ATOMIC_XCHG_FLOAT_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2104 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
2105 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \
2106 KMP_XCHG_REAL##BITS(lhs, rhs); \
2116 #define OP_CMPXCHG_WR(TYPE, BITS, OP) \
2118 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2119 TYPE old_value, new_value; \
2121 old_value = temp_val; \
2123 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
2124 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
2125 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
2127 old_value = temp_val; \
2133 #define ATOMIC_CMPXCHG_WR(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2134 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
2135 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \
2136 OP_CMPXCHG_WR(TYPE, BITS, OP) \
2145 #define ATOMIC_CRITICAL_WR(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2146 ATOMIC_BEGIN(TYPE_ID, OP_ID, TYPE, void) \
2147 OP_GOMP_CRITICAL(OP, GOMP_FLAG) \
2148 OP_CRITICAL(OP, LCK_ID) \
2152 ATOMIC_XCHG_WR(fixed1, wr, kmp_int8, 8, =,
2154 ATOMIC_XCHG_WR(fixed2, wr, kmp_int16, 16, =,
2156 ATOMIC_XCHG_WR(fixed4, wr, kmp_int32, 32, =,
2159 ATOMIC_CMPXCHG_WR(fixed8, wr, kmp_int64, 64, =,
2162 ATOMIC_XCHG_WR(fixed8, wr, kmp_int64, 64, =,
2166 ATOMIC_XCHG_FLOAT_WR(float4, wr, kmp_real32, 32, =,
2169 ATOMIC_CMPXCHG_WR(float8, wr, kmp_real64, 64, =,
2172 ATOMIC_XCHG_FLOAT_WR(float8, wr, kmp_real64, 64, =,
2176 ATOMIC_CRITICAL_WR(float10, wr,
long double, =, 10r,
2179 ATOMIC_CRITICAL_WR(float16, wr, QUAD_LEGACY, =, 16r,
2182 ATOMIC_CRITICAL_WR(cmplx4, wr, kmp_cmplx32, =, 8c, 1)
2183 ATOMIC_CRITICAL_WR(cmplx8, wr, kmp_cmplx64, =, 16c,
2185 ATOMIC_CRITICAL_WR(cmplx10, wr, kmp_cmplx80, =, 20c,
2188 ATOMIC_CRITICAL_WR(cmplx16, wr, CPLX128_LEG, =, 32c,
2191 ATOMIC_CRITICAL_WR(float16, a16_wr, Quad_a16_t, =, 16r,
2193 ATOMIC_CRITICAL_WR(cmplx16, a16_wr, kmp_cmplx128_a16_t, =, 32c,
2206 #define ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, RET_TYPE) \
2207 RET_TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, \
2208 TYPE *lhs, TYPE rhs, int flag) { \
2209 KMP_DEBUG_ASSERT(__kmp_init_serial); \
2210 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
2218 #define OP_CRITICAL_CPT(OP, LCK_ID) \
2219 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2223 new_value = (*lhs); \
2225 new_value = (*lhs); \
2229 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2232 #define OP_UPDATE_CRITICAL_CPT(TYPE, OP, LCK_ID) \
2233 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2236 (*lhs) = (TYPE)((*lhs)OP rhs); \
2237 new_value = (*lhs); \
2239 new_value = (*lhs); \
2240 (*lhs) = (TYPE)((*lhs)OP rhs); \
2243 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2247 #ifdef KMP_GOMP_COMPAT
2248 #define OP_GOMP_CRITICAL_CPT(TYPE, OP, FLAG) \
2249 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2251 OP_UPDATE_CRITICAL_CPT(TYPE, OP, 0); \
2254 #define OP_GOMP_CRITICAL_CPT(TYPE, OP, FLAG)
2264 #define OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2266 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2267 TYPE old_value, new_value; \
2269 old_value = temp_val; \
2270 new_value = (TYPE)(old_value OP rhs); \
2271 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
2272 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
2273 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
2275 old_value = temp_val; \
2276 new_value = (TYPE)(old_value OP rhs); \
2285 #define ATOMIC_CMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2286 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2289 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \
2290 OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2294 #define ATOMIC_FIXED_ADD_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2295 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2296 TYPE old_value, new_value; \
2298 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \
2300 old_value = KMP_TEST_THEN_ADD##BITS(lhs, OP rhs); \
2302 return old_value OP rhs; \
2308 ATOMIC_FIXED_ADD_CPT(fixed4, add_cpt, kmp_int32, 32, +,
2310 ATOMIC_FIXED_ADD_CPT(fixed4, sub_cpt, kmp_int32, 32, -,
2312 ATOMIC_FIXED_ADD_CPT(fixed8, add_cpt, kmp_int64, 64, +,
2314 ATOMIC_FIXED_ADD_CPT(fixed8, sub_cpt, kmp_int64, 64, -,
2317 ATOMIC_CMPXCHG_CPT(float4, add_cpt, kmp_real32, 32, +,
2319 ATOMIC_CMPXCHG_CPT(float4, sub_cpt, kmp_real32, 32, -,
2321 ATOMIC_CMPXCHG_CPT(float8, add_cpt, kmp_real64, 64, +,
2323 ATOMIC_CMPXCHG_CPT(float8, sub_cpt, kmp_real64, 64, -,
2338 ATOMIC_CMPXCHG_CPT(fixed1, add_cpt, kmp_int8, 8, +,
2340 ATOMIC_CMPXCHG_CPT(fixed1, andb_cpt, kmp_int8, 8, &,
2342 ATOMIC_CMPXCHG_CPT(fixed1, div_cpt, kmp_int8, 8, /,
2344 ATOMIC_CMPXCHG_CPT(fixed1u, div_cpt, kmp_uint8, 8, /,
2346 ATOMIC_CMPXCHG_CPT(fixed1, mul_cpt, kmp_int8, 8, *,
2348 ATOMIC_CMPXCHG_CPT(fixed1, orb_cpt, kmp_int8, 8, |,
2350 ATOMIC_CMPXCHG_CPT(fixed1, shl_cpt, kmp_int8, 8, <<,
2352 ATOMIC_CMPXCHG_CPT(fixed1, shr_cpt, kmp_int8, 8, >>,
2354 ATOMIC_CMPXCHG_CPT(fixed1u, shr_cpt, kmp_uint8, 8, >>,
2356 ATOMIC_CMPXCHG_CPT(fixed1, sub_cpt, kmp_int8, 8, -,
2358 ATOMIC_CMPXCHG_CPT(fixed1, xor_cpt, kmp_int8, 8, ^,
2360 ATOMIC_CMPXCHG_CPT(fixed2, add_cpt, kmp_int16, 16, +,
2362 ATOMIC_CMPXCHG_CPT(fixed2, andb_cpt, kmp_int16, 16, &,
2364 ATOMIC_CMPXCHG_CPT(fixed2, div_cpt, kmp_int16, 16, /,
2366 ATOMIC_CMPXCHG_CPT(fixed2u, div_cpt, kmp_uint16, 16, /,
2368 ATOMIC_CMPXCHG_CPT(fixed2, mul_cpt, kmp_int16, 16, *,
2370 ATOMIC_CMPXCHG_CPT(fixed2, orb_cpt, kmp_int16, 16, |,
2372 ATOMIC_CMPXCHG_CPT(fixed2, shl_cpt, kmp_int16, 16, <<,
2374 ATOMIC_CMPXCHG_CPT(fixed2, shr_cpt, kmp_int16, 16, >>,
2376 ATOMIC_CMPXCHG_CPT(fixed2u, shr_cpt, kmp_uint16, 16, >>,
2378 ATOMIC_CMPXCHG_CPT(fixed2, sub_cpt, kmp_int16, 16, -,
2380 ATOMIC_CMPXCHG_CPT(fixed2, xor_cpt, kmp_int16, 16, ^,
2382 ATOMIC_CMPXCHG_CPT(fixed4, andb_cpt, kmp_int32, 32, &,
2384 ATOMIC_CMPXCHG_CPT(fixed4, div_cpt, kmp_int32, 32, /,
2386 ATOMIC_CMPXCHG_CPT(fixed4u, div_cpt, kmp_uint32, 32, /,
2388 ATOMIC_CMPXCHG_CPT(fixed4, mul_cpt, kmp_int32, 32, *,
2390 ATOMIC_CMPXCHG_CPT(fixed4, orb_cpt, kmp_int32, 32, |,
2392 ATOMIC_CMPXCHG_CPT(fixed4, shl_cpt, kmp_int32, 32, <<,
2394 ATOMIC_CMPXCHG_CPT(fixed4, shr_cpt, kmp_int32, 32, >>,
2396 ATOMIC_CMPXCHG_CPT(fixed4u, shr_cpt, kmp_uint32, 32, >>,
2398 ATOMIC_CMPXCHG_CPT(fixed4, xor_cpt, kmp_int32, 32, ^,
2400 ATOMIC_CMPXCHG_CPT(fixed8, andb_cpt, kmp_int64, 64, &,
2402 ATOMIC_CMPXCHG_CPT(fixed8, div_cpt, kmp_int64, 64, /,
2404 ATOMIC_CMPXCHG_CPT(fixed8u, div_cpt, kmp_uint64, 64, /,
2406 ATOMIC_CMPXCHG_CPT(fixed8, mul_cpt, kmp_int64, 64, *,
2408 ATOMIC_CMPXCHG_CPT(fixed8, orb_cpt, kmp_int64, 64, |,
2410 ATOMIC_CMPXCHG_CPT(fixed8, shl_cpt, kmp_int64, 64, <<,
2412 ATOMIC_CMPXCHG_CPT(fixed8, shr_cpt, kmp_int64, 64, >>,
2414 ATOMIC_CMPXCHG_CPT(fixed8u, shr_cpt, kmp_uint64, 64, >>,
2416 ATOMIC_CMPXCHG_CPT(fixed8, xor_cpt, kmp_int64, 64, ^,
2418 ATOMIC_CMPXCHG_CPT(float4, div_cpt, kmp_real32, 32, /,
2420 ATOMIC_CMPXCHG_CPT(float4, mul_cpt, kmp_real32, 32, *,
2422 ATOMIC_CMPXCHG_CPT(float8, div_cpt, kmp_real64, 64, /,
2424 ATOMIC_CMPXCHG_CPT(float8, mul_cpt, kmp_real64, 64, *,
2436 #define ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
2437 TYPE __kmpc_atomic_##TYPE_ID##_##OP_ID##_##RTYPE_ID( \
2438 ident_t *id_ref, int gtid, TYPE *lhs, RTYPE rhs, int flag) { \
2439 KMP_DEBUG_ASSERT(__kmp_init_serial); \
2441 ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID "_" #RTYPE_ID ": T#%d\n", \
2445 #define ATOMIC_CMPXCHG_CPT_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \
2446 RTYPE, LCK_ID, MASK, GOMP_FLAG) \
2447 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
2449 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \
2450 OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2454 #define ATOMIC_CRITICAL_CPT_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \
2455 LCK_ID, GOMP_FLAG) \
2456 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
2458 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \
2459 OP_UPDATE_CRITICAL_CPT(TYPE, OP, LCK_ID) \
2462 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, add_cpt, 8, +, fp, _Quad, 1i, 0,
2464 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, add_cpt, 8, +, fp, _Quad, 1i, 0,
2466 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, sub_cpt, 8, -, fp, _Quad, 1i, 0,
2468 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, sub_cpt, 8, -, fp, _Quad, 1i, 0,
2470 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, mul_cpt, 8, *, fp, _Quad, 1i, 0,
2472 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, mul_cpt, 8, *, fp, _Quad, 1i, 0,
2474 ATOMIC_CMPXCHG_CPT_MIX(fixed1,
char, div_cpt, 8, /, fp, _Quad, 1i, 0,
2476 ATOMIC_CMPXCHG_CPT_MIX(fixed1u, uchar, div_cpt, 8, /, fp, _Quad, 1i, 0,
2479 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, add_cpt, 16, +, fp, _Quad, 2i, 1,
2481 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, add_cpt, 16, +, fp, _Quad, 2i, 1,
2483 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, sub_cpt, 16, -, fp, _Quad, 2i, 1,
2485 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, sub_cpt, 16, -, fp, _Quad, 2i, 1,
2487 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, mul_cpt, 16, *, fp, _Quad, 2i, 1,
2489 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, mul_cpt, 16, *, fp, _Quad, 2i, 1,
2491 ATOMIC_CMPXCHG_CPT_MIX(fixed2,
short, div_cpt, 16, /, fp, _Quad, 2i, 1,
2493 ATOMIC_CMPXCHG_CPT_MIX(fixed2u, ushort, div_cpt, 16, /, fp, _Quad, 2i, 1,
2496 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, add_cpt, 32, +, fp, _Quad, 4i, 3,
2498 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, add_cpt, 32, +, fp, _Quad, 4i, 3,
2500 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, sub_cpt, 32, -, fp, _Quad, 4i, 3,
2502 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, sub_cpt, 32, -, fp, _Quad, 4i, 3,
2504 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, mul_cpt, 32, *, fp, _Quad, 4i, 3,
2506 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, mul_cpt, 32, *, fp, _Quad, 4i, 3,
2508 ATOMIC_CMPXCHG_CPT_MIX(fixed4, kmp_int32, div_cpt, 32, /, fp, _Quad, 4i, 3,
2510 ATOMIC_CMPXCHG_CPT_MIX(fixed4u, kmp_uint32, div_cpt, 32, /, fp, _Quad, 4i, 3,
2513 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, add_cpt, 64, +, fp, _Quad, 8i, 7,
2515 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, add_cpt, 64, +, fp, _Quad, 8i, 7,
2517 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, sub_cpt, 64, -, fp, _Quad, 8i, 7,
2519 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, sub_cpt, 64, -, fp, _Quad, 8i, 7,
2521 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, mul_cpt, 64, *, fp, _Quad, 8i, 7,
2523 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, mul_cpt, 64, *, fp, _Quad, 8i, 7,
2525 ATOMIC_CMPXCHG_CPT_MIX(fixed8, kmp_int64, div_cpt, 64, /, fp, _Quad, 8i, 7,
2527 ATOMIC_CMPXCHG_CPT_MIX(fixed8u, kmp_uint64, div_cpt, 64, /, fp, _Quad, 8i, 7,
2530 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, add_cpt, 32, +, fp, _Quad, 4r, 3,
2532 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, sub_cpt, 32, -, fp, _Quad, 4r, 3,
2534 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, mul_cpt, 32, *, fp, _Quad, 4r, 3,
2536 ATOMIC_CMPXCHG_CPT_MIX(float4, kmp_real32, div_cpt, 32, /, fp, _Quad, 4r, 3,
2539 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, add_cpt, 64, +, fp, _Quad, 8r, 7,
2541 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, sub_cpt, 64, -, fp, _Quad, 8r, 7,
2543 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, mul_cpt, 64, *, fp, _Quad, 8r, 7,
2545 ATOMIC_CMPXCHG_CPT_MIX(float8, kmp_real64, div_cpt, 64, /, fp, _Quad, 8r, 7,
2548 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, add_cpt, +, fp, _Quad, 10r,
2550 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, sub_cpt, -, fp, _Quad, 10r,
2552 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, mul_cpt, *, fp, _Quad, 10r,
2554 ATOMIC_CRITICAL_CPT_MIX(float10,
long double, div_cpt, /, fp, _Quad, 10r,
2568 #define OP_CRITICAL_L_CPT(OP, LCK_ID) \
2569 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2573 (*lhs) = new_value; \
2575 new_value = (*lhs); \
2579 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid);
2582 #ifdef KMP_GOMP_COMPAT
2583 #define OP_GOMP_CRITICAL_L_CPT(OP, FLAG) \
2584 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2586 OP_CRITICAL_L_CPT(OP, 0); \
2590 #define OP_GOMP_CRITICAL_L_CPT(OP, FLAG)
2595 #define ATOMIC_CMPX_L_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2596 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2599 OP_GOMP_CRITICAL_L_CPT(= *lhs OP, GOMP_FLAG) \
2600 OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2603 ATOMIC_CMPX_L_CPT(fixed1, andl_cpt,
char, 8, &&,
2605 ATOMIC_CMPX_L_CPT(fixed1, orl_cpt,
char, 8, ||,
2607 ATOMIC_CMPX_L_CPT(fixed2, andl_cpt,
short, 16, &&,
2609 ATOMIC_CMPX_L_CPT(fixed2, orl_cpt,
short, 16, ||,
2611 ATOMIC_CMPX_L_CPT(fixed4, andl_cpt, kmp_int32, 32, &&,
2613 ATOMIC_CMPX_L_CPT(fixed4, orl_cpt, kmp_int32, 32, ||,
2615 ATOMIC_CMPX_L_CPT(fixed8, andl_cpt, kmp_int64, 64, &&,
2617 ATOMIC_CMPX_L_CPT(fixed8, orl_cpt, kmp_int64, 64, ||,
2629 #define MIN_MAX_CRITSECT_CPT(OP, LCK_ID) \
2630 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2632 if (*lhs OP rhs) { \
2638 new_value = old_value; \
2642 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2646 #ifdef KMP_GOMP_COMPAT
2647 #define GOMP_MIN_MAX_CRITSECT_CPT(OP, FLAG) \
2648 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2650 MIN_MAX_CRITSECT_CPT(OP, 0); \
2653 #define GOMP_MIN_MAX_CRITSECT_CPT(OP, FLAG)
2657 #define MIN_MAX_CMPXCHG_CPT(TYPE, BITS, OP) \
2659 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2662 old_value = temp_val; \
2663 while (old_value OP rhs && \
2664 !KMP_COMPARE_AND_STORE_ACQ##BITS( \
2665 (kmp_int##BITS *)lhs, \
2666 *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
2667 *VOLATILE_CAST(kmp_int##BITS *) & rhs)) { \
2669 old_value = temp_val; \
2679 #define MIN_MAX_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2680 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2681 TYPE new_value, old_value; \
2682 if (*lhs OP rhs) { \
2683 GOMP_MIN_MAX_CRITSECT_CPT(OP, GOMP_FLAG) \
2684 MIN_MAX_CRITSECT_CPT(OP, LCK_ID) \
2689 #define MIN_MAX_COMPXCHG_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2690 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2691 TYPE new_value, old_value; \
2693 if (*lhs OP rhs) { \
2694 GOMP_MIN_MAX_CRITSECT_CPT(OP, GOMP_FLAG) \
2695 MIN_MAX_CMPXCHG_CPT(TYPE, BITS, OP) \
2700 MIN_MAX_COMPXCHG_CPT(fixed1, max_cpt,
char, 8, <,
2702 MIN_MAX_COMPXCHG_CPT(fixed1, min_cpt,
char, 8, >,
2704 MIN_MAX_COMPXCHG_CPT(fixed2, max_cpt,
short, 16, <,
2706 MIN_MAX_COMPXCHG_CPT(fixed2, min_cpt,
short, 16, >,
2708 MIN_MAX_COMPXCHG_CPT(fixed4, max_cpt, kmp_int32, 32, <,
2710 MIN_MAX_COMPXCHG_CPT(fixed4, min_cpt, kmp_int32, 32, >,
2712 MIN_MAX_COMPXCHG_CPT(fixed8, max_cpt, kmp_int64, 64, <,
2714 MIN_MAX_COMPXCHG_CPT(fixed8, min_cpt, kmp_int64, 64, >,
2716 MIN_MAX_COMPXCHG_CPT(float4, max_cpt, kmp_real32, 32, <,
2718 MIN_MAX_COMPXCHG_CPT(float4, min_cpt, kmp_real32, 32, >,
2720 MIN_MAX_COMPXCHG_CPT(float8, max_cpt, kmp_real64, 64, <,
2722 MIN_MAX_COMPXCHG_CPT(float8, min_cpt, kmp_real64, 64, >,
2724 MIN_MAX_CRITICAL_CPT(float10, max_cpt,
long double, <, 10r,
2726 MIN_MAX_CRITICAL_CPT(float10, min_cpt,
long double, >, 10r,
2729 MIN_MAX_CRITICAL_CPT(float16, max_cpt, QUAD_LEGACY, <, 16r,
2731 MIN_MAX_CRITICAL_CPT(float16, min_cpt, QUAD_LEGACY, >, 16r,
2734 MIN_MAX_CRITICAL_CPT(float16, max_a16_cpt, Quad_a16_t, <, 16r,
2736 MIN_MAX_CRITICAL_CPT(float16, min_a16_cpt, Quad_a16_t, >, 16r,
2742 #ifdef KMP_GOMP_COMPAT
2743 #define OP_GOMP_CRITICAL_EQV_CPT(OP, FLAG) \
2744 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2746 OP_CRITICAL_CPT(OP, 0); \
2749 #define OP_GOMP_CRITICAL_EQV_CPT(OP, FLAG)
2752 #define ATOMIC_CMPX_EQV_CPT(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2753 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2756 OP_GOMP_CRITICAL_EQV_CPT(^= (TYPE) ~, GOMP_FLAG) \
2757 OP_CMPXCHG_CPT(TYPE, BITS, OP) \
2762 ATOMIC_CMPXCHG_CPT(fixed1, neqv_cpt, kmp_int8, 8, ^,
2764 ATOMIC_CMPXCHG_CPT(fixed2, neqv_cpt, kmp_int16, 16, ^,
2766 ATOMIC_CMPXCHG_CPT(fixed4, neqv_cpt, kmp_int32, 32, ^,
2768 ATOMIC_CMPXCHG_CPT(fixed8, neqv_cpt, kmp_int64, 64, ^,
2770 ATOMIC_CMPX_EQV_CPT(fixed1, eqv_cpt, kmp_int8, 8, ^~,
2772 ATOMIC_CMPX_EQV_CPT(fixed2, eqv_cpt, kmp_int16, 16, ^~,
2774 ATOMIC_CMPX_EQV_CPT(fixed4, eqv_cpt, kmp_int32, 32, ^~,
2776 ATOMIC_CMPX_EQV_CPT(fixed8, eqv_cpt, kmp_int64, 64, ^~,
2785 #define ATOMIC_CRITICAL_CPT(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2786 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2788 OP_GOMP_CRITICAL_CPT(TYPE, OP, GOMP_FLAG) \
2789 OP_UPDATE_CRITICAL_CPT(TYPE, OP, LCK_ID) \
2795 #define OP_CRITICAL_CPT_WRK(OP, LCK_ID) \
2796 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2806 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2810 #ifdef KMP_GOMP_COMPAT
2811 #define OP_GOMP_CRITICAL_CPT_WRK(OP, FLAG) \
2812 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2814 OP_CRITICAL_CPT_WRK(OP## =, 0); \
2817 #define OP_GOMP_CRITICAL_CPT_WRK(OP, FLAG)
2821 #define ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \
2822 void __kmpc_atomic_##TYPE_ID##_##OP_ID(ident_t *id_ref, int gtid, TYPE *lhs, \
2823 TYPE rhs, TYPE *out, int flag) { \
2824 KMP_DEBUG_ASSERT(__kmp_init_serial); \
2825 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_" #OP_ID ": T#%d\n", gtid));
2828 #define ATOMIC_CRITICAL_CPT_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
2829 ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \
2830 OP_GOMP_CRITICAL_CPT_WRK(OP, GOMP_FLAG) \
2831 OP_CRITICAL_CPT_WRK(OP## =, LCK_ID) \
2837 ATOMIC_CRITICAL_CPT(float10, add_cpt,
long double, +, 10r,
2839 ATOMIC_CRITICAL_CPT(float10, sub_cpt,
long double, -, 10r,
2841 ATOMIC_CRITICAL_CPT(float10, mul_cpt,
long double, *, 10r,
2843 ATOMIC_CRITICAL_CPT(float10, div_cpt,
long double, /, 10r,
2847 ATOMIC_CRITICAL_CPT(float16, add_cpt, QUAD_LEGACY, +, 16r,
2849 ATOMIC_CRITICAL_CPT(float16, sub_cpt, QUAD_LEGACY, -, 16r,
2851 ATOMIC_CRITICAL_CPT(float16, mul_cpt, QUAD_LEGACY, *, 16r,
2853 ATOMIC_CRITICAL_CPT(float16, div_cpt, QUAD_LEGACY, /, 16r,
2856 ATOMIC_CRITICAL_CPT(float16, add_a16_cpt, Quad_a16_t, +, 16r,
2858 ATOMIC_CRITICAL_CPT(float16, sub_a16_cpt, Quad_a16_t, -, 16r,
2860 ATOMIC_CRITICAL_CPT(float16, mul_a16_cpt, Quad_a16_t, *, 16r,
2862 ATOMIC_CRITICAL_CPT(float16, div_a16_cpt, Quad_a16_t, /, 16r,
2870 ATOMIC_CRITICAL_CPT_WRK(cmplx4, add_cpt, kmp_cmplx32, +, 8c,
2872 ATOMIC_CRITICAL_CPT_WRK(cmplx4, sub_cpt, kmp_cmplx32, -, 8c,
2874 ATOMIC_CRITICAL_CPT_WRK(cmplx4, mul_cpt, kmp_cmplx32, *, 8c,
2876 ATOMIC_CRITICAL_CPT_WRK(cmplx4, div_cpt, kmp_cmplx32, /, 8c,
2879 ATOMIC_CRITICAL_CPT(cmplx8, add_cpt, kmp_cmplx64, +, 16c,
2881 ATOMIC_CRITICAL_CPT(cmplx8, sub_cpt, kmp_cmplx64, -, 16c,
2883 ATOMIC_CRITICAL_CPT(cmplx8, mul_cpt, kmp_cmplx64, *, 16c,
2885 ATOMIC_CRITICAL_CPT(cmplx8, div_cpt, kmp_cmplx64, /, 16c,
2887 ATOMIC_CRITICAL_CPT(cmplx10, add_cpt, kmp_cmplx80, +, 20c,
2889 ATOMIC_CRITICAL_CPT(cmplx10, sub_cpt, kmp_cmplx80, -, 20c,
2891 ATOMIC_CRITICAL_CPT(cmplx10, mul_cpt, kmp_cmplx80, *, 20c,
2893 ATOMIC_CRITICAL_CPT(cmplx10, div_cpt, kmp_cmplx80, /, 20c,
2896 ATOMIC_CRITICAL_CPT(cmplx16, add_cpt, CPLX128_LEG, +, 32c,
2898 ATOMIC_CRITICAL_CPT(cmplx16, sub_cpt, CPLX128_LEG, -, 32c,
2900 ATOMIC_CRITICAL_CPT(cmplx16, mul_cpt, CPLX128_LEG, *, 32c,
2902 ATOMIC_CRITICAL_CPT(cmplx16, div_cpt, CPLX128_LEG, /, 32c,
2905 ATOMIC_CRITICAL_CPT(cmplx16, add_a16_cpt, kmp_cmplx128_a16_t, +, 32c,
2907 ATOMIC_CRITICAL_CPT(cmplx16, sub_a16_cpt, kmp_cmplx128_a16_t, -, 32c,
2909 ATOMIC_CRITICAL_CPT(cmplx16, mul_a16_cpt, kmp_cmplx128_a16_t, *, 32c,
2911 ATOMIC_CRITICAL_CPT(cmplx16, div_a16_cpt, kmp_cmplx128_a16_t, /, 32c,
2926 #define OP_CRITICAL_CPT_REV(TYPE, OP, LCK_ID) \
2927 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2931 (*lhs) = (TYPE)((rhs)OP(*lhs)); \
2932 new_value = (*lhs); \
2934 new_value = (*lhs); \
2935 (*lhs) = (TYPE)((rhs)OP(*lhs)); \
2937 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
2941 #ifdef KMP_GOMP_COMPAT
2942 #define OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, FLAG) \
2943 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
2945 OP_CRITICAL_CPT_REV(TYPE, OP, 0); \
2948 #define OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, FLAG)
2958 #define OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \
2960 TYPE KMP_ATOMIC_VOLATILE temp_val; \
2961 TYPE old_value, new_value; \
2963 old_value = temp_val; \
2964 new_value = (TYPE)(rhs OP old_value); \
2965 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
2966 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
2967 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
2969 old_value = temp_val; \
2970 new_value = (TYPE)(rhs OP old_value); \
2979 #define ATOMIC_CMPXCHG_CPT_REV(TYPE_ID, OP_ID, TYPE, BITS, OP, GOMP_FLAG) \
2980 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
2983 OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \
2984 OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \
2987 ATOMIC_CMPXCHG_CPT_REV(fixed1, div_cpt_rev, kmp_int8, 8, /,
2989 ATOMIC_CMPXCHG_CPT_REV(fixed1u, div_cpt_rev, kmp_uint8, 8, /,
2991 ATOMIC_CMPXCHG_CPT_REV(fixed1, shl_cpt_rev, kmp_int8, 8, <<,
2993 ATOMIC_CMPXCHG_CPT_REV(fixed1, shr_cpt_rev, kmp_int8, 8, >>,
2995 ATOMIC_CMPXCHG_CPT_REV(fixed1u, shr_cpt_rev, kmp_uint8, 8, >>,
2997 ATOMIC_CMPXCHG_CPT_REV(fixed1, sub_cpt_rev, kmp_int8, 8, -,
2999 ATOMIC_CMPXCHG_CPT_REV(fixed2, div_cpt_rev, kmp_int16, 16, /,
3001 ATOMIC_CMPXCHG_CPT_REV(fixed2u, div_cpt_rev, kmp_uint16, 16, /,
3003 ATOMIC_CMPXCHG_CPT_REV(fixed2, shl_cpt_rev, kmp_int16, 16, <<,
3005 ATOMIC_CMPXCHG_CPT_REV(fixed2, shr_cpt_rev, kmp_int16, 16, >>,
3007 ATOMIC_CMPXCHG_CPT_REV(fixed2u, shr_cpt_rev, kmp_uint16, 16, >>,
3009 ATOMIC_CMPXCHG_CPT_REV(fixed2, sub_cpt_rev, kmp_int16, 16, -,
3011 ATOMIC_CMPXCHG_CPT_REV(fixed4, div_cpt_rev, kmp_int32, 32, /,
3013 ATOMIC_CMPXCHG_CPT_REV(fixed4u, div_cpt_rev, kmp_uint32, 32, /,
3015 ATOMIC_CMPXCHG_CPT_REV(fixed4, shl_cpt_rev, kmp_int32, 32, <<,
3017 ATOMIC_CMPXCHG_CPT_REV(fixed4, shr_cpt_rev, kmp_int32, 32, >>,
3019 ATOMIC_CMPXCHG_CPT_REV(fixed4u, shr_cpt_rev, kmp_uint32, 32, >>,
3021 ATOMIC_CMPXCHG_CPT_REV(fixed4, sub_cpt_rev, kmp_int32, 32, -,
3023 ATOMIC_CMPXCHG_CPT_REV(fixed8, div_cpt_rev, kmp_int64, 64, /,
3025 ATOMIC_CMPXCHG_CPT_REV(fixed8u, div_cpt_rev, kmp_uint64, 64, /,
3027 ATOMIC_CMPXCHG_CPT_REV(fixed8, shl_cpt_rev, kmp_int64, 64, <<,
3029 ATOMIC_CMPXCHG_CPT_REV(fixed8, shr_cpt_rev, kmp_int64, 64, >>,
3031 ATOMIC_CMPXCHG_CPT_REV(fixed8u, shr_cpt_rev, kmp_uint64, 64, >>,
3033 ATOMIC_CMPXCHG_CPT_REV(fixed8, sub_cpt_rev, kmp_int64, 64, -,
3035 ATOMIC_CMPXCHG_CPT_REV(float4, div_cpt_rev, kmp_real32, 32, /,
3037 ATOMIC_CMPXCHG_CPT_REV(float4, sub_cpt_rev, kmp_real32, 32, -,
3039 ATOMIC_CMPXCHG_CPT_REV(float8, div_cpt_rev, kmp_real64, 64, /,
3041 ATOMIC_CMPXCHG_CPT_REV(float8, sub_cpt_rev, kmp_real64, 64, -,
3051 #define ATOMIC_CRITICAL_CPT_REV(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, GOMP_FLAG) \
3052 ATOMIC_BEGIN_CPT(TYPE_ID, OP_ID, TYPE, TYPE) \
3055 OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \
3056 OP_CRITICAL_CPT_REV(TYPE, OP, LCK_ID) \
3061 ATOMIC_CRITICAL_CPT_REV(float10, sub_cpt_rev,
long double, -, 10r,
3063 ATOMIC_CRITICAL_CPT_REV(float10, div_cpt_rev,
long double, /, 10r,
3067 ATOMIC_CRITICAL_CPT_REV(float16, sub_cpt_rev, QUAD_LEGACY, -, 16r,
3069 ATOMIC_CRITICAL_CPT_REV(float16, div_cpt_rev, QUAD_LEGACY, /, 16r,
3072 ATOMIC_CRITICAL_CPT_REV(float16, sub_a16_cpt_rev, Quad_a16_t, -, 16r,
3074 ATOMIC_CRITICAL_CPT_REV(float16, div_a16_cpt_rev, Quad_a16_t, /, 16r,
3084 #define OP_CRITICAL_CPT_REV_WRK(OP, LCK_ID) \
3085 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3088 (*lhs) = (rhs)OP(*lhs); \
3092 (*lhs) = (rhs)OP(*lhs); \
3095 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3099 #ifdef KMP_GOMP_COMPAT
3100 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP, FLAG) \
3101 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
3103 OP_CRITICAL_CPT_REV_WRK(OP, 0); \
3106 #define OP_GOMP_CRITICAL_CPT_REV_WRK(OP, FLAG)
3110 #define ATOMIC_CRITICAL_CPT_REV_WRK(TYPE_ID, OP_ID, TYPE, OP, LCK_ID, \
3112 ATOMIC_BEGIN_WRK(TYPE_ID, OP_ID, TYPE) \
3113 OP_GOMP_CRITICAL_CPT_REV_WRK(OP, GOMP_FLAG) \
3114 OP_CRITICAL_CPT_REV_WRK(OP, LCK_ID) \
3120 ATOMIC_CRITICAL_CPT_REV_WRK(cmplx4, sub_cpt_rev, kmp_cmplx32, -, 8c,
3122 ATOMIC_CRITICAL_CPT_REV_WRK(cmplx4, div_cpt_rev, kmp_cmplx32, /, 8c,
3125 ATOMIC_CRITICAL_CPT_REV(cmplx8, sub_cpt_rev, kmp_cmplx64, -, 16c,
3127 ATOMIC_CRITICAL_CPT_REV(cmplx8, div_cpt_rev, kmp_cmplx64, /, 16c,
3129 ATOMIC_CRITICAL_CPT_REV(cmplx10, sub_cpt_rev, kmp_cmplx80, -, 20c,
3131 ATOMIC_CRITICAL_CPT_REV(cmplx10, div_cpt_rev, kmp_cmplx80, /, 20c,
3134 ATOMIC_CRITICAL_CPT_REV(cmplx16, sub_cpt_rev, CPLX128_LEG, -, 32c,
3136 ATOMIC_CRITICAL_CPT_REV(cmplx16, div_cpt_rev, CPLX128_LEG, /, 32c,
3139 ATOMIC_CRITICAL_CPT_REV(cmplx16, sub_a16_cpt_rev, kmp_cmplx128_a16_t, -, 32c,
3141 ATOMIC_CRITICAL_CPT_REV(cmplx16, div_a16_cpt_rev, kmp_cmplx128_a16_t, /, 32c,
3155 #define ATOMIC_CMPXCHG_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, BITS, OP, RTYPE_ID, \
3156 RTYPE, LCK_ID, MASK, GOMP_FLAG) \
3157 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
3159 OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \
3160 OP_CMPXCHG_CPT_REV(TYPE, BITS, OP) \
3164 #define ATOMIC_CRITICAL_CPT_REV_MIX(TYPE_ID, TYPE, OP_ID, OP, RTYPE_ID, RTYPE, \
3165 LCK_ID, GOMP_FLAG) \
3166 ATOMIC_BEGIN_CPT_MIX(TYPE_ID, OP_ID, TYPE, RTYPE_ID, RTYPE) \
3168 OP_GOMP_CRITICAL_CPT_REV(TYPE, OP, GOMP_FLAG) \
3169 OP_CRITICAL_CPT_REV(TYPE, OP, LCK_ID) \
3172 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1,
char, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0,
3174 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1u, uchar, sub_cpt_rev, 8, -, fp, _Quad, 1i, 0,
3176 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1,
char, div_cpt_rev, 8, /, fp, _Quad, 1i, 0,
3178 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed1u, uchar, div_cpt_rev, 8, /, fp, _Quad, 1i, 0,
3181 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2,
short, sub_cpt_rev, 16, -, fp, _Quad, 2i, 1,
3183 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2u, ushort, sub_cpt_rev, 16, -, fp, _Quad, 2i,
3186 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2,
short, div_cpt_rev, 16, /, fp, _Quad, 2i, 1,
3188 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed2u, ushort, div_cpt_rev, 16, /, fp, _Quad, 2i,
3192 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4, kmp_int32, sub_cpt_rev, 32, -, fp, _Quad, 4i,
3194 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4u, kmp_uint32, sub_cpt_rev, 32, -, fp, _Quad,
3196 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4, kmp_int32, div_cpt_rev, 32, /, fp, _Quad, 4i,
3198 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed4u, kmp_uint32, div_cpt_rev, 32, /, fp, _Quad,
3201 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8, kmp_int64, sub_cpt_rev, 64, -, fp, _Quad, 8i,
3204 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8u, kmp_uint64, sub_cpt_rev, 64, -, fp, _Quad,
3207 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8, kmp_int64, div_cpt_rev, 64, /, fp, _Quad, 8i,
3210 ATOMIC_CMPXCHG_CPT_REV_MIX(fixed8u, kmp_uint64, div_cpt_rev, 64, /, fp, _Quad,
3214 ATOMIC_CMPXCHG_CPT_REV_MIX(float4, kmp_real32, sub_cpt_rev, 32, -, fp, _Quad,
3217 ATOMIC_CMPXCHG_CPT_REV_MIX(float4, kmp_real32, div_cpt_rev, 32, /, fp, _Quad,
3221 ATOMIC_CMPXCHG_CPT_REV_MIX(float8, kmp_real64, sub_cpt_rev, 64, -, fp, _Quad,
3224 ATOMIC_CMPXCHG_CPT_REV_MIX(float8, kmp_real64, div_cpt_rev, 64, /, fp, _Quad,
3228 ATOMIC_CRITICAL_CPT_REV_MIX(float10,
long double, sub_cpt_rev, -, fp, _Quad,
3230 ATOMIC_CRITICAL_CPT_REV_MIX(float10,
long double, div_cpt_rev, /, fp, _Quad,
3237 #define ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3238 TYPE __kmpc_atomic_##TYPE_ID##_swp(ident_t *id_ref, int gtid, TYPE *lhs, \
3240 KMP_DEBUG_ASSERT(__kmp_init_serial); \
3241 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid));
3243 #define CRITICAL_SWP(LCK_ID) \
3244 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3246 old_value = (*lhs); \
3249 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3253 #ifdef KMP_GOMP_COMPAT
3254 #define GOMP_CRITICAL_SWP(FLAG) \
3255 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
3260 #define GOMP_CRITICAL_SWP(FLAG)
3263 #define ATOMIC_XCHG_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \
3264 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3266 GOMP_CRITICAL_SWP(GOMP_FLAG) \
3267 old_value = KMP_XCHG_FIXED##BITS(lhs, rhs); \
3271 #define ATOMIC_XCHG_FLOAT_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \
3272 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3274 GOMP_CRITICAL_SWP(GOMP_FLAG) \
3275 old_value = KMP_XCHG_REAL##BITS(lhs, rhs); \
3280 #define CMPXCHG_SWP(TYPE, BITS) \
3282 TYPE KMP_ATOMIC_VOLATILE temp_val; \
3283 TYPE old_value, new_value; \
3285 old_value = temp_val; \
3287 while (!KMP_COMPARE_AND_STORE_ACQ##BITS( \
3288 (kmp_int##BITS *)lhs, *VOLATILE_CAST(kmp_int##BITS *) & old_value, \
3289 *VOLATILE_CAST(kmp_int##BITS *) & new_value)) { \
3291 old_value = temp_val; \
3298 #define ATOMIC_CMPXCHG_SWP(TYPE_ID, TYPE, BITS, GOMP_FLAG) \
3299 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3302 GOMP_CRITICAL_SWP(GOMP_FLAG) \
3303 CMPXCHG_SWP(TYPE, BITS) \
3306 ATOMIC_XCHG_SWP(fixed1, kmp_int8, 8, KMP_ARCH_X86)
3307 ATOMIC_XCHG_SWP(fixed2, kmp_int16, 16, KMP_ARCH_X86)
3308 ATOMIC_XCHG_SWP(fixed4, kmp_int32, 32, KMP_ARCH_X86)
3310 ATOMIC_XCHG_FLOAT_SWP(float4, kmp_real32, 32,
3314 ATOMIC_CMPXCHG_SWP(fixed8, kmp_int64, 64,
3316 ATOMIC_CMPXCHG_SWP(float8, kmp_real64, 64,
3319 ATOMIC_XCHG_SWP(fixed8, kmp_int64, 64, KMP_ARCH_X86)
3320 ATOMIC_XCHG_FLOAT_SWP(float8, kmp_real64, 64,
3327 #define ATOMIC_CRITICAL_SWP(TYPE_ID, TYPE, LCK_ID, GOMP_FLAG) \
3328 ATOMIC_BEGIN_SWP(TYPE_ID, TYPE) \
3330 GOMP_CRITICAL_SWP(GOMP_FLAG) \
3331 CRITICAL_SWP(LCK_ID) \
3339 #define ATOMIC_BEGIN_SWP_WRK(TYPE_ID, TYPE) \
3340 void __kmpc_atomic_##TYPE_ID##_swp(ident_t *id_ref, int gtid, TYPE *lhs, \
3341 TYPE rhs, TYPE *out) { \
3342 KMP_DEBUG_ASSERT(__kmp_init_serial); \
3343 KA_TRACE(100, ("__kmpc_atomic_" #TYPE_ID "_swp: T#%d\n", gtid));
3345 #define CRITICAL_SWP_WRK(LCK_ID) \
3346 __kmp_acquire_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3351 __kmp_release_atomic_lock(&ATOMIC_LOCK##LCK_ID, gtid); \
3355 #ifdef KMP_GOMP_COMPAT
3356 #define GOMP_CRITICAL_SWP_WRK(FLAG) \
3357 if ((FLAG) && (__kmp_atomic_mode == 2)) { \
3359 CRITICAL_SWP_WRK(0); \
3362 #define GOMP_CRITICAL_SWP_WRK(FLAG)
3366 #define ATOMIC_CRITICAL_SWP_WRK(TYPE_ID, TYPE, LCK_ID, GOMP_FLAG) \
3367 ATOMIC_BEGIN_SWP_WRK(TYPE_ID, TYPE) \
3369 GOMP_CRITICAL_SWP_WRK(GOMP_FLAG) \
3370 CRITICAL_SWP_WRK(LCK_ID) \
3374 ATOMIC_CRITICAL_SWP(float10,
long double, 10r, 1)
3376 ATOMIC_CRITICAL_SWP(float16, QUAD_LEGACY, 16r, 1)
3379 ATOMIC_CRITICAL_SWP_WRK(cmplx4, kmp_cmplx32, 8c, 1)
3384 ATOMIC_CRITICAL_SWP(cmplx8, kmp_cmplx64, 16c, 1)
3385 ATOMIC_CRITICAL_SWP(cmplx10, kmp_cmplx80, 20c, 1)
3387 ATOMIC_CRITICAL_SWP(cmplx16, CPLX128_LEG, 32c, 1)
3389 ATOMIC_CRITICAL_SWP(float16_a16, Quad_a16_t, 16r,
3391 ATOMIC_CRITICAL_SWP(cmplx16_a16, kmp_cmplx128_a16_t, 32c,
3405 void __kmpc_atomic_1(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3406 void (*f)(
void *,
void *,
void *)) {
3407 KMP_DEBUG_ASSERT(__kmp_init_serial);
3410 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3416 kmp_int8 old_value, new_value;
3418 old_value = *(kmp_int8 *)lhs;
3419 (*f)(&new_value, &old_value, rhs);
3422 while (!KMP_COMPARE_AND_STORE_ACQ8((kmp_int8 *)lhs, *(kmp_int8 *)&old_value,
3423 *(kmp_int8 *)&new_value)) {
3426 old_value = *(kmp_int8 *)lhs;
3427 (*f)(&new_value, &old_value, rhs);
3434 #ifdef KMP_GOMP_COMPAT
3435 if (__kmp_atomic_mode == 2) {
3436 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3439 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_1i, gtid);
3441 (*f)(lhs, lhs, rhs);
3443 #ifdef KMP_GOMP_COMPAT
3444 if (__kmp_atomic_mode == 2) {
3445 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3448 __kmp_release_atomic_lock(&__kmp_atomic_lock_1i, gtid);
3452 void __kmpc_atomic_2(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3453 void (*f)(
void *,
void *,
void *)) {
3455 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3457 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
3460 !((kmp_uintptr_t)lhs & 0x1)
3463 kmp_int16 old_value, new_value;
3465 old_value = *(kmp_int16 *)lhs;
3466 (*f)(&new_value, &old_value, rhs);
3469 while (!KMP_COMPARE_AND_STORE_ACQ16(
3470 (kmp_int16 *)lhs, *(kmp_int16 *)&old_value, *(kmp_int16 *)&new_value)) {
3473 old_value = *(kmp_int16 *)lhs;
3474 (*f)(&new_value, &old_value, rhs);
3481 #ifdef KMP_GOMP_COMPAT
3482 if (__kmp_atomic_mode == 2) {
3483 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3486 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_2i, gtid);
3488 (*f)(lhs, lhs, rhs);
3490 #ifdef KMP_GOMP_COMPAT
3491 if (__kmp_atomic_mode == 2) {
3492 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3495 __kmp_release_atomic_lock(&__kmp_atomic_lock_2i, gtid);
3499 void __kmpc_atomic_4(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3500 void (*f)(
void *,
void *,
void *)) {
3501 KMP_DEBUG_ASSERT(__kmp_init_serial);
3506 #
if KMP_ARCH_X86 || KMP_ARCH_X86_64
3509 !((kmp_uintptr_t)lhs & 0x3)
3512 kmp_int32 old_value, new_value;
3514 old_value = *(kmp_int32 *)lhs;
3515 (*f)(&new_value, &old_value, rhs);
3518 while (!KMP_COMPARE_AND_STORE_ACQ32(
3519 (kmp_int32 *)lhs, *(kmp_int32 *)&old_value, *(kmp_int32 *)&new_value)) {
3522 old_value = *(kmp_int32 *)lhs;
3523 (*f)(&new_value, &old_value, rhs);
3531 #ifdef KMP_GOMP_COMPAT
3532 if (__kmp_atomic_mode == 2) {
3533 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3536 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_4i, gtid);
3538 (*f)(lhs, lhs, rhs);
3540 #ifdef KMP_GOMP_COMPAT
3541 if (__kmp_atomic_mode == 2) {
3542 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3545 __kmp_release_atomic_lock(&__kmp_atomic_lock_4i, gtid);
3549 void __kmpc_atomic_8(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3550 void (*f)(
void *,
void *,
void *)) {
3551 KMP_DEBUG_ASSERT(__kmp_init_serial);
3554 #
if KMP_ARCH_X86 && defined(KMP_GOMP_COMPAT)
3556 #elif KMP_ARCH_X86 || KMP_ARCH_X86_64
3559 !((kmp_uintptr_t)lhs & 0x7)
3562 kmp_int64 old_value, new_value;
3564 old_value = *(kmp_int64 *)lhs;
3565 (*f)(&new_value, &old_value, rhs);
3567 while (!KMP_COMPARE_AND_STORE_ACQ64(
3568 (kmp_int64 *)lhs, *(kmp_int64 *)&old_value, *(kmp_int64 *)&new_value)) {
3571 old_value = *(kmp_int64 *)lhs;
3572 (*f)(&new_value, &old_value, rhs);
3580 #ifdef KMP_GOMP_COMPAT
3581 if (__kmp_atomic_mode == 2) {
3582 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3585 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_8i, gtid);
3587 (*f)(lhs, lhs, rhs);
3589 #ifdef KMP_GOMP_COMPAT
3590 if (__kmp_atomic_mode == 2) {
3591 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3594 __kmp_release_atomic_lock(&__kmp_atomic_lock_8i, gtid);
3598 void __kmpc_atomic_10(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3599 void (*f)(
void *,
void *,
void *)) {
3600 KMP_DEBUG_ASSERT(__kmp_init_serial);
3602 #ifdef KMP_GOMP_COMPAT
3603 if (__kmp_atomic_mode == 2) {
3604 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3607 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_10r, gtid);
3609 (*f)(lhs, lhs, rhs);
3611 #ifdef KMP_GOMP_COMPAT
3612 if (__kmp_atomic_mode == 2) {
3613 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3616 __kmp_release_atomic_lock(&__kmp_atomic_lock_10r, gtid);
3619 void __kmpc_atomic_16(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3620 void (*f)(
void *,
void *,
void *)) {
3621 KMP_DEBUG_ASSERT(__kmp_init_serial);
3623 #ifdef KMP_GOMP_COMPAT
3624 if (__kmp_atomic_mode == 2) {
3625 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3628 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_16c, gtid);
3630 (*f)(lhs, lhs, rhs);
3632 #ifdef KMP_GOMP_COMPAT
3633 if (__kmp_atomic_mode == 2) {
3634 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3637 __kmp_release_atomic_lock(&__kmp_atomic_lock_16c, gtid);
3640 void __kmpc_atomic_20(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3641 void (*f)(
void *,
void *,
void *)) {
3642 KMP_DEBUG_ASSERT(__kmp_init_serial);
3644 #ifdef KMP_GOMP_COMPAT
3645 if (__kmp_atomic_mode == 2) {
3646 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3649 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_20c, gtid);
3651 (*f)(lhs, lhs, rhs);
3653 #ifdef KMP_GOMP_COMPAT
3654 if (__kmp_atomic_mode == 2) {
3655 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3658 __kmp_release_atomic_lock(&__kmp_atomic_lock_20c, gtid);
3661 void __kmpc_atomic_32(
ident_t *id_ref,
int gtid,
void *lhs,
void *rhs,
3662 void (*f)(
void *,
void *,
void *)) {
3663 KMP_DEBUG_ASSERT(__kmp_init_serial);
3665 #ifdef KMP_GOMP_COMPAT
3666 if (__kmp_atomic_mode == 2) {
3667 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3670 __kmp_acquire_atomic_lock(&__kmp_atomic_lock_32c, gtid);
3672 (*f)(lhs, lhs, rhs);
3674 #ifdef KMP_GOMP_COMPAT
3675 if (__kmp_atomic_mode == 2) {
3676 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3679 __kmp_release_atomic_lock(&__kmp_atomic_lock_32c, gtid);
3685 void __kmpc_atomic_start(
void) {
3686 int gtid = __kmp_entry_gtid();
3687 KA_TRACE(20, (
"__kmpc_atomic_start: T#%d\n", gtid));
3688 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid);
3691 void __kmpc_atomic_end(
void) {
3692 int gtid = __kmp_get_gtid();
3693 KA_TRACE(20, (
"__kmpc_atomic_end: T#%d\n", gtid));
3694 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid);
3697 #if KMP_ARCH_X86 || KMP_ARCH_X86_64
3715 bool __kmpc_atomic_bool_1_cas(
ident_t *loc,
int gtid,
char *x,
char e,
char d) {
3716 return KMP_COMPARE_AND_STORE_ACQ8(x, e, d);
3718 bool __kmpc_atomic_bool_2_cas(
ident_t *loc,
int gtid,
short *x,
short e,
3720 return KMP_COMPARE_AND_STORE_ACQ16(x, e, d);
3722 bool __kmpc_atomic_bool_4_cas(
ident_t *loc,
int gtid, kmp_int32 *x, kmp_int32 e,
3724 return KMP_COMPARE_AND_STORE_ACQ32(x, e, d);
3726 bool __kmpc_atomic_bool_8_cas(
ident_t *loc,
int gtid, kmp_int64 *x, kmp_int64 e,
3728 return KMP_COMPARE_AND_STORE_ACQ64(x, e, d);
3745 char __kmpc_atomic_val_1_cas(
ident_t *loc,
int gtid,
char *x,
char e,
char d) {
3746 return KMP_COMPARE_AND_STORE_RET8(x, e, d);
3748 short __kmpc_atomic_val_2_cas(
ident_t *loc,
int gtid,
short *x,
short e,
3750 return KMP_COMPARE_AND_STORE_RET16(x, e, d);
3752 kmp_int32 __kmpc_atomic_val_4_cas(
ident_t *loc,
int gtid, kmp_int32 *x,
3753 kmp_int32 e, kmp_int32 d) {
3754 return KMP_COMPARE_AND_STORE_RET32(x, e, d);
3756 kmp_int64 __kmpc_atomic_val_8_cas(
ident_t *loc,
int gtid, kmp_int64 *x,
3757 kmp_int64 e, kmp_int64 d) {
3758 return KMP_COMPARE_AND_STORE_RET64(x, e, d);
3777 bool __kmpc_atomic_bool_1_cas_cpt(
ident_t *loc,
int gtid,
char *x,
char e,
3779 char old = KMP_COMPARE_AND_STORE_RET8(x, e, d);
3782 KMP_ASSERT(pv != NULL);
3786 bool __kmpc_atomic_bool_2_cas_cpt(
ident_t *loc,
int gtid,
short *x,
short e,
3787 short d,
short *pv) {
3788 short old = KMP_COMPARE_AND_STORE_RET16(x, e, d);
3791 KMP_ASSERT(pv != NULL);
3795 bool __kmpc_atomic_bool_4_cas_cpt(
ident_t *loc,
int gtid, kmp_int32 *x,
3796 kmp_int32 e, kmp_int32 d, kmp_int32 *pv) {
3797 kmp_int32 old = KMP_COMPARE_AND_STORE_RET32(x, e, d);
3800 KMP_ASSERT(pv != NULL);
3804 bool __kmpc_atomic_bool_8_cas_cpt(
ident_t *loc,
int gtid, kmp_int64 *x,
3805 kmp_int64 e, kmp_int64 d, kmp_int64 *pv) {
3806 kmp_int64 old = KMP_COMPARE_AND_STORE_RET64(x, e, d);
3809 KMP_ASSERT(pv != NULL);
3830 char __kmpc_atomic_val_1_cas_cpt(
ident_t *loc,
int gtid,
char *x,
char e,
3832 char old = KMP_COMPARE_AND_STORE_RET8(x, e, d);
3833 KMP_ASSERT(pv != NULL);
3834 *pv = old == e ? d : old;
3837 short __kmpc_atomic_val_2_cas_cpt(
ident_t *loc,
int gtid,
short *x,
short e,
3838 short d,
short *pv) {
3839 short old = KMP_COMPARE_AND_STORE_RET16(x, e, d);
3840 KMP_ASSERT(pv != NULL);
3841 *pv = old == e ? d : old;
3844 kmp_int32 __kmpc_atomic_val_4_cas_cpt(
ident_t *loc,
int gtid, kmp_int32 *x,
3845 kmp_int32 e, kmp_int32 d, kmp_int32 *pv) {
3846 kmp_int32 old = KMP_COMPARE_AND_STORE_RET32(x, e, d);
3847 KMP_ASSERT(pv != NULL);
3848 *pv = old == e ? d : old;
3851 kmp_int64 __kmpc_atomic_val_8_cas_cpt(
ident_t *loc,
int gtid, kmp_int64 *x,
3852 kmp_int64 e, kmp_int64 d, kmp_int64 *pv) {
3853 kmp_int64 old = KMP_COMPARE_AND_STORE_RET64(x, e, d);
3854 KMP_ASSERT(pv != NULL);
3855 *pv = old == e ? d : old;