Skip to content

Commit 6bafb38

Browse files
hahnjohahnjo
committed
[hist] Implement atomic increment and addition
std::atomic_ref in C++20 will offer a portable way to apply atomic operations to a referenced object, but for now implement a minimal version ourselves with compiler builtins.
1 parent 7501685 commit 6bafb38

File tree

3 files changed

+217
-0
lines changed

3 files changed

+217
-0
lines changed

‎hist/histv7/inc/ROOT/RHistUtils.hxx‎

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,12 @@
55
#ifndef ROOT_RHistUtils
66
#define ROOT_RHistUtils
77

8+
#include <type_traits>
9+
10+
#ifdef _MSC_VER
11+
#include <intrin.h>
12+
#endif
13+
814
namespace ROOT {
915
namespace Experimental {
1016
namespace Internal {
@@ -16,6 +22,180 @@ struct LastType<T> {
1622
using type = T;
1723
};
1824

25+
#ifdef _MSC_VER
26+
namespace MSVC {
27+
template <std::size_t N>
28+
struct AtomicOps {};
29+
30+
template <>
31+
struct AtomicOps<1> {
32+
static void Load(const void *ptr, void *ret)
33+
{
34+
*static_cast<char *>(ret) = __iso_volatile_load8(static_cast<const char *>(ptr));
35+
}
36+
static void Add(void *ptr, const void *val)
37+
{
38+
_InterlockedExchangeAdd8(static_cast<char *>(ptr), *static_cast<const char *>(val));
39+
}
40+
static bool CompareExchange(void *ptr, void *expected, const void *desired)
41+
{
42+
// MSVC functions have the arguments reversed...
43+
char expectedVal = *static_cast<char *>(expected);
44+
char previous =
45+
_InterlockedCompareExchange8(static_cast<char *>(ptr), *static_cast<const char *>(desired), expectedVal);
46+
if (previous == expectedVal) {
47+
return true;
48+
}
49+
*static_cast<char *>(expected) = previous;
50+
return false;
51+
}
52+
};
53+
54+
template <>
55+
struct AtomicOps<2> {
56+
static void Load(const void *ptr, void *ret)
57+
{
58+
*static_cast<short *>(ret) = __iso_volatile_load16(static_cast<const short *>(ptr));
59+
}
60+
static void Add(void *ptr, const void *val)
61+
{
62+
_InterlockedExchangeAdd16(static_cast<short *>(ptr), *static_cast<const short *>(val));
63+
}
64+
static bool CompareExchange(void *ptr, void *expected, const void *desired)
65+
{
66+
// MSVC functions have the arguments reversed...
67+
short expectedVal = *static_cast<short *>(expected);
68+
short previous =
69+
_InterlockedCompareExchange16(static_cast<short *>(ptr), *static_cast<const short *>(desired), expectedVal);
70+
if (previous == expectedVal) {
71+
return true;
72+
}
73+
*static_cast<short *>(expected) = previous;
74+
return false;
75+
}
76+
};
77+
78+
template <>
79+
struct AtomicOps<4> {
80+
static void Load(const void *ptr, void *ret)
81+
{
82+
*static_cast<int *>(ret) = __iso_volatile_load32(static_cast<const int *>(ptr));
83+
}
84+
static void Add(void *ptr, const void *val)
85+
{
86+
_InterlockedExchangeAdd(static_cast<long *>(ptr), *static_cast<const long *>(val));
87+
}
88+
static bool CompareExchange(void *ptr, void *expected, const void *desired)
89+
{
90+
// MSVC functions have the arguments reversed...
91+
long expectedVal = *static_cast<long *>(expected);
92+
long previous =
93+
_InterlockedCompareExchange(static_cast<long *>(ptr), *static_cast<const long *>(desired), expectedVal);
94+
if (previous == expectedVal) {
95+
return true;
96+
}
97+
*static_cast<long *>(expected) = previous;
98+
return false;
99+
}
100+
};
101+
102+
template <>
103+
struct AtomicOps<8> {
104+
static void Load(const void *ptr, void *ret)
105+
{
106+
*static_cast<__int64 *>(ret) = __iso_volatile_load64(static_cast<const __int64 *>(ptr));
107+
}
108+
static void Add(void *ptr, const void *val);
109+
static bool CompareExchange(void *ptr, void *expected, const void *desired)
110+
{
111+
// MSVC functions have the arguments reversed...
112+
__int64 expectedVal = *static_cast<__int64 *>(expected);
113+
__int64 previous = _InterlockedCompareExchange64(static_cast<__int64 *>(ptr),
114+
*static_cast<const __int64 *>(desired), expectedVal);
115+
if (previous == expectedVal) {
116+
return true;
117+
}
118+
*static_cast<__int64 *>(expected) = previous;
119+
return false;
120+
}
121+
};
122+
} // namespace MSVC
123+
#endif
124+
125+
template <typename T>
126+
void AtomicLoad(const T *ptr, T *ret)
127+
{
128+
#ifndef _MSC_VER
129+
__atomic_load(ptr, ret, __ATOMIC_RELAXED);
130+
#else
131+
MSVC::AtomicOps<sizeof(T)>::Load(ptr, ret);
132+
#endif
133+
}
134+
135+
template <typename T>
136+
bool AtomicCompareExchange(T *ptr, T *expected, T *desired)
137+
{
138+
#ifndef _MSC_VER
139+
return __atomic_compare_exchange(ptr, expected, desired, /*weak=*/false, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
140+
#else
141+
return MSVC::AtomicOps<sizeof(T)>::CompareExchange(ptr, expected, desired);
142+
#endif
143+
}
144+
145+
template <typename T>
146+
void AtomicAddCompareExchangeLoop(T *ptr, T val)
147+
{
148+
T expected;
149+
AtomicLoad(ptr, &expected);
150+
T desired = expected + val;
151+
while (!AtomicCompareExchange(ptr, &expected, &desired)) {
152+
// expected holds the new value; try again.
153+
desired = expected + val;
154+
}
155+
}
156+
157+
#ifdef _MSC_VER
158+
namespace MSVC {
159+
inline void AtomicOps<8>::Add(void *ptr, const void *val)
160+
{
161+
#if _WIN64
162+
_InterlockedExchangeAdd64(static_cast<__int64 *>(ptr), *static_cast<const __int64 *>(val));
163+
#else
164+
AtomicAddCompareExchangeLoop(static_cast<__int64 *>(ptr), *static_cast<const __int64 *>(val));
165+
#endif
166+
}
167+
} // namespace MSVC
168+
#endif
169+
170+
template <typename T>
171+
std::enable_if_t<std::is_integral_v<T>> AtomicAdd(T *ptr, T val)
172+
{
173+
#ifndef _MSC_VER
174+
__atomic_fetch_add(ptr, val, __ATOMIC_RELAXED);
175+
#else
176+
MSVC::AtomicOps<sizeof(T)>::Add(ptr, &val);
177+
#endif
178+
}
179+
180+
template <typename T>
181+
std::enable_if_t<std::is_floating_point_v<T>> AtomicAdd(T *ptr, T val)
182+
{
183+
AtomicAddCompareExchangeLoop(ptr, val);
184+
}
185+
186+
// For adding a double-precision weight to a single-precision bin content type, cast the argument once before the
187+
// compare-exchange loop.
188+
static inline void AtomicAdd(float *ptr, double val)
189+
{
190+
AtomicAdd(ptr, static_cast<float>(val));
191+
}
192+
193+
template <typename T>
194+
std::enable_if_t<std::is_arithmetic_v<T>> AtomicInc(T *ptr)
195+
{
196+
AtomicAdd(ptr, static_cast<T>(1));
197+
}
198+
19199
} // namespace Internal
20200
} // namespace Experimental
21201
} // namespace ROOT

‎hist/histv7/test/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
HIST_ADD_GTEST(hist_atomic hist_atomic.cxx)
12
HIST_ADD_GTEST(hist_axes hist_axes.cxx)
23
HIST_ADD_GTEST(hist_categorical hist_categorical.cxx)
34
HIST_ADD_GTEST(hist_engine hist_engine.cxx)

‎hist/histv7/test/hist_atomic.cxx‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#include <ROOT/RHistUtils.hxx>
2+
3+
#include <gtest/gtest.h>
4+
5+
#ifndef TYPED_TEST_SUITE
6+
#define TYPED_TEST_SUITE TYPED_TEST_CASE
7+
#endif
8+
9+
template <typename T>
10+
class RHistAtomic : public testing::Test {};
11+
12+
using AtomicTypes = testing::Types<char, short, int, long, long long, float, double>;
13+
TYPED_TEST_SUITE(RHistAtomic, AtomicTypes);
14+
15+
TYPED_TEST(RHistAtomic, AtomicInc)
16+
{
17+
TypeParam a = 1;
18+
ROOT::Experimental::Internal::AtomicInc(&a);
19+
EXPECT_EQ(a, 2);
20+
}
21+
22+
TYPED_TEST(RHistAtomic, AtomicAdd)
23+
{
24+
TypeParam a = 1;
25+
const TypeParam b = 2;
26+
ROOT::Experimental::Internal::AtomicAdd(&a, b);
27+
EXPECT_EQ(a, 3);
28+
}
29+
30+
TEST(AtomicAdd, FloatDouble)
31+
{
32+
float a = 1;
33+
const double b = 2;
34+
ROOT::Experimental::Internal::AtomicAdd(&a, b);
35+
EXPECT_EQ(a, 3);
36+
}

0 commit comments

Comments
 (0)