1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
|
// SPDX-License-Identifier: GPL-2.0+
//
// An earlier version of this file appeared in the companion webpage for
// "Frightening small children and disconcerting grown-ups: Concurrency
// in the Linux kernel" by Alglave, Maranget, McKenney, Parri, and Stern,
// which appeared in ASPLOS 2018.
// ONCE
READ_ONCE(X) __load{ONCE}(X)
WRITE_ONCE(X,V) { __store{ONCE}(X,V); }
// Release Acquire and friends
smp_store_release(X,V) { __store{RELEASE}(*X,V); }
smp_load_acquire(X) __load{ACQUIRE}(*X)
rcu_assign_pointer(X,V) { __store{RELEASE}(X,V); }
rcu_dereference(X) __load{ONCE}(X)
smp_store_mb(X,V) { __store{ONCE}(X,V); __fence{MB}; }
// Fences
smp_mb() { __fence{MB}; }
smp_rmb() { __fence{rmb}; }
smp_wmb() { __fence{wmb}; }
smp_mb__before_atomic() { __fence{before-atomic}; }
smp_mb__after_atomic() { __fence{after-atomic}; }
smp_mb__after_spinlock() { __fence{after-spinlock}; }
smp_mb__after_unlock_lock() { __fence{after-unlock-lock}; }
smp_mb__after_srcu_read_unlock() { __fence{after-srcu-read-unlock}; }
barrier() { __fence{barrier}; }
// Exchange
xchg(X,V) __xchg{MB}(X,V)
xchg_relaxed(X,V) __xchg{ONCE}(X,V)
xchg_release(X,V) __xchg{RELEASE}(X,V)
xchg_acquire(X,V) __xchg{ACQUIRE}(X,V)
cmpxchg(X,V,W) __cmpxchg{MB}(X,V,W)
cmpxchg_relaxed(X,V,W) __cmpxchg{ONCE}(X,V,W)
cmpxchg_acquire(X,V,W) __cmpxchg{ACQUIRE}(X,V,W)
cmpxchg_release(X,V,W) __cmpxchg{RELEASE}(X,V,W)
// Spinlocks
spin_lock(X) { __lock(X); }
spin_unlock(X) { __unlock(X); }
spin_trylock(X) __trylock(X)
spin_is_locked(X) __islocked(X)
// RCU
rcu_read_lock() { __fence{rcu-lock}; }
rcu_read_unlock() { __fence{rcu-unlock}; }
synchronize_rcu() { __fence{sync-rcu}; }
synchronize_rcu_expedited() { __fence{sync-rcu}; }
// SRCU
srcu_read_lock(X) __load{srcu-lock}(*X)
srcu_read_unlock(X,Y) { __store{srcu-unlock}(*X,Y); }
srcu_down_read(X) __load{srcu-lock}(*X)
srcu_up_read(X,Y) { __store{srcu-unlock}(*X,Y); }
synchronize_srcu(X) { __srcu{sync-srcu}(X); }
synchronize_srcu_expedited(X) { __srcu{sync-srcu}(X); }
// Atomic
atomic_read(X) READ_ONCE(*X)
atomic_set(X,V) { WRITE_ONCE(*X,V); }
atomic_read_acquire(X) smp_load_acquire(X)
atomic_set_release(X,V) { smp_store_release(X,V); }
atomic_add(V,X) { __atomic_op{NORETURN}(X,+,V); }
atomic_sub(V,X) { __atomic_op{NORETURN}(X,-,V); }
atomic_and(V,X) { __atomic_op{NORETURN}(X,&,V); }
atomic_or(V,X) { __atomic_op{NORETURN}(X,|,V); }
atomic_xor(V,X) { __atomic_op{NORETURN}(X,^,V); }
atomic_inc(X) { __atomic_op{NORETURN}(X,+,1); }
atomic_dec(X) { __atomic_op{NORETURN}(X,-,1); }
atomic_andnot(V,X) { __atomic_op{NORETURN}(X,&~,V); }
atomic_add_return(V,X) __atomic_op_return{MB}(X,+,V)
atomic_add_return_relaxed(V,X) __atomic_op_return{ONCE}(X,+,V)
atomic_add_return_acquire(V,X) __atomic_op_return{ACQUIRE}(X,+,V)
atomic_add_return_release(V,X) __atomic_op_return{RELEASE}(X,+,V)
atomic_fetch_add(V,X) __atomic_fetch_op{MB}(X,+,V)
atomic_fetch_add_relaxed(V,X) __atomic_fetch_op{ONCE}(X,+,V)
atomic_fetch_add_acquire(V,X) __atomic_fetch_op{ACQUIRE}(X,+,V)
atomic_fetch_add_release(V,X) __atomic_fetch_op{RELEASE}(X,+,V)
atomic_fetch_and(V,X) __atomic_fetch_op{MB}(X,&,V)
atomic_fetch_and_relaxed(V,X) __atomic_fetch_op{ONCE}(X,&,V)
atomic_fetch_and_acquire(V,X) __atomic_fetch_op{ACQUIRE}(X,&,V)
atomic_fetch_and_release(V,X) __atomic_fetch_op{RELEASE}(X,&,V)
atomic_fetch_or(V,X) __atomic_fetch_op{MB}(X,|,V)
atomic_fetch_or_relaxed(V,X) __atomic_fetch_op{ONCE}(X,|,V)
atomic_fetch_or_acquire(V,X) __atomic_fetch_op{ACQUIRE}(X,|,V)
atomic_fetch_or_release(V,X) __atomic_fetch_op{RELEASE}(X,|,V)
atomic_fetch_xor(V,X) __atomic_fetch_op{MB}(X,^,V)
atomic_fetch_xor_relaxed(V,X) __atomic_fetch_op{ONCE}(X,^,V)
atomic_fetch_xor_acquire(V,X) __atomic_fetch_op{ACQUIRE}(X,^,V)
atomic_fetch_xor_release(V,X) __atomic_fetch_op{RELEASE}(X,^,V)
atomic_inc_return(X) __atomic_op_return{MB}(X,+,1)
atomic_inc_return_relaxed(X) __atomic_op_return{ONCE}(X,+,1)
atomic_inc_return_acquire(X) __atomic_op_return{ACQUIRE}(X,+,1)
atomic_inc_return_release(X) __atomic_op_return{RELEASE}(X,+,1)
atomic_fetch_inc(X) __atomic_fetch_op{MB}(X,+,1)
atomic_fetch_inc_relaxed(X) __atomic_fetch_op{ONCE}(X,+,1)
atomic_fetch_inc_acquire(X) __atomic_fetch_op{ACQUIRE}(X,+,1)
atomic_fetch_inc_release(X) __atomic_fetch_op{RELEASE}(X,+,1)
atomic_sub_return(V,X) __atomic_op_return{MB}(X,-,V)
atomic_sub_return_relaxed(V,X) __atomic_op_return{ONCE}(X,-,V)
atomic_sub_return_acquire(V,X) __atomic_op_return{ACQUIRE}(X,-,V)
atomic_sub_return_release(V,X) __atomic_op_return{RELEASE}(X,-,V)
atomic_fetch_sub(V,X) __atomic_fetch_op{MB}(X,-,V)
atomic_fetch_sub_relaxed(V,X) __atomic_fetch_op{ONCE}(X,-,V)
atomic_fetch_sub_acquire(V,X) __atomic_fetch_op{ACQUIRE}(X,-,V)
atomic_fetch_sub_release(V,X) __atomic_fetch_op{RELEASE}(X,-,V)
atomic_dec_return(X) __atomic_op_return{MB}(X,-,1)
atomic_dec_return_relaxed(X) __atomic_op_return{ONCE}(X,-,1)
atomic_dec_return_acquire(X) __atomic_op_return{ACQUIRE}(X,-,1)
atomic_dec_return_release(X) __atomic_op_return{RELEASE}(X,-,1)
atomic_fetch_dec(X) __atomic_fetch_op{MB}(X,-,1)
atomic_fetch_dec_relaxed(X) __atomic_fetch_op{ONCE}(X,-,1)
atomic_fetch_dec_acquire(X) __atomic_fetch_op{ACQUIRE}(X,-,1)
atomic_fetch_dec_release(X) __atomic_fetch_op{RELEASE}(X,-,1)
atomic_xchg(X,V) __xchg{MB}(X,V)
atomic_xchg_relaxed(X,V) __xchg{ONCE}(X,V)
atomic_xchg_release(X,V) __xchg{RELEASE}(X,V)
atomic_xchg_acquire(X,V) __xchg{ACQUIRE}(X,V)
atomic_cmpxchg(X,V,W) __cmpxchg{MB}(X,V,W)
atomic_cmpxchg_relaxed(X,V,W) __cmpxchg{ONCE}(X,V,W)
atomic_cmpxchg_acquire(X,V,W) __cmpxchg{ACQUIRE}(X,V,W)
atomic_cmpxchg_release(X,V,W) __cmpxchg{RELEASE}(X,V,W)
atomic_sub_and_test(V,X) __atomic_op_return{MB}(X,-,V) == 0
atomic_dec_and_test(X) __atomic_op_return{MB}(X,-,1) == 0
atomic_inc_and_test(X) __atomic_op_return{MB}(X,+,1) == 0
atomic_add_negative(V,X) __atomic_op_return{MB}(X,+,V) < 0
atomic_add_negative_relaxed(V,X) __atomic_op_return{ONCE}(X,+,V) < 0
atomic_add_negative_acquire(V,X) __atomic_op_return{ACQUIRE}(X,+,V) < 0
atomic_add_negative_release(V,X) __atomic_op_return{RELEASE}(X,+,V) < 0
atomic_fetch_andnot(V,X) __atomic_fetch_op{MB}(X,&~,V)
atomic_fetch_andnot_acquire(V,X) __atomic_fetch_op{ACQUIRE}(X,&~,V)
atomic_fetch_andnot_release(V,X) __atomic_fetch_op{RELEASE}(X,&~,V)
atomic_fetch_andnot_relaxed(V,X) __atomic_fetch_op{ONCE}(X,&~,V)
atomic_add_unless(X,V,W) __atomic_add_unless{MB}(X,V,W)
|
