Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 :
3 : // Generated by scripts/atomic/gen-atomic-fallback.sh
4 : // DO NOT MODIFY THIS FILE DIRECTLY
5 :
6 : #ifndef _LINUX_ATOMIC_FALLBACK_H
7 : #define _LINUX_ATOMIC_FALLBACK_H
8 :
9 : #include <linux/compiler.h>
10 :
11 : #ifndef arch_xchg_relaxed
12 : #define arch_xchg_acquire arch_xchg
13 : #define arch_xchg_release arch_xchg
14 : #define arch_xchg_relaxed arch_xchg
15 : #else /* arch_xchg_relaxed */
16 :
17 : #ifndef arch_xchg_acquire
18 : #define arch_xchg_acquire(...) \
19 : __atomic_op_acquire(arch_xchg, __VA_ARGS__)
20 : #endif
21 :
22 : #ifndef arch_xchg_release
23 : #define arch_xchg_release(...) \
24 : __atomic_op_release(arch_xchg, __VA_ARGS__)
25 : #endif
26 :
27 : #ifndef arch_xchg
28 : #define arch_xchg(...) \
29 : __atomic_op_fence(arch_xchg, __VA_ARGS__)
30 : #endif
31 :
32 : #endif /* arch_xchg_relaxed */
33 :
34 : #ifndef arch_cmpxchg_relaxed
35 : #define arch_cmpxchg_acquire arch_cmpxchg
36 : #define arch_cmpxchg_release arch_cmpxchg
37 : #define arch_cmpxchg_relaxed arch_cmpxchg
38 : #else /* arch_cmpxchg_relaxed */
39 :
40 : #ifndef arch_cmpxchg_acquire
41 : #define arch_cmpxchg_acquire(...) \
42 : __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__)
43 : #endif
44 :
45 : #ifndef arch_cmpxchg_release
46 : #define arch_cmpxchg_release(...) \
47 : __atomic_op_release(arch_cmpxchg, __VA_ARGS__)
48 : #endif
49 :
50 : #ifndef arch_cmpxchg
51 : #define arch_cmpxchg(...) \
52 : __atomic_op_fence(arch_cmpxchg, __VA_ARGS__)
53 : #endif
54 :
55 : #endif /* arch_cmpxchg_relaxed */
56 :
57 : #ifndef arch_cmpxchg64_relaxed
58 : #define arch_cmpxchg64_acquire arch_cmpxchg64
59 : #define arch_cmpxchg64_release arch_cmpxchg64
60 : #define arch_cmpxchg64_relaxed arch_cmpxchg64
61 : #else /* arch_cmpxchg64_relaxed */
62 :
63 : #ifndef arch_cmpxchg64_acquire
64 : #define arch_cmpxchg64_acquire(...) \
65 : __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__)
66 : #endif
67 :
68 : #ifndef arch_cmpxchg64_release
69 : #define arch_cmpxchg64_release(...) \
70 : __atomic_op_release(arch_cmpxchg64, __VA_ARGS__)
71 : #endif
72 :
73 : #ifndef arch_cmpxchg64
74 : #define arch_cmpxchg64(...) \
75 : __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)
76 : #endif
77 :
78 : #endif /* arch_cmpxchg64_relaxed */
79 :
80 : #ifndef arch_try_cmpxchg_relaxed
81 : #ifdef arch_try_cmpxchg
82 : #define arch_try_cmpxchg_acquire arch_try_cmpxchg
83 : #define arch_try_cmpxchg_release arch_try_cmpxchg
84 : #define arch_try_cmpxchg_relaxed arch_try_cmpxchg
85 : #endif /* arch_try_cmpxchg */
86 :
87 : #ifndef arch_try_cmpxchg
88 : #define arch_try_cmpxchg(_ptr, _oldp, _new) \
89 : ({ \
90 : typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
91 : ___r = arch_cmpxchg((_ptr), ___o, (_new)); \
92 : if (unlikely(___r != ___o)) \
93 : *___op = ___r; \
94 : likely(___r == ___o); \
95 : })
96 : #endif /* arch_try_cmpxchg */
97 :
98 : #ifndef arch_try_cmpxchg_acquire
99 : #define arch_try_cmpxchg_acquire(_ptr, _oldp, _new) \
100 : ({ \
101 : typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
102 : ___r = arch_cmpxchg_acquire((_ptr), ___o, (_new)); \
103 : if (unlikely(___r != ___o)) \
104 : *___op = ___r; \
105 : likely(___r == ___o); \
106 : })
107 : #endif /* arch_try_cmpxchg_acquire */
108 :
109 : #ifndef arch_try_cmpxchg_release
110 : #define arch_try_cmpxchg_release(_ptr, _oldp, _new) \
111 : ({ \
112 : typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
113 : ___r = arch_cmpxchg_release((_ptr), ___o, (_new)); \
114 : if (unlikely(___r != ___o)) \
115 : *___op = ___r; \
116 : likely(___r == ___o); \
117 : })
118 : #endif /* arch_try_cmpxchg_release */
119 :
120 : #ifndef arch_try_cmpxchg_relaxed
121 : #define arch_try_cmpxchg_relaxed(_ptr, _oldp, _new) \
122 : ({ \
123 : typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
124 : ___r = arch_cmpxchg_relaxed((_ptr), ___o, (_new)); \
125 : if (unlikely(___r != ___o)) \
126 : *___op = ___r; \
127 : likely(___r == ___o); \
128 : })
129 : #endif /* arch_try_cmpxchg_relaxed */
130 :
131 : #else /* arch_try_cmpxchg_relaxed */
132 :
133 : #ifndef arch_try_cmpxchg_acquire
134 : #define arch_try_cmpxchg_acquire(...) \
135 : __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)
136 : #endif
137 :
138 : #ifndef arch_try_cmpxchg_release
139 : #define arch_try_cmpxchg_release(...) \
140 : __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)
141 : #endif
142 :
143 : #ifndef arch_try_cmpxchg
144 : #define arch_try_cmpxchg(...) \
145 : __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)
146 : #endif
147 :
148 : #endif /* arch_try_cmpxchg_relaxed */
149 :
150 : #ifndef arch_atomic_read_acquire
151 : static __always_inline int
152 : arch_atomic_read_acquire(const atomic_t *v)
153 : {
154 : int ret;
155 :
156 : if (__native_word(atomic_t)) {
157 0 : ret = smp_load_acquire(&(v)->counter);
158 : } else {
159 : ret = arch_atomic_read(v);
160 : __atomic_acquire_fence();
161 : }
162 :
163 : return ret;
164 : }
165 : #define arch_atomic_read_acquire arch_atomic_read_acquire
166 : #endif
167 :
168 : #ifndef arch_atomic_set_release
169 : static __always_inline void
170 : arch_atomic_set_release(atomic_t *v, int i)
171 : {
172 : if (__native_word(atomic_t)) {
173 0 : smp_store_release(&(v)->counter, i);
174 : } else {
175 : __atomic_release_fence();
176 : arch_atomic_set(v, i);
177 : }
178 : }
179 : #define arch_atomic_set_release arch_atomic_set_release
180 : #endif
181 :
182 : #ifndef arch_atomic_add_return_relaxed
183 : #define arch_atomic_add_return_acquire arch_atomic_add_return
184 : #define arch_atomic_add_return_release arch_atomic_add_return
185 : #define arch_atomic_add_return_relaxed arch_atomic_add_return
186 : #else /* arch_atomic_add_return_relaxed */
187 :
188 : #ifndef arch_atomic_add_return_acquire
189 : static __always_inline int
190 : arch_atomic_add_return_acquire(int i, atomic_t *v)
191 : {
192 : int ret = arch_atomic_add_return_relaxed(i, v);
193 : __atomic_acquire_fence();
194 : return ret;
195 : }
196 : #define arch_atomic_add_return_acquire arch_atomic_add_return_acquire
197 : #endif
198 :
199 : #ifndef arch_atomic_add_return_release
200 : static __always_inline int
201 : arch_atomic_add_return_release(int i, atomic_t *v)
202 : {
203 : __atomic_release_fence();
204 : return arch_atomic_add_return_relaxed(i, v);
205 : }
206 : #define arch_atomic_add_return_release arch_atomic_add_return_release
207 : #endif
208 :
209 : #ifndef arch_atomic_add_return
210 : static __always_inline int
211 : arch_atomic_add_return(int i, atomic_t *v)
212 : {
213 : int ret;
214 : __atomic_pre_full_fence();
215 : ret = arch_atomic_add_return_relaxed(i, v);
216 : __atomic_post_full_fence();
217 : return ret;
218 : }
219 : #define arch_atomic_add_return arch_atomic_add_return
220 : #endif
221 :
222 : #endif /* arch_atomic_add_return_relaxed */
223 :
224 : #ifndef arch_atomic_fetch_add_relaxed
225 : #define arch_atomic_fetch_add_acquire arch_atomic_fetch_add
226 : #define arch_atomic_fetch_add_release arch_atomic_fetch_add
227 : #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add
228 : #else /* arch_atomic_fetch_add_relaxed */
229 :
230 : #ifndef arch_atomic_fetch_add_acquire
231 : static __always_inline int
232 : arch_atomic_fetch_add_acquire(int i, atomic_t *v)
233 : {
234 : int ret = arch_atomic_fetch_add_relaxed(i, v);
235 : __atomic_acquire_fence();
236 : return ret;
237 : }
238 : #define arch_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire
239 : #endif
240 :
241 : #ifndef arch_atomic_fetch_add_release
242 : static __always_inline int
243 : arch_atomic_fetch_add_release(int i, atomic_t *v)
244 : {
245 : __atomic_release_fence();
246 : return arch_atomic_fetch_add_relaxed(i, v);
247 : }
248 : #define arch_atomic_fetch_add_release arch_atomic_fetch_add_release
249 : #endif
250 :
251 : #ifndef arch_atomic_fetch_add
252 : static __always_inline int
253 : arch_atomic_fetch_add(int i, atomic_t *v)
254 : {
255 : int ret;
256 : __atomic_pre_full_fence();
257 : ret = arch_atomic_fetch_add_relaxed(i, v);
258 : __atomic_post_full_fence();
259 : return ret;
260 : }
261 : #define arch_atomic_fetch_add arch_atomic_fetch_add
262 : #endif
263 :
264 : #endif /* arch_atomic_fetch_add_relaxed */
265 :
266 : #ifndef arch_atomic_sub_return_relaxed
267 : #define arch_atomic_sub_return_acquire arch_atomic_sub_return
268 : #define arch_atomic_sub_return_release arch_atomic_sub_return
269 : #define arch_atomic_sub_return_relaxed arch_atomic_sub_return
270 : #else /* arch_atomic_sub_return_relaxed */
271 :
272 : #ifndef arch_atomic_sub_return_acquire
273 : static __always_inline int
274 : arch_atomic_sub_return_acquire(int i, atomic_t *v)
275 : {
276 : int ret = arch_atomic_sub_return_relaxed(i, v);
277 : __atomic_acquire_fence();
278 : return ret;
279 : }
280 : #define arch_atomic_sub_return_acquire arch_atomic_sub_return_acquire
281 : #endif
282 :
283 : #ifndef arch_atomic_sub_return_release
284 : static __always_inline int
285 : arch_atomic_sub_return_release(int i, atomic_t *v)
286 : {
287 : __atomic_release_fence();
288 : return arch_atomic_sub_return_relaxed(i, v);
289 : }
290 : #define arch_atomic_sub_return_release arch_atomic_sub_return_release
291 : #endif
292 :
293 : #ifndef arch_atomic_sub_return
294 : static __always_inline int
295 : arch_atomic_sub_return(int i, atomic_t *v)
296 : {
297 : int ret;
298 : __atomic_pre_full_fence();
299 : ret = arch_atomic_sub_return_relaxed(i, v);
300 : __atomic_post_full_fence();
301 : return ret;
302 : }
303 : #define arch_atomic_sub_return arch_atomic_sub_return
304 : #endif
305 :
306 : #endif /* arch_atomic_sub_return_relaxed */
307 :
308 : #ifndef arch_atomic_fetch_sub_relaxed
309 : #define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub
310 : #define arch_atomic_fetch_sub_release arch_atomic_fetch_sub
311 : #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub
312 : #else /* arch_atomic_fetch_sub_relaxed */
313 :
314 : #ifndef arch_atomic_fetch_sub_acquire
315 : static __always_inline int
316 : arch_atomic_fetch_sub_acquire(int i, atomic_t *v)
317 : {
318 : int ret = arch_atomic_fetch_sub_relaxed(i, v);
319 : __atomic_acquire_fence();
320 : return ret;
321 : }
322 : #define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire
323 : #endif
324 :
325 : #ifndef arch_atomic_fetch_sub_release
326 : static __always_inline int
327 : arch_atomic_fetch_sub_release(int i, atomic_t *v)
328 : {
329 : __atomic_release_fence();
330 : return arch_atomic_fetch_sub_relaxed(i, v);
331 : }
332 : #define arch_atomic_fetch_sub_release arch_atomic_fetch_sub_release
333 : #endif
334 :
335 : #ifndef arch_atomic_fetch_sub
336 : static __always_inline int
337 : arch_atomic_fetch_sub(int i, atomic_t *v)
338 : {
339 : int ret;
340 : __atomic_pre_full_fence();
341 : ret = arch_atomic_fetch_sub_relaxed(i, v);
342 : __atomic_post_full_fence();
343 : return ret;
344 : }
345 : #define arch_atomic_fetch_sub arch_atomic_fetch_sub
346 : #endif
347 :
348 : #endif /* arch_atomic_fetch_sub_relaxed */
349 :
350 : #ifndef arch_atomic_inc
351 : static __always_inline void
352 : arch_atomic_inc(atomic_t *v)
353 : {
354 : arch_atomic_add(1, v);
355 : }
356 : #define arch_atomic_inc arch_atomic_inc
357 : #endif
358 :
359 : #ifndef arch_atomic_inc_return_relaxed
360 : #ifdef arch_atomic_inc_return
361 : #define arch_atomic_inc_return_acquire arch_atomic_inc_return
362 : #define arch_atomic_inc_return_release arch_atomic_inc_return
363 : #define arch_atomic_inc_return_relaxed arch_atomic_inc_return
364 : #endif /* arch_atomic_inc_return */
365 :
366 : #ifndef arch_atomic_inc_return
367 : static __always_inline int
368 : arch_atomic_inc_return(atomic_t *v)
369 : {
370 0 : return arch_atomic_add_return(1, v);
371 : }
372 : #define arch_atomic_inc_return arch_atomic_inc_return
373 : #endif
374 :
375 : #ifndef arch_atomic_inc_return_acquire
376 : static __always_inline int
377 : arch_atomic_inc_return_acquire(atomic_t *v)
378 : {
379 : return arch_atomic_add_return_acquire(1, v);
380 : }
381 : #define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
382 : #endif
383 :
384 : #ifndef arch_atomic_inc_return_release
385 : static __always_inline int
386 : arch_atomic_inc_return_release(atomic_t *v)
387 : {
388 : return arch_atomic_add_return_release(1, v);
389 : }
390 : #define arch_atomic_inc_return_release arch_atomic_inc_return_release
391 : #endif
392 :
393 : #ifndef arch_atomic_inc_return_relaxed
394 : static __always_inline int
395 : arch_atomic_inc_return_relaxed(atomic_t *v)
396 : {
397 : return arch_atomic_add_return_relaxed(1, v);
398 : }
399 : #define arch_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed
400 : #endif
401 :
402 : #else /* arch_atomic_inc_return_relaxed */
403 :
404 : #ifndef arch_atomic_inc_return_acquire
405 : static __always_inline int
406 : arch_atomic_inc_return_acquire(atomic_t *v)
407 : {
408 : int ret = arch_atomic_inc_return_relaxed(v);
409 : __atomic_acquire_fence();
410 : return ret;
411 : }
412 : #define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
413 : #endif
414 :
415 : #ifndef arch_atomic_inc_return_release
416 : static __always_inline int
417 : arch_atomic_inc_return_release(atomic_t *v)
418 : {
419 : __atomic_release_fence();
420 : return arch_atomic_inc_return_relaxed(v);
421 : }
422 : #define arch_atomic_inc_return_release arch_atomic_inc_return_release
423 : #endif
424 :
425 : #ifndef arch_atomic_inc_return
426 : static __always_inline int
427 : arch_atomic_inc_return(atomic_t *v)
428 : {
429 : int ret;
430 : __atomic_pre_full_fence();
431 : ret = arch_atomic_inc_return_relaxed(v);
432 : __atomic_post_full_fence();
433 : return ret;
434 : }
435 : #define arch_atomic_inc_return arch_atomic_inc_return
436 : #endif
437 :
438 : #endif /* arch_atomic_inc_return_relaxed */
439 :
440 : #ifndef arch_atomic_fetch_inc_relaxed
441 : #ifdef arch_atomic_fetch_inc
442 : #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc
443 : #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc
444 : #define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc
445 : #endif /* arch_atomic_fetch_inc */
446 :
447 : #ifndef arch_atomic_fetch_inc
448 : static __always_inline int
449 : arch_atomic_fetch_inc(atomic_t *v)
450 : {
451 0 : return arch_atomic_fetch_add(1, v);
452 : }
453 : #define arch_atomic_fetch_inc arch_atomic_fetch_inc
454 : #endif
455 :
456 : #ifndef arch_atomic_fetch_inc_acquire
457 : static __always_inline int
458 : arch_atomic_fetch_inc_acquire(atomic_t *v)
459 : {
460 : return arch_atomic_fetch_add_acquire(1, v);
461 : }
462 : #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
463 : #endif
464 :
465 : #ifndef arch_atomic_fetch_inc_release
466 : static __always_inline int
467 : arch_atomic_fetch_inc_release(atomic_t *v)
468 : {
469 : return arch_atomic_fetch_add_release(1, v);
470 : }
471 : #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
472 : #endif
473 :
474 : #ifndef arch_atomic_fetch_inc_relaxed
475 : static __always_inline int
476 : arch_atomic_fetch_inc_relaxed(atomic_t *v)
477 : {
478 : return arch_atomic_fetch_add_relaxed(1, v);
479 : }
480 : #define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc_relaxed
481 : #endif
482 :
483 : #else /* arch_atomic_fetch_inc_relaxed */
484 :
485 : #ifndef arch_atomic_fetch_inc_acquire
486 : static __always_inline int
487 : arch_atomic_fetch_inc_acquire(atomic_t *v)
488 : {
489 : int ret = arch_atomic_fetch_inc_relaxed(v);
490 : __atomic_acquire_fence();
491 : return ret;
492 : }
493 : #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
494 : #endif
495 :
496 : #ifndef arch_atomic_fetch_inc_release
497 : static __always_inline int
498 : arch_atomic_fetch_inc_release(atomic_t *v)
499 : {
500 : __atomic_release_fence();
501 : return arch_atomic_fetch_inc_relaxed(v);
502 : }
503 : #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
504 : #endif
505 :
506 : #ifndef arch_atomic_fetch_inc
507 : static __always_inline int
508 : arch_atomic_fetch_inc(atomic_t *v)
509 : {
510 : int ret;
511 : __atomic_pre_full_fence();
512 : ret = arch_atomic_fetch_inc_relaxed(v);
513 : __atomic_post_full_fence();
514 : return ret;
515 : }
516 : #define arch_atomic_fetch_inc arch_atomic_fetch_inc
517 : #endif
518 :
519 : #endif /* arch_atomic_fetch_inc_relaxed */
520 :
521 : #ifndef arch_atomic_dec
522 : static __always_inline void
523 : arch_atomic_dec(atomic_t *v)
524 : {
525 : arch_atomic_sub(1, v);
526 : }
527 : #define arch_atomic_dec arch_atomic_dec
528 : #endif
529 :
530 : #ifndef arch_atomic_dec_return_relaxed
531 : #ifdef arch_atomic_dec_return
532 : #define arch_atomic_dec_return_acquire arch_atomic_dec_return
533 : #define arch_atomic_dec_return_release arch_atomic_dec_return
534 : #define arch_atomic_dec_return_relaxed arch_atomic_dec_return
535 : #endif /* arch_atomic_dec_return */
536 :
537 : #ifndef arch_atomic_dec_return
538 : static __always_inline int
539 : arch_atomic_dec_return(atomic_t *v)
540 : {
541 0 : return arch_atomic_sub_return(1, v);
542 : }
543 : #define arch_atomic_dec_return arch_atomic_dec_return
544 : #endif
545 :
546 : #ifndef arch_atomic_dec_return_acquire
547 : static __always_inline int
548 : arch_atomic_dec_return_acquire(atomic_t *v)
549 : {
550 : return arch_atomic_sub_return_acquire(1, v);
551 : }
552 : #define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
553 : #endif
554 :
555 : #ifndef arch_atomic_dec_return_release
556 : static __always_inline int
557 : arch_atomic_dec_return_release(atomic_t *v)
558 : {
559 : return arch_atomic_sub_return_release(1, v);
560 : }
561 : #define arch_atomic_dec_return_release arch_atomic_dec_return_release
562 : #endif
563 :
564 : #ifndef arch_atomic_dec_return_relaxed
565 : static __always_inline int
566 : arch_atomic_dec_return_relaxed(atomic_t *v)
567 : {
568 : return arch_atomic_sub_return_relaxed(1, v);
569 : }
570 : #define arch_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed
571 : #endif
572 :
573 : #else /* arch_atomic_dec_return_relaxed */
574 :
575 : #ifndef arch_atomic_dec_return_acquire
576 : static __always_inline int
577 : arch_atomic_dec_return_acquire(atomic_t *v)
578 : {
579 : int ret = arch_atomic_dec_return_relaxed(v);
580 : __atomic_acquire_fence();
581 : return ret;
582 : }
583 : #define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
584 : #endif
585 :
586 : #ifndef arch_atomic_dec_return_release
587 : static __always_inline int
588 : arch_atomic_dec_return_release(atomic_t *v)
589 : {
590 : __atomic_release_fence();
591 : return arch_atomic_dec_return_relaxed(v);
592 : }
593 : #define arch_atomic_dec_return_release arch_atomic_dec_return_release
594 : #endif
595 :
596 : #ifndef arch_atomic_dec_return
597 : static __always_inline int
598 : arch_atomic_dec_return(atomic_t *v)
599 : {
600 : int ret;
601 : __atomic_pre_full_fence();
602 : ret = arch_atomic_dec_return_relaxed(v);
603 : __atomic_post_full_fence();
604 : return ret;
605 : }
606 : #define arch_atomic_dec_return arch_atomic_dec_return
607 : #endif
608 :
609 : #endif /* arch_atomic_dec_return_relaxed */
610 :
611 : #ifndef arch_atomic_fetch_dec_relaxed
612 : #ifdef arch_atomic_fetch_dec
613 : #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec
614 : #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec
615 : #define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec
616 : #endif /* arch_atomic_fetch_dec */
617 :
618 : #ifndef arch_atomic_fetch_dec
619 : static __always_inline int
620 : arch_atomic_fetch_dec(atomic_t *v)
621 : {
622 : return arch_atomic_fetch_sub(1, v);
623 : }
624 : #define arch_atomic_fetch_dec arch_atomic_fetch_dec
625 : #endif
626 :
627 : #ifndef arch_atomic_fetch_dec_acquire
628 : static __always_inline int
629 : arch_atomic_fetch_dec_acquire(atomic_t *v)
630 : {
631 : return arch_atomic_fetch_sub_acquire(1, v);
632 : }
633 : #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
634 : #endif
635 :
636 : #ifndef arch_atomic_fetch_dec_release
637 : static __always_inline int
638 : arch_atomic_fetch_dec_release(atomic_t *v)
639 : {
640 : return arch_atomic_fetch_sub_release(1, v);
641 : }
642 : #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
643 : #endif
644 :
645 : #ifndef arch_atomic_fetch_dec_relaxed
646 : static __always_inline int
647 : arch_atomic_fetch_dec_relaxed(atomic_t *v)
648 : {
649 : return arch_atomic_fetch_sub_relaxed(1, v);
650 : }
651 : #define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec_relaxed
652 : #endif
653 :
654 : #else /* arch_atomic_fetch_dec_relaxed */
655 :
656 : #ifndef arch_atomic_fetch_dec_acquire
657 : static __always_inline int
658 : arch_atomic_fetch_dec_acquire(atomic_t *v)
659 : {
660 : int ret = arch_atomic_fetch_dec_relaxed(v);
661 : __atomic_acquire_fence();
662 : return ret;
663 : }
664 : #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
665 : #endif
666 :
667 : #ifndef arch_atomic_fetch_dec_release
668 : static __always_inline int
669 : arch_atomic_fetch_dec_release(atomic_t *v)
670 : {
671 : __atomic_release_fence();
672 : return arch_atomic_fetch_dec_relaxed(v);
673 : }
674 : #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
675 : #endif
676 :
677 : #ifndef arch_atomic_fetch_dec
678 : static __always_inline int
679 : arch_atomic_fetch_dec(atomic_t *v)
680 : {
681 : int ret;
682 : __atomic_pre_full_fence();
683 : ret = arch_atomic_fetch_dec_relaxed(v);
684 : __atomic_post_full_fence();
685 : return ret;
686 : }
687 : #define arch_atomic_fetch_dec arch_atomic_fetch_dec
688 : #endif
689 :
690 : #endif /* arch_atomic_fetch_dec_relaxed */
691 :
692 : #ifndef arch_atomic_fetch_and_relaxed
693 : #define arch_atomic_fetch_and_acquire arch_atomic_fetch_and
694 : #define arch_atomic_fetch_and_release arch_atomic_fetch_and
695 : #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and
696 : #else /* arch_atomic_fetch_and_relaxed */
697 :
698 : #ifndef arch_atomic_fetch_and_acquire
699 : static __always_inline int
700 : arch_atomic_fetch_and_acquire(int i, atomic_t *v)
701 : {
702 : int ret = arch_atomic_fetch_and_relaxed(i, v);
703 : __atomic_acquire_fence();
704 : return ret;
705 : }
706 : #define arch_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire
707 : #endif
708 :
709 : #ifndef arch_atomic_fetch_and_release
710 : static __always_inline int
711 : arch_atomic_fetch_and_release(int i, atomic_t *v)
712 : {
713 : __atomic_release_fence();
714 : return arch_atomic_fetch_and_relaxed(i, v);
715 : }
716 : #define arch_atomic_fetch_and_release arch_atomic_fetch_and_release
717 : #endif
718 :
719 : #ifndef arch_atomic_fetch_and
720 : static __always_inline int
721 : arch_atomic_fetch_and(int i, atomic_t *v)
722 : {
723 : int ret;
724 : __atomic_pre_full_fence();
725 : ret = arch_atomic_fetch_and_relaxed(i, v);
726 : __atomic_post_full_fence();
727 : return ret;
728 : }
729 : #define arch_atomic_fetch_and arch_atomic_fetch_and
730 : #endif
731 :
732 : #endif /* arch_atomic_fetch_and_relaxed */
733 :
734 : #ifndef arch_atomic_andnot
735 : static __always_inline void
736 : arch_atomic_andnot(int i, atomic_t *v)
737 : {
738 : arch_atomic_and(~i, v);
739 : }
740 : #define arch_atomic_andnot arch_atomic_andnot
741 : #endif
742 :
743 : #ifndef arch_atomic_fetch_andnot_relaxed
744 : #ifdef arch_atomic_fetch_andnot
745 : #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot
746 : #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot
747 : #define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot
748 : #endif /* arch_atomic_fetch_andnot */
749 :
750 : #ifndef arch_atomic_fetch_andnot
751 : static __always_inline int
752 : arch_atomic_fetch_andnot(int i, atomic_t *v)
753 : {
754 : return arch_atomic_fetch_and(~i, v);
755 : }
756 : #define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
757 : #endif
758 :
759 : #ifndef arch_atomic_fetch_andnot_acquire
760 : static __always_inline int
761 : arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
762 : {
763 : return arch_atomic_fetch_and_acquire(~i, v);
764 : }
765 : #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
766 : #endif
767 :
768 : #ifndef arch_atomic_fetch_andnot_release
769 : static __always_inline int
770 : arch_atomic_fetch_andnot_release(int i, atomic_t *v)
771 : {
772 : return arch_atomic_fetch_and_release(~i, v);
773 : }
774 : #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
775 : #endif
776 :
777 : #ifndef arch_atomic_fetch_andnot_relaxed
778 : static __always_inline int
779 : arch_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
780 : {
781 : return arch_atomic_fetch_and_relaxed(~i, v);
782 : }
783 : #define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed
784 : #endif
785 :
786 : #else /* arch_atomic_fetch_andnot_relaxed */
787 :
788 : #ifndef arch_atomic_fetch_andnot_acquire
789 : static __always_inline int
790 : arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
791 : {
792 : int ret = arch_atomic_fetch_andnot_relaxed(i, v);
793 : __atomic_acquire_fence();
794 : return ret;
795 : }
796 : #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
797 : #endif
798 :
799 : #ifndef arch_atomic_fetch_andnot_release
800 : static __always_inline int
801 : arch_atomic_fetch_andnot_release(int i, atomic_t *v)
802 : {
803 : __atomic_release_fence();
804 : return arch_atomic_fetch_andnot_relaxed(i, v);
805 : }
806 : #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
807 : #endif
808 :
809 : #ifndef arch_atomic_fetch_andnot
810 : static __always_inline int
811 : arch_atomic_fetch_andnot(int i, atomic_t *v)
812 : {
813 : int ret;
814 : __atomic_pre_full_fence();
815 : ret = arch_atomic_fetch_andnot_relaxed(i, v);
816 : __atomic_post_full_fence();
817 : return ret;
818 : }
819 : #define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
820 : #endif
821 :
822 : #endif /* arch_atomic_fetch_andnot_relaxed */
823 :
824 : #ifndef arch_atomic_fetch_or_relaxed
825 : #define arch_atomic_fetch_or_acquire arch_atomic_fetch_or
826 : #define arch_atomic_fetch_or_release arch_atomic_fetch_or
827 : #define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or
828 : #else /* arch_atomic_fetch_or_relaxed */
829 :
830 : #ifndef arch_atomic_fetch_or_acquire
831 : static __always_inline int
832 : arch_atomic_fetch_or_acquire(int i, atomic_t *v)
833 : {
834 : int ret = arch_atomic_fetch_or_relaxed(i, v);
835 : __atomic_acquire_fence();
836 : return ret;
837 : }
838 : #define arch_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire
839 : #endif
840 :
841 : #ifndef arch_atomic_fetch_or_release
842 : static __always_inline int
843 : arch_atomic_fetch_or_release(int i, atomic_t *v)
844 : {
845 : __atomic_release_fence();
846 : return arch_atomic_fetch_or_relaxed(i, v);
847 : }
848 : #define arch_atomic_fetch_or_release arch_atomic_fetch_or_release
849 : #endif
850 :
851 : #ifndef arch_atomic_fetch_or
852 : static __always_inline int
853 : arch_atomic_fetch_or(int i, atomic_t *v)
854 : {
855 : int ret;
856 : __atomic_pre_full_fence();
857 : ret = arch_atomic_fetch_or_relaxed(i, v);
858 : __atomic_post_full_fence();
859 : return ret;
860 : }
861 : #define arch_atomic_fetch_or arch_atomic_fetch_or
862 : #endif
863 :
864 : #endif /* arch_atomic_fetch_or_relaxed */
865 :
866 : #ifndef arch_atomic_fetch_xor_relaxed
867 : #define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor
868 : #define arch_atomic_fetch_xor_release arch_atomic_fetch_xor
869 : #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor
870 : #else /* arch_atomic_fetch_xor_relaxed */
871 :
872 : #ifndef arch_atomic_fetch_xor_acquire
873 : static __always_inline int
874 : arch_atomic_fetch_xor_acquire(int i, atomic_t *v)
875 : {
876 : int ret = arch_atomic_fetch_xor_relaxed(i, v);
877 : __atomic_acquire_fence();
878 : return ret;
879 : }
880 : #define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire
881 : #endif
882 :
883 : #ifndef arch_atomic_fetch_xor_release
884 : static __always_inline int
885 : arch_atomic_fetch_xor_release(int i, atomic_t *v)
886 : {
887 : __atomic_release_fence();
888 : return arch_atomic_fetch_xor_relaxed(i, v);
889 : }
890 : #define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release
891 : #endif
892 :
893 : #ifndef arch_atomic_fetch_xor
894 : static __always_inline int
895 : arch_atomic_fetch_xor(int i, atomic_t *v)
896 : {
897 : int ret;
898 : __atomic_pre_full_fence();
899 : ret = arch_atomic_fetch_xor_relaxed(i, v);
900 : __atomic_post_full_fence();
901 : return ret;
902 : }
903 : #define arch_atomic_fetch_xor arch_atomic_fetch_xor
904 : #endif
905 :
906 : #endif /* arch_atomic_fetch_xor_relaxed */
907 :
908 : #ifndef arch_atomic_xchg_relaxed
909 : #define arch_atomic_xchg_acquire arch_atomic_xchg
910 : #define arch_atomic_xchg_release arch_atomic_xchg
911 : #define arch_atomic_xchg_relaxed arch_atomic_xchg
912 : #else /* arch_atomic_xchg_relaxed */
913 :
914 : #ifndef arch_atomic_xchg_acquire
915 : static __always_inline int
916 : arch_atomic_xchg_acquire(atomic_t *v, int i)
917 : {
918 : int ret = arch_atomic_xchg_relaxed(v, i);
919 : __atomic_acquire_fence();
920 : return ret;
921 : }
922 : #define arch_atomic_xchg_acquire arch_atomic_xchg_acquire
923 : #endif
924 :
925 : #ifndef arch_atomic_xchg_release
926 : static __always_inline int
927 : arch_atomic_xchg_release(atomic_t *v, int i)
928 : {
929 : __atomic_release_fence();
930 : return arch_atomic_xchg_relaxed(v, i);
931 : }
932 : #define arch_atomic_xchg_release arch_atomic_xchg_release
933 : #endif
934 :
935 : #ifndef arch_atomic_xchg
936 : static __always_inline int
937 : arch_atomic_xchg(atomic_t *v, int i)
938 : {
939 : int ret;
940 : __atomic_pre_full_fence();
941 : ret = arch_atomic_xchg_relaxed(v, i);
942 : __atomic_post_full_fence();
943 : return ret;
944 : }
945 : #define arch_atomic_xchg arch_atomic_xchg
946 : #endif
947 :
948 : #endif /* arch_atomic_xchg_relaxed */
949 :
950 : #ifndef arch_atomic_cmpxchg_relaxed
951 : #define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg
952 : #define arch_atomic_cmpxchg_release arch_atomic_cmpxchg
953 : #define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg
954 : #else /* arch_atomic_cmpxchg_relaxed */
955 :
956 : #ifndef arch_atomic_cmpxchg_acquire
957 : static __always_inline int
958 : arch_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
959 : {
960 : int ret = arch_atomic_cmpxchg_relaxed(v, old, new);
961 : __atomic_acquire_fence();
962 : return ret;
963 : }
964 : #define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire
965 : #endif
966 :
967 : #ifndef arch_atomic_cmpxchg_release
968 : static __always_inline int
969 : arch_atomic_cmpxchg_release(atomic_t *v, int old, int new)
970 : {
971 : __atomic_release_fence();
972 : return arch_atomic_cmpxchg_relaxed(v, old, new);
973 : }
974 : #define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release
975 : #endif
976 :
977 : #ifndef arch_atomic_cmpxchg
978 : static __always_inline int
979 : arch_atomic_cmpxchg(atomic_t *v, int old, int new)
980 : {
981 : int ret;
982 : __atomic_pre_full_fence();
983 : ret = arch_atomic_cmpxchg_relaxed(v, old, new);
984 : __atomic_post_full_fence();
985 : return ret;
986 : }
987 : #define arch_atomic_cmpxchg arch_atomic_cmpxchg
988 : #endif
989 :
990 : #endif /* arch_atomic_cmpxchg_relaxed */
991 :
992 : #ifndef arch_atomic_try_cmpxchg_relaxed
993 : #ifdef arch_atomic_try_cmpxchg
994 : #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg
995 : #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg
996 : #define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg
997 : #endif /* arch_atomic_try_cmpxchg */
998 :
999 : #ifndef arch_atomic_try_cmpxchg
1000 : static __always_inline bool
1001 : arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
1002 : {
1003 : int r, o = *old;
1004 : r = arch_atomic_cmpxchg(v, o, new);
1005 : if (unlikely(r != o))
1006 : *old = r;
1007 : return likely(r == o);
1008 : }
1009 : #define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
1010 : #endif
1011 :
1012 : #ifndef arch_atomic_try_cmpxchg_acquire
1013 : static __always_inline bool
1014 : arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
1015 : {
1016 : int r, o = *old;
1017 : r = arch_atomic_cmpxchg_acquire(v, o, new);
1018 : if (unlikely(r != o))
1019 : *old = r;
1020 : return likely(r == o);
1021 : }
1022 : #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
1023 : #endif
1024 :
1025 : #ifndef arch_atomic_try_cmpxchg_release
1026 : static __always_inline bool
1027 : arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
1028 : {
1029 : int r, o = *old;
1030 : r = arch_atomic_cmpxchg_release(v, o, new);
1031 : if (unlikely(r != o))
1032 : *old = r;
1033 : return likely(r == o);
1034 : }
1035 : #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
1036 : #endif
1037 :
1038 : #ifndef arch_atomic_try_cmpxchg_relaxed
1039 : static __always_inline bool
1040 : arch_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
1041 : {
1042 : int r, o = *old;
1043 : r = arch_atomic_cmpxchg_relaxed(v, o, new);
1044 : if (unlikely(r != o))
1045 : *old = r;
1046 : return likely(r == o);
1047 : }
1048 : #define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg_relaxed
1049 : #endif
1050 :
1051 : #else /* arch_atomic_try_cmpxchg_relaxed */
1052 :
1053 : #ifndef arch_atomic_try_cmpxchg_acquire
1054 : static __always_inline bool
1055 : arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
1056 : {
1057 : bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
1058 : __atomic_acquire_fence();
1059 : return ret;
1060 : }
1061 : #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
1062 : #endif
1063 :
1064 : #ifndef arch_atomic_try_cmpxchg_release
1065 : static __always_inline bool
1066 : arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
1067 : {
1068 : __atomic_release_fence();
1069 : return arch_atomic_try_cmpxchg_relaxed(v, old, new);
1070 : }
1071 : #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
1072 : #endif
1073 :
1074 : #ifndef arch_atomic_try_cmpxchg
1075 : static __always_inline bool
1076 : arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
1077 : {
1078 : bool ret;
1079 : __atomic_pre_full_fence();
1080 : ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
1081 : __atomic_post_full_fence();
1082 : return ret;
1083 : }
1084 : #define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
1085 : #endif
1086 :
1087 : #endif /* arch_atomic_try_cmpxchg_relaxed */
1088 :
1089 : #ifndef arch_atomic_sub_and_test
1090 : /**
1091 : * arch_atomic_sub_and_test - subtract value from variable and test result
1092 : * @i: integer value to subtract
1093 : * @v: pointer of type atomic_t
1094 : *
1095 : * Atomically subtracts @i from @v and returns
1096 : * true if the result is zero, or false for all
1097 : * other cases.
1098 : */
1099 : static __always_inline bool
1100 : arch_atomic_sub_and_test(int i, atomic_t *v)
1101 : {
1102 : return arch_atomic_sub_return(i, v) == 0;
1103 : }
1104 : #define arch_atomic_sub_and_test arch_atomic_sub_and_test
1105 : #endif
1106 :
1107 : #ifndef arch_atomic_dec_and_test
1108 : /**
1109 : * arch_atomic_dec_and_test - decrement and test
1110 : * @v: pointer of type atomic_t
1111 : *
1112 : * Atomically decrements @v by 1 and
1113 : * returns true if the result is 0, or false for all other
1114 : * cases.
1115 : */
1116 : static __always_inline bool
1117 : arch_atomic_dec_and_test(atomic_t *v)
1118 : {
1119 : return arch_atomic_dec_return(v) == 0;
1120 : }
1121 : #define arch_atomic_dec_and_test arch_atomic_dec_and_test
1122 : #endif
1123 :
1124 : #ifndef arch_atomic_inc_and_test
1125 : /**
1126 : * arch_atomic_inc_and_test - increment and test
1127 : * @v: pointer of type atomic_t
1128 : *
1129 : * Atomically increments @v by 1
1130 : * and returns true if the result is zero, or false for all
1131 : * other cases.
1132 : */
1133 : static __always_inline bool
1134 : arch_atomic_inc_and_test(atomic_t *v)
1135 : {
1136 : return arch_atomic_inc_return(v) == 0;
1137 : }
1138 : #define arch_atomic_inc_and_test arch_atomic_inc_and_test
1139 : #endif
1140 :
1141 : #ifndef arch_atomic_add_negative
1142 : /**
1143 : * arch_atomic_add_negative - add and test if negative
1144 : * @i: integer value to add
1145 : * @v: pointer of type atomic_t
1146 : *
1147 : * Atomically adds @i to @v and returns true
1148 : * if the result is negative, or false when
1149 : * result is greater than or equal to zero.
1150 : */
1151 : static __always_inline bool
1152 : arch_atomic_add_negative(int i, atomic_t *v)
1153 : {
1154 : return arch_atomic_add_return(i, v) < 0;
1155 : }
1156 : #define arch_atomic_add_negative arch_atomic_add_negative
1157 : #endif
1158 :
1159 : #ifndef arch_atomic_fetch_add_unless
1160 : /**
1161 : * arch_atomic_fetch_add_unless - add unless the number is already a given value
1162 : * @v: pointer of type atomic_t
1163 : * @a: the amount to add to v...
1164 : * @u: ...unless v is equal to u.
1165 : *
1166 : * Atomically adds @a to @v, so long as @v was not already @u.
1167 : * Returns original value of @v
1168 : */
1169 : static __always_inline int
1170 : arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
1171 : {
1172 638 : int c = arch_atomic_read(v);
1173 :
1174 : do {
1175 638 : if (unlikely(c == u))
1176 : break;
1177 1276 : } while (!arch_atomic_try_cmpxchg(v, &c, c + a));
1178 :
1179 : return c;
1180 : }
1181 : #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
1182 : #endif
1183 :
1184 : #ifndef arch_atomic_add_unless
1185 : /**
1186 : * arch_atomic_add_unless - add unless the number is already a given value
1187 : * @v: pointer of type atomic_t
1188 : * @a: the amount to add to v...
1189 : * @u: ...unless v is equal to u.
1190 : *
1191 : * Atomically adds @a to @v, if @v was not already @u.
1192 : * Returns true if the addition was done.
1193 : */
1194 : static __always_inline bool
1195 : arch_atomic_add_unless(atomic_t *v, int a, int u)
1196 : {
1197 638 : return arch_atomic_fetch_add_unless(v, a, u) != u;
1198 : }
1199 : #define arch_atomic_add_unless arch_atomic_add_unless
1200 : #endif
1201 :
1202 : #ifndef arch_atomic_inc_not_zero
1203 : /**
1204 : * arch_atomic_inc_not_zero - increment unless the number is zero
1205 : * @v: pointer of type atomic_t
1206 : *
1207 : * Atomically increments @v by 1, if @v is non-zero.
1208 : * Returns true if the increment was done.
1209 : */
1210 : static __always_inline bool
1211 : arch_atomic_inc_not_zero(atomic_t *v)
1212 : {
1213 0 : return arch_atomic_add_unless(v, 1, 0);
1214 : }
1215 : #define arch_atomic_inc_not_zero arch_atomic_inc_not_zero
1216 : #endif
1217 :
1218 : #ifndef arch_atomic_inc_unless_negative
1219 : static __always_inline bool
1220 : arch_atomic_inc_unless_negative(atomic_t *v)
1221 : {
1222 0 : int c = arch_atomic_read(v);
1223 :
1224 : do {
1225 0 : if (unlikely(c < 0))
1226 : return false;
1227 0 : } while (!arch_atomic_try_cmpxchg(v, &c, c + 1));
1228 :
1229 : return true;
1230 : }
1231 : #define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative
1232 : #endif
1233 :
1234 : #ifndef arch_atomic_dec_unless_positive
1235 : static __always_inline bool
1236 : arch_atomic_dec_unless_positive(atomic_t *v)
1237 : {
1238 0 : int c = arch_atomic_read(v);
1239 :
1240 : do {
1241 0 : if (unlikely(c > 0))
1242 : return false;
1243 0 : } while (!arch_atomic_try_cmpxchg(v, &c, c - 1));
1244 :
1245 : return true;
1246 : }
1247 : #define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive
1248 : #endif
1249 :
1250 : #ifndef arch_atomic_dec_if_positive
1251 : static __always_inline int
1252 : arch_atomic_dec_if_positive(atomic_t *v)
1253 : {
1254 0 : int dec, c = arch_atomic_read(v);
1255 :
1256 : do {
1257 0 : dec = c - 1;
1258 0 : if (unlikely(dec < 0))
1259 : break;
1260 0 : } while (!arch_atomic_try_cmpxchg(v, &c, dec));
1261 :
1262 : return dec;
1263 : }
1264 : #define arch_atomic_dec_if_positive arch_atomic_dec_if_positive
1265 : #endif
1266 :
1267 : #ifdef CONFIG_GENERIC_ATOMIC64
1268 : #include <asm-generic/atomic64.h>
1269 : #endif
1270 :
1271 : #ifndef arch_atomic64_read_acquire
1272 : static __always_inline s64
1273 : arch_atomic64_read_acquire(const atomic64_t *v)
1274 : {
1275 : s64 ret;
1276 :
1277 : if (__native_word(atomic64_t)) {
1278 : ret = smp_load_acquire(&(v)->counter);
1279 : } else {
1280 : ret = arch_atomic64_read(v);
1281 : __atomic_acquire_fence();
1282 : }
1283 :
1284 : return ret;
1285 : }
1286 : #define arch_atomic64_read_acquire arch_atomic64_read_acquire
1287 : #endif
1288 :
1289 : #ifndef arch_atomic64_set_release
1290 : static __always_inline void
1291 : arch_atomic64_set_release(atomic64_t *v, s64 i)
1292 : {
1293 : if (__native_word(atomic64_t)) {
1294 : smp_store_release(&(v)->counter, i);
1295 : } else {
1296 : __atomic_release_fence();
1297 : arch_atomic64_set(v, i);
1298 : }
1299 : }
1300 : #define arch_atomic64_set_release arch_atomic64_set_release
1301 : #endif
1302 :
1303 : #ifndef arch_atomic64_add_return_relaxed
1304 : #define arch_atomic64_add_return_acquire arch_atomic64_add_return
1305 : #define arch_atomic64_add_return_release arch_atomic64_add_return
1306 : #define arch_atomic64_add_return_relaxed arch_atomic64_add_return
1307 : #else /* arch_atomic64_add_return_relaxed */
1308 :
1309 : #ifndef arch_atomic64_add_return_acquire
1310 : static __always_inline s64
1311 : arch_atomic64_add_return_acquire(s64 i, atomic64_t *v)
1312 : {
1313 : s64 ret = arch_atomic64_add_return_relaxed(i, v);
1314 : __atomic_acquire_fence();
1315 : return ret;
1316 : }
1317 : #define arch_atomic64_add_return_acquire arch_atomic64_add_return_acquire
1318 : #endif
1319 :
1320 : #ifndef arch_atomic64_add_return_release
1321 : static __always_inline s64
1322 : arch_atomic64_add_return_release(s64 i, atomic64_t *v)
1323 : {
1324 : __atomic_release_fence();
1325 : return arch_atomic64_add_return_relaxed(i, v);
1326 : }
1327 : #define arch_atomic64_add_return_release arch_atomic64_add_return_release
1328 : #endif
1329 :
1330 : #ifndef arch_atomic64_add_return
1331 : static __always_inline s64
1332 : arch_atomic64_add_return(s64 i, atomic64_t *v)
1333 : {
1334 : s64 ret;
1335 : __atomic_pre_full_fence();
1336 : ret = arch_atomic64_add_return_relaxed(i, v);
1337 : __atomic_post_full_fence();
1338 : return ret;
1339 : }
1340 : #define arch_atomic64_add_return arch_atomic64_add_return
1341 : #endif
1342 :
1343 : #endif /* arch_atomic64_add_return_relaxed */
1344 :
1345 : #ifndef arch_atomic64_fetch_add_relaxed
1346 : #define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add
1347 : #define arch_atomic64_fetch_add_release arch_atomic64_fetch_add
1348 : #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add
1349 : #else /* arch_atomic64_fetch_add_relaxed */
1350 :
1351 : #ifndef arch_atomic64_fetch_add_acquire
1352 : static __always_inline s64
1353 : arch_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
1354 : {
1355 : s64 ret = arch_atomic64_fetch_add_relaxed(i, v);
1356 : __atomic_acquire_fence();
1357 : return ret;
1358 : }
1359 : #define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire
1360 : #endif
1361 :
1362 : #ifndef arch_atomic64_fetch_add_release
1363 : static __always_inline s64
1364 : arch_atomic64_fetch_add_release(s64 i, atomic64_t *v)
1365 : {
1366 : __atomic_release_fence();
1367 : return arch_atomic64_fetch_add_relaxed(i, v);
1368 : }
1369 : #define arch_atomic64_fetch_add_release arch_atomic64_fetch_add_release
1370 : #endif
1371 :
1372 : #ifndef arch_atomic64_fetch_add
1373 : static __always_inline s64
1374 : arch_atomic64_fetch_add(s64 i, atomic64_t *v)
1375 : {
1376 : s64 ret;
1377 : __atomic_pre_full_fence();
1378 : ret = arch_atomic64_fetch_add_relaxed(i, v);
1379 : __atomic_post_full_fence();
1380 : return ret;
1381 : }
1382 : #define arch_atomic64_fetch_add arch_atomic64_fetch_add
1383 : #endif
1384 :
1385 : #endif /* arch_atomic64_fetch_add_relaxed */
1386 :
1387 : #ifndef arch_atomic64_sub_return_relaxed
1388 : #define arch_atomic64_sub_return_acquire arch_atomic64_sub_return
1389 : #define arch_atomic64_sub_return_release arch_atomic64_sub_return
1390 : #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return
1391 : #else /* arch_atomic64_sub_return_relaxed */
1392 :
1393 : #ifndef arch_atomic64_sub_return_acquire
1394 : static __always_inline s64
1395 : arch_atomic64_sub_return_acquire(s64 i, atomic64_t *v)
1396 : {
1397 : s64 ret = arch_atomic64_sub_return_relaxed(i, v);
1398 : __atomic_acquire_fence();
1399 : return ret;
1400 : }
1401 : #define arch_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire
1402 : #endif
1403 :
1404 : #ifndef arch_atomic64_sub_return_release
1405 : static __always_inline s64
1406 : arch_atomic64_sub_return_release(s64 i, atomic64_t *v)
1407 : {
1408 : __atomic_release_fence();
1409 : return arch_atomic64_sub_return_relaxed(i, v);
1410 : }
1411 : #define arch_atomic64_sub_return_release arch_atomic64_sub_return_release
1412 : #endif
1413 :
1414 : #ifndef arch_atomic64_sub_return
1415 : static __always_inline s64
1416 : arch_atomic64_sub_return(s64 i, atomic64_t *v)
1417 : {
1418 : s64 ret;
1419 : __atomic_pre_full_fence();
1420 : ret = arch_atomic64_sub_return_relaxed(i, v);
1421 : __atomic_post_full_fence();
1422 : return ret;
1423 : }
1424 : #define arch_atomic64_sub_return arch_atomic64_sub_return
1425 : #endif
1426 :
1427 : #endif /* arch_atomic64_sub_return_relaxed */
1428 :
1429 : #ifndef arch_atomic64_fetch_sub_relaxed
1430 : #define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub
1431 : #define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub
1432 : #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub
1433 : #else /* arch_atomic64_fetch_sub_relaxed */
1434 :
1435 : #ifndef arch_atomic64_fetch_sub_acquire
1436 : static __always_inline s64
1437 : arch_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1438 : {
1439 : s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);
1440 : __atomic_acquire_fence();
1441 : return ret;
1442 : }
1443 : #define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire
1444 : #endif
1445 :
1446 : #ifndef arch_atomic64_fetch_sub_release
1447 : static __always_inline s64
1448 : arch_atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1449 : {
1450 : __atomic_release_fence();
1451 : return arch_atomic64_fetch_sub_relaxed(i, v);
1452 : }
1453 : #define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release
1454 : #endif
1455 :
1456 : #ifndef arch_atomic64_fetch_sub
1457 : static __always_inline s64
1458 : arch_atomic64_fetch_sub(s64 i, atomic64_t *v)
1459 : {
1460 : s64 ret;
1461 : __atomic_pre_full_fence();
1462 : ret = arch_atomic64_fetch_sub_relaxed(i, v);
1463 : __atomic_post_full_fence();
1464 : return ret;
1465 : }
1466 : #define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
1467 : #endif
1468 :
1469 : #endif /* arch_atomic64_fetch_sub_relaxed */
1470 :
1471 : #ifndef arch_atomic64_inc
1472 : static __always_inline void
1473 : arch_atomic64_inc(atomic64_t *v)
1474 : {
1475 : arch_atomic64_add(1, v);
1476 : }
1477 : #define arch_atomic64_inc arch_atomic64_inc
1478 : #endif
1479 :
1480 : #ifndef arch_atomic64_inc_return_relaxed
1481 : #ifdef arch_atomic64_inc_return
1482 : #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return
1483 : #define arch_atomic64_inc_return_release arch_atomic64_inc_return
1484 : #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return
1485 : #endif /* arch_atomic64_inc_return */
1486 :
1487 : #ifndef arch_atomic64_inc_return
1488 : static __always_inline s64
1489 : arch_atomic64_inc_return(atomic64_t *v)
1490 : {
1491 0 : return arch_atomic64_add_return(1, v);
1492 : }
1493 : #define arch_atomic64_inc_return arch_atomic64_inc_return
1494 : #endif
1495 :
1496 : #ifndef arch_atomic64_inc_return_acquire
1497 : static __always_inline s64
1498 : arch_atomic64_inc_return_acquire(atomic64_t *v)
1499 : {
1500 : return arch_atomic64_add_return_acquire(1, v);
1501 : }
1502 : #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
1503 : #endif
1504 :
1505 : #ifndef arch_atomic64_inc_return_release
1506 : static __always_inline s64
1507 : arch_atomic64_inc_return_release(atomic64_t *v)
1508 : {
1509 : return arch_atomic64_add_return_release(1, v);
1510 : }
1511 : #define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
1512 : #endif
1513 :
1514 : #ifndef arch_atomic64_inc_return_relaxed
1515 : static __always_inline s64
1516 : arch_atomic64_inc_return_relaxed(atomic64_t *v)
1517 : {
1518 0 : return arch_atomic64_add_return_relaxed(1, v);
1519 : }
1520 : #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed
1521 : #endif
1522 :
1523 : #else /* arch_atomic64_inc_return_relaxed */
1524 :
1525 : #ifndef arch_atomic64_inc_return_acquire
1526 : static __always_inline s64
1527 : arch_atomic64_inc_return_acquire(atomic64_t *v)
1528 : {
1529 : s64 ret = arch_atomic64_inc_return_relaxed(v);
1530 : __atomic_acquire_fence();
1531 : return ret;
1532 : }
1533 : #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
1534 : #endif
1535 :
1536 : #ifndef arch_atomic64_inc_return_release
1537 : static __always_inline s64
1538 : arch_atomic64_inc_return_release(atomic64_t *v)
1539 : {
1540 : __atomic_release_fence();
1541 : return arch_atomic64_inc_return_relaxed(v);
1542 : }
1543 : #define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
1544 : #endif
1545 :
1546 : #ifndef arch_atomic64_inc_return
1547 : static __always_inline s64
1548 : arch_atomic64_inc_return(atomic64_t *v)
1549 : {
1550 : s64 ret;
1551 : __atomic_pre_full_fence();
1552 : ret = arch_atomic64_inc_return_relaxed(v);
1553 : __atomic_post_full_fence();
1554 : return ret;
1555 : }
1556 : #define arch_atomic64_inc_return arch_atomic64_inc_return
1557 : #endif
1558 :
1559 : #endif /* arch_atomic64_inc_return_relaxed */
1560 :
1561 : #ifndef arch_atomic64_fetch_inc_relaxed
1562 : #ifdef arch_atomic64_fetch_inc
1563 : #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc
1564 : #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc
1565 : #define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc
1566 : #endif /* arch_atomic64_fetch_inc */
1567 :
1568 : #ifndef arch_atomic64_fetch_inc
1569 : static __always_inline s64
1570 : arch_atomic64_fetch_inc(atomic64_t *v)
1571 : {
1572 : return arch_atomic64_fetch_add(1, v);
1573 : }
1574 : #define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
1575 : #endif
1576 :
1577 : #ifndef arch_atomic64_fetch_inc_acquire
1578 : static __always_inline s64
1579 : arch_atomic64_fetch_inc_acquire(atomic64_t *v)
1580 : {
1581 : return arch_atomic64_fetch_add_acquire(1, v);
1582 : }
1583 : #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
1584 : #endif
1585 :
1586 : #ifndef arch_atomic64_fetch_inc_release
1587 : static __always_inline s64
1588 : arch_atomic64_fetch_inc_release(atomic64_t *v)
1589 : {
1590 : return arch_atomic64_fetch_add_release(1, v);
1591 : }
1592 : #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
1593 : #endif
1594 :
1595 : #ifndef arch_atomic64_fetch_inc_relaxed
1596 : static __always_inline s64
1597 : arch_atomic64_fetch_inc_relaxed(atomic64_t *v)
1598 : {
1599 : return arch_atomic64_fetch_add_relaxed(1, v);
1600 : }
1601 : #define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc_relaxed
1602 : #endif
1603 :
1604 : #else /* arch_atomic64_fetch_inc_relaxed */
1605 :
1606 : #ifndef arch_atomic64_fetch_inc_acquire
1607 : static __always_inline s64
1608 : arch_atomic64_fetch_inc_acquire(atomic64_t *v)
1609 : {
1610 : s64 ret = arch_atomic64_fetch_inc_relaxed(v);
1611 : __atomic_acquire_fence();
1612 : return ret;
1613 : }
1614 : #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
1615 : #endif
1616 :
1617 : #ifndef arch_atomic64_fetch_inc_release
1618 : static __always_inline s64
1619 : arch_atomic64_fetch_inc_release(atomic64_t *v)
1620 : {
1621 : __atomic_release_fence();
1622 : return arch_atomic64_fetch_inc_relaxed(v);
1623 : }
1624 : #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
1625 : #endif
1626 :
1627 : #ifndef arch_atomic64_fetch_inc
1628 : static __always_inline s64
1629 : arch_atomic64_fetch_inc(atomic64_t *v)
1630 : {
1631 : s64 ret;
1632 : __atomic_pre_full_fence();
1633 : ret = arch_atomic64_fetch_inc_relaxed(v);
1634 : __atomic_post_full_fence();
1635 : return ret;
1636 : }
1637 : #define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
1638 : #endif
1639 :
1640 : #endif /* arch_atomic64_fetch_inc_relaxed */
1641 :
1642 : #ifndef arch_atomic64_dec
1643 : static __always_inline void
1644 : arch_atomic64_dec(atomic64_t *v)
1645 : {
1646 : arch_atomic64_sub(1, v);
1647 : }
1648 : #define arch_atomic64_dec arch_atomic64_dec
1649 : #endif
1650 :
1651 : #ifndef arch_atomic64_dec_return_relaxed
1652 : #ifdef arch_atomic64_dec_return
1653 : #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return
1654 : #define arch_atomic64_dec_return_release arch_atomic64_dec_return
1655 : #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return
1656 : #endif /* arch_atomic64_dec_return */
1657 :
1658 : #ifndef arch_atomic64_dec_return
1659 : static __always_inline s64
1660 : arch_atomic64_dec_return(atomic64_t *v)
1661 : {
1662 0 : return arch_atomic64_sub_return(1, v);
1663 : }
1664 : #define arch_atomic64_dec_return arch_atomic64_dec_return
1665 : #endif
1666 :
1667 : #ifndef arch_atomic64_dec_return_acquire
1668 : static __always_inline s64
1669 : arch_atomic64_dec_return_acquire(atomic64_t *v)
1670 : {
1671 : return arch_atomic64_sub_return_acquire(1, v);
1672 : }
1673 : #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
1674 : #endif
1675 :
1676 : #ifndef arch_atomic64_dec_return_release
1677 : static __always_inline s64
1678 : arch_atomic64_dec_return_release(atomic64_t *v)
1679 : {
1680 : return arch_atomic64_sub_return_release(1, v);
1681 : }
1682 : #define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
1683 : #endif
1684 :
1685 : #ifndef arch_atomic64_dec_return_relaxed
1686 : static __always_inline s64
1687 : arch_atomic64_dec_return_relaxed(atomic64_t *v)
1688 : {
1689 : return arch_atomic64_sub_return_relaxed(1, v);
1690 : }
1691 : #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed
1692 : #endif
1693 :
1694 : #else /* arch_atomic64_dec_return_relaxed */
1695 :
1696 : #ifndef arch_atomic64_dec_return_acquire
1697 : static __always_inline s64
1698 : arch_atomic64_dec_return_acquire(atomic64_t *v)
1699 : {
1700 : s64 ret = arch_atomic64_dec_return_relaxed(v);
1701 : __atomic_acquire_fence();
1702 : return ret;
1703 : }
1704 : #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
1705 : #endif
1706 :
1707 : #ifndef arch_atomic64_dec_return_release
1708 : static __always_inline s64
1709 : arch_atomic64_dec_return_release(atomic64_t *v)
1710 : {
1711 : __atomic_release_fence();
1712 : return arch_atomic64_dec_return_relaxed(v);
1713 : }
1714 : #define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
1715 : #endif
1716 :
1717 : #ifndef arch_atomic64_dec_return
1718 : static __always_inline s64
1719 : arch_atomic64_dec_return(atomic64_t *v)
1720 : {
1721 : s64 ret;
1722 : __atomic_pre_full_fence();
1723 : ret = arch_atomic64_dec_return_relaxed(v);
1724 : __atomic_post_full_fence();
1725 : return ret;
1726 : }
1727 : #define arch_atomic64_dec_return arch_atomic64_dec_return
1728 : #endif
1729 :
1730 : #endif /* arch_atomic64_dec_return_relaxed */
1731 :
1732 : #ifndef arch_atomic64_fetch_dec_relaxed
1733 : #ifdef arch_atomic64_fetch_dec
1734 : #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec
1735 : #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec
1736 : #define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec
1737 : #endif /* arch_atomic64_fetch_dec */
1738 :
1739 : #ifndef arch_atomic64_fetch_dec
1740 : static __always_inline s64
1741 : arch_atomic64_fetch_dec(atomic64_t *v)
1742 : {
1743 : return arch_atomic64_fetch_sub(1, v);
1744 : }
1745 : #define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
1746 : #endif
1747 :
1748 : #ifndef arch_atomic64_fetch_dec_acquire
1749 : static __always_inline s64
1750 : arch_atomic64_fetch_dec_acquire(atomic64_t *v)
1751 : {
1752 : return arch_atomic64_fetch_sub_acquire(1, v);
1753 : }
1754 : #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
1755 : #endif
1756 :
1757 : #ifndef arch_atomic64_fetch_dec_release
1758 : static __always_inline s64
1759 : arch_atomic64_fetch_dec_release(atomic64_t *v)
1760 : {
1761 : return arch_atomic64_fetch_sub_release(1, v);
1762 : }
1763 : #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1764 : #endif
1765 :
1766 : #ifndef arch_atomic64_fetch_dec_relaxed
1767 : static __always_inline s64
1768 : arch_atomic64_fetch_dec_relaxed(atomic64_t *v)
1769 : {
1770 : return arch_atomic64_fetch_sub_relaxed(1, v);
1771 : }
1772 : #define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec_relaxed
1773 : #endif
1774 :
1775 : #else /* arch_atomic64_fetch_dec_relaxed */
1776 :
1777 : #ifndef arch_atomic64_fetch_dec_acquire
1778 : static __always_inline s64
1779 : arch_atomic64_fetch_dec_acquire(atomic64_t *v)
1780 : {
1781 : s64 ret = arch_atomic64_fetch_dec_relaxed(v);
1782 : __atomic_acquire_fence();
1783 : return ret;
1784 : }
1785 : #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
1786 : #endif
1787 :
1788 : #ifndef arch_atomic64_fetch_dec_release
1789 : static __always_inline s64
1790 : arch_atomic64_fetch_dec_release(atomic64_t *v)
1791 : {
1792 : __atomic_release_fence();
1793 : return arch_atomic64_fetch_dec_relaxed(v);
1794 : }
1795 : #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1796 : #endif
1797 :
1798 : #ifndef arch_atomic64_fetch_dec
1799 : static __always_inline s64
1800 : arch_atomic64_fetch_dec(atomic64_t *v)
1801 : {
1802 : s64 ret;
1803 : __atomic_pre_full_fence();
1804 : ret = arch_atomic64_fetch_dec_relaxed(v);
1805 : __atomic_post_full_fence();
1806 : return ret;
1807 : }
1808 : #define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
1809 : #endif
1810 :
1811 : #endif /* arch_atomic64_fetch_dec_relaxed */
1812 :
1813 : #ifndef arch_atomic64_fetch_and_relaxed
1814 : #define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and
1815 : #define arch_atomic64_fetch_and_release arch_atomic64_fetch_and
1816 : #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and
1817 : #else /* arch_atomic64_fetch_and_relaxed */
1818 :
1819 : #ifndef arch_atomic64_fetch_and_acquire
1820 : static __always_inline s64
1821 : arch_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
1822 : {
1823 : s64 ret = arch_atomic64_fetch_and_relaxed(i, v);
1824 : __atomic_acquire_fence();
1825 : return ret;
1826 : }
1827 : #define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire
1828 : #endif
1829 :
1830 : #ifndef arch_atomic64_fetch_and_release
1831 : static __always_inline s64
1832 : arch_atomic64_fetch_and_release(s64 i, atomic64_t *v)
1833 : {
1834 : __atomic_release_fence();
1835 : return arch_atomic64_fetch_and_relaxed(i, v);
1836 : }
1837 : #define arch_atomic64_fetch_and_release arch_atomic64_fetch_and_release
1838 : #endif
1839 :
1840 : #ifndef arch_atomic64_fetch_and
1841 : static __always_inline s64
1842 : arch_atomic64_fetch_and(s64 i, atomic64_t *v)
1843 : {
1844 : s64 ret;
1845 : __atomic_pre_full_fence();
1846 : ret = arch_atomic64_fetch_and_relaxed(i, v);
1847 : __atomic_post_full_fence();
1848 : return ret;
1849 : }
1850 : #define arch_atomic64_fetch_and arch_atomic64_fetch_and
1851 : #endif
1852 :
1853 : #endif /* arch_atomic64_fetch_and_relaxed */
1854 :
1855 : #ifndef arch_atomic64_andnot
1856 : static __always_inline void
1857 : arch_atomic64_andnot(s64 i, atomic64_t *v)
1858 : {
1859 0 : arch_atomic64_and(~i, v);
1860 : }
1861 : #define arch_atomic64_andnot arch_atomic64_andnot
1862 : #endif
1863 :
1864 : #ifndef arch_atomic64_fetch_andnot_relaxed
1865 : #ifdef arch_atomic64_fetch_andnot
1866 : #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot
1867 : #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot
1868 : #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot
1869 : #endif /* arch_atomic64_fetch_andnot */
1870 :
1871 : #ifndef arch_atomic64_fetch_andnot
1872 : static __always_inline s64
1873 : arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
1874 : {
1875 : return arch_atomic64_fetch_and(~i, v);
1876 : }
1877 : #define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
1878 : #endif
1879 :
1880 : #ifndef arch_atomic64_fetch_andnot_acquire
1881 : static __always_inline s64
1882 : arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1883 : {
1884 : return arch_atomic64_fetch_and_acquire(~i, v);
1885 : }
1886 : #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
1887 : #endif
1888 :
1889 : #ifndef arch_atomic64_fetch_andnot_release
1890 : static __always_inline s64
1891 : arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1892 : {
1893 : return arch_atomic64_fetch_and_release(~i, v);
1894 : }
1895 : #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
1896 : #endif
1897 :
1898 : #ifndef arch_atomic64_fetch_andnot_relaxed
1899 : static __always_inline s64
1900 : arch_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1901 : {
1902 : return arch_atomic64_fetch_and_relaxed(~i, v);
1903 : }
1904 : #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed
1905 : #endif
1906 :
1907 : #else /* arch_atomic64_fetch_andnot_relaxed */
1908 :
1909 : #ifndef arch_atomic64_fetch_andnot_acquire
1910 : static __always_inline s64
1911 : arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1912 : {
1913 : s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
1914 : __atomic_acquire_fence();
1915 : return ret;
1916 : }
1917 : #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
1918 : #endif
1919 :
1920 : #ifndef arch_atomic64_fetch_andnot_release
1921 : static __always_inline s64
1922 : arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1923 : {
1924 : __atomic_release_fence();
1925 : return arch_atomic64_fetch_andnot_relaxed(i, v);
1926 : }
1927 : #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
1928 : #endif
1929 :
1930 : #ifndef arch_atomic64_fetch_andnot
1931 : static __always_inline s64
1932 : arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
1933 : {
1934 : s64 ret;
1935 : __atomic_pre_full_fence();
1936 : ret = arch_atomic64_fetch_andnot_relaxed(i, v);
1937 : __atomic_post_full_fence();
1938 : return ret;
1939 : }
1940 : #define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
1941 : #endif
1942 :
1943 : #endif /* arch_atomic64_fetch_andnot_relaxed */
1944 :
1945 : #ifndef arch_atomic64_fetch_or_relaxed
1946 : #define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or
1947 : #define arch_atomic64_fetch_or_release arch_atomic64_fetch_or
1948 : #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or
1949 : #else /* arch_atomic64_fetch_or_relaxed */
1950 :
1951 : #ifndef arch_atomic64_fetch_or_acquire
1952 : static __always_inline s64
1953 : arch_atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1954 : {
1955 : s64 ret = arch_atomic64_fetch_or_relaxed(i, v);
1956 : __atomic_acquire_fence();
1957 : return ret;
1958 : }
1959 : #define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or_acquire
1960 : #endif
1961 :
1962 : #ifndef arch_atomic64_fetch_or_release
1963 : static __always_inline s64
1964 : arch_atomic64_fetch_or_release(s64 i, atomic64_t *v)
1965 : {
1966 : __atomic_release_fence();
1967 : return arch_atomic64_fetch_or_relaxed(i, v);
1968 : }
1969 : #define arch_atomic64_fetch_or_release arch_atomic64_fetch_or_release
1970 : #endif
1971 :
1972 : #ifndef arch_atomic64_fetch_or
1973 : static __always_inline s64
1974 : arch_atomic64_fetch_or(s64 i, atomic64_t *v)
1975 : {
1976 : s64 ret;
1977 : __atomic_pre_full_fence();
1978 : ret = arch_atomic64_fetch_or_relaxed(i, v);
1979 : __atomic_post_full_fence();
1980 : return ret;
1981 : }
1982 : #define arch_atomic64_fetch_or arch_atomic64_fetch_or
1983 : #endif
1984 :
1985 : #endif /* arch_atomic64_fetch_or_relaxed */
1986 :
1987 : #ifndef arch_atomic64_fetch_xor_relaxed
1988 : #define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor
1989 : #define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor
1990 : #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor
1991 : #else /* arch_atomic64_fetch_xor_relaxed */
1992 :
1993 : #ifndef arch_atomic64_fetch_xor_acquire
1994 : static __always_inline s64
1995 : arch_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1996 : {
1997 : s64 ret = arch_atomic64_fetch_xor_relaxed(i, v);
1998 : __atomic_acquire_fence();
1999 : return ret;
2000 : }
2001 : #define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor_acquire
2002 : #endif
2003 :
2004 : #ifndef arch_atomic64_fetch_xor_release
2005 : static __always_inline s64
2006 : arch_atomic64_fetch_xor_release(s64 i, atomic64_t *v)
2007 : {
2008 : __atomic_release_fence();
2009 : return arch_atomic64_fetch_xor_relaxed(i, v);
2010 : }
2011 : #define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release
2012 : #endif
2013 :
2014 : #ifndef arch_atomic64_fetch_xor
2015 : static __always_inline s64
2016 : arch_atomic64_fetch_xor(s64 i, atomic64_t *v)
2017 : {
2018 : s64 ret;
2019 : __atomic_pre_full_fence();
2020 : ret = arch_atomic64_fetch_xor_relaxed(i, v);
2021 : __atomic_post_full_fence();
2022 : return ret;
2023 : }
2024 : #define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
2025 : #endif
2026 :
2027 : #endif /* arch_atomic64_fetch_xor_relaxed */
2028 :
2029 : #ifndef arch_atomic64_xchg_relaxed
2030 : #define arch_atomic64_xchg_acquire arch_atomic64_xchg
2031 : #define arch_atomic64_xchg_release arch_atomic64_xchg
2032 : #define arch_atomic64_xchg_relaxed arch_atomic64_xchg
2033 : #else /* arch_atomic64_xchg_relaxed */
2034 :
2035 : #ifndef arch_atomic64_xchg_acquire
2036 : static __always_inline s64
2037 : arch_atomic64_xchg_acquire(atomic64_t *v, s64 i)
2038 : {
2039 : s64 ret = arch_atomic64_xchg_relaxed(v, i);
2040 : __atomic_acquire_fence();
2041 : return ret;
2042 : }
2043 : #define arch_atomic64_xchg_acquire arch_atomic64_xchg_acquire
2044 : #endif
2045 :
2046 : #ifndef arch_atomic64_xchg_release
2047 : static __always_inline s64
2048 : arch_atomic64_xchg_release(atomic64_t *v, s64 i)
2049 : {
2050 : __atomic_release_fence();
2051 : return arch_atomic64_xchg_relaxed(v, i);
2052 : }
2053 : #define arch_atomic64_xchg_release arch_atomic64_xchg_release
2054 : #endif
2055 :
2056 : #ifndef arch_atomic64_xchg
2057 : static __always_inline s64
2058 : arch_atomic64_xchg(atomic64_t *v, s64 i)
2059 : {
2060 : s64 ret;
2061 : __atomic_pre_full_fence();
2062 : ret = arch_atomic64_xchg_relaxed(v, i);
2063 : __atomic_post_full_fence();
2064 : return ret;
2065 : }
2066 : #define arch_atomic64_xchg arch_atomic64_xchg
2067 : #endif
2068 :
2069 : #endif /* arch_atomic64_xchg_relaxed */
2070 :
2071 : #ifndef arch_atomic64_cmpxchg_relaxed
2072 : #define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg
2073 : #define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg
2074 : #define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg
2075 : #else /* arch_atomic64_cmpxchg_relaxed */
2076 :
2077 : #ifndef arch_atomic64_cmpxchg_acquire
2078 : static __always_inline s64
2079 : arch_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
2080 : {
2081 : s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
2082 : __atomic_acquire_fence();
2083 : return ret;
2084 : }
2085 : #define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire
2086 : #endif
2087 :
2088 : #ifndef arch_atomic64_cmpxchg_release
2089 : static __always_inline s64
2090 : arch_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
2091 : {
2092 : __atomic_release_fence();
2093 : return arch_atomic64_cmpxchg_relaxed(v, old, new);
2094 : }
2095 : #define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release
2096 : #endif
2097 :
2098 : #ifndef arch_atomic64_cmpxchg
2099 : static __always_inline s64
2100 : arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
2101 : {
2102 : s64 ret;
2103 : __atomic_pre_full_fence();
2104 : ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
2105 : __atomic_post_full_fence();
2106 : return ret;
2107 : }
2108 : #define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
2109 : #endif
2110 :
2111 : #endif /* arch_atomic64_cmpxchg_relaxed */
2112 :
2113 : #ifndef arch_atomic64_try_cmpxchg_relaxed
2114 : #ifdef arch_atomic64_try_cmpxchg
2115 : #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg
2116 : #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg
2117 : #define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg
2118 : #endif /* arch_atomic64_try_cmpxchg */
2119 :
2120 : #ifndef arch_atomic64_try_cmpxchg
2121 : static __always_inline bool
2122 : arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2123 : {
2124 : s64 r, o = *old;
2125 : r = arch_atomic64_cmpxchg(v, o, new);
2126 : if (unlikely(r != o))
2127 : *old = r;
2128 : return likely(r == o);
2129 : }
2130 : #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
2131 : #endif
2132 :
2133 : #ifndef arch_atomic64_try_cmpxchg_acquire
2134 : static __always_inline bool
2135 : arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2136 : {
2137 : s64 r, o = *old;
2138 : r = arch_atomic64_cmpxchg_acquire(v, o, new);
2139 : if (unlikely(r != o))
2140 : *old = r;
2141 : return likely(r == o);
2142 : }
2143 : #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
2144 : #endif
2145 :
2146 : #ifndef arch_atomic64_try_cmpxchg_release
2147 : static __always_inline bool
2148 : arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2149 : {
2150 : s64 r, o = *old;
2151 : r = arch_atomic64_cmpxchg_release(v, o, new);
2152 : if (unlikely(r != o))
2153 : *old = r;
2154 : return likely(r == o);
2155 : }
2156 : #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
2157 : #endif
2158 :
2159 : #ifndef arch_atomic64_try_cmpxchg_relaxed
2160 : static __always_inline bool
2161 : arch_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
2162 : {
2163 : s64 r, o = *old;
2164 : r = arch_atomic64_cmpxchg_relaxed(v, o, new);
2165 : if (unlikely(r != o))
2166 : *old = r;
2167 : return likely(r == o);
2168 : }
2169 : #define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg_relaxed
2170 : #endif
2171 :
2172 : #else /* arch_atomic64_try_cmpxchg_relaxed */
2173 :
2174 : #ifndef arch_atomic64_try_cmpxchg_acquire
2175 : static __always_inline bool
2176 : arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2177 : {
2178 : bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2179 : __atomic_acquire_fence();
2180 : return ret;
2181 : }
2182 : #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
2183 : #endif
2184 :
2185 : #ifndef arch_atomic64_try_cmpxchg_release
2186 : static __always_inline bool
2187 : arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2188 : {
2189 : __atomic_release_fence();
2190 : return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2191 : }
2192 : #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
2193 : #endif
2194 :
2195 : #ifndef arch_atomic64_try_cmpxchg
2196 : static __always_inline bool
2197 : arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2198 : {
2199 : bool ret;
2200 : __atomic_pre_full_fence();
2201 : ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2202 : __atomic_post_full_fence();
2203 : return ret;
2204 : }
2205 : #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
2206 : #endif
2207 :
2208 : #endif /* arch_atomic64_try_cmpxchg_relaxed */
2209 :
2210 : #ifndef arch_atomic64_sub_and_test
2211 : /**
2212 : * arch_atomic64_sub_and_test - subtract value from variable and test result
2213 : * @i: integer value to subtract
2214 : * @v: pointer of type atomic64_t
2215 : *
2216 : * Atomically subtracts @i from @v and returns
2217 : * true if the result is zero, or false for all
2218 : * other cases.
2219 : */
2220 : static __always_inline bool
2221 : arch_atomic64_sub_and_test(s64 i, atomic64_t *v)
2222 : {
2223 : return arch_atomic64_sub_return(i, v) == 0;
2224 : }
2225 : #define arch_atomic64_sub_and_test arch_atomic64_sub_and_test
2226 : #endif
2227 :
2228 : #ifndef arch_atomic64_dec_and_test
2229 : /**
2230 : * arch_atomic64_dec_and_test - decrement and test
2231 : * @v: pointer of type atomic64_t
2232 : *
2233 : * Atomically decrements @v by 1 and
2234 : * returns true if the result is 0, or false for all other
2235 : * cases.
2236 : */
2237 : static __always_inline bool
2238 : arch_atomic64_dec_and_test(atomic64_t *v)
2239 : {
2240 : return arch_atomic64_dec_return(v) == 0;
2241 : }
2242 : #define arch_atomic64_dec_and_test arch_atomic64_dec_and_test
2243 : #endif
2244 :
2245 : #ifndef arch_atomic64_inc_and_test
2246 : /**
2247 : * arch_atomic64_inc_and_test - increment and test
2248 : * @v: pointer of type atomic64_t
2249 : *
2250 : * Atomically increments @v by 1
2251 : * and returns true if the result is zero, or false for all
2252 : * other cases.
2253 : */
2254 : static __always_inline bool
2255 : arch_atomic64_inc_and_test(atomic64_t *v)
2256 : {
2257 : return arch_atomic64_inc_return(v) == 0;
2258 : }
2259 : #define arch_atomic64_inc_and_test arch_atomic64_inc_and_test
2260 : #endif
2261 :
2262 : #ifndef arch_atomic64_add_negative
2263 : /**
2264 : * arch_atomic64_add_negative - add and test if negative
2265 : * @i: integer value to add
2266 : * @v: pointer of type atomic64_t
2267 : *
2268 : * Atomically adds @i to @v and returns true
2269 : * if the result is negative, or false when
2270 : * result is greater than or equal to zero.
2271 : */
2272 : static __always_inline bool
2273 : arch_atomic64_add_negative(s64 i, atomic64_t *v)
2274 : {
2275 : return arch_atomic64_add_return(i, v) < 0;
2276 : }
2277 : #define arch_atomic64_add_negative arch_atomic64_add_negative
2278 : #endif
2279 :
2280 : #ifndef arch_atomic64_fetch_add_unless
2281 : /**
2282 : * arch_atomic64_fetch_add_unless - add unless the number is already a given value
2283 : * @v: pointer of type atomic64_t
2284 : * @a: the amount to add to v...
2285 : * @u: ...unless v is equal to u.
2286 : *
2287 : * Atomically adds @a to @v, so long as @v was not already @u.
2288 : * Returns original value of @v
2289 : */
2290 : static __always_inline s64
2291 : arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
2292 : {
2293 0 : s64 c = arch_atomic64_read(v);
2294 :
2295 : do {
2296 0 : if (unlikely(c == u))
2297 : break;
2298 0 : } while (!arch_atomic64_try_cmpxchg(v, &c, c + a));
2299 :
2300 : return c;
2301 : }
2302 : #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
2303 : #endif
2304 :
2305 : #ifndef arch_atomic64_add_unless
2306 : /**
2307 : * arch_atomic64_add_unless - add unless the number is already a given value
2308 : * @v: pointer of type atomic64_t
2309 : * @a: the amount to add to v...
2310 : * @u: ...unless v is equal to u.
2311 : *
2312 : * Atomically adds @a to @v, if @v was not already @u.
2313 : * Returns true if the addition was done.
2314 : */
2315 : static __always_inline bool
2316 : arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
2317 : {
2318 0 : return arch_atomic64_fetch_add_unless(v, a, u) != u;
2319 : }
2320 : #define arch_atomic64_add_unless arch_atomic64_add_unless
2321 : #endif
2322 :
2323 : #ifndef arch_atomic64_inc_not_zero
2324 : /**
2325 : * arch_atomic64_inc_not_zero - increment unless the number is zero
2326 : * @v: pointer of type atomic64_t
2327 : *
2328 : * Atomically increments @v by 1, if @v is non-zero.
2329 : * Returns true if the increment was done.
2330 : */
2331 : static __always_inline bool
2332 : arch_atomic64_inc_not_zero(atomic64_t *v)
2333 : {
2334 0 : return arch_atomic64_add_unless(v, 1, 0);
2335 : }
2336 : #define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero
2337 : #endif
2338 :
2339 : #ifndef arch_atomic64_inc_unless_negative
2340 : static __always_inline bool
2341 : arch_atomic64_inc_unless_negative(atomic64_t *v)
2342 : {
2343 : s64 c = arch_atomic64_read(v);
2344 :
2345 : do {
2346 : if (unlikely(c < 0))
2347 : return false;
2348 : } while (!arch_atomic64_try_cmpxchg(v, &c, c + 1));
2349 :
2350 : return true;
2351 : }
2352 : #define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative
2353 : #endif
2354 :
2355 : #ifndef arch_atomic64_dec_unless_positive
2356 : static __always_inline bool
2357 : arch_atomic64_dec_unless_positive(atomic64_t *v)
2358 : {
2359 : s64 c = arch_atomic64_read(v);
2360 :
2361 : do {
2362 : if (unlikely(c > 0))
2363 : return false;
2364 : } while (!arch_atomic64_try_cmpxchg(v, &c, c - 1));
2365 :
2366 : return true;
2367 : }
2368 : #define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive
2369 : #endif
2370 :
2371 : #ifndef arch_atomic64_dec_if_positive
2372 : static __always_inline s64
2373 : arch_atomic64_dec_if_positive(atomic64_t *v)
2374 : {
2375 0 : s64 dec, c = arch_atomic64_read(v);
2376 :
2377 : do {
2378 0 : dec = c - 1;
2379 0 : if (unlikely(dec < 0))
2380 : break;
2381 0 : } while (!arch_atomic64_try_cmpxchg(v, &c, dec));
2382 :
2383 : return dec;
2384 : }
2385 : #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
2386 : #endif
2387 :
2388 : #endif /* _LINUX_ATOMIC_FALLBACK_H */
2389 : // 8e2cc06bc0d2c0967d2f8424762bd48555ee40ae
|
