118#define rte_compiler_barrier() do { \
119 asm volatile ("" : : : "memory"); \
148#ifdef RTE_FORCE_INTRINSICS
152 return __sync_bool_compare_and_swap(dst, exp, src);
171static inline uint16_t
174#ifdef RTE_FORCE_INTRINSICS
175static inline uint16_t
178#if defined(__clang__)
179 return __atomic_exchange_n(dst, val, __ATOMIC_SEQ_CST);
181 return __atomic_exchange_2(dst, val, __ATOMIC_SEQ_CST);
196#define RTE_ATOMIC16_INIT(val) { (val) }
249 __sync_fetch_and_add(&v->
cnt, inc);
263 __sync_fetch_and_sub(&v->
cnt, dec);
275#ifdef RTE_FORCE_INTRINSICS
292#ifdef RTE_FORCE_INTRINSICS
316 return __sync_add_and_fetch(&v->
cnt, inc);
336 return __sync_sub_and_fetch(&v->
cnt, dec);
352#ifdef RTE_FORCE_INTRINSICS
355 return __sync_add_and_fetch(&v->
cnt, 1) == 0;
372#ifdef RTE_FORCE_INTRINSICS
375 return __sync_sub_and_fetch(&v->
cnt, 1) == 0;
392#ifdef RTE_FORCE_INTRINSICS
431#ifdef RTE_FORCE_INTRINSICS
435 return __sync_bool_compare_and_swap(dst, exp, src);
454static inline uint32_t
457#ifdef RTE_FORCE_INTRINSICS
458static inline uint32_t
461#if defined(__clang__)
462 return __atomic_exchange_n(dst, val, __ATOMIC_SEQ_CST);
464 return __atomic_exchange_4(dst, val, __ATOMIC_SEQ_CST);
479#define RTE_ATOMIC32_INIT(val) { (val) }
532 __sync_fetch_and_add(&v->
cnt, inc);
546 __sync_fetch_and_sub(&v->
cnt, dec);
558#ifdef RTE_FORCE_INTRINSICS
575#ifdef RTE_FORCE_INTRINSICS
599 return __sync_add_and_fetch(&v->
cnt, inc);
619 return __sync_sub_and_fetch(&v->
cnt, dec);
635#ifdef RTE_FORCE_INTRINSICS
638 return __sync_add_and_fetch(&v->
cnt, 1) == 0;
655#ifdef RTE_FORCE_INTRINSICS
658 return __sync_sub_and_fetch(&v->
cnt, 1) == 0;
675#ifdef RTE_FORCE_INTRINSICS
713#ifdef RTE_FORCE_INTRINSICS
717 return __sync_bool_compare_and_swap(dst, exp, src);
736static inline uint64_t
739#ifdef RTE_FORCE_INTRINSICS
740static inline uint64_t
743#if defined(__clang__)
744 return __atomic_exchange_n(dst, val, __ATOMIC_SEQ_CST);
746 return __atomic_exchange_8(dst, val, __ATOMIC_SEQ_CST);
761#define RTE_ATOMIC64_INIT(val) { (val) }
772#ifdef RTE_FORCE_INTRINSICS
782 while (success == 0) {
802#ifdef RTE_FORCE_INTRINSICS
812 while (success == 0) {
834#ifdef RTE_FORCE_INTRINSICS
844 while (success == 0) {
864#ifdef RTE_FORCE_INTRINSICS
868 __sync_fetch_and_add(&v->
cnt, inc);
883#ifdef RTE_FORCE_INTRINSICS
887 __sync_fetch_and_sub(&v->
cnt, dec);
900#ifdef RTE_FORCE_INTRINSICS
917#ifdef RTE_FORCE_INTRINSICS
941#ifdef RTE_FORCE_INTRINSICS
945 return __sync_add_and_fetch(&v->
cnt, inc);
965#ifdef RTE_FORCE_INTRINSICS
969 return __sync_sub_and_fetch(&v->
cnt, dec);
986#ifdef RTE_FORCE_INTRINSICS
1006#ifdef RTE_FORCE_INTRINSICS
1026#ifdef RTE_FORCE_INTRINSICS
1041#ifdef RTE_FORCE_INTRINSICS
1059 __extension__ __int128 int128;
1109 const rte_int128_t *src,
static int rte_atomic16_dec_and_test(rte_atomic16_t *v)
static void rte_atomic16_dec(rte_atomic16_t *v)
static int rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
static int rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
static void rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
static int rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
static void rte_atomic_thread_fence(int memorder)
static int rte_atomic64_test_and_set(rte_atomic64_t *v)
static void rte_io_rmb(void)
static void rte_rmb(void)
static void rte_atomic32_clear(rte_atomic32_t *v)
static int64_t rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
static void rte_io_mb(void)
static void rte_io_wmb(void)
static int rte_atomic32_inc_and_test(rte_atomic32_t *v)
static int rte_atomic64_dec_and_test(rte_atomic64_t *v)
static void rte_atomic64_clear(rte_atomic64_t *v)
static void rte_smp_mb(void)
static int16_t rte_atomic16_sub_return(rte_atomic16_t *v, int16_t dec)
static void rte_atomic16_set(rte_atomic16_t *v, int16_t new_value)
static void rte_atomic16_clear(rte_atomic16_t *v)
static void rte_atomic16_add(rte_atomic16_t *v, int16_t inc)
static uint32_t rte_atomic32_exchange(volatile uint32_t *dst, uint32_t val)
static void rte_atomic32_inc(rte_atomic32_t *v)
static void rte_smp_wmb(void)
static void rte_atomic16_sub(rte_atomic16_t *v, int16_t dec)
static int rte_atomic32_test_and_set(rte_atomic32_t *v)
static void rte_atomic32_dec(rte_atomic32_t *v)
static uint16_t rte_atomic16_exchange(volatile uint16_t *dst, uint16_t val)
static uint64_t rte_atomic64_exchange(volatile uint64_t *dst, uint64_t val)
static void rte_atomic64_dec(rte_atomic64_t *v)
static int32_t rte_atomic32_sub_return(rte_atomic32_t *v, int32_t dec)
static void rte_atomic16_init(rte_atomic16_t *v)
static void rte_smp_rmb(void)
static void rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
static __rte_experimental int rte_atomic128_cmp_exchange(rte_int128_t *dst, rte_int128_t *exp, const rte_int128_t *src, unsigned int weak, int success, int failure)
static int rte_atomic16_test_and_set(rte_atomic16_t *v)
static int32_t rte_atomic32_read(const rte_atomic32_t *v)
static void rte_wmb(void)
static void rte_atomic32_add(rte_atomic32_t *v, int32_t inc)
static void rte_atomic32_set(rte_atomic32_t *v, int32_t new_value)
static void rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
static void rte_atomic32_sub(rte_atomic32_t *v, int32_t dec)
static void rte_atomic64_inc(rte_atomic64_t *v)
static int16_t rte_atomic16_read(const rte_atomic16_t *v)
static int64_t rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
static void rte_atomic64_init(rte_atomic64_t *v)
static int16_t rte_atomic16_add_return(rte_atomic16_t *v, int16_t inc)
static void rte_atomic32_init(rte_atomic32_t *v)
static void rte_atomic16_inc(rte_atomic16_t *v)
static int64_t rte_atomic64_read(rte_atomic64_t *v)
static int rte_atomic64_inc_and_test(rte_atomic64_t *v)
static int rte_atomic32_dec_and_test(rte_atomic32_t *v)
static int rte_atomic16_inc_and_test(rte_atomic16_t *v)
static int32_t rte_atomic32_add_return(rte_atomic32_t *v, int32_t inc)
__extension__ struct rte_eth_link __rte_aligned(8)