34 #ifndef _RTE_MEMPOOL_H_
35 #define _RTE_MEMPOOL_H_
67 #include <sys/queue.h>
80 #define RTE_MEMPOOL_HEADER_COOKIE1 0xbadbadbadadd2e55ULL
81 #define RTE_MEMPOOL_HEADER_COOKIE2 0xf2eef2eedadd2e55ULL
82 #define RTE_MEMPOOL_TRAILER_COOKIE 0xadd2e55badbadbadULL
84 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
88 struct rte_mempool_debug_stats {
91 uint64_t get_success_bulk;
92 uint64_t get_success_objs;
93 uint64_t get_fail_bulk;
94 uint64_t get_fail_objs;
98 #if RTE_MEMPOOL_CACHE_MAX_SIZE > 0
102 struct rte_mempool_cache {
108 void *objs[RTE_MEMPOOL_CACHE_MAX_SIZE * 3];
123 #define RTE_MEMPOOL_NAMESIZE 32
124 #define RTE_MEMPOOL_MZ_PREFIX "MP_"
127 #define RTE_MEMPOOL_MZ_FORMAT RTE_MEMPOOL_MZ_PREFIX "%s"
129 #ifdef RTE_LIBRTE_XEN_DOM0
132 #define RTE_MEMPOOL_OBJ_NAME "%s_" RTE_MEMPOOL_MZ_PREFIX "elt"
136 #define RTE_MEMPOOL_OBJ_NAME RTE_MEMPOOL_MZ_FORMAT
140 #define MEMPOOL_PG_SHIFT_MAX (sizeof(uintptr_t) * CHAR_BIT - 1)
143 #define MEMPOOL_PG_NUM_DEFAULT 1
145 #ifndef RTE_MEMPOOL_ALIGN
146 #define RTE_MEMPOOL_ALIGN RTE_CACHE_LINE_SIZE
149 #define RTE_MEMPOOL_ALIGN_MASK (RTE_MEMPOOL_ALIGN - 1)
161 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
173 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
197 #if RTE_MEMPOOL_CACHE_MAX_SIZE > 0
199 struct rte_mempool_cache local_cache[RTE_MAX_LCORE];
202 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
204 struct rte_mempool_debug_stats stats[RTE_MAX_LCORE];
222 #define MEMPOOL_F_NO_SPREAD 0x0001
223 #define MEMPOOL_F_NO_CACHE_ALIGN 0x0002
224 #define MEMPOOL_F_SP_PUT 0x0004
225 #define MEMPOOL_F_SC_GET 0x0008
237 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
238 #define __MEMPOOL_STAT_ADD(mp, name, n) do { \
239 unsigned __lcore_id = rte_lcore_id(); \
240 if (__lcore_id < RTE_MAX_LCORE) { \
241 mp->stats[__lcore_id].name##_objs += n; \
242 mp->stats[__lcore_id].name##_bulk += 1; \
246 #define __MEMPOOL_STAT_ADD(mp, name, n) do {} while(0)
257 #define MEMPOOL_HEADER_SIZE(mp, pgn) (sizeof(*(mp)) + \
258 RTE_ALIGN_CEIL(((pgn) - RTE_DIM((mp)->elt_pa)) * \
259 sizeof ((mp)->elt_pa[0]), RTE_CACHE_LINE_SIZE))
264 #define MEMPOOL_IS_CONTIG(mp) \
265 ((mp)->pg_num == MEMPOOL_PG_NUM_DEFAULT && \
266 (mp)->phys_addr == (mp)->elt_pa[0])
310 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
311 #ifndef __INTEL_COMPILER
312 #pragma GCC diagnostic ignored "-Wcast-qual"
314 static inline void __mempool_check_cookies(
const struct rte_mempool *mp,
315 void *
const *obj_table_const,
316 unsigned n,
int free)
327 tmp = (
void *) obj_table_const;
328 obj_table = (
void **) tmp;
334 rte_panic(
"MEMPOOL: object is owned by another "
337 hdr = __mempool_get_header(obj);
338 cookie = hdr->cookie;
344 "obj=%p, mempool=%p, cookie=%" PRIx64
"\n",
345 obj, (
const void *) mp, cookie);
346 rte_panic(
"MEMPOOL: bad header cookie (put)\n");
350 else if (free == 1) {
354 "obj=%p, mempool=%p, cookie=%" PRIx64
"\n",
355 obj, (
const void *) mp, cookie);
356 rte_panic(
"MEMPOOL: bad header cookie (get)\n");
360 else if (free == 2) {
365 "obj=%p, mempool=%p, cookie=%" PRIx64
"\n",
366 obj, (
const void *) mp, cookie);
367 rte_panic(
"MEMPOOL: bad header cookie (audit)\n");
370 tlr = __mempool_get_trailer(obj);
371 cookie = tlr->cookie;
375 "obj=%p, mempool=%p, cookie=%" PRIx64
"\n",
376 obj, (
const void *) mp, cookie);
377 rte_panic(
"MEMPOOL: bad trailer cookie\n");
381 #ifndef __INTEL_COMPILER
382 #pragma GCC diagnostic error "-Wcast-qual"
385 #define __mempool_check_cookies(mp, obj_table_const, n, free) do {} while(0)
432 uint32_t elt_num,
size_t elt_sz,
size_t align,
433 const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift,
540 int socket_id,
unsigned flags);
640 int socket_id,
unsigned flags,
void *vaddr,
643 #ifdef RTE_LIBRTE_XEN_DOM0
726 rte_dom0_mempool_create(
const char *
name,
unsigned n,
unsigned elt_size,
730 int socket_id,
unsigned flags);
755 static inline void __attribute__((always_inline))
756 __mempool_put_bulk(struct
rte_mempool *mp,
void * const *obj_table,
757 unsigned n,
int is_mp)
759 #if RTE_MEMPOOL_CACHE_MAX_SIZE > 0
760 struct rte_mempool_cache *cache;
764 uint32_t cache_size = mp->cache_size;
765 uint32_t flushthresh = mp->cache_flushthresh;
769 __MEMPOOL_STAT_ADD(mp, put, n);
771 #if RTE_MEMPOOL_CACHE_MAX_SIZE > 0
773 if (
unlikely(cache_size == 0 || is_mp == 0 ||
774 lcore_id >= RTE_MAX_LCORE))
778 if (
unlikely(n > RTE_MEMPOOL_CACHE_MAX_SIZE))
781 cache = &mp->local_cache[lcore_id];
782 cache_objs = &cache->objs[cache->len];
792 for (index = 0; index < n; ++index, obj_table++)
793 cache_objs[index] = *obj_table;
797 if (cache->len >= flushthresh) {
799 cache->len - cache_size);
800 cache->len = cache_size;
809 #ifdef RTE_LIBRTE_MEMPOOL_DEBUG
812 rte_panic(
"cannot put objects in mempool\n");
816 rte_panic(
"cannot put objects in mempool\n");
837 static inline void __attribute__((always_inline))
841 __mempool_check_cookies(mp, obj_table, n, 0);
842 __mempool_put_bulk(mp, obj_table, n, 1);
859 __mempool_check_cookies(mp, obj_table, n, 0);
860 __mempool_put_bulk(mp, obj_table, n, 0);
877 static inline void __attribute__((always_inline))
881 __mempool_check_cookies(mp, obj_table, n, 0);
893 static inline void __attribute__((always_inline))
907 static inline void __attribute__((always_inline))
925 static inline void __attribute__((always_inline))
945 static inline int __attribute__((always_inline))
946 __mempool_get_bulk(struct
rte_mempool *mp,
void **obj_table,
947 unsigned n,
int is_mc)
950 #if RTE_MEMPOOL_CACHE_MAX_SIZE > 0
951 struct rte_mempool_cache *cache;
955 uint32_t cache_size = mp->cache_size;
958 if (
unlikely(cache_size == 0 || is_mc == 0 ||
959 n >= cache_size || lcore_id >= RTE_MAX_LCORE))
962 cache = &mp->local_cache[lcore_id];
963 cache_objs = cache->objs;
966 if (cache->len < n) {
968 uint32_t req = n + (cache_size - cache->len);
986 for (index = 0, len = cache->len - 1; index < n; ++index, len--, obj_table++)
987 *obj_table = cache_objs[len];
991 __MEMPOOL_STAT_ADD(mp, get_success, n);
1005 __MEMPOOL_STAT_ADD(mp, get_fail, n);
1007 __MEMPOOL_STAT_ADD(mp, get_success, n);
1030 static inline int __attribute__((always_inline))
1034 ret = __mempool_get_bulk(mp, obj_table, n, 1);
1036 __mempool_check_cookies(mp, obj_table, n, 1);
1059 static inline int __attribute__((always_inline))
1063 ret = __mempool_get_bulk(mp, obj_table, n, 0);
1065 __mempool_check_cookies(mp, obj_table, n, 1);
1091 static inline int __attribute__((always_inline))
1095 ret = __mempool_get_bulk(mp, obj_table, n,
1098 __mempool_check_cookies(mp, obj_table, n, 1);
1118 static inline int __attribute__((always_inline))
1140 static inline int __attribute__((always_inline))
1166 static inline int __attribute__((always_inline))
1203 static inline unsigned
1263 off = (
const char *)elt - (
const char *)mp->
elt_va_start;
static int rte_mempool_mc_get(struct rte_mempool *mp, void **obj_p)
static int rte_mempool_get(struct rte_mempool *mp, void **obj_p)
struct rte_mempool * rte_mempool_lookup(const char *name)
#define MEMPOOL_PG_NUM_DEFAULT
#define MEMPOOL_HEADER_SIZE(mp, pgn)
void(* rte_mempool_obj_iter_t)(void *, void *, void *, uint32_t)
phys_addr_t elt_pa[MEMPOOL_PG_NUM_DEFAULT]
unsigned rte_mempool_count(const struct rte_mempool *mp)
static int rte_mempool_sc_get(struct rte_mempool *mp, void **obj_p)
static int rte_ring_mp_enqueue_bulk(struct rte_ring *r, void *const *obj_table, unsigned n)
#define RTE_MEMPOOL_NAMESIZE
void rte_mempool_list_dump(FILE *f)
static void rte_mempool_sp_put_bulk(struct rte_mempool *mp, void *const *obj_table, unsigned n)
uint32_t cache_flushthresh
struct rte_mempool * rte_mempool_create(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, rte_mempool_ctor_t *mp_init, void *mp_init_arg, rte_mempool_obj_ctor_t *obj_init, void *obj_init_arg, int socket_id, unsigned flags)
#define RTE_LOG(l, t,...)
static int rte_mempool_empty(const struct rte_mempool *mp)
int rte_eal_has_hugepages(void)
static int rte_ring_sp_enqueue_bulk(struct rte_ring *r, void *const *obj_table, unsigned n)
void( rte_mempool_ctor_t)(struct rte_mempool *, void *)
void rte_mempool_dump(FILE *f, const struct rte_mempool *mp)
char name[RTE_MEMPOOL_NAMESIZE]
static int rte_mempool_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned n)
uint32_t rte_mempool_obj_iter(void *vaddr, uint32_t elt_num, size_t elt_sz, size_t align, const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift, rte_mempool_obj_iter_t obj_iter, void *obj_iter_arg)
phys_addr_t rte_mem_virt2phy(const void *virt)
void rte_mempool_audit(const struct rte_mempool *mp)
static int rte_ring_mc_dequeue_bulk(struct rte_ring *r, void **obj_table, unsigned n)
#define RTE_MEMPOOL_HEADER_COOKIE1
static int rte_ring_sc_dequeue_bulk(struct rte_ring *r, void **obj_table, unsigned n)
#define RTE_PTR_ADD(ptr, x)
static unsigned rte_mempool_free_count(const struct rte_mempool *mp)
static void rte_mempool_mp_put_bulk(struct rte_mempool *mp, void *const *obj_table, unsigned n)
static int rte_mempool_full(const struct rte_mempool *mp)
static unsigned rte_lcore_id(void)
static void rte_mempool_sp_put(struct rte_mempool *mp, void *obj)
void( rte_mempool_obj_ctor_t)(struct rte_mempool *, void *, void *, unsigned)
static int rte_mempool_mc_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned n)
static void rte_mempool_put_bulk(struct rte_mempool *mp, void *const *obj_table, unsigned n)
uint32_t rte_mempool_calc_obj_size(uint32_t elt_size, uint32_t flags, struct rte_mempool_objsz *sz)
void rte_log_set_history(int enable)
struct rte_mempool * rte_mempool_xmem_create(const char *name, unsigned n, unsigned elt_size, unsigned cache_size, unsigned private_data_size, rte_mempool_ctor_t *mp_init, void *mp_init_arg, rte_mempool_obj_ctor_t *obj_init, void *obj_init_arg, int socket_id, unsigned flags, void *vaddr, const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift)
unsigned private_data_size
static int rte_mempool_sc_get_bulk(struct rte_mempool *mp, void **obj_table, unsigned n)
void rte_mempool_walk(void(*func)(const struct rte_mempool *, void *arg), void *arg)
#define RTE_PTR_SUB(ptr, x)
static struct rte_mempool * rte_mempool_from_obj(void *obj)
uint32_t pg_num __rte_cache_aligned
#define RTE_MEMPOOL_HEADER_COOKIE2
size_t rte_mempool_xmem_size(uint32_t elt_num, size_t elt_sz, uint32_t pg_shift)
#define RTE_MEMPOOL_TRAILER_COOKIE
static void rte_mempool_mp_put(struct rte_mempool *mp, void *obj)
static void * rte_mempool_get_priv(struct rte_mempool *mp)
static void rte_mempool_put(struct rte_mempool *mp, void *obj)
ssize_t rte_mempool_xmem_usage(void *vaddr, uint32_t elt_num, size_t elt_sz, const phys_addr_t paddr[], uint32_t pg_num, uint32_t pg_shift)
static phys_addr_t rte_mempool_virt2phy(const struct rte_mempool *mp, const void *elt)
#define __rte_cache_aligned