Windows basic memory management
Subsystems (2)
PATCH v4 5/8 eal/windows: replace sys/queue.h with a complete one from FreeBSD
In reply to Dmitry KozlyukLimited version imported previously lacks at least SLIST macros. Import a complete file from FreeBSD, since its license exception is already approved by Technical Board.
--- lib/librte_eal/windows/include/sys/queue.h | 663 +++++++++++++++++++-- 1 file changed, 601 insertions(+), 62 deletions(-)
lib/librte_eal/windows/include/sys/queue.h +601 -62
@@ -8,7 +8,36 @@#define _SYS_QUEUE_H_/* -* This file defines tail queues. +* This file defines four types of data structures: singly-linked lists, +* singly-linked tail queues, lists and tail queues. +* +* A singly-linked list is headed by a single forward pointer. The elements +* are singly linked for minimum space and pointer manipulation overhead at +* the expense of O(n) removal for arbitrary elements. New elements can be +* added to the list after an existing element or at the head of the list. +* Elements being removed from the head of the list should use the explicit +* macro for this purpose for optimum efficiency. A singly-linked list may +* only be traversed in the forward direction. Singly-linked lists are ideal +* for applications with large datasets and few or no removals or for +* implementing a LIFO queue. +* +* A singly-linked tail queue is headed by a pair of pointers, one to the +* head of the list and the other to the tail of the list. The elements are +* singly linked for minimum space and pointer manipulation overhead at the +* expense of O(n) removal for arbitrary elements. New elements can be added +* to the list after an existing element, at the head of the list, or at the +* end of the list. Elements being removed from the head of the tail queue +* should use the explicit macro for this purpose for optimum efficiency. +* A singly-linked tail queue may only be traversed in the forward direction. +* Singly-linked tail queues are ideal for applications with large datasets +* and few or no removals or for implementing a FIFO queue. +* +* A list is headed by a single forward pointer (or an array of forward +* pointers for a hash table header). The elements are doubly linked +* so that an arbitrary element can be removed without a need to +* traverse the list. New elements can be added to the list before +* or after an existing element or at the head of the list. A list +* may be traversed in either direction. * * A tail queue is headed by a pair of pointers, one to the head of the * list and the other to the tail of the list. The elements are doubly
@@ -17,65 +46,93 @@* after an existing element, at the head of the list, or at the end of * the list. A tail queue may be traversed in either direction. * +* For details on the use of these macros, see the queue(3) manual page. +* * Below is a summary of implemented functions where: * + means the macro is available * - means the macro is not available * s means the macro is available but is slow (runs in O(n) time) * -* TAILQ -* _HEAD + -* _CLASS_HEAD + -* _HEAD_INITIALIZER + -* _ENTRY + -* _CLASS_ENTRY + -* _INIT + -* _EMPTY + -* _FIRST + -* _NEXT + -* _PREV + -* _LAST + -* _LAST_FAST + -* _FOREACH + -* _FOREACH_FROM + -* _FOREACH_SAFE + -* _FOREACH_FROM_SAFE + -* _FOREACH_REVERSE + -* _FOREACH_REVERSE_FROM + -* _FOREACH_REVERSE_SAFE + -* _FOREACH_REVERSE_FROM_SAFE + -* _INSERT_HEAD + -* _INSERT_BEFORE + -* _INSERT_AFTER + -* _INSERT_TAIL + -* _CONCAT + -* _REMOVE_AFTER - -* _REMOVE_HEAD - -* _REMOVE + -* _SWAP + +* SLIST LIST STAILQ TAILQ +* _HEAD + + + + +* _CLASS_HEAD + + + + +* _HEAD_INITIALIZER + + + + +* _ENTRY + + + + +* _CLASS_ENTRY + + + + +* _INIT + + + + +* _EMPTY + + + + +* _FIRST + + + + +* _NEXT + + + + +* _PREV - + - + +* _LAST - - + + +* _LAST_FAST - - - + +* _FOREACH + + + + +* _FOREACH_FROM + + + + +* _FOREACH_SAFE + + + + +* _FOREACH_FROM_SAFE + + + + +* _FOREACH_REVERSE - - - + +* _FOREACH_REVERSE_FROM - - - + +* _FOREACH_REVERSE_SAFE - - - + +* _FOREACH_REVERSE_FROM_SAFE - - - + +* _INSERT_HEAD + + + + +* _INSERT_BEFORE - + - + +* _INSERT_AFTER + + + + +* _INSERT_TAIL - - + + +* _CONCAT s s + + +* _REMOVE_AFTER + - + - +* _REMOVE_HEAD + - + - +* _REMOVE s + s + +* _SWAP + + + + * */ - -#ifdef __cplusplus -extern "C" { +#ifdef QUEUE_MACRO_DEBUG +#warn Use QUEUE_MACRO_DEBUG_TRACE and/or QUEUE_MACRO_DEBUG_TRASH +#define QUEUE_MACRO_DEBUG_TRACE +#define QUEUE_MACRO_DEBUG_TRASH #endif -/* - * List definitions. - */ -#define LIST_HEAD(name, type) \ -struct name { \ - struct type *lh_first; /* first element */ \ -} +#ifdef QUEUE_MACRO_DEBUG_TRACE +/* Store the last 2 places the queue element or head was altered */ +struct qm_trace { +unsigned long lastline; +unsigned long prevline; +const char *lastfile; +const char *prevfile; +}; + +#define TRACEBUF struct qm_trace trace; +#define TRACEBUF_INITIALIZER { __LINE__, 0, __FILE__, NULL } , + +#define QMD_TRACE_HEAD (head) do { \ + (head)->trace.prevline = (head)->trace.lastline; \ + (head)->trace.prevfile = (head)->trace.lastfile; \ + (head)->trace.lastline = __LINE__; \ + (head)->trace.lastfile = __FILE__; \ +} while (0) +#define QMD_TRACE_ELEM (elem) do { \ + (elem)->trace.prevline = (elem)->trace.lastline; \ + (elem)->trace.prevfile = (elem)->trace.lastfile; \ + (elem)->trace.lastline = __LINE__; \ + (elem)->trace.lastfile = __FILE__; \ +} while (0) + +#else /* !QUEUE_MACRO_DEBUG_TRACE */ #define QMD_TRACE_ELEM (elem)#define QMD_TRACE_HEAD (head)#define TRACEBUF #define TRACEBUF_INITIALIZER +#endif /* QUEUE_MACRO_DEBUG_TRACE */ +#ifdef QUEUE_MACRO_DEBUG_TRASH +#define QMD_SAVELINK (name, link) void **name = (void *)&(link) +#define TRASHIT (x) do {(x) = (void *)-1;} while (0) +#define QMD_IS_TRASHED (x) ((x) == (void *)(intptr_t)-1) +#else /* !QUEUE_MACRO_DEBUG_TRASH */ +#define QMD_SAVELINK (name, link)#define TRASHIT (x)#define QMD_IS_TRASHED (x) 0 - -#define QMD_SAVELINK(name, link) +#endif /* QUEUE_MACRO_DEBUG_TRASH */ #ifdef __cplusplus/*
@@ -86,6 +143,445 @@ struct name { \#define QUEUE_TYPEOF (type) struct type#endif +/* +* Singly-linked List declarations. +*/ +#define SLIST_HEAD (name, type) \ +struct name { \ + struct type *slh_first;/* first element */ \ +} + +#define SLIST_CLASS_HEAD(name, type) \ +struct name { \ + class type *slh_first; /* first element */ \ +} + +#define SLIST_HEAD_INITIALIZER (head) \ + { NULL } + +#define SLIST_ENTRY(type) \ +struct { \ + struct type *sle_next; /* next element */ \ +} + +#define SLIST_CLASS_ENTRY(type) \ +struct { \ + class type *sle_next; /* next element */ \ +} + +/* +* Singly-linked List functions. +*/ +#if (defined(_KERNEL) && defined(INVARIANTS )) +#define QMD_SLIST_CHECK_PREVPTR (prevp, elm) do { \ + if (*(prevp) != (elm)) \ + panic("Bad prevptr *(%p) == %p != %p", \ + (prevp), *(prevp), (elm)); \ +} while (0) +#else +#define QMD_SLIST_CHECK_PREVPTR (prevp, elm) +#endif + +#define SLIST_CONCAT (head1, head2, type, field) do { \ + QUEUE_TYPEOF(type) *curelm = SLIST_FIRST(head1); \ + if (curelm == NULL) { \ + if ((SLIST_FIRST(head1) = SLIST_FIRST(head2)) != NULL) \ + SLIST_INIT(head2); \ + } else if (SLIST_FIRST(head2) != NULL) { \ + while (SLIST_NEXT(curelm, field) != NULL) \ + curelm = SLIST_NEXT(curelm, field); \ + SLIST_NEXT(curelm, field) = SLIST_FIRST(head2); \ + SLIST_INIT(head2); \ + } \ +} while (0) + +#define SLIST_EMPTY (head) ((head)->slh_first == NULL) + +#define SLIST_FIRST (head) ((head)->slh_first) + +#define SLIST_FOREACH (var, head, field) \ + for ((var) = SLIST_FIRST((head)); \ + (var); \ + (var) = SLIST_NEXT((var), field)) + +#define SLIST_FOREACH_FROM (var, head, field) \ + for ((var) = ((var) ? (var) : SLIST_FIRST((head))); \ + (var); \ + (var) = SLIST_NEXT((var), field)) + +#define SLIST_FOREACH_SAFE (var, head, field, tvar) \ + for ((var) = SLIST_FIRST((head)); \ + (var) && ((tvar) = SLIST_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define SLIST_FOREACH_FROM_SAFE (var, head, field, tvar) \ + for ((var) = ((var) ? (var) : SLIST_FIRST((head))); \ + (var) && ((tvar) = SLIST_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define SLIST_FOREACH_PREVPTR (var, varp, head, field) \ + for ((varp) = &SLIST_FIRST((head)); \ + ((var) = *(varp)) != NULL; \ + (varp) = &SLIST_NEXT((var), field)) + +#define SLIST_INIT (head) do { \ + SLIST_FIRST((head)) = NULL; \ +} while (0) + +#define SLIST_INSERT_AFTER (slistelm, elm, field) do { \ + SLIST_NEXT((elm), field) = SLIST_NEXT((slistelm), field); \ + SLIST_NEXT((slistelm), field) = (elm); \ +} while (0) + +#define SLIST_INSERT_HEAD (head, elm, field) do { \ + SLIST_NEXT((elm), field) = SLIST_FIRST((head)); \ + SLIST_FIRST((head)) = (elm); \ +} while (0) + +#define SLIST_NEXT (elm, field) ((elm)->field.sle_next) + +#define SLIST_REMOVE (head, elm, type, field) do { \ + QMD_SAVELINK(oldnext, (elm)->field.sle_next); \ + if (SLIST_FIRST((head)) == (elm)) { \ + SLIST_REMOVE_HEAD((head), field); \ + } \ + else { \ + QUEUE_TYPEOF(type) *curelm = SLIST_FIRST(head); \ + while (SLIST_NEXT(curelm, field) != (elm)) \ + curelm = SLIST_NEXT(curelm, field); \ + SLIST_REMOVE_AFTER(curelm, field); \ + } \ + TRASHIT(*oldnext); \ +} while (0) + +#define SLIST_REMOVE_AFTER (elm, field) do { \ + SLIST_NEXT(elm, field) = \ + SLIST_NEXT(SLIST_NEXT(elm, field), field); \ +} while (0) + +#define SLIST_REMOVE_HEAD (head, field) do { \ + SLIST_FIRST((head)) = SLIST_NEXT(SLIST_FIRST((head)), field); \ +} while (0) + +#define SLIST_REMOVE_PREVPTR (prevp, elm, field) do { \ + QMD_SLIST_CHECK_PREVPTR(prevp, elm); \ + *(prevp) = SLIST_NEXT(elm, field); \ + TRASHIT((elm)->field.sle_next); \ +} while (0) + +#define SLIST_SWAP (head1, head2, type) do { \ + QUEUE_TYPEOF(type) *swap_first = SLIST_FIRST(head1); \ + SLIST_FIRST(head1) = SLIST_FIRST(head2); \ + SLIST_FIRST(head2) = swap_first; \ +} while (0) + +/* +* Singly-linked Tail queue declarations. +*/ +#define STAILQ_HEAD (name, type) \ +struct name { \ + struct type *stqh_first;/* first element */ \ +struct type **stqh_last;/* addr of last next element */ \ +} + +#define STAILQ_CLASS_HEAD(name, type) \ +struct name { \ + class type *stqh_first; /* first element */ \ + class type **stqh_last; /* addr of last next element */ \ +} + +#define STAILQ_HEAD_INITIALIZER (head) \ + { NULL, &(head).stqh_first } + +#define STAILQ_ENTRY(type) \ +struct { \ + struct type *stqe_next; /* next element */ \ +} + +#define STAILQ_CLASS_ENTRY(type) \ +struct { \ + class type *stqe_next; /* next element */ \ +} + +/* +* Singly-linked Tail queue functions. +*/ +#define STAILQ_CONCAT (head1, head2) do { \ + if (!STAILQ_EMPTY((head2))) { \ + *(head1)->stqh_last = (head2)->stqh_first; \ + (head1)->stqh_last = (head2)->stqh_last; \ + STAILQ_INIT((head2)); \ + } \ +} while (0) + +#define STAILQ_EMPTY (head) ((head)->stqh_first == NULL) + +#define STAILQ_FIRST (head) ((head)->stqh_first) + +#define STAILQ_FOREACH (var, head, field) \ + for((var) = STAILQ_FIRST((head)); \ + (var); \ + (var) = STAILQ_NEXT((var), field)) + +#define STAILQ_FOREACH_FROM (var, head, field) \ + for ((var) = ((var) ? (var) : STAILQ_FIRST((head))); \ + (var); \ + (var) = STAILQ_NEXT((var), field)) + +#define STAILQ_FOREACH_SAFE (var, head, field, tvar) \ + for ((var) = STAILQ_FIRST((head)); \ + (var) && ((tvar) = STAILQ_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define STAILQ_FOREACH_FROM_SAFE (var, head, field, tvar) \ + for ((var) = ((var) ? (var) : STAILQ_FIRST((head))); \ + (var) && ((tvar) = STAILQ_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define STAILQ_INIT (head) do { \ + STAILQ_FIRST((head)) = NULL; \ + (head)->stqh_last = &STAILQ_FIRST((head)); \ +} while (0) + +#define STAILQ_INSERT_AFTER (head, tqelm, elm, field) do { \ + if ((STAILQ_NEXT((elm), field) = STAILQ_NEXT((tqelm), field)) == NULL)\ + (head)->stqh_last = &STAILQ_NEXT((elm), field); \ + STAILQ_NEXT((tqelm), field) = (elm); \ +} while (0) + +#define STAILQ_INSERT_HEAD (head, elm, field) do { \ + if ((STAILQ_NEXT((elm), field) = STAILQ_FIRST((head))) == NULL) \ + (head)->stqh_last = &STAILQ_NEXT((elm), field); \ + STAILQ_FIRST((head)) = (elm); \ +} while (0) + +#define STAILQ_INSERT_TAIL (head, elm, field) do { \ + STAILQ_NEXT((elm), field) = NULL; \ + *(head)->stqh_last = (elm); \ + (head)->stqh_last = &STAILQ_NEXT((elm), field); \ +} while (0) + +#define STAILQ_LAST (head, type, field) \ + (STAILQ_EMPTY((head)) ? NULL : \ + __containerof((head)->stqh_last, \ + QUEUE_TYPEOF(type), field.stqe_next)) + +#define STAILQ_NEXT (elm, field) ((elm)->field.stqe_next) + +#define STAILQ_REMOVE (head, elm, type, field) do { \ + QMD_SAVELINK(oldnext, (elm)->field.stqe_next); \ + if (STAILQ_FIRST((head)) == (elm)) { \ + STAILQ_REMOVE_HEAD((head), field); \ + } \ + else { \ + QUEUE_TYPEOF(type) *curelm = STAILQ_FIRST(head); \ + while (STAILQ_NEXT(curelm, field) != (elm)) \ + curelm = STAILQ_NEXT(curelm, field); \ + STAILQ_REMOVE_AFTER(head, curelm, field); \ + } \ + TRASHIT(*oldnext); \ +} while (0) + +#define STAILQ_REMOVE_AFTER (head, elm, field) do { \ + if ((STAILQ_NEXT(elm, field) = \ + STAILQ_NEXT(STAILQ_NEXT(elm, field), field)) == NULL) \ + (head)->stqh_last = &STAILQ_NEXT((elm), field); \ +} while (0) + +#define STAILQ_REMOVE_HEAD (head, field) do { \ + if ((STAILQ_FIRST((head)) = \ + STAILQ_NEXT(STAILQ_FIRST((head)), field)) == NULL) \ + (head)->stqh_last = &STAILQ_FIRST((head)); \ +} while (0) + +#define STAILQ_SWAP (head1, head2, type) do { \ + QUEUE_TYPEOF(type) *swap_first = STAILQ_FIRST(head1); \ + QUEUE_TYPEOF(type) **swap_last = (head1)->stqh_last; \ + STAILQ_FIRST(head1) = STAILQ_FIRST(head2); \ + (head1)->stqh_last = (head2)->stqh_last; \ + STAILQ_FIRST(head2) = swap_first; \ + (head2)->stqh_last = swap_last; \ + if (STAILQ_EMPTY(head1)) \ + (head1)->stqh_last = &STAILQ_FIRST(head1); \ + if (STAILQ_EMPTY(head2)) \ + (head2)->stqh_last = &STAILQ_FIRST(head2); \ +} while (0) + + +/* +* List declarations. +*/ +#define LIST_HEAD(name, type) \ +struct name { \ + struct type *lh_first; /* first element */ \ +} + +#define LIST_CLASS_HEAD(name, type) \ +struct name { \ + class type *lh_first; /* first element */ \ +} + +#define LIST_HEAD_INITIALIZER (head) \ + { NULL } + +#define LIST_ENTRY (type) \ +struct { \ + struct type *le_next;/* next element */ \ +struct type **le_prev;/* address of previous next element */ \ +} + +#define LIST_CLASS_ENTRY(type) \ +struct { \ + class type *le_next; /* next element */ \ + class type **le_prev; /* address of previous next element */ \ +} + +/* +* List functions. +*/ + +#if (defined(_KERNEL) && defined(INVARIANTS )) +/* +* QMD_LIST_CHECK_HEAD(LIST_HEAD *head, LIST_ENTRY NAME) +* +* If the list is non-empty, validates that the first element of the list +* points back at 'head.' +*/ +#define QMD_LIST_CHECK_HEAD (head, field) do { \ + if (LIST_FIRST((head)) != NULL && \ + LIST_FIRST((head))->field.le_prev != \ + &LIST_FIRST((head))) \ + panic("Bad list head %p first->prev != head", (head)); \ +} while (0) + +/* +* QMD_LIST_CHECK_NEXT(TYPE *elm, LIST_ENTRY NAME) +* +* If an element follows 'elm' in the list, validates that the next element +* points back at 'elm.' +*/ +#define QMD_LIST_CHECK_NEXT (elm, field) do { \ + if (LIST_NEXT((elm), field) != NULL && \ + LIST_NEXT((elm), field)->field.le_prev != \ + &((elm)->field.le_next)) \ + panic("Bad link elm %p next->prev != elm", (elm)); \ +} while (0) + +/* +* QMD_LIST_CHECK_PREV(TYPE *elm, LIST_ENTRY NAME) +* +* Validates that the previous element (or head of the list) points to 'elm.' +*/ +#define QMD_LIST_CHECK_PREV (elm, field) do { \ + if (*(elm)->field.le_prev != (elm)) \ + panic("Bad link elm %p prev->next != elm", (elm)); \ +} while (0) +#else +#define QMD_LIST_CHECK_HEAD (head, field) +#define QMD_LIST_CHECK_NEXT (elm, field) +#define QMD_LIST_CHECK_PREV (elm, field) +#endif /* (_KERNEL && INVARIANTS) */ + +#define LIST_CONCAT (head1, head2, type, field) do { \ + QUEUE_TYPEOF(type) *curelm = LIST_FIRST(head1); \ + if (curelm == NULL) { \ + if ((LIST_FIRST(head1) = LIST_FIRST(head2)) != NULL) { \ + LIST_FIRST(head2)->field.le_prev = \ + &LIST_FIRST((head1)); \ + LIST_INIT(head2); \ + } \ + } else if (LIST_FIRST(head2) != NULL) { \ + while (LIST_NEXT(curelm, field) != NULL) \ + curelm = LIST_NEXT(curelm, field); \ + LIST_NEXT(curelm, field) = LIST_FIRST(head2); \ + LIST_FIRST(head2)->field.le_prev = &LIST_NEXT(curelm, field); \ + LIST_INIT(head2); \ + } \ +} while (0) + +#define LIST_EMPTY (head) ((head)->lh_first == NULL) + +#define LIST_FIRST (head) ((head)->lh_first) + +#define LIST_FOREACH (var, head, field) \ + for ((var) = LIST_FIRST((head)); \ + (var); \ + (var) = LIST_NEXT((var), field)) + +#define LIST_FOREACH_FROM (var, head, field) \ + for ((var) = ((var) ? (var) : LIST_FIRST((head))); \ + (var); \ + (var) = LIST_NEXT((var), field)) + +#define LIST_FOREACH_SAFE (var, head, field, tvar) \ + for ((var) = LIST_FIRST((head)); \ + (var) && ((tvar) = LIST_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define LIST_FOREACH_FROM_SAFE (var, head, field, tvar) \ + for ((var) = ((var) ? (var) : LIST_FIRST((head))); \ + (var) && ((tvar) = LIST_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define LIST_INIT (head) do { \ + LIST_FIRST((head)) = NULL; \ +} while (0) + +#define LIST_INSERT_AFTER (listelm, elm, field) do { \ + QMD_LIST_CHECK_NEXT(listelm, field); \ + if ((LIST_NEXT((elm), field) = LIST_NEXT((listelm), field)) != NULL)\ + LIST_NEXT((listelm), field)->field.le_prev = \ + &LIST_NEXT((elm), field); \ + LIST_NEXT((listelm), field) = (elm); \ + (elm)->field.le_prev = &LIST_NEXT((listelm), field); \ +} while (0) + +#define LIST_INSERT_BEFORE (listelm, elm, field) do { \ + QMD_LIST_CHECK_PREV(listelm, field); \ + (elm)->field.le_prev = (listelm)->field.le_prev; \ + LIST_NEXT((elm), field) = (listelm); \ + *(listelm)->field.le_prev = (elm); \ + (listelm)->field.le_prev = &LIST_NEXT((elm), field); \ +} while (0) + +#define LIST_INSERT_HEAD (head, elm, field) do { \ + QMD_LIST_CHECK_HEAD((head), field); \ + if ((LIST_NEXT((elm), field) = LIST_FIRST((head))) != NULL) \ + LIST_FIRST((head))->field.le_prev = &LIST_NEXT((elm), field);\ + LIST_FIRST((head)) = (elm); \ + (elm)->field.le_prev = &LIST_FIRST((head)); \ +} while (0) + +#define LIST_NEXT (elm, field) ((elm)->field.le_next) + +#define LIST_PREV (elm, head, type, field) \ + ((elm)->field.le_prev == &LIST_FIRST((head)) ? NULL : \ + __containerof((elm)->field.le_prev, \ + QUEUE_TYPEOF(type), field.le_next)) + +#define LIST_REMOVE (elm, field) do { \ + QMD_SAVELINK(oldnext, (elm)->field.le_next); \ + QMD_SAVELINK(oldprev, (elm)->field.le_prev); \ + QMD_LIST_CHECK_NEXT(elm, field); \ + QMD_LIST_CHECK_PREV(elm, field); \ + if (LIST_NEXT((elm), field) != NULL) \ + LIST_NEXT((elm), field)->field.le_prev = \ + (elm)->field.le_prev; \ + *(elm)->field.le_prev = LIST_NEXT((elm), field); \ + TRASHIT(*oldnext); \ + TRASHIT(*oldprev); \ +} while (0) + +#define LIST_SWAP (head1, head2, type, field) do { \ + QUEUE_TYPEOF(type) *swap_tmp = LIST_FIRST(head1); \ + LIST_FIRST((head1)) = LIST_FIRST((head2)); \ + LIST_FIRST((head2)) = swap_tmp; \ + if ((swap_tmp = LIST_FIRST((head1))) != NULL) \ + swap_tmp->field.le_prev = &LIST_FIRST((head1)); \ + if ((swap_tmp = LIST_FIRST((head2))) != NULL) \ + swap_tmp->field.le_prev = &LIST_FIRST((head2)); \ +} while (0) +/* * Tail queue declarations. */
@@ -123,10 +619,58 @@ struct { \/* * Tail queue functions. */ +#if (defined(_KERNEL) && defined(INVARIANTS )) +/* +* QMD_TAILQ_CHECK_HEAD(TAILQ_HEAD *head, TAILQ_ENTRY NAME) +* +* If the tailq is non-empty, validates that the first element of the tailq +* points back at 'head.' +*/ +#define QMD_TAILQ_CHECK_HEAD (head, field) do { \ + if (!TAILQ_EMPTY(head) && \ + TAILQ_FIRST((head))->field.tqe_prev != \ + &TAILQ_FIRST((head))) \ + panic("Bad tailq head %p first->prev != head", (head)); \ +} while (0) + +/* +* QMD_TAILQ_CHECK_TAIL(TAILQ_HEAD *head, TAILQ_ENTRY NAME) +* +* Validates that the tail of the tailq is a pointer to pointer to NULL. +*/ +#define QMD_TAILQ_CHECK_TAIL (head, field) do { \ + if (*(head)->tqh_last != NULL) \ + panic("Bad tailq NEXT(%p->tqh_last) != NULL", (head)); \ +} while (0) + +/* +* QMD_TAILQ_CHECK_NEXT(TYPE *elm, TAILQ_ENTRY NAME) +* +* If an element follows 'elm' in the tailq, validates that the next element +* points back at 'elm.' +*/ +#define QMD_TAILQ_CHECK_NEXT (elm, field) do { \ + if (TAILQ_NEXT((elm), field) != NULL && \ + TAILQ_NEXT((elm), field)->field.tqe_prev != \ + &((elm)->field.tqe_next)) \ + panic("Bad link elm %p next->prev != elm", (elm)); \ +} while (0) + +/* +* QMD_TAILQ_CHECK_PREV(TYPE *elm, TAILQ_ENTRY NAME) +* +* Validates that the previous element (or head of the tailq) points to 'elm.' +*/ +#define QMD_TAILQ_CHECK_PREV (elm, field) do { \ + if (*(elm)->field.tqe_prev != (elm)) \ + panic("Bad link elm %p prev->next != elm", (elm)); \ +} while (0) +#else #define QMD_TAILQ_CHECK_HEAD (head, field)#define QMD_TAILQ_CHECK_TAIL (head, headname)#define QMD_TAILQ_CHECK_NEXT (elm, field)#define QMD_TAILQ_CHECK_PREV (elm, field) +#endif /* (_KERNEL && INVARIANTS) */ #define TAILQ_CONCAT (head1, head2, field) do { \ if (!TAILQ_EMPTY(head2)) { \
@@ -191,9 +735,8 @@ struct { \#define TAILQ_INSERT_AFTER (head, listelm, elm, field) do { \ QMD_TAILQ_CHECK_NEXT(listelm, field); \ - TAILQ_NEXT((elm), field) = TAILQ_NEXT((listelm), field); \ - if (TAILQ_NEXT((listelm), field) != NULL) \ - TAILQ_NEXT((elm), field)->field.tqe_prev = \ + if ((TAILQ_NEXT((elm), field) = TAILQ_NEXT((listelm), field)) != NULL)\ + TAILQ_NEXT((elm), field)->field.tqe_prev = \ &TAILQ_NEXT((elm), field); \ else { \ (head)->tqh_last = &TAILQ_NEXT((elm), field); \
@@ -217,8 +760,7 @@ struct { \#define TAILQ_INSERT_HEAD (head, elm, field) do { \ QMD_TAILQ_CHECK_HEAD(head, field); \ - TAILQ_NEXT((elm), field) = TAILQ_FIRST((head)); \ - if (TAILQ_FIRST((head)) != NULL) \ + if ((TAILQ_NEXT((elm), field) = TAILQ_FIRST((head))) != NULL) \ TAILQ_FIRST((head))->field.tqe_prev = \ &TAILQ_NEXT((elm), field); \ else \
@@ -250,21 +792,24 @@ struct { \* you may want to prefetch the last data element. */ #define TAILQ_LAST_FAST (head, type, field) \ - (TAILQ_EMPTY(head) ? NULL : __containerof((head)->tqh_last, \ - QUEUE_TYPEOF(type), field.tqe_next)) + (TAILQ_EMPTY(head) ? NULL : __containerof((head)->tqh_last, QUEUE_TYPEOF(type), field.tqe_next))#define TAILQ_NEXT (elm, field) ((elm)->field.tqe_next)#define TAILQ_PREV (elm, headname, field) \ (*(((struct headname *)((elm)->field.tqe_prev))->tqh_last)) +#define TAILQ_PREV_FAST (elm, head, type, field) \ + ((elm)->field.tqe_prev == &(head)->tqh_first ? NULL : \ + __containerof((elm)->field.tqe_prev, QUEUE_TYPEOF(type), field.tqe_next)) +#define TAILQ_REMOVE (head, elm, field) do { \ QMD_SAVELINK(oldnext, (elm)->field.tqe_next); \ QMD_SAVELINK(oldprev, (elm)->field.tqe_prev); \ QMD_TAILQ_CHECK_NEXT(elm, field); \ QMD_TAILQ_CHECK_PREV(elm, field); \ if ((TAILQ_NEXT((elm), field)) != NULL) \ - TAILQ_NEXT((elm), field)->field.tqe_prev = \ + TAILQ_NEXT((elm), field)->field.tqe_prev = \ (elm)->field.tqe_prev; \ else { \ (head)->tqh_last = (elm)->field.tqe_prev; \
@@ -277,26 +822,20 @@ struct { \ } while (0)#define TAILQ_SWAP (head1, head2, type, field) do { \ - QUEUE_TYPEOF(type) * swap_first = (head1)->tqh_first; \ - QUEUE_TYPEOF(type) * *swap_last = (head1)->tqh_last; \ + QUEUE_TYPEOF(type) *swap_first = (head1)->tqh_first; \ + QUEUE_TYPEOF(type) **swap_last = (head1)->tqh_last; \ (head1)->tqh_first = (head2)->tqh_first; \ (head1)->tqh_last = (head2)->tqh_last; \ (head2)->tqh_first = swap_first; \ (head2)->tqh_last = swap_last; \ - swap_first = (head1)->tqh_first; \ - if (swap_first != NULL) \ + if ((swap_first = (head1)->tqh_first) != NULL) \ swap_first->field.tqe_prev = &(head1)->tqh_first; \ else \ (head1)->tqh_last = &(head1)->tqh_first; \ - swap_first = (head2)->tqh_first; \ - if (swap_first != NULL) \ + if ((swap_first = (head2)->tqh_first) != NULL) \ swap_first->field.tqe_prev = &(head2)->tqh_first; \ else \ (head2)->tqh_last = &(head2)->tqh_first; \ } while (0) -#ifdef __cplusplus -} -#endif - -#endif /* _SYS_QUEUE_H_ */ +#endif /* !_SYS_QUEUE_H_ */
Signature · 1 line
2.25.1
PATCH v4 6/8 eal/windows: improve CPU and NUMA node detection
In reply to Dmitry Kozlyuk1. Map CPU cores to their respective NUMA nodes as reported by system.
2. Support systems with more than 64 cores (multiple processor groups).
3. Fix magic constants, styling issues, and compiler warnings.
4. Add EAL private function to map DPDK socket ID to NUMA node number.
--- lib/librte_eal/windows/eal_lcore.c | 185 +++++++++++++++++---------- lib/librte_eal/windows/eal_windows.h | 10 ++ 2 files changed, 124 insertions(+), 71 deletions(-)
lib/librte_eal/windows/eal_lcore.c +114 -71
@@ -3,103 +3,146 @@*/ #include <pthread.h> +#include <stdbool.h> #include <stdint.h> #include <rte_common.h> +#include <rte_debug.h> +#include <rte_lcore.h> +#include <rte_os.h> #include "eal_private.h" #include "eal_thread.h" #include "eal_windows.h" -/* global data structure that contains the CPU map */ -static struct _wcpu_map { -unsigned int total_procs; -unsigned int proc_sockets; -unsigned int proc_cores; -unsigned int reserved; -struct _win_lcore_map { -uint8_t socket_id; -uint8_t core_id; - } wlcore_map[RTE_MAX_LCORE ]; -} wcpu_map = {0 }; - -/* -* Create a map of all processors and associated cores on the system -*/ +/** Number of logical processors (cores) in a processor group (32 or 64). */ +#define EAL_PROCESSOR_GROUP_SIZE (sizeof(KAFFINITY) * CHAR_BIT) + +struct lcore_map { +uint8_t socket_id; +uint8_t core_id; +}; + +struct socket_map { +uint16_t node_id; +}; + +struct cpu_map { +unsigned int socket_count; +unsigned int lcore_count; +struct lcore_map lcores[RTE_MAX_LCORE ]; +struct socket_map sockets[RTE_MAX_NUMA_NODES ]; +}; + +static struct cpu_map cpu_map = {0 }; +void -eal_create_cpu_map () +eal_create_cpu_map (void ) { - wcpu_map.total_procs = -GetActiveProcessorCount (ALL_PROCESSOR_GROUPS ); - -LOGICAL_PROCESSOR_RELATIONSHIP lprocRel; -DWORD lprocInfoSize =0 ; -BOOL ht_enabled = FALSE; - -/* First get the processor package information */ - lprocRel = RelationProcessorPackage; -/* Determine the size of buffer we need (pass NULL) */ -GetLogicalProcessorInformationEx (lprocRel,NULL , &lprocInfoSize); - wcpu_map.proc_sockets = lprocInfoSize /48 ; - - lprocInfoSize =0 ; -/* Next get the processor core information */ - lprocRel = RelationProcessorCore; -GetLogicalProcessorInformationEx (lprocRel,NULL , &lprocInfoSize); - wcpu_map.proc_cores = lprocInfoSize /48 ; - -if (wcpu_map.total_procs > wcpu_map.proc_cores) - ht_enabled = TRUE; - -/* Distribute the socket and core ids appropriately -* across the logical cores. For now, split the cores -* equally across the sockets. -*/ -unsigned int lcore =0 ; -for (unsigned int socket =0 ; socket < - wcpu_map.proc_sockets; ++socket) { -for (unsigned int core =0 ; - core < (wcpu_map.proc_cores / wcpu_map.proc_sockets); - ++core) { - wcpu_map.wlcore_map[lcore] - .socket_id = socket; - wcpu_map.wlcore_map[lcore] - .core_id = core; - lcore++; -if (ht_enabled) { - wcpu_map.wlcore_map[lcore] - .socket_id = socket; - wcpu_map.wlcore_map[lcore] - .core_id = core; - lcore++; +SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *infos, *info; +DWORD infos_size; +bool full = false; + + infos_size =0 ; +if (!GetLogicalProcessorInformationEx ( + RelationNumaNode,NULL , &infos_size)) { +DWORD error =GetLastError (); +if (error !=ERROR_INSUFFICIENT_BUFFER ) { +rte_panic ("cannot get NUMA node info size, error %lu" , +GetLastError ()); + } + } + + infos =malloc (infos_size); +if (infos ==NULL ) { +rte_panic ("cannot allocate memory for NUMA node information" ); +return ; + } + +if (!GetLogicalProcessorInformationEx ( + RelationNumaNode, infos, &infos_size)) { +rte_panic ("cannot get NUMA node information, error %lu" , +GetLastError ()); + } + + info = infos; +while ((uint8_t *)info - (uint8_t *)infos < infos_size) { +unsigned int node_id = info->NumaNode.NodeNumber; +GROUP_AFFINITY *cores = &info->NumaNode.GroupMask; +struct lcore_map *lcore; +unsigned int i, socket_id; + +/* NUMA node may be reported multiple times if it includes +* cores from different processor groups, e. g. 80 cores +* of a physical processor comprise one NUMA node, but two +* processor groups, because group size is limited by 32/64. +*/ +for (socket_id =0 ; socket_id < cpu_map.socket_count; + socket_id++) { +if (cpu_map.sockets[socket_id].node_id == node_id) +break ; + } + +if (socket_id == cpu_map.socket_count) { +if (socket_id ==RTE_DIM (cpu_map.sockets)) { + full = true; + goto exit; } + + cpu_map.sockets[socket_id].node_id = node_id; + cpu_map.socket_count++; + } + +for (i =0 ; i <EAL_PROCESSOR_GROUP_SIZE ; i++) { +if ((cores->Mask & ((KAFFINITY )1 << i)) ==0 ) +continue ; + +if (cpu_map.lcore_count ==RTE_DIM (cpu_map.lcores)) { + full = true; + goto exit; + } + + lcore = &cpu_map.lcores[cpu_map.lcore_count]; + lcore->socket_id = socket_id; + lcore->core_id = + cores->Group *EAL_PROCESSOR_GROUP_SIZE + i; + cpu_map.lcore_count++; } + + info = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)( + (uint8_t *)info + info->Size); + } + +exit: +if (full) { +/* RTE_LOG() may not be available, but this is important. */ +fprintf (stderr,"Enumerated maximum of %u NUMA nodes and %u cores\n" , + cpu_map.socket_count, cpu_map.lcore_count); } + +free (infos); } -/* -* Check if a cpu is present by the presence of the cpu information for it -*/ int eal_cpu_detected (unsigned int lcore_id) { -return (lcore_id < wcpu_map.total_procs); +return lcore_id < cpu_map.lcore_count; } -/* -* Get CPU socket id for a logical core -*/ unsigned eal_cpu_socket_id (unsigned int lcore_id) { -return wcpu_map.wlcore_map[lcore_id].socket_id; +return cpu_map.lcores[lcore_id].socket_id; } -/* -* Get CPU socket id (NUMA node) for a logical core -*/ unsigned eal_cpu_core_id (unsigned int lcore_id) { -return wcpu_map.wlcore_map[lcore_id].core_id; +return cpu_map.lcores[lcore_id].core_id; +} + +unsigned int +eal_socket_numa_node (unsigned int socket_id) +{ +return cpu_map.sockets[socket_id].node_id; }
lib/librte_eal/windows/eal_windows.h +10 -0
@@ -26,4 +26,14 @@ void eal_create_cpu_map(void);*/ int eal_thread_create (pthread_t *thread); +/** +* Get system NUMA node number for a socket ID. +* +* @param socket_id +* Valid EAL socket ID. +* @return +* NUMA node number to use with Win32 API. +*/ +unsigned int eal_socket_numa_node (unsigned int socket_id); +#endif /* _EAL_WINDOWS_H_ */
Signature · 1 line
2.25.1
PATCH v4 7/8 eal/windows: initialize hugepage info
In reply to Dmitry KozlyukAdd hugepages discovery ("large pages" in Windows terminology) and update documentation for required privilege setup. Only 2MB hugepages are supported and their number is estimated roughly due to the lack or unstable status of suitable OS APIs.
--- config/meson.build | 2 + doc/guides/windows_gsg/build_dpdk.rst | 20 ----- doc/guides/windows_gsg/index.rst | 1 + doc/guides/windows_gsg/run_apps.rst | 47 +++++++++++ lib/librte_eal/windows/eal.c | 14 ++++ lib/librte_eal/windows/eal_hugepages.c | 108 +++++++++++++++++++++++++ lib/librte_eal/windows/meson.build | 1 + 7 files changed, 173 insertions(+), 20 deletions(-) create mode 100644 doc/guides/windows_gsg/run_apps.rst create mode 100644 lib/librte_eal/windows/eal_hugepages.c
config/meson.build +2 -0
@@ -271,6 +271,8 @@ if is_windows if cc.get_id() == 'gcc' add_project_arguments('-D__USE_MINGW_ANSI_STDIO', language: 'c') endif + + add_project_link_arguments('-ladvapi32', language: 'c') endif if get_option('b_lto')
doc/guides/windows_gsg/build_dpdk.rst +0 -20
@@ -111,23 +111,3 @@ Depending on the distribution, paths in this file may need adjustments. meson --cross-file config/x86/meson_mingw.txt -Dexamples=helloworld build ninja -C build - - -Run the helloworld example -========================== - -Navigate to the examples in the build directory and run `dpdk-helloworld.exe`. - -.. code-block:: console - - cd C:\Users\me\dpdk\build\examples - dpdk-helloworld.exe - hello from core 1 - hello from core 3 - hello from core 0 - hello from core 2 - -Note for MinGW-w64: applications are linked to ``libwinpthread-1.dll`` -by default. To run the example, either add toolchain executables directory -to the PATH or copy the library to the working directory. -Alternatively, static linking may be used (mind the LGPLv2.1 license).
doc/guides/windows_gsg/index.rst +1 -0
@@ -12,3 +12,4 @@ Getting Started Guide for Windows intro build_dpdk + run_apps
doc/guides/windows_gsg/run_apps.rst new file +47 -0
47 lines not on this page · raw message
lib/librte_eal/windows/eal.c +14 -0
@@ -18,8 +18,11 @@#include <eal_options.h> #include <eal_private.h> +#include "eal_hugepages.h" #include "eal_windows.h" +#define MEMSIZE_IF_NO_HUGE_PAGE (64ULL * 1024ULL * 1024ULL) +/* Allow the application to print its usage message too if set */ static rte_usage_hook_t rte_application_usage_hook;
@@ -242,6 +245,17 @@ rte_eal_init(int argc, char **argv)if (fctret <0 )exit (1 ); +if (!internal_config.no_hugetlbfs && (eal_hugepage_info_init () <0 )) { +rte_eal_init_alert ("Cannot get hugepage information" ); + rte_errno =EACCES ; +return -1 ; + } + +if (internal_config.memory ==0 && !internal_config.force_sockets) { +if (internal_config.no_hugetlbfs) + internal_config.memory =MEMSIZE_IF_NO_HUGE_PAGE ; + } +eal_thread_init_master (rte_config.master_lcore);RTE_LCORE_FOREACH_SLAVE (i) {
lib/librte_eal/windows/eal_hugepages.c new file +108 -0
108 lines not on this page · raw message
lib/librte_eal/windows/meson.build +1 -0
@@ -6,6 +6,7 @@ subdir('include') sources += files( 'eal.c', 'eal_debug.c', + 'eal_hugepages.c', 'eal_lcore.c', 'eal_thread.c', 'getopt.c',
Signature · 1 line
2.25.1
PATCH v4 8/8 eal/windows: implement basic memory management
In reply to Dmitry KozlyukBasic memory management supports core libraries and PMDs operating in IOVA as PA mode. It uses a kernel-mode driver, virt2phys, to obtain IOVAs of hugepages allocated from user-mode. Multi-process mode is not implemented and is forcefully disabled at startup.
--- config/meson.build | 12 +- doc/guides/windows_gsg/run_apps.rst | 54 +- lib/librte_eal/common/eal_common_memzone.c | 7 + lib/librte_eal/common/meson.build | 10 + lib/librte_eal/common/rte_malloc.c | 9 + lib/librte_eal/rte_eal_exports.def | 119 ++ lib/librte_eal/windows/eal.c | 144 ++ lib/librte_eal/windows/eal_memalloc.c | 418 ++++++ lib/librte_eal/windows/eal_memory.c | 1155 +++++++++++++++++ lib/librte_eal/windows/eal_mp.c | 103 ++ lib/librte_eal/windows/eal_windows.h | 90 ++ lib/librte_eal/windows/include/meson.build | 1 + lib/librte_eal/windows/include/rte_os.h | 4 + .../windows/include/rte_virt2phys.h | 34 + lib/librte_eal/windows/include/rte_windows.h | 2 + lib/librte_eal/windows/include/unistd.h | 3 + lib/librte_eal/windows/meson.build | 5 + 17 files changed, 2164 insertions(+), 6 deletions(-) create mode 100644 lib/librte_eal/windows/eal_memalloc.c create mode 100644 lib/librte_eal/windows/eal_memory.c create mode 100644 lib/librte_eal/windows/eal_mp.c create mode 100644 lib/librte_eal/windows/include/rte_virt2phys.h
config/meson.build +9 -3
@@ -264,15 +264,21 @@ if is_freebsd endif if is_windows - # Minimum supported API is Windows 7. - add_project_arguments('-D_WIN32_WINNT=0x0601', language: 'c') + # VirtualAlloc2() is available since Windows 10 / Server 2016. + add_project_arguments('-D_WIN32_WINNT=0x0A00', language: 'c') # Use MinGW-w64 stdio, because DPDK assumes ANSI-compliant formatting. if cc.get_id() == 'gcc' add_project_arguments('-D__USE_MINGW_ANSI_STDIO', language: 'c') endif - add_project_link_arguments('-ladvapi32', language: 'c') + # Contrary to docs, VirtualAlloc2() is exported by mincore.lib + # in Windows SDK, while MinGW exports it by advapi32.a. + if is_ms_linker + add_project_link_arguments('-lmincore', language: 'c') + endif + + add_project_link_arguments('-ladvapi32', '-lsetupapi', language: 'c') endif if get_option('b_lto')
doc/guides/windows_gsg/run_apps.rst +51 -3
68 lines not on this page · raw message
lib/librte_eal/common/eal_common_memzone.c +7 -0
@@ -19,7 +19,14 @@#include <rte_errno.h> #include <rte_string_fns.h> #include <rte_common.h> + +#ifndef RTE_EXEC_ENV_WINDOWS #include <rte_eal_trace.h> +#else +#define rte_eal_trace_memzone_reserve (...) +#define rte_eal_trace_memzone_lookup (...) +#define rte_eal_trace_memzone_free (...) +#endif #include "malloc_heap.h" #include "malloc_elem.h"
lib/librte_eal/common/meson.build +10 -0
@@ -9,11 +9,21 @@ if is_windows 'eal_common_class.c', 'eal_common_devargs.c', 'eal_common_errno.c', + 'eal_common_fbarray.c', 'eal_common_launch.c', 'eal_common_lcore.c', 'eal_common_log.c', + 'eal_common_mcfg.c', + 'eal_common_memalloc.c', + 'eal_common_memory.c', + 'eal_common_memzone.c', 'eal_common_options.c', + 'eal_common_string_fns.c', + 'eal_common_tailqs.c', 'eal_common_thread.c', + 'malloc_elem.c', + 'malloc_heap.c', + 'rte_malloc.c', 'rte_option.c', ) subdir_done()
lib/librte_eal/common/rte_malloc.c +9 -0
@@ -20,7 +20,16 @@#include <rte_lcore.h> #include <rte_common.h> #include <rte_spinlock.h> + +#ifndef RTE_EXEC_ENV_WINDOWS #include <rte_eal_trace.h> +#else +/* Suppress -Wempty-body for tracepoints used as "if" body. */ +#define rte_eal_trace_mem_malloc (...) do {} while (0) +#define rte_eal_trace_mem_zmalloc (...) do {} while (0) +#define rte_eal_trace_mem_realloc (...) do {} while (0) +#define rte_eal_trace_mem_free (...) do {} while (0) +#endif #include <rte_malloc.h> #include "malloc_elem.h"
lib/librte_eal/rte_eal_exports.def +119 -0
128 lines not on this page · raw message
lib/librte_eal/windows/eal.c +144 -0
168 lines not on this page · raw message
lib/librte_eal/windows/eal_memalloc.c new file +418 -0
418 lines not on this page · raw message
lib/librte_eal/windows/eal_memory.c new file +1155 -0
1,155 lines not on this page · raw message
lib/librte_eal/windows/eal_mp.c new file +103 -0
103 lines not on this page · raw message
lib/librte_eal/windows/eal_windows.h +90 -0
102 lines not on this page · raw message
lib/librte_eal/windows/include/meson.build +1 -0
@@ -5,5 +5,6 @@ includes += include_directories('.') headers += files( 'rte_os.h', + 'rte_virt2phys.h', 'rte_windows.h', )
lib/librte_eal/windows/include/rte_os.h +4 -0
@@ -36,6 +36,10 @@ extern "C" {#define strncasecmp (s1, s2, count) _strnicmp(s1, s2, count) +#define open _open +#define close _close +#define unlink _unlink +/* cpu_set macros implementation */ #define RTE_CPU_AND (dst, src1, src2) CPU_AND(dst, src1, src2)#define RTE_CPU_OR (dst, src1, src2) CPU_OR(dst, src1, src2)
lib/librte_eal/windows/include/rte_virt2phys.h new file +34 -0
@@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright (c) 2020 Dmitry Kozlyuk +*/ + +/** +* @file virt2phys driver interface +*/ + +/** +* Driver device interface GUID {539c2135-793a-4926-afec-d3a1b61bbc8a}. +*/ +DEFINE_GUID (GUID_DEVINTERFACE_VIRT2PHYS , +0x539c2135 ,0x793a ,0x4926 , +0xaf ,0xec ,0xd3 ,0xa1 ,0xb6 ,0x1b ,0xbc ,0x8a ); + +/** +* Driver device type for IO control codes. +*/ +#define VIRT2PHYS_DEVTYPE 0x8000 + +/** +* Translate a valid non-paged virtual address to a physical address. +* +* Note: A physical address zero (0) is reported if input address +* is paged out or not mapped. However, if input is a valid mapping +* of I/O port 0x0000, output is also zero. There is no way +* to distinguish between these cases by return value only. +* +* Input: a non-paged virtual address (PVOID). +* +* Output: the corresponding physical address (LARGE_INTEGER). +*/ +#define IOCTL_VIRT2PHYS_TRANSLATE CTL_CODE( \ + VIRT2PHYS_DEVTYPE, 0x800, METHOD_BUFFERED, FILE_ANY_ACCESS)
lib/librte_eal/windows/include/rte_windows.h +2 -0
@@ -23,6 +23,8 @@#include <basetsd.h> #include <psapi.h> +#include <setupapi.h> +#include <winioctl.h> /* Have GUIDs defined. */ #ifndef INITGUID
lib/librte_eal/windows/include/unistd.h +3 -0
@@ -9,4 +9,7 @@* as Microsoft libc does not contain unistd.h. This may be removed * in future releases. */ + +#include <io.h> +#endif /* _UNISTD_H_ */
lib/librte_eal/windows/meson.build +5 -0
@@ -8,6 +8,11 @@ sources += files( 'eal_debug.c', 'eal_hugepages.c', 'eal_lcore.c', + 'eal_memalloc.c', + 'eal_memory.c', + 'eal_mp.c', 'eal_thread.c', 'getopt.c', ) + +dpdk_conf.set10('RTE_EAL_NUMA_AWARE_HUGEPAGES', true)
Signature · 1 line
2.25.1
Re: [dpdk-dev] [PATCH v4 8/8] eal/windows: implement basic memory management
Quoted 7 lines · Dmitry Kozlyuk
Basic memory management supports core libraries and PMDs operating in IOVA as PA mode. It uses a kernel-mode driver, virt2phys, to obtain IOVAs of hugepages allocated from user-mode. Multi-process mode is not implemented and is forcefully disabled at startup.
Signed-off-by Dmitry Kozlyuk <[email protected]>---
[snip]
Quoted 12 lines
lib/librte_eal/windows/eal_memory.c new file +12
-0+void * +eal_mem_reserve (void *requested_addr,size_t size,int flags) +{ +void *virt; + +/* Windows requires hugepages to be committed. */ +if (flags &EAL_RESERVE_HUGEPAGES ) { + rte_errno =ENOTSUP ; +return NULL ; + } + + virt = VirtualAlloc2(GetCurrentProcess(), requested_addr, size,
lib/librte_eal/windows/eal_memory.c line 289
Quoted 10 lines
lib/librte_eal/windows/eal_memory.c new file +10
-0+MEM_RESERVE |MEM_RESERVE_PLACEHOLDER ,PAGE_NOACCESS , + NULL, 0); +if (virt ==NULL ) { +DWORD err =GetLastError (); +RTE_LOG_WIN32_ERR ("VirtualAlloc2()" ); +set_errno_from_win32_alloc_error (err); + } + +if ((flags &EAL_RESERVE_FORCE_ADDRESS ) && (virt != requested_addr)) { +if (!VirtualFree (virt,0 ,MEM_RELEASE ))
Shouldn't this be VirtualFreeEx() here?
lib/librte_eal/windows/eal_memory.c line 297
Quoted 8 lines
lib/librte_eal/windows/eal_memory.c new file +8
-0+RTE_LOG_WIN32_ERR ("VirtualFree()" ); + rte_errno =ENOMEM ; +return NULL ; + } + +return virt; +} +
ranjit m.
Re: [dpdk-dev] [PATCH v4 8/8] eal/windows: implement basic memory management
Quoted 1 line · On 2020-04-28 18:18 GMT-0700 Ranjit Menon wrote:
On 4/28/2020 4:50 PM, Dmitry Kozlyuk wrote:
[snip]
Quoted 10 lines
Quoted 10 lines
lib/librte_eal/windows/eal_memory.c new file +10
-0+void * +eal_mem_reserve (void *requested_addr,size_t size,int flags) +{ +void *virt; + +/* Windows requires hugepages to be committed. */ +if (flags &EAL_RESERVE_HUGEPAGES ) { + rte_errno =ENOTSUP ; +return NULL ; + }
Quoted 8 lines
lib/librte_eal/windows/eal_memory.c line 285
Quoted 8 lines
lib/librte_eal/windows/eal_memory.c new file +8
-0+ + virt =VirtualAlloc2 (GetCurrentProcess (), requested_addr, size, +MEM_RESERVE |MEM_RESERVE_PLACEHOLDER ,PAGE_NOACCESS , +NULL ,0 ); +if (virt ==NULL ) { +DWORD err =GetLastError (); +RTE_LOG_WIN32_ERR ("VirtualAlloc2()" ); +set_errno_from_win32_alloc_error (err);
return NULL;
is also missing here, thanks for making me re-check this part.
Quoted 6 lines
lib/librte_eal/windows/eal_memory.c line 289
Quoted 4 lines
lib/librte_eal/windows/eal_memory.c new file +4
-0+ } + +if ((flags &EAL_RESERVE_FORCE_ADDRESS ) && (virt != requested_addr)) { +if (!VirtualFree (virt,0 ,MEM_RELEASE ))Shouldn't this be
VirtualFreeEx()here?
You're right, there were a few more places like this within the file.
Signature · 1 line
Dmitry Kozlyuk
Re: [dpdk-dev] [PATCH v4 8/8] eal/windows: implement basic memory management
In reply to Dmitry KozlyukQuoted 7 lines · Dmitry Kozlyuk
Basic memory management supports core libraries and PMDs operating in IOVA as PA mode. It uses a kernel-mode driver, virt2phys, to obtain IOVAs of hugepages allocated from user-mode. Multi-process mode is not implemented and is forcefully disabled at startup.
Signed-off-by Dmitry Kozlyuk <[email protected]>---
Lots of duplication... I wonder if it would be possible to share at least some of this code in common. Tracking down bugs because of duplicated code desync is always a pain...
<snip>
Quoted 7 lines
lib/librte_eal/common/eal_common_memzone.c +0
-0@@ -19,7 +19,14 @@#include <rte_errno.h> #include <rte_string_fns.h>
lib/librte_eal/common/eal_common_memzone.c line 30
Quoted 10 lines
lib/librte_eal/common/eal_common_memzone.c +7
-0#include <rte_common.h> + +#ifndef RTE_EXEC_ENV_WINDOWS #include <rte_eal_trace.h> +#else +#define rte_eal_trace_memzone_reserve (...) +#define rte_eal_trace_memzone_lookup (...) +#define rte_eal_trace_memzone_free (...) +#endif
Is it possible for rte_eal_trace.h to implement this workaround instead? It wouldn't be very wise to have to have this in each file that depends on rte_eal_trace.h.
Quoted 10 lines
lib/librte_eal/common/eal_common_memzone.c +0
-0#include "malloc_heap.h" #include "malloc_elem.h" lib/librte_eal/common/meson.build +0
-0@@ -9,11 +9,21 @@ if is_windows 'eal_common_class.c', 'eal_common_devargs.c', 'eal_common_errno.c',
<snip>
lib/librte_eal/common/meson.build rte_eal_init(int argc, char **argv) · line 353
Quoted 12 lines
/* Launch threads, called at application init(). */int
rte_eal_init(int argc, char **argv)lib/librte_eal/common/meson.build +5
-0@@ -245,6 +346,13 @@ rte_eal_init(int argc, char **argv) if (fctret < 0) exit(1); + /* Prevent creation of shared memory files. */ + if (internal_config.no_shconf == 0) { + RTE_LOG(WARNING, EAL, "Multi-process support is requested, " + "but not available.\n"); + internal_config.no_shconf = 1;
In the future i would like to deprecate no_shconf because it's a strict subset of in_memory mode and serves the same purpose. Might i suggest using in_memory flag instead? IIRC no_shconf is automatically set when you set in_memory mode.
Signature · 2 lines
Thanks, Anatoly
Re: [dpdk-dev] [PATCH v4 8/8] eal/windows: implement basic memory management
Quoted 4 lines · Anatoly
On 29-Apr-20 12:50 AM, Dmitry Kozlyuk wrote:
Lots of duplication... I wonder if it would be possible to share at least some of this code in common. Tracking down bugs because of duplicated code desync is always a pain...
This was the main question of the cover letter :)
Dmitry Malloy explained to me recently that even internally Windows has no notion of preallocated hugepages and "memory types" (as memseg_primary_init describes it). Since Windows EAL is not going to support multi-process any time soon (if ever), maybe these reservations are not needed and memory manger should create MSLs and enforce socket limits dynamically? This way most of the duplicated code can be removed, I think. Or does MSL reservation serve some other purposes?
Quoted 13 lines
Quoted 13 lines
lib/librte_eal/common/eal_common_memzone.c +4
-0@@ -19,7 +19,14 @@#include <rte_errno.h> #include <rte_string_fns.h> #include <rte_common.h> + +#ifndef RTE_EXEC_ENV_WINDOWS #include <rte_eal_trace.h> +#else +#define rte_eal_trace_memzone_reserve (...)
Quoted 8 lines
lib/librte_eal/common/eal_common_memzone.c line 30
Quoted 4 lines
lib/librte_eal/common/eal_common_memzone.c +3
-0+#define rte_eal_trace_memzone_lookup (...) +#define rte_eal_trace_memzone_free (...) +#endif Is it possible for
rte_eal_trace.hto implement this workaround instead? It wouldn't be very wise to have to have this in each file that depends onrte_eal_trace.h.
I can add a patch that makes each tracepoint a no-op on Windows.
We discussed this issue (spreading workarounds) 2020-04-30 on Windows community call. The proper solution would be supporting trace on Windows, but IIRC no one is yet directly assigned to do that.
[snip]
Quoted 10 lines
lib/librte_eal/common/eal_common_memzone.c
Quoted 5 lines
lib/librte_eal/common/eal_common_memzone.c +5
-0+/* Prevent creation of shared memory files. */ +if (internal_config.no_shconf ==0 ) { +RTE_LOG (WARNING ,EAL ,"Multi-process support is requested, " +"but not available.\n" ); + internal_config.no_shconf =1 ;In the future i would like to deprecate
no_shconfbecause it's a strict subset ofin_memorymode and serves the same purpose. Might i suggest usingin_memoryflag instead? IIRCno_shconfis automatically set when you setin_memorymode.
OK, thanks.
Signature · 1 line
Dmitry Kozlyuk
Re: [dpdk-dev] [PATCH v4 8/8] eal/windows: implement basic memory management
Quoted 6 lines · Dmitry Kozlyuk
Quoted 4 lines · Anatoly
On 29-Apr-20 12:50 AM, Dmitry Kozlyuk wrote:
Lots of duplication... I wonder if it would be possible to share at least some of this code in common. Tracking down bugs because of duplicated code desync is always a pain...
Quoted 8 lines
This was the main question of the cover letter :)
Dmitry Malloy explained to me recently that even internally Windows has no notion of preallocated hugepages and "memory types" (asmemseg_primary_initdescribes it). Since Windows EAL is not going to support multi-process any time soon (if ever), maybe these reservations are not needed and memory manger should create MSLs and enforce socket limits dynamically? This way most of the duplicated code can be removed, I think. Or does MSL reservation serve some other purposes?
MSL reservation serves the purpose of dynamically expanding memory usage. If there is no notion of NUMA nodes or multiple page sizes, then you can greatly simplify the code, but you'd still need *some* usage of MSL's if you plan to support dynamically allocating memory, or supporting externally allocated memory (i assume it's out of scope for now, since you can't do IOVA as VA).
So, yes, you could greatly simplify the memory management code *if* you were to go FreeBSD way and not allow dynamic page reservation. If you do, however, then i would guess that you'd end up writing something that's largely similar to existing Linux code (minus multiprocess) and so would just be duplicating effort.
Quoted 23 lines
Quoted 14 lines
Quoted 14 lines
lib/librte_eal/common/eal_common_memzone.c +4
-0@@ -19,7 +19,14 @@#include <rte_errno.h> #include <rte_string_fns.h> #include <rte_common.h> + +#ifndef RTE_EXEC_ENV_WINDOWS #include <rte_eal_trace.h> +#else +#define rte_eal_trace_memzone_reserve (...)Quoted 8 lines
lib/librte_eal/common/eal_common_memzone.c line 30
Quoted 4 lines
lib/librte_eal/common/eal_common_memzone.c +3
-0+#define rte_eal_trace_memzone_lookup (...) +#define rte_eal_trace_memzone_free (...) +#endif Is it possible for
rte_eal_trace.hto implement this workaround instead? It wouldn't be very wise to have to have this in each file that depends onrte_eal_trace.h.
Quoted 5 lines
I can add a patch that makes each tracepoint a no-op on Windows.
We discussed this issue (spreading workarounds) 2020-04-30 on Windows community call. The proper solution would be supporting trace on Windows, but IIRC no one is yet directly assigned to do that.
Apologies, i'm not plugged into those discussions :)
Signature · 2 lines
Thanks, Anatoly
Re: [dpdk-dev] [PATCH v4 8/8] eal/windows: implement basic memory management
Quoted 16 lines · Anatoly
Quoted 6 lines · Dmitry Kozlyuk
Quoted 4 lines · Anatoly
On 29-Apr-20 12:50 AM, Dmitry Kozlyuk wrote:
Lots of duplication... I wonder if it would be possible to share at least some of this code in common. Tracking down bugs because of duplicated code desync is always a pain...Quoted 8 lines
This was the main question of the cover letter :)
Dmitry Malloy explained to me recently that even internally Windows has no notion of preallocated hugepages and "memory types" (asmemseg_primary_initdescribes it). Since Windows EAL is not going to support multi-process any time soon (if ever), maybe these reservations are not needed and memory manger should create MSLs and enforce socket limits dynamically? This way most of the duplicated code can be removed, I think. Or does MSL reservation serve some other purposes?
Quoted 2 lines
MSL reservation serves the purpose of dynamically expanding memory usage.
But expansion is limited during init, because alloc_more_mem_on_socket() works with existing MSLs, correct? No going to change anything there, just trying to understand MM internals.
Quoted 5 lines
If there is no notion of NUMA nodes or multiple page sizes, then you can greatly simplify the code, but you'd still need *some* usage of MSL's if you plan to support dynamically allocating memory, or supporting externally allocated memory (i assume it's out of scope for now, since you can't do IOVA as VA).
Windows is NUMA-aware and it supports both 2MB and 1GB hugepages (although Windows EAL does not at the moment, because Win32 API is not yet official). What I meant is that Windows does not reserve hugepages like Linux does with vm.nr_hugepages or hugepage-related kernel options. So logic duplicated from Linux EAL makes sense for Windows. The bulk of it can be extracted to some common file, but it will not be truly common, rather "everything but FreeBSD". Against it is a point that Windows MM may change significantly, but I honestly can't come up with an example of how can those duplicated parts may require adjustments.
Quoted 5 lines
So, yes, you could greatly simplify the memory management code *if* you were to go FreeBSD way and not allow dynamic page reservation. If you do, however, then i would guess that you'd end up writing something that's largely similar to existing Linux code (minus multiprocess) and so would just be duplicating effort.
Signature · 1 line
Dmitry Kozlyuk
Re: [dpdk-dev] [PATCH v4 8/8] eal/windows: implement basic memory management
Quoted 17 lines · Dmitry Kozlyuk
Quoted 16 lines · Anatoly
Quoted 14 lines · Dmitry Kozlyuk
Quoted 4 lines · Anatoly
On 29-Apr-20 12:50 AM, Dmitry Kozlyuk wrote:
Lots of duplication... I wonder if it would be possible to share at least some of this code in common. Tracking down bugs because of duplicated code desync is always a pain...This was the main question of the cover letter :)
Dmitry Malloy explained to me recently that even internally Windows has no notion of preallocated hugepages and "memory types" (asmemseg_primary_initdescribes it). Since Windows EAL is not going to support multi-process any time soon (if ever), maybe these reservations are not needed and memory manger should create MSLs and enforce socket limits dynamically? This way most of the duplicated code can be removed, I think. Or does MSL reservation serve some other purposes?
Quoted 6 lines
Quoted 2 lines
MSL reservation serves the purpose of dynamically expanding memory usage.
But expansion is limited during init, because
alloc_more_mem_on_socket()works with existing MSLs, correct? No going to change anything there, just trying to understand MM internals.
Yes, system memory MSLs will stay the same for the duration of the program, they are not allocated on the fly. External memory will create/destroy MSLs but those too aren't allocated dynamically - there's a fixed number of MSLs, and if you run out, well, you're out of luck.
So no, the MSLs themselves don't get allocated/deallocated at runtime *if* they belong to system (internal) memory.
Quoted 7 lines
Quoted 6 lines
If there is no notion of NUMA nodes or multiple page sizes, then you can greatly simplify the code, but you'd still need *some* usage of MSL's if you plan to support dynamically allocating memory, or supporting externally allocated memory (i assume it's out of scope for now, since you can't do IOVA as VA).
Quoted 9 lines
Windows is NUMA-aware and it supports both 2MB and 1GB hugepages (although Windows EAL does not at the moment, because Win32 API is not yet official). What I meant is that Windows does not reserve hugepages like Linux does with vm.nr_hugepages or hugepage-related kernel options. So logic duplicated from Linux EAL makes sense for Windows. The bulk of it can be extracted to some common file, but it will not be truly common, rather "everything but FreeBSD". Against it is a point that Windows MM may change significantly, but I honestly can't come up with an example of how can those duplicated parts may require adjustments.
Fair enough. It's your EAL, you can do as you like :)
Quoted 7 lines
Quoted 6 lines
So, yes, you could greatly simplify the memory management code *if* you were to go FreeBSD way and not allow dynamic page reservation. If you do, however, then i would guess that you'd end up writing something that's largely similar to existing Linux code (minus multiprocess) and so would just be duplicating effort.
Signature · 2 lines
Thanks, Anatoly
Re: [dpdk-dev] [PATCH v4 8/8] eal/windows: implement basic memory management
In reply to Dmitry KozlyukHi Dmitry,
I'm using your latest memory management patchset and getting an error in
the function VirualAlloc2 in eal_mem_commit, error code: 0x57 (ERROR_INVALID_PARAMETER). I'm using Windows server 2019 build 17763, and followed the steps to Grant *Lock pages in memory* Privilege.
The parameters that are sent to the function are:GetCurrentProcess() is -1.requested_addr is 0x0000025b`93800000.
Size is 0x200000 (sysInfo.dwAllocationGranularity is 0x10000). Flags is 0x20007000.
Also, Socket_id is 0.
The call stack is:
00 dpdk_mempool_test!eal_mem_commit+0x253
01 dpdk_mempool_test!alloc_seg+0x1b0
02 dpdk_mempool_test!alloc_seg_walk+0x2a1
03 dpdk_mempool_test!rte_memseg_list_walk_thread_unsafe+0x81 04 dpdk_mempool_test!eal_memalloc_alloc_seg_bulk+0x1a5 05 dpdk_mempool_test!alloc_pages_on_heap+0x13a
06 dpdk_mempool_test!try_expand_heap_primary+0x1dc 07 dpdk_mempool_test!try_expand_heap+0xf5
08 dpdk_mempool_test!alloc_more_mem_on_socket+0x693 09 dpdk_mempool_test!malloc_heap_alloc_on_heap_id+0x2a7 0a dpdk_mempool_test!malloc_heap_alloc+0x184
0b dpdk_mempool_test!malloc_socket+0xf9
0c dpdk_mempool_test!rte_malloc_socket+0x39
0d dpdk_mempool_test!rte_zmalloc_socket+0x31
0e dpdk_mempool_test!rte_zmalloc+0x2d
0f dpdk_mempool_test!rte_mempool_create_empty+0x1c9 10 dpdk_mempool_test!rte_mempool_create+0xf8
Quoted 2,616 lines
-----Original Message-----
From: Dmitry Kozlyuk <[email protected]>
Sent: Wednesday, April 29, 2020 2:50 AM
To: [email protected]
Cc: Dmitry Malloy (MESHCHANINOV) <[email protected]>; Narcisa Ana Maria Vasile <[email protected]>; Fady Bader <[email protected]>; Tal Shnaiderman <[email protected]>; Dmitry Kozlyuk <[email protected]>; Thomas Monjalon <[email protected]>; Harini Ramakrishnan <[email protected]>; Omar Cardona <[email protected]>; Pallavi Kadam <[email protected]>; Ranjit Menon <[email protected]>; John McNamara <[email protected]>; Marko Kovacevic <[email protected]>; Anatoly Burakov <[email protected]>
Subject: [PATCH v4 8/8] eal/windows: implement basic memory managementBasic memory management supports core libraries and PMDs operating in IOVA as PA mode. It uses a kernel-mode driver, virt2phys, to obtain IOVAs of hugepages allocated from user-mode. Multi-process mode is not implemented and is forcefully disabled at startup.
Signed-off-by Dmitry Kozlyuk <[email protected]>--- config/meson.build | 12 +- doc/guides/windows_gsg/run_apps.rst | 54 +- lib/librte_eal/common/eal_common_memzone.c | 7 + lib/librte_eal/common/meson.build | 10 + lib/librte_eal/common/rte_malloc.c | 9 + lib/librte_eal/rte_eal_exports.def | 119 ++ lib/librte_eal/windows/eal.c | 144 ++ lib/librte_eal/windows/eal_memalloc.c | 418 ++++++ lib/librte_eal/windows/eal_memory.c | 1155 +++++++++++++++++ lib/librte_eal/windows/eal_mp.c | 103 ++ lib/librte_eal/windows/eal_windows.h | 90 ++ lib/librte_eal/windows/include/meson.build | 1 + lib/librte_eal/windows/include/rte_os.h | 4 + .../windows/include/rte_virt2phys.h | 34 + lib/librte_eal/windows/include/rte_windows.h | 2 + lib/librte_eal/windows/include/unistd.h | 3 + lib/librte_eal/windows/meson.build | 5 + 17 files changed, 2164 insertions(+), 6 deletions(-) create mode 100644 lib/librte_eal/windows/eal_memalloc.c create mode 100644 lib/librte_eal/windows/eal_memory.c create mode 100644 lib/librte_eal/windows/eal_mp.c create mode 100644 lib/librte_eal/windows/include/rte_virt2phys.hconfig/meson.build +2
-2@@ -264,15 +264,21 @@ if is_freebsd endif if is_windows - # Minimum supported API is Windows 7. - add_project_arguments('-D_WIN32_WINNT=0x0601', language: 'c') + # VirtualAlloc2() is available since Windows 10 / Server 2016. + add_project_arguments('-D_WIN32_WINNT=0x0A00', language: 'c') # Use MinGW-w64 stdio, because DPDK assumes ANSI-compliantformatting.
config/meson.build +0
-0if cc.get_id() == 'gcc' add_project_arguments('-D__USE_MINGW_ANSI_STDIO',language: 'c')
config/meson.build +7
-1endif - add_project_link_arguments('-ladvapi32', language: 'c') + # Contrary to docs, VirtualAlloc2() is exported by mincore.lib + # in Windows SDK, while MinGW exports it by advapi32.a. + if is_ms_linker + add_project_link_arguments('-lmincore', language: 'c') + endif + + add_project_link_arguments('-ladvapi32', '-lsetupapi', language: 'c') endif if get_option('b_lto')doc/guides/windows_gsg/run_apps.rst +2
-2@@ -7,10 +7,10 @@ Running DPDK Applications Grant *Lock pages in memory* Privilege -------------------------------------- -Use of hugepages ("large pages" in Windows terminolocy) requires +Use of hugepages ("large pages" in Windows terminology) requires ``SeLockMemoryPrivilege`` for the user running an application. -1. Open *Local Security Policy* snap in, either: +1. Open *Local Security Policy* snap-in, either: * Control Panel / Computer Management / Local Security Policy; * or Win+R, type ``secpol``, press Enter.@@ -24,7 +24,55 @@ Use of hugepages ("large pages" in Windowsterminolocy) requires
See
Large-Page Support_ in MSDN for details.-.. _Large-page Support:
https://eur03.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdocs.
microsoft.com%2Fen-us%2Fwindows%2Fwin32%2Fmemory%2Flarge-page- support&data=02%7C01%7Cfady%40mellanox.com%7C3c6bd806786c47 9c1e3008d7ebcef213%7Ca652971c7d2e4d9ba6a4d149256f461b%7C0%7C0%7 C637237146372884132&sdata=l41r9A%2FflDmhDInC9L84zvPkO4efbRImC YmtI9IOkT4%3D&reserved=0
+.. _Large-Page Support:
https://eur03.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdocs.
microsoft.com%2Fen-us%2Fwindows%2Fwin32%2Fmemory%2Flarge-page- support&data=02%7C01%7Cfady%40mellanox.com%7C3c6bd806786c47 9c1e3008d7ebcef213%7Ca652971c7d2e4d9ba6a4d149256f461b%7C0%7C0%7 C637237146372884132&sdata=l41r9A%2FflDmhDInC9L84zvPkO4efbRImC YmtI9IOkT4%3D&reserved=0doc/guides/windows_gsg/run_apps.rst +38
-0+ + +Load virt2phys Driver +--------------------- + +Access to physical addresses is provided by a kernel-mode driver, virt2phys. +It is mandatory at least for using hardware PMDs, but may also be required +for mempools. + +Refer to documentation in ``dpdk-kmods`` repository for details on system +setup, driver build and installation. This driver is not signed, so signature +checking must be disabled to load it. + +.. warning:: + + Disabling driver signature enforcement weakens OS security. + It is discouraged in production environments. + +Compiled package consists of ``virt2phys.inf``, ``virt2phys.cat``, +and ``virt2phys.sys``. It can be installed as follows +from Elevated Command Prompt: + +.. code-block:: console + + pnputil /add-driver Z:\path\to\virt2phys.inf /install + +On Windows Server additional steps are required: + +1. From Device Manager, Action menu, select "Add legacy hardware". +2. It will launch the "Add Hardware Wizard". Click "Next". +3. Select second option "Install the hardware that I manually select + from a list (Advanced)". +4. On the next screen, "Kernel bypass" will be shown as a device class. +5. Select it, and click "Next". +6. The previously installed drivers will now be installed for the + "Virtual to physical address translator" device. + +When loaded successfully, the driver is shown in *Device Manager* as*Virtual
doc/guides/windows_gsg/run_apps.rst +10
-0+to physical address translator* device under *Kernel bypass* category. +Installed driver persists across reboots. + +If DPDK is unable to communicate with the driver, a warning is printed +on initialization (debug-level logs provide more details): + +.. code-block:: text + + EAL: Cannot open virt2phys driver interface + Run the ``helloworld`` Examplelib/librte_eal/common/eal_common_memzone.c +7
-0@@ -19,7 +19,14 @@#include <rte_errno.h> #include <rte_string_fns.h> #include <rte_common.h> + +#ifndef RTE_EXEC_ENV_WINDOWS #include <rte_eal_trace.h> +#else +#define rte_eal_trace_memzone_reserve (...) +#define rte_eal_trace_memzone_lookup (...) +#define rte_eal_trace_memzone_free (...) +#endif #include "malloc_heap.h" #include "malloc_elem.h" lib/librte_eal/common/meson.build +10
-0@@ -9,11 +9,21 @@ if is_windows 'eal_common_class.c', 'eal_common_devargs.c', 'eal_common_errno.c', + 'eal_common_fbarray.c', 'eal_common_launch.c', 'eal_common_lcore.c', 'eal_common_log.c', + 'eal_common_mcfg.c', + 'eal_common_memalloc.c', + 'eal_common_memory.c', + 'eal_common_memzone.c', 'eal_common_options.c', + 'eal_common_string_fns.c', + 'eal_common_tailqs.c', 'eal_common_thread.c', + 'malloc_elem.c', + 'malloc_heap.c', + 'rte_malloc.c', 'rte_option.c', ) subdir_done()lib/librte_eal/common/rte_malloc.c +9
-0@@ -20,7 +20,16 @@#include <rte_lcore.h> #include <rte_common.h> #include <rte_spinlock.h> + +#ifndef RTE_EXEC_ENV_WINDOWS #include <rte_eal_trace.h> +#else +/* Suppress -Wempty-body for tracepoints used as "if" body. */ +#define rte_eal_trace_mem_malloc (...) do {} while (0) +#define rte_eal_trace_mem_zmalloc (...) do {} while (0) +#define rte_eal_trace_mem_realloc (...) do {} while (0) +#define rte_eal_trace_mem_free (...) do {} while (0) +#endif #include <rte_malloc.h> #include "malloc_elem.h" lib/librte_eal/rte_eal_exports.def +119
-0@@ -1,9 +1,128 @@ EXPORTS __rte_panic + rte_calloc + rte_calloc_socket rte_eal_get_configuration + rte_eal_has_hugepages rte_eal_init + rte_eal_iova_mode rte_eal_mp_remote_launch rte_eal_mp_wait_lcore + rte_eal_process_type rte_eal_remote_launch + rte_eal_tailq_lookup + rte_eal_tailq_register + rte_eal_using_phys_addrs + rte_free rte_log + rte_malloc + rte_malloc_dump_stats + rte_malloc_get_socket_stats + rte_malloc_set_limit + rte_malloc_socket + rte_malloc_validate + rte_malloc_virt2iova + rte_mcfg_mem_read_lock + rte_mcfg_mem_read_unlock + rte_mcfg_mem_write_lock + rte_mcfg_mem_write_unlock + rte_mcfg_mempool_read_lock + rte_mcfg_mempool_read_unlock + rte_mcfg_mempool_write_lock + rte_mcfg_mempool_write_unlock + rte_mcfg_tailq_read_lock + rte_mcfg_tailq_read_unlock + rte_mcfg_tailq_write_lock + rte_mcfg_tailq_write_unlock + rte_mem_lock_page + rte_mem_virt2iova + rte_mem_virt2phy + rte_memory_get_nchannel + rte_memory_get_nrank + rte_memzone_dump + rte_memzone_free + rte_memzone_lookup + rte_memzone_reserve + rte_memzone_reserve_aligned + rte_memzone_reserve_bounded + rte_memzone_walk rte_vlog + rte_realloc + rte_zmalloc + rte_zmalloc_socket + + rte_mp_action_register + rte_mp_action_unregister + rte_mp_reply + rte_mp_sendmsg + + rte_fbarray_attach + rte_fbarray_destroy + rte_fbarray_detach + rte_fbarray_dump_metadata + rte_fbarray_find_contig_free + rte_fbarray_find_contig_used + rte_fbarray_find_idx + rte_fbarray_find_next_free + rte_fbarray_find_next_n_free + rte_fbarray_find_next_n_used + rte_fbarray_find_next_used + rte_fbarray_get + rte_fbarray_init + rte_fbarray_is_used + rte_fbarray_set_free + rte_fbarray_set_used + rte_malloc_dump_heaps + rte_mem_alloc_validator_register + rte_mem_alloc_validator_unregister + rte_mem_check_dma_mask + rte_mem_event_callback_register + rte_mem_event_callback_unregister + rte_mem_iova2virt + rte_mem_virt2memseg + rte_mem_virt2memseg_list + rte_memseg_contig_walk + rte_memseg_list_walk + rte_memseg_walk + rte_mp_request_async + rte_mp_request_sync + + rte_fbarray_find_prev_free + rte_fbarray_find_prev_n_free + rte_fbarray_find_prev_n_used + rte_fbarray_find_prev_used + rte_fbarray_find_rev_contig_free + rte_fbarray_find_rev_contig_used + rte_memseg_contig_walk_thread_unsafe + rte_memseg_list_walk_thread_unsafe + rte_memseg_walk_thread_unsafe + + rte_malloc_heap_create + rte_malloc_heap_destroy + rte_malloc_heap_get_socket + rte_malloc_heap_memory_add + rte_malloc_heap_memory_attach + rte_malloc_heap_memory_detach + rte_malloc_heap_memory_remove + rte_malloc_heap_socket_is_external + rte_mem_check_dma_mask_thread_unsafe + rte_mem_set_dma_mask + rte_memseg_get_fd + rte_memseg_get_fd_offset + rte_memseg_get_fd_offset_thread_unsafe + rte_memseg_get_fd_thread_unsafe + + rte_extmem_attach + rte_extmem_detach + rte_extmem_register + rte_extmem_unregister + + rte_fbarray_find_biggest_free + rte_fbarray_find_biggest_used + rte_fbarray_find_rev_biggest_free + rte_fbarray_find_rev_biggest_used + + rte_get_page_size + rte_mem_lock + rte_mem_map + rte_mem_unmaplib/librte_eal/windows/eal.c +101
-0@@ -93,6 +93,24 @@ eal_proc_type_detect(void)return ptype; } +enum rte_proc_type_t +rte_eal_process_type (void ) +{ +return rte_config.process_type; +} + +int +rte_eal_has_hugepages (void ) +{ +return !internal_config.no_hugetlbfs; +} + +enum rte_iova_mode +rte_eal_iova_mode (void ) +{ +return rte_config.iova_mode; +} +/* display usage */ static void eal_usage (const char *prgname)@@ -224,6 +242,89 @@ rte_eal_init_alert(const char *msg)RTE_LOG (ERR ,EAL ,"%s\n" , msg); } +int +eal_file_truncate (int fd,ssize_t size) +{ +HANDLE handle; +DWORD ret; +LONG low = (LONG )((size_t )size); +LONG high = (LONG )((size_t )size >>32 ); + + handle = (HANDLE )_get_osfhandle (fd); +if (handle ==INVALID_HANDLE_VALUE ) { + rte_errno =EBADF ; +return -1 ; + } + + ret =SetFilePointer (handle, low, &high,FILE_BEGIN ); +if (ret ==INVALID_SET_FILE_POINTER ) { +RTE_LOG_WIN32_ERR ("SetFilePointer()" ); + rte_errno =EINVAL ; +return -1 ; + } + +return 0 ; +} + +static int +lock_file (HANDLE handle,enum eal_flock_op op,enum eal_flock_mode mode) +{ +DWORD sys_flags =0 ; +OVERLAPPED overlapped; + +if (op ==EAL_FLOCK_EXCLUSIVE ) + sys_flags |=LOCKFILE_EXCLUSIVE_LOCK ; +if (mode ==EAL_FLOCK_RETURN ) + sys_flags |=LOCKFILE_FAIL_IMMEDIATELY ; + +memset (&overlapped,0 ,sizeof (overlapped)); +if (!LockFileEx (handle, sys_flags,0 ,0 ,0 , &overlapped)) { +if ((sys_flags &LOCKFILE_FAIL_IMMEDIATELY ) && + (GetLastError () ==ERROR_IO_PENDING )) { + rte_errno =EWOULDBLOCK ; + }else { +RTE_LOG_WIN32_ERR ("LockFileEx()" ); + rte_errno =EINVAL ; + } +return -1 ; + } + +return 0 ; +} + +static int +unlock_file (HANDLE handle) +{ +if (!UnlockFileEx (handle,0 ,0 ,0 ,NULL )) { +RTE_LOG_WIN32_ERR ("UnlockFileEx()" ); + rte_errno =EINVAL ; +return -1 ; + } +return 0 ; +} + +int +eal_file_lock (int fd,enum eal_flock_op op,enum eal_flock_mode mode) +{ +HANDLE handle = (HANDLE )_get_osfhandle (fd); + +if (handle ==INVALID_HANDLE_VALUE ) { + rte_errno =EBADF ; +return -1 ; + } + +switch (op) { +case EAL_FLOCK_EXCLUSIVE : +case EAL_FLOCK_SHARED : +return lock_file (handle, op, mode); +case EAL_FLOCK_UNLOCK : +return unlock_file (handle); +default : + rte_errno =EINVAL ; +return -1 ; + } +} +/* Launch threads, called at application init(). */ int
rte_eal_init(int argc, char **argv)lib/librte_eal/windows/eal.c +3
-0@@ -245,6 +346,13 @@ rte_eal_init(int argc, char **argv)if (fctret <0 )exit (1 ); +/* Prevent creation of shared memory files. */ +if (internal_config.no_shconf ==0 ) { + RTE_LOG(WARNING, EAL, "Multi-process support isrequested, "
lib/librte_eal/windows/eal.c +4
-0+ "but not available.\n"); + internal_config.no_shconf =1 ; + } +if (!internal_config.no_hugetlbfs && (eal_hugepage_info_init () <0 )){
lib/librte_eal/windows/eal.c +36
-0rte_eal_init_alert ("Cannot get hugepage information" ); rte_errno =EACCES ;@@ -256,6 +364,42 @@ rte_eal_init(int argc, char **argv) internal_config.memory =MEMSIZE_IF_NO_HUGE_PAGE ; } +if (eal_mem_win32api_init () <0 ) { +rte_eal_init_alert ("Cannot access Win32 memory management"); + rte_errno =ENOTSUP ; +return -1 ; + } + +if (eal_mem_virt2iova_init () <0 ) { +/* Non-fatal error if physical addresses are not required. */ +RTE_LOG (WARNING ,EAL ,"Cannot access virt2phys driver, " +"PA will not be available\n" ); + } + +if (rte_eal_memzone_init () <0 ) { +rte_eal_init_alert ("Cannot init memzone" ); + rte_errno =ENODEV ; +return -1 ; + } + +if (rte_eal_memory_init () <0 ) { +rte_eal_init_alert ("Cannot init memory" ); + rte_errno =ENOMEM ; +return -1 ; + } + +if (rte_eal_malloc_heap_init () <0 ) { +rte_eal_init_alert ("Cannot init malloc heap" ); + rte_errno =ENODEV ; +return -1 ; + } + +if (rte_eal_tailqs_init () <0 ) { +rte_eal_init_alert ("Cannot init tail queues for objects" ); + rte_errno =EFAULT ; +return -1 ; + } +eal_thread_init_master (rte_config.master_lcore);lib/librte_eal/windows/eal.c +0
-0RTE_LCORE_FOREACH_SLAVE (i) {lib/librte_eal/windows/eal_memalloc.c new file +57
-0@@ -0,0 +1,418 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright (c) 2020 Dmitry Kozlyuk +*/ + +#include <rte_errno.h> +#include <rte_os.h> +#include <rte_windows.h> + +#include "eal_internal_cfg.h" +#include "eal_memalloc.h" +#include "eal_memcfg.h" +#include "eal_private.h" +#include "eal_windows.h" + +int +eal_memalloc_get_seg_fd (int list_idx,int seg_idx) +{ +/* Hugepages have no assiciated files in Windows. */ +RTE_SET_USED (list_idx); +RTE_SET_USED (seg_idx); +EAL_LOG_NOT_IMPLEMENTED (); +return -1 ; +} + +int +eal_memalloc_get_seg_fd_offset (int list_idx,int seg_idx,size_t *offset) +{ +/* Hugepages have no assiciated files in Windows. */ +RTE_SET_USED (list_idx); +RTE_SET_USED (seg_idx); +RTE_SET_USED (offset); +EAL_LOG_NOT_IMPLEMENTED (); +return -1 ; +} + +static int +alloc_seg (struct rte_memseg *ms,void *requested_addr,int socket_id, +struct hugepage_info *hi) +{ +HANDLE current_process; +unsigned int numa_node; +size_t alloc_sz; +void *addr; +rte_iova_t iova =RTE_BAD_IOVA ; +PSAPI_WORKING_SET_EX_INFORMATION info; +PSAPI_WORKING_SET_EX_BLOCK *page; + +if (ms->len >0 ) { +/* If a segment is already allocated as needed, return it. */ +if ((ms->addr == requested_addr) && + (ms->socket_id == socket_id) && + (ms->hugepage_sz == hi->hugepage_sz)) { +return 0 ; + } + +/* Bugcheck, should not happen. */ + RTE_LOG(DEBUG, EAL, "Attempted to reallocate segment %p"
lib/librte_eal/windows/eal_memalloc.c new file +19
-0+"(size %zu) on socket %d" , ms->addr, + ms->len, ms->socket_id); +return -1 ; + } + + current_process =GetCurrentProcess (); + numa_node =eal_socket_numa_node (socket_id); + alloc_sz = hi->hugepage_sz; + +if (requested_addr ==NULL ) { +/* Request a new chunk of memory from OS. */ + addr =eal_mem_alloc_socket (alloc_sz, socket_id); +if (addr ==NULL ) { +RTE_LOG (DEBUG ,EAL ,"Cannot allocate %zu bytes " +"on socket %d\n" , alloc_sz, socket_id); +return -1 ; + } + }else { +/* Requested address is already reserved, commit memory. */
lib/librte_eal/windows/eal_memalloc.c new file +1
-0+ addr = eal_mem_commit(requested_addr, alloc_sz,
socket_id);lib/librte_eal/windows/eal_memalloc.c new file +2
-0+if (addr ==NULL ) { + RTE_LOG(DEBUG, EAL, "Cannot commit reservedmemory %p "
lib/librte_eal/windows/eal_memalloc.c new file +34
-0+"(size %zu) on socket %d\n" , + requested_addr, alloc_sz, socket_id); +return -1 ; + } + } + +/* Force OS to allocate a physical page and select a NUMA node. +* Hugepages are not pageable in Windows, so there's no race +* for physical address. +*/ + *(volatile int *)addr = *(volatile int *)addr; + +/* Only try to obtain IOVA if it's available, so that applications +* that do not need IOVA can use this allocator. +*/ +if (rte_eal_using_phys_addrs ()) { + iova =rte_mem_virt2iova (addr); +if (iova ==RTE_BAD_IOVA ) { +RTE_LOG (DEBUG ,EAL , +"Cannot get IOVA of allocated segment\n" ); + goto error; + } + } + +/* Only "Ex" function can handle hugepages. */ + info.VirtualAddress = addr; +if (!QueryWorkingSetEx (current_process, &info,sizeof (info))) { +RTE_LOG_WIN32_ERR ("QueryWorkingSetEx()" ); + goto error; + } + + page = &info.VirtualAttributes; +if (!page->Valid || !page->LargePage) { + RTE_LOG(DEBUG, EAL, "Got regular page instead of ahugepage\n");
lib/librte_eal/windows/eal_memalloc.c new file +5
-0+ goto error; + } +if (page->Node != numa_node) { + RTE_LOG(DEBUG, EAL, + "NUMA node hint %u (socket %d) not respected, got%u\n",
lib/librte_eal/windows/eal_memalloc.c new file +45
-0+ numa_node, socket_id, page->Node); + goto error; + } + + ms->addr = addr; + ms->hugepage_sz = hi->hugepage_sz; + ms->len = alloc_sz; + ms->nchannel =rte_memory_get_nchannel (); + ms->nrank =rte_memory_get_nrank (); + ms->iova = iova; + ms->socket_id = socket_id; + +return 0 ; + +error: +/* Only jump here when `addr` and `alloc_sz` are valid. */ +eal_mem_decommit (addr, alloc_sz); +return -1 ; +} + +static int +free_seg (struct rte_memseg *ms) +{ +if (eal_mem_decommit (ms->addr, ms->len)) +return -1 ; + +/* Must clear the segment, because alloc_seg() inspects it. */ +memset (ms,0 ,sizeof (*ms)); +return 0 ; +} + +struct alloc_walk_param { +struct hugepage_info *hi; +struct rte_memseg **ms; +size_t page_sz; +unsigned int segs_allocated; +unsigned int n_segs; +int socket; +bool exact; +}; + +static int +alloc_seg_walk (const struct rte_memseg_list *msl,void *arg) +{ + struct rte_mem_config *mcfg = rte_eal_get_configuration()-Quoted 1 line
mem_config;lib/librte_eal/windows/eal_memalloc.c new file +30
-0+struct alloc_walk_param *wa = arg; +struct rte_memseg_list *cur_msl; +size_t page_sz; +int cur_idx, start_idx, j; +unsigned int msl_idx, need, i; + +if (msl->page_sz != wa->page_sz) +return 0 ; +if (msl->socket_id != wa->socket) +return 0 ; + + page_sz = (size_t )msl->page_sz; + + msl_idx = msl - mcfg->memsegs; + cur_msl = &mcfg->memsegs[msl_idx]; + + need = wa->n_segs; + +/* try finding space in memseg list */ +if (wa->exact) { +/* if we require exact number of pages in a list, find them */ + cur_idx =rte_fbarray_find_next_n_free ( + &cur_msl->memseg_arr,0 , need); +if (cur_idx <0 ) +return 0 ; + start_idx = cur_idx; + }else { +int cur_len; + +/* we don't require exact number of pages, so we're going to go
lib/librte_eal/windows/eal_memalloc.c new file +22
-0+* for best-effort allocation. that means finding the biggest +* unused block, and going with that. +*/ + cur_idx =rte_fbarray_find_biggest_free ( + &cur_msl->memseg_arr,0 ); +if (cur_idx <0 ) +return 0 ; + start_idx = cur_idx; +/* adjust the size to possibly be smaller than original +* request, but do not allow it to be bigger. +*/ + cur_len =rte_fbarray_find_contig_free ( + &cur_msl->memseg_arr, cur_idx); + need =RTE_MIN (need, (unsigned int )cur_len); + } + +for (i =0 ; i < need; i++, cur_idx++) { +struct rte_memseg *cur; +void *map_addr; + + cur =rte_fbarray_get (&cur_msl->memseg_arr, cur_idx); + map_addr = RTE_PTR_ADD(cur_msl->base_va, cur_idx *
page_sz);lib/librte_eal/windows/eal_memalloc.c new file +3
-0+ +if (alloc_seg (cur, map_addr, wa->socket, wa->hi)) { + RTE_LOG(DEBUG, EAL, "attempted to allocate %isegments, "
lib/librte_eal/windows/eal_memalloc.c new file +10
-0+ "but only %i were allocated\n", need, i); + +/* if exact number wasn't requested, stop */ +if (!wa->exact) + goto out; + +/* clean up */ +for (j = start_idx; j < cur_idx; j++) { +struct rte_memseg *tmp; + struct rte_fbarray *arr = &cur_msl-Quoted 1 line
memseg_arr;lib/librte_eal/windows/eal_memalloc.c new file +6
-0+ + tmp =rte_fbarray_get (arr, j); +rte_fbarray_set_free (arr, j); + +if (free_seg (tmp)) + RTE_LOG(DEBUG, EAL, "Cannot freepage\n");
lib/librte_eal/windows/eal_memalloc.c new file +4
-0+ } +/* clear the list */ +if (wa->ms) + memset(wa->ms, 0, sizeof(*wa->ms) * wa-Quoted 1 line
n_segs);lib/librte_eal/windows/eal_memalloc.c new file +26
-0+ +return -1 ; + } +if (wa->ms) + wa->ms[i] = cur; + +rte_fbarray_set_used (&cur_msl->memseg_arr, cur_idx); + } + +out: + wa->segs_allocated = i; +if (i >0 ) + cur_msl->version++; + +/* if we didn't allocate any segments, move on to the next list */ +return i >0 ; +} + +struct free_walk_param { +struct hugepage_info *hi; +struct rte_memseg *ms; +}; +static int +free_seg_walk (const struct rte_memseg_list *msl,void *arg) +{ + struct rte_mem_config *mcfg = rte_eal_get_configuration()-Quoted 1 line
mem_config;lib/librte_eal/windows/eal_memalloc.c new file +37
-0+struct rte_memseg_list *found_msl; +struct free_walk_param *wa = arg; +uintptr_t start_addr, end_addr; +int msl_idx, seg_idx, ret; + + start_addr = (uintptr_t ) msl->base_va; + end_addr = start_addr + msl->len; + +if ((uintptr_t )wa->ms->addr < start_addr || + (uintptr_t )wa->ms->addr >= end_addr) +return 0 ; + + msl_idx = msl - mcfg->memsegs; + seg_idx =RTE_PTR_DIFF (wa->ms->addr, start_addr) / msl->page_sz; + +/* msl is const */ + found_msl = &mcfg->memsegs[msl_idx]; + found_msl->version++; + +rte_fbarray_set_free (&found_msl->memseg_arr, seg_idx); + + ret =free_seg (wa->ms); + +return (ret <0 ) ? (-1 ) :1 ; +} + +int +eal_memalloc_alloc_seg_bulk (struct rte_memseg **ms,int n_segs, +size_t page_sz,int socket,bool exact) +{ +unsigned int i; +int ret =-1 ; +struct alloc_walk_param wa; +struct hugepage_info *hi =NULL ; + +if (internal_config.legacy_mem) { + RTE_LOG(ERR, EAL, "dynamic allocation not supported inlegacy mode\n");
lib/librte_eal/windows/eal_memalloc.c new file +5
-0+return -ENOTSUP ; + } + +for (i =0 ; i < internal_config.num_hugepage_sizes; i++) { + struct hugepage_info *hpi =&internal_config.hugepage_info[i];
lib/librte_eal/windows/eal_memalloc.c new file +7
-0+if (page_sz == hpi->hugepage_sz) { + hi = hpi; +break ; + } + } +if (!hi) { + RTE_LOG(ERR, EAL, "cannot find relevant hugepage_infoentry\n");
lib/librte_eal/windows/eal_memalloc.c new file +51
-0+return -1 ; + } + +memset (&wa,0 ,sizeof (wa)); + wa.exact = exact; + wa.hi = hi; + wa.ms = ms; + wa.n_segs = n_segs; + wa.page_sz = page_sz; + wa.socket = socket; + wa.segs_allocated =0 ; + +/* memalloc is locked, so it's safe to use thread-unsafe version */ + ret =rte_memseg_list_walk_thread_unsafe (alloc_seg_walk, &wa); +if (ret ==0 ) { +RTE_LOG (ERR ,EAL ,"cannot find suitable memseg_list\n" ); + ret =-1 ; + }else if (ret >0 ) { + ret = (int )wa.segs_allocated; + } + +return ret; +} + +struct rte_memseg * +eal_memalloc_alloc_seg (size_t page_sz,int socket) +{ +struct rte_memseg *ms =NULL ; +eal_memalloc_alloc_seg_bulk (&ms,1 , page_sz, socket, true); +return ms; +} + +int +eal_memalloc_free_seg_bulk (struct rte_memseg **ms,int n_segs) +{ +int seg, ret =0 ; + +/* dynamic free not supported in legacy mode */ +if (internal_config.legacy_mem) +return -1 ; + +for (seg =0 ; seg < n_segs; seg++) { +struct rte_memseg *cur = ms[seg]; +struct hugepage_info *hi =NULL ; +struct free_walk_param wa; +size_t i; +int walk_res; + +/* if this page is marked as unfreeable, fail */ +if (cur->flags &RTE_MEMSEG_FLAG_DO_NOT_FREE ) { + RTE_LOG(DEBUG, EAL, "Page is not allowed to befreed\n");
lib/librte_eal/windows/eal_memalloc.c new file +14
-0+ ret =-1 ; +continue ; + } + +memset (&wa,0 ,sizeof (wa)); + +for (i =0 ; i <RTE_DIM (internal_config.hugepage_info); + i++) { + hi = &internal_config.hugepage_info[i]; +if (cur->hugepage_sz == hi->hugepage_sz) +break ; + } +if (i ==RTE_DIM (internal_config.hugepage_info)) { + RTE_LOG(ERR, EAL, "Can't find relevant
hugepage_infoentry\n");lib/librte_eal/windows/eal_memalloc.c new file +8
-0+ ret =-1 ; +continue ; + } + + wa.ms = cur; + wa.hi = hi; + +/* memalloc is locked, so it's safe to use thread-unsafe version
lib/librte_eal/windows/eal_memalloc.c new file +2
-0+*/ + walk_res =rte_memseg_list_walk_thread_unsafe(free_seg_walk,
lib/librte_eal/windows/eal_memalloc.c new file +30
-0+ &wa); +if (walk_res ==1 ) +continue ; +if (walk_res ==0 ) +RTE_LOG (ERR ,EAL ,"Couldn't find memseg list\n" ); + ret =-1 ; + } +return ret; +} + +int +eal_memalloc_free_seg (struct rte_memseg *ms) +{ +return eal_memalloc_free_seg_bulk (&ms,1 ); +} + +int +eal_memalloc_sync_with_primary (void ) +{ +/* No multi-process support. */ +EAL_LOG_NOT_IMPLEMENTED (); +return -1 ; +} + +int +eal_memalloc_init (void ) +{ +/* No action required. */ +return 0 ; +}lib/librte_eal/windows/eal_memory.c new file +26
-0@@ -0,0 +1,1155 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright (c) 2010-2014 Intel Corporation (functions from Linux EAL) +* Copyright (c) 2020 Dmitry Kozlyuk (Windows specifics) +*/ + +#include <inttypes.h> +#include <io.h> + +#include <rte_errno.h> +#include <rte_memory.h> + +#include "eal_internal_cfg.h" +#include "eal_memalloc.h" +#include "eal_memcfg.h" +#include "eal_options.h" +#include "eal_private.h" +#include "eal_windows.h" + +#include <rte_virt2phys.h> + +/* MinGW-w64 headers lack VirtualAlloc2() in some distributions. +* Provide a copy of definitions and code to load it dynamically. +* Note: definitions are copied verbatim from Microsoft documentation +* and don't follow DPDK code style. +* +* MEM_RESERVE_PLACEHOLDER being defined means VirtualAlloc2() is present too.
+ */lib/librte_eal/windows/eal_memory.c new file +3
-0+#ifndef MEM_PRESERVE_PLACEHOLDER + +/* https://eur03.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdocs.
microsoft.com%2Fen-us%2Fwindows%2Fwin32%2Fapi%2Fwinnt%2Fne- winnt- mem_extended_parameter_type&data=02%7C01%7Cfady%40mellano x.com%7C3c6bd806786c479c1e3008d7ebcef213%7Ca652971c7d2e4d9ba6a4d 149256f461b%7C0%7C0%7C637237146372884132&sdata=Pd0bUVDAN8e iV5zORXJ9r0ZmzIwsfOaeL650gXPpQww%3D&reserved=0 */lib/librte_eal/windows/eal_memory.c new file +13
-0+typedef enum MEM_EXTENDED_PARAMETER_TYPE { + MemExtendedParameterInvalidType, + MemExtendedParameterAddressRequirements, + MemExtendedParameterNumaNode, + MemExtendedParameterPartitionHandle, + MemExtendedParameterMax, + MemExtendedParameterUserPhysicalHandle, + MemExtendedParameterAttributeFlags +} *PMEM_EXTENDED_PARAMETER_TYPE ; + +#define MEM_EXTENDED_PARAMETER_TYPE_BITS 4 + +/* https://eur03.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdocs.
microsoft.com%2Fen-us%2Fwindows%2Fwin32%2Fapi%2Fwinnt%2Fns- winnt- mem_extended_parameter&data=02%7C01%7Cfady%40mellanox.com %7C3c6bd806786c479c1e3008d7ebcef213%7Ca652971c7d2e4d9ba6a4d149256 f461b%7C0%7C0%7C637237146372884132&sdata=s5nguLunGkdr2hgJUS MIqV5fw7Qo1SDfo0TC%2BA3CFfY%3D&reserved=0 */lib/librte_eal/windows/eal_memory.c new file +4
-0+typedef struct MEM_EXTENDED_PARAMETER { + struct { + DWORD64 Type : MEM_EXTENDED_PARAMETER_TYPE_BITS; + DWORD64 Reserved : 64 -
MEM_EXTENDED_PARAMETER_TYPE_BITS;lib/librte_eal/windows/eal_memory.c new file +11
-0+ }DUMMYSTRUCTNAME ; +union { +DWORD64 ULong64; +PVOID Pointer; +SIZE_T Size; +HANDLE Handle; +DWORD ULong; + }DUMMYUNIONNAME ; +}MEM_EXTENDED_PARAMETER , *PMEM_EXTENDED_PARAMETER ; + +/* https://eur03.safelinks.protection.outlook.com/?url=https%3A%2F%2Fdocs.
microsoft.com%2Fen-us%2Fwindows%2Fwin32%2Fapi%2Fmemoryapi%2Fnf- memoryapi- virtualalloc2&data=02%7C01%7Cfady%40mellanox.com%7C3c6bd806786 c479c1e3008d7ebcef213%7Ca652971c7d2e4d9ba6a4d149256f461b%7C0%7C0 %7C637237146372884132&sdata=1SsOS8O2lRVD8yqDoTsBM8vTMRlvdJL fTT38FMAcoec%3D&reserved=0 */lib/librte_eal/windows/eal_memory.c new file +26
-0+typedef PVOID (*VirtualAlloc2_type )( +HANDLE Process, +PVOID BaseAddress, +SIZE_T Size, +ULONG AllocationType, +ULONG PageProtection, +MEM_EXTENDED_PARAMETER *ExtendedParameters, +ULONG ParameterCount +); + +/* VirtualAlloc2() flags. */ +#define MEM_COALESCE_PLACEHOLDERS 0x00000001 +#define MEM_PRESERVE_PLACEHOLDER 0x00000002 +#define MEM_REPLACE_PLACEHOLDER 0x00004000 +#define MEM_RESERVE_PLACEHOLDER 0x00040000 + +/* Named exactly as the function, so that user code does not depend +* on it being found at compile time or dynamically. +*/ +static VirtualAlloc2_type VirtualAlloc2; + +int +eal_mem_win32api_init (void ) +{ +/* Contrary to the docs, VirtualAlloc2() is not in kernel32.dll, +* see https://eur03.safelinks.protection.outlook.com/?url=https%3A%2F%2Fgithu
b.com%2FMicrosoftDocs%2Ffeedback%2Fissues%2F1129&data=02%7C 01%7Cfady%40mellanox.com%7C3c6bd806786c479c1e3008d7ebcef213%7Ca6 52971c7d2e4d9ba6a4d149256f461b%7C0%7C0%7C637237146372884132& ;sdata=tf%2BSJSNeKcOu9uDinDwsTYsME4R%2BHfdRery%2BUNRWflQ%3D& amp;reserved=0.lib/librte_eal/windows/eal_memory.c new file +14
-0+*/ +static const char library_name[] ="kernelbase.dll" ; +static const char function[] ="VirtualAlloc2" ; + +HMODULE library =NULL ; +int ret =0 ; + +/* Already done. */ +if (VirtualAlloc2 !=NULL ) +return 0 ; + + library =LoadLibraryA (library_name); +if (library ==NULL ) { + RTE_LOG_WIN32_ERR("LoadLibraryA(\"%s\")",
library_name);lib/librte_eal/windows/eal_memory.c new file +77
-0+ return -1; + } + + VirtualAlloc2 = (VirtualAlloc2_type)( + (void *)GetProcAddress (library, function)); +if (VirtualAlloc2 ==NULL ) { +RTE_LOG_WIN32_ERR ("GetProcAddress(\"%s\", \"%s\")\n" , + library_name, function); + +/* Contrary to the docs, Server 2016 is not supported. */ +RTE_LOG (ERR ,EAL ,"Windows 10 or Windows Server 2019 " +" is required for memory management\n" ); + ret =-1 ; + } + +FreeLibrary (library); + +return ret; +} + +#else + +/* Stub in case VirtualAlloc2() is provided by the compiler. */ +int +eal_mem_win32api_init (void ) +{ +return 0 ; +} + +#endif /* defined(MEM_RESERVE_PLACEHOLDER) */ + +static HANDLE virt2phys_device =INVALID_HANDLE_VALUE ; + +int +eal_mem_virt2iova_init (void ) +{ +HDEVINFO list =INVALID_HANDLE_VALUE ; +SP_DEVICE_INTERFACE_DATA ifdata; +SP_DEVICE_INTERFACE_DETAIL_DATA *detail =NULL ; +DWORD detail_size; +int ret =-1 ; + + list =SetupDiGetClassDevs ( + &GUID_DEVINTERFACE_VIRT2PHYS ,NULL ,NULL , +DIGCF_DEVICEINTERFACE |DIGCF_PRESENT ); +if (list ==INVALID_HANDLE_VALUE ) { +RTE_LOG_WIN32_ERR ("SetupDiGetClassDevs()" ); + goto exit; + } + + ifdata.cbSize =sizeof (ifdata); +if (!SetupDiEnumDeviceInterfaces ( + list,NULL , &GUID_DEVINTERFACE_VIRT2PHYS ,0 , &ifdata)) { +RTE_LOG_WIN32_ERR ("SetupDiEnumDeviceInterfaces()" ); + goto exit; + } + +if (!SetupDiGetDeviceInterfaceDetail ( + list, &ifdata,NULL ,0 , &detail_size,NULL )) { +if (GetLastError () !=ERROR_INSUFFICIENT_BUFFER ) { +RTE_LOG_WIN32_ERR ( +"SetupDiGetDeviceInterfaceDetail(probe)" ); + goto exit; + } + } + + detail =malloc (detail_size); +if (detail ==NULL ) { +RTE_LOG (ERR ,EAL ,"Cannot allocate virt2phys " +"device interface detail data\n" ); + goto exit; + } + + detail->cbSize =sizeof (*detail); +if (!SetupDiGetDeviceInterfaceDetail ( + list, &ifdata, detail, detail_size,NULL ,NULL )) { +RTE_LOG_WIN32_ERR("SetupDiGetDeviceInterfaceDetail(read)");lib/librte_eal/windows/eal_memory.c new file +4
-0+ goto exit; + } + + RTE_LOG(DEBUG, EAL, "Found virt2phys device: %s\n", detail-Quoted 1 line
DevicePath);
lib/librte_eal/windows/eal_memory.c new file +34
-0+ + virt2phys_device =CreateFile ( + detail->DevicePath,0 ,0 ,NULL ,OPEN_EXISTING ,0 ,NULL ); +if (virt2phys_device ==INVALID_HANDLE_VALUE ) { +RTE_LOG_WIN32_ERR ("CreateFile()" ); + goto exit; + } + +/* Indicate success. */ + ret =0 ; + +exit: +if (detail !=NULL ) +free (detail); +if (list !=INVALID_HANDLE_VALUE ) +SetupDiDestroyDeviceInfoList (list); + +return ret; +} + +phys_addr_t +rte_mem_virt2phy (const void *virt) +{ +LARGE_INTEGER phys; +DWORD bytes_returned; + +if (virt2phys_device ==INVALID_HANDLE_VALUE ) +return RTE_BAD_PHYS_ADDR ; + +if (!DeviceIoControl ( + virt2phys_device,IOCTL_VIRT2PHYS_TRANSLATE , + &virt,sizeof (virt), &phys,sizeof (phys), + &bytes_returned,NULL )) { +RTE_LOG_WIN32_ERR("DeviceIoControl(IOCTL_VIRT2PHYS_TRANSL
ATE)");lib/librte_eal/windows/eal_memory.c new file +23
-0+return RTE_BAD_PHYS_ADDR ; + } + +return phys.QuadPart; +} + +/* Windows currently only supports IOVA as PA. */ +rte_iova_t +rte_mem_virt2iova (const void *virt) +{ +phys_addr_t phys; + +if (virt2phys_device ==INVALID_HANDLE_VALUE ) +return RTE_BAD_IOVA ; + + phys =rte_mem_virt2phy (virt); +if (phys ==RTE_BAD_PHYS_ADDR ) +return RTE_BAD_IOVA ; + +return (rte_iova_t )phys; +} + +/* Always using physical addresses under Windows if they can be obtained. */
lib/librte_eal/windows/eal_memory.c new file +44
-0+int +rte_eal_using_phys_addrs (void ) +{ +return virt2phys_device !=INVALID_HANDLE_VALUE ; +} + +/* Approximate error mapping from VirtualAlloc2() to POSIX mmap(3). */ +static void +set_errno_from_win32_alloc_error (DWORD code) +{ +switch (code) { +case ERROR_SUCCESS : + rte_errno =0 ; +break ; + +case ERROR_INVALID_ADDRESS : +/* A valid requested address is not available. */ +case ERROR_COMMITMENT_LIMIT : +/* May occcur when committing regular memory. */ +case ERROR_NO_SYSTEM_RESOURCES : +/* Occurs when the system runs out of hugepages. */ + rte_errno =ENOMEM ; +break ; + +case ERROR_INVALID_PARAMETER : +default : + rte_errno =EINVAL ; +break ; + } +} + +void * +eal_mem_reserve (void *requested_addr,size_t size,int flags) +{ +void *virt; + +/* Windows requires hugepages to be committed. */ +if (flags &EAL_RESERVE_HUGEPAGES ) { + rte_errno =ENOTSUP ; +return NULL ; + } + + virt = VirtualAlloc2(GetCurrentProcess(), requested_addr, size, + MEM_RESERVE | MEM_RESERVE_PLACEHOLDER,
PAGE_NOACCESS,lib/librte_eal/windows/eal_memory.c new file +8
-0+ NULL, 0); +if (virt ==NULL ) { +DWORD err =GetLastError (); +RTE_LOG_WIN32_ERR ("VirtualAlloc2()" ); +set_errno_from_win32_alloc_error (err); + } + + if ((flags & EAL_RESERVE_FORCE_ADDRESS) && (virt !=
requested_addr)) {lib/librte_eal/windows/eal_memory.c new file +17
-0+if (!VirtualFree (virt,0 ,MEM_RELEASE )) +RTE_LOG_WIN32_ERR ("VirtualFree()" ); + rte_errno =ENOMEM ; +return NULL ; + } + +return virt; +} + +void * +eal_mem_alloc (size_t size,size_t page_size) +{ +if (page_size !=0 ) +return eal_mem_alloc_socket (size,SOCKET_ID_ANY ); + + return VirtualAlloc( + NULL, size, MEM_RESERVE | MEM_COMMIT,
PAGE_READWRITE);lib/librte_eal/windows/eal_memory.c new file +37
-0+} + +void * +eal_mem_alloc_socket (size_t size,int socket_id) +{ +DWORD flags =MEM_RESERVE |MEM_COMMIT ; +void *addr; + + flags =MEM_RESERVE |MEM_COMMIT |MEM_LARGE_PAGES ; + addr =VirtualAllocExNuma (GetCurrentProcess (),NULL , size, flags, +PAGE_READWRITE ,eal_socket_numa_node (socket_id)); +if (addr ==NULL ) + rte_errno =ENOMEM ; +return addr; +} + +void * +eal_mem_commit (void *requested_addr,size_t size,int socket_id) +{ +MEM_EXTENDED_PARAMETER param; +DWORD param_count =0 ; +DWORD flags; +void *addr; + +if (requested_addr !=NULL ) { +MEMORY_BASIC_INFORMATION info; +if (VirtualQuery (requested_addr, &info,sizeof (info)) ==0 ) { +RTE_LOG_WIN32_ERR ("VirtualQuery()" ); +return NULL ; + } + +/* Split reserved region if only a part is committed. */ + flags =MEM_RELEASE |MEM_PRESERVE_PLACEHOLDER ; +if ((info.RegionSize > size) && + !VirtualFree (requested_addr, size, flags)) { + RTE_LOG_WIN32_ERR("VirtualFree(%p, %zu, " + "<split placeholder>)", requested_addr,size);
lib/librte_eal/windows/eal_memory.c new file +32
-0+return NULL ; + } + } + +if (socket_id !=SOCKET_ID_ANY ) { + param_count =1 ; +memset (¶m,0 ,sizeof (param)); + param.Type = MemExtendedParameterNumaNode; + param.ULong =eal_socket_numa_node (socket_id); + } + + flags =MEM_RESERVE |MEM_COMMIT |MEM_LARGE_PAGES ; +if (requested_addr !=NULL ) + flags |=MEM_REPLACE_PLACEHOLDER ; + + addr =VirtualAlloc2 (GetCurrentProcess (), requested_addr, size, + flags,PAGE_READWRITE , ¶m, param_count); +if (addr ==NULL ) { +DWORD err =GetLastError (); +RTE_LOG_WIN32_ERR ("VirtualAlloc2(%p, %zu, " +"<replace placeholder>)" , addr, size); +set_errno_from_win32_alloc_error (err); +return NULL ; + } + +return addr; +} + +int +eal_mem_decommit (void *addr,size_t size) +{ +/* Decommit memory, which might be a part of a larger reserved region.
lib/librte_eal/windows/eal_memory.c new file +1
-0+* Allocator commits hugepage-sized placeholders, so there's no need
lib/librte_eal/windows/eal_memory.c new file +3
-0+* to coalesce placeholders back into region, they can be reused as is. +*/ + if (!VirtualFree(addr, size, MEM_RELEASE |
MEM_PRESERVE_PLACEHOLDER)) {lib/librte_eal/windows/eal_memory.c new file +1
-0+ RTE_LOG_WIN32_ERR("VirtualFree(%p, %zu, ...)", addr,size);
lib/librte_eal/windows/eal_memory.c new file +39
-0+return -1 ; + } +return 0 ; +} + +/** +* Free a reserved memory region in full or in part. +* +* @param addr +* Starting address of the area to free. +* @param size +* Number of bytes to free. Must be a multiple of page size. +* @param reserved +* Fail if the region is not in reserved state. +* @return +* * 0 on successful deallocation; +* * 1 if region mut be in reserved state but it is not; +* * (-1) on system API failures. +*/ +static int +mem_free (void *addr,size_t size,bool reserved) +{ +MEMORY_BASIC_INFORMATION info; +HANDLE process; + +if (VirtualQuery (addr, &info,sizeof (info)) ==0 ) { +RTE_LOG_WIN32_ERR ("VirtualQuery()" ); +return -1 ; + } + +if (reserved && (info.State !=MEM_RESERVE )) +return 1 ; + + process =GetCurrentProcess (); + +/* Free complete region. */ +if ((addr == info.AllocationBase) && (size == info.RegionSize)) { +if (!VirtualFreeEx (process, addr,0 ,MEM_RELEASE )) { + RTE_LOG_WIN32_ERR("VirtualFree(%p, 0,
MEM_RELEASE)",lib/librte_eal/windows/eal_memory.c new file +10
-0+ addr); + } +return 0 ; + } + +/* Split the part to be freed and the remaining reservation. */ +if (!VirtualFreeEx (process, addr, size, +MEM_RELEASE |MEM_PRESERVE_PLACEHOLDER )) { + RTE_LOG_WIN32_ERR("VirtualFree(%p, %zu, " + "MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER)",addr, size);
lib/librte_eal/windows/eal_memory.c new file +6
-0+return -1 ; + } + +/* Actually free reservation part. */ +if (!VirtualFreeEx (process, addr,0 ,MEM_RELEASE )) { + RTE_LOG_WIN32_ERR("VirtualFree(%p, 0, MEM_RELEASE)",addr);
lib/librte_eal/windows/eal_memory.c new file +51
-0+return -1 ; + } + +return 0 ; +} + +void +eal_mem_free (void *virt,size_t size) +{ +mem_free (virt, size, false); +} + +int +eal_mem_set_dump (void *virt,size_t size,bool dump) +{ +RTE_SET_USED (virt); +RTE_SET_USED (size); +RTE_SET_USED (dump); + +/* Windows does not dump reserved memory by default. +* +* There is <werapi.h> to include or exclude regions from the dump, +* but this is not currently required by EAL. +*/ + + rte_errno =ENOTSUP ; +return -1 ; +} + +void * +rte_mem_map (void *requested_addr,size_t size,int prot,int flags, +int fd,size_t offset) +{ +HANDLE file_handle =INVALID_HANDLE_VALUE ; +HANDLE mapping_handle =INVALID_HANDLE_VALUE ; +DWORD sys_prot =0 ; +DWORD sys_access =0 ; +DWORD size_high = (DWORD )(size >>32 ); +DWORD size_low = (DWORD )size; +DWORD offset_high = (DWORD )(offset >>32 ); +DWORD offset_low = (DWORD )offset; +LPVOID virt =NULL ; + +if (prot &RTE_PROT_EXECUTE ) { +if (prot &RTE_PROT_READ ) { + sys_prot =PAGE_EXECUTE_READ ; + sys_access =FILE_MAP_READ |FILE_MAP_EXECUTE ; + } +if (prot &RTE_PROT_WRITE ) { + sys_prot =PAGE_EXECUTE_READWRITE ; + sys_access = FILE_MAP_WRITE |
FILE_MAP_EXECUTE;lib/librte_eal/windows/eal_memory.c new file +27
-0+ } + }else { +if (prot &RTE_PROT_READ ) { + sys_prot =PAGE_READONLY ; + sys_access =FILE_MAP_READ ; + } +if (prot &RTE_PROT_WRITE ) { + sys_prot =PAGE_READWRITE ; + sys_access =FILE_MAP_WRITE ; + } + } + +if (flags &RTE_MAP_PRIVATE ) + sys_access |=FILE_MAP_COPY ; + +if ((flags &RTE_MAP_ANONYMOUS ) ==0 ) + file_handle = (HANDLE )_get_osfhandle (fd); + + mapping_handle =CreateFileMapping ( + file_handle,NULL , sys_prot, size_high, size_low,NULL ); +if (mapping_handle ==INVALID_HANDLE_VALUE ) { +RTE_LOG_WIN32_ERR ("CreateFileMapping()" ); +return NULL ; + } + +/* There is a race for the requested_addr between mem_free() +* and MapViewOfFileEx(). MapViewOfFile3() that can replace a reserved
lib/librte_eal/windows/eal_memory.c new file +23
-0+* region with a mapping in a single operation, but it does not support +* private mappings. +*/ +if (requested_addr !=NULL ) { +int ret =mem_free (requested_addr, size, true); +if (ret) { +if (ret >0 ) { +RTE_LOG (ERR ,EAL ,"Cannot map memory " +"to a region not reserved\n" ); + rte_errno =EADDRNOTAVAIL ; + } +return NULL ; + } + } + + virt =MapViewOfFileEx (mapping_handle, sys_access, + offset_high, offset_low, size, requested_addr); +if (!virt) { +RTE_LOG_WIN32_ERR ("MapViewOfFileEx()" ); +return NULL ; + } + +if ((flags &RTE_MAP_FORCE_ADDRESS ) && (virt != requested_addr)){
lib/librte_eal/windows/eal_memory.c new file +81
-0+if (!UnmapViewOfFile (virt)) +RTE_LOG_WIN32_ERR ("UnmapViewOfFile()" ); + virt =NULL ; + } + +if (!CloseHandle (mapping_handle)) +RTE_LOG_WIN32_ERR ("CloseHandle()" ); + +return virt; +} + +int +rte_mem_unmap (void *virt,size_t size) +{ +RTE_SET_USED (size); + +if (!UnmapViewOfFile (virt)) { + rte_errno =GetLastError (); +RTE_LOG_WIN32_ERR ("UnmapViewOfFile()" ); +return -1 ; + } +return 0 ; +} + +uint64_t +eal_get_baseaddr (void ) +{ +/* Windows strategy for memory allocation is undocumented. +* Returning 0 here effectively disables address guessing +* unless user provides an address hint. +*/ +return 0 ; +} + +size_t +rte_get_page_size (void ) +{ +SYSTEM_INFO info; +GetSystemInfo (&info); +return info.dwPageSize; +} + +int +rte_mem_lock (const void *virt,size_t size) +{ +/* VirtualLock() takes `void*`, work around compiler warning. */ +void *addr = (void *)((uintptr_t )virt); + +if (!VirtualLock (addr, size)) { +RTE_LOG_WIN32_ERR ("VirtualLock()" ); +return -1 ; + } + +return 0 ; +} + +static int +memseg_list_init (struct rte_memseg_list *msl,uint64_t page_sz, +int n_segs,int socket_id,int type_msl_idx) +{ +return eal_memseg_list_init ( + msl, page_sz, n_segs, socket_id, type_msl_idx, true); +} + +static int +memseg_list_alloc (struct rte_memseg_list *msl) +{ +return eal_memseg_list_alloc (msl,0 ); +} + +/* +* Remaining code in this file largely duplicates Linux EAL. +* Although Windows EAL supports only one hugepage size currently, +* code structure and comments are preserved so that changes may be +* easily ported until duplication is removed. +*/ + +static int +memseg_primary_init (void ) +{ + struct rte_mem_config *mcfg = rte_eal_get_configuration()-Quoted 1 line
mem_config;lib/librte_eal/windows/eal_memory.c new file +16
-0+struct memtype { +uint64_t page_sz; +int socket_id; + } *memtypes =NULL ; +int i, hpi_idx, msl_idx, ret =-1 ;/* fail unless told to succeed */ +struct rte_memseg_list *msl; +uint64_t max_mem, max_mem_per_type; +unsigned int max_seglists_per_type; +unsigned int n_memtypes, cur_type; + +/* no-huge does not need this at all */ +if (internal_config.no_hugetlbfs) +return 0 ; + +/* +* figuring out amount of memory we're going to have is a long and very
lib/librte_eal/windows/eal_memory.c new file +1
-0+* involved process. the basic element we're operating with is a memory
lib/librte_eal/windows/eal_memory.c new file +1
-0+* type, defined as a combination of NUMA node ID and page size (so that
lib/librte_eal/windows/eal_memory.c new file +4
-0+* e.g. 2 sockets with 2 page sizes yield 4 memory types in total). +* +* deciding amount of memory going towards each memory type is a +* balancing act between maximum segments per type, maximum memory per
lib/librte_eal/windows/eal_memory.c new file +1
-0+* type, and number of detected NUMA nodes. the goal is to make sure
lib/librte_eal/windows/eal_memory.c new file +3
-0+* each memory type gets at least one memseg list. +* +* the total amount of memory is limited by RTE_MAX_MEM_MB value.
lib/librte_eal/windows/eal_memory.c new file +3
-0+* +* the total amount of memory per type is limited by either +* RTE_MAX_MEM_MB_PER_TYPE, or by RTE_MAX_MEM_MB divided by the number
lib/librte_eal/windows/eal_memory.c new file +1
-0+* of detected NUMA nodes. additionally, maximum number of segments per
lib/librte_eal/windows/eal_memory.c new file +1
-0+* type is also limited by RTE_MAX_MEMSEG_PER_TYPE. this is because for
lib/librte_eal/windows/eal_memory.c new file +1
-0+* smaller page sizes, it can take hundreds of thousands of segments to
lib/librte_eal/windows/eal_memory.c new file +4
-0+* reach the above specified per-type memory limits. +* +* additionally, each type may have multiple memseg lists associated +* with it, each limited by either RTE_MAX_MEM_MB_PER_LIST for bigger
lib/librte_eal/windows/eal_memory.c new file +1
-0+* page sizes, or RTE_MAX_MEMSEG_PER_LIST segments for smaller ones.
lib/librte_eal/windows/eal_memory.c new file +2
-0+* +* the number of memseg lists per type is decided based on the above
lib/librte_eal/windows/eal_memory.c new file +1
-0+* limits, and also taking number of detected NUMA nodes, to make sure
lib/librte_eal/windows/eal_memory.c new file +1
-0+* that we don't run out of memseg lists before we populate all NUMA
lib/librte_eal/windows/eal_memory.c new file +5
-0+* nodes with memory. +* +* we do this in three stages. first, we collect the number of types. +* then, we figure out memory constraints and populate the list of +* would-be memseg lists. then, we go ahead and allocate the memseg
lib/librte_eal/windows/eal_memory.c new file +5
-0+* lists. +*/ + +/* create space for mem types */ + n_memtypes = internal_config.num_hugepage_sizes *
rte_socket_count();lib/librte_eal/windows/eal_memory.c new file +3
-0+ memtypes =calloc (n_memtypes,sizeof (*memtypes)); +if (memtypes ==NULL ) { + RTE_LOG(ERR, EAL, "Cannot allocate space for memorytypes\n");
lib/librte_eal/windows/eal_memory.c new file +25
-0+return -1 ; + } + +/* populate mem types */ + cur_type =0 ; +for (hpi_idx =0 ; hpi_idx < (int ) internal_config.num_hugepage_sizes; + hpi_idx++) { +struct hugepage_info *hpi; +uint64_t hugepage_sz; + + hpi = &internal_config.hugepage_info[hpi_idx]; + hugepage_sz = hpi->hugepage_sz; + +for (i =0 ; i < (int )rte_socket_count (); i++, cur_type++) { +int socket_id =rte_socket_id_by_idx (i); + + memtypes[cur_type].page_sz = hugepage_sz; + memtypes[cur_type].socket_id = socket_id; + +RTE_LOG (DEBUG ,EAL ,"Detected memory type: " +"socket_id:%u hugepage_sz:%" PRIu64"\n" , + socket_id, hugepage_sz); + } + } +/* number of memtypes could have been lower due to no NUMA support */
lib/librte_eal/windows/eal_memory.c new file +5
-0+ n_memtypes = cur_type; + +/* set up limits for types */ + max_mem = (uint64_t )RTE_MAX_MEM_MB <<20 ; + max_mem_per_type =RTE_MIN((uint64_t)RTE_MAX_MEM_MB_PER_TYPE << 20,
lib/librte_eal/windows/eal_memory.c new file +10
-0+ max_mem / n_memtypes); + +/* +* limit maximum number of segment lists per type to ensure there's +* space for memseg lists for all NUMA nodes with all page sizes +*/ + max_seglists_per_type =RTE_MAX_MEMSEG_LISTS / n_memtypes; + +if (max_seglists_per_type ==0 ) { + RTE_LOG(ERR, EAL, "Cannot accommodate all memory types,please increase %s\n",
lib/librte_eal/windows/eal_memory.c new file +18
-0+ RTE_STR(CONFIG_RTE_MAX_MEMSEG_LISTS)); + goto out; + } + +/* go through all mem types and create segment lists */ + msl_idx =0 ; +for (cur_type =0 ; cur_type < n_memtypes; cur_type++) { +unsigned int cur_seglist, n_seglists, n_segs; +unsigned int max_segs_per_type, max_segs_per_list; +struct memtype *type = &memtypes[cur_type]; +uint64_t max_mem_per_list, pagesz; +int socket_id; + + pagesz = type->page_sz; + socket_id = type->socket_id; + +/* +* we need to create segment lists for this type. we must take
lib/librte_eal/windows/eal_memory.c new file +3
-0+* into account the following things: +* +* 1. total amount of memory we can use for this memory type
lib/librte_eal/windows/eal_memory.c new file +2
-0+* 2. total amount of memory per memseg list allowed +* 3. number of segments needed to fit the amount of memory
lib/librte_eal/windows/eal_memory.c new file +8
-0+* 4. number of segments allowed per type +* 5. number of segments allowed per memseg list +* 6. number of memseg lists we are allowed to take up +*/ + +/* calculate how much segments we will need in total */ + max_segs_per_type = max_mem_per_type / pagesz; +/* limit number of segments to maximum allowed per type */
lib/librte_eal/windows/eal_memory.c new file +2
-0+ max_segs_per_type = RTE_MIN(max_segs_per_type, + (unsignedint)RTE_MAX_MEMSEG_PER_TYPE);
lib/librte_eal/windows/eal_memory.c new file +3
-0+/* limit number of segments to maximum allowed per list */ + max_segs_per_list = RTE_MIN(max_segs_per_type, + (unsignedint)RTE_MAX_MEMSEG_PER_LIST);
lib/librte_eal/windows/eal_memory.c new file +2
-0+ +/* calculate how much memory we can have per segment list */
lib/librte_eal/windows/eal_memory.c new file +2
-0+ max_mem_per_list = RTE_MIN(max_segs_per_list * pagesz, + (uint64_t)RTE_MAX_MEM_MB_PER_LIST <<20);
lib/librte_eal/windows/eal_memory.c new file +2
-0+ +/* calculate how many segments each segment list will have */
lib/librte_eal/windows/eal_memory.c new file +1
-0+ n_segs = RTE_MIN(max_segs_per_list, max_mem_per_list /pagesz);
lib/librte_eal/windows/eal_memory.c new file +6
-0+ +/* calculate how many segment lists we can have */ + n_seglists =RTE_MIN (max_segs_per_type / n_segs, + max_mem_per_type / max_mem_per_list); + +/* limit number of segment lists according to our maximum */
lib/librte_eal/windows/eal_memory.c new file +4
-0+ n_seglists =RTE_MIN (n_seglists, max_seglists_per_type); + +RTE_LOG (DEBUG ,EAL ,"Creating %i segment lists: " + "n_segs:%i socket_id:%i hugepage_sz:%"PRIu64 "\n",
lib/librte_eal/windows/eal_memory.c new file +7
-0+ n_seglists, n_segs, socket_id, pagesz); + +/* create all segment lists */ +for (cur_seglist =0 ; cur_seglist < n_seglists; cur_seglist++) { +if (msl_idx >=RTE_MAX_MEMSEG_LISTS ) { + RTE_LOG(ERR, EAL, + "No more space in memseg lists,please increase %s\n",
+RTE_STR(CONFIG_RTE_MAX_MEMSEG_LISTS));lib/librte_eal/windows/eal_memory.c new file +10
-0+ goto out; + } + msl = &mcfg->memsegs[msl_idx++]; + +if (memseg_list_init (msl, pagesz, n_segs, + socket_id, cur_seglist)) + goto out; + +if (memseg_list_alloc (msl)) { + RTE_LOG(ERR, EAL, "Cannot allocate VAspace for memseg list\n");
lib/librte_eal/windows/eal_memory.c new file +34
-0+ goto out; + } + } + } +/* we're successful */ + ret =0 ; +out: +free (memtypes); +return ret; +} + +static int +memseg_secondary_init (void ) +{ +EAL_LOG_NOT_IMPLEMENTED (); +return -1 ; +} + +int +rte_eal_memseg_init (void ) +{ +if (rte_eal_process_type () ==RTE_PROC_PRIMARY ) +return memseg_primary_init (); +return memseg_secondary_init (); +} + +static inline uint64_t +get_socket_mem_size (int socket) +{ +uint64_t size =0 ; +unsigned int i; + +for (i =0 ; i < internal_config.num_hugepage_sizes; i++) { + struct hugepage_info *hpi =&internal_config.hugepage_info[i];
lib/librte_eal/windows/eal_memory.c new file +32
-0+ size += hpi->hugepage_sz * hpi->num_pages[socket]; + } + +return size; +} + +static int +calc_num_pages_per_socket (uint64_t *memory, +struct hugepage_info *hp_info, +struct hugepage_info *hp_used, +unsigned int num_hp_info) +{ +unsigned int socket, j, i =0 ; +unsigned int requested, available; +int total_num_pages =0 ; +uint64_t remaining_mem, cur_mem; +uint64_t total_mem = internal_config.memory; + +if (num_hp_info ==0 ) +return -1 ; + +/* if specific memory amounts per socket weren't requested */ +if (internal_config.force_sockets ==0 ) { +size_t total_size; +int cpu_per_socket[RTE_MAX_NUMA_NODES ]; +size_t default_size; +unsigned int lcore_id; + +/* Compute number of cores per socket */ +memset (cpu_per_socket,0 ,sizeof (cpu_per_socket)); +RTE_LCORE_FOREACH (lcore_id) { +cpu_per_socket[rte_lcore_to_socket_id(lcore_id)]++;lib/librte_eal/windows/eal_memory.c new file +4
-0+ } + +/* +* Automatically spread requested memory amongst detected
lib/librte_eal/windows/eal_memory.c new file +1
-0+* sockets according to number of cores from cpu mask present
lib/librte_eal/windows/eal_memory.c new file +4
-0+* on each socket. +*/ + total_size = internal_config.memory; + for (socket = 0; socket < RTE_MAX_NUMA_NODES &&
total_size!= 0;lib/librte_eal/windows/eal_memory.c new file +9
-0+ socket++) { + +/* Set memory amount per socket */ + default_size = internal_config.memory * + cpu_per_socket[socket] /rte_lcore_count (); + +/* Limit to maximum available memory on socket */ + default_size = RTE_MIN( + default_size,
get_socket_mem_size(socket));lib/librte_eal/windows/eal_memory.c new file +11
-0+ +/* Update sizes */ + memory[socket] = default_size; + total_size -= default_size; + } + +/* +* If some memory is remaining, try to allocate it by getting +* all available memory from sockets, one after the other. +*/ + for (socket = 0; socket < RTE_MAX_NUMA_NODES &&
total_size!= 0;lib/librte_eal/windows/eal_memory.c new file +4
-0+ socket++) { +/* take whatever is available */ + default_size = RTE_MIN( + get_socket_mem_size(socket) -memory[socket],
lib/librte_eal/windows/eal_memory.c new file +9
-0+ total_size); + +/* Update sizes */ + memory[socket] += default_size; + total_size -= default_size; + } + } + + for (socket = 0; socket < RTE_MAX_NUMA_NODES && total_mem !=0;
lib/librte_eal/windows/eal_memory.c new file +7
-0+ socket++) { +/* skips if the memory on specific socket wasn't requested */ +for (i =0 ; i < num_hp_info && memory[socket] !=0 ; i++) { +strncpy (hp_used[i].hugedir, hp_info[i].hugedir, +sizeof (hp_used[i].hugedir)); + hp_used[i].num_pages[socket] = RTE_MIN( + memory[socket] /hp_info[i].hugepage_sz,
lib/librte_eal/windows/eal_memory.c new file +9
-0+ hp_info[i].num_pages[socket]); + + cur_mem = hp_used[i].num_pages[socket] * + hp_used[i].hugepage_sz; + + memory[socket] -= cur_mem; + total_mem -= cur_mem; + + total_num_pages +=hp_used[i].num_pages[socket];
lib/librte_eal/windows/eal_memory.c new file +14
-0+ +/* check if we have met all memory requests */ +if (memory[socket] ==0 ) +break ; + +/* Check if we have any more pages left at this size, +* if so, move on to next size. +*/ +if (hp_used[i].num_pages[socket] == + hp_info[i].num_pages[socket]) +continue ; + +/* At this point we know that there are more pages +* available that are bigger than the memory we want,
lib/librte_eal/windows/eal_memory.c new file +6
-0+* so lets see if we can get enough from other page +* sizes. +*/ + remaining_mem =0 ; +for (j = i+1 ; j < num_hp_info; j++) + remaining_mem += hp_info[j].hugepage_sz*
lib/librte_eal/windows/eal_memory.c new file +8
-0+ hp_info[j].num_pages[socket]; + +/* Is there enough other memory? +* If not, allocate another page and quit. +*/ +if (remaining_mem < memory[socket]) { + cur_mem = RTE_MIN( + memory[socket],hp_info[i].hugepage_sz);
lib/librte_eal/windows/eal_memory.c new file +13
-0+ memory[socket] -= cur_mem; + total_mem -= cur_mem; + hp_used[i].num_pages[socket]++; + total_num_pages++; +break ;/* we are done with this socket*/ + } + } +/* if we didn't satisfy all memory requirements per socket */ +if (memory[socket] >0 && + internal_config.socket_mem[socket] !=0 ) { +/* to prevent icc errors */ + requested = (unsigned int)( + internal_config.socket_mem[socket] /
0x100000);lib/librte_eal/windows/eal_memory.c new file +2
-0+ available = requested - + ((unsigned int)(memory[socket] /
0x100000));lib/librte_eal/windows/eal_memory.c new file +1
-0+ RTE_LOG(ERR, EAL, "Not enough memory availableon "
lib/librte_eal/windows/eal_memory.c new file +1
-0+ "socket %u! Requested: %uMB, available:%uMB\n",
lib/librte_eal/windows/eal_memory.c new file +8
-0+ socket, requested, available); +return -1 ; + } + } + +/* if we didn't satisfy total memory requirements */ +if (total_mem >0 ) { + requested = (unsigned int) (internal_config.memory /
0x100000);lib/librte_eal/windows/eal_memory.c new file +1
-0+ available = requested - (unsigned int) (total_mem /
0x100000);lib/librte_eal/windows/eal_memory.c new file +29
-0+RTE_LOG (ERR ,EAL ,"Not enough memory available! " +"Requested: %uMB, available: %uMB\n" , + requested, available); +return -1 ; + } +return total_num_pages; +} + +/* Limit is checked by validator itself, nothing left to analyze.*/ +static int +limits_callback (int socket_id,size_t cur_limit,size_t new_len) +{ +RTE_SET_USED (socket_id); +RTE_SET_USED (cur_limit); +RTE_SET_USED (new_len); +return -1 ; +} + +static int +eal_hugepage_init (void ) +{ +struct hugepage_info used_hp[MAX_HUGEPAGE_SIZES ]; +uint64_t memory[RTE_MAX_NUMA_NODES ]; +int hp_sz_idx, socket_id; + +memset (used_hp,0 ,sizeof (used_hp)); + + for (hp_sz_idx = 0; + hp_sz_idx < (int)internal_config.num_hugepage_sizes;
lib/librte_eal/windows/eal_memory.c new file +9
-0+ hp_sz_idx++) { +/* also initialize used_hp hugepage sizes in used_hp */ +struct hugepage_info *hpi; + hpi = &internal_config.hugepage_info[hp_sz_idx]; + used_hp[hp_sz_idx].hugepage_sz = hpi->hugepage_sz; + } + +/* make a copy of socket_mem, needed for balanced allocation. */ + for (socket_id = 0; socket_id < RTE_MAX_NUMA_NODES;socket_id++)
lib/librte_eal/windows/eal_memory.c new file +1
-0+ memory[socket_id] =internal_config.socket_mem[socket_id];
lib/librte_eal/windows/eal_memory.c new file +9
-0+ +/* calculate final number of pages */ +if (calc_num_pages_per_socket (memory, + internal_config.hugepage_info, used_hp, + internal_config.num_hugepage_sizes) <0 ) +return -1 ; + + for (hp_sz_idx = 0; + hp_sz_idx <(int)internal_config.num_hugepage_sizes;
lib/librte_eal/windows/eal_memory.c new file +6
-0+ hp_sz_idx++) { +for (socket_id =0 ; socket_id <RTE_MAX_NUMA_NODES ; + socket_id++) { +struct rte_memseg **pages; +struct hugepage_info *hpi = &used_hp[hp_sz_idx]; + unsigned int num_pages = hpi-Quoted 1 line
num_pages[socket_id];
lib/librte_eal/windows/eal_memory.c new file +7
-0+unsigned int num_pages_alloc; + +if (num_pages ==0 ) +continue ; + +RTE_LOG (DEBUG ,EAL , + "Allocating %u pages of size %" PRIu64 "M onsocket %i\n",
lib/librte_eal/windows/eal_memory.c new file +1
-0+ num_pages, hpi->hugepage_sz >> 20,
socket_id);lib/librte_eal/windows/eal_memory.c new file +3
-0+ +/* we may not be able to allocate all pages in one go, +* because we break up our memory map into multiple
lib/librte_eal/windows/eal_memory.c new file +15
-0+* memseg lists. therefore, try allocating multiple +* times and see if we can get the desired number of +* pages from multiple allocations. +*/ + + num_pages_alloc =0 ; + do { + int i, cur_pages, needed; + + needed = num_pages - num_pages_alloc; + + pages = malloc(sizeof(*pages) * needed); + + /* do not request exact number of pages */ + cur_pages =eal_memalloc_alloc_seg_bulk(pages,
lib/librte_eal/windows/eal_memory.c new file +12
-0+ needed, hpi->hugepage_sz, + socket_id, false); +if (cur_pages <=0 ) { +free (pages); +return -1 ; + } + +/* mark preallocated pages as unfreeable */ +for (i =0 ; i < cur_pages; i++) { +struct rte_memseg *ms = pages[i]; + ms->flags |= +RTE_MEMSEG_FLAG_DO_NOT_FREE;lib/librte_eal/windows/eal_memory.c new file +17
-0+ } +free (pages); + + num_pages_alloc += cur_pages; + }while (num_pages_alloc != num_pages); + } + } +/* if socket limits were specified, set them */ +if (internal_config.force_socket_limits) { +unsigned int i; +for (i =0 ; i <RTE_MAX_NUMA_NODES ; i++) { +uint64_t limit = internal_config.socket_limit[i]; +if (limit ==0 ) +continue ; +if (rte_mem_alloc_validator_register ("socket-limit" , + limits_callback, i, limit)) + RTE_LOG(ERR, EAL, "Failed to register socket"
lib/librte_eal/windows/eal_memory.c new file +65
-0+ "limits validator callback\n"); + } + } +return 0 ; +} + +static int +eal_nohuge_init(void) +{ + struct rte_mem_config *mcfg; + struct rte_memseg_list *msl; + int n_segs, cur_seg; + uint64_t page_sz; + void *addr; + struct rte_fbarray *arr; + struct rte_memseg *ms; + + mcfg = rte_eal_get_configuration()->mem_config; + + /* nohuge mode is legacy mode */ + internal_config.legacy_mem = 1; + + /* create a memseg list */ + msl = &mcfg->memsegs[0]; + + page_sz = RTE_PGSIZE_4K; + n_segs = internal_config.memory / page_sz; + + if (rte_fbarray_init(&msl->memseg_arr, "nohugemem", n_segs, + sizeof(struct rte_memseg))) { + RTE_LOG(ERR, EAL, "Cannot allocate memseg list\n"); + return -1; + } + + addr = eal_mem_alloc(internal_config.memory, 0); + if (addr == NULL) { + RTE_LOG(ERR, EAL, "Cannot allocate %zu bytes", + internal_config.memory); + return -1; + } + + msl->base_va = addr; + msl->page_sz = page_sz; + msl->socket_id = 0; + msl->len = internal_config.memory; + msl->heap = 1; + + /* populate memsegs. each memseg is one page long */ + for (cur_seg = 0; cur_seg < n_segs; cur_seg++) { + arr = &msl->memseg_arr; + + ms = rte_fbarray_get(arr, cur_seg); + ms->iova = RTE_BAD_IOVA; + ms->addr = addr; + ms->hugepage_sz = page_sz; + ms->socket_id = 0; + ms->len = page_sz; + + rte_fbarray_set_used(arr, cur_seg); + + addr = RTE_PTR_ADD(addr, (size_t)page_sz); + } + + if (mcfg->dma_maskbits && + rte_mem_check_dma_mask_thread_unsafe(mcfg-Quoted 1 line
dma_maskbits)) {lib/librte_eal/windows/eal_memory.c new file +3
-0+ RTE_LOG(ERR, EAL, + "%s(): couldn't allocate memory due to IOVA " + "exceeding limits of current DMA mask.\n",
__func__);lib/librte_eal/windows/eal_memory.c new file +19
-0+return -1 ; + } + +return 0 ; +} + +int +rte_eal_hugepage_init (void ) +{ +return internal_config.no_hugetlbfs ? +eal_nohuge_init () :eal_hugepage_init (); +} + +int +rte_eal_hugepage_attach (void ) +{ +EAL_LOG_NOT_IMPLEMENTED (); +return -1 ; +}lib/librte_eal/windows/eal_mp.c new file +51
-0@@ -0,0 +1,103 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright (c) 2020 Dmitry Kozlyuk +*/ + +/** +* @file Multiprocess support stubs +* +* Stubs must log an error until implemented. If success is required +* for non-multiprocess operation, stub must log a warning and a comment +* must document what requires success emulation. +*/ + +#include <rte_eal.h> +#include <rte_errno.h> + +#include "eal_private.h" +#include "eal_windows.h" +#include "malloc_mp.h" + +void +rte_mp_channel_cleanup (void ) +{ +EAL_LOG_NOT_IMPLEMENTED (); +} + +int +rte_mp_action_register (const char *name,rte_mp_t action) +{ +RTE_SET_USED (name); +RTE_SET_USED (action); +EAL_LOG_NOT_IMPLEMENTED (); +return -1 ; +} + +void +rte_mp_action_unregister (const char *name) +{ +RTE_SET_USED (name); +EAL_LOG_NOT_IMPLEMENTED (); +} + +int +rte_mp_sendmsg (struct rte_mp_msg *msg) +{ +RTE_SET_USED (msg); +EAL_LOG_NOT_IMPLEMENTED (); +return -1 ; +} + +int +rte_mp_request_sync(struct rte_mp_msg *req, struct rte_mp_reply*reply,
lib/librte_eal/windows/eal_mp.c new file +52
-0+ const struct timespec *ts) +{ +RTE_SET_USED (req); +RTE_SET_USED (reply); +RTE_SET_USED (ts); +EAL_LOG_NOT_IMPLEMENTED (); +return -1 ; +} + +int +rte_mp_request_async (struct rte_mp_msg *req,const struct timespec *ts, +rte_mp_async_reply_t clb) +{ +RTE_SET_USED (req); +RTE_SET_USED (ts); +RTE_SET_USED (clb); +EAL_LOG_NOT_IMPLEMENTED (); +return -1 ; +} + +int +rte_mp_reply (struct rte_mp_msg *msg,const char *peer) +{ +RTE_SET_USED (msg); +RTE_SET_USED (peer); +EAL_LOG_NOT_IMPLEMENTED (); +return -1 ; +} + +int +register_mp_requests (void ) +{ +/* Non-stub function succeeds if multi-process is not supported. */ +EAL_LOG_STUB (); +return 0 ; +} + +int +request_to_primary (struct malloc_mp_req *req) +{ +RTE_SET_USED (req); +EAL_LOG_NOT_IMPLEMENTED (); +return -1 ; +} + +int +request_sync (void ) +{ +/* Common memory allocator depends on this function success. */ +EAL_LOG_STUB (); +return 0 ; +}lib/librte_eal/windows/eal_windows.h +16
-0@@ -9,8 +9,24 @@* @file Facilities private to Windows EAL */ +#include <rte_errno.h> #include <rte_windows.h> +/** +* Log current function as not implemented and set rte_errno. +*/ +#define EAL_LOG_NOT_IMPLEMENTED () \ + do { \ + RTE_LOG(DEBUG, EAL, "%s() is not implemented\n", __func__); \ + rte_errno =ENOTSUP ; \ + }while (0 ) + +/** +* Log current function as a stub. + */ +#define EAL_LOG_STUB () \ + RTE_LOG(DEBUG, EAL, "Windows: %s() is a stub\n", __func__) +/** * Create a map of processors and cores on the system. */
lib/librte_eal/windows/eal_windows.h +74
-0@@ -36,4 +52,78 @@ int eal_thread_create(pthread_t *thread);*/ unsigned int eal_socket_numa_node (unsigned int socket_id); +/** +* Open virt2phys driver interface device. +* +* @return 0 on success, (-1) on failure. +*/ +int eal_mem_virt2iova_init (void ); + +/** +* Locate Win32 memory management routines in system libraries. +* +* @return 0 on success, (-1) on failure. +*/ +int eal_mem_win32api_init (void ); + +/** +* Allocate a contiguous chunk of virtual memory. +* +* Use eal_mem_free() to free allocated memory. +* +* @param size +* Number of bytes to allocate. +* @param page_size +* If non-zero, means memory must be allocated in hugepages +* of the specified size. The *size* parameter must then be +* a multiple of the largest hugepage size requested. +* @return +* Address of allocated memory, NULL on failure and rte_errno is set. +*/ +void *eal_mem_alloc (size_t size,size_t page_size); + +/** +* Allocate new memory in hugepages on the specified NUMA node. +* +* @param size +* Number of bytes to allocate. Must be a multiple of huge page size. +* @param socket_id +* Socket ID. +* @return +* Address of the memory allocated on success or NULL on failure. +*/ +void *eal_mem_alloc_socket (size_t size,int socket_id); + +/** +* Commit memory previously reserved with eal_mem_reserve() +* or decommitted from hugepages by eal_mem_decommit(). +* +* @param requested_addr +* Address within a reserved region. Must not be NULL. +* @param size +* Number of bytes to commit. Must be a multiple of page size. +* @param socket_id +* Socket ID to allocate on. Can be SOCKET_ID_ANY. +* @return +* On success, address of the committed memory, that is, requested_addr. +* On failure, NULL and rte_errno is set. +*/ +void *eal_mem_commit (void *requested_addr,size_t size,int socket_id); + +/** +* Put allocated or committed memory back into reserved state. +* +* @param addr +* Address of the region to decommit. +* @param size +* Number of bytes to decommit. +* +* The *addr* and *size* must match location and size +* of a previously allocated or committed region. +* +* @return +* 0 on success, (-1) on failure. +*/ +int eal_mem_decommit (void *addr,size_t size); +#endif /* _EAL_WINDOWS_H_ */ lib/librte_eal/windows/include/meson.build +1
-0@@ -5,5 +5,6 @@ includes += include_directories('.') headers += files( 'rte_os.h', + 'rte_virt2phys.h', 'rte_windows.h', )lib/librte_eal/windows/include/rte_os.h +4
-0@@ -36,6 +36,10 @@ extern "C" {#define strncasecmp (s1, s2, count) _strnicmp(s1, s2, count) +#define open _open +#define close _close +#define unlink _unlink +/* cpu_set macros implementation */ #define RTE_CPU_AND (dst, src1, src2) CPU_AND(dst, src1, src2)#define RTE_CPU_OR (dst, src1, src2) CPU_OR(dst, src1, src2)lib/librte_eal/windows/include/rte_virt2phys.h new file +34
-0@@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright (c) 2020 Dmitry Kozlyuk +*/ + +/** +* @file virt2phys driver interface +*/ + +/** +* Driver device interface GUID {539c2135-793a-4926-afec-d3a1b61bbc8a}. +*/ +DEFINE_GUID (GUID_DEVINTERFACE_VIRT2PHYS , +0x539c2135 ,0x793a ,0x4926 , +0xaf ,0xec ,0xd3 ,0xa1 ,0xb6 ,0x1b ,0xbc ,0x8a ); + +/** +* Driver device type for IO control codes. +*/ +#define VIRT2PHYS_DEVTYPE 0x8000 + +/** +* Translate a valid non-paged virtual address to a physical address. +* +* Note: A physical address zero (0) is reported if input address +* is paged out or not mapped. However, if input is a valid mapping +* of I/O port 0x0000, output is also zero. There is no way +* to distinguish between these cases by return value only. +* +* Input: a non-paged virtual address (PVOID). +* +* Output: the corresponding physical address (LARGE_INTEGER). +*/ +#define IOCTL_VIRT2PHYS_TRANSLATE CTL_CODE( \ + VIRT2PHYS_DEVTYPE, 0x800, METHOD_BUFFERED,
FILE_ANY_ACCESS)lib/librte_eal/windows/include/rte_windows.h +2
-0@@ -23,6 +23,8 @@#include <basetsd.h> #include <psapi.h> +#include <setupapi.h> +#include <winioctl.h> /* Have GUIDs defined. */ #ifndef INITGUID lib/librte_eal/windows/include/unistd.h +3
-0@@ -9,4 +9,7 @@* as Microsoft libc does not contain unistd.h. This may be removed * in future releases. */ + +#include <io.h> +#endif /* _UNISTD_H_ */ lib/librte_eal/windows/meson.build +5
-1@@ -8,6 +8,11 @@ sources += files( 'eal_debug.c', 'eal_hugepages.c', 'eal_lcore.c', + 'eal_memalloc.c', + 'eal_memory.c', + 'eal_mp.c', 'eal_thread.c', 'getopt.c', ) + +dpdk_conf.set10('RTE_EAL_NUMA_AWARE_HUGEPAGES', true) --2.25.1
Re: [dpdk-dev] [PATCH v4 8/8] eal/windows: implement basic memory management
Quoted 23 lines · On Wed, 13 May 2020 08:24:12 +0000 Fady Bader <[email protected]> wrote:
Hi Dmitry,
I'm using your latest memory management patchset and getting an error in the function VirualAlloc2 ineal_mem_commit, error code:0x57(ERROR_INVALID_PARAMETER). I'm using Windows server 2019 build 17763, and followed the steps to Grant *Lock pages in memory* Privilege.The parameters that are sent to the function are:
GetCurrentProcess()is -1.requested_addris 0x0000025b`93800000.
Size is0x200000(sysInfo.dwAllocationGranularity is0x10000). Flags is0x20007000.
Also,Socket_idis 0.The call stack is:
00 dpdk_mempool_test!eal_mem_commit+0x253
01 dpdk_mempool_test!alloc_seg+0x1b0
02 dpdk_mempool_test!alloc_seg_walk+0x2a1
03 dpdk_mempool_test!rte_memseg_list_walk_thread_unsafe+0x81 04 dpdk_mempool_test!eal_memalloc_alloc_seg_bulk+0x1a5 05 dpdk_mempool_test!alloc_pages_on_heap+0x13a
06 dpdk_mempool_test!try_expand_heap_primary+0x1dc 07 dpdk_mempool_test!try_expand_heap+0xf5
08 dpdk_mempool_test!alloc_more_mem_on_socket+0x693
Quoted 8 lines
09 dpdk_mempool_test!malloc_heap_alloc_on_heap_id+0x2a7 0a dpdk_mempool_test!malloc_heap_alloc+0x184 0b dpdk_mempool_test!malloc_socket+0xf9
0c dpdk_mempool_test!rte_malloc_socket+0x39
0d dpdk_mempool_test!rte_zmalloc_socket+0x31 0e dpdk_mempool_test!rte_zmalloc+0x2d
0f dpdk_mempool_test!rte_mempool_create_empty+0x1c9 10 dpdk_mempool_test!rte_mempool_create+0xf8
Hi Fady,
Can you share the code snippet causing this?
--
Dmitry Kozlyuk
Re: [dpdk-dev] [PATCH v4 8/8] eal/windows: implement basic memory management
Quoted 55 lines
-----Original Message-----
From: Dmitry Kozlyuk <[email protected]>
Sent: Wednesday, May 13, 2020 11:43 AM
To: Fady Bader <[email protected]>
Cc: [email protected]; Dmitry Malloy (MESHCHANINOV) <[email protected]>; Narcisa Ana Maria Vasile <[email protected]>; Tal Shnaiderman <[email protected]>; Thomas Monjalon <[email protected]>; Harini Ramakrishnan <[email protected]>; Omar Cardona <[email protected]>; Pallavi Kadam <[email protected]>; Ranjit Menon <[email protected]>; John McNamara <[email protected]>; Marko Kovacevic <[email protected]>; Anatoly Burakov <[email protected]>
Subject: Re: [PATCH v4 8/8] eal/windows: implement basic memory managementQuoted 4 lines · On Wed, 13 May 2020 08:24:12 +0000 Fady Bader <[email protected]> wrote:
Hi Dmitry,
I'm using your latest memory management patchset and getting an error in the function VirualAlloc2 ineal_mem_commit, error code:0x57(ERROR_INVALID_PARAMETER). I'm using Windows server 2019 build17763,
Quoted 19 lines
and followed the steps to Grant *Lock pages in memory* Privilege.
The parameters that are sent to the function are:
GetCurrentProcess()is -1.requested_addris 0x0000025b`93800000.
Size is0x200000(sysInfo.dwAllocationGranularity is0x10000). Flags is0x20007000.
Also,Socket_idis 0.The call stack is:
00 dpdk_mempool_test!eal_mem_commit+0x253
01 dpdk_mempool_test!alloc_seg+0x1b0
02 dpdk_mempool_test!alloc_seg_walk+0x2a1
03 dpdk_mempool_test!rte_memseg_list_walk_thread_unsafe+0x81 04 dpdk_mempool_test!eal_memalloc_alloc_seg_bulk+0x1a5 05 dpdk_mempool_test!alloc_pages_on_heap+0x13a
06 dpdk_mempool_test!try_expand_heap_primary+0x1dc 07 dpdk_mempool_test!try_expand_heap+0xf5
08 dpdk_mempool_test!alloc_more_mem_on_socket+0x693Quoted 8 lines
09 dpdk_mempool_test!malloc_heap_alloc_on_heap_id+0x2a7 0a dpdk_mempool_test!malloc_heap_alloc+0x184 0b dpdk_mempool_test!malloc_socket+0xf9
0c dpdk_mempool_test!rte_malloc_socket+0x39
0d dpdk_mempool_test!rte_zmalloc_socket+0x31 0e dpdk_mempool_test!rte_zmalloc+0x2d
0f dpdk_mempool_test!rte_mempool_create_empty+0x1c9 10 dpdk_mempool_test!rte_mempool_create+0xf8
Quoted 4 lines
Hi Fady,
Can you share the code snippet causing this?
[snip]
+
+void*
+eal_mem_commit(void *requested_addr, size_t size, int socket_id) +{
+ MEM_EXTENDED_PARAMETER param;
+ DWORD param_count = 0;
+ DWORD flags;
+ void *addr;
+
+ if (requested_addr != NULL) {
+ MEMORY_BASIC_INFORMATION info;
+ if (VirtualQuery(requested_addr, &info, sizeof(info)) == 0) {
+ RTE_LOG_WIN32_ERR("VirtualQuery()");
+ return NULL;
+ }
+
+ /* Split reserved region if only a part is committed. */
+ flags = MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER;
+ if ((info.RegionSize > size) &&
+ !VirtualFree(requested_addr, size, flags)) {
+ RTE_LOG_WIN32_ERR("VirtualFree(%p, %zu, "
+ "<split placeholder>)", requested_addr, size);
+ return NULL;
+ }
+ }
+
+ if (socket_id != SOCKET_ID_ANY) {
+ param_count = 1;
+ memset(¶m, 0, sizeof(param));
+ param.Type = MemExtendedParameterNumaNode;
+ param.ULong = eal_socket_numa_node(socket_id);
+ }
+
+ flags = MEM_RESERVE | MEM_COMMIT | MEM_LARGE_PAGES;
+ if (requested_addr != NULL)
+ flags |= MEM_REPLACE_PLACEHOLDER;
+
+ addr = VirtualAlloc2(GetCurrentProcess(), requested_addr, size,
+ flags, PAGE_READWRITE, ¶m, param_count);
+ if (addr == NULL) {
+ DWORD err = GetLastError();
+ RTE_LOG_WIN32_ERR("VirtualAlloc2(%p, %zu, "
+ "<replace placeholder>)", addr, size);
+ set_errno_from_win32_alloc_error(err);
+ return NULL;
+ }
+
+ return addr;
+}
+
Quoted 2 lines
--
Dmitry Kozlyuk
Re: [dpdk-dev] [PATCH v4 8/8] eal/windows: implement basic memory management
Quoted 39 lines
-----Original Message-----
From: Fady Bader
Sent: Wednesday, May 13, 2020 12:09 PM
To: Dmitry Kozlyuk <[email protected]>
Cc: [email protected]; Dmitry Malloy (MESHCHANINOV) <[email protected]>; Narcisa Ana Maria Vasile <[email protected]>; Tal Shnaiderman <[email protected]>; Thomas Monjalon <[email protected]>; Harini Ramakrishnan <[email protected]>; Omar Cardona <[email protected]>; Pallavi Kadam <[email protected]>; Ranjit Menon <[email protected]>; John McNamara <[email protected]>; Marko Kovacevic <[email protected]>; Anatoly Burakov <[email protected]>
Subject: RE: [PATCH v4 8/8] eal/windows: implement basic memory managementQuoted 7 lines
-----Original Message-----
From: Dmitry Kozlyuk <[email protected]>
Sent: Wednesday, May 13, 2020 11:43 AM
To: Fady Bader <[email protected]>
Cc: [email protected]; Dmitry Malloy (MESHCHANINOV) <[email protected]>; Narcisa Ana Maria Vasile <[email protected]>; Tal ShnaidermanQuoted 10 lines
Thomas Monjalon <[email protected]>; Harini Ramakrishnan <[email protected]>; Omar Cardona <[email protected]>; Pallavi Kadam <[email protected]>; Ranjit Menon <[email protected]>; John McNamara <[email protected]>; Marko Kovacevic <[email protected]>; Anatoly Burakov <[email protected]>
Subject: Re: [PATCH v4 8/8] eal/windows: implement basic memory management
Quoted 42 lines
Quoted 7 lines
Quoted 4 lines · On Wed, 13 May 2020 08:24:12 +0000 Fady Bader <[email protected]> wrote:
Hi Dmitry,
I'm using your latest memory management patchset and getting an error in the function VirualAlloc2 ineal_mem_commit, error code:0x57(ERROR_INVALID_PARAMETER). I'm using Windows server 2019build
Quoted 33 lines
17763,
Quoted 19 lines
and followed the steps to Grant *Lock pages in memory* Privilege.
The parameters that are sent to the function are:
GetCurrentProcess()is -1.requested_addris 0x0000025b`93800000.
Size is0x200000(sysInfo.dwAllocationGranularity is0x10000). Flags is0x20007000.
Also,Socket_idis 0.The call stack is:
00 dpdk_mempool_test!eal_mem_commit+0x253
01 dpdk_mempool_test!alloc_seg+0x1b0
02 dpdk_mempool_test!alloc_seg_walk+0x2a1
03 dpdk_mempool_test!rte_memseg_list_walk_thread_unsafe+0x81 04 dpdk_mempool_test!eal_memalloc_alloc_seg_bulk+0x1a5 05 dpdk_mempool_test!alloc_pages_on_heap+0x13a
06 dpdk_mempool_test!try_expand_heap_primary+0x1dc 07 dpdk_mempool_test!try_expand_heap+0xf5
08 dpdk_mempool_test!alloc_more_mem_on_socket+0x693Quoted 8 lines
09 dpdk_mempool_test!malloc_heap_alloc_on_heap_id+0x2a7 0a dpdk_mempool_test!malloc_heap_alloc+0x184 0b dpdk_mempool_test!malloc_socket+0xf9
0c dpdk_mempool_test!rte_malloc_socket+0x39
0d dpdk_mempool_test!rte_zmalloc_socket+0x31 0e dpdk_mempool_test!rte_zmalloc+0x2d
0f dpdk_mempool_test!rte_mempool_create_empty+0x1c9 10 dpdk_mempool_test!rte_mempool_create+0xf8Hi Fady,
Can you share the code snippet causing this?
I got it from dpdk\app\test\test_mempool
Line 496:
/* create a mempool (without cache) */mp_nocache = rte_mempool_create("test_nocache", MEMPOOL_SIZE,
MEMPOOL_ELT_SIZE, 0, 0, NULL, NULL, my_obj_init, NULL, SOCKET_ID_ANY, 0);
Quoted 3 lines
Quoted 3 lines
--
Dmitry Kozlyuk
Re: [dpdk-dev] [PATCH v4 8/8] eal/windows: implement basic memory management
In reply to Fady BaderQuoted 54 lines · On Wed, 13 May 2020 09:09:22 +0000 Fady Bader <[email protected]> wrote:
Quoted 54 lines
-----Original Message-----
From: Dmitry Kozlyuk <[email protected]>
Sent: Wednesday, May 13, 2020 11:43 AM
To: Fady Bader <[email protected]>
Cc: [email protected]; Dmitry Malloy (MESHCHANINOV) <[email protected]>; Narcisa Ana Maria Vasile <[email protected]>; Tal Shnaiderman <[email protected]>; Thomas Monjalon <[email protected]>; Harini Ramakrishnan <[email protected]>; Omar Cardona <[email protected]>; Pallavi Kadam <[email protected]>; Ranjit Menon <[email protected]>; John McNamara <[email protected]>; Marko Kovacevic <[email protected]>; Anatoly Burakov <[email protected]>
Subject: Re: [PATCH v4 8/8] eal/windows: implement basic memory managementQuoted 5 lines · On Wed, 13 May 2020 08:24:12 +0000 Fady Bader <[email protected]> wrote:
Hi Dmitry,
I'm using your latest memory management patchset and getting an error in the function VirualAlloc2 ineal_mem_commit, error code:0x57(ERROR_INVALID_PARAMETER). I'm using Windows server 2019 build17763,
Quoted 27 lines
and followed the steps to Grant *Lock pages in memory* Privilege.
The parameters that are sent to the function are:
GetCurrentProcess()is -1.requested_addris 0x0000025b`93800000.
Size is0x200000(sysInfo.dwAllocationGranularity is0x10000). Flags is0x20007000.
Also,Socket_idis 0.The call stack is:
00 dpdk_mempool_test!eal_mem_commit+0x253
01 dpdk_mempool_test!alloc_seg+0x1b0
02 dpdk_mempool_test!alloc_seg_walk+0x2a1
03 dpdk_mempool_test!rte_memseg_list_walk_thread_unsafe+0x81 04 dpdk_mempool_test!eal_memalloc_alloc_seg_bulk+0x1a5 05 dpdk_mempool_test!alloc_pages_on_heap+0x13a
06 dpdk_mempool_test!try_expand_heap_primary+0x1dc 07 dpdk_mempool_test!try_expand_heap+0xf5
08 dpdk_mempool_test!alloc_more_mem_on_socket+0x693 09 dpdk_mempool_test!malloc_heap_alloc_on_heap_id+0x2a7 0a dpdk_mempool_test!malloc_heap_alloc+0x184
0b dpdk_mempool_test!malloc_socket+0xf9
0c dpdk_mempool_test!rte_malloc_socket+0x39
0d dpdk_mempool_test!rte_zmalloc_socket+0x31
0e dpdk_mempool_test!rte_zmalloc+0x2d
0f dpdk_mempool_test!rte_mempool_create_empty+0x1c9 10 dpdk_mempool_test!rte_mempool_create+0xf8
Quoted 6 lines
Quoted 4 lines
Hi Fady,
Can you share the code snippet causing this?
[snip]
[snip]
I meant the code of the application that calls rte_mempool_create(). Or is it one of the DPDK test applications?
--
Dmitry Kozlyuk
Re: [dpdk-dev] [PATCH v4 8/8] eal/windows: implement basic memory management
Quoted 79 lines
-----Original Message-----
From: Dmitry Kozlyuk <[email protected]>
Sent: Wednesday, May 13, 2020 12:39 PM
To: Fady Bader <[email protected]>
Cc: [email protected]; Dmitry Malloy (MESHCHANINOV) <[email protected]>; Narcisa Ana Maria Vasile <[email protected]>; Tal Shnaiderman <[email protected]>; Thomas Monjalon <[email protected]>; Harini Ramakrishnan <[email protected]>; Omar Cardona <[email protected]>; Pallavi Kadam <[email protected]>; Ranjit Menon <[email protected]>; John McNamara <[email protected]>; Marko Kovacevic <[email protected]>; Anatoly Burakov <[email protected]>
Subject: Re: [PATCH v4 8/8] eal/windows: implement basic memory managementQuoted 8 lines · On Wed, 13 May 2020 09:09:22 +0000 Fady Bader <[email protected]> wrote:
Quoted 8 lines
-----Original Message-----
From: Dmitry Kozlyuk <[email protected]>
Sent: Wednesday, May 13, 2020 11:43 AM
To: Fady Bader <[email protected]>
Cc: [email protected]; Dmitry Malloy (MESHCHANINOV) <[email protected]>; Narcisa Ana Maria Vasile <[email protected]>; Tal Shnaiderman <[email protected]>; Thomas Monjalon <[email protected]>;Harini
Quoted 48 lines
Quoted 48 lines
Ramakrishnan <[email protected]>; Omar Cardona <[email protected]>; Pallavi Kadam <[email protected]>; Ranjit Menon <[email protected]>; John McNamara <[email protected]>; Marko Kovacevic <[email protected]>; Anatoly Burakov <[email protected]>
Subject: Re: [PATCH v4 8/8] eal/windows: implement basic memory managementQuoted 5 lines · On Wed, 13 May 2020 08:24:12 +0000 Fady Bader <[email protected]> wrote:
Hi Dmitry,
I'm using your latest memory management patchset and getting an error in the function VirualAlloc2 ineal_mem_commit, error code:0x57(ERROR_INVALID_PARAMETER). I'm using Windows server 2019 build17763,
Quoted 19 lines
and followed the steps to Grant *Lock pages in memory* Privilege.
The parameters that are sent to the function are:
GetCurrentProcess()is -1.requested_addris 0x0000025b`93800000.
Size is0x200000(sysInfo.dwAllocationGranularity is0x10000). Flags is0x20007000.
Also,Socket_idis 0.The call stack is:
00 dpdk_mempool_test!eal_mem_commit+0x253
01 dpdk_mempool_test!alloc_seg+0x1b0
02 dpdk_mempool_test!alloc_seg_walk+0x2a1
03 dpdk_mempool_test!rte_memseg_list_walk_thread_unsafe+0x81 04 dpdk_mempool_test!eal_memalloc_alloc_seg_bulk+0x1a5 05 dpdk_mempool_test!alloc_pages_on_heap+0x13a
06 dpdk_mempool_test!try_expand_heap_primary+0x1dc 07 dpdk_mempool_test!try_expand_heap+0xf5
08 dpdk_mempool_test!alloc_more_mem_on_socket+0x693Quoted 8 lines
09 dpdk_mempool_test!malloc_heap_alloc_on_heap_id+0x2a7 0a dpdk_mempool_test!malloc_heap_alloc+0x184 0b dpdk_mempool_test!malloc_socket+0xf9
0c dpdk_mempool_test!rte_malloc_socket+0x39
0d dpdk_mempool_test!rte_zmalloc_socket+0x31 0e dpdk_mempool_test!rte_zmalloc+0x2d
0f dpdk_mempool_test!rte_mempool_create_empty+0x1c9 10 dpdk_mempool_test!rte_mempool_create+0xf8Hi Fady,
Quoted 8 lines
Quoted 4 lines
Quoted 2 lines
Can you share the code snippet causing this?
[snip]
[snip]
I meant the code of the application that calls
rte_mempool_create(). Or is it one of the DPDK test applications?
I got it from dpdk\app\test\test_mempool
Line 496:
/* create a mempool (without cache) */mp_nocache = rte_mempool_create("test_nocache", MEMPOOL_SIZE,
MEMPOOL_ELT_SIZE, 0, 0, NULL, NULL, my_obj_init, NULL, SOCKET_ID_ANY, 0);
Quoted 3 lines
--
Dmitry Kozlyuk
Re: [dpdk-dev] [PATCH v4 8/8] eal/windows: implement basic memory management
On Wed, 13 May 2020 12:25:10 +0000
Fady Bader <[email protected]> wrote:
[snip]
Quoted 6 lines
Quoted 3 lines
I meant the code of the application that calls
rte_mempool_create(). Or is it one of the DPDK test applications?I got it from dpdk\app\test\test_mempool
Quoted 8 lines
Line 496:
/* create a mempool (without cache) */mp_nocache= rte_mempool_create("test_nocache",MEMPOOL_SIZE,MEMPOOL_ELT_SIZE, 0, 0, NULL, NULL, my_obj_init, NULL, SOCKET_ID_ANY, 0);
For building this code you must have enabled librte_ring, librte_mempool, and drivers/mempool, and I assume you build test code without librte_cmdline somehow. This are nontrivial changes, so I can't be sure to reproduce them exactly. Can you please share a complete patch?
Meanwhile, I observe a similar issue where rte_mempool_create() fails to allocate memory and hangs when compiled with Clang, succeeds with native MinGW, but still hangs with cross MinGW. I'm investigating it.
Testing patch follows, the snippet added is in
examples/helloworld/main.
---
config/meson.build +1 -1
@@ -117,7 +117,7 @@ if not is_windows endif # use pthreads if available for the platform -if not is_ms_linker +if cc.find_library('pthread', required: false).found() add_project_link_arguments('-pthread', language: 'c') dpdk_extra_ldflags += '-pthread' endif
drivers/meson.build +26 -25
98 lines not on this page · raw message
examples/helloworld/main.c +21 -0
@@ -14,6 +14,8 @@#include <rte_per_lcore.h> #include <rte_lcore.h> #include <rte_debug.h> +#include <rte_mempool.h> +#include <rte_errno.h> static int lcore_hello(__rte_unused void *arg)
@@ -29,11 +31,30 @@ main(int argc, char **argv) {int ret;unsigned lcore_id; +struct rte_mempool *pool; + +rte_log_set_level (RTE_LOGTYPE_EAL ,RTE_LOG_DEBUG ); ret =rte_eal_init (argc, argv);if (ret <0 )rte_panic ("Cannot init EAL\n" ); + pool =rte_mempool_create ( +"test_mempool" , + (1 <<18 ) -1 , + (1 <<12 ), +1 <<9 , +0 , +NULL ,NULL ,NULL ,NULL , +SOCKET_ID_ANY , +0 ); +if (!pool) { +RTE_LOG (ERR ,USER1 ,"cannot create mempool: %d\n" , rte_errno); +return EXIT_FAILURE ; + } + +rte_mempool_free (pool); +/* call lcore_hello() on every slave lcore */ RTE_LCORE_FOREACH_SLAVE (lcore_id) {rte_eal_remote_launch (lcore_hello,NULL , lcore_id);
examples/meson.build +1 -1
@@ -83,7 +83,7 @@ foreach example: examples includes = [include_directories(example)] deps = ['eal', 'mempool', 'net', 'mbuf', 'ethdev', 'cmdline'] if is_windows - deps = ['eal'] # only supported lib on Windows currently + deps = ['eal', 'mempool', 'ring'] # only supported lib on Windows currently endif subdir(example)
lib/librte_mempool/rte_mempool.c +12 -13
@@ -12,7 +12,6 @@#include <inttypes.h> #include <errno.h> #include <sys/queue.h> -#include <sys/mman.h> #include <rte_common.h> #include <rte_log.h>
@@ -148,7 +147,7 @@ get_min_page_size(int socket_id)rte_memseg_list_walk (find_min_pagesz, &wa); -return wa.min ==SIZE_MAX ? (size_t )getpagesize () : wa.min; +return wa.min ==SIZE_MAX ? (size_t )rte_get_page_size () : wa.min; }
@@ -526,7 +525,7 @@ rte_mempool_get_page_size(struct rte_mempool *mp, size_t *pg_sz)else if (rte_eal_has_hugepages () || alloc_in_ext_mem) *pg_sz =get_min_page_size (mp->socket_id);else - *pg_sz =getpagesize (); + *pg_sz =rte_get_page_size ();rte_mempool_trace_get_page_size (mp, *pg_sz);return 0 ;
@@ -686,7 +685,7 @@ get_anon_size(const struct rte_mempool *mp)size_t min_chunk_size;size_t align; - pg_sz =getpagesize (); + pg_sz =rte_get_page_size (); pg_shift =rte_bsf32 (pg_sz); size =rte_mempool_ops_calc_mem_size (mp, mp->size, pg_shift, &min_chunk_size, &align);
@@ -710,7 +709,7 @@ rte_mempool_memchunk_anon_free(struct rte_mempool_memhdr *memhdr,if (size <0 )return ; -munmap (opaque, size); +rte_mem_unmap (opaque, size); }/* populate the mempool with an anonymous mapping */
@@ -740,20 +739,20 @@ rte_mempool_populate_anon(struct rte_mempool *mp) }/* get chunk of virtually continuous memory */ - addr =mmap (NULL , size,PROT_READ |PROT_WRITE , -MAP_SHARED |MAP_ANONYMOUS ,-1 ,0 ); -if (addr ==MAP_FAILED ) { - rte_errno = errno; + addr =rte_mem_map (NULL , size,RTE_PROT_READ |RTE_PROT_WRITE , +RTE_MAP_SHARED |RTE_MAP_ANONYMOUS ,-1 ,0 ); +if (addr ==NULL ) {return 0 ; }/* can't use MMAP_LOCKED, it does not exist on BSD */ -if (mlock (addr, size) <0 ) { - rte_errno = errno; -munmap (addr, size); +if (rte_mem_lock (addr, size) <0 ) { + ret = rte_errno; +rte_mem_unmap (addr, size); + rte_errno = ret;return 0 ; } - ret =rte_mempool_populate_virt (mp, addr, size,getpagesize (), + ret =rte_mempool_populate_virt (mp, addr, size,rte_get_page_size (), rte_mempool_memchunk_anon_free, addr);if (ret ==0 )/* should not happen */ ret = -ENOBUFS ;
lib/meson.build +32 -20
91 lines not on this page · raw message
--
Dmitry Kozlyuk
Re: [dpdk-dev] [PATCH v4 8/8] eal/windows: implement basic memory management
Quoted 19 lines · On Mon, 18 May 2020 03:17:04 +0300 Dmitry Kozlyuk <[email protected]>…
On Wed, 13 May 2020 12:25:10 +0000
Fady Bader <[email protected]> wrote:
[snip]Quoted 7 lines
Quoted 4 lines
I meant the code of the application that calls
rte_mempool_create(). Or is it one of the DPDK test applications?I got it from dpdk\app\test\test_mempool
Quoted 8 lines
Line 496:
/* create a mempool (without cache) */mp_nocache= rte_mempool_create("test_nocache",MEMPOOL_SIZE,MEMPOOL_ELT_SIZE, 0, 0, NULL, NULL, my_obj_init, NULL, SOCKET_ID_ANY, 0);
Quoted 5 lines
For building this code you must have enabled
librte_ring,librte_mempool, anddrivers/mempool, and I assume you build test code withoutlibrte_cmdlinesomehow. This are nontrivial changes, so I can't be sure to reproduce them exactly. Can you please share a complete patch?
Never mind, managed to reproduce it.
Quoted 3 lines
Meanwhile, I observe a similar issue where
rte_mempool_create()fails to allocate memory and hangs when compiled with Clang, succeeds with native MinGW, but still hangs with cross MinGW. I'm investigating it.
I must have messed up with my build setup, because I observe this no more, now that I'm testing it from scratch with different toolchains and their versions.
--
Dmitry Kozlyuk
v5 cover Windows basic memory management
In reply to Dmitry KozlyukNote: cover letter updated for v5.
This patchset implements basic MM with the following features:
* Hugepages are dynamically allocated in user-mode.
* Only 2MB hugepages are supported.
* IOVA is always PA, obtained through kernel-mode driver.
* No 32-bit support (presumably not demanded).
* Ni multi-process support (it is forcefully disabled).
* No-huge mode for testing without IOVA is available.
Testing revealed Windows Server 2019 does not allow allocating hugepage memory at a reserved address, despite advertised API. So allocator has to temporary free the region to be allocated. This creates in inherent race condition. This issue is being discussed with Microsoft privately.
New EAL public functions for memory mapping are introduced to mitigate OS differences in DPDK libraries and applications: rte_mem_map, rte_mem_unmap, rte_mem_lock, rte_get_page_size.
To support common MM routines, internal wrappers for low-level memory reservation and file management are introduced. These changes affect Linux and FreeBSD EAL. Shared code is placed unded /unix/ subdirectory (suggested by Thomas).
To avoid code duplication between Linux and Windows EAL, common code for EALs supporting dynamic memory allocation is extracted (discussed with Anatoly Burakov in v4 thread). This is a separate patch to ease the review, but it can be merged with the previous one.
EAL tracepoints save size_t values as long, which is invalid on Windows. New size_t emitter for tracepoints is introduced (suggested by Jerin Jacob to Fady Bader, see [1]). Also, to avoid workaround in every file using the tracepoints, stubs are added to Windows EAL.
Entire <sys/queue.h> is imported from FreeBSD, replacing existing partial import. There is already a license exception for this file. The file is imported as-is, so it causes a bunch of checkpatch warnings.
[1]: http://mails.dpdk.org/archives/dev/2020-May/168076.html
---
v5:
* Fix allocation and deallocation on Windows Server (Fady Bader).
* Replace remaining VirtualFree with VirtualFreeEx (Ranjit Menon).
* Fix errors in eal_get_virtual_area (Anatoly Burakov).
* Fix error handling and documentation for rte_mem_lock (Anatoly Burakov).
* Extract common code for EALs w/dynamic allocation (Anatoly Burakov).
* Use POSIX value for rte_errno after rte_mem_unmap() on Windows.
* Add stubs to use tracing functions without workarounds.
v4:
* Rebase on ToT, drop patches merged into master.
* Rearrange patches to split Windows code (Jerin).
* Fix Linux and FreeBSD build with make (Ophir).
* Use int instead of enum to hold a set of flags (Anatoly).
* Rename eal_mem_reserve items and fix their description (Anatoly).
* Add eal_mem_set_dump() wrapper around madvise (Anatoly).
* Don't claim Windows Server 2016 support due to lack of API (Tal).
* Replace enum rte_page_sizes with a set of #defines (Jerin).
* Fix documentation, SPDX tags, logging (Thomas).
v3:
* Fix Linux build on and aarch64 and 32-bit x86 (reported by CI).
* Fix logic and error handling while allocating segments.
* Fix Unix rte_mem_map(): return NULL on failure.
* Fix some checkpatch.sh issues:
* Do not return positive errno, use DWORD for GetLastError().
* Make dpdk-kmods source files non-executable.
* Improve GSG for Windows Server (suggested by Ranjit Menon).
v2:
* Rebase on ToT. Move all new code shared between Linux and FreeBSD to /unix/ subdirectory, also factor out some existing code there.
* Improve description of Clang issue with rte_page_sizes on Windows. Restore -fstrict-enum for EAL. Check running, not target compiler.
* Use EAL prefix for private facilities instead if RTE.
* Improve documentation comments for new functions.
* Remove co-installer for virt2phys. Add a typecast for clarity.
* Document virt2phys in user guide, improve its own README.
* Explicitly and forcefully disable multi-process.
Dmitry Kozlyuk (11):
eal: replace rte_page_sizes with a set of constants
eal: introduce internal wrappers for file operations
eal: introduce memory management wrappers
eal/mem: extract common code for memseg list initialization eal/mem: extract common code for dynamic memory allocation
trace: add size_t field emitter
eal/windows: add tracing support stubs
eal/windows: replace sys/queue.h with a complete one from FreeBSD eal/windows: improve CPU and NUMA node detection eal/windows: initialize hugepage info
eal/windows: implement basic memory management
config/meson.build | 12 +- doc/guides/windows_gsg/build_dpdk.rst | 20 - doc/guides/windows_gsg/index.rst | 1 + doc/guides/windows_gsg/run_apps.rst | 95 +++ lib/librte_eal/common/eal_common_dynmem.c | 498 ++++++++++++ lib/librte_eal/common/eal_common_fbarray.c | 58 +- lib/librte_eal/common/eal_common_memory.c | 152 +++- lib/librte_eal/common/eal_common_thread.c | 5 +- lib/librte_eal/common/eal_private.h | 228 +++++- lib/librte_eal/common/meson.build | 16 + lib/librte_eal/common/rte_malloc.c | 9 + lib/librte_eal/freebsd/Makefile | 5 + lib/librte_eal/freebsd/eal_memory.c | 94 +-- lib/librte_eal/include/rte_eal_trace.h | 8 +- lib/librte_eal/include/rte_memory.h | 111 ++- lib/librte_eal/include/rte_trace_point.h | 1 + lib/librte_eal/linux/Makefile | 5 + lib/librte_eal/linux/eal_memalloc.c | 5 +- lib/librte_eal/linux/eal_memory.c | 605 +-------------- lib/librte_eal/meson.build | 4 + lib/librte_eal/rte_eal_exports.def | 119 +++ lib/librte_eal/rte_eal_version.map | 6 + lib/librte_eal/unix/eal_unix.c | 51 ++ lib/librte_eal/unix/eal_unix_memory.c | 152 ++++ lib/librte_eal/unix/meson.build | 7 + lib/librte_eal/windows/eal.c | 190 +++++ lib/librte_eal/windows/eal_hugepages.c | 108 +++ lib/librte_eal/windows/eal_lcore.c | 185 +++-- lib/librte_eal/windows/eal_memalloc.c | 442 +++++++++++ lib/librte_eal/windows/eal_memory.c | 710 ++++++++++++++++++ lib/librte_eal/windows/eal_mp.c | 103 +++ lib/librte_eal/windows/eal_windows.h | 85 +++ lib/librte_eal/windows/include/meson.build | 1 + lib/librte_eal/windows/include/rte_os.h | 9 + .../windows/include/rte_virt2phys.h | 34 + lib/librte_eal/windows/include/rte_windows.h | 2 + lib/librte_eal/windows/include/sys/queue.h | 663 ++++++++++++++-- lib/librte_eal/windows/include/unistd.h | 3 + lib/librte_eal/windows/meson.build | 6 + lib/librte_mempool/rte_mempool_trace.h | 10 +- 40 files changed, 3903 insertions(+), 915 deletions(-) create mode 100644 doc/guides/windows_gsg/run_apps.rst create mode 100644 lib/librte_eal/common/eal_common_dynmem.c create mode 100644 lib/librte_eal/unix/eal_unix.c create mode 100644 lib/librte_eal/unix/eal_unix_memory.c create mode 100644 lib/librte_eal/unix/meson.build create mode 100644 lib/librte_eal/windows/eal_hugepages.c create mode 100644 lib/librte_eal/windows/eal_memalloc.c create mode 100644 lib/librte_eal/windows/eal_memory.c create mode 100644 lib/librte_eal/windows/eal_mp.c create mode 100644 lib/librte_eal/windows/include/rte_virt2phys.h
Signature · 1 line
2.25.4
PATCH v5 1/11 eal: replace rte_page_sizes with a set of constants
Clang on Windows follows MS ABI where enum values are limited to 2^31-1. Enum rte_page_sizes has members valued above this limit, which get wrapped to zero, resulting in compilation error (duplicate values in enum). Using MS ABI is mandatory for Windows EAL to call Win32 APIs.
Remove rte_page_sizes and replace its values with #define's. This enumeration is not used in public API, so there's no ABI breakage.
---
Release notes for 20.08 don't exits yet, so not adding anything.
lib/librte_eal/include/rte_memory.h | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-)
lib/librte_eal/include/rte_memory.h +10 -13
@@ -24,19 +24,16 @@ extern "C" {#include <rte_config.h> #include <rte_fbarray.h> -__extension__ -enum rte_page_sizes { -RTE_PGSIZE_4K =1ULL <<12 , -RTE_PGSIZE_64K =1ULL <<16 , -RTE_PGSIZE_256K =1ULL <<18 , -RTE_PGSIZE_2M =1ULL <<21 , -RTE_PGSIZE_16M =1ULL <<24 , -RTE_PGSIZE_256M =1ULL <<28 , -RTE_PGSIZE_512M =1ULL <<29 , -RTE_PGSIZE_1G =1ULL <<30 , -RTE_PGSIZE_4G =1ULL <<32 , -RTE_PGSIZE_16G =1ULL <<34 , -}; +#define RTE_PGSIZE_4K (1ULL << 12) +#define RTE_PGSIZE_64K (1ULL << 16) +#define RTE_PGSIZE_256K (1ULL << 18) +#define RTE_PGSIZE_2M (1ULL << 21) +#define RTE_PGSIZE_16M (1ULL << 24) +#define RTE_PGSIZE_256M (1ULL << 28) +#define RTE_PGSIZE_512M (1ULL << 29) +#define RTE_PGSIZE_1G (1ULL << 30) +#define RTE_PGSIZE_4G (1ULL << 32) +#define RTE_PGSIZE_16G (1ULL << 34)#define SOCKET_ID_ANY -1/**< Any NUMA socket. */
Signature · 1 line
2.25.4
PATCH v5 2/11 eal: introduce internal wrappers for file operations
In reply to Dmitry KozlyukEAL common code uses file locking and truncation. Introduce OS-independent wrappers in order to support both Linux/FreeBSD and Windows:
* eal_file_lock: lock or unlock an open file.
* eal_file_truncate: enforce a given size for an open file.
Wrappers follow POSIX semantics, but interface is not POSIX, so that it can be made more clean, e.g. by not mixing locking operation and behaviour on conflict.
Implementation for Linux and FreeBSD is placed in "unix" subdirectory, which is intended for common code between the two. Files should be named after the ones from which the code is factored in OS subdirectory.
--- lib/librte_eal/common/eal_common_fbarray.c | 21 ++++----- lib/librte_eal/common/eal_private.h | 47 ++++++++++++++++++++ lib/librte_eal/freebsd/Makefile | 4 ++ lib/librte_eal/linux/Makefile | 4 ++ lib/librte_eal/meson.build | 4 ++ lib/librte_eal/unix/eal_unix.c | 51 ++++++++++++++++++++++ lib/librte_eal/unix/meson.build | 6 +++ 7 files changed, 124 insertions(+), 13 deletions(-) create mode 100644 lib/librte_eal/unix/eal_unix.c create mode 100644 lib/librte_eal/unix/meson.build
lib/librte_eal/common/eal_common_fbarray.c +8 -13
@@ -8,8 +8,8 @@#include <sys/mman.h> #include <stdint.h> #include <errno.h> -#include <sys/file.h> #include <string.h> +#include <unistd.h> #include <rte_common.h> #include <rte_log.h>
@@ -85,10 +85,8 @@ resize_and_map(int fd, void *addr, size_t len)char path[PATH_MAX ];void *map_addr; -if (ftruncate (fd, len)) { +if (eal_file_truncate (fd, len)) {RTE_LOG (ERR ,EAL ,"Cannot truncate %s\n" , path); -/* pass errno up the chain */ - rte_errno = errno;return -1 ; }
@@ -778,7 +776,8 @@ rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len, __func__, path, strerror(errno)); rte_errno = errno; goto fail; - }else if (flock (fd,LOCK_EX |LOCK_NB )) { + }else if (eal_file_lock ( + fd,EAL_FLOCK_EXCLUSIVE ,EAL_FLOCK_RETURN )) {RTE_LOG (DEBUG ,EAL ,"%s(): couldn't lock %s: %s\n" , __func__, path,strerror (errno)); rte_errno =EBUSY ;
@@ -789,10 +788,8 @@ rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,* still attach to it, but no other process could reinitialize * it. */ -if (flock (fd,LOCK_SH |LOCK_NB )) { - rte_errno = errno; +if (eal_file_lock (fd,EAL_FLOCK_SHARED ,EAL_FLOCK_RETURN )) goto fail; - }if (resize_and_map (fd, data, mmap_len)) goto fail;
@@ -895,10 +892,8 @@ rte_fbarray_attach(struct rte_fbarray *arr) }/* lock the file, to let others know we're using it */ -if (flock (fd,LOCK_SH |LOCK_NB )) { - rte_errno = errno; +if (eal_file_lock (fd,EAL_FLOCK_SHARED ,EAL_FLOCK_RETURN )) goto fail; - }if (resize_and_map (fd, data, mmap_len)) goto fail;
@@ -1025,7 +1020,7 @@ rte_fbarray_destroy(struct rte_fbarray *arr)* has been detached by all other processes */ fd = tmp->fd; -if (flock (fd,LOCK_EX |LOCK_NB )) { +if (eal_file_lock (fd,EAL_FLOCK_EXCLUSIVE ,EAL_FLOCK_RETURN )) {RTE_LOG (DEBUG ,EAL ,"Cannot destroy fbarray - another process is using it\n" ); rte_errno =EBUSY ; ret =-1 ;
@@ -1042,7 +1037,7 @@ rte_fbarray_destroy(struct rte_fbarray *arr)* we're still holding an exclusive lock, so drop it to * shared. */ -flock (fd,LOCK_SH |LOCK_NB ); +eal_file_lock (fd,EAL_FLOCK_SHARED ,EAL_FLOCK_RETURN ); ret =-1 ; goto out;
lib/librte_eal/common/eal_private.h +47 -0
51 lines not on this page · raw message
lib/librte_eal/freebsd/Makefile +4 -0
@@ -7,6 +7,7 @@ LIB = librte_eal.a ARCH_DIR ?= $(RTE_ARCH) VPATH += $(RTE_SDK)/lib/librte_eal/$(ARCH_DIR) +VPATH += $(RTE_SDK)/lib/librte_eal/unix VPATH += $(RTE_SDK)/lib/librte_eal/common CFLAGS += -I$(SRCDIR)/include
@@ -74,6 +75,9 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_service.c SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_random.c SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_reciprocal.c +# from unix dir +SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += eal_unix.c + # from arch dir SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_cpuflags.c SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_hypervisor.c
lib/librte_eal/linux/Makefile +4 -0
@@ -7,6 +7,7 @@ LIB = librte_eal.a ARCH_DIR ?= $(RTE_ARCH) VPATH += $(RTE_SDK)/lib/librte_eal/$(ARCH_DIR) +VPATH += $(RTE_SDK)/lib/librte_eal/unix VPATH += $(RTE_SDK)/lib/librte_eal/common CFLAGS += -I$(SRCDIR)/include
@@ -81,6 +82,9 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_service.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_random.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_reciprocal.c +# from unix dir +SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_unix.c + # from arch dir SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_cpuflags.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_hypervisor.c
lib/librte_eal/meson.build +4 -0
@@ -6,6 +6,10 @@ subdir('include') subdir('common') +if not is_windows + subdir('unix') +endif + dpdk_conf.set('RTE_EXEC_ENV_' + exec_env.to_upper(), 1) subdir(exec_env)
lib/librte_eal/unix/eal_unix.c new file +51 -0
51 lines not on this page · raw message
lib/librte_eal/unix/meson.build new file +6 -0
@@ -0,0 +1,6 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2020 Dmitry Kozlyuk + +sources += files( + 'eal_unix.c', +)
Signature · 1 line
2.25.4
Re: [dpdk-dev] [PATCH v5 02/11] eal: introduce internal wrappers for file operations
25/05/2020 02:37, Dmitry Kozlyuk:
Quoted 2 lines
*
eal_file_lock: lock or unlock an open file.
*eal_file_truncate: enforce a given size for an open file.
[...]
Quoted 3 lines
Implementation for Linux and FreeBSD is placed in "unix" subdirectory, which is intended for common code between the two. Files should be named after the ones from which the code is factored in OS subdirectory.
[...]
Quoted 1 line
lib/librte_eal/unix/eal_unix.c | 51 ++++++++++++++++++++++
Why naming this file eal_unix?
If it's truly global, it should be unix/eal.c
If it's only about file operations, it could be unix/eal_file.c
Please update MAINTAINERS when creating a new file. All files or directories must be listed in MAINTAINERS, even if there is no maintainer for them.
Re: [dpdk-dev] [PATCH v5 02/11] eal: introduce internal wrappers for file operations
Quoted 4 lines · On Thu, 28 May 2020 09:59:13 +0200 Thomas Monjalon <[email protected]> wrote:
25/05/2020 02:37, Dmitry Kozlyuk:
Quoted 2 lines
*
eal_file_lock: lock or unlock an open file.
*eal_file_truncate: enforce a given size for an open file.[...]
Quoted 8 lines
Quoted 3 lines
Implementation for Linux and FreeBSD is placed in "unix" subdirectory, which is intended for common code between the two. Files should be named after the ones from which the code is factored in OS subdirectory.
[...]
Quoted 1 line
lib/librte_eal/unix/eal_unix.c | 51 ++++++++++++++++++++++Why naming this file
eal_unix?
If it's truly global, it should beunix/eal.c
We've already discussed this: Makefiles are written in such way that files in common and arch directories must have distinct names for OS-specific ones. Until Makefiles exist, this is consistent naming with the rest of librte_eal.
Quoted 1 line
If it's only about file operations, it could be
unix/eal_file.c
Good suggestion, given that more file-related functions are expected to be extracted here in future (e.g. for tracing).
Quoted 3 lines
Please update
MAINTAINERSwhen creating a new file. All files or directories must be listed inMAINTAINERS, even if there is no maintainer for them.
Will do.
Signature · 1 line
Dmitry Kozlyuk
Re: [dpdk-dev] [PATCH v5 02/11] eal: introduce internal wrappers for file operations
28/05/2020 12:09, Dmitry Kozlyuk:
Quoted 11 lines
Quoted 8 lines · On Thu, 28 May 2020 09:59:13 +0200 Thomas Monjalon <[email protected]> wrote:
25/05/2020 02:37, Dmitry Kozlyuk:
Quoted 2 lines
*
eal_file_lock: lock or unlock an open file.
*eal_file_truncate: enforce a given size for an open file.[...]
Quoted 3 lines
Implementation for Linux and FreeBSD is placed in "unix" subdirectory, which is intended for common code between the two. Files should be named after the ones from which the code is factored in OS subdirectory.
[...]
Quoted 8 lines
Quoted 4 lines
Quoted 1 line
lib/librte_eal/unix/eal_unix.c | 51 ++++++++++++++++++++++Why naming this file
eal_unix?
If it's truly global, it should beunix/eal.cWe've already discussed this: Makefiles are written in such way that files in common and arch directories must have distinct names for OS-specific ones. Until Makefiles exist, this is consistent naming with the rest of
librte_eal.
Oh right.
Quoted 10 lines
Quoted 1 line
If it's only about file operations, it could be
unix/eal_file.cGood suggestion, given that more file-related functions are expected to be extracted here in future (e.g. for tracing).
Quoted 3 lines
Please update
MAINTAINERSwhen creating a new file. All files or directories must be listed inMAINTAINERS, even if there is no maintainer for them.Will do.
Thanks
PATCH v5 3/11 eal: introduce memory management wrappers
In reply to Dmitry KozlyukIntroduce OS-independent wrappers for memory management operations used across DPDK and specifically in common code of EAL:
* rte_mem_map()
* rte_mem_unmap()
* rte_get_page_size()
* rte_mem_lock()
Windows uses different APIs for memory mapping and reservation, while Unices reserve memory by mapping it. Introduce EAL private functions to support memory reservation in common code:
* eal_mem_reserve()
* eal_mem_free()
* eal_mem_set_dump()
Wrappers follow POSIX semantics limited to DPDK tasks, but their signatures deliberately differ from POSIX ones to be more safe and expressive.
--- lib/librte_eal/common/eal_common_fbarray.c | 37 +++-- lib/librte_eal/common/eal_common_memory.c | 60 +++----- lib/librte_eal/common/eal_private.h | 78 ++++++++++- lib/librte_eal/freebsd/Makefile | 1 + lib/librte_eal/include/rte_memory.h | 88 ++++++++++++ lib/librte_eal/linux/Makefile | 1 + lib/librte_eal/linux/eal_memalloc.c | 5 +- lib/librte_eal/rte_eal_version.map | 6 + lib/librte_eal/unix/eal_unix_memory.c | 152 +++++++++++++++++++++ lib/librte_eal/unix/meson.build | 1 + 10 files changed, 365 insertions(+), 64 deletions(-) create mode 100644 lib/librte_eal/unix/eal_unix_memory.c
lib/librte_eal/common/eal_common_fbarray.c +17 -20
@@ -5,15 +5,15 @@#include <fcntl.h> #include <inttypes.h> #include <limits.h> -#include <sys/mman.h> #include <stdint.h> #include <errno.h> #include <string.h> #include <unistd.h> #include <rte_common.h> -#include <rte_log.h> #include <rte_errno.h> +#include <rte_log.h> +#include <rte_memory.h> #include <rte_spinlock.h> #include <rte_tailq.h>
@@ -90,12 +90,9 @@ resize_and_map(int fd, void *addr, size_t len)return -1 ; } - map_addr =mmap (addr, len,PROT_READ |PROT_WRITE , -MAP_SHARED |MAP_FIXED , fd,0 ); + map_addr =rte_mem_map (addr, len,RTE_PROT_READ |RTE_PROT_WRITE , +RTE_MAP_SHARED |RTE_MAP_FORCE_ADDRESS , fd,0 );if (map_addr != addr) { -RTE_LOG (ERR ,EAL ,"mmap() failed: %s\n" ,strerror (errno)); -/* pass errno up the chain */ - rte_errno = errno;return -1 ; }return 0 ;
@@ -733,7 +730,7 @@ rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,return -1 ; } - page_sz =sysconf (_SC_PAGESIZE); + page_sz =rte_get_page_size ();if (page_sz == (size_t )-1 ) {free (ma);return -1 ;
@@ -754,9 +751,11 @@ rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,if (internal_config.no_shconf) {/* remap virtual area as writable */ -void *new_data =mmap (data, mmap_len,PROT_READ |PROT_WRITE , -MAP_FIXED |MAP_PRIVATE |MAP_ANONYMOUS , fd,0 ); -if (new_data ==MAP_FAILED ) { +static const int flags =RTE_MAP_FORCE_ADDRESS | +RTE_MAP_PRIVATE |RTE_MAP_ANONYMOUS ; +void *new_data =rte_mem_map (data, mmap_len, +RTE_PROT_READ |RTE_PROT_WRITE , flags, fd,0 ); +if (new_data ==NULL ) {RTE_LOG (DEBUG ,EAL ,"%s(): couldn't remap anonymous memory: %s\n" , __func__,strerror (errno)); goto fail;
@@ -821,7 +820,7 @@ rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,return 0 ; fail:if (data) -munmap (data, mmap_len); +rte_mem_unmap (data, mmap_len);if (fd >=0 )close (fd);free (ma);
@@ -859,7 +858,7 @@ rte_fbarray_attach(struct rte_fbarray *arr)return -1 ; } - page_sz =sysconf (_SC_PAGESIZE); + page_sz =rte_get_page_size ();if (page_sz == (size_t )-1 ) {free (ma);return -1 ;
@@ -911,7 +910,7 @@ rte_fbarray_attach(struct rte_fbarray *arr)return 0 ; fail:if (data) -munmap (data, mmap_len); +rte_mem_unmap (data, mmap_len);if (fd >=0 )close (fd);free (ma);
@@ -939,8 +938,7 @@ rte_fbarray_detach(struct rte_fbarray *arr)* really do anything about it, things will blow up either way. */ -size_t page_sz =sysconf (_SC_PAGESIZE); - +size_t page_sz =rte_get_page_size ();if (page_sz == (size_t )-1 )return -1 ;
@@ -959,7 +957,7 @@ rte_fbarray_detach(struct rte_fbarray *arr) goto out; } -munmap (arr->data, mmap_len); +rte_mem_unmap (arr->data, mmap_len);/* area is unmapped, close fd and remove the tailq entry */ if (tmp->fd >=0 )
@@ -994,8 +992,7 @@ rte_fbarray_destroy(struct rte_fbarray *arr)* really do anything about it, things will blow up either way. */ -size_t page_sz =sysconf (_SC_PAGESIZE); - +size_t page_sz =rte_get_page_size ();if (page_sz == (size_t )-1 )return -1 ;
@@ -1044,7 +1041,7 @@ rte_fbarray_destroy(struct rte_fbarray *arr) }close (fd); } -munmap (arr->data, mmap_len); +rte_mem_unmap (arr->data, mmap_len);/* area is unmapped, remove the tailq entry */ TAILQ_REMOVE (&mem_area_tailq, tmp, next);
lib/librte_eal/common/eal_common_memory.c +22 -38
136 lines not on this page · raw message
lib/librte_eal/common/eal_private.h +75 -3
106 lines not on this page · raw message
lib/librte_eal/freebsd/Makefile +1 -0
@@ -77,6 +77,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_reciprocal.c # from unix dir SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += eal_unix.c +SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += eal_unix_memory.c # from arch dir SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_cpuflags.c
lib/librte_eal/include/rte_memory.h +88 -0
94 lines not on this page · raw message
lib/librte_eal/linux/Makefile +1 -0
@@ -84,6 +84,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_reciprocal.c # from unix dir SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_unix.c +SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_unix_memory.c # from arch dir SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_cpuflags.c
lib/librte_eal/linux/eal_memalloc.c +2 -3
@@ -630,7 +630,7 @@ alloc_seg(struct rte_memseg *ms, void *addr, int socket_id, mapped:munmap (addr, alloc_sz); unmapped: - flags =MAP_FIXED ; + flags =EAL_RESERVE_FORCE_ADDRESS ; new_addr =eal_get_virtual_area (addr, &alloc_sz, alloc_sz,0 , flags);if (new_addr != addr) {if (new_addr !=NULL )
@@ -687,8 +687,7 @@ free_seg(struct rte_memseg *ms, struct hugepage_info *hi,return -1 ; } -if (madvise (ms->addr, ms->len,MADV_DONTDUMP ) !=0 ) -RTE_LOG (DEBUG ,EAL ,"madvise failed: %s\n" ,strerror (errno)); +eal_mem_set_dump (ms->addr, ms->len, false); exit_early = false;
lib/librte_eal/rte_eal_version.map +6 -0
@@ -386,4 +386,10 @@ EXPERIMENTAL { rte_trace_point_lookup; rte_trace_regexp; rte_trace_save; + + # added in 20.08 + rte_get_page_size; + rte_mem_lock; + rte_mem_map; + rte_mem_unmap; };
lib/librte_eal/unix/eal_unix_memory.c new file +152 -0
152 lines not on this page · raw message
lib/librte_eal/unix/meson.build +1 -0
@@ -3,4 +3,5 @@ sources += files( 'eal_unix.c', + 'eal_unix_memory.c', )
Signature · 1 line
2.25.4
Re: [dpdk-dev] [PATCH v5 03/11] eal: introduce memory management wrappers
Are wrappers 100% are required.
Would it be simpler (and less invasive) to have a windows_compat.h that plugged this holes? I am not sure on the standard approach here - so I will leave this to others.
Outside of that - do these symbols really require experimental status. Are they really likely to change?
Ray K
Quoted 754 lines · Dmitry Kozlyuk
Introduce OS-independent wrappers for memory management operations used across DPDK and specifically in common code of EAL:
*
rte_mem_map()
*rte_mem_unmap()
*rte_get_page_size()
*rte_mem_lock()Windows uses different APIs for memory mapping and reservation, while Unices reserve memory by mapping it. Introduce EAL private functions to support memory reservation in common code:
*
eal_mem_reserve()
*eal_mem_free()
*eal_mem_set_dump()Wrappers follow POSIX semantics limited to DPDK tasks, but their signatures deliberately differ from POSIX ones to be more safe and expressive.
Signed-off-by Dmitry Kozlyuk <[email protected]>--- lib/librte_eal/common/eal_common_fbarray.c | 37 +++-- lib/librte_eal/common/eal_common_memory.c | 60 +++----- lib/librte_eal/common/eal_private.h | 78 ++++++++++- lib/librte_eal/freebsd/Makefile | 1 + lib/librte_eal/include/rte_memory.h | 88 ++++++++++++ lib/librte_eal/linux/Makefile | 1 + lib/librte_eal/linux/eal_memalloc.c | 5 +- lib/librte_eal/rte_eal_version.map | 6 + lib/librte_eal/unix/eal_unix_memory.c | 152 +++++++++++++++++++++ lib/librte_eal/unix/meson.build | 1 + 10 files changed, 365 insertions(+), 64 deletions(-) create mode 100644 lib/librte_eal/unix/eal_unix_memory.clib/librte_eal/common/eal_common_fbarray.c +17
-20@@ -5,15 +5,15 @@#include <fcntl.h> #include <inttypes.h> #include <limits.h> -#include <sys/mman.h> #include <stdint.h> #include <errno.h> #include <string.h> #include <unistd.h> #include <rte_common.h> -#include <rte_log.h> #include <rte_errno.h> +#include <rte_log.h> +#include <rte_memory.h> #include <rte_spinlock.h> #include <rte_tailq.h> @@ -90,12 +90,9 @@ resize_and_map(int fd, void *addr, size_t len)return -1 ; } - map_addr =mmap (addr, len,PROT_READ |PROT_WRITE , -MAP_SHARED |MAP_FIXED , fd,0 ); + map_addr =rte_mem_map (addr, len,RTE_PROT_READ |RTE_PROT_WRITE , +RTE_MAP_SHARED |RTE_MAP_FORCE_ADDRESS , fd,0 );if (map_addr != addr) { -RTE_LOG (ERR ,EAL ,"mmap() failed: %s\n" ,strerror (errno)); -/* pass errno up the chain */ - rte_errno = errno;return -1 ; }return 0 ;@@ -733,7 +730,7 @@ rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,return -1 ; } - page_sz =sysconf (_SC_PAGESIZE); + page_sz =rte_get_page_size ();if (page_sz == (size_t )-1 ) {free (ma);return -1 ;@@ -754,9 +751,11 @@ rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,if (internal_config.no_shconf) {/* remap virtual area as writable */ -void *new_data =mmap (data, mmap_len,PROT_READ |PROT_WRITE , -MAP_FIXED |MAP_PRIVATE |MAP_ANONYMOUS , fd,0 ); -if (new_data ==MAP_FAILED ) { +static const int flags =RTE_MAP_FORCE_ADDRESS | +RTE_MAP_PRIVATE |RTE_MAP_ANONYMOUS ; +void *new_data =rte_mem_map (data, mmap_len, +RTE_PROT_READ |RTE_PROT_WRITE , flags, fd,0 ); +if (new_data ==NULL ) {RTE_LOG (DEBUG ,EAL ,"%s(): couldn't remap anonymous memory: %s\n" , __func__,strerror (errno)); goto fail;@@ -821,7 +820,7 @@ rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,return 0 ; fail:if (data) -munmap (data, mmap_len); +rte_mem_unmap (data, mmap_len);if (fd >=0 )close (fd);free (ma);@@ -859,7 +858,7 @@ rte_fbarray_attach(struct rte_fbarray *arr)return -1 ; } - page_sz =sysconf (_SC_PAGESIZE); + page_sz =rte_get_page_size ();if (page_sz == (size_t )-1 ) {free (ma);return -1 ;@@ -911,7 +910,7 @@ rte_fbarray_attach(struct rte_fbarray *arr)return 0 ; fail:if (data) -munmap (data, mmap_len); +rte_mem_unmap (data, mmap_len);if (fd >=0 )close (fd);free (ma);@@ -939,8 +938,7 @@ rte_fbarray_detach(struct rte_fbarray *arr)* really do anything about it, things will blow up either way. */ -size_t page_sz =sysconf (_SC_PAGESIZE); - +size_t page_sz =rte_get_page_size ();if (page_sz == (size_t )-1 )return -1 ;@@ -959,7 +957,7 @@ rte_fbarray_detach(struct rte_fbarray *arr) goto out; } -munmap (arr->data, mmap_len); +rte_mem_unmap (arr->data, mmap_len);/* area is unmapped, close fd and remove the tailq entry */ if (tmp->fd >=0 )@@ -994,8 +992,7 @@ rte_fbarray_destroy(struct rte_fbarray *arr)* really do anything about it, things will blow up either way. */ -size_t page_sz =sysconf (_SC_PAGESIZE); - +size_t page_sz =rte_get_page_size ();if (page_sz == (size_t )-1 )return -1 ;@@ -1044,7 +1041,7 @@ rte_fbarray_destroy(struct rte_fbarray *arr) }close (fd); } -munmap (arr->data, mmap_len); +rte_mem_unmap (arr->data, mmap_len);/* area is unmapped, remove the tailq entry */ TAILQ_REMOVE (&mem_area_tailq, tmp, next);lib/librte_eal/common/eal_common_memory.c +22
-38@@ -11,7 +11,6 @@#include <string.h> #include <unistd.h> #include <inttypes.h> -#include <sys/mman.h> #include <sys/queue.h> #include <rte_fbarray.h> @@ -40,18 +39,10 @@static void *next_baseaddr;static uint64_t system_page_sz; -#ifdef RTE_EXEC_ENV_LINUX -#define RTE_DONTDUMP MADV_DONTDUMP -#elif definedRTE_EXEC_ENV_FREEBSD -#define RTE_DONTDUMP MADV_NOCORE -#else -#error "madvise doesn't support this OS" -#endif -#define MAX_MMAP_WITH_DEFINED_ADDR_TRIES 5void *eal_get_virtual_area (void *requested_addr,size_t *size, -size_t page_sz,int flags,int mmap_flags) +size_t page_sz,int flags,int reserve_flags) {bool addr_is_hint, allow_shrink, unmap, no_align;uint64_t map_sz;@@ -59,9 +50,7 @@ eal_get_virtual_area(void *requested_addr, size_t *size,uint8_t try =0 ;if (system_page_sz ==0 ) - system_page_sz =sysconf (_SC_PAGESIZE); - - mmap_flags |=MAP_PRIVATE |MAP_ANONYMOUS ; + system_page_sz =rte_get_page_size ();RTE_LOG (DEBUG ,EAL ,"Ask a virtual area of 0x%zx bytes\n" , *size);@@ -105,24 +94,24 @@ eal_get_virtual_area(void *requested_addr, size_t *size,return NULL ; } - mapped_addr =mmap (requested_addr, (size_t )map_sz,PROT_NONE , - mmap_flags,-1 ,0 ); -if (mapped_addr ==MAP_FAILED && allow_shrink) + mapped_addr =eal_mem_reserve ( + requested_addr, (size_t )map_sz, reserve_flags); +if ((mapped_addr ==NULL ) && allow_shrink) *size -= page_sz; -if (mapped_addr !=MAP_FAILED && addr_is_hint && - mapped_addr != requested_addr) { +if ((mapped_addr !=NULL ) && addr_is_hint && + (mapped_addr != requested_addr)) { try++; next_baseaddr =RTE_PTR_ADD (next_baseaddr, page_sz);if (try <=MAX_MMAP_WITH_DEFINED_ADDR_TRIES ) {/* hint was not used. Try with another offset */ -munmap (mapped_addr, map_sz); - mapped_addr =MAP_FAILED ; +eal_mem_free (mapped_addr, map_sz); + mapped_addr =NULL ; requested_addr = next_baseaddr; } } }while ((allow_shrink || addr_is_hint) && - mapped_addr ==MAP_FAILED && *size >0 ); + (mapped_addr ==NULL ) && (*size >0 ));/* align resulting address - if map failed, we will ignore the value * anyway, so no need to add additional checks. @@ -132,20 +121,17 @@ eal_get_virtual_area(void *requested_addr, size_t *size,if (*size ==0 ) {RTE_LOG (ERR ,EAL ,"Cannot get a virtual area of any size: %s\n" , -strerror (errno)); - rte_errno = errno; +strerror (rte_errno));return NULL ; - }else if (mapped_addr ==MAP_FAILED ) { + }else if (mapped_addr ==NULL ) {RTE_LOG (ERR ,EAL ,"Cannot get a virtual area: %s\n" , -strerror (errno)); -/* pass errno up the call chain */ - rte_errno = errno; +strerror (rte_errno));return NULL ; }else if (requested_addr !=NULL && !addr_is_hint && aligned_addr != requested_addr) {RTE_LOG (ERR ,EAL ,"Cannot get a virtual area at requested address: %p (got %p)\n" , requested_addr, aligned_addr); -munmap (mapped_addr, map_sz); +eal_mem_free (mapped_addr, map_sz); rte_errno =EADDRNOTAVAIL ;return NULL ; } else if (requested_addr != NULL && addr_is_hint &&@@ -161,7 +147,7 @@ eal_get_virtual_area(void *requested_addr, size_t *size, aligned_addr, *size);if (unmap) { -munmap (mapped_addr, map_sz); +eal_mem_free (mapped_addr, map_sz); }else if (!no_align) {void *map_end, *aligned_end;size_t before_len, after_len;@@ -179,19 +165,17 @@ eal_get_virtual_area(void *requested_addr, size_t *size,/* unmap space before aligned mmap address */ before_len =RTE_PTR_DIFF (aligned_addr, mapped_addr);if (before_len >0 ) -munmap (mapped_addr, before_len); +eal_mem_free (mapped_addr, before_len);/* unmap space after aligned end mmap address */ after_len =RTE_PTR_DIFF (map_end, aligned_end);if (after_len >0 ) -munmap (aligned_end, after_len); +eal_mem_free (aligned_end, after_len); }if (!unmap) {/* Exclude these pages from a core dump. */ -if (madvise (aligned_addr, *size,RTE_DONTDUMP ) !=0 ) -RTE_LOG (DEBUG ,EAL ,"madvise failed: %s\n" , -strerror (errno)); +eal_mem_set_dump (aligned_addr, *size, false); }return aligned_addr;@@ -547,10 +531,10 @@ rte_eal_memdevice_init(void)int rte_mem_lock_page (const void *virt) { -unsigned long virtual = (unsigned long )virt; -int page_size =getpagesize (); -unsigned long aligned = (virtual & ~(page_size -1 )); -return mlock ((void *)aligned, page_size); +uintptr_t virtual = (uintptr_t )virt; +size_t page_size =rte_get_page_size (); +uintptr_t aligned =RTE_PTR_ALIGN_FLOOR (virtual, page_size); +return rte_mem_lock ((void *)aligned, page_size); }int lib/librte_eal/common/eal_private.h +75
-3@@ -11,6 +11,7 @@#include <rte_dev.h> #include <rte_lcore.h> +#include <rte_memory.h> /** * Structure storing internal configuration (per-lcore) @@ -202,6 +203,24 @@ int rte_eal_alarm_init(void);*/ int rte_eal_check_module (const char *module_name); +/** +* Memory reservation flags. +*/ +enum eal_mem_reserve_flags { +/** +* Reserve hugepages. May be unsupported by some platforms. +*/ +EAL_RESERVE_HUGEPAGES =1 <<0 , +/** +* Force reserving memory at the requested address. +* This can be a destructive action depending on the implementation. +* +* @see RTE_MAP_FORCE_ADDRESS for description of possible consequences +* (although implementations are not required to use it). +*/ +EAL_RESERVE_FORCE_ADDRESS =1 <<1 +}; +/** * Get virtual area of specified size from the OS. * @@ -215,8 +234,8 @@ int rte_eal_check_module(const char *module_name);* Page size on which to align requested virtual area. * @param flags * EAL_VIRTUAL_AREA_* flags. -* @param mmap_flags -* Extra flags passed directly to mmap(). +* @param reserve_flags +* Extra flags passed directly to rte_mem_reserve(). * * @return * Virtual area address if successful. @@ -233,7 +252,7 @@ int rte_eal_check_module(const char *module_name);/**< immediately unmap reserved virtual area. */ void *eal_get_virtual_area (void *requested_addr,size_t *size, -size_t page_sz,int flags,int mmap_flags); +size_t page_sz,int flags,int reserve_flags);/** * Get cpu core_id. @@ -467,4 +486,57 @@ eal_file_lock(int fd, enum eal_flock_op op, enum eal_flock_mode mode);int eal_file_truncate (int fd,ssize_t size); +/** +* Reserve a region of virtual memory. +* +* Use eal_mem_free() to free reserved memory. +* +* @param requested_addr +* A desired reservation addressm which must be page-aligned. +* The system might not respect it. +* NULL means the address will be chosen by the system. +* @param size +* Reservation size. Must be a multiple of system page size. +* @param flags +* Reservation options, a combination of eal_mem_reserve_flags. +* @returns +* Starting address of the reserved area on success, NULL on failure. +* Callers must not access this memory until remapping it. +*/ +void * +eal_mem_reserve (void *requested_addr,size_t size,int flags); + +/** +* Free memory obtained by eal_mem_reserve() or eal_mem_alloc(). +* +* If *virt* and *size* describe a part of the reserved region, +* only this part of the region is freed (accurately up to the system +* page size). If *virt* points to allocated memory, *size* must match +* the one specified on allocation. The behavior is undefined +* if the memory pointed by *virt* is obtained from another source +* than listed above. +* +* @param virt +* A virtual address in a region previously reserved. +* @param size +* Number of bytes to unreserve. +*/ +void +eal_mem_free (void *virt,size_t size); + +/** +* Configure memory region inclusion into core dumps. +* +* @param virt +* Starting address of the region. +* @param size +* Size of the region. +* @param dump +* True to include memory into core dumps, false to exclude. +* @return +* 0 on success, (-1) on failure and rte_errno is set. +*/ +int +eal_mem_set_dump (void *virt,size_t size,bool dump); +#endif /* _EAL_PRIVATE_H_ */ lib/librte_eal/freebsd/Makefile +1
-0@@ -77,6 +77,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_reciprocal.c # from unix dir SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += eal_unix.c +SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += eal_unix_memory.c # from arch dir SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_cpuflags.clib/librte_eal/include/rte_memory.h +88
-0@@ -82,6 +82,94 @@ struct rte_memseg_list {struct rte_fbarray memseg_arr; }; +/** +* Memory protection flags. +*/ +enum rte_mem_prot { +RTE_PROT_READ =1 <<0 ,/**< Read access. */ +RTE_PROT_WRITE =1 <<1 ,/**< Write access. */ +RTE_PROT_EXECUTE =1 <<2 /**< Code execution. */ +}; + +/** +* Additional flags for memory mapping. +*/ +enum rte_map_flags { +/** Changes to the mapped memory are visible to other processes. */ +RTE_MAP_SHARED =1 <<0 , +/** Mapping is not backed by a regular file. */ +RTE_MAP_ANONYMOUS =1 <<1 , +/** Copy-on-write mapping, changes are invisible to other processes. */ +RTE_MAP_PRIVATE =1 <<2 , +/** +* Force mapping to the requested address. This flag should be used +* with caution, because to fulfill the request implementation +* may remove all other mappings in the requested region. However, +* it is not required to do so, thus mapping with this flag may fail. +*/ +RTE_MAP_FORCE_ADDRESS =1 <<3 +}; + +/** +* Map a portion of an opened file or the page file into memory. +* +* This function is similar to POSIX mmap(3) with common MAP_ANONYMOUS +* extension, except for the return value. +* +* @param requested_addr +* Desired virtual address for mapping. Can be NULL to let OS choose. +* @param size +* Size of the mapping in bytes. +* @param prot +* Protection flags, a combination of rte_mem_prot values. +* @param flags +* Addtional mapping flags, a combination of rte_map_flags. +* @param fd +* Mapped file descriptor. Can be negative for anonymous mapping. +* @param offset +* Offset of the mapped region in fd. Must be 0 for anonymous mappings. +* @return +* Mapped address or NULL on failure and rte_errno is set to OS error. +*/ +__rte_experimental +void * +rte_mem_map (void *requested_addr,size_t size,int prot,int flags, +int fd,size_t offset); + +/** +* OS-independent implementation of POSIX munmap(3). +*/ +__rte_experimental +int +rte_mem_unmap (void *virt,size_t size); + +/** +* Get system page size. This function never fails. +* +* @return +* Page size in bytes. +*/ +__rte_experimental +size_t +rte_get_page_size (void ); + +/** +* Lock in physical memory all pages crossed by the address region. +* +* @param virt +* Base virtual address of the region. +* @param size +* Size of the region. +* @return +* 0 on success, negative on error. +* +* @see rte_get_page_size() to retrieve the page size. +* @see rte_mem_lock_page() to lock an entire single page. +*/ +__rte_experimental +int +rte_mem_lock (const void *virt,size_t size); +/** * Lock page in physical memory and prevent from swapping. * lib/librte_eal/linux/Makefile +1
-0@@ -84,6 +84,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_reciprocal.c # from unix dir SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_unix.c +SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_unix_memory.c # from arch dir SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_cpuflags.clib/librte_eal/linux/eal_memalloc.c +2
-3@@ -630,7 +630,7 @@ alloc_seg(struct rte_memseg *ms, void *addr, int socket_id, mapped:munmap (addr, alloc_sz); unmapped: - flags =MAP_FIXED ; + flags =EAL_RESERVE_FORCE_ADDRESS ; new_addr =eal_get_virtual_area (addr, &alloc_sz, alloc_sz,0 , flags);if (new_addr != addr) {if (new_addr !=NULL )@@ -687,8 +687,7 @@ free_seg(struct rte_memseg *ms, struct hugepage_info *hi,return -1 ; } -if (madvise (ms->addr, ms->len,MADV_DONTDUMP ) !=0 ) -RTE_LOG (DEBUG ,EAL ,"madvise failed: %s\n" ,strerror (errno)); +eal_mem_set_dump (ms->addr, ms->len, false); exit_early = false;lib/librte_eal/rte_eal_version.map +6
-0@@ -386,4 +386,10 @@ EXPERIMENTAL { rte_trace_point_lookup; rte_trace_regexp; rte_trace_save; + + # added in 20.08 + rte_get_page_size; + rte_mem_lock; + rte_mem_map; + rte_mem_unmap; };lib/librte_eal/unix/eal_unix_memory.c new file +152
-0@@ -0,0 +1,152 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Dmitry Kozlyuk +*/ + +#include <string.h> +#include <sys/mman.h> +#include <unistd.h> + +#include <rte_errno.h> +#include <rte_log.h> +#include <rte_memory.h> + +#include "eal_private.h" + +#ifdef RTE_EXEC_ENV_LINUX +#define EAL_DONTDUMP MADV_DONTDUMP +#define EAL_DODUMP MADV_DODUMP +#elif definedRTE_EXEC_ENV_FREEBSD +#define EAL_DONTDUMP MADV_NOCORE +#define EAL_DODUMP MADV_CORE +#else +#error "madvise doesn't support this OS" +#endif + +static void * +mem_map (void *requested_addr,size_t size,int prot,int flags, +int fd,size_t offset) +{ +void *virt =mmap (requested_addr, size, prot, flags, fd, offset); +if (virt ==MAP_FAILED ) { +RTE_LOG (DEBUG ,EAL , +"Cannot mmap(%p, 0x%zx, 0x%x, 0x%x, %d, 0x%zx): %s\n" , + requested_addr, size, prot, flags, fd, offset, +strerror (errno)); + rte_errno = errno; +return NULL ; + } +return virt; +} + +static int +mem_unmap (void *virt,size_t size) +{ +int ret =munmap (virt, size); +if (ret <0 ) { +RTE_LOG (DEBUG ,EAL ,"Cannot munmap(%p, 0x%zx): %s\n" , + virt, size,strerror (errno)); + rte_errno = errno; + } +return ret; +} + +void * +eal_mem_reserve (void *requested_addr,size_t size,int flags) +{ +int sys_flags =MAP_PRIVATE |MAP_ANONYMOUS ; + +if (flags &EAL_RESERVE_HUGEPAGES ) { +#ifdef MAP_HUGETLB + sys_flags |=MAP_HUGETLB ; +#else + rte_errno =ENOTSUP ; +return NULL ; +#endif + } + +if (flags &EAL_RESERVE_FORCE_ADDRESS ) + sys_flags |=MAP_FIXED ; + +return mem_map (requested_addr, size,PROT_NONE , sys_flags,-1 ,0 ); +} + +void +eal_mem_free (void *virt,size_t size) +{ +mem_unmap (virt, size); +} + +int +eal_mem_set_dump (void *virt,size_t size,bool dump) +{ +int flags = dump ?EAL_DODUMP :EAL_DONTDUMP ; +int ret =madvise (virt, size, flags); +if (ret) { +RTE_LOG (DEBUG ,EAL ,"madvise(%p, %#zx, %d) failed: %s\n" , + virt, size, flags,strerror (rte_errno)); + rte_errno = errno; + } +return ret; +} + +static int +mem_rte_to_sys_prot (int prot) +{ +int sys_prot =PROT_NONE ; + +if (prot &RTE_PROT_READ ) + sys_prot |=PROT_READ ; +if (prot &RTE_PROT_WRITE ) + sys_prot |=PROT_WRITE ; +if (prot &RTE_PROT_EXECUTE ) + sys_prot |=PROT_EXEC ; + +return sys_prot; +} + +void * +rte_mem_map (void *requested_addr,size_t size,int prot,int flags, +int fd,size_t offset) +{ +int sys_flags =0 ; +int sys_prot; + + sys_prot =mem_rte_to_sys_prot (prot); + +if (flags &RTE_MAP_SHARED ) + sys_flags |=MAP_SHARED ; +if (flags &RTE_MAP_ANONYMOUS ) + sys_flags |=MAP_ANONYMOUS ; +if (flags &RTE_MAP_PRIVATE ) + sys_flags |=MAP_PRIVATE ; +if (flags &RTE_MAP_FORCE_ADDRESS ) + sys_flags |=MAP_FIXED ; + +return mem_map (requested_addr, size, sys_prot, sys_flags, fd, offset); +} + +int +rte_mem_unmap (void *virt,size_t size) +{ +return mem_unmap (virt, size); +} + +size_t +rte_get_page_size (void ) +{ +static size_t page_size; + +if (!page_size) + page_size =sysconf (_SC_PAGESIZE); + +return page_size; +} + +int +rte_mem_lock (const void *virt,size_t size) +{ +int ret =mlock (virt, size); +if (ret) + rte_errno = errno; +return ret; +}lib/librte_eal/unix/meson.build +1
-0@@ -3,4 +3,5 @@ sources += files( 'eal_unix.c', + 'eal_unix_memory.c', )
Re: [dpdk-dev] [PATCH v5 03/11] eal: introduce memory management wrappers
Answers below is the summary of discussion with Thomas, Ranjit, Tal, et al.
Quoted 3 lines · On Wed, 27 May 2020 07:33:32 +0100 Ray Kinsella <[email protected]> wrote:
Are wrappers 100% are required.
Would it be simpler (and less invasive) to have awindows_compat.hthat plugged this holes? I am not sure on the standard approach here - so I will leave this to others.
With wrappers, we control API and semantics, which is limited compared to the underlying syscalls. It is also cleaner not to export non-RTE symbols from DPDK libraries. Regarding invasion: it requires little change, factoring out some common error logging in the process.
Quoted 2 lines
Outside of that - do these symbols really require experimental status. Are they really likely to change?
Indeed, the wrappers should be internal, not experimental. Will fix in v6.
Signature · 1 line
Dmitry Kozlyuk
Re: [dpdk-dev] [PATCH v5 03/11] eal: introduce memory management wrappers
In reply to Dmitry KozlyukQuoted 22 lines · Dmitry Kozlyuk
Introduce OS-independent wrappers for memory management operations used across DPDK and specifically in common code of EAL:
*
rte_mem_map()
*rte_mem_unmap()
*rte_get_page_size()
*rte_mem_lock()Windows uses different APIs for memory mapping and reservation, while Unices reserve memory by mapping it. Introduce EAL private functions to support memory reservation in common code:
*
eal_mem_reserve()
*eal_mem_free()
*eal_mem_set_dump()Wrappers follow POSIX semantics limited to DPDK tasks, but their signatures deliberately differ from POSIX ones to be more safe and expressive.
Signed-off-by Dmitry Kozlyuk <[email protected]>---
<snip>
Quoted 9 lines
lib/librte_eal/common/eal_common_fbarray.c +1
-1- page_sz =sysconf (_SC_PAGESIZE); + page_sz =rte_get_page_size ();if (page_sz == (size_t )-1 ) {free (ma);return -1 ;@@ -754,9 +751,11 @@ rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,if (internal_config.no_shconf) {/* remap virtual area as writable */
lib/librte_eal/common/eal_common_fbarray.c rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len, · line 760
Quoted 10 lines
lib/librte_eal/common/eal_common_fbarray.c +5
-3-void *new_data =mmap (data, mmap_len,PROT_READ |PROT_WRITE , -MAP_FIXED |MAP_PRIVATE |MAP_ANONYMOUS , fd,0 ); -if (new_data ==MAP_FAILED ) { +static const int flags =RTE_MAP_FORCE_ADDRESS | +RTE_MAP_PRIVATE |RTE_MAP_ANONYMOUS ; +void *new_data =rte_mem_map (data, mmap_len, +RTE_PROT_READ |RTE_PROT_WRITE , flags, fd,0 ); +if (new_data ==NULL ) {RTE_LOG (DEBUG ,EAL ,"%s(): couldn't remap anonymous memory: %s\n" , __func__,strerror (errno));
I believe this should be rte_strerror(rte_errno) instead of strerror(errno).
Quoted 12 lines
lib/librte_eal/common/eal_common_fbarray.c +1
-1goto fail;@@ -821,7 +820,7 @@ rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,return 0 ; fail:if (data) -munmap (data, mmap_len); +rte_mem_unmap (data, mmap_len);if (fd >=0 )close (fd);free (ma);@@ -859,7 +858,7 @@ rte_fbarray_attach(struct rte_fbarray *arr)return -1 ;
<snip>
Quoted 18 lines
lib/librte_eal/include/rte_memory.h +17
-0+/** +* Memory protection flags. +*/ +enum rte_mem_prot { +RTE_PROT_READ =1 <<0 ,/**< Read access. */ +RTE_PROT_WRITE =1 <<1 ,/**< Write access. */ +RTE_PROT_EXECUTE =1 <<2 /**< Code execution. */ +}; + +/** +* Additional flags for memory mapping. +*/ +enum rte_map_flags { +/** Changes to the mapped memory are visible to other processes. */ +RTE_MAP_SHARED =1 <<0 , +/** Mapping is not backed by a regular file. */ +RTE_MAP_ANONYMOUS =1 <<1 ,
lib/librte_eal/include/rte_memory.h struct rte_memseg_list { · line 111
Quoted 10 lines
lib/librte_eal/include/rte_memory.h +10
-0+/** Copy-on-write mapping, changes are invisible to other processes. */ +RTE_MAP_PRIVATE =1 <<2 , +/** +* Force mapping to the requested address. This flag should be used +* with caution, because to fulfill the request implementation +* may remove all other mappings in the requested region. However, +* it is not required to do so, thus mapping with this flag may fail. +*/ + RTE_MAP_FORCE_ADDRESS = 1 << 3 +};
I have no strong opinion on this, but it feels like the fact that these are enums is a relic from the times where you used enum everywhere :) i have a feeling that DPDK codebase prefers #define's for this usage, while what you have here is more of a C++ thing.
Signature · 2 lines
Thanks, Anatoly
Re: [dpdk-dev] [PATCH v5 03/11] eal: introduce memory management wrappers
28/05/2020 13:26, Burakov, Anatoly:
Quoted 25 lines · Dmitry Kozlyuk
Quoted 24 lines
lib/librte_eal/include/rte_memory.h +24
-0+/** +* Memory protection flags. +*/ +enum rte_mem_prot { +RTE_PROT_READ =1 <<0 ,/**< Read access. */ +RTE_PROT_WRITE =1 <<1 ,/**< Write access. */ +RTE_PROT_EXECUTE =1 <<2 /**< Code execution. */ +}; + +/** +* Additional flags for memory mapping. +*/ +enum rte_map_flags { +/** Changes to the mapped memory are visible to other processes. */ +RTE_MAP_SHARED =1 <<0 , +/** Mapping is not backed by a regular file. */ +RTE_MAP_ANONYMOUS =1 <<1 , +/** Copy-on-write mapping, changes are invisible to other processes. */ +RTE_MAP_PRIVATE =1 <<2 , +/** +* Force mapping to the requested address. This flag should be used +* with caution, because to fulfill the request implementation +* may remove all other mappings in the requested region. However, +* it is not required to do so, thus mapping with this flag may fail.
Quoted 8 lines
lib/librte_eal/include/rte_memory.h struct rte_memseg_list { · line 111
Quoted 3 lines
lib/librte_eal/include/rte_memory.h +3
-0+*/ + RTE_MAP_FORCE_ADDRESS = 1 << 3 +};I have no strong opinion on this, but it feels like the fact that these are enums is a relic from the times where you used enum everywhere :) i have a feeling that DPDK codebase prefers #define's for this usage, while what you have here is more of a C++ thing.
The benefit of using an enum is to explicitly name the type of the variables, serving documentation purpose.
+1 for the enums
Re: [dpdk-dev] [PATCH v5 03/11] eal: introduce memory management wrappers
In reply to Dmitry KozlyukQuoted 22 lines · Dmitry Kozlyuk
Introduce OS-independent wrappers for memory management operations used across DPDK and specifically in common code of EAL:
*
rte_mem_map()
*rte_mem_unmap()
*rte_get_page_size()
*rte_mem_lock()Windows uses different APIs for memory mapping and reservation, while Unices reserve memory by mapping it. Introduce EAL private functions to support memory reservation in common code:
*
eal_mem_reserve()
*eal_mem_free()
*eal_mem_set_dump()Wrappers follow POSIX semantics limited to DPDK tasks, but their signatures deliberately differ from POSIX ones to be more safe and expressive.
Signed-off-by Dmitry Kozlyuk <[email protected]>---
<snip>
lib/librte_eal/common/eal_common_memory.c eal_get_virtual_area(void *requested_addr, size_t *size, · line 128
Quoted 6 lines
lib/librte_eal/common/eal_common_memory.c +2
-3+ }else if (mapped_addr ==NULL ) {RTE_LOG (ERR ,EAL ,"Cannot get a virtual area: %s\n" , -strerror (errno)); -/* pass errno up the call chain */ - rte_errno = errno; +strerror (rte_errno));
Also, please check that you're using rte_strerror with rte_errno :)
Signature · 2 lines
Thanks, Anatoly
PATCH v5 4/11 eal/mem: extract common code for memseg list initialization
In reply to Dmitry KozlyukAll supported OS create memory segment lists (MSL) and reserve VA space for them in a nearly identical way. Move common code into EAL private functions to reduce duplication.
--- lib/librte_eal/common/eal_common_memory.c | 92 +++++++++++++++++ lib/librte_eal/common/eal_private.h | 62 ++++++++++++ lib/librte_eal/freebsd/eal_memory.c | 94 ++++-------------- lib/librte_eal/linux/eal_memory.c | 115 +++++----------------- 4 files changed, 195 insertions(+), 168 deletions(-)
lib/librte_eal/common/eal_common_memory.c +92 -0
@@ -24,6 +24,7 @@#include "eal_private.h" #include "eal_internal_cfg.h" #include "eal_memcfg.h" +#include "eal_options.h" #include "malloc_heap.h" /*
@@ -181,6 +182,97 @@ eal_get_virtual_area(void *requested_addr, size_t *size,return aligned_addr; } +int +eal_memseg_list_init_named (struct rte_memseg_list *msl,const char *name, +uint64_t page_sz,int n_segs,int socket_id,bool heap) +{ +if (rte_fbarray_init (&msl->memseg_arr, name, n_segs, +sizeof (struct rte_memseg ))) { +RTE_LOG (ERR ,EAL ,"Cannot allocate memseg list: %s\n" , +rte_strerror (rte_errno)); +return -1 ; + } + + msl->page_sz = page_sz; + msl->socket_id = socket_id; + msl->base_va =NULL ; + msl->heap = heap; + +RTE_LOG (DEBUG ,EAL , +"Memseg list allocated at socket %i, page size 0x%zxkB\n" , + socket_id, (size_t )page_sz >>10 ); + +return 0 ; +} + +int +eal_memseg_list_init (struct rte_memseg_list *msl,uint64_t page_sz, +int n_segs,int socket_id,int type_msl_idx,bool heap) +{ +char name[RTE_FBARRAY_NAME_LEN ]; + +snprintf (name,sizeof (name),MEMSEG_LIST_FMT , page_sz >>10 , socket_id, + type_msl_idx); + +return eal_memseg_list_init_named ( + msl, name, page_sz, n_segs, socket_id, heap); +} + +int +eal_memseg_list_alloc (struct rte_memseg_list *msl,int reserve_flags) +{ +uint64_t page_sz; +size_t mem_sz; +void *addr; + + page_sz = msl->page_sz; + mem_sz = page_sz * msl->memseg_arr.len; + + addr =eal_get_virtual_area ( + msl->base_va, &mem_sz, page_sz,0 , reserve_flags); +if (addr ==NULL ) { +if (rte_errno ==EADDRNOTAVAIL ) +RTE_LOG (ERR ,EAL ,"Cannot reserve %llu bytes at [%p] - " +"please use '--" OPT_BASE_VIRTADDR "' option\n" , + (unsigned long long )mem_sz, msl->base_va); +else +RTE_LOG (ERR ,EAL ,"Cannot reserve memory\n" ); +return -1 ; + } + msl->base_va = addr; + msl->len = mem_sz; + +RTE_LOG (DEBUG ,EAL ,"VA reserved for memseg list at %p, size %zx\n" , + addr, mem_sz); + +return 0 ; +} + +void +eal_memseg_list_populate (struct rte_memseg_list *msl,void *addr,int n_segs) +{ +uint64_t page_sz = msl->page_sz; +int i; + +for (i =0 ; i < n_segs; i++) { +struct rte_fbarray *arr = &msl->memseg_arr; +struct rte_memseg *ms =rte_fbarray_get (arr, i); + +if (rte_eal_iova_mode () ==RTE_IOVA_VA ) + ms->iova = (uintptr_t )addr; +else + ms->iova =RTE_BAD_IOVA ; + ms->addr = addr; + ms->hugepage_sz = page_sz; + ms->socket_id =0 ; + ms->len = page_sz; + +rte_fbarray_set_used (arr, i); + + addr =RTE_PTR_ADD (addr, page_sz); + } +} +static struct rte_memseg *virt2memseg (const void *addr,const struct rte_memseg_list *msl) {
lib/librte_eal/common/eal_private.h +62 -0
68 lines not on this page · raw message
lib/librte_eal/freebsd/eal_memory.c +18 -76
156 lines not on this page · raw message
lib/librte_eal/linux/eal_memory.c +23 -92
216 lines not on this page · raw message
Signature · 1 line
2.25.4
Re: [dpdk-dev] [PATCH v5 04/11] eal/mem: extract common code for memseg list initialization
25/05/2020 02:37, Dmitry Kozlyuk:
Quoted 10 lines
All supported OS create memory segment lists (MSL) and reserve VA space for them in a nearly identical way. Move common code into EAL private functions to reduce duplication.
Signed-off-by Dmitry Kozlyuk <[email protected]>---
lib/librte_eal/common/eal_common_memory.c +4
-0+void +eal_memseg_list_populate (struct rte_memseg_list *msl,void *addr,int n_segs) +{ +uint64_t page_sz = msl->page_sz;
[...]
lib/librte_eal/common/eal_common_memory.c eal_get_virtual_area(void *requested_addr, size_t *size, · line 272
Quoted 1 line
lib/librte_eal/common/eal_common_memory.c +1
-0+ addr =RTE_PTR_ADD (addr, page_sz);
This is an error in 32-bit compilation:
lib/librte_eal/common/eal_common_memory.c:
In function ‘eal_memseg_list_populate’: rte_common.h:215:30: error:
cast to pointer from integer of different size [-Werror=int-to-pointer-cast]
215 | #define RTE_PTR_ADD(ptr, x) ((void*)((uintptr_t)(ptr) + (x)))
| ^
The original code was doing a cast to size_t.
lib/librte_eal/linux/eal_memory.c eal_legacy_hugepage_init(void) · line 1453
Quoted 3 lines
lib/librte_eal/linux/eal_memory.c +0
-1- addr =RTE_PTR_ADD (addr, (size_t )page_sz);
I believe the correct cast should be uintptr_t.
Maybe it would be even more correct to do this cast inside RTE_PTR_ADD?
Re: [dpdk-dev] [PATCH v5 04/11] eal/mem: extract common code for memseg list initialization
Quoted 22 lines · On Thu, 28 May 2020 09:31:54 +0200 Thomas Monjalon <[email protected]> wrote:
25/05/2020 02:37, Dmitry Kozlyuk:
Quoted 10 lines
All supported OS create memory segment lists (MSL) and reserve VA space for them in a nearly identical way. Move common code into EAL private functions to reduce duplication.
Signed-off-by Dmitry Kozlyuk <[email protected]>---
lib/librte_eal/common/eal_common_memory.c +4
-0+void +eal_memseg_list_populate (struct rte_memseg_list *msl,void *addr,int n_segs) +{ +uint64_t page_sz = msl->page_sz;[...]
lib/librte_eal/common/eal_common_memory.c eal_get_virtual_area(void *requested_addr, size_t *size, · line 272
Quoted 1 line
lib/librte_eal/common/eal_common_memory.c +1
-0+ addr =RTE_PTR_ADD (addr, page_sz);This is an error in 32-bit compilation:
lib/librte_eal/common/eal_common_memory.c:
In function ‘eal_memseg_list_populate’: rte_common.h:215:30: error:
cast to pointer from integer of different size [-Werror=int-to-pointer-cast]
215 | #defineRTE_PTR_ADD(ptr, x)((void*)((uintptr_t)(ptr) + (x)))
| ^
Quoted 8 lines
The original code was doing a cast to
size_t.lib/librte_eal/linux/eal_memory.c eal_legacy_hugepage_init(void) · line 1453
Quoted 3 lines
lib/librte_eal/linux/eal_memory.c +0
-1- addr =RTE_PTR_ADD (addr, (size_t )page_sz);I believe the correct cast should be
uintptr_t.
Maybe it would be even more correct to do this cast insideRTE_PTR_ADD?
Ack, this is the issue I mentioned in the Community Call letter. I think size_t is a more suitable type for page_*sz* than uintptr_t.
Signature · 1 line
Dmitry Kozlyuk
Re: [dpdk-dev] [PATCH v5 04/11] eal/mem: extract common code for memseg list initialization
In reply to Dmitry KozlyukQuoted 6 lines · Dmitry Kozlyuk
All supported OS create memory segment lists (MSL) and reserve VA space for them in a nearly identical way. Move common code into EAL private functions to reduce duplication.
Signed-off-by Dmitry Kozlyuk <[email protected]>---
<snip>
Quoted 6 lines
lib/librte_eal/common/eal_common_memory.c +6
-0+eal_memseg_list_alloc(struct rte_memseg_list *msl, int reserve_flags) +{ +uint64_t page_sz; +size_t mem_sz; +void *addr; +
lib/librte_eal/common/eal_common_memory.c eal_get_virtual_area(void *requested_addr, size_t *size, · line 237
Quoted 10 lines
lib/librte_eal/common/eal_common_memory.c +10
-0+ page_sz = msl->page_sz; + mem_sz = page_sz * msl->memseg_arr.len; + + addr =eal_get_virtual_area ( + msl->base_va, &mem_sz, page_sz,0 , reserve_flags); +if (addr ==NULL ) { +if (rte_errno ==EADDRNOTAVAIL ) +RTE_LOG (ERR ,EAL ,"Cannot reserve %llu bytes at [%p] - " +"please use '--" OPT_BASE_VIRTADDR "' option\n" , + (unsigned long long )mem_sz, msl->base_va);
Do all OS's support this EAL option?
Quoted 10 lines
lib/librte_eal/common/eal_common_memory.c +10
-0+else +RTE_LOG (ERR ,EAL ,"Cannot reserve memory\n" ); +return -1 ; + } + msl->base_va = addr; + msl->len = mem_sz; + +RTE_LOG (DEBUG ,EAL ,"VA reserved for memseg list at %p, size %zx\n" , + addr, mem_sz); +
<snip>
Quoted 19 lines
lib/librte_eal/linux/eal_memory.c +1
-14-#define MEMSEG_LIST_FMT "memseg-%" PRIu64 "k-%i-%i"static int -alloc_memseg_list (struct rte_memseg_list *msl,uint64_t page_sz, +memseg_list_init (struct rte_memseg_list *msl,uint64_t page_sz,int n_segs,int socket_id,int type_msl_idx) { -char name[RTE_FBARRAY_NAME_LEN ]; - -snprintf (name,sizeof (name),MEMSEG_LIST_FMT , page_sz >>10 , socket_id, - type_msl_idx); -if (rte_fbarray_init (&msl->memseg_arr, name, n_segs, -sizeof (struct rte_memseg ))) { -RTE_LOG (ERR ,EAL ,"Cannot allocate memseg list: %s\n" , -rte_strerror (rte_errno)); -return -1 ; - } - - msl->page_sz = page_sz;
lib/librte_eal/linux/eal_memory.c free_memseg_list(struct rte_memseg_list *msl) · line 820
Quoted 10 lines
lib/librte_eal/linux/eal_memory.c +2
-8- msl->socket_id = socket_id; - msl->base_va =NULL ; - msl->heap =1 ;/* mark it as a heap segment */ - -RTE_LOG (DEBUG ,EAL ,"Memseg list allocated: 0x%zxkB at socket %i\n" , - (size_t )page_sz >>10 , socket_id); - -return 0 ; +return eal_memseg_list_init ( + msl, page_sz, n_segs, socket_id, type_msl_idx, true);
Here and in similar places: I wonder if there's value of keeping memseg_list_init function instead of just calling eal_memseg_list_init() directly from where this is called?
Signature · 2 lines
Thanks, Anatoly
Re: [dpdk-dev] [PATCH v5 04/11] eal/mem: extract common code for memseg list initialization
Quoted 20 lines · Anatoly
Quoted 6 lines · Dmitry Kozlyuk
All supported OS create memory segment lists (MSL) and reserve VA space for them in a nearly identical way. Move common code into EAL private functions to reduce duplication.
Signed-off-by Dmitry Kozlyuk <[email protected]>---
<snip>
Quoted 10 lines
lib/librte_eal/common/eal_common_memory.c +10
-0+eal_memseg_list_alloc(struct rte_memseg_list *msl, int reserve_flags) +{ + uint64_t page_sz; +size_t mem_sz; +void *addr; + + page_sz = msl->page_sz; + mem_sz = page_sz * msl->memseg_arr.len; + + addr = eal_get_virtual_area(
Quoted 8 lines
lib/librte_eal/common/eal_common_memory.c eal_get_virtual_area(void *requested_addr, size_t *size, · line 237
Quoted 6 lines
lib/librte_eal/common/eal_common_memory.c +6
-0+ msl->base_va, &mem_sz, page_sz, 0, reserve_flags); +if (addr ==NULL ) { +if (rte_errno ==EADDRNOTAVAIL ) +RTE_LOG (ERR ,EAL ,"Cannot reserve %llu bytes at [%p] - " +"please use '--" OPT_BASE_VIRTADDR "' option\n" , + (unsigned long long )mem_sz, msl->base_va);Do all OS's support this EAL option?
Supported, yes; meaningful, not quite: for Windows, we start with address 0 (let the OS choose) and using the option can hardly help. Probably Linux and FreeBSD EALs should print this hint. For Windows, we can leave the option, but not print misleading hint.
Signature · 1 line
Dmitry Kozlyuk
Re: [dpdk-dev] [PATCH v5 04/11] eal/mem: extract common code for memseg list initialization
Quoted 32 lines · Dmitry Kozlyuk
Quoted 20 lines · Anatoly
Quoted 6 lines · Dmitry Kozlyuk
All supported OS create memory segment lists (MSL) and reserve VA space for them in a nearly identical way. Move common code into EAL private functions to reduce duplication.
Signed-off-by Dmitry Kozlyuk <[email protected]>---
<snip>
Quoted 10 lines
lib/librte_eal/common/eal_common_memory.c +10
-0+eal_memseg_list_alloc(struct rte_memseg_list *msl, int reserve_flags) +{ + uint64_t page_sz; +size_t mem_sz; +void *addr; + + page_sz = msl->page_sz; + mem_sz = page_sz * msl->memseg_arr.len; + + addr = eal_get_virtual_area(Quoted 8 lines
lib/librte_eal/common/eal_common_memory.c eal_get_virtual_area(void *requested_addr, size_t *size, · line 237
Quoted 6 lines
lib/librte_eal/common/eal_common_memory.c +6
-0+ msl->base_va, &mem_sz, page_sz, 0, reserve_flags); +if (addr ==NULL ) { +if (rte_errno ==EADDRNOTAVAIL ) +RTE_LOG (ERR ,EAL ,"Cannot reserve %llu bytes at [%p] - " +"please use '--" OPT_BASE_VIRTADDR "' option\n" , + (unsigned long long )mem_sz, msl->base_va);Do all OS's support this EAL option?
Quoted 5 lines
Supported, yes; meaningful, not quite: for Windows, we start with address 0 (let the OS choose) and using the option can hardly help. Probably Linux and FreeBSD EALs should print this hint. For Windows, we can leave the option, but not print misleading hint.
We keep rte_errno when exiting this function, do we not? How about we just check it in the caller? It's a bit of a hack, but will avoid OS-specific #ifdef-ery in the common code. Not sure if it's worth it though, over just having an #idef :) Up to you!
Signature · 2 lines
Thanks, Anatoly
Re: [dpdk-dev] [PATCH v5 04/11] eal/mem: extract common code for memseg list initialization
In reply to Dmitry KozlyukQuoted 6 lines · Dmitry Kozlyuk
All supported OS create memory segment lists (MSL) and reserve VA space for them in a nearly identical way. Move common code into EAL private functions to reduce duplication.
Signed-off-by Dmitry Kozlyuk <[email protected]>---
<snip>
Quoted 12 lines
lib/librte_eal/common/eal_common_memory.c +12
-0+void +eal_memseg_list_populate (struct rte_memseg_list *msl,void *addr,int n_segs) +{ +uint64_t page_sz = msl->page_sz; +int i; + +for (i =0 ; i < n_segs; i++) { +struct rte_fbarray *arr = &msl->memseg_arr; +struct rte_memseg *ms =rte_fbarray_get (arr, i); + +if (rte_eal_iova_mode () ==RTE_IOVA_VA ) + ms->iova = (uintptr_t )addr;
lib/librte_eal/common/eal_common_memory.c eal_get_virtual_area(void *requested_addr, size_t *size, · line 272
Quoted 10 lines
lib/librte_eal/common/eal_common_memory.c +10
-0+else + ms->iova =RTE_BAD_IOVA ; + ms->addr = addr; + ms->hugepage_sz = page_sz; + ms->socket_id =0 ; + ms->len = page_sz; + +rte_fbarray_set_used (arr, i); + + addr =RTE_PTR_ADD (addr, page_sz);
This breaks 32-bit build. I believe page_sz should be size_t, not uint64_t.
Signature · 2 lines
Thanks, Anatoly
PATCH v5 5/11 eal/mem: extract common code for dynamic memory allocation
In reply to Dmitry KozlyukCode in Linux EAL that supports dynamic memory allocation (as opposed to static allocation used by FreeBSD) is not OS-dependent and can be reused by Windows EAL. Move such code to a file compiled only for the OS that require it.
--- lib/librte_eal/common/eal_common_dynmem.c | 498 ++++++++++++++++++++++ lib/librte_eal/common/eal_private.h | 43 +- lib/librte_eal/common/meson.build | 4 + lib/librte_eal/linux/eal_memory.c | 494 +-------------------- 4 files changed, 547 insertions(+), 492 deletions(-) create mode 100644 lib/librte_eal/common/eal_common_dynmem.c
lib/librte_eal/common/eal_common_dynmem.c new file +498 -0
@@ -0,0 +1,498 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2010-2014 Intel Corporation. +* Copyright(c) 2013 6WIND S.A. +*/ + +#include <inttypes.h> +#include <string.h> + +#include <rte_log.h> + +#include "eal_internal_cfg.h" +#include "eal_memalloc.h" +#include "eal_memcfg.h" +#include "eal_private.h" + +/** @file Functions common to EALs that support dynamic memory allocation. */ + +int +eal_dynmem_memseg_lists_init (void ) +{ +struct rte_mem_config *mcfg =rte_eal_get_configuration ()->mem_config; +struct memtype { +uint64_t page_sz; +int socket_id; + } *memtypes =NULL ; +int i, hpi_idx, msl_idx, ret =-1 ;/* fail unless told to succeed */ +struct rte_memseg_list *msl; +uint64_t max_mem, max_mem_per_type; +unsigned int max_seglists_per_type; +unsigned int n_memtypes, cur_type; + +/* no-huge does not need this at all */ +if (internal_config.no_hugetlbfs) +return 0 ; + +/* +* figuring out amount of memory we're going to have is a long and very +* involved process. the basic element we're operating with is a memory +* type, defined as a combination of NUMA node ID and page size (so that +* e.g. 2 sockets with 2 page sizes yield 4 memory types in total). +* +* deciding amount of memory going towards each memory type is a +* balancing act between maximum segments per type, maximum memory per +* type, and number of detected NUMA nodes. the goal is to make sure +* each memory type gets at least one memseg list. +* +* the total amount of memory is limited by RTE_MAX_MEM_MB value. +* +* the total amount of memory per type is limited by either +* RTE_MAX_MEM_MB_PER_TYPE, or by RTE_MAX_MEM_MB divided by the number +* of detected NUMA nodes. additionally, maximum number of segments per +* type is also limited by RTE_MAX_MEMSEG_PER_TYPE. this is because for +* smaller page sizes, it can take hundreds of thousands of segments to +* reach the above specified per-type memory limits. +* +* additionally, each type may have multiple memseg lists associated +* with it, each limited by either RTE_MAX_MEM_MB_PER_LIST for bigger +* page sizes, or RTE_MAX_MEMSEG_PER_LIST segments for smaller ones. +* +* the number of memseg lists per type is decided based on the above +* limits, and also taking number of detected NUMA nodes, to make sure +* that we don't run out of memseg lists before we populate all NUMA +* nodes with memory. +* +* we do this in three stages. first, we collect the number of types. +* then, we figure out memory constraints and populate the list of +* would-be memseg lists. then, we go ahead and allocate the memseg +* lists. +*/ + +/* create space for mem types */ + n_memtypes = internal_config.num_hugepage_sizes *rte_socket_count (); + memtypes =calloc (n_memtypes,sizeof (*memtypes)); +if (memtypes ==NULL ) { +RTE_LOG (ERR ,EAL ,"Cannot allocate space for memory types\n" ); +return -1 ; + } + +/* populate mem types */ + cur_type =0 ; +for (hpi_idx =0 ; hpi_idx < (int ) internal_config.num_hugepage_sizes; + hpi_idx++) { +struct hugepage_info *hpi; +uint64_t hugepage_sz; + + hpi = &internal_config.hugepage_info[hpi_idx]; + hugepage_sz = hpi->hugepage_sz; + +for (i =0 ; i < (int )rte_socket_count (); i++, cur_type++) { +int socket_id =rte_socket_id_by_idx (i); + +#ifndef RTE_EAL_NUMA_AWARE_HUGEPAGES +/* we can still sort pages by socket in legacy mode */ +if (!internal_config.legacy_mem && socket_id >0 ) +break ; +#endif + memtypes[cur_type].page_sz = hugepage_sz; + memtypes[cur_type].socket_id = socket_id; + +RTE_LOG (DEBUG ,EAL ,"Detected memory type: " +"socket_id:%u hugepage_sz:%" PRIu64"\n" , + socket_id, hugepage_sz); + } + } +/* number of memtypes could have been lower due to no NUMA support */ + n_memtypes = cur_type; + +/* set up limits for types */ + max_mem = (uint64_t )RTE_MAX_MEM_MB <<20 ; + max_mem_per_type =RTE_MIN ((uint64_t )RTE_MAX_MEM_MB_PER_TYPE <<20 , + max_mem / n_memtypes); +/* +* limit maximum number of segment lists per type to ensure there's +* space for memseg lists for all NUMA nodes with all page sizes +*/ + max_seglists_per_type =RTE_MAX_MEMSEG_LISTS / n_memtypes; + +if (max_seglists_per_type ==0 ) { +RTE_LOG (ERR ,EAL ,"Cannot accommodate all memory types, please increase %s\n" , +RTE_STR (CONFIG_RTE_MAX_MEMSEG_LISTS )); + goto out; + } + +/* go through all mem types and create segment lists */ + msl_idx =0 ; +for (cur_type =0 ; cur_type < n_memtypes; cur_type++) { +unsigned int cur_seglist, n_seglists, n_segs; +unsigned int max_segs_per_type, max_segs_per_list; +struct memtype *type = &memtypes[cur_type]; +uint64_t max_mem_per_list, pagesz; +int socket_id; + + pagesz = type->page_sz; + socket_id = type->socket_id; + +/* +* we need to create segment lists for this type. we must take +* into account the following things: +* +* 1. total amount of memory we can use for this memory type +* 2. total amount of memory per memseg list allowed +* 3. number of segments needed to fit the amount of memory +* 4. number of segments allowed per type +* 5. number of segments allowed per memseg list +* 6. number of memseg lists we are allowed to take up +*/ + +/* calculate how much segments we will need in total */ + max_segs_per_type = max_mem_per_type / pagesz; +/* limit number of segments to maximum allowed per type */ + max_segs_per_type =RTE_MIN (max_segs_per_type, + (unsigned int )RTE_MAX_MEMSEG_PER_TYPE ); +/* limit number of segments to maximum allowed per list */ + max_segs_per_list =RTE_MIN (max_segs_per_type, + (unsigned int )RTE_MAX_MEMSEG_PER_LIST ); + +/* calculate how much memory we can have per segment list */ + max_mem_per_list =RTE_MIN (max_segs_per_list * pagesz, + (uint64_t )RTE_MAX_MEM_MB_PER_LIST <<20 ); + +/* calculate how many segments each segment list will have */ + n_segs =RTE_MIN (max_segs_per_list, max_mem_per_list / pagesz); + +/* calculate how many segment lists we can have */ + n_seglists =RTE_MIN (max_segs_per_type / n_segs, + max_mem_per_type / max_mem_per_list); + +/* limit number of segment lists according to our maximum */ + n_seglists =RTE_MIN (n_seglists, max_seglists_per_type); + +RTE_LOG (DEBUG ,EAL ,"Creating %i segment lists: " +"n_segs:%i socket_id:%i hugepage_sz:%" PRIu64"\n" , + n_seglists, n_segs, socket_id, pagesz); + +/* create all segment lists */ +for (cur_seglist =0 ; cur_seglist < n_seglists; cur_seglist++) { +if (msl_idx >=RTE_MAX_MEMSEG_LISTS ) { +RTE_LOG (ERR ,EAL , +"No more space in memseg lists, please increase %s\n" , +RTE_STR (CONFIG_RTE_MAX_MEMSEG_LISTS )); + goto out; + } + msl = &mcfg->memsegs[msl_idx++]; + +if (eal_memseg_list_init (msl, pagesz, n_segs, + socket_id, cur_seglist, true)) + goto out; + +if (eal_memseg_list_alloc (msl,0 )) { +RTE_LOG (ERR ,EAL ,"Cannot allocate VA space for memseg list\n" ); + goto out; + } + } + } +/* we're successful */ + ret =0 ; +out: +free (memtypes); +return ret; +} + +static int __rte_unused +hugepage_count_walk (const struct rte_memseg_list *msl,void *arg) +{ +struct hugepage_info *hpi = arg; + +if (msl->page_sz != hpi->hugepage_sz) +return 0 ; + + hpi->num_pages[msl->socket_id] += msl->memseg_arr.len; +return 0 ; +} + +static int +limits_callback (int socket_id,size_t cur_limit,size_t new_len) +{ +RTE_SET_USED (socket_id); +RTE_SET_USED (cur_limit); +RTE_SET_USED (new_len); +return -1 ; +} + +int +eal_dynmem_hugepage_init (void ) +{ +struct hugepage_info used_hp[MAX_HUGEPAGE_SIZES ]; +uint64_t memory[RTE_MAX_NUMA_NODES ]; +int hp_sz_idx, socket_id; + +memset (used_hp,0 ,sizeof (used_hp)); + +for (hp_sz_idx =0 ; + hp_sz_idx < (int ) internal_config.num_hugepage_sizes; + hp_sz_idx++) { +#ifndef RTE_ARCH_64 +struct hugepage_info dummy; +unsigned int i; +#endif +/* also initialize used_hp hugepage sizes in used_hp */ +struct hugepage_info *hpi; + hpi = &internal_config.hugepage_info[hp_sz_idx]; + used_hp[hp_sz_idx].hugepage_sz = hpi->hugepage_sz; + +#ifndef RTE_ARCH_64 +/* for 32-bit, limit number of pages on socket to whatever we've +* preallocated, as we cannot allocate more. +*/ +memset (&dummy,0 ,sizeof (dummy)); + dummy.hugepage_sz = hpi->hugepage_sz; +if (rte_memseg_list_walk (hugepage_count_walk, &dummy) <0 ) +return -1 ; + +for (i =0 ; i <RTE_DIM (dummy.num_pages); i++) { + hpi->num_pages[i] =RTE_MIN (hpi->num_pages[i], + dummy.num_pages[i]); + } +#endif + } + +/* make a copy of socket_mem, needed for balanced allocation. */ +for (hp_sz_idx =0 ; hp_sz_idx <RTE_MAX_NUMA_NODES ; hp_sz_idx++) + memory[hp_sz_idx] = internal_config.socket_mem[hp_sz_idx]; + +/* calculate final number of pages */ +if (eal_dynmem_calc_num_pages_per_socket (memory, + internal_config.hugepage_info, used_hp, + internal_config.num_hugepage_sizes) <0 ) +return -1 ; + +for (hp_sz_idx =0 ; + hp_sz_idx < (int )internal_config.num_hugepage_sizes; + hp_sz_idx++) { +for (socket_id =0 ; socket_id <RTE_MAX_NUMA_NODES ; + socket_id++) { +struct rte_memseg **pages; +struct hugepage_info *hpi = &used_hp[hp_sz_idx]; +unsigned int num_pages = hpi->num_pages[socket_id]; +unsigned int num_pages_alloc; + +if (num_pages ==0 ) +continue ; + +RTE_LOG (DEBUG ,EAL , +"Allocating %u pages of size %" PRIu64"M " +"on socket %i\n" , + num_pages, hpi->hugepage_sz >>20 , socket_id); + +/* we may not be able to allocate all pages in one go, +* because we break up our memory map into multiple +* memseg lists. therefore, try allocating multiple +* times and see if we can get the desired number of +* pages from multiple allocations. +*/ + + num_pages_alloc =0 ; +do { +int i, cur_pages, needed; + + needed = num_pages - num_pages_alloc; + + pages =malloc (sizeof (*pages) * needed); + +/* do not request exact number of pages */ + cur_pages =eal_memalloc_alloc_seg_bulk (pages, + needed, hpi->hugepage_sz, + socket_id, false); +if (cur_pages <=0 ) { +free (pages); +return -1 ; + } + +/* mark preallocated pages as unfreeable */ +for (i =0 ; i < cur_pages; i++) { +struct rte_memseg *ms = pages[i]; + ms->flags |= +RTE_MEMSEG_FLAG_DO_NOT_FREE ; + } +free (pages); + + num_pages_alloc += cur_pages; + }while (num_pages_alloc != num_pages); + } + } + +/* if socket limits were specified, set them */ +if (internal_config.force_socket_limits) { +unsigned int i; +for (i =0 ; i <RTE_MAX_NUMA_NODES ; i++) { +uint64_t limit = internal_config.socket_limit[i]; +if (limit ==0 ) +continue ; +if (rte_mem_alloc_validator_register ("socket-limit" , + limits_callback, i, limit)) +RTE_LOG (ERR ,EAL ,"Failed to register socket limits validator callback\n" ); + } + } +return 0 ; +} + +__rte_unused /* function is unused on 32-bit builds */ +static inline uint64_t +get_socket_mem_size (int socket) +{ +uint64_t size =0 ; +unsigned int i; + +for (i =0 ; i < internal_config.num_hugepage_sizes; i++) { +struct hugepage_info *hpi = &internal_config.hugepage_info[i]; + size += hpi->hugepage_sz * hpi->num_pages[socket]; + } + +return size; +} + +int +eal_dynmem_calc_num_pages_per_socket ( +uint64_t *memory,struct hugepage_info *hp_info, +struct hugepage_info *hp_used,unsigned int num_hp_info) +{ +unsigned int socket, j, i =0 ; +unsigned int requested, available; +int total_num_pages =0 ; +uint64_t remaining_mem, cur_mem; +uint64_t total_mem = internal_config.memory; + +if (num_hp_info ==0 ) +return -1 ; + +/* if specific memory amounts per socket weren't requested */ +if (internal_config.force_sockets ==0 ) { +size_t total_size; +int cpu_per_socket[RTE_MAX_NUMA_NODES ]; +size_t default_size; +unsigned int lcore_id; + +/* Compute number of cores per socket */ +memset (cpu_per_socket,0 ,sizeof (cpu_per_socket)); +RTE_LCORE_FOREACH (lcore_id) { + cpu_per_socket[rte_lcore_to_socket_id (lcore_id)]++; + } + +/* +* Automatically spread requested memory amongst detected +* sockets according to number of cores from cpu mask present +* on each socket. +*/ + total_size = internal_config.memory; +for (socket =0 ; socket <RTE_MAX_NUMA_NODES && total_size !=0 ; + socket++) { + +/* Set memory amount per socket */ + default_size = internal_config.memory * + cpu_per_socket[socket] /rte_lcore_count (); + +/* Limit to maximum available memory on socket */ + default_size =RTE_MIN ( + default_size,get_socket_mem_size (socket)); + +/* Update sizes */ + memory[socket] = default_size; + total_size -= default_size; + } + +/* +* If some memory is remaining, try to allocate it by getting +* all available memory from sockets, one after the other. +*/ +for (socket =0 ; socket <RTE_MAX_NUMA_NODES && total_size !=0 ; + socket++) { +/* take whatever is available */ + default_size =RTE_MIN ( +get_socket_mem_size (socket) - memory[socket], + total_size); + +/* Update sizes */ + memory[socket] += default_size; + total_size -= default_size; + } + } + +for (socket =0 ; socket <RTE_MAX_NUMA_NODES && total_mem !=0 ; + socket++) { +/* skips if the memory on specific socket wasn't requested */ +for (i =0 ; i < num_hp_info && memory[socket] !=0 ; i++) { +strncpy (hp_used[i].hugedir, hp_info[i].hugedir, +sizeof (hp_used[i].hugedir)); + hp_used[i].num_pages[socket] =RTE_MIN ( + memory[socket] / hp_info[i].hugepage_sz, + hp_info[i].num_pages[socket]); + + cur_mem = hp_used[i].num_pages[socket] * + hp_used[i].hugepage_sz; + + memory[socket] -= cur_mem; + total_mem -= cur_mem; + + total_num_pages += hp_used[i].num_pages[socket]; + +/* check if we have met all memory requests */ +if (memory[socket] ==0 ) +break ; + +/* Check if we have any more pages left at this size, +* if so, move on to next size. +*/ +if (hp_used[i].num_pages[socket] == + hp_info[i].num_pages[socket]) +continue ; + +/* At this point we know that there are more pages +* available that are bigger than the memory we want, +* so lets see if we can get enough from other page +* sizes. +*/ + remaining_mem =0 ; +for (j = i+1 ; j < num_hp_info; j++) + remaining_mem += hp_info[j].hugepage_sz * + hp_info[j].num_pages[socket]; + +/* Is there enough other memory? +* If not, allocate another page and quit. +*/ +if (remaining_mem < memory[socket]) { + cur_mem =RTE_MIN ( + memory[socket], hp_info[i].hugepage_sz); + memory[socket] -= cur_mem; + total_mem -= cur_mem; + hp_used[i].num_pages[socket]++; + total_num_pages++; +break ;/* we are done with this socket*/ + } + } +/* if we didn't satisfy all memory requirements per socket */ +if (memory[socket] >0 && + internal_config.socket_mem[socket] !=0 ) { +/* to prevent icc errors */ + requested = (unsigned int )( + internal_config.socket_mem[socket] /0x100000 ); + available = requested - + ((unsigned int )(memory[socket] /0x100000 )); +RTE_LOG (ERR ,EAL ,"Not enough memory available on " +"socket %u! Requested: %uMB, available: %uMB\n" , + socket, requested, available); +return -1 ; + } + } + +/* if we didn't satisfy total memory requirements */ +if (total_mem >0 ) { + requested = (unsigned int ) (internal_config.memory /0x100000 ); + available = requested - (unsigned int ) (total_mem /0x100000 ); +RTE_LOG (ERR ,EAL ,"Not enough memory available! " +"Requested: %uMB, available: %uMB\n" , + requested, available); +return -1 ; + } +return total_num_pages; +}
lib/librte_eal/common/eal_private.h +42 -1
61 lines not on this page · raw message
lib/librte_eal/common/meson.build +4 -0
@@ -56,3 +56,7 @@ sources += files( 'rte_reciprocal.c', 'rte_service.c', ) + +if is_linux + sources += files('eal_common_dynmem.c') +endif
lib/librte_eal/linux/eal_memory.c +3 -491
524 lines not on this page · raw message
Signature · 1 line
2.25.4
Re: [dpdk-dev] [PATCH v5 05/11] eal/mem: extract common code for dynamic memory allocation
25/05/2020 02:37, Dmitry Kozlyuk:
lib/librte_eal/linux/eal_memory.c rte_eal_hugepage_init(void) · line 1636
Quoted 13 lines
Code in Linux EAL that supports dynamic memory allocation (as opposed to static allocation used by FreeBSD) is not OS-dependent and can be reused by Windows EAL. Move such code to a file compiled only for the OS that require it.
Signed-off-by Dmitry Kozlyuk <[email protected]>---
lib/librte_eal/linux/eal_memory.c +1
-1@@ -1943,7 +1633,7 @@ rte_eal_hugepage_init(void) {return internal_config.legacy_mem ?eal_legacy_hugepage_init () : -eal_hugepage_init (); +eal_dynmem_hugepage_init ();
There is a compilation issue, building clang+shared:
undefined reference to `eal_dynmem_hugepage_init'
Re: [dpdk-dev] [PATCH v5 05/11] eal/mem: extract common code for dynamic memory allocation
In reply to Dmitry KozlyukQuoted 7 lines · Dmitry Kozlyuk
Code in Linux EAL that supports dynamic memory allocation (as opposed to static allocation used by FreeBSD) is not OS-dependent and can be reused by Windows EAL. Move such code to a file compiled only for the OS that require it.
Signed-off-by Dmitry Kozlyuk <[email protected]>---
I believe you forgot to add dynmem to Makefile.
Quoted 9 lines
lib/librte_eal/common/eal_common_dynmem.c new file +9
-0+eal_dynmem_calc_num_pages_per_socket ( + uint64_t *memory, struct hugepage_info *hp_info, + struct hugepage_info *hp_used, unsigned int num_hp_info) +{ +unsigned int socket, j, i =0 ; +unsigned int requested, available; +int total_num_pages =0 ; +uint64_t remaining_mem, cur_mem; +uint64_t total_mem = internal_config.memory;
lib/librte_eal/common/eal_common_dynmem.c line 374
Quoted 10 lines
lib/librte_eal/common/eal_common_dynmem.c new file +10
-0+ +if (num_hp_info ==0 ) +return -1 ; + +/* if specific memory amounts per socket weren't requested */ +if (internal_config.force_sockets ==0 ) { +size_t total_size; +int cpu_per_socket[RTE_MAX_NUMA_NODES ]; +size_t default_size; +unsigned int lcore_id;
Comparing code from eal_memory.c and this one, it seems like you've dropped all 32-bit code from this function. Is that intentional?
Signature · 2 lines
Thanks, Anatoly
Re: [dpdk-dev] [PATCH v5 05/11] eal/mem: extract common code for dynamic memory allocation
Quoted 10 lines · Anatoly
Quoted 7 lines · Dmitry Kozlyuk
Code in Linux EAL that supports dynamic memory allocation (as opposed to static allocation used by FreeBSD) is not OS-dependent and can be reused by Windows EAL. Move such code to a file compiled only for the OS that require it.
Signed-off-by Dmitry Kozlyuk <[email protected]>---
I believe you forgot to add dynmem to
Makefile.
Right, thanks.
Quoted 15 lines
Quoted 15 lines
lib/librte_eal/common/eal_common_dynmem.c new file +14
-0+eal_dynmem_calc_num_pages_per_socket ( + uint64_t *memory, struct hugepage_info *hp_info, + struct hugepage_info *hp_used, unsigned int num_hp_info) +{ +unsigned int socket, j, i =0 ; +unsigned int requested, available; +int total_num_pages =0 ; +uint64_t remaining_mem, cur_mem; +uint64_t total_mem = internal_config.memory; + +if (num_hp_info ==0 ) +return -1 ; + +/* if specific memory amounts per socket weren't requested */
Quoted 8 lines
lib/librte_eal/common/eal_common_dynmem.c line 374
Quoted 5 lines
lib/librte_eal/common/eal_common_dynmem.c new file +5
-0+if (internal_config.force_sockets ==0 ) { +size_t total_size; +int cpu_per_socket[RTE_MAX_NUMA_NODES ]; +size_t default_size; +unsigned int lcore_id;Comparing code from
eal_memory.cand this one, it seems like you've dropped all 32-bit code from this function. Is that intentional?
No, it's a mistake.
Signature · 1 line
Dmitry Kozlyuk
PATCH v5 6/11 trace: add size_t field emitter
In reply to Dmitry KozlyukIt is not guaranteed that sizeof(long) == sizeof(size_t). On Windows, sizeof(long) == 4 and sizeof(size_t) == 8 for 64-bit programs. Tracepoints using "long" field emitter are therefore invalid there. Add dedicated field emitter for size_t and use it to store size_t values in all existing tracepoints.
--- lib/librte_eal/include/rte_eal_trace.h | 8 ++++---- lib/librte_eal/include/rte_trace_point.h | 1 + lib/librte_mempool/rte_mempool_trace.h | 10 +++++----- 3 files changed, 10 insertions(+), 9 deletions(-)
lib/librte_eal/include/rte_eal_trace.h +4 -4
@@ -143,7 +143,7 @@ RTE_TRACE_POINT( RTE_TRACE_POINT_ARGS(const char *type, size_t size, unsigned int align, int socket, void *ptr),rte_trace_point_emit_string (type); -rte_trace_point_emit_long (size); +rte_trace_point_emit_size_t (size);rte_trace_point_emit_u32 (align);rte_trace_point_emit_int (socket);rte_trace_point_emit_ptr (ptr);
@@ -154,7 +154,7 @@ RTE_TRACE_POINT( RTE_TRACE_POINT_ARGS(const char *type, size_t size, unsigned int align, int socket, void *ptr),rte_trace_point_emit_string (type); -rte_trace_point_emit_long (size); +rte_trace_point_emit_size_t (size);rte_trace_point_emit_u32 (align);rte_trace_point_emit_int (socket);rte_trace_point_emit_ptr (ptr);
@@ -164,7 +164,7 @@ RTE_TRACE_POINT( rte_eal_trace_mem_realloc, RTE_TRACE_POINT_ARGS(size_t size, unsigned int align, int socket, void *ptr), -rte_trace_point_emit_long (size); +rte_trace_point_emit_size_t (size);rte_trace_point_emit_u32 (align);rte_trace_point_emit_int (socket);rte_trace_point_emit_ptr (ptr);
@@ -183,7 +183,7 @@ RTE_TRACE_POINT( unsigned int flags, unsigned int align, unsigned int bound, const void *mz),rte_trace_point_emit_string (name ); -rte_trace_point_emit_long (len); +rte_trace_point_emit_size_t (len);rte_trace_point_emit_int (socket_id);rte_trace_point_emit_u32 (flags);rte_trace_point_emit_u32 (align);
lib/librte_eal/include/rte_trace_point.h +1 -0
@@ -395,6 +395,7 @@ do { \#define rte_trace_point_emit_i8 (in) __rte_trace_point_emit(in, int8_t)#define rte_trace_point_emit_int (in) __rte_trace_point_emit(in, int32_t)#define rte_trace_point_emit_long (in) __rte_trace_point_emit(in, long) +#define rte_trace_point_emit_size_t (in) __rte_trace_point_emit(in, size_t)#define rte_trace_point_emit_float (in) __rte_trace_point_emit(in, float)#define rte_trace_point_emit_double (in) __rte_trace_point_emit(in, double)#define rte_trace_point_emit_ptr (in) __rte_trace_point_emit(in, uintptr_t)
lib/librte_mempool/rte_mempool_trace.h +5 -5
@@ -72,7 +72,7 @@ RTE_TRACE_POINT(rte_trace_point_emit_string (mempool->name);rte_trace_point_emit_ptr (vaddr);rte_trace_point_emit_u64 (iova); -rte_trace_point_emit_long (len); +rte_trace_point_emit_size_t (len);rte_trace_point_emit_ptr (free_cb);rte_trace_point_emit_ptr (opaque); )
@@ -84,8 +84,8 @@ RTE_TRACE_POINT(rte_trace_point_emit_ptr (mempool);rte_trace_point_emit_string (mempool->name);rte_trace_point_emit_ptr (addr); -rte_trace_point_emit_long (len); -rte_trace_point_emit_long (pg_sz); +rte_trace_point_emit_size_t (len); +rte_trace_point_emit_size_t (pg_sz);rte_trace_point_emit_ptr (free_cb);rte_trace_point_emit_ptr (opaque); )
@@ -126,7 +126,7 @@ RTE_TRACE_POINT( RTE_TRACE_POINT_ARGS(struct rte_mempool *mempool, size_t pg_sz),rte_trace_point_emit_ptr (mempool);rte_trace_point_emit_string (mempool->name); -rte_trace_point_emit_long (pg_sz); +rte_trace_point_emit_size_t (pg_sz); ) RTE_TRACE_POINT(
@@ -139,7 +139,7 @@ RTE_TRACE_POINT(rte_trace_point_emit_u32 (max_objs);rte_trace_point_emit_ptr (vaddr);rte_trace_point_emit_u64 (iova); -rte_trace_point_emit_long (len); +rte_trace_point_emit_size_t (len);rte_trace_point_emit_ptr (obj_cb);rte_trace_point_emit_ptr (obj_cb_arg); )
Signature · 1 line
2.25.4
Re: [dpdk-dev] [PATCH v5 06/11] trace: add size_t field emitter
Quoted 9 lines · Dmitry Kozlyuk
It is not guaranteed that sizeof(long) ==
sizeof(size_t). On Windows, sizeof(long) == 4 andsizeof(size_t)== 8 for 64-bit programs. Tracepoints using "long" field emitter are therefore invalid there. Add dedicated field emitter forsize_tand use it to storesize_tvalues in all existing tracepoints.Signed-off-by Dmitry Kozlyuk <[email protected]>---
The Doxygen comment for the new emit function is missing.
See https://github.com/DPDK/dpdk/blob/master/lib/librte_eal/include/rte_trace_point.h#L138
Other than the above nit, rest looks good to me.
PATCH v5 7/11 eal/windows: add tracing support stubs
In reply to Dmitry KozlyukEAL common code depends on tracepoint calls, but generic implementation cannot be enabled on Windows due to missing standard library facilities. Add stub functions to support tracepoint compilation, so that common code does not have to conditionally include tracepoints until proper support is added.
--- lib/librte_eal/common/eal_common_thread.c | 5 +--- lib/librte_eal/common/meson.build | 1 + lib/librte_eal/windows/eal.c | 34 ++++++++++++++++++++++- 3 files changed, 35 insertions(+), 5 deletions(-)
lib/librte_eal/common/eal_common_thread.c +1 -4
@@ -15,9 +15,7 @@#include <rte_lcore.h> #include <rte_memory.h> #include <rte_log.h> -#ifndef RTE_EXEC_ENV_WINDOWS #include <rte_trace_point.h> -#endif #include "eal_internal_cfg.h" #include "eal_private.h"
@@ -169,9 +167,8 @@ static void *rte_thread_init(void *arg)free (params); } -#ifndef RTE_EXEC_ENV_WINDOWS __rte_trace_mem_per_thread_alloc (); -#endif +return start_routine (routine_arg); }
lib/librte_eal/common/meson.build +1 -0
@@ -14,6 +14,7 @@ if is_windows 'eal_common_log.c', 'eal_common_options.c', 'eal_common_thread.c', + 'eal_common_trace_points.c', ) subdir_done() endif
lib/librte_eal/windows/eal.c +33 -1
@@ -17,6 +17,7 @@#include <eal_filesystem.h> #include <eal_options.h> #include <eal_private.h> +#include <rte_trace_point.h> #include "eal_windows.h"
@@ -221,7 +222,38 @@ rte_eal_init_alert(const char *msg)RTE_LOG (ERR ,EAL ,"%s\n" , msg); } -/* Launch threads, called at application init(). */ +/* Stubs to enable EAL trace point compilation +* until eal_common_trace.c can be compiled. +*/ + +RTE_DEFINE_PER_LCORE(volatile int, trace_point_sz); +RTE_DEFINE_PER_LCORE(void *, trace_mem); + +void +__rte_trace_mem_per_thread_alloc (void ) +{ +} + +void +__rte_trace_point_emit_field (size_t sz,const char *field, +const char *type) +{ +RTE_SET_USED (sz); +RTE_SET_USED (field); +RTE_SET_USED (type); +} + +int +__rte_trace_point_register (rte_trace_point_t *trace,const char *name, +void (*register_fn)(void )) +{ +RTE_SET_USED (trace); +RTE_SET_USED (name); +RTE_SET_USED (register_fn); +return -ENOTSUP ; +} + +/* Launch threads, called at application init(). */ int rte_eal_init (int argc,char **argv) {
Signature · 1 line
2.25.4
PATCH v5 8/11 eal/windows: replace sys/queue.h with a complete one from FreeBSD
In reply to Dmitry KozlyukLimited version imported previously lacks at least SLIST macros. Import a complete file from FreeBSD, since its license exception is already approved by Technical Board.
--- lib/librte_eal/windows/include/sys/queue.h | 663 +++++++++++++++++++-- 1 file changed, 601 insertions(+), 62 deletions(-)
lib/librte_eal/windows/include/sys/queue.h +601 -62
@@ -8,7 +8,36 @@#define _SYS_QUEUE_H_/* -* This file defines tail queues. +* This file defines four types of data structures: singly-linked lists, +* singly-linked tail queues, lists and tail queues. +* +* A singly-linked list is headed by a single forward pointer. The elements +* are singly linked for minimum space and pointer manipulation overhead at +* the expense of O(n) removal for arbitrary elements. New elements can be +* added to the list after an existing element or at the head of the list. +* Elements being removed from the head of the list should use the explicit +* macro for this purpose for optimum efficiency. A singly-linked list may +* only be traversed in the forward direction. Singly-linked lists are ideal +* for applications with large datasets and few or no removals or for +* implementing a LIFO queue. +* +* A singly-linked tail queue is headed by a pair of pointers, one to the +* head of the list and the other to the tail of the list. The elements are +* singly linked for minimum space and pointer manipulation overhead at the +* expense of O(n) removal for arbitrary elements. New elements can be added +* to the list after an existing element, at the head of the list, or at the +* end of the list. Elements being removed from the head of the tail queue +* should use the explicit macro for this purpose for optimum efficiency. +* A singly-linked tail queue may only be traversed in the forward direction. +* Singly-linked tail queues are ideal for applications with large datasets +* and few or no removals or for implementing a FIFO queue. +* +* A list is headed by a single forward pointer (or an array of forward +* pointers for a hash table header). The elements are doubly linked +* so that an arbitrary element can be removed without a need to +* traverse the list. New elements can be added to the list before +* or after an existing element or at the head of the list. A list +* may be traversed in either direction. * * A tail queue is headed by a pair of pointers, one to the head of the * list and the other to the tail of the list. The elements are doubly
@@ -17,65 +46,93 @@* after an existing element, at the head of the list, or at the end of * the list. A tail queue may be traversed in either direction. * +* For details on the use of these macros, see the queue(3) manual page. +* * Below is a summary of implemented functions where: * + means the macro is available * - means the macro is not available * s means the macro is available but is slow (runs in O(n) time) * -* TAILQ -* _HEAD + -* _CLASS_HEAD + -* _HEAD_INITIALIZER + -* _ENTRY + -* _CLASS_ENTRY + -* _INIT + -* _EMPTY + -* _FIRST + -* _NEXT + -* _PREV + -* _LAST + -* _LAST_FAST + -* _FOREACH + -* _FOREACH_FROM + -* _FOREACH_SAFE + -* _FOREACH_FROM_SAFE + -* _FOREACH_REVERSE + -* _FOREACH_REVERSE_FROM + -* _FOREACH_REVERSE_SAFE + -* _FOREACH_REVERSE_FROM_SAFE + -* _INSERT_HEAD + -* _INSERT_BEFORE + -* _INSERT_AFTER + -* _INSERT_TAIL + -* _CONCAT + -* _REMOVE_AFTER - -* _REMOVE_HEAD - -* _REMOVE + -* _SWAP + +* SLIST LIST STAILQ TAILQ +* _HEAD + + + + +* _CLASS_HEAD + + + + +* _HEAD_INITIALIZER + + + + +* _ENTRY + + + + +* _CLASS_ENTRY + + + + +* _INIT + + + + +* _EMPTY + + + + +* _FIRST + + + + +* _NEXT + + + + +* _PREV - + - + +* _LAST - - + + +* _LAST_FAST - - - + +* _FOREACH + + + + +* _FOREACH_FROM + + + + +* _FOREACH_SAFE + + + + +* _FOREACH_FROM_SAFE + + + + +* _FOREACH_REVERSE - - - + +* _FOREACH_REVERSE_FROM - - - + +* _FOREACH_REVERSE_SAFE - - - + +* _FOREACH_REVERSE_FROM_SAFE - - - + +* _INSERT_HEAD + + + + +* _INSERT_BEFORE - + - + +* _INSERT_AFTER + + + + +* _INSERT_TAIL - - + + +* _CONCAT s s + + +* _REMOVE_AFTER + - + - +* _REMOVE_HEAD + - + - +* _REMOVE s + s + +* _SWAP + + + + * */ - -#ifdef __cplusplus -extern "C" { +#ifdef QUEUE_MACRO_DEBUG +#warn Use QUEUE_MACRO_DEBUG_TRACE and/or QUEUE_MACRO_DEBUG_TRASH +#define QUEUE_MACRO_DEBUG_TRACE +#define QUEUE_MACRO_DEBUG_TRASH #endif -/* - * List definitions. - */ -#define LIST_HEAD(name, type) \ -struct name { \ - struct type *lh_first; /* first element */ \ -} +#ifdef QUEUE_MACRO_DEBUG_TRACE +/* Store the last 2 places the queue element or head was altered */ +struct qm_trace { +unsigned long lastline; +unsigned long prevline; +const char *lastfile; +const char *prevfile; +}; + +#define TRACEBUF struct qm_trace trace; +#define TRACEBUF_INITIALIZER { __LINE__, 0, __FILE__, NULL } , + +#define QMD_TRACE_HEAD (head) do { \ + (head)->trace.prevline = (head)->trace.lastline; \ + (head)->trace.prevfile = (head)->trace.lastfile; \ + (head)->trace.lastline = __LINE__; \ + (head)->trace.lastfile = __FILE__; \ +} while (0) +#define QMD_TRACE_ELEM (elem) do { \ + (elem)->trace.prevline = (elem)->trace.lastline; \ + (elem)->trace.prevfile = (elem)->trace.lastfile; \ + (elem)->trace.lastline = __LINE__; \ + (elem)->trace.lastfile = __FILE__; \ +} while (0) + +#else /* !QUEUE_MACRO_DEBUG_TRACE */ #define QMD_TRACE_ELEM (elem)#define QMD_TRACE_HEAD (head)#define TRACEBUF #define TRACEBUF_INITIALIZER +#endif /* QUEUE_MACRO_DEBUG_TRACE */ +#ifdef QUEUE_MACRO_DEBUG_TRASH +#define QMD_SAVELINK (name, link) void **name = (void *)&(link) +#define TRASHIT (x) do {(x) = (void *)-1;} while (0) +#define QMD_IS_TRASHED (x) ((x) == (void *)(intptr_t)-1) +#else /* !QUEUE_MACRO_DEBUG_TRASH */ +#define QMD_SAVELINK (name, link)#define TRASHIT (x)#define QMD_IS_TRASHED (x) 0 - -#define QMD_SAVELINK(name, link) +#endif /* QUEUE_MACRO_DEBUG_TRASH */ #ifdef __cplusplus/*
@@ -86,6 +143,445 @@ struct name { \#define QUEUE_TYPEOF (type) struct type#endif +/* +* Singly-linked List declarations. +*/ +#define SLIST_HEAD (name, type) \ +struct name { \ + struct type *slh_first;/* first element */ \ +} + +#define SLIST_CLASS_HEAD(name, type) \ +struct name { \ + class type *slh_first; /* first element */ \ +} + +#define SLIST_HEAD_INITIALIZER (head) \ + { NULL } + +#define SLIST_ENTRY(type) \ +struct { \ + struct type *sle_next; /* next element */ \ +} + +#define SLIST_CLASS_ENTRY(type) \ +struct { \ + class type *sle_next; /* next element */ \ +} + +/* +* Singly-linked List functions. +*/ +#if (defined(_KERNEL) && defined(INVARIANTS )) +#define QMD_SLIST_CHECK_PREVPTR (prevp, elm) do { \ + if (*(prevp) != (elm)) \ + panic("Bad prevptr *(%p) == %p != %p", \ + (prevp), *(prevp), (elm)); \ +} while (0) +#else +#define QMD_SLIST_CHECK_PREVPTR (prevp, elm) +#endif + +#define SLIST_CONCAT (head1, head2, type, field) do { \ + QUEUE_TYPEOF(type) *curelm = SLIST_FIRST(head1); \ + if (curelm == NULL) { \ + if ((SLIST_FIRST(head1) = SLIST_FIRST(head2)) != NULL) \ + SLIST_INIT(head2); \ + } else if (SLIST_FIRST(head2) != NULL) { \ + while (SLIST_NEXT(curelm, field) != NULL) \ + curelm = SLIST_NEXT(curelm, field); \ + SLIST_NEXT(curelm, field) = SLIST_FIRST(head2); \ + SLIST_INIT(head2); \ + } \ +} while (0) + +#define SLIST_EMPTY (head) ((head)->slh_first == NULL) + +#define SLIST_FIRST (head) ((head)->slh_first) + +#define SLIST_FOREACH (var, head, field) \ + for ((var) = SLIST_FIRST((head)); \ + (var); \ + (var) = SLIST_NEXT((var), field)) + +#define SLIST_FOREACH_FROM (var, head, field) \ + for ((var) = ((var) ? (var) : SLIST_FIRST((head))); \ + (var); \ + (var) = SLIST_NEXT((var), field)) + +#define SLIST_FOREACH_SAFE (var, head, field, tvar) \ + for ((var) = SLIST_FIRST((head)); \ + (var) && ((tvar) = SLIST_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define SLIST_FOREACH_FROM_SAFE (var, head, field, tvar) \ + for ((var) = ((var) ? (var) : SLIST_FIRST((head))); \ + (var) && ((tvar) = SLIST_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define SLIST_FOREACH_PREVPTR (var, varp, head, field) \ + for ((varp) = &SLIST_FIRST((head)); \ + ((var) = *(varp)) != NULL; \ + (varp) = &SLIST_NEXT((var), field)) + +#define SLIST_INIT (head) do { \ + SLIST_FIRST((head)) = NULL; \ +} while (0) + +#define SLIST_INSERT_AFTER (slistelm, elm, field) do { \ + SLIST_NEXT((elm), field) = SLIST_NEXT((slistelm), field); \ + SLIST_NEXT((slistelm), field) = (elm); \ +} while (0) + +#define SLIST_INSERT_HEAD (head, elm, field) do { \ + SLIST_NEXT((elm), field) = SLIST_FIRST((head)); \ + SLIST_FIRST((head)) = (elm); \ +} while (0) + +#define SLIST_NEXT (elm, field) ((elm)->field.sle_next) + +#define SLIST_REMOVE (head, elm, type, field) do { \ + QMD_SAVELINK(oldnext, (elm)->field.sle_next); \ + if (SLIST_FIRST((head)) == (elm)) { \ + SLIST_REMOVE_HEAD((head), field); \ + } \ + else { \ + QUEUE_TYPEOF(type) *curelm = SLIST_FIRST(head); \ + while (SLIST_NEXT(curelm, field) != (elm)) \ + curelm = SLIST_NEXT(curelm, field); \ + SLIST_REMOVE_AFTER(curelm, field); \ + } \ + TRASHIT(*oldnext); \ +} while (0) + +#define SLIST_REMOVE_AFTER (elm, field) do { \ + SLIST_NEXT(elm, field) = \ + SLIST_NEXT(SLIST_NEXT(elm, field), field); \ +} while (0) + +#define SLIST_REMOVE_HEAD (head, field) do { \ + SLIST_FIRST((head)) = SLIST_NEXT(SLIST_FIRST((head)), field); \ +} while (0) + +#define SLIST_REMOVE_PREVPTR (prevp, elm, field) do { \ + QMD_SLIST_CHECK_PREVPTR(prevp, elm); \ + *(prevp) = SLIST_NEXT(elm, field); \ + TRASHIT((elm)->field.sle_next); \ +} while (0) + +#define SLIST_SWAP (head1, head2, type) do { \ + QUEUE_TYPEOF(type) *swap_first = SLIST_FIRST(head1); \ + SLIST_FIRST(head1) = SLIST_FIRST(head2); \ + SLIST_FIRST(head2) = swap_first; \ +} while (0) + +/* +* Singly-linked Tail queue declarations. +*/ +#define STAILQ_HEAD (name, type) \ +struct name { \ + struct type *stqh_first;/* first element */ \ +struct type **stqh_last;/* addr of last next element */ \ +} + +#define STAILQ_CLASS_HEAD(name, type) \ +struct name { \ + class type *stqh_first; /* first element */ \ + class type **stqh_last; /* addr of last next element */ \ +} + +#define STAILQ_HEAD_INITIALIZER (head) \ + { NULL, &(head).stqh_first } + +#define STAILQ_ENTRY(type) \ +struct { \ + struct type *stqe_next; /* next element */ \ +} + +#define STAILQ_CLASS_ENTRY(type) \ +struct { \ + class type *stqe_next; /* next element */ \ +} + +/* +* Singly-linked Tail queue functions. +*/ +#define STAILQ_CONCAT (head1, head2) do { \ + if (!STAILQ_EMPTY((head2))) { \ + *(head1)->stqh_last = (head2)->stqh_first; \ + (head1)->stqh_last = (head2)->stqh_last; \ + STAILQ_INIT((head2)); \ + } \ +} while (0) + +#define STAILQ_EMPTY (head) ((head)->stqh_first == NULL) + +#define STAILQ_FIRST (head) ((head)->stqh_first) + +#define STAILQ_FOREACH (var, head, field) \ + for((var) = STAILQ_FIRST((head)); \ + (var); \ + (var) = STAILQ_NEXT((var), field)) + +#define STAILQ_FOREACH_FROM (var, head, field) \ + for ((var) = ((var) ? (var) : STAILQ_FIRST((head))); \ + (var); \ + (var) = STAILQ_NEXT((var), field)) + +#define STAILQ_FOREACH_SAFE (var, head, field, tvar) \ + for ((var) = STAILQ_FIRST((head)); \ + (var) && ((tvar) = STAILQ_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define STAILQ_FOREACH_FROM_SAFE (var, head, field, tvar) \ + for ((var) = ((var) ? (var) : STAILQ_FIRST((head))); \ + (var) && ((tvar) = STAILQ_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define STAILQ_INIT (head) do { \ + STAILQ_FIRST((head)) = NULL; \ + (head)->stqh_last = &STAILQ_FIRST((head)); \ +} while (0) + +#define STAILQ_INSERT_AFTER (head, tqelm, elm, field) do { \ + if ((STAILQ_NEXT((elm), field) = STAILQ_NEXT((tqelm), field)) == NULL)\ + (head)->stqh_last = &STAILQ_NEXT((elm), field); \ + STAILQ_NEXT((tqelm), field) = (elm); \ +} while (0) + +#define STAILQ_INSERT_HEAD (head, elm, field) do { \ + if ((STAILQ_NEXT((elm), field) = STAILQ_FIRST((head))) == NULL) \ + (head)->stqh_last = &STAILQ_NEXT((elm), field); \ + STAILQ_FIRST((head)) = (elm); \ +} while (0) + +#define STAILQ_INSERT_TAIL (head, elm, field) do { \ + STAILQ_NEXT((elm), field) = NULL; \ + *(head)->stqh_last = (elm); \ + (head)->stqh_last = &STAILQ_NEXT((elm), field); \ +} while (0) + +#define STAILQ_LAST (head, type, field) \ + (STAILQ_EMPTY((head)) ? NULL : \ + __containerof((head)->stqh_last, \ + QUEUE_TYPEOF(type), field.stqe_next)) + +#define STAILQ_NEXT (elm, field) ((elm)->field.stqe_next) + +#define STAILQ_REMOVE (head, elm, type, field) do { \ + QMD_SAVELINK(oldnext, (elm)->field.stqe_next); \ + if (STAILQ_FIRST((head)) == (elm)) { \ + STAILQ_REMOVE_HEAD((head), field); \ + } \ + else { \ + QUEUE_TYPEOF(type) *curelm = STAILQ_FIRST(head); \ + while (STAILQ_NEXT(curelm, field) != (elm)) \ + curelm = STAILQ_NEXT(curelm, field); \ + STAILQ_REMOVE_AFTER(head, curelm, field); \ + } \ + TRASHIT(*oldnext); \ +} while (0) + +#define STAILQ_REMOVE_AFTER (head, elm, field) do { \ + if ((STAILQ_NEXT(elm, field) = \ + STAILQ_NEXT(STAILQ_NEXT(elm, field), field)) == NULL) \ + (head)->stqh_last = &STAILQ_NEXT((elm), field); \ +} while (0) + +#define STAILQ_REMOVE_HEAD (head, field) do { \ + if ((STAILQ_FIRST((head)) = \ + STAILQ_NEXT(STAILQ_FIRST((head)), field)) == NULL) \ + (head)->stqh_last = &STAILQ_FIRST((head)); \ +} while (0) + +#define STAILQ_SWAP (head1, head2, type) do { \ + QUEUE_TYPEOF(type) *swap_first = STAILQ_FIRST(head1); \ + QUEUE_TYPEOF(type) **swap_last = (head1)->stqh_last; \ + STAILQ_FIRST(head1) = STAILQ_FIRST(head2); \ + (head1)->stqh_last = (head2)->stqh_last; \ + STAILQ_FIRST(head2) = swap_first; \ + (head2)->stqh_last = swap_last; \ + if (STAILQ_EMPTY(head1)) \ + (head1)->stqh_last = &STAILQ_FIRST(head1); \ + if (STAILQ_EMPTY(head2)) \ + (head2)->stqh_last = &STAILQ_FIRST(head2); \ +} while (0) + + +/* +* List declarations. +*/ +#define LIST_HEAD(name, type) \ +struct name { \ + struct type *lh_first; /* first element */ \ +} + +#define LIST_CLASS_HEAD(name, type) \ +struct name { \ + class type *lh_first; /* first element */ \ +} + +#define LIST_HEAD_INITIALIZER (head) \ + { NULL } + +#define LIST_ENTRY (type) \ +struct { \ + struct type *le_next;/* next element */ \ +struct type **le_prev;/* address of previous next element */ \ +} + +#define LIST_CLASS_ENTRY(type) \ +struct { \ + class type *le_next; /* next element */ \ + class type **le_prev; /* address of previous next element */ \ +} + +/* +* List functions. +*/ + +#if (defined(_KERNEL) && defined(INVARIANTS )) +/* +* QMD_LIST_CHECK_HEAD(LIST_HEAD *head, LIST_ENTRY NAME) +* +* If the list is non-empty, validates that the first element of the list +* points back at 'head.' +*/ +#define QMD_LIST_CHECK_HEAD (head, field) do { \ + if (LIST_FIRST((head)) != NULL && \ + LIST_FIRST((head))->field.le_prev != \ + &LIST_FIRST((head))) \ + panic("Bad list head %p first->prev != head", (head)); \ +} while (0) + +/* +* QMD_LIST_CHECK_NEXT(TYPE *elm, LIST_ENTRY NAME) +* +* If an element follows 'elm' in the list, validates that the next element +* points back at 'elm.' +*/ +#define QMD_LIST_CHECK_NEXT (elm, field) do { \ + if (LIST_NEXT((elm), field) != NULL && \ + LIST_NEXT((elm), field)->field.le_prev != \ + &((elm)->field.le_next)) \ + panic("Bad link elm %p next->prev != elm", (elm)); \ +} while (0) + +/* +* QMD_LIST_CHECK_PREV(TYPE *elm, LIST_ENTRY NAME) +* +* Validates that the previous element (or head of the list) points to 'elm.' +*/ +#define QMD_LIST_CHECK_PREV (elm, field) do { \ + if (*(elm)->field.le_prev != (elm)) \ + panic("Bad link elm %p prev->next != elm", (elm)); \ +} while (0) +#else +#define QMD_LIST_CHECK_HEAD (head, field) +#define QMD_LIST_CHECK_NEXT (elm, field) +#define QMD_LIST_CHECK_PREV (elm, field) +#endif /* (_KERNEL && INVARIANTS) */ + +#define LIST_CONCAT (head1, head2, type, field) do { \ + QUEUE_TYPEOF(type) *curelm = LIST_FIRST(head1); \ + if (curelm == NULL) { \ + if ((LIST_FIRST(head1) = LIST_FIRST(head2)) != NULL) { \ + LIST_FIRST(head2)->field.le_prev = \ + &LIST_FIRST((head1)); \ + LIST_INIT(head2); \ + } \ + } else if (LIST_FIRST(head2) != NULL) { \ + while (LIST_NEXT(curelm, field) != NULL) \ + curelm = LIST_NEXT(curelm, field); \ + LIST_NEXT(curelm, field) = LIST_FIRST(head2); \ + LIST_FIRST(head2)->field.le_prev = &LIST_NEXT(curelm, field); \ + LIST_INIT(head2); \ + } \ +} while (0) + +#define LIST_EMPTY (head) ((head)->lh_first == NULL) + +#define LIST_FIRST (head) ((head)->lh_first) + +#define LIST_FOREACH (var, head, field) \ + for ((var) = LIST_FIRST((head)); \ + (var); \ + (var) = LIST_NEXT((var), field)) + +#define LIST_FOREACH_FROM (var, head, field) \ + for ((var) = ((var) ? (var) : LIST_FIRST((head))); \ + (var); \ + (var) = LIST_NEXT((var), field)) + +#define LIST_FOREACH_SAFE (var, head, field, tvar) \ + for ((var) = LIST_FIRST((head)); \ + (var) && ((tvar) = LIST_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define LIST_FOREACH_FROM_SAFE (var, head, field, tvar) \ + for ((var) = ((var) ? (var) : LIST_FIRST((head))); \ + (var) && ((tvar) = LIST_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define LIST_INIT (head) do { \ + LIST_FIRST((head)) = NULL; \ +} while (0) + +#define LIST_INSERT_AFTER (listelm, elm, field) do { \ + QMD_LIST_CHECK_NEXT(listelm, field); \ + if ((LIST_NEXT((elm), field) = LIST_NEXT((listelm), field)) != NULL)\ + LIST_NEXT((listelm), field)->field.le_prev = \ + &LIST_NEXT((elm), field); \ + LIST_NEXT((listelm), field) = (elm); \ + (elm)->field.le_prev = &LIST_NEXT((listelm), field); \ +} while (0) + +#define LIST_INSERT_BEFORE (listelm, elm, field) do { \ + QMD_LIST_CHECK_PREV(listelm, field); \ + (elm)->field.le_prev = (listelm)->field.le_prev; \ + LIST_NEXT((elm), field) = (listelm); \ + *(listelm)->field.le_prev = (elm); \ + (listelm)->field.le_prev = &LIST_NEXT((elm), field); \ +} while (0) + +#define LIST_INSERT_HEAD (head, elm, field) do { \ + QMD_LIST_CHECK_HEAD((head), field); \ + if ((LIST_NEXT((elm), field) = LIST_FIRST((head))) != NULL) \ + LIST_FIRST((head))->field.le_prev = &LIST_NEXT((elm), field);\ + LIST_FIRST((head)) = (elm); \ + (elm)->field.le_prev = &LIST_FIRST((head)); \ +} while (0) + +#define LIST_NEXT (elm, field) ((elm)->field.le_next) + +#define LIST_PREV (elm, head, type, field) \ + ((elm)->field.le_prev == &LIST_FIRST((head)) ? NULL : \ + __containerof((elm)->field.le_prev, \ + QUEUE_TYPEOF(type), field.le_next)) + +#define LIST_REMOVE (elm, field) do { \ + QMD_SAVELINK(oldnext, (elm)->field.le_next); \ + QMD_SAVELINK(oldprev, (elm)->field.le_prev); \ + QMD_LIST_CHECK_NEXT(elm, field); \ + QMD_LIST_CHECK_PREV(elm, field); \ + if (LIST_NEXT((elm), field) != NULL) \ + LIST_NEXT((elm), field)->field.le_prev = \ + (elm)->field.le_prev; \ + *(elm)->field.le_prev = LIST_NEXT((elm), field); \ + TRASHIT(*oldnext); \ + TRASHIT(*oldprev); \ +} while (0) + +#define LIST_SWAP (head1, head2, type, field) do { \ + QUEUE_TYPEOF(type) *swap_tmp = LIST_FIRST(head1); \ + LIST_FIRST((head1)) = LIST_FIRST((head2)); \ + LIST_FIRST((head2)) = swap_tmp; \ + if ((swap_tmp = LIST_FIRST((head1))) != NULL) \ + swap_tmp->field.le_prev = &LIST_FIRST((head1)); \ + if ((swap_tmp = LIST_FIRST((head2))) != NULL) \ + swap_tmp->field.le_prev = &LIST_FIRST((head2)); \ +} while (0) +/* * Tail queue declarations. */
@@ -123,10 +619,58 @@ struct { \/* * Tail queue functions. */ +#if (defined(_KERNEL) && defined(INVARIANTS )) +/* +* QMD_TAILQ_CHECK_HEAD(TAILQ_HEAD *head, TAILQ_ENTRY NAME) +* +* If the tailq is non-empty, validates that the first element of the tailq +* points back at 'head.' +*/ +#define QMD_TAILQ_CHECK_HEAD (head, field) do { \ + if (!TAILQ_EMPTY(head) && \ + TAILQ_FIRST((head))->field.tqe_prev != \ + &TAILQ_FIRST((head))) \ + panic("Bad tailq head %p first->prev != head", (head)); \ +} while (0) + +/* +* QMD_TAILQ_CHECK_TAIL(TAILQ_HEAD *head, TAILQ_ENTRY NAME) +* +* Validates that the tail of the tailq is a pointer to pointer to NULL. +*/ +#define QMD_TAILQ_CHECK_TAIL (head, field) do { \ + if (*(head)->tqh_last != NULL) \ + panic("Bad tailq NEXT(%p->tqh_last) != NULL", (head)); \ +} while (0) + +/* +* QMD_TAILQ_CHECK_NEXT(TYPE *elm, TAILQ_ENTRY NAME) +* +* If an element follows 'elm' in the tailq, validates that the next element +* points back at 'elm.' +*/ +#define QMD_TAILQ_CHECK_NEXT (elm, field) do { \ + if (TAILQ_NEXT((elm), field) != NULL && \ + TAILQ_NEXT((elm), field)->field.tqe_prev != \ + &((elm)->field.tqe_next)) \ + panic("Bad link elm %p next->prev != elm", (elm)); \ +} while (0) + +/* +* QMD_TAILQ_CHECK_PREV(TYPE *elm, TAILQ_ENTRY NAME) +* +* Validates that the previous element (or head of the tailq) points to 'elm.' +*/ +#define QMD_TAILQ_CHECK_PREV (elm, field) do { \ + if (*(elm)->field.tqe_prev != (elm)) \ + panic("Bad link elm %p prev->next != elm", (elm)); \ +} while (0) +#else #define QMD_TAILQ_CHECK_HEAD (head, field)#define QMD_TAILQ_CHECK_TAIL (head, headname)#define QMD_TAILQ_CHECK_NEXT (elm, field)#define QMD_TAILQ_CHECK_PREV (elm, field) +#endif /* (_KERNEL && INVARIANTS) */ #define TAILQ_CONCAT (head1, head2, field) do { \ if (!TAILQ_EMPTY(head2)) { \
@@ -191,9 +735,8 @@ struct { \#define TAILQ_INSERT_AFTER (head, listelm, elm, field) do { \ QMD_TAILQ_CHECK_NEXT(listelm, field); \ - TAILQ_NEXT((elm), field) = TAILQ_NEXT((listelm), field); \ - if (TAILQ_NEXT((listelm), field) != NULL) \ - TAILQ_NEXT((elm), field)->field.tqe_prev = \ + if ((TAILQ_NEXT((elm), field) = TAILQ_NEXT((listelm), field)) != NULL)\ + TAILQ_NEXT((elm), field)->field.tqe_prev = \ &TAILQ_NEXT((elm), field); \ else { \ (head)->tqh_last = &TAILQ_NEXT((elm), field); \
@@ -217,8 +760,7 @@ struct { \#define TAILQ_INSERT_HEAD (head, elm, field) do { \ QMD_TAILQ_CHECK_HEAD(head, field); \ - TAILQ_NEXT((elm), field) = TAILQ_FIRST((head)); \ - if (TAILQ_FIRST((head)) != NULL) \ + if ((TAILQ_NEXT((elm), field) = TAILQ_FIRST((head))) != NULL) \ TAILQ_FIRST((head))->field.tqe_prev = \ &TAILQ_NEXT((elm), field); \ else \
@@ -250,21 +792,24 @@ struct { \* you may want to prefetch the last data element. */ #define TAILQ_LAST_FAST (head, type, field) \ - (TAILQ_EMPTY(head) ? NULL : __containerof((head)->tqh_last, \ - QUEUE_TYPEOF(type), field.tqe_next)) + (TAILQ_EMPTY(head) ? NULL : __containerof((head)->tqh_last, QUEUE_TYPEOF(type), field.tqe_next))#define TAILQ_NEXT (elm, field) ((elm)->field.tqe_next)#define TAILQ_PREV (elm, headname, field) \ (*(((struct headname *)((elm)->field.tqe_prev))->tqh_last)) +#define TAILQ_PREV_FAST (elm, head, type, field) \ + ((elm)->field.tqe_prev == &(head)->tqh_first ? NULL : \ + __containerof((elm)->field.tqe_prev, QUEUE_TYPEOF(type), field.tqe_next)) +#define TAILQ_REMOVE (head, elm, field) do { \ QMD_SAVELINK(oldnext, (elm)->field.tqe_next); \ QMD_SAVELINK(oldprev, (elm)->field.tqe_prev); \ QMD_TAILQ_CHECK_NEXT(elm, field); \ QMD_TAILQ_CHECK_PREV(elm, field); \ if ((TAILQ_NEXT((elm), field)) != NULL) \ - TAILQ_NEXT((elm), field)->field.tqe_prev = \ + TAILQ_NEXT((elm), field)->field.tqe_prev = \ (elm)->field.tqe_prev; \ else { \ (head)->tqh_last = (elm)->field.tqe_prev; \
@@ -277,26 +822,20 @@ struct { \ } while (0)#define TAILQ_SWAP (head1, head2, type, field) do { \ - QUEUE_TYPEOF(type) * swap_first = (head1)->tqh_first; \ - QUEUE_TYPEOF(type) * *swap_last = (head1)->tqh_last; \ + QUEUE_TYPEOF(type) *swap_first = (head1)->tqh_first; \ + QUEUE_TYPEOF(type) **swap_last = (head1)->tqh_last; \ (head1)->tqh_first = (head2)->tqh_first; \ (head1)->tqh_last = (head2)->tqh_last; \ (head2)->tqh_first = swap_first; \ (head2)->tqh_last = swap_last; \ - swap_first = (head1)->tqh_first; \ - if (swap_first != NULL) \ + if ((swap_first = (head1)->tqh_first) != NULL) \ swap_first->field.tqe_prev = &(head1)->tqh_first; \ else \ (head1)->tqh_last = &(head1)->tqh_first; \ - swap_first = (head2)->tqh_first; \ - if (swap_first != NULL) \ + if ((swap_first = (head2)->tqh_first) != NULL) \ swap_first->field.tqe_prev = &(head2)->tqh_first; \ else \ (head2)->tqh_last = &(head2)->tqh_first; \ } while (0) -#ifdef __cplusplus -} -#endif - -#endif /* _SYS_QUEUE_H_ */ +#endif /* !_SYS_QUEUE_H_ */
Signature · 1 line
2.25.4
PATCH v5 9/11 eal/windows: improve CPU and NUMA node detection
In reply to Dmitry Kozlyuk1. Map CPU cores to their respective NUMA nodes as reported by system.
2. Support systems with more than 64 cores (multiple processor groups).
3. Fix magic constants, styling issues, and compiler warnings.
4. Add EAL private function to map DPDK socket ID to NUMA node number.
--- lib/librte_eal/windows/eal_lcore.c | 185 +++++++++++++++++---------- lib/librte_eal/windows/eal_windows.h | 10 ++ 2 files changed, 124 insertions(+), 71 deletions(-)
lib/librte_eal/windows/eal_lcore.c +114 -71
@@ -3,103 +3,146 @@*/ #include <pthread.h> +#include <stdbool.h> #include <stdint.h> #include <rte_common.h> +#include <rte_debug.h> +#include <rte_lcore.h> +#include <rte_os.h> #include "eal_private.h" #include "eal_thread.h" #include "eal_windows.h" -/* global data structure that contains the CPU map */ -static struct _wcpu_map { -unsigned int total_procs; -unsigned int proc_sockets; -unsigned int proc_cores; -unsigned int reserved; -struct _win_lcore_map { -uint8_t socket_id; -uint8_t core_id; - } wlcore_map[RTE_MAX_LCORE ]; -} wcpu_map = {0 }; - -/* -* Create a map of all processors and associated cores on the system -*/ +/** Number of logical processors (cores) in a processor group (32 or 64). */ +#define EAL_PROCESSOR_GROUP_SIZE (sizeof(KAFFINITY) * CHAR_BIT) + +struct lcore_map { +uint8_t socket_id; +uint8_t core_id; +}; + +struct socket_map { +uint16_t node_id; +}; + +struct cpu_map { +unsigned int socket_count; +unsigned int lcore_count; +struct lcore_map lcores[RTE_MAX_LCORE ]; +struct socket_map sockets[RTE_MAX_NUMA_NODES ]; +}; + +static struct cpu_map cpu_map = {0 }; +void -eal_create_cpu_map () +eal_create_cpu_map (void ) { - wcpu_map.total_procs = -GetActiveProcessorCount (ALL_PROCESSOR_GROUPS ); - -LOGICAL_PROCESSOR_RELATIONSHIP lprocRel; -DWORD lprocInfoSize =0 ; -BOOL ht_enabled = FALSE; - -/* First get the processor package information */ - lprocRel = RelationProcessorPackage; -/* Determine the size of buffer we need (pass NULL) */ -GetLogicalProcessorInformationEx (lprocRel,NULL , &lprocInfoSize); - wcpu_map.proc_sockets = lprocInfoSize /48 ; - - lprocInfoSize =0 ; -/* Next get the processor core information */ - lprocRel = RelationProcessorCore; -GetLogicalProcessorInformationEx (lprocRel,NULL , &lprocInfoSize); - wcpu_map.proc_cores = lprocInfoSize /48 ; - -if (wcpu_map.total_procs > wcpu_map.proc_cores) - ht_enabled = TRUE; - -/* Distribute the socket and core ids appropriately -* across the logical cores. For now, split the cores -* equally across the sockets. -*/ -unsigned int lcore =0 ; -for (unsigned int socket =0 ; socket < - wcpu_map.proc_sockets; ++socket) { -for (unsigned int core =0 ; - core < (wcpu_map.proc_cores / wcpu_map.proc_sockets); - ++core) { - wcpu_map.wlcore_map[lcore] - .socket_id = socket; - wcpu_map.wlcore_map[lcore] - .core_id = core; - lcore++; -if (ht_enabled) { - wcpu_map.wlcore_map[lcore] - .socket_id = socket; - wcpu_map.wlcore_map[lcore] - .core_id = core; - lcore++; +SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *infos, *info; +DWORD infos_size; +bool full = false; + + infos_size =0 ; +if (!GetLogicalProcessorInformationEx ( + RelationNumaNode,NULL , &infos_size)) { +DWORD error =GetLastError (); +if (error !=ERROR_INSUFFICIENT_BUFFER ) { +rte_panic ("cannot get NUMA node info size, error %lu" , +GetLastError ()); + } + } + + infos =malloc (infos_size); +if (infos ==NULL ) { +rte_panic ("cannot allocate memory for NUMA node information" ); +return ; + } + +if (!GetLogicalProcessorInformationEx ( + RelationNumaNode, infos, &infos_size)) { +rte_panic ("cannot get NUMA node information, error %lu" , +GetLastError ()); + } + + info = infos; +while ((uint8_t *)info - (uint8_t *)infos < infos_size) { +unsigned int node_id = info->NumaNode.NodeNumber; +GROUP_AFFINITY *cores = &info->NumaNode.GroupMask; +struct lcore_map *lcore; +unsigned int i, socket_id; + +/* NUMA node may be reported multiple times if it includes +* cores from different processor groups, e. g. 80 cores +* of a physical processor comprise one NUMA node, but two +* processor groups, because group size is limited by 32/64. +*/ +for (socket_id =0 ; socket_id < cpu_map.socket_count; + socket_id++) { +if (cpu_map.sockets[socket_id].node_id == node_id) +break ; + } + +if (socket_id == cpu_map.socket_count) { +if (socket_id ==RTE_DIM (cpu_map.sockets)) { + full = true; + goto exit; } + + cpu_map.sockets[socket_id].node_id = node_id; + cpu_map.socket_count++; + } + +for (i =0 ; i <EAL_PROCESSOR_GROUP_SIZE ; i++) { +if ((cores->Mask & ((KAFFINITY )1 << i)) ==0 ) +continue ; + +if (cpu_map.lcore_count ==RTE_DIM (cpu_map.lcores)) { + full = true; + goto exit; + } + + lcore = &cpu_map.lcores[cpu_map.lcore_count]; + lcore->socket_id = socket_id; + lcore->core_id = + cores->Group *EAL_PROCESSOR_GROUP_SIZE + i; + cpu_map.lcore_count++; } + + info = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)( + (uint8_t *)info + info->Size); + } + +exit: +if (full) { +/* RTE_LOG() may not be available, but this is important. */ +fprintf (stderr,"Enumerated maximum of %u NUMA nodes and %u cores\n" , + cpu_map.socket_count, cpu_map.lcore_count); } + +free (infos); } -/* -* Check if a cpu is present by the presence of the cpu information for it -*/ int eal_cpu_detected (unsigned int lcore_id) { -return (lcore_id < wcpu_map.total_procs); +return lcore_id < cpu_map.lcore_count; } -/* -* Get CPU socket id for a logical core -*/ unsigned eal_cpu_socket_id (unsigned int lcore_id) { -return wcpu_map.wlcore_map[lcore_id].socket_id; +return cpu_map.lcores[lcore_id].socket_id; } -/* -* Get CPU socket id (NUMA node) for a logical core -*/ unsigned eal_cpu_core_id (unsigned int lcore_id) { -return wcpu_map.wlcore_map[lcore_id].core_id; +return cpu_map.lcores[lcore_id].core_id; +} + +unsigned int +eal_socket_numa_node (unsigned int socket_id) +{ +return cpu_map.sockets[socket_id].node_id; }
lib/librte_eal/windows/eal_windows.h +10 -0
@@ -26,4 +26,14 @@ void eal_create_cpu_map(void);*/ int eal_thread_create (pthread_t *thread); +/** +* Get system NUMA node number for a socket ID. +* +* @param socket_id +* Valid EAL socket ID. +* @return +* NUMA node number to use with Win32 API. +*/ +unsigned int eal_socket_numa_node (unsigned int socket_id); +#endif /* _EAL_WINDOWS_H_ */
Signature · 1 line
2.25.4
PATCH v5 10/11 eal/windows: initialize hugepage info
In reply to Dmitry KozlyukAdd hugepages discovery ("large pages" in Windows terminology) and update documentation for required privilege setup. Only 2MB hugepages are supported and their number is estimated roughly due to the lack or unstable status of suitable OS APIs.
--- config/meson.build | 2 + doc/guides/windows_gsg/build_dpdk.rst | 20 ----- doc/guides/windows_gsg/index.rst | 1 + doc/guides/windows_gsg/run_apps.rst | 47 +++++++++++ lib/librte_eal/windows/eal.c | 14 ++++ lib/librte_eal/windows/eal_hugepages.c | 108 +++++++++++++++++++++++++ lib/librte_eal/windows/meson.build | 1 + 7 files changed, 173 insertions(+), 20 deletions(-) create mode 100644 doc/guides/windows_gsg/run_apps.rst create mode 100644 lib/librte_eal/windows/eal_hugepages.c
config/meson.build +2 -0
@@ -268,6 +268,8 @@ if is_windows if cc.get_id() == 'gcc' add_project_arguments('-D__USE_MINGW_ANSI_STDIO', language: 'c') endif + + add_project_link_arguments('-ladvapi32', language: 'c') endif if get_option('b_lto')
doc/guides/windows_gsg/build_dpdk.rst +0 -20
@@ -111,23 +111,3 @@ Depending on the distribution, paths in this file may need adjustments. meson --cross-file config/x86/meson_mingw.txt -Dexamples=helloworld build ninja -C build - - -Run the helloworld example -========================== - -Navigate to the examples in the build directory and run `dpdk-helloworld.exe`. - -.. code-block:: console - - cd C:\Users\me\dpdk\build\examples - dpdk-helloworld.exe - hello from core 1 - hello from core 3 - hello from core 0 - hello from core 2 - -Note for MinGW-w64: applications are linked to ``libwinpthread-1.dll`` -by default. To run the example, either add toolchain executables directory -to the PATH or copy the library to the working directory. -Alternatively, static linking may be used (mind the LGPLv2.1 license).
doc/guides/windows_gsg/index.rst +1 -0
@@ -12,3 +12,4 @@ Getting Started Guide for Windows intro build_dpdk + run_apps
doc/guides/windows_gsg/run_apps.rst new file +47 -0
47 lines not on this page · raw message
lib/librte_eal/windows/eal.c +14 -0
@@ -19,8 +19,11 @@#include <eal_private.h> #include <rte_trace_point.h> +#include "eal_hugepages.h" #include "eal_windows.h" +#define MEMSIZE_IF_NO_HUGE_PAGE (64ULL * 1024ULL * 1024ULL) +/* Allow the application to print its usage message too if set */ static rte_usage_hook_t rte_application_usage_hook;
@@ -276,6 +279,17 @@ rte_eal_init(int argc, char **argv)if (fctret <0 )exit (1 ); +if (!internal_config.no_hugetlbfs && (eal_hugepage_info_init () <0 )) { +rte_eal_init_alert ("Cannot get hugepage information" ); + rte_errno =EACCES ; +return -1 ; + } + +if (internal_config.memory ==0 && !internal_config.force_sockets) { +if (internal_config.no_hugetlbfs) + internal_config.memory =MEMSIZE_IF_NO_HUGE_PAGE ; + } +eal_thread_init_master (rte_config.master_lcore);RTE_LCORE_FOREACH_SLAVE (i) {
lib/librte_eal/windows/eal_hugepages.c new file +108 -0
108 lines not on this page · raw message
lib/librte_eal/windows/meson.build +1 -0
@@ -6,6 +6,7 @@ subdir('include') sources += files( 'eal.c', 'eal_debug.c', + 'eal_hugepages.c', 'eal_lcore.c', 'eal_log.c', 'eal_thread.c',
Signature · 1 line
2.25.4
PATCH v5 11/11 eal/windows: implement basic memory management
In reply to Dmitry KozlyukBasic memory management supports core libraries and PMDs operating in IOVA as PA mode. It uses a kernel-mode driver, virt2phys, to obtain IOVAs of hugepages allocated from user-mode. Multi-process mode is not implemented and is forcefully disabled at startup.
--- config/meson.build | 12 +- doc/guides/windows_gsg/run_apps.rst | 54 +- lib/librte_eal/common/meson.build | 11 + lib/librte_eal/common/rte_malloc.c | 9 + lib/librte_eal/rte_eal_exports.def | 119 +++ lib/librte_eal/windows/eal.c | 146 +++- lib/librte_eal/windows/eal_memalloc.c | 442 +++++++++++ lib/librte_eal/windows/eal_memory.c | 710 ++++++++++++++++++ lib/librte_eal/windows/eal_mp.c | 103 +++ lib/librte_eal/windows/eal_windows.h | 75 ++ lib/librte_eal/windows/include/meson.build | 1 + lib/librte_eal/windows/include/rte_os.h | 9 + .../windows/include/rte_virt2phys.h | 34 + lib/librte_eal/windows/include/rte_windows.h | 2 + lib/librte_eal/windows/include/unistd.h | 3 + lib/librte_eal/windows/meson.build | 5 + 16 files changed, 1728 insertions(+), 7 deletions(-) create mode 100644 lib/librte_eal/windows/eal_memalloc.c create mode 100644 lib/librte_eal/windows/eal_memory.c create mode 100644 lib/librte_eal/windows/eal_mp.c create mode 100644 lib/librte_eal/windows/include/rte_virt2phys.h
config/meson.build +9 -3
@@ -261,15 +261,21 @@ if is_freebsd endif if is_windows - # Minimum supported API is Windows 7. - add_project_arguments('-D_WIN32_WINNT=0x0601', language: 'c') + # VirtualAlloc2() is available since Windows 10 / Server 2016. + add_project_arguments('-D_WIN32_WINNT=0x0A00', language: 'c') # Use MinGW-w64 stdio, because DPDK assumes ANSI-compliant formatting. if cc.get_id() == 'gcc' add_project_arguments('-D__USE_MINGW_ANSI_STDIO', language: 'c') endif - add_project_link_arguments('-ladvapi32', language: 'c') + # Contrary to docs, VirtualAlloc2() is exported by mincore.lib + # in Windows SDK, while MinGW exports it by advapi32.a. + if is_ms_linker + add_project_link_arguments('-lmincore', language: 'c') + endif + + add_project_link_arguments('-ladvapi32', '-lsetupapi', language: 'c') endif if get_option('b_lto')
doc/guides/windows_gsg/run_apps.rst +51 -3
68 lines not on this page · raw message
lib/librte_eal/common/meson.build +11 -0
@@ -8,13 +8,24 @@ if is_windows 'eal_common_bus.c', 'eal_common_class.c', 'eal_common_devargs.c', + 'eal_common_dynmem.c', 'eal_common_errno.c', + 'eal_common_fbarray.c', 'eal_common_launch.c', 'eal_common_lcore.c', 'eal_common_log.c', + 'eal_common_mcfg.c', + 'eal_common_memalloc.c', + 'eal_common_memory.c', + 'eal_common_memzone.c', 'eal_common_options.c', + 'eal_common_string_fns.c', + 'eal_common_tailqs.c', 'eal_common_thread.c', 'eal_common_trace_points.c', + 'malloc_elem.c', + 'malloc_heap.c', + 'rte_malloc.c', ) subdir_done() endif
lib/librte_eal/common/rte_malloc.c +9 -0
@@ -20,7 +20,16 @@#include <rte_lcore.h> #include <rte_common.h> #include <rte_spinlock.h> + +#ifndef RTE_EXEC_ENV_WINDOWS #include <rte_eal_trace.h> +#else +/* Suppress -Wempty-body for tracepoints used as "if" body. */ +#define rte_eal_trace_mem_malloc (...) do {} while (0) +#define rte_eal_trace_mem_zmalloc (...) do {} while (0) +#define rte_eal_trace_mem_realloc (...) do {} while (0) +#define rte_eal_trace_mem_free (...) do {} while (0) +#endif #include <rte_malloc.h> #include "malloc_elem.h"
lib/librte_eal/rte_eal_exports.def +119 -0
128 lines not on this page · raw message
lib/librte_eal/windows/eal.c +145 -1
176 lines not on this page · raw message
lib/librte_eal/windows/eal_memalloc.c new file +442 -0
442 lines not on this page · raw message
lib/librte_eal/windows/eal_memory.c new file +710 -0
710 lines not on this page · raw message
lib/librte_eal/windows/eal_mp.c new file +103 -0
103 lines not on this page · raw message
lib/librte_eal/windows/eal_windows.h +75 -0
87 lines not on this page · raw message
lib/librte_eal/windows/include/meson.build +1 -0
@@ -5,5 +5,6 @@ includes += include_directories('.') headers += files( 'rte_os.h', + 'rte_virt2phys.h', 'rte_windows.h', )
lib/librte_eal/windows/include/rte_os.h +9 -0
@@ -36,6 +36,15 @@ extern "C" {#define strncasecmp (s1, s2, count) _strnicmp(s1, s2, count) +/* MinGW-w64 7.0.0 defines _open() and open() as inlines, 6.0.0 doesn't. +* Other pairs of functions are only defined, not declared. +*/ +#if !definedRTE_TOOLCHAIN_GCC || definedNO_OLDNAMES +#define open _open +#endif +#define close _close +#define unlink _unlink +/* cpu_set macros implementation */ #define RTE_CPU_AND (dst, src1, src2) CPU_AND(dst, src1, src2)#define RTE_CPU_OR (dst, src1, src2) CPU_OR(dst, src1, src2)
lib/librte_eal/windows/include/rte_virt2phys.h new file +34 -0
@@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright (c) 2020 Dmitry Kozlyuk +*/ + +/** +* @file virt2phys driver interface +*/ + +/** +* Driver device interface GUID {539c2135-793a-4926-afec-d3a1b61bbc8a}. +*/ +DEFINE_GUID (GUID_DEVINTERFACE_VIRT2PHYS , +0x539c2135 ,0x793a ,0x4926 , +0xaf ,0xec ,0xd3 ,0xa1 ,0xb6 ,0x1b ,0xbc ,0x8a ); + +/** +* Driver device type for IO control codes. +*/ +#define VIRT2PHYS_DEVTYPE 0x8000 + +/** +* Translate a valid non-paged virtual address to a physical address. +* +* Note: A physical address zero (0) is reported if input address +* is paged out or not mapped. However, if input is a valid mapping +* of I/O port 0x0000, output is also zero. There is no way +* to distinguish between these cases by return value only. +* +* Input: a non-paged virtual address (PVOID). +* +* Output: the corresponding physical address (LARGE_INTEGER). +*/ +#define IOCTL_VIRT2PHYS_TRANSLATE CTL_CODE( \ + VIRT2PHYS_DEVTYPE, 0x800, METHOD_BUFFERED, FILE_ANY_ACCESS)
lib/librte_eal/windows/include/rte_windows.h +2 -0
@@ -23,6 +23,8 @@#include <basetsd.h> #include <psapi.h> +#include <setupapi.h> +#include <winioctl.h> /* Have GUIDs defined. */ #ifndef INITGUID
lib/librte_eal/windows/include/unistd.h +3 -0
@@ -9,4 +9,7 @@* as Microsoft libc does not contain unistd.h. This may be removed * in future releases. */ + +#include <io.h> +#endif /* _UNISTD_H_ */
lib/librte_eal/windows/meson.build +5 -0
@@ -9,7 +9,12 @@ sources += files( 'eal_hugepages.c', 'eal_lcore.c', 'eal_log.c', + 'eal_memalloc.c', + 'eal_memory.c', + 'eal_mp.c', 'eal_thread.c', 'fnmatch.c', 'getopt.c', ) + +dpdk_conf.set10('RTE_EAL_NUMA_AWARE_HUGEPAGES', true)
Signature · 1 line
2.25.4
v6 cover Windows basic memory management
In reply to Dmitry KozlyukThis patchset implements basic MM with the following features:
* Hugepages are dynamically allocated in user-mode.
* Only 2MB hugepages are supported.
* IOVA is always PA, obtained through kernel-mode driver.
* No 32-bit support (presumably not demanded).
* Ni multi-process support (it is forcefully disabled).
* No-huge mode for testing without IOVA is available.
Testing revealed Windows Server 2019 does not allow allocating hugepage memory at a reserved address, despite advertised API. So allocator has to temporary free the region to be allocated. This creates in inherent race condition. This issue is being discussed with Microsoft privately.
New EAL public functions for memory mapping are introduced to mitigate OS differences in DPDK libraries and applications: rte_mem_map, rte_mem_unmap, rte_mem_lock, rte_mem_page_size.
To support common MM routines, internal wrappers for low-level memory reservation and file management are introduced. These changes affect Linux and FreeBSD EAL. Shared code is placed unded /unix/ subdirectory (suggested by Thomas).
To avoid code duplication between Linux and Windows EAL, common code for EALs supporting dynamic memory allocation is extracted (discussed with Anatoly Burakov in v4 thread). This is a separate patch to ease the review, but it can be merged with the previous one.
EAL tracepoints save size_t values as long, which is invalid on Windows. New size_t emitter for tracepoints is introduced (suggested by Jerin Jacob to Fady Bader, see [1]). Also, to avoid workaround in every file using the tracepoints, stubs are added to Windows EAL.
Entire <sys/queue.h> is imported from FreeBSD, replacing existing partial import. There is already a license exception for this file. The file is imported as-is, so it causes a bunch of checkpatch warnings.
[1]: http://mails.dpdk.org/archives/dev/2020-May/168076.html
---
v6:
* Fix 32-bit build on x86 (CI).
* Fix Makefile build (Anatoly Burakov, Thomas Monjalon).
* Restore 32-bit common code (Anatoly Burakov).
* Fix error reporting in memory management (Anatoly Burakov).
* Add Doxygen comment for size_t tracepoint emitter (Jerin Jacob).
* Update MAINTAINERS for new files and new code (Thomas Monjalon).
* Rename rte_get_page_size to rte_mem_page_size.
* Mark DPDK-only wrappers internal, move them to separate file.
* Get rid of warnings in enabled common code with Clang on Windows.
v5:
* Fix allocation and deallocation on Windows Server (Fady Bader).
* Replace remaining VirtualFree with VirtualFreeEx (Ranjit Menon).
* Fix errors in eal_get_virtual_area (Anatoly Burakov).
* Fix error handling and documentation for rte_mem_lock (Anatoly Burakov).
* Extract common code for EALs w/dynamic allocation (Anatoly Burakov).
* Use POSIX value for rte_errno after rte_mem_unmap() on Windows.
* Add stubs to use tracing functions without workarounds.
v4:
* Rebase on ToT, drop patches merged into master.
* Rearrange patches to split Windows code (Jerin).
* Fix Linux and FreeBSD build with make (Ophir).
* Use int instead of enum to hold a set of flags (Anatoly).
* Rename eal_mem_reserve items and fix their description (Anatoly).
* Add eal_mem_set_dump() wrapper around madvise (Anatoly).
* Don't claim Windows Server 2016 support due to lack of API (Tal).
* Replace enum rte_page_sizes with a set of #defines (Jerin).
* Fix documentation, SPDX tags, logging (Thomas).
v3:
* Fix Linux build on and aarch64 and 32-bit x86 (reported by CI).
* Fix logic and error handling while allocating segments.
* Fix Unix rte_mem_map(): return NULL on failure.
* Fix some checkpatch.sh issues:
* Do not return positive errno, use DWORD for GetLastError().
* Make dpdk-kmods source files non-executable.
* Improve GSG for Windows Server (suggested by Ranjit Menon).
v2:
* Rebase on ToT. Move all new code shared between Linux and FreeBSD to /unix/ subdirectory, also factor out some existing code there.
* Improve description of Clang issue with rte_page_sizes on Windows. Restore -fstrict-enum for EAL. Check running, not target compiler.
* Use EAL prefix for private facilities instead if RTE.
* Improve documentation comments for new functions.
* Remove co-installer for virt2phys. Add a typecast for clarity.
* Document virt2phys in user guide, improve its own README.
* Explicitly and forcefully disable multi-process.
Dmitry Kozlyuk (11):
eal: replace rte_page_sizes with a set of constants
eal: introduce internal wrappers for file operations
eal: introduce memory management wrappers
eal/mem: extract common code for memseg list initialization eal/mem: extract common code for dynamic memory allocation
trace: add size_t field emitter
eal/windows: add tracing support stubs
eal/windows: replace sys/queue.h with a complete one from FreeBSD eal/windows: improve CPU and NUMA node detection eal/windows: initialize hugepage info
eal/windows: implement basic memory management
MAINTAINERS | 9 + config/meson.build | 12 +- doc/guides/rel_notes/release_20_08.rst | 2 + doc/guides/windows_gsg/build_dpdk.rst | 20 - doc/guides/windows_gsg/index.rst | 1 + doc/guides/windows_gsg/run_apps.rst | 95 +++ lib/librte_eal/common/eal_common_dynmem.c | 521 +++++++++++++ lib/librte_eal/common/eal_common_fbarray.c | 71 +- lib/librte_eal/common/eal_common_memory.c | 158 +++- lib/librte_eal/common/eal_common_thread.c | 5 +- lib/librte_eal/common/eal_private.h | 255 ++++++- lib/librte_eal/common/meson.build | 16 + lib/librte_eal/common/rte_malloc.c | 1 + lib/librte_eal/freebsd/Makefile | 5 + lib/librte_eal/freebsd/eal_memory.c | 98 +-- lib/librte_eal/include/rte_eal_memory.h | 93 +++ lib/librte_eal/include/rte_eal_trace.h | 8 +- lib/librte_eal/include/rte_memory.h | 23 +- lib/librte_eal/include/rte_trace_point.h | 3 + lib/librte_eal/linux/Makefile | 6 + lib/librte_eal/linux/eal_memalloc.c | 5 +- lib/librte_eal/linux/eal_memory.c | 614 +-------------- lib/librte_eal/meson.build | 4 + lib/librte_eal/rte_eal_exports.def | 119 +++ lib/librte_eal/rte_eal_version.map | 9 + lib/librte_eal/unix/eal_file.c | 76 ++ lib/librte_eal/unix/eal_unix_memory.c | 152 ++++ lib/librte_eal/unix/meson.build | 7 + lib/librte_eal/windows/eal.c | 107 +++ lib/librte_eal/windows/eal_file.c | 123 +++ lib/librte_eal/windows/eal_hugepages.c | 108 +++ lib/librte_eal/windows/eal_lcore.c | 185 +++-- lib/librte_eal/windows/eal_memalloc.c | 441 +++++++++++ lib/librte_eal/windows/eal_memory.c | 710 ++++++++++++++++++ lib/librte_eal/windows/eal_mp.c | 103 +++ lib/librte_eal/windows/eal_windows.h | 85 +++ lib/librte_eal/windows/include/meson.build | 1 + lib/librte_eal/windows/include/rte_os.h | 17 + .../windows/include/rte_virt2phys.h | 34 + lib/librte_eal/windows/include/rte_windows.h | 2 + lib/librte_eal/windows/include/sys/queue.h | 663 ++++++++++++++-- lib/librte_eal/windows/include/unistd.h | 3 + lib/librte_eal/windows/meson.build | 7 + lib/librte_mempool/rte_mempool_trace.h | 10 +- 44 files changed, 4048 insertions(+), 939 deletions(-) create mode 100644 doc/guides/windows_gsg/run_apps.rst create mode 100644 lib/librte_eal/common/eal_common_dynmem.c create mode 100644 lib/librte_eal/include/rte_eal_memory.h create mode 100644 lib/librte_eal/unix/eal_file.c create mode 100644 lib/librte_eal/unix/eal_unix_memory.c create mode 100644 lib/librte_eal/unix/meson.build create mode 100644 lib/librte_eal/windows/eal_file.c create mode 100644 lib/librte_eal/windows/eal_hugepages.c create mode 100644 lib/librte_eal/windows/eal_memalloc.c create mode 100644 lib/librte_eal/windows/eal_memory.c create mode 100644 lib/librte_eal/windows/eal_mp.c create mode 100644 lib/librte_eal/windows/include/rte_virt2phys.h
Signature · 1 line
2.25.4
PATCH v6 1/11 eal: replace rte_page_sizes with a set of constants
Clang on Windows follows MS ABI where enum values are limited to 2^31-1. Enum rte_page_sizes has members valued above this limit, which get wrapped to zero, resulting in compilation error (duplicate values in enum). Using MS ABI is mandatory for Windows EAL to call Win32 APIs.
Remove rte_page_sizes and replace its values with #define's. This enumeration is not used in public API, so there's no ABI breakage. Announce API changes for 20.08 in documentation.
--- doc/guides/rel_notes/release_20_08.rst | 2 ++ lib/librte_eal/include/rte_memory.h | 23 ++++++++++------------- 2 files changed, 12 insertions(+), 13 deletions(-)
doc/guides/rel_notes/release_20_08.rst +2 -0
@@ -85,6 +85,8 @@ API Changes Also, make sure to start the actual text at the margin. ========================================================= +* ``rte_page_sizes`` enumeration is replaced with ``RTE_PGSIZE_xxx`` defines. + ABI Changes -----------
lib/librte_eal/include/rte_memory.h +10 -13
@@ -24,19 +24,16 @@ extern "C" {#include <rte_config.h> #include <rte_fbarray.h> -__extension__ -enum rte_page_sizes { -RTE_PGSIZE_4K =1ULL <<12 , -RTE_PGSIZE_64K =1ULL <<16 , -RTE_PGSIZE_256K =1ULL <<18 , -RTE_PGSIZE_2M =1ULL <<21 , -RTE_PGSIZE_16M =1ULL <<24 , -RTE_PGSIZE_256M =1ULL <<28 , -RTE_PGSIZE_512M =1ULL <<29 , -RTE_PGSIZE_1G =1ULL <<30 , -RTE_PGSIZE_4G =1ULL <<32 , -RTE_PGSIZE_16G =1ULL <<34 , -}; +#define RTE_PGSIZE_4K (1ULL << 12) +#define RTE_PGSIZE_64K (1ULL << 16) +#define RTE_PGSIZE_256K (1ULL << 18) +#define RTE_PGSIZE_2M (1ULL << 21) +#define RTE_PGSIZE_16M (1ULL << 24) +#define RTE_PGSIZE_256M (1ULL << 28) +#define RTE_PGSIZE_512M (1ULL << 29) +#define RTE_PGSIZE_1G (1ULL << 30) +#define RTE_PGSIZE_4G (1ULL << 32) +#define RTE_PGSIZE_16G (1ULL << 34)#define SOCKET_ID_ANY -1/**< Any NUMA socket. */
Signature · 1 line
2.25.4
Re: [dpdk-dev] [PATCH v6 01/11] eal: replace rte_page_sizes with a set of constants
Quoted 11 lines · On Wed, 3 Jun 2020 02:03:19 +0300 Dmitry Kozlyuk <[email protected]>…
Clang on Windows follows MS ABI where enum values are limited to 2^31-1. Enum
rte_page_sizeshas members valued above this limit, which get wrapped to zero, resulting in compilation error (duplicate values in enum). Using MS ABI is mandatory for Windows EAL to call Win32 APIs.Remove
rte_page_sizesand replace its values with #define's. This enumeration is not used in public API, so there's no ABI breakage. Announce API changes for 20.08 in documentation.Suggested-by Jerin Jacob <[email protected]>Signed-off-by Dmitry Kozlyuk <[email protected]>
In this case #define makes more sense.
PATCH v6 2/11 eal: introduce internal wrappers for file operations
In reply to Dmitry KozlyukIntroduce OS-independent wrappers in order to support common EAL code on Unix and Windows:
* eal_file_create: open an existing file.
* eal_file_open: create a file or open it if exists.
* eal_file_lock: lock or unlock an open file.
* eal_file_truncate: enforce a given size for an open file.
Implementation for Linux and FreeBSD is placed in "unix" subdirectory, which is intended for common code between the two. These thin wrappers require no special maintenance.
Common code supporting multi-process doesn't use the new wrappers, because it is inherently Unix-specific and would impose excessive requirements on the wrappers.
--- MAINTAINERS | 3 + lib/librte_eal/common/eal_common_fbarray.c | 31 ++++----- lib/librte_eal/common/eal_private.h | 74 +++++++++++++++++++++ lib/librte_eal/freebsd/Makefile | 4 ++ lib/librte_eal/linux/Makefile | 4 ++ lib/librte_eal/meson.build | 4 ++ lib/librte_eal/unix/eal_file.c | 76 ++++++++++++++++++++++ lib/librte_eal/unix/meson.build | 6 ++ 8 files changed, 183 insertions(+), 19 deletions(-) create mode 100644 lib/librte_eal/unix/eal_file.c create mode 100644 lib/librte_eal/unix/meson.build
MAINTAINERS +3 -0
@@ -323,6 +323,9 @@ FreeBSD UIO M: Bruce Richardson <[email protected]> F: kernel/freebsd/nic_uio/ +Unix shared files +F: lib/librte_eal/unix/ + Windows support M: Harini Ramakrishnan <[email protected]> M: Omar Cardona <[email protected]>
lib/librte_eal/common/eal_common_fbarray.c +12 -19
@@ -8,8 +8,8 @@#include <sys/mman.h> #include <stdint.h> #include <errno.h> -#include <sys/file.h> #include <string.h> +#include <unistd.h> #include <rte_common.h> #include <rte_log.h>
@@ -85,10 +85,8 @@ resize_and_map(int fd, void *addr, size_t len)char path[PATH_MAX ];void *map_addr; -if (ftruncate (fd, len)) { +if (eal_file_truncate (fd, len)) {RTE_LOG (ERR ,EAL ,"Cannot truncate %s\n" , path); -/* pass errno up the chain */ - rte_errno = errno;return -1 ; }
@@ -772,15 +770,15 @@ rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,* and see if we succeed. If we don't, someone else is using it * already. */ - fd =open (path,O_CREAT |O_RDWR ,0600 ); + fd =eal_file_create (path);if (fd <0 ) {RTE_LOG (DEBUG ,EAL ,"%s(): couldn't open %s: %s\n" , - __func__, path,strerror (errno)); - rte_errno = errno; + __func__, path,rte_strerror (rte_errno)); goto fail; - }else if (flock (fd,LOCK_EX |LOCK_NB )) { + }else if (eal_file_lock ( + fd,EAL_FLOCK_EXCLUSIVE ,EAL_FLOCK_RETURN )) {RTE_LOG (DEBUG ,EAL ,"%s(): couldn't lock %s: %s\n" , - __func__, path,strerror (errno)); + __func__, path,rte_strerror (rte_errno)); rte_errno =EBUSY ; goto fail; }
@@ -789,10 +787,8 @@ rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,* still attach to it, but no other process could reinitialize * it. */ -if (flock (fd,LOCK_SH |LOCK_NB )) { - rte_errno = errno; +if (eal_file_lock (fd,EAL_FLOCK_SHARED ,EAL_FLOCK_RETURN )) goto fail; - }if (resize_and_map (fd, data, mmap_len)) goto fail;
@@ -888,17 +884,14 @@ rte_fbarray_attach(struct rte_fbarray *arr)eal_get_fbarray_path (path,sizeof (path), arr->name); - fd =open (path,O_RDWR ); + fd =eal_file_open (path, true);if (fd <0 ) { - rte_errno = errno; goto fail; }/* lock the file, to let others know we're using it */ -if (flock (fd,LOCK_SH |LOCK_NB )) { - rte_errno = errno; +if (eal_file_lock (fd,EAL_FLOCK_SHARED ,EAL_FLOCK_RETURN )) goto fail; - }if (resize_and_map (fd, data, mmap_len)) goto fail;
@@ -1025,7 +1018,7 @@ rte_fbarray_destroy(struct rte_fbarray *arr)* has been detached by all other processes */ fd = tmp->fd; -if (flock (fd,LOCK_EX |LOCK_NB )) { +if (eal_file_lock (fd,EAL_FLOCK_EXCLUSIVE ,EAL_FLOCK_RETURN )) {RTE_LOG (DEBUG ,EAL ,"Cannot destroy fbarray - another process is using it\n" ); rte_errno =EBUSY ; ret =-1 ;
@@ -1042,7 +1035,7 @@ rte_fbarray_destroy(struct rte_fbarray *arr)* we're still holding an exclusive lock, so drop it to * shared. */ -flock (fd,LOCK_SH |LOCK_NB ); +eal_file_lock (fd,EAL_FLOCK_SHARED ,EAL_FLOCK_RETURN ); ret =-1 ; goto out;
lib/librte_eal/common/eal_private.h +74 -0
78 lines not on this page · raw message
lib/librte_eal/freebsd/Makefile +4 -0
@@ -7,6 +7,7 @@ LIB = librte_eal.a ARCH_DIR ?= $(RTE_ARCH) VPATH += $(RTE_SDK)/lib/librte_eal/$(ARCH_DIR) +VPATH += $(RTE_SDK)/lib/librte_eal/unix VPATH += $(RTE_SDK)/lib/librte_eal/common CFLAGS += -I$(SRCDIR)/include
@@ -74,6 +75,9 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_service.c SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_random.c SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_reciprocal.c +# from unix dir +SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += eal_file.c + # from arch dir SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_cpuflags.c SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_hypervisor.c
lib/librte_eal/linux/Makefile +4 -0
@@ -7,6 +7,7 @@ LIB = librte_eal.a ARCH_DIR ?= $(RTE_ARCH) VPATH += $(RTE_SDK)/lib/librte_eal/$(ARCH_DIR) +VPATH += $(RTE_SDK)/lib/librte_eal/unix VPATH += $(RTE_SDK)/lib/librte_eal/common CFLAGS += -I$(SRCDIR)/include
@@ -81,6 +82,9 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_service.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_random.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_reciprocal.c +# from unix dir +SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_file.c + # from arch dir SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_cpuflags.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_hypervisor.c
lib/librte_eal/meson.build +4 -0
@@ -6,6 +6,10 @@ subdir('include') subdir('common') +if not is_windows + subdir('unix') +endif + dpdk_conf.set('RTE_EXEC_ENV_' + exec_env.to_upper(), 1) subdir(exec_env)
lib/librte_eal/unix/eal_file.c new file +76 -0
76 lines not on this page · raw message
lib/librte_eal/unix/meson.build new file +6 -0
@@ -0,0 +1,6 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2020 Dmitry Kozlyuk + +sources += files( + 'eal_file.c', +)
Signature · 1 line
2.25.4
Re: [dpdk-dev] [PATCH v6 02/11] eal: introduce internal wrappers for file operations
Quoted 303 lines · Dmitry Kozlyuk
Introduce OS-independent wrappers in order to support common EAL code on Unix and Windows:
*
eal_file_create: open an existing file.
*eal_file_open: create a file or open it if exists.
*eal_file_lock: lock or unlock an open file.
*eal_file_truncate: enforce a given size for an open file.Implementation for Linux and FreeBSD is placed in "unix" subdirectory, which is intended for common code between the two. These thin wrappers require no special maintenance.
Common code supporting multi-process doesn't use the new wrappers, because it is inherently Unix-specific and would impose excessive requirements on the wrappers.
Signed-off-by Dmitry Kozlyuk <[email protected]>--- MAINTAINERS | 3 + lib/librte_eal/common/eal_common_fbarray.c | 31 ++++----- lib/librte_eal/common/eal_private.h | 74 +++++++++++++++++++++ lib/librte_eal/freebsd/Makefile | 4 ++ lib/librte_eal/linux/Makefile | 4 ++ lib/librte_eal/meson.build | 4 ++ lib/librte_eal/unix/eal_file.c | 76 ++++++++++++++++++++++ lib/librte_eal/unix/meson.build | 6 ++ 8 files changed, 183 insertions(+), 19 deletions(-) create mode 100644 lib/librte_eal/unix/eal_file.c create mode 100644 lib/librte_eal/unix/meson.buildMAINTAINERS +3
-0@@ -323,6 +323,9 @@ FreeBSD UIO M: Bruce Richardson <[email protected]> F: kernel/freebsd/nic_uio/ +Unix shared files +F: lib/librte_eal/unix/ + Windows support M: Harini Ramakrishnan <[email protected]> M: Omar Cardona <[email protected]>lib/librte_eal/common/eal_common_fbarray.c +12
-19@@ -8,8 +8,8 @@#include <sys/mman.h> #include <stdint.h> #include <errno.h> -#include <sys/file.h> #include <string.h> +#include <unistd.h> #include <rte_common.h> #include <rte_log.h> @@ -85,10 +85,8 @@ resize_and_map(int fd, void *addr, size_t len)char path[PATH_MAX ];void *map_addr; -if (ftruncate (fd, len)) { +if (eal_file_truncate (fd, len)) {RTE_LOG (ERR ,EAL ,"Cannot truncate %s\n" , path); -/* pass errno up the chain */ - rte_errno = errno;return -1 ; }@@ -772,15 +770,15 @@ rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,* and see if we succeed. If we don't, someone else is using it * already. */ - fd =open (path,O_CREAT |O_RDWR ,0600 ); + fd =eal_file_create (path);if (fd <0 ) {RTE_LOG (DEBUG ,EAL ,"%s(): couldn't open %s: %s\n" , - __func__, path,strerror (errno)); - rte_errno = errno; + __func__, path,rte_strerror (rte_errno)); goto fail; - }else if (flock (fd,LOCK_EX |LOCK_NB )) { + }else if (eal_file_lock ( + fd,EAL_FLOCK_EXCLUSIVE ,EAL_FLOCK_RETURN )) {RTE_LOG (DEBUG ,EAL ,"%s(): couldn't lock %s: %s\n" , - __func__, path,strerror (errno)); + __func__, path,rte_strerror (rte_errno)); rte_errno =EBUSY ; goto fail; }@@ -789,10 +787,8 @@ rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,* still attach to it, but no other process could reinitialize * it. */ -if (flock (fd,LOCK_SH |LOCK_NB )) { - rte_errno = errno; +if (eal_file_lock (fd,EAL_FLOCK_SHARED ,EAL_FLOCK_RETURN )) goto fail; - }if (resize_and_map (fd, data, mmap_len)) goto fail;@@ -888,17 +884,14 @@ rte_fbarray_attach(struct rte_fbarray *arr)eal_get_fbarray_path (path,sizeof (path), arr->name); - fd =open (path,O_RDWR ); + fd =eal_file_open (path, true);if (fd <0 ) { - rte_errno = errno; goto fail; }/* lock the file, to let others know we're using it */ -if (flock (fd,LOCK_SH |LOCK_NB )) { - rte_errno = errno; +if (eal_file_lock (fd,EAL_FLOCK_SHARED ,EAL_FLOCK_RETURN )) goto fail; - }if (resize_and_map (fd, data, mmap_len)) goto fail;@@ -1025,7 +1018,7 @@ rte_fbarray_destroy(struct rte_fbarray *arr)* has been detached by all other processes */ fd = tmp->fd; -if (flock (fd,LOCK_EX |LOCK_NB )) { +if (eal_file_lock (fd,EAL_FLOCK_EXCLUSIVE ,EAL_FLOCK_RETURN )) {RTE_LOG (DEBUG ,EAL ,"Cannot destroy fbarray - another process is using it\n" ); rte_errno =EBUSY ; ret =-1 ;@@ -1042,7 +1035,7 @@ rte_fbarray_destroy(struct rte_fbarray *arr)* we're still holding an exclusive lock, so drop it to * shared. */ -flock (fd,LOCK_SH |LOCK_NB ); +eal_file_lock (fd,EAL_FLOCK_SHARED ,EAL_FLOCK_RETURN ); ret =-1 ; goto out;lib/librte_eal/common/eal_private.h +74
-0@@ -420,4 +420,78 @@ eal_malloc_no_trace(const char *type, size_t size, unsigned int align);void eal_free_no_trace (void *addr); +/** +* Create a file or open it if exits. +* +* Newly created file is only accessible to the owner (0600 equivalent). +* Returned descriptor is always read/write. +* +* @param path +* Path to the file. +* @return +* Open file descriptor on success, (-1) on failure and rte_errno is set. +*/ +int +eal_file_create (const char *path); + +/** +* Open an existing file. +* +* @param path +* Path to the file. +* @param writable +* Whether to open file read/write or read-only. +* @return +* Open file descriptor on success, (-1) on failure and rte_errno is set. +*/ +int +eal_file_open (const char *path,bool writable); + +/** File locking operation. */ +enum eal_flock_op { +EAL_FLOCK_SHARED ,/**< Acquire a shared lock. */ +EAL_FLOCK_EXCLUSIVE ,/**< Acquire an exclusive lock. */ +EAL_FLOCK_UNLOCK /**< Release a previously taken lock. */ +}; + +/** Behavior on file locking conflict. */ +enum eal_flock_mode { +EAL_FLOCK_WAIT ,/**< Wait until the file gets unlocked to lock it. */ +EAL_FLOCK_RETURN /**< Return immediately if the file is locked. */ +}; + +/** +* Lock or unlock the file. +* +* On failure @code rte_errno @endcode is set to the error code +* specified by POSIX flock(3) description. +* +* @param fd +* Opened file descriptor. +* @param op +* Operation to perform. +* @param mode +* Behavior on conflict. +* @return +* 0 on success, (-1) on failure. +*/ +int +eal_file_lock (int fd,enum eal_flock_op op,enum eal_flock_mode mode); + +/** +* Truncate or extend the file to the specified size. +* +* On failure @code rte_errno @endcode is set to the error code +* specified by POSIX ftruncate(3) description. +* +* @param fd +* Opened file descriptor. +* @param size +* Desired file size. +* @return +* 0 on success, (-1) on failure. +*/ +int +eal_file_truncate (int fd,ssize_t size); +#endif /* _EAL_PRIVATE_H_ */ lib/librte_eal/freebsd/Makefile +4
-0@@ -7,6 +7,7 @@ LIB = librte_eal.a ARCH_DIR ?= $(RTE_ARCH) VPATH += $(RTE_SDK)/lib/librte_eal/$(ARCH_DIR) +VPATH += $(RTE_SDK)/lib/librte_eal/unix VPATH += $(RTE_SDK)/lib/librte_eal/common CFLAGS += -I$(SRCDIR)/include@@ -74,6 +75,9 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_service.c SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_random.c SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_reciprocal.c +# from unix dir +SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += eal_file.c + # from arch dir SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_cpuflags.c SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_hypervisor.clib/librte_eal/linux/Makefile +4
-0@@ -7,6 +7,7 @@ LIB = librte_eal.a ARCH_DIR ?= $(RTE_ARCH) VPATH += $(RTE_SDK)/lib/librte_eal/$(ARCH_DIR) +VPATH += $(RTE_SDK)/lib/librte_eal/unix VPATH += $(RTE_SDK)/lib/librte_eal/common CFLAGS += -I$(SRCDIR)/include@@ -81,6 +82,9 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_service.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_random.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_reciprocal.c +# from unix dir +SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_file.c + # from arch dir SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_cpuflags.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_hypervisor.clib/librte_eal/meson.build +4
-0@@ -6,6 +6,10 @@ subdir('include') subdir('common') +if not is_windows + subdir('unix') +endif + dpdk_conf.set('RTE_EXEC_ENV_' + exec_env.to_upper(), 1) subdir(exec_env)lib/librte_eal/unix/eal_file.c new file +14
-0@@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright(c) 2020 Dmitry Kozlyuk +*/ + +#include <sys/file.h> +#include <sys/mman.h> +#include <unistd.h> + +#include <rte_errno.h> + +#include "eal_private.h" + +int +eal_file_create (const char *path)
lib/librte_eal/unix/eal_file.c line 24
Quoted 10 lines
lib/librte_eal/unix/eal_file.c new file +10
-0+{ +int ret; + + ret =open (path,O_CREAT |O_RDWR ,0600 ); +if (ret <0 ) + rte_errno = errno; + +return ret; +} +
You don't need this call if you support the oflags option in the open call below.
lib/librte_eal/unix/eal_file.c line 37
Quoted 13 lines
lib/librte_eal/unix/eal_file.c new file +13
-0+int +eal_file_open (const char *path,bool writable) +{ +int ret, flags; + + flags = writable ?O_RDWR :O_RDONLY ; + ret =open (path, flags); +if (ret <0 ) + rte_errno = errno; + +return ret; +} +
why are you changing this api from the posix file format (with oflags specified). As far as I can see both unix and windows platforms support that
Quoted 55 lines
lib/librte_eal/unix/eal_file.c new file +39
-0+int +eal_file_truncate (int fd,ssize_t size) +{ +int ret; + + ret =ftruncate (fd, size); +if (ret) + rte_errno = errno; + +return ret; +} + +int +eal_file_lock (int fd,enum eal_flock_op op,enum eal_flock_mode mode) +{ +int sys_flags =0 ; +int ret; + +if (mode ==EAL_FLOCK_RETURN ) + sys_flags |=LOCK_NB ; + +switch (op) { +case EAL_FLOCK_EXCLUSIVE : + sys_flags |=LOCK_EX ; +break ; +case EAL_FLOCK_SHARED : + sys_flags |=LOCK_SH ; +break ; +case EAL_FLOCK_UNLOCK : + sys_flags |=LOCK_UN ; +break ; + } + + ret =flock (fd, sys_flags); +if (ret) + rte_errno = errno; + +return ret; +}lib/librte_eal/unix/meson.build new file +6
-0@@ -0,0 +1,6 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2020 Dmitry Kozlyuk + +sources += files( + 'eal_file.c', +)Signature · 1 line
2.25.4
Re: [dpdk-dev] [PATCH v6 02/11] eal: introduce internal wrappers for file operations
On Wed, 3 Jun 2020 08:07:59 -0400
Neil Horman <[email protected]> wrote:
[snip]
Quoted 6 lines
Quoted 6 lines
lib/librte_eal/unix/eal_file.c new file +6
-0+int +eal_file_create (const char *path) +{ +int ret; + + ret =open (path,O_CREAT |O_RDWR ,0600 );
Quoted 8 lines
lib/librte_eal/unix/eal_file.c line 24
Quoted 6 lines
lib/librte_eal/unix/eal_file.c new file +6
-0+if (ret <0 ) + rte_errno = errno; + +return ret; +} +You don't need this call if you support the oflags option in the open call below.
See below.
Quoted 7 lines
Quoted 7 lines
lib/librte_eal/unix/eal_file.c new file +7
-0+int +eal_file_open (const char *path,bool writable) +{ +int ret, flags; + + flags = writable ?O_RDWR :O_RDONLY ; + ret =open (path, flags);
Quoted 8 lines
lib/librte_eal/unix/eal_file.c line 24
Quoted 6 lines
lib/librte_eal/unix/eal_file.c new file +6
-0+if (ret <0 ) + rte_errno = errno; + +return ret; +} +why are you changing this api from the posix file format (with oflags specified). As far as I can see both unix and windows platforms support that
There is a number of caveats, which IMO make this approach better:
1. Filesystem permissions on Windows are complicated. Supporting anything other than 0600 would add a lot of code, while EAL doesn't really need it. Microsoft's open() takes not permission bits, but a set of flags.
2. Restricted interface prevents EAL developers from accidentally using features not supported on all platforms via a seemingly rich API.
3. Microsoft CRT (the one Clang is using) deprecates open() in favor of _sopen_s() and issues a warning, and we're targeting -Werror. Disabling all such warnings (_CRT_SECURE_NO_DEPRECATE) doesn't seem right when CRT vendor encourages using alternatives. This is the primary reason for open() wrappers in v6.
Signature · 1 line
Dmitry Kozlyuk
Re: [dpdk-dev] [PATCH v6 02/11] eal: introduce internal wrappers for file operations
Quoted 43 lines · Dmitry Kozlyuk
On Wed, 3 Jun 2020 08:07:59 -0400
Neil Horman <[email protected]> wrote:[snip]
Quoted 6 lines
Quoted 6 lines
lib/librte_eal/unix/eal_file.c new file +6
-0+int +eal_file_create (const char *path) +{ +int ret; + + ret =open (path,O_CREAT |O_RDWR ,0600 );Quoted 8 lines
lib/librte_eal/unix/eal_file.c line 24
Quoted 6 lines
lib/librte_eal/unix/eal_file.c new file +6
-0+if (ret <0 ) + rte_errno = errno; + +return ret; +} +You don't need this call if you support the oflags option in the open call below.
See below.
Quoted 7 lines
Quoted 7 lines
lib/librte_eal/unix/eal_file.c new file +7
-0+int +eal_file_open (const char *path,bool writable) +{ +int ret, flags; + + flags = writable ?O_RDWR :O_RDONLY ; + ret =open (path, flags);Quoted 8 lines
lib/librte_eal/unix/eal_file.c line 24
Quoted 6 lines
lib/librte_eal/unix/eal_file.c new file +6
-0+if (ret <0 ) + rte_errno = errno; + +return ret; +} +why are you changing this api from the posix file format (with oflags specified). As far as I can see both unix and windows platforms support that
There is a number of caveats, which IMO make this approach better:
1. Filesystem permissions on Windows are complicated. Supporting anything other than 0600 would add a lot of code, while EAL doesn't really need it. Microsoft's
open()takes not permission bits, but a set of flags.
Quoted 9 lines
2. Restricted interface prevents EAL developers from accidentally using features not supported on all platforms via a seemingly rich API.
3. Microsoft CRT (the one Clang is using) deprecates
open()in favor of_sopen_s()and issues a warning, and we're targeting -Werror. Disabling all such warnings (_CRT_SECURE_NO_DEPRECATE) doesn't seem right when CRT vendor encourages using alternatives. This is the primary reason foropen()wrappers in v6.
that seems a bit shortsighted to me. By creating wrappers that restrict functionality to the least common demoninator of supported platforms restricts what all platforms are capable of. For example, theres no reason that the eal library shouldn't be able to open a file O_TRUNC or O_SYNC just because its complex to do it on a single platform.
The API should be written to support the full range of functionality on all platforms, and the individual implementations should write the code to make that happen, or return an error that its unsupported on this particular platform.
I'm not saying that you have to implement everything now, but you shouldn't restrict the API from being able to do so in the future. Otherwise, in the future, if someone wants to implement O_TRUNC support (just to site an example), they're going to have to make a change to the API above, and alter the implementation for all the platforms anyway. You may as well make the API robust enough to support that now.
Neil
Quoted 3 lines
Signature · 1 line
Dmitry Kozlyuk
Re: [dpdk-dev] [PATCH v6 02/11] eal: introduce internal wrappers for file operations
Quoted 54 lines · On Thu, 4 Jun 2020 17:07:07 -0400 Neil Horman <[email protected]> wrote:
Quoted 43 lines · Dmitry Kozlyuk
On Wed, 3 Jun 2020 08:07:59 -0400
Neil Horman <[email protected]> wrote:[snip]
Quoted 6 lines
Quoted 6 lines
lib/librte_eal/unix/eal_file.c new file +6
-0+int +eal_file_create (const char *path) +{ +int ret; + + ret =open (path,O_CREAT |O_RDWR ,0600 );Quoted 8 lines
lib/librte_eal/unix/eal_file.c line 24
Quoted 6 lines
lib/librte_eal/unix/eal_file.c new file +6
-0+if (ret <0 ) + rte_errno = errno; + +return ret; +} +You don't need this call if you support the oflags option in the open call below.
See below.
Quoted 7 lines
Quoted 7 lines
lib/librte_eal/unix/eal_file.c new file +7
-0+int +eal_file_open (const char *path,bool writable) +{ +int ret, flags; + + flags = writable ?O_RDWR :O_RDONLY ; + ret =open (path, flags);Quoted 8 lines
lib/librte_eal/unix/eal_file.c line 24
Quoted 6 lines
lib/librte_eal/unix/eal_file.c new file +6
-0+if (ret <0 ) + rte_errno = errno; + +return ret; +} +why are you changing this api from the posix file format (with oflags specified). As far as I can see both unix and windows platforms support that
There is a number of caveats, which IMO make this approach better:
1. Filesystem permissions on Windows are complicated. Supporting anything other than 0600 would add a lot of code, while EAL doesn't really need it. Microsoft's
open()takes not permission bits, but a set of flags.Quoted 9 lines
2. Restricted interface prevents EAL developers from accidentally using features not supported on all platforms via a seemingly rich API.
3. Microsoft CRT (the one Clang is using) deprecates
open()in favor of_sopen_s()and issues a warning, and we're targeting -Werror. Disabling all such warnings (_CRT_SECURE_NO_DEPRECATE) doesn't seem right when CRT vendor encourages using alternatives. This is the primary reason foropen()wrappers in v6.
Quoted 5 lines
that seems a bit shortsighted to me. By creating wrappers that restrict functionality to the least common demoninator of supported platforms restricts what all platforms are capable of. For example, theres no reason that the eal library shouldn't be able to open a file
O_TRUNCorO_SYNCjust because its complex to do it on a single platform.
The purpose of these wrappers is to maximize reuse of common code. It doesn't require POSIX par se, it's just implemented in terms of API that had been available on all supported OSes until Windows target was introduced. Wrapper interface is derived from common code requirements.
Quoted 3 lines
The API should be written to support the full range of functionality on all platforms, and the individual implementations should write the code to make that happen, or return an error that its unsupported on this particular platform.
IMO, common code, by definition, should avoid partial support of anything.
Quoted 6 lines
I'm not saying that you have to implement everything now, but you shouldn't restrict the API from being able to do so in the future. Otherwise, in the future, if someone wants to implement
O_TRUNCsupport (just to site an example), they're going to have to make a change to the API above, and alter the implementation for all the platforms anyway. You may as well make the API robust enough to support that now.
I agree that these particular wrappers can have a lot more options, so probably flags would be better. However, I wouldn't add parameters that have partial support, namely, permissions. What do you think of the following (names shortened)?
enum mode {
RO = 0, /* write-only is not portable */
RW = 1,
CREATE = 2 /* always 0600 equivalent */
};eal_file_open(const char *path, int mode);
Signature · 1 line
Dmitry Kozlyuk
Re: [dpdk-dev] [PATCH v6 02/11] eal: introduce internal wrappers for file operations
Quoted 63 lines · Dmitry Kozlyuk
Quoted 54 lines · On Thu, 4 Jun 2020 17:07:07 -0400 Neil Horman <[email protected]> wrote:
Quoted 43 lines · Dmitry Kozlyuk
On Wed, 3 Jun 2020 08:07:59 -0400
Neil Horman <[email protected]> wrote:[snip]
Quoted 6 lines
Quoted 6 lines
lib/librte_eal/unix/eal_file.c new file +6
-0+int +eal_file_create (const char *path) +{ +int ret; + + ret =open (path,O_CREAT |O_RDWR ,0600 );Quoted 8 lines
lib/librte_eal/unix/eal_file.c line 24
Quoted 6 lines
lib/librte_eal/unix/eal_file.c new file +6
-0+if (ret <0 ) + rte_errno = errno; + +return ret; +} +You don't need this call if you support the oflags option in the open call below.
See below.
Quoted 7 lines
Quoted 7 lines
lib/librte_eal/unix/eal_file.c new file +7
-0+int +eal_file_open (const char *path,bool writable) +{ +int ret, flags; + + flags = writable ?O_RDWR :O_RDONLY ; + ret =open (path, flags);Quoted 8 lines
lib/librte_eal/unix/eal_file.c line 24
Quoted 6 lines
lib/librte_eal/unix/eal_file.c new file +6
-0+if (ret <0 ) + rte_errno = errno; + +return ret; +} +why are you changing this api from the posix file format (with oflags specified). As far as I can see both unix and windows platforms support that
There is a number of caveats, which IMO make this approach better:
1. Filesystem permissions on Windows are complicated. Supporting anything other than 0600 would add a lot of code, while EAL doesn't really need it. Microsoft's
open()takes not permission bits, but a set of flags.Quoted 9 lines
2. Restricted interface prevents EAL developers from accidentally using features not supported on all platforms via a seemingly rich API.
3. Microsoft CRT (the one Clang is using) deprecates
open()in favor of_sopen_s()and issues a warning, and we're targeting -Werror. Disabling all such warnings (_CRT_SECURE_NO_DEPRECATE) doesn't seem right when CRT vendor encourages using alternatives. This is the primary reason foropen()wrappers in v6.Quoted 5 lines
that seems a bit shortsighted to me. By creating wrappers that restrict functionality to the least common demoninator of supported platforms restricts what all platforms are capable of. For example, theres no reason that the eal library shouldn't be able to open a file
O_TRUNCorO_SYNCjust because its complex to do it on a single platform.
Quoted 5 lines
The purpose of these wrappers is to maximize reuse of common code. It doesn't require POSIX par se, it's just implemented in terms of API that had been available on all supported OSes until Windows target was introduced. Wrapper interface is derived from common code requirements.
Sure, and I'm fine with that. What I'm concerned about is implementing wrappers that define their APIs in terms of whats currently implemented. Theres no reason that the existing in use feature set won't be built upon in the future, and the API should be able to handle that.
Quoted 6 lines
Quoted 3 lines
The API should be written to support the full range of functionality on all platforms, and the individual implementations should write the code to make that happen, or return an error that its unsupported on this particular platform.
IMO, common code, by definition, should avoid partial support of anything.
I disagree. Anytime you abstract an implementation to a more generic api, you have the possibility that a given implementation won't offer full support for all of the APIs features, and thats ok, as long as there are no users of the features, and a given implmentation properly returns an error when their usage is attempted. The expectation then is, that the user of the feature will add the feature to all implementations, so that the code can remain portable. What you shouldn't do is define the API such that those features can't be implemented without having to change the API, as that runs the potential risk of having to modify the ABI. Thats probably not the case here, but the notion stands. If you write the API to encompass the superset of supported platforms features, the rest is implementation details.
Quoted 10 lines
Quoted 6 lines
I'm not saying that you have to implement everything now, but you shouldn't restrict the API from being able to do so in the future. Otherwise, in the future, if someone wants to implement
O_TRUNCsupport (just to site an example), they're going to have to make a change to the API above, and alter the implementation for all the platforms anyway. You may as well make the API robust enough to support that now.I agree that these particular wrappers can have a lot more options, so probably flags would be better. However, I wouldn't add parameters that have partial support, namely, permissions.
But windows does offer file and folder permissions:
https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-2000-server/bb727008(v=technet.10)?redirectedfrom=MSDN
As you said, they are complicated, and the security model is different, but those are details that can be worked out/emulated when the need arises.
Quoted 11 lines
What do you think of the following
(names shortened)?enum mode { RO = 0, /* write-only is not portable */ RW = 1, CREATE = 2 /* always 0600 equivalent */ };
eal_file_open(const char *path, int mode);
Yeah, that makes sense to me. I'd be good with that.
Neil
Quoted 3 lines
Signature · 1 line
Dmitry Kozlyuk
PATCH v6 3/11 eal: introduce memory management wrappers
In reply to Dmitry KozlyukIntroduce OS-independent wrappers for memory management operations used across DPDK and specifically in common code of EAL:
* rte_mem_map()
* rte_mem_unmap()
* rte_mem_page_size()
* rte_mem_lock()
Windows uses different APIs for memory mapping and reservation, while Unices reserve memory by mapping it. Introduce EAL private functions to support memory reservation in common code:
* eal_mem_reserve()
* eal_mem_free()
* eal_mem_set_dump()
Wrappers follow POSIX semantics limited to DPDK tasks, but their signatures deliberately differ from POSIX ones to be more safe and expressive. New symbols are internal. Being thin wrappers, they require no special maintenance.
--- lib/librte_eal/common/eal_common_fbarray.c | 40 +++--- lib/librte_eal/common/eal_common_memory.c | 61 ++++----- lib/librte_eal/common/eal_private.h | 78 ++++++++++- lib/librte_eal/freebsd/Makefile | 1 + lib/librte_eal/include/rte_eal_memory.h | 93 +++++++++++++ lib/librte_eal/linux/Makefile | 1 + lib/librte_eal/linux/eal_memalloc.c | 5 +- lib/librte_eal/rte_eal_version.map | 9 ++ lib/librte_eal/unix/eal_unix_memory.c | 152 +++++++++++++++++++++ lib/librte_eal/unix/meson.build | 1 + 10 files changed, 376 insertions(+), 65 deletions(-) create mode 100644 lib/librte_eal/include/rte_eal_memory.h create mode 100644 lib/librte_eal/unix/eal_unix_memory.c
lib/librte_eal/common/eal_common_fbarray.c +19 -21
@@ -5,15 +5,16 @@#include <fcntl.h> #include <inttypes.h> #include <limits.h> -#include <sys/mman.h> #include <stdint.h> #include <errno.h> #include <string.h> #include <unistd.h> #include <rte_common.h> -#include <rte_log.h> +#include <rte_eal_memory.h> #include <rte_errno.h> +#include <rte_log.h> +#include <rte_memory.h> #include <rte_spinlock.h> #include <rte_tailq.h>
@@ -90,12 +91,9 @@ resize_and_map(int fd, void *addr, size_t len)return -1 ; } - map_addr =mmap (addr, len,PROT_READ |PROT_WRITE , -MAP_SHARED |MAP_FIXED , fd,0 ); + map_addr =rte_mem_map (addr, len,RTE_PROT_READ |RTE_PROT_WRITE , +RTE_MAP_SHARED |RTE_MAP_FORCE_ADDRESS , fd,0 );if (map_addr != addr) { -RTE_LOG (ERR ,EAL ,"mmap() failed: %s\n" ,strerror (errno)); -/* pass errno up the chain */ - rte_errno = errno;return -1 ; }return 0 ;
@@ -733,7 +731,7 @@ rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,return -1 ; } - page_sz =sysconf (_SC_PAGESIZE); + page_sz =rte_mem_page_size ();if (page_sz == (size_t )-1 ) {free (ma);return -1 ;
@@ -754,11 +752,13 @@ rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,if (internal_config.no_shconf) {/* remap virtual area as writable */ -void *new_data =mmap (data, mmap_len,PROT_READ |PROT_WRITE , -MAP_FIXED |MAP_PRIVATE |MAP_ANONYMOUS , fd,0 ); -if (new_data ==MAP_FAILED ) { +static const int flags =RTE_MAP_FORCE_ADDRESS | +RTE_MAP_PRIVATE |RTE_MAP_ANONYMOUS ; +void *new_data =rte_mem_map (data, mmap_len, +RTE_PROT_READ |RTE_PROT_WRITE , flags, fd,0 ); +if (new_data ==NULL ) {RTE_LOG (DEBUG ,EAL ,"%s(): couldn't remap anonymous memory: %s\n" , - __func__,strerror (errno)); + __func__,rte_strerror (rte_errno)); goto fail; } }else {
@@ -820,7 +820,7 @@ rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,return 0 ; fail:if (data) -munmap (data, mmap_len); +rte_mem_unmap (data, mmap_len);if (fd >=0 )close (fd);free (ma);
@@ -858,7 +858,7 @@ rte_fbarray_attach(struct rte_fbarray *arr)return -1 ; } - page_sz =sysconf (_SC_PAGESIZE); + page_sz =rte_mem_page_size ();if (page_sz == (size_t )-1 ) {free (ma);return -1 ;
@@ -909,7 +909,7 @@ rte_fbarray_attach(struct rte_fbarray *arr)return 0 ; fail:if (data) -munmap (data, mmap_len); +rte_mem_unmap (data, mmap_len);if (fd >=0 )close (fd);free (ma);
@@ -937,8 +937,7 @@ rte_fbarray_detach(struct rte_fbarray *arr)* really do anything about it, things will blow up either way. */ -size_t page_sz =sysconf (_SC_PAGESIZE); - +size_t page_sz =rte_mem_page_size ();if (page_sz == (size_t )-1 )return -1 ;
@@ -957,7 +956,7 @@ rte_fbarray_detach(struct rte_fbarray *arr) goto out; } -munmap (arr->data, mmap_len); +rte_mem_unmap (arr->data, mmap_len);/* area is unmapped, close fd and remove the tailq entry */ if (tmp->fd >=0 )
@@ -992,8 +991,7 @@ rte_fbarray_destroy(struct rte_fbarray *arr)* really do anything about it, things will blow up either way. */ -size_t page_sz =sysconf (_SC_PAGESIZE); - +size_t page_sz =rte_mem_page_size ();if (page_sz == (size_t )-1 )return -1 ;
@@ -1042,7 +1040,7 @@ rte_fbarray_destroy(struct rte_fbarray *arr) }close (fd); } -munmap (arr->data, mmap_len); +rte_mem_unmap (arr->data, mmap_len);/* area is unmapped, remove the tailq entry */ TAILQ_REMOVE (&mem_area_tailq, tmp, next);
lib/librte_eal/common/eal_common_memory.c +23 -38
143 lines not on this page · raw message
lib/librte_eal/common/eal_private.h +75 -3
106 lines not on this page · raw message
lib/librte_eal/freebsd/Makefile +1 -0
@@ -77,6 +77,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_reciprocal.c # from unix dir SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += eal_file.c +SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += eal_unix_memory.c # from arch dir SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_cpuflags.c
lib/librte_eal/include/rte_eal_memory.h new file +93 -0
93 lines not on this page · raw message
lib/librte_eal/linux/Makefile +1 -0
@@ -84,6 +84,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_reciprocal.c # from unix dir SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_file.c +SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_unix_memory.c # from arch dir SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_cpuflags.c
lib/librte_eal/linux/eal_memalloc.c +2 -3
@@ -630,7 +630,7 @@ alloc_seg(struct rte_memseg *ms, void *addr, int socket_id, mapped:munmap (addr, alloc_sz); unmapped: - flags =MAP_FIXED ; + flags =EAL_RESERVE_FORCE_ADDRESS ; new_addr =eal_get_virtual_area (addr, &alloc_sz, alloc_sz,0 , flags);if (new_addr != addr) {if (new_addr !=NULL )
@@ -687,8 +687,7 @@ free_seg(struct rte_memseg *ms, struct hugepage_info *hi,return -1 ; } -if (madvise (ms->addr, ms->len,MADV_DONTDUMP ) !=0 ) -RTE_LOG (DEBUG ,EAL ,"madvise failed: %s\n" ,strerror (errno)); +eal_mem_set_dump (ms->addr, ms->len, false); exit_early = false;
lib/librte_eal/rte_eal_version.map +9 -0
@@ -387,3 +387,12 @@ EXPERIMENTAL { rte_trace_regexp; rte_trace_save; }; + +INTERNAL { + global: + + rte_mem_lock; + rte_mem_map; + rte_mem_page_size; + rte_mem_unmap; +};
lib/librte_eal/unix/eal_unix_memory.c new file +152 -0
152 lines not on this page · raw message
lib/librte_eal/unix/meson.build +1 -0
@@ -3,4 +3,5 @@ sources += files( 'eal_file.c', + 'eal_unix_memory.c', )
Signature · 1 line
2.25.4
PATCH v6 4/11 eal/mem: extract common code for memseg list initialization
In reply to Dmitry KozlyukAll supported OS create memory segment lists (MSL) and reserve VA space for them in a nearly identical way. Move common code into EAL private functions to reduce duplication.
--- lib/librte_eal/common/eal_common_memory.c | 97 ++++++++++++++++++ lib/librte_eal/common/eal_private.h | 62 ++++++++++++ lib/librte_eal/freebsd/eal_memory.c | 94 ++++-------------- lib/librte_eal/linux/eal_memory.c | 115 +++++----------------- 4 files changed, 200 insertions(+), 168 deletions(-)
lib/librte_eal/common/eal_common_memory.c +97 -0
@@ -25,6 +25,7 @@#include "eal_private.h" #include "eal_internal_cfg.h" #include "eal_memcfg.h" +#include "eal_options.h" #include "malloc_heap.h" /*
@@ -182,6 +183,102 @@ eal_get_virtual_area(void *requested_addr, size_t *size,return aligned_addr; } +int +eal_memseg_list_init_named (struct rte_memseg_list *msl,const char *name, +uint64_t page_sz,int n_segs,int socket_id,bool heap) +{ +if (rte_fbarray_init (&msl->memseg_arr, name, n_segs, +sizeof (struct rte_memseg ))) { +RTE_LOG (ERR ,EAL ,"Cannot allocate memseg list: %s\n" , +rte_strerror (rte_errno)); +return -1 ; + } + + msl->page_sz = page_sz; + msl->socket_id = socket_id; + msl->base_va =NULL ; + msl->heap = heap; + +RTE_LOG (DEBUG ,EAL , +"Memseg list allocated at socket %i, page size 0x%zxkB\n" , + socket_id, (size_t )page_sz >>10 ); + +return 0 ; +} + +int +eal_memseg_list_init (struct rte_memseg_list *msl,uint64_t page_sz, +int n_segs,int socket_id,int type_msl_idx,bool heap) +{ +char name[RTE_FBARRAY_NAME_LEN ]; + +snprintf (name,sizeof (name),MEMSEG_LIST_FMT , page_sz >>10 , socket_id, + type_msl_idx); + +return eal_memseg_list_init_named ( + msl, name, page_sz, n_segs, socket_id, heap); +} + +int +eal_memseg_list_alloc (struct rte_memseg_list *msl,int reserve_flags) +{ +size_t page_sz, mem_sz; +void *addr; + + page_sz = msl->page_sz; + mem_sz = page_sz * msl->memseg_arr.len; + + addr =eal_get_virtual_area ( + msl->base_va, &mem_sz, page_sz,0 , reserve_flags); +if (addr ==NULL ) { +#ifndef RTE_EXEC_ENV_WINDOWS +/* The hint would be misleading on Windows, but this function +* is called from many places, including common code, +* so don't duplicate the message. +*/ +if (rte_errno ==EADDRNOTAVAIL ) +RTE_LOG (ERR ,EAL ,"Cannot reserve %llu bytes at [%p] - " +"please use '--" OPT_BASE_VIRTADDR "' option\n" , + (unsigned long long )mem_sz, msl->base_va); +else +RTE_LOG (ERR ,EAL ,"Cannot reserve memory\n" ); +#endif +return -1 ; + } + msl->base_va = addr; + msl->len = mem_sz; + +RTE_LOG (DEBUG ,EAL ,"VA reserved for memseg list at %p, size %zx\n" , + addr, mem_sz); + +return 0 ; +} + +void +eal_memseg_list_populate (struct rte_memseg_list *msl,void *addr,int n_segs) +{ +size_t page_sz = msl->page_sz; +int i; + +for (i =0 ; i < n_segs; i++) { +struct rte_fbarray *arr = &msl->memseg_arr; +struct rte_memseg *ms =rte_fbarray_get (arr, i); + +if (rte_eal_iova_mode () ==RTE_IOVA_VA ) + ms->iova = (uintptr_t )addr; +else + ms->iova =RTE_BAD_IOVA ; + ms->addr = addr; + ms->hugepage_sz = page_sz; + ms->socket_id =0 ; + ms->len = page_sz; + +rte_fbarray_set_used (arr, i); + + addr =RTE_PTR_ADD (addr, page_sz); + } +} +static struct rte_memseg *virt2memseg (const void *addr,const struct rte_memseg_list *msl) {
lib/librte_eal/common/eal_private.h +62 -0
68 lines not on this page · raw message
lib/librte_eal/freebsd/eal_memory.c +18 -76
156 lines not on this page · raw message
lib/librte_eal/linux/eal_memory.c +23 -92
216 lines not on this page · raw message
Signature · 1 line
2.25.4
Re: [dpdk-dev] [PATCH v6 04/11] eal/mem: extract common code for memseg list initialization
Quoted 6 lines · Dmitry Kozlyuk
All supported OS create memory segment lists (MSL) and reserve VA space for them in a nearly identical way. Move common code into EAL private functions to reduce duplication.
Signed-off-by Dmitry Kozlyuk <[email protected]>---
<snip>
Quoted 14 lines
lib/librte_eal/common/eal_common_memory.c +14
-0+int +eal_memseg_list_alloc (struct rte_memseg_list *msl,int reserve_flags) +{ +size_t page_sz, mem_sz; +void *addr; + + page_sz = msl->page_sz; + mem_sz = page_sz * msl->memseg_arr.len; + + addr =eal_get_virtual_area ( + msl->base_va, &mem_sz, page_sz,0 , reserve_flags); +if (addr ==NULL ) { +#ifndef RTE_EXEC_ENV_WINDOWS +/* The hint would be misleading on Windows, but this function
lib/librte_eal/common/eal_common_memory.c eal_get_virtual_area(void *requested_addr, size_t *size, · line 245
Quoted 10 lines
lib/librte_eal/common/eal_common_memory.c +10
-0+* is called from many places, including common code, +* so don't duplicate the message. +*/ +if (rte_errno ==EADDRNOTAVAIL ) +RTE_LOG (ERR ,EAL ,"Cannot reserve %llu bytes at [%p] - " +"please use '--" OPT_BASE_VIRTADDR "' option\n" , + (unsigned long long )mem_sz, msl->base_va); +else +RTE_LOG (ERR ,EAL ,"Cannot reserve memory\n" ); +#endif
You're left without any error messages on Windows. How about:
const char *err_str = "Cannot reserve memory\n";
#ifndef RTE_EXEC_ENV_WINDOWS
if (rte_errno == EADDRNOTAVAIL)
err_str = ...
#endifRTE_LOG(ERR, EAL, err_str);
or something like that?
Signature · 2 lines
Thanks, Anatoly
Re: [dpdk-dev] [PATCH v6 04/11] eal/mem: extract common code for memseg list initialization
Quoted 37 lines · Anatoly
Quoted 6 lines · Dmitry Kozlyuk
All supported OS create memory segment lists (MSL) and reserve VA space for them in a nearly identical way. Move common code into EAL private functions to reduce duplication.
Signed-off-by Dmitry Kozlyuk <[email protected]>---
<snip>
Quoted 14 lines
lib/librte_eal/common/eal_common_memory.c +14
-0+int +eal_memseg_list_alloc (struct rte_memseg_list *msl,int reserve_flags) +{ +size_t page_sz, mem_sz; +void *addr; + + page_sz = msl->page_sz; + mem_sz = page_sz * msl->memseg_arr.len; + + addr =eal_get_virtual_area ( + msl->base_va, &mem_sz, page_sz,0 , reserve_flags); +if (addr ==NULL ) { +#ifndef RTE_EXEC_ENV_WINDOWS +/* The hint would be misleading on Windows, but this function lib/librte_eal/common/eal_common_memory.c eal_get_virtual_area(void *requested_addr, size_t *size, · line 245
Quoted 10 lines
lib/librte_eal/common/eal_common_memory.c +10
-0+* is called from many places, including common code, +* so don't duplicate the message. +*/ +if (rte_errno ==EADDRNOTAVAIL ) +RTE_LOG (ERR ,EAL ,"Cannot reserve %llu bytes at [%p] - " +"please use '--" OPT_BASE_VIRTADDR "' option\n" , + (unsigned long long )mem_sz, msl->base_va); +else +RTE_LOG (ERR ,EAL ,"Cannot reserve memory\n" ); +#endif You're left without any error messages on Windows. How about:
Quoted 9 lines
const char
*err_str= "Cannot reserve memory\n";
#ifndefRTE_EXEC_ENV_WINDOWS
if (rte_errno== EADDRNOTAVAIL)err_str = ...#endif
RTE_LOG(ERR, EAL, err_str);or something like that?
How about removing generic error message here completely and printing more specific messages at call sites? In fact, almost all of them already do this. It would be more helpful in tracking down errors.
Signature · 1 line
Dmitry Kozlyuk
Re: [dpdk-dev] [PATCH v6 04/11] eal/mem: extract common code for memseg list initialization
Quoted 47 lines · Dmitry Kozlyuk
Quoted 44 lines · Anatoly
Quoted 6 lines · Dmitry Kozlyuk
All supported OS create memory segment lists (MSL) and reserve VA space for them in a nearly identical way. Move common code into EAL private functions to reduce duplication.
Signed-off-by Dmitry Kozlyuk <[email protected]>---
<snip>
Quoted 14 lines
lib/librte_eal/common/eal_common_memory.c +14
-0+int +eal_memseg_list_alloc (struct rte_memseg_list *msl,int reserve_flags) +{ +size_t page_sz, mem_sz; +void *addr; + + page_sz = msl->page_sz; + mem_sz = page_sz * msl->memseg_arr.len; + + addr =eal_get_virtual_area ( + msl->base_va, &mem_sz, page_sz,0 , reserve_flags); +if (addr ==NULL ) { +#ifndef RTE_EXEC_ENV_WINDOWS +/* The hint would be misleading on Windows, but this function lib/librte_eal/common/eal_common_memory.c eal_get_virtual_area(void *requested_addr, size_t *size, · line 245
Quoted 10 lines
lib/librte_eal/common/eal_common_memory.c +10
-0+* is called from many places, including common code, +* so don't duplicate the message. +*/ +if (rte_errno ==EADDRNOTAVAIL ) +RTE_LOG (ERR ,EAL ,"Cannot reserve %llu bytes at [%p] - " +"please use '--" OPT_BASE_VIRTADDR "' option\n" , + (unsigned long long )mem_sz, msl->base_va); +else +RTE_LOG (ERR ,EAL ,"Cannot reserve memory\n" ); +#endif You're left without any error messages on Windows. How about:
const char
*err_str= "Cannot reserve memory\n";
#ifndefRTE_EXEC_ENV_WINDOWS
if (rte_errno== EADDRNOTAVAIL)err_str = ...#endif
RTE_LOG(ERR, EAL, err_str);
Quoted 7 lines
Quoted 2 lines
or something like that?
How about removing generic error message here completely and printing more specific messages at call sites? In fact, almost all of them already do this. It would be more helpful in tracking down errors.
Agreed, let's do that :) We do pass up the rte_errno, correct? So, we should be able to do that.
Signature · 2 lines
Thanks, Anatoly
Re: [dpdk-dev] [PATCH v6 04/11] eal/mem: extract common code for memseg list initialization
On Wed, 10 Jun 2020 11:26:22 +0100
"Burakov, Anatoly" <[email protected]> wrote:
[snip]
Quoted 28 lines
Quoted 28 lines
Quoted 27 lines
Quoted 5 lines
lib/librte_eal/common/eal_common_memory.c +5
-0+ addr =eal_get_virtual_area ( + msl->base_va, &mem_sz, page_sz,0 , reserve_flags); +if (addr ==NULL ) { +#ifndef RTE_EXEC_ENV_WINDOWS +/* The hint would be misleading on Windows, but this function lib/librte_eal/common/eal_common_memory.c eal_get_virtual_area(void *requested_addr, size_t *size, · line 245
Quoted 10 lines
lib/librte_eal/common/eal_common_memory.c +10
-0+* is called from many places, including common code, +* so don't duplicate the message. +*/ +if (rte_errno ==EADDRNOTAVAIL ) +RTE_LOG (ERR ,EAL ,"Cannot reserve %llu bytes at [%p] - " +"please use '--" OPT_BASE_VIRTADDR "' option\n" , + (unsigned long long )mem_sz, msl->base_va); +else +RTE_LOG (ERR ,EAL ,"Cannot reserve memory\n" ); +#endif You're left without any error messages on Windows. How about:
const char
*err_str= "Cannot reserve memory\n";
#ifndefRTE_EXEC_ENV_WINDOWS
if (rte_errno== EADDRNOTAVAIL)err_str = ...#endif
RTE_LOG(ERR, EAL, err_str);or something like that?
Quoted 7 lines
Quoted 4 lines
How about removing generic error message here completely and printing more specific messages at call sites? In fact, almost all of them already do this. It would be more helpful in tracking down errors.
Agreed, let's do that :) We do pass up the
rte_errno, correct? So, we should be able to do that.
Actually, callers don't need rte_errno, because we only have to distinguish EADDRNOTAVAIL here, and eal_get_virtual_area() already prints precise diagnostics at WARNING and ERR level. rte_errno is preserved, however.
Signature · 1 line
Dmitry Kozlyuk
Re: [dpdk-dev] [PATCH v6 04/11] eal/mem: extract common code for memseg list initialization
Quoted 37 lines · Dmitry Kozlyuk
On Wed, 10 Jun 2020 11:26:22 +0100
"Burakov, Anatoly" <[email protected]> wrote:[snip]
Quoted 33 lines
Quoted 32 lines
Quoted 18 lines
Quoted 5 lines
lib/librte_eal/common/eal_common_memory.c +5
-0+ addr =eal_get_virtual_area ( + msl->base_va, &mem_sz, page_sz,0 , reserve_flags); +if (addr ==NULL ) { +#ifndef RTE_EXEC_ENV_WINDOWS +/* The hint would be misleading on Windows, but this function lib/librte_eal/common/eal_common_memory.c eal_get_virtual_area(void *requested_addr, size_t *size, · line 245
Quoted 10 lines
lib/librte_eal/common/eal_common_memory.c +10
-0+* is called from many places, including common code, +* so don't duplicate the message. +*/ +if (rte_errno ==EADDRNOTAVAIL ) +RTE_LOG (ERR ,EAL ,"Cannot reserve %llu bytes at [%p] - " +"please use '--" OPT_BASE_VIRTADDR "' option\n" , + (unsigned long long )mem_sz, msl->base_va); +else +RTE_LOG (ERR ,EAL ,"Cannot reserve memory\n" ); +#endif You're left without any error messages on Windows. How about:
Quoted 9 lines
const char
*err_str= "Cannot reserve memory\n";
#ifndefRTE_EXEC_ENV_WINDOWS
if (rte_errno== EADDRNOTAVAIL)err_str = ...#endif
RTE_LOG(ERR, EAL, err_str);or something like that?
How about removing generic error message here completely and printing more specific messages at call sites? In fact, almost all of them already do this. It would be more helpful in tracking down errors.
Quoted 7 lines
Quoted 2 lines
Agreed, let's do that :) We do pass up the
rte_errno, correct? So, we should be able to do that.Actually, callers don't need
rte_errno, because we only have to distinguish EADDRNOTAVAIL here, andeal_get_virtual_area()already prints precise diagnostics at WARNING and ERR level.rte_errnois preserved, however.
Not sure i agree, we still need the "--base-virtaddr" hint, and we can only do that from the caller (without #ifdef-ery here), so we do need rte_errno for that.
Signature · 2 lines
Thanks, Anatoly
Re: [dpdk-dev] [PATCH v6 04/11] eal/mem: extract common code for memseg list initialization
Quoted 41 lines · Anatoly
Quoted 40 lines · Dmitry Kozlyuk
On Wed, 10 Jun 2020 11:26:22 +0100
"Burakov, Anatoly" <[email protected]> wrote:[snip]
Quoted 35 lines
Quoted 25 lines
Quoted 25 lines
Quoted 5 lines
lib/librte_eal/common/eal_common_memory.c +5
-0+ addr =eal_get_virtual_area ( + msl->base_va, &mem_sz, page_sz,0 , reserve_flags); +if (addr ==NULL ) { +#ifndef RTE_EXEC_ENV_WINDOWS +/* The hint would be misleading on Windows, but this function lib/librte_eal/common/eal_common_memory.c eal_get_virtual_area(void *requested_addr, size_t *size, · line 245
Quoted 10 lines
lib/librte_eal/common/eal_common_memory.c +10
-0+* is called from many places, including common code, +* so don't duplicate the message. +*/ +if (rte_errno ==EADDRNOTAVAIL ) +RTE_LOG (ERR ,EAL ,"Cannot reserve %llu bytes at [%p] - " +"please use '--" OPT_BASE_VIRTADDR "' option\n" , + (unsigned long long )mem_sz, msl->base_va); +else +RTE_LOG (ERR ,EAL ,"Cannot reserve memory\n" ); +#endif You're left without any error messages on Windows. How about:
const char
*err_str= "Cannot reserve memory\n";
#ifndefRTE_EXEC_ENV_WINDOWS
if (rte_errno== EADDRNOTAVAIL)err_str = ...#endif
RTE_LOG(ERR, EAL, err_str);Quoted 7 lines
Quoted 2 lines
or something like that?
How about removing generic error message here completely and printing more specific messages at call sites? In fact, almost all of them already do this. It would be more helpful in tracking down errors.
Agreed, let's do that :) We do pass up the
rte_errno, correct? So, we should be able to do that.
Quoted 8 lines
Quoted 4 lines
Actually, callers don't need
rte_errno, because we only have to distinguish EADDRNOTAVAIL here, andeal_get_virtual_area()already prints precise diagnostics at WARNING and ERR level.rte_errnois preserved, however.Not sure i agree, we still need the "--base-virtaddr" hint, and we can only do that from the caller (without #ifdef-ery here), so we do need
rte_errnofor that.
Maybe we're talking about different things. The "--base-virtaddr" hint is printed from eal_memseg_list_alloc() on Unices for EADDRNOTAVAIL. This is handy to avoid duplicating the hint and to provide context, so let's keep it despite #ifndef.
Otherwise, a generic error is printed from the same function (mistakenly not on Windows in v6). This generic error adds nothing to eal_get_virtual_area() logs and also doesn't help to known which exact eal_memseg_list_alloc() failed. If instead callers printed their own messages, it would be clear which call failed and in which context. Generic error can than be removed, eal_memseg_list_alloc() code simplified. Callers can inspect rte_errno if they ever need it, but really they don't, because hint is printed by eal_memseg_list_alloc() and eal_get_virtual_area() prints even more precise logs. This is what I did in v8.
Signature · 1 line
Dmitry Kozlyuk
Re: [dpdk-dev] [PATCH v6 04/11] eal/mem: extract common code for memseg list initialization
Quoted 58 lines · Dmitry Kozlyuk
Quoted 41 lines · Anatoly
Quoted 40 lines · Dmitry Kozlyuk
On Wed, 10 Jun 2020 11:26:22 +0100
"Burakov, Anatoly" <[email protected]> wrote:[snip]
Quoted 35 lines
Quoted 25 lines
Quoted 25 lines
Quoted 5 lines
(quoted diff) +5
-0+ addr = eal_get_virtual_area( + msl->base_va, &mem_sz, page_sz, 0, reserve_flags); + if (addr == NULL) { +#ifndef RTE_EXEC_ENV_WINDOWS + /* The hint would be misleading on Windows, but this functionReview
Quoted 10 lines
(quoted diff) +10
-0+ * is called from many places, including common code, + * so don't duplicate the message. + */ + if (rte_errno == EADDRNOTAVAIL) + RTE_LOG(ERR, EAL, "Cannot reserve %llu bytes at [%p] - " + "please use '--" OPT_BASE_VIRTADDR "' option\n", + (unsigned long long)mem_sz, msl->base_va); + else + RTE_LOG(ERR, EAL, "Cannot reserve memory\n"); +#endifYou're left without any error messages on Windows. How about:
const char
*err_str= "Cannot reserve memory\n";
#ifndefRTE_EXEC_ENV_WINDOWS
if (rte_errno== EADDRNOTAVAIL)err_str = ...#endif
RTE_LOG(ERR, EAL, err_str);Quoted 7 lines
Quoted 2 lines
or something like that?
How about removing generic error message here completely and printing more specific messages at call sites? In fact, almost all of them already do this. It would be more helpful in tracking down errors.
Agreed, let's do that :) We do pass up the
rte_errno, correct? So, we should be able to do that.Quoted 8 lines
Quoted 4 lines
Actually, callers don't need
rte_errno, because we only have to distinguish EADDRNOTAVAIL here, andeal_get_virtual_area()already prints precise diagnostics at WARNING and ERR level.rte_errnois preserved, however.Not sure i agree, we still need the "--base-virtaddr" hint, and we can only do that from the caller (without #ifdef-ery here), so we do need
rte_errnofor that.Maybe we're talking about different things. The "--base-virtaddr" hint is printed from
eal_memseg_list_alloc()on Unices for EADDRNOTAVAIL. This is handy to avoid duplicating the hint and to provide context, so let's keep it despite #ifndef.
Quoted 10 lines
Otherwise, a generic error is printed from the same function (mistakenly not on Windows in v6). This generic error adds nothing to
eal_get_virtual_area()logs and also doesn't help to known which exacteal_memseg_list_alloc()failed. If instead callers printed their own messages, it would be clear which call failed and in which context. Generic error can than be removed,eal_memseg_list_alloc()code simplified. Callers can inspectrte_errnoif they ever need it, but really they don't, because hint is printed byeal_memseg_list_alloc()andeal_get_virtual_area()prints even more precise logs. This is what I did in v8.
Right, OK :)
Signature · 2 lines
Thanks, Anatoly
PATCH v6 5/11 eal/mem: extract common code for dynamic memory allocation
In reply to Dmitry KozlyukCode in Linux EAL that supports dynamic memory allocation (as opposed to static allocation used by FreeBSD) is not OS-dependent and can be reused by Windows EAL. Move such code to a file compiled only for the OS that require it. Keep Anatoly Burakov maintainer of extracted code.
--- MAINTAINERS | 1 + lib/librte_eal/common/eal_common_dynmem.c | 521 +++++++++++++++++++++ lib/librte_eal/common/eal_private.h | 43 +- lib/librte_eal/common/meson.build | 4 + lib/librte_eal/freebsd/eal_memory.c | 12 +- lib/librte_eal/linux/Makefile | 1 + lib/librte_eal/linux/eal_memory.c | 523 +--------------------- 7 files changed, 582 insertions(+), 523 deletions(-) create mode 100644 lib/librte_eal/common/eal_common_dynmem.c
MAINTAINERS +1 -0
@@ -208,6 +208,7 @@ F: lib/librte_eal/include/rte_fbarray.h F: lib/librte_eal/include/rte_mem* F: lib/librte_eal/include/rte_malloc.h F: lib/librte_eal/common/*malloc* +F: lib/librte_eal/common/eal_common_dynmem.c F: lib/librte_eal/common/eal_common_fbarray.c F: lib/librte_eal/common/eal_common_mem* F: lib/librte_eal/common/eal_hugepages.h
lib/librte_eal/common/eal_common_dynmem.c new file +521 -0
521 lines not on this page · raw message
lib/librte_eal/common/eal_private.h +42 -1
61 lines not on this page · raw message
lib/librte_eal/common/meson.build +4 -0
@@ -56,3 +56,7 @@ sources += files( 'rte_reciprocal.c', 'rte_service.c', ) + +if is_linux + sources += files('eal_common_dynmem.c') +endif
lib/librte_eal/freebsd/eal_memory.c +2 -10
@@ -317,14 +317,6 @@ get_mem_amount(uint64_t page_sz, uint64_t max_mem)return RTE_ALIGN (area_sz, page_sz); } -static int -memseg_list_init (struct rte_memseg_list *msl,uint64_t page_sz, -int n_segs,int socket_id,int type_msl_idx) -{ -return eal_memseg_list_init ( - msl, page_sz, n_segs, socket_id, type_msl_idx, false); -} -static int memseg_list_alloc (struct rte_memseg_list *msl) {
@@ -421,8 +413,8 @@ memseg_primary_init(void) cur_max_mem); n_segs = cur_mem / hugepage_sz; -if (memseg_list_init (msl, hugepage_sz, n_segs, -0 , type_msl_idx)) +if (eal_memseg_list_init (msl, hugepage_sz, n_segs, +0 , type_msl_idx, false))return -1 ; total_segs += msl->memseg_arr.len;
lib/librte_eal/linux/Makefile +1 -0
@@ -50,6 +50,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_common_timer.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_common_memzone.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_common_log.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_common_launch.c +SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_common_dynmem.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_common_mcfg.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_common_memalloc.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_common_memory.c
lib/librte_eal/linux/eal_memory.c +11 -512
586 lines not on this page · raw message
Signature · 1 line
2.25.4
PATCH v6 6/11 trace: add size_t field emitter
In reply to Dmitry KozlyukIt is not guaranteed that sizeof(long) == sizeof(size_t). On Windows, sizeof(long) == 4 and sizeof(size_t) == 8 for 64-bit programs. Tracepoints using "long" field emitter are therefore invalid there. Add dedicated field emitter for size_t and use it to store size_t values in all existing tracepoints.
--- lib/librte_eal/include/rte_eal_trace.h | 8 ++++---- lib/librte_eal/include/rte_trace_point.h | 3 +++ lib/librte_mempool/rte_mempool_trace.h | 10 +++++----- 3 files changed, 12 insertions(+), 9 deletions(-)
lib/librte_eal/include/rte_eal_trace.h +4 -4
@@ -143,7 +143,7 @@ RTE_TRACE_POINT( RTE_TRACE_POINT_ARGS(const char *type, size_t size, unsigned int align, int socket, void *ptr),rte_trace_point_emit_string (type); -rte_trace_point_emit_long (size); +rte_trace_point_emit_size_t (size);rte_trace_point_emit_u32 (align);rte_trace_point_emit_int (socket);rte_trace_point_emit_ptr (ptr);
@@ -154,7 +154,7 @@ RTE_TRACE_POINT( RTE_TRACE_POINT_ARGS(const char *type, size_t size, unsigned int align, int socket, void *ptr),rte_trace_point_emit_string (type); -rte_trace_point_emit_long (size); +rte_trace_point_emit_size_t (size);rte_trace_point_emit_u32 (align);rte_trace_point_emit_int (socket);rte_trace_point_emit_ptr (ptr);
@@ -164,7 +164,7 @@ RTE_TRACE_POINT( rte_eal_trace_mem_realloc, RTE_TRACE_POINT_ARGS(size_t size, unsigned int align, int socket, void *ptr), -rte_trace_point_emit_long (size); +rte_trace_point_emit_size_t (size);rte_trace_point_emit_u32 (align);rte_trace_point_emit_int (socket);rte_trace_point_emit_ptr (ptr);
@@ -183,7 +183,7 @@ RTE_TRACE_POINT( unsigned int flags, unsigned int align, unsigned int bound, const void *mz),rte_trace_point_emit_string (name ); -rte_trace_point_emit_long (len); +rte_trace_point_emit_size_t (len);rte_trace_point_emit_int (socket_id);rte_trace_point_emit_u32 (flags);rte_trace_point_emit_u32 (align);
lib/librte_eal/include/rte_trace_point.h +3 -0
@@ -138,6 +138,8 @@ _tp _args \#define rte_trace_point_emit_int (val)/** Tracepoint function payload for long datatype */ #define rte_trace_point_emit_long (val) +/** Tracepoint function payload for size_t datatype */ +#define rte_trace_point_emit_size_t (val)/** Tracepoint function payload for float datatype */ #define rte_trace_point_emit_float (val)/** Tracepoint function payload for double datatype */
@@ -395,6 +397,7 @@ do { \#define rte_trace_point_emit_i8 (in) __rte_trace_point_emit(in, int8_t)#define rte_trace_point_emit_int (in) __rte_trace_point_emit(in, int32_t)#define rte_trace_point_emit_long (in) __rte_trace_point_emit(in, long) +#define rte_trace_point_emit_size_t (in) __rte_trace_point_emit(in, size_t)#define rte_trace_point_emit_float (in) __rte_trace_point_emit(in, float)#define rte_trace_point_emit_double (in) __rte_trace_point_emit(in, double)#define rte_trace_point_emit_ptr (in) __rte_trace_point_emit(in, uintptr_t)
lib/librte_mempool/rte_mempool_trace.h +5 -5
@@ -72,7 +72,7 @@ RTE_TRACE_POINT(rte_trace_point_emit_string (mempool->name);rte_trace_point_emit_ptr (vaddr);rte_trace_point_emit_u64 (iova); -rte_trace_point_emit_long (len); +rte_trace_point_emit_size_t (len);rte_trace_point_emit_ptr (free_cb);rte_trace_point_emit_ptr (opaque); )
@@ -84,8 +84,8 @@ RTE_TRACE_POINT(rte_trace_point_emit_ptr (mempool);rte_trace_point_emit_string (mempool->name);rte_trace_point_emit_ptr (addr); -rte_trace_point_emit_long (len); -rte_trace_point_emit_long (pg_sz); +rte_trace_point_emit_size_t (len); +rte_trace_point_emit_size_t (pg_sz);rte_trace_point_emit_ptr (free_cb);rte_trace_point_emit_ptr (opaque); )
@@ -126,7 +126,7 @@ RTE_TRACE_POINT( RTE_TRACE_POINT_ARGS(struct rte_mempool *mempool, size_t pg_sz),rte_trace_point_emit_ptr (mempool);rte_trace_point_emit_string (mempool->name); -rte_trace_point_emit_long (pg_sz); +rte_trace_point_emit_size_t (pg_sz); ) RTE_TRACE_POINT(
@@ -139,7 +139,7 @@ RTE_TRACE_POINT(rte_trace_point_emit_u32 (max_objs);rte_trace_point_emit_ptr (vaddr);rte_trace_point_emit_u64 (iova); -rte_trace_point_emit_long (len); +rte_trace_point_emit_size_t (len);rte_trace_point_emit_ptr (obj_cb);rte_trace_point_emit_ptr (obj_cb_arg); )
Signature · 1 line
2.25.4
Re: [dpdk-dev] [PATCH v6 06/11] trace: add size_t field emitter
Quoted 8 lines · Dmitry Kozlyuk
It is not guaranteed that sizeof(long) ==
sizeof(size_t). On Windows, sizeof(long) == 4 andsizeof(size_t)== 8 for 64-bit programs. Tracepoints using "long" field emitter are therefore invalid there. Add dedicated field emitter forsize_tand use it to storesize_tvalues in all existing tracepoints.Signed-off-by Dmitry Kozlyuk <[email protected]>
Quoted 112 lines
--- lib/librte_eal/include/rte_eal_trace.h | 8 ++++---- lib/librte_eal/include/rte_trace_point.h | 3 +++ lib/librte_mempool/rte_mempool_trace.h | 10 +++++----- 3 files changed, 12 insertions(+), 9 deletions(-)lib/librte_eal/include/rte_eal_trace.h +4
-4@@ -143,7 +143,7 @@ RTE_TRACE_POINT( RTE_TRACE_POINT_ARGS(const char *type, size_t size, unsigned int align, int socket, void *ptr),rte_trace_point_emit_string (type); -rte_trace_point_emit_long (size); +rte_trace_point_emit_size_t (size);rte_trace_point_emit_u32 (align);rte_trace_point_emit_int (socket);rte_trace_point_emit_ptr (ptr);@@ -154,7 +154,7 @@ RTE_TRACE_POINT( RTE_TRACE_POINT_ARGS(const char *type, size_t size, unsigned int align, int socket, void *ptr),rte_trace_point_emit_string (type); -rte_trace_point_emit_long (size); +rte_trace_point_emit_size_t (size);rte_trace_point_emit_u32 (align);rte_trace_point_emit_int (socket);rte_trace_point_emit_ptr (ptr);@@ -164,7 +164,7 @@ RTE_TRACE_POINT( rte_eal_trace_mem_realloc, RTE_TRACE_POINT_ARGS(size_t size, unsigned int align, int socket, void *ptr), -rte_trace_point_emit_long (size); +rte_trace_point_emit_size_t (size);rte_trace_point_emit_u32 (align);rte_trace_point_emit_int (socket);rte_trace_point_emit_ptr (ptr);@@ -183,7 +183,7 @@ RTE_TRACE_POINT( unsigned int flags, unsigned int align, unsigned int bound, const void *mz),rte_trace_point_emit_string (name ); -rte_trace_point_emit_long (len); +rte_trace_point_emit_size_t (len);rte_trace_point_emit_int (socket_id);rte_trace_point_emit_u32 (flags);rte_trace_point_emit_u32 (align);lib/librte_eal/include/rte_trace_point.h +3
-0@@ -138,6 +138,8 @@ _tp _args \#define rte_trace_point_emit_int (val)/** Tracepoint function payload for long datatype */ #define rte_trace_point_emit_long (val) +/** Tracepoint function payload for size_t datatype */ +#define rte_trace_point_emit_size_t (val)/** Tracepoint function payload for float datatype */ #define rte_trace_point_emit_float (val)/** Tracepoint function payload for double datatype */ @@ -395,6 +397,7 @@ do { \#define rte_trace_point_emit_i8 (in) __rte_trace_point_emit(in, int8_t)#define rte_trace_point_emit_int (in) __rte_trace_point_emit(in, int32_t)#define rte_trace_point_emit_long (in) __rte_trace_point_emit(in, long) +#define rte_trace_point_emit_size_t (in) __rte_trace_point_emit(in, size_t)#define rte_trace_point_emit_float (in) __rte_trace_point_emit(in, float)#define rte_trace_point_emit_double (in) __rte_trace_point_emit(in, double)#define rte_trace_point_emit_ptr (in) __rte_trace_point_emit(in, uintptr_t)lib/librte_mempool/rte_mempool_trace.h +5
-6@@ -72,7 +72,7 @@ RTE_TRACE_POINT(rte_trace_point_emit_string (mempool->name);rte_trace_point_emit_ptr (vaddr);rte_trace_point_emit_u64 (iova); -rte_trace_point_emit_long (len); +rte_trace_point_emit_size_t (len);rte_trace_point_emit_ptr (free_cb);rte_trace_point_emit_ptr (opaque); )@@ -84,8 +84,8 @@ RTE_TRACE_POINT(rte_trace_point_emit_ptr (mempool);rte_trace_point_emit_string (mempool->name);rte_trace_point_emit_ptr (addr); -rte_trace_point_emit_long (len); -rte_trace_point_emit_long (pg_sz); +rte_trace_point_emit_size_t (len); +rte_trace_point_emit_size_t (pg_sz);rte_trace_point_emit_ptr (free_cb);rte_trace_point_emit_ptr (opaque); )@@ -126,7 +126,7 @@ RTE_TRACE_POINT( RTE_TRACE_POINT_ARGS(struct rte_mempool *mempool, size_t pg_sz),rte_trace_point_emit_ptr (mempool);rte_trace_point_emit_string (mempool->name); -rte_trace_point_emit_long (pg_sz); +rte_trace_point_emit_size_t (pg_sz); ) RTE_TRACE_POINT(@@ -139,7 +139,7 @@ RTE_TRACE_POINT(rte_trace_point_emit_u32 (max_objs);rte_trace_point_emit_ptr (vaddr);rte_trace_point_emit_u64 (iova); -rte_trace_point_emit_long (len); +rte_trace_point_emit_size_t (len);rte_trace_point_emit_ptr (obj_cb);rte_trace_point_emit_ptr (obj_cb_arg); ) --2.25.4
PATCH v6 7/11 eal/windows: add tracing support stubs
In reply to Dmitry KozlyukEAL common code depends on tracepoint calls, but generic implementation cannot be enabled on Windows due to missing standard library facilities. Add stub functions to support tracepoint compilation, so that common code does not have to conditionally include tracepoints until proper support is added.
--- lib/librte_eal/common/eal_common_thread.c | 5 +--- lib/librte_eal/common/meson.build | 1 + lib/librte_eal/windows/eal.c | 34 ++++++++++++++++++++++- 3 files changed, 35 insertions(+), 5 deletions(-)
lib/librte_eal/common/eal_common_thread.c +1 -4
@@ -15,9 +15,7 @@#include <rte_lcore.h> #include <rte_memory.h> #include <rte_log.h> -#ifndef RTE_EXEC_ENV_WINDOWS #include <rte_trace_point.h> -#endif #include "eal_internal_cfg.h" #include "eal_private.h"
@@ -169,9 +167,8 @@ static void *rte_thread_init(void *arg)free (params); } -#ifndef RTE_EXEC_ENV_WINDOWS __rte_trace_mem_per_thread_alloc (); -#endif +return start_routine (routine_arg); }
lib/librte_eal/common/meson.build +1 -0
@@ -14,6 +14,7 @@ if is_windows 'eal_common_log.c', 'eal_common_options.c', 'eal_common_thread.c', + 'eal_common_trace_points.c', ) subdir_done() endif
lib/librte_eal/windows/eal.c +33 -1
@@ -17,6 +17,7 @@#include <eal_filesystem.h> #include <eal_options.h> #include <eal_private.h> +#include <rte_trace_point.h> #include "eal_windows.h"
@@ -221,7 +222,38 @@ rte_eal_init_alert(const char *msg)RTE_LOG (ERR ,EAL ,"%s\n" , msg); } -/* Launch threads, called at application init(). */ +/* Stubs to enable EAL trace point compilation +* until eal_common_trace.c can be compiled. +*/ + +RTE_DEFINE_PER_LCORE(volatile int, trace_point_sz); +RTE_DEFINE_PER_LCORE(void *, trace_mem); + +void +__rte_trace_mem_per_thread_alloc (void ) +{ +} + +void +__rte_trace_point_emit_field (size_t sz,const char *field, +const char *type) +{ +RTE_SET_USED (sz); +RTE_SET_USED (field); +RTE_SET_USED (type); +} + +int +__rte_trace_point_register (rte_trace_point_t *trace,const char *name, +void (*register_fn)(void )) +{ +RTE_SET_USED (trace); +RTE_SET_USED (name); +RTE_SET_USED (register_fn); +return -ENOTSUP ; +} + +/* Launch threads, called at application init(). */ int rte_eal_init (int argc,char **argv) {
Signature · 1 line
2.25.4
PATCH v6 8/11 eal/windows: replace sys/queue.h with a complete one from FreeBSD
In reply to Dmitry KozlyukLimited version imported previously lacks at least SLIST macros. Import a complete file from FreeBSD, since its license exception is already approved by Technical Board.
--- lib/librte_eal/windows/include/sys/queue.h | 663 +++++++++++++++++++-- 1 file changed, 601 insertions(+), 62 deletions(-)
lib/librte_eal/windows/include/sys/queue.h +601 -62
@@ -8,7 +8,36 @@#define _SYS_QUEUE_H_/* -* This file defines tail queues. +* This file defines four types of data structures: singly-linked lists, +* singly-linked tail queues, lists and tail queues. +* +* A singly-linked list is headed by a single forward pointer. The elements +* are singly linked for minimum space and pointer manipulation overhead at +* the expense of O(n) removal for arbitrary elements. New elements can be +* added to the list after an existing element or at the head of the list. +* Elements being removed from the head of the list should use the explicit +* macro for this purpose for optimum efficiency. A singly-linked list may +* only be traversed in the forward direction. Singly-linked lists are ideal +* for applications with large datasets and few or no removals or for +* implementing a LIFO queue. +* +* A singly-linked tail queue is headed by a pair of pointers, one to the +* head of the list and the other to the tail of the list. The elements are +* singly linked for minimum space and pointer manipulation overhead at the +* expense of O(n) removal for arbitrary elements. New elements can be added +* to the list after an existing element, at the head of the list, or at the +* end of the list. Elements being removed from the head of the tail queue +* should use the explicit macro for this purpose for optimum efficiency. +* A singly-linked tail queue may only be traversed in the forward direction. +* Singly-linked tail queues are ideal for applications with large datasets +* and few or no removals or for implementing a FIFO queue. +* +* A list is headed by a single forward pointer (or an array of forward +* pointers for a hash table header). The elements are doubly linked +* so that an arbitrary element can be removed without a need to +* traverse the list. New elements can be added to the list before +* or after an existing element or at the head of the list. A list +* may be traversed in either direction. * * A tail queue is headed by a pair of pointers, one to the head of the * list and the other to the tail of the list. The elements are doubly
@@ -17,65 +46,93 @@* after an existing element, at the head of the list, or at the end of * the list. A tail queue may be traversed in either direction. * +* For details on the use of these macros, see the queue(3) manual page. +* * Below is a summary of implemented functions where: * + means the macro is available * - means the macro is not available * s means the macro is available but is slow (runs in O(n) time) * -* TAILQ -* _HEAD + -* _CLASS_HEAD + -* _HEAD_INITIALIZER + -* _ENTRY + -* _CLASS_ENTRY + -* _INIT + -* _EMPTY + -* _FIRST + -* _NEXT + -* _PREV + -* _LAST + -* _LAST_FAST + -* _FOREACH + -* _FOREACH_FROM + -* _FOREACH_SAFE + -* _FOREACH_FROM_SAFE + -* _FOREACH_REVERSE + -* _FOREACH_REVERSE_FROM + -* _FOREACH_REVERSE_SAFE + -* _FOREACH_REVERSE_FROM_SAFE + -* _INSERT_HEAD + -* _INSERT_BEFORE + -* _INSERT_AFTER + -* _INSERT_TAIL + -* _CONCAT + -* _REMOVE_AFTER - -* _REMOVE_HEAD - -* _REMOVE + -* _SWAP + +* SLIST LIST STAILQ TAILQ +* _HEAD + + + + +* _CLASS_HEAD + + + + +* _HEAD_INITIALIZER + + + + +* _ENTRY + + + + +* _CLASS_ENTRY + + + + +* _INIT + + + + +* _EMPTY + + + + +* _FIRST + + + + +* _NEXT + + + + +* _PREV - + - + +* _LAST - - + + +* _LAST_FAST - - - + +* _FOREACH + + + + +* _FOREACH_FROM + + + + +* _FOREACH_SAFE + + + + +* _FOREACH_FROM_SAFE + + + + +* _FOREACH_REVERSE - - - + +* _FOREACH_REVERSE_FROM - - - + +* _FOREACH_REVERSE_SAFE - - - + +* _FOREACH_REVERSE_FROM_SAFE - - - + +* _INSERT_HEAD + + + + +* _INSERT_BEFORE - + - + +* _INSERT_AFTER + + + + +* _INSERT_TAIL - - + + +* _CONCAT s s + + +* _REMOVE_AFTER + - + - +* _REMOVE_HEAD + - + - +* _REMOVE s + s + +* _SWAP + + + + * */ - -#ifdef __cplusplus -extern "C" { +#ifdef QUEUE_MACRO_DEBUG +#warn Use QUEUE_MACRO_DEBUG_TRACE and/or QUEUE_MACRO_DEBUG_TRASH +#define QUEUE_MACRO_DEBUG_TRACE +#define QUEUE_MACRO_DEBUG_TRASH #endif -/* - * List definitions. - */ -#define LIST_HEAD(name, type) \ -struct name { \ - struct type *lh_first; /* first element */ \ -} +#ifdef QUEUE_MACRO_DEBUG_TRACE +/* Store the last 2 places the queue element or head was altered */ +struct qm_trace { +unsigned long lastline; +unsigned long prevline; +const char *lastfile; +const char *prevfile; +}; + +#define TRACEBUF struct qm_trace trace; +#define TRACEBUF_INITIALIZER { __LINE__, 0, __FILE__, NULL } , + +#define QMD_TRACE_HEAD (head) do { \ + (head)->trace.prevline = (head)->trace.lastline; \ + (head)->trace.prevfile = (head)->trace.lastfile; \ + (head)->trace.lastline = __LINE__; \ + (head)->trace.lastfile = __FILE__; \ +} while (0) +#define QMD_TRACE_ELEM (elem) do { \ + (elem)->trace.prevline = (elem)->trace.lastline; \ + (elem)->trace.prevfile = (elem)->trace.lastfile; \ + (elem)->trace.lastline = __LINE__; \ + (elem)->trace.lastfile = __FILE__; \ +} while (0) + +#else /* !QUEUE_MACRO_DEBUG_TRACE */ #define QMD_TRACE_ELEM (elem)#define QMD_TRACE_HEAD (head)#define TRACEBUF #define TRACEBUF_INITIALIZER +#endif /* QUEUE_MACRO_DEBUG_TRACE */ +#ifdef QUEUE_MACRO_DEBUG_TRASH +#define QMD_SAVELINK (name, link) void **name = (void *)&(link) +#define TRASHIT (x) do {(x) = (void *)-1;} while (0) +#define QMD_IS_TRASHED (x) ((x) == (void *)(intptr_t)-1) +#else /* !QUEUE_MACRO_DEBUG_TRASH */ +#define QMD_SAVELINK (name, link)#define TRASHIT (x)#define QMD_IS_TRASHED (x) 0 - -#define QMD_SAVELINK(name, link) +#endif /* QUEUE_MACRO_DEBUG_TRASH */ #ifdef __cplusplus/*
@@ -86,6 +143,445 @@ struct name { \#define QUEUE_TYPEOF (type) struct type#endif +/* +* Singly-linked List declarations. +*/ +#define SLIST_HEAD (name, type) \ +struct name { \ + struct type *slh_first;/* first element */ \ +} + +#define SLIST_CLASS_HEAD(name, type) \ +struct name { \ + class type *slh_first; /* first element */ \ +} + +#define SLIST_HEAD_INITIALIZER (head) \ + { NULL } + +#define SLIST_ENTRY(type) \ +struct { \ + struct type *sle_next; /* next element */ \ +} + +#define SLIST_CLASS_ENTRY(type) \ +struct { \ + class type *sle_next; /* next element */ \ +} + +/* +* Singly-linked List functions. +*/ +#if (defined(_KERNEL) && defined(INVARIANTS )) +#define QMD_SLIST_CHECK_PREVPTR (prevp, elm) do { \ + if (*(prevp) != (elm)) \ + panic("Bad prevptr *(%p) == %p != %p", \ + (prevp), *(prevp), (elm)); \ +} while (0) +#else +#define QMD_SLIST_CHECK_PREVPTR (prevp, elm) +#endif + +#define SLIST_CONCAT (head1, head2, type, field) do { \ + QUEUE_TYPEOF(type) *curelm = SLIST_FIRST(head1); \ + if (curelm == NULL) { \ + if ((SLIST_FIRST(head1) = SLIST_FIRST(head2)) != NULL) \ + SLIST_INIT(head2); \ + } else if (SLIST_FIRST(head2) != NULL) { \ + while (SLIST_NEXT(curelm, field) != NULL) \ + curelm = SLIST_NEXT(curelm, field); \ + SLIST_NEXT(curelm, field) = SLIST_FIRST(head2); \ + SLIST_INIT(head2); \ + } \ +} while (0) + +#define SLIST_EMPTY (head) ((head)->slh_first == NULL) + +#define SLIST_FIRST (head) ((head)->slh_first) + +#define SLIST_FOREACH (var, head, field) \ + for ((var) = SLIST_FIRST((head)); \ + (var); \ + (var) = SLIST_NEXT((var), field)) + +#define SLIST_FOREACH_FROM (var, head, field) \ + for ((var) = ((var) ? (var) : SLIST_FIRST((head))); \ + (var); \ + (var) = SLIST_NEXT((var), field)) + +#define SLIST_FOREACH_SAFE (var, head, field, tvar) \ + for ((var) = SLIST_FIRST((head)); \ + (var) && ((tvar) = SLIST_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define SLIST_FOREACH_FROM_SAFE (var, head, field, tvar) \ + for ((var) = ((var) ? (var) : SLIST_FIRST((head))); \ + (var) && ((tvar) = SLIST_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define SLIST_FOREACH_PREVPTR (var, varp, head, field) \ + for ((varp) = &SLIST_FIRST((head)); \ + ((var) = *(varp)) != NULL; \ + (varp) = &SLIST_NEXT((var), field)) + +#define SLIST_INIT (head) do { \ + SLIST_FIRST((head)) = NULL; \ +} while (0) + +#define SLIST_INSERT_AFTER (slistelm, elm, field) do { \ + SLIST_NEXT((elm), field) = SLIST_NEXT((slistelm), field); \ + SLIST_NEXT((slistelm), field) = (elm); \ +} while (0) + +#define SLIST_INSERT_HEAD (head, elm, field) do { \ + SLIST_NEXT((elm), field) = SLIST_FIRST((head)); \ + SLIST_FIRST((head)) = (elm); \ +} while (0) + +#define SLIST_NEXT (elm, field) ((elm)->field.sle_next) + +#define SLIST_REMOVE (head, elm, type, field) do { \ + QMD_SAVELINK(oldnext, (elm)->field.sle_next); \ + if (SLIST_FIRST((head)) == (elm)) { \ + SLIST_REMOVE_HEAD((head), field); \ + } \ + else { \ + QUEUE_TYPEOF(type) *curelm = SLIST_FIRST(head); \ + while (SLIST_NEXT(curelm, field) != (elm)) \ + curelm = SLIST_NEXT(curelm, field); \ + SLIST_REMOVE_AFTER(curelm, field); \ + } \ + TRASHIT(*oldnext); \ +} while (0) + +#define SLIST_REMOVE_AFTER (elm, field) do { \ + SLIST_NEXT(elm, field) = \ + SLIST_NEXT(SLIST_NEXT(elm, field), field); \ +} while (0) + +#define SLIST_REMOVE_HEAD (head, field) do { \ + SLIST_FIRST((head)) = SLIST_NEXT(SLIST_FIRST((head)), field); \ +} while (0) + +#define SLIST_REMOVE_PREVPTR (prevp, elm, field) do { \ + QMD_SLIST_CHECK_PREVPTR(prevp, elm); \ + *(prevp) = SLIST_NEXT(elm, field); \ + TRASHIT((elm)->field.sle_next); \ +} while (0) + +#define SLIST_SWAP (head1, head2, type) do { \ + QUEUE_TYPEOF(type) *swap_first = SLIST_FIRST(head1); \ + SLIST_FIRST(head1) = SLIST_FIRST(head2); \ + SLIST_FIRST(head2) = swap_first; \ +} while (0) + +/* +* Singly-linked Tail queue declarations. +*/ +#define STAILQ_HEAD (name, type) \ +struct name { \ + struct type *stqh_first;/* first element */ \ +struct type **stqh_last;/* addr of last next element */ \ +} + +#define STAILQ_CLASS_HEAD(name, type) \ +struct name { \ + class type *stqh_first; /* first element */ \ + class type **stqh_last; /* addr of last next element */ \ +} + +#define STAILQ_HEAD_INITIALIZER (head) \ + { NULL, &(head).stqh_first } + +#define STAILQ_ENTRY(type) \ +struct { \ + struct type *stqe_next; /* next element */ \ +} + +#define STAILQ_CLASS_ENTRY(type) \ +struct { \ + class type *stqe_next; /* next element */ \ +} + +/* +* Singly-linked Tail queue functions. +*/ +#define STAILQ_CONCAT (head1, head2) do { \ + if (!STAILQ_EMPTY((head2))) { \ + *(head1)->stqh_last = (head2)->stqh_first; \ + (head1)->stqh_last = (head2)->stqh_last; \ + STAILQ_INIT((head2)); \ + } \ +} while (0) + +#define STAILQ_EMPTY (head) ((head)->stqh_first == NULL) + +#define STAILQ_FIRST (head) ((head)->stqh_first) + +#define STAILQ_FOREACH (var, head, field) \ + for((var) = STAILQ_FIRST((head)); \ + (var); \ + (var) = STAILQ_NEXT((var), field)) + +#define STAILQ_FOREACH_FROM (var, head, field) \ + for ((var) = ((var) ? (var) : STAILQ_FIRST((head))); \ + (var); \ + (var) = STAILQ_NEXT((var), field)) + +#define STAILQ_FOREACH_SAFE (var, head, field, tvar) \ + for ((var) = STAILQ_FIRST((head)); \ + (var) && ((tvar) = STAILQ_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define STAILQ_FOREACH_FROM_SAFE (var, head, field, tvar) \ + for ((var) = ((var) ? (var) : STAILQ_FIRST((head))); \ + (var) && ((tvar) = STAILQ_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define STAILQ_INIT (head) do { \ + STAILQ_FIRST((head)) = NULL; \ + (head)->stqh_last = &STAILQ_FIRST((head)); \ +} while (0) + +#define STAILQ_INSERT_AFTER (head, tqelm, elm, field) do { \ + if ((STAILQ_NEXT((elm), field) = STAILQ_NEXT((tqelm), field)) == NULL)\ + (head)->stqh_last = &STAILQ_NEXT((elm), field); \ + STAILQ_NEXT((tqelm), field) = (elm); \ +} while (0) + +#define STAILQ_INSERT_HEAD (head, elm, field) do { \ + if ((STAILQ_NEXT((elm), field) = STAILQ_FIRST((head))) == NULL) \ + (head)->stqh_last = &STAILQ_NEXT((elm), field); \ + STAILQ_FIRST((head)) = (elm); \ +} while (0) + +#define STAILQ_INSERT_TAIL (head, elm, field) do { \ + STAILQ_NEXT((elm), field) = NULL; \ + *(head)->stqh_last = (elm); \ + (head)->stqh_last = &STAILQ_NEXT((elm), field); \ +} while (0) + +#define STAILQ_LAST (head, type, field) \ + (STAILQ_EMPTY((head)) ? NULL : \ + __containerof((head)->stqh_last, \ + QUEUE_TYPEOF(type), field.stqe_next)) + +#define STAILQ_NEXT (elm, field) ((elm)->field.stqe_next) + +#define STAILQ_REMOVE (head, elm, type, field) do { \ + QMD_SAVELINK(oldnext, (elm)->field.stqe_next); \ + if (STAILQ_FIRST((head)) == (elm)) { \ + STAILQ_REMOVE_HEAD((head), field); \ + } \ + else { \ + QUEUE_TYPEOF(type) *curelm = STAILQ_FIRST(head); \ + while (STAILQ_NEXT(curelm, field) != (elm)) \ + curelm = STAILQ_NEXT(curelm, field); \ + STAILQ_REMOVE_AFTER(head, curelm, field); \ + } \ + TRASHIT(*oldnext); \ +} while (0) + +#define STAILQ_REMOVE_AFTER (head, elm, field) do { \ + if ((STAILQ_NEXT(elm, field) = \ + STAILQ_NEXT(STAILQ_NEXT(elm, field), field)) == NULL) \ + (head)->stqh_last = &STAILQ_NEXT((elm), field); \ +} while (0) + +#define STAILQ_REMOVE_HEAD (head, field) do { \ + if ((STAILQ_FIRST((head)) = \ + STAILQ_NEXT(STAILQ_FIRST((head)), field)) == NULL) \ + (head)->stqh_last = &STAILQ_FIRST((head)); \ +} while (0) + +#define STAILQ_SWAP (head1, head2, type) do { \ + QUEUE_TYPEOF(type) *swap_first = STAILQ_FIRST(head1); \ + QUEUE_TYPEOF(type) **swap_last = (head1)->stqh_last; \ + STAILQ_FIRST(head1) = STAILQ_FIRST(head2); \ + (head1)->stqh_last = (head2)->stqh_last; \ + STAILQ_FIRST(head2) = swap_first; \ + (head2)->stqh_last = swap_last; \ + if (STAILQ_EMPTY(head1)) \ + (head1)->stqh_last = &STAILQ_FIRST(head1); \ + if (STAILQ_EMPTY(head2)) \ + (head2)->stqh_last = &STAILQ_FIRST(head2); \ +} while (0) + + +/* +* List declarations. +*/ +#define LIST_HEAD(name, type) \ +struct name { \ + struct type *lh_first; /* first element */ \ +} + +#define LIST_CLASS_HEAD(name, type) \ +struct name { \ + class type *lh_first; /* first element */ \ +} + +#define LIST_HEAD_INITIALIZER (head) \ + { NULL } + +#define LIST_ENTRY (type) \ +struct { \ + struct type *le_next;/* next element */ \ +struct type **le_prev;/* address of previous next element */ \ +} + +#define LIST_CLASS_ENTRY(type) \ +struct { \ + class type *le_next; /* next element */ \ + class type **le_prev; /* address of previous next element */ \ +} + +/* +* List functions. +*/ + +#if (defined(_KERNEL) && defined(INVARIANTS )) +/* +* QMD_LIST_CHECK_HEAD(LIST_HEAD *head, LIST_ENTRY NAME) +* +* If the list is non-empty, validates that the first element of the list +* points back at 'head.' +*/ +#define QMD_LIST_CHECK_HEAD (head, field) do { \ + if (LIST_FIRST((head)) != NULL && \ + LIST_FIRST((head))->field.le_prev != \ + &LIST_FIRST((head))) \ + panic("Bad list head %p first->prev != head", (head)); \ +} while (0) + +/* +* QMD_LIST_CHECK_NEXT(TYPE *elm, LIST_ENTRY NAME) +* +* If an element follows 'elm' in the list, validates that the next element +* points back at 'elm.' +*/ +#define QMD_LIST_CHECK_NEXT (elm, field) do { \ + if (LIST_NEXT((elm), field) != NULL && \ + LIST_NEXT((elm), field)->field.le_prev != \ + &((elm)->field.le_next)) \ + panic("Bad link elm %p next->prev != elm", (elm)); \ +} while (0) + +/* +* QMD_LIST_CHECK_PREV(TYPE *elm, LIST_ENTRY NAME) +* +* Validates that the previous element (or head of the list) points to 'elm.' +*/ +#define QMD_LIST_CHECK_PREV (elm, field) do { \ + if (*(elm)->field.le_prev != (elm)) \ + panic("Bad link elm %p prev->next != elm", (elm)); \ +} while (0) +#else +#define QMD_LIST_CHECK_HEAD (head, field) +#define QMD_LIST_CHECK_NEXT (elm, field) +#define QMD_LIST_CHECK_PREV (elm, field) +#endif /* (_KERNEL && INVARIANTS) */ + +#define LIST_CONCAT (head1, head2, type, field) do { \ + QUEUE_TYPEOF(type) *curelm = LIST_FIRST(head1); \ + if (curelm == NULL) { \ + if ((LIST_FIRST(head1) = LIST_FIRST(head2)) != NULL) { \ + LIST_FIRST(head2)->field.le_prev = \ + &LIST_FIRST((head1)); \ + LIST_INIT(head2); \ + } \ + } else if (LIST_FIRST(head2) != NULL) { \ + while (LIST_NEXT(curelm, field) != NULL) \ + curelm = LIST_NEXT(curelm, field); \ + LIST_NEXT(curelm, field) = LIST_FIRST(head2); \ + LIST_FIRST(head2)->field.le_prev = &LIST_NEXT(curelm, field); \ + LIST_INIT(head2); \ + } \ +} while (0) + +#define LIST_EMPTY (head) ((head)->lh_first == NULL) + +#define LIST_FIRST (head) ((head)->lh_first) + +#define LIST_FOREACH (var, head, field) \ + for ((var) = LIST_FIRST((head)); \ + (var); \ + (var) = LIST_NEXT((var), field)) + +#define LIST_FOREACH_FROM (var, head, field) \ + for ((var) = ((var) ? (var) : LIST_FIRST((head))); \ + (var); \ + (var) = LIST_NEXT((var), field)) + +#define LIST_FOREACH_SAFE (var, head, field, tvar) \ + for ((var) = LIST_FIRST((head)); \ + (var) && ((tvar) = LIST_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define LIST_FOREACH_FROM_SAFE (var, head, field, tvar) \ + for ((var) = ((var) ? (var) : LIST_FIRST((head))); \ + (var) && ((tvar) = LIST_NEXT((var), field), 1); \ + (var) = (tvar)) + +#define LIST_INIT (head) do { \ + LIST_FIRST((head)) = NULL; \ +} while (0) + +#define LIST_INSERT_AFTER (listelm, elm, field) do { \ + QMD_LIST_CHECK_NEXT(listelm, field); \ + if ((LIST_NEXT((elm), field) = LIST_NEXT((listelm), field)) != NULL)\ + LIST_NEXT((listelm), field)->field.le_prev = \ + &LIST_NEXT((elm), field); \ + LIST_NEXT((listelm), field) = (elm); \ + (elm)->field.le_prev = &LIST_NEXT((listelm), field); \ +} while (0) + +#define LIST_INSERT_BEFORE (listelm, elm, field) do { \ + QMD_LIST_CHECK_PREV(listelm, field); \ + (elm)->field.le_prev = (listelm)->field.le_prev; \ + LIST_NEXT((elm), field) = (listelm); \ + *(listelm)->field.le_prev = (elm); \ + (listelm)->field.le_prev = &LIST_NEXT((elm), field); \ +} while (0) + +#define LIST_INSERT_HEAD (head, elm, field) do { \ + QMD_LIST_CHECK_HEAD((head), field); \ + if ((LIST_NEXT((elm), field) = LIST_FIRST((head))) != NULL) \ + LIST_FIRST((head))->field.le_prev = &LIST_NEXT((elm), field);\ + LIST_FIRST((head)) = (elm); \ + (elm)->field.le_prev = &LIST_FIRST((head)); \ +} while (0) + +#define LIST_NEXT (elm, field) ((elm)->field.le_next) + +#define LIST_PREV (elm, head, type, field) \ + ((elm)->field.le_prev == &LIST_FIRST((head)) ? NULL : \ + __containerof((elm)->field.le_prev, \ + QUEUE_TYPEOF(type), field.le_next)) + +#define LIST_REMOVE (elm, field) do { \ + QMD_SAVELINK(oldnext, (elm)->field.le_next); \ + QMD_SAVELINK(oldprev, (elm)->field.le_prev); \ + QMD_LIST_CHECK_NEXT(elm, field); \ + QMD_LIST_CHECK_PREV(elm, field); \ + if (LIST_NEXT((elm), field) != NULL) \ + LIST_NEXT((elm), field)->field.le_prev = \ + (elm)->field.le_prev; \ + *(elm)->field.le_prev = LIST_NEXT((elm), field); \ + TRASHIT(*oldnext); \ + TRASHIT(*oldprev); \ +} while (0) + +#define LIST_SWAP (head1, head2, type, field) do { \ + QUEUE_TYPEOF(type) *swap_tmp = LIST_FIRST(head1); \ + LIST_FIRST((head1)) = LIST_FIRST((head2)); \ + LIST_FIRST((head2)) = swap_tmp; \ + if ((swap_tmp = LIST_FIRST((head1))) != NULL) \ + swap_tmp->field.le_prev = &LIST_FIRST((head1)); \ + if ((swap_tmp = LIST_FIRST((head2))) != NULL) \ + swap_tmp->field.le_prev = &LIST_FIRST((head2)); \ +} while (0) +/* * Tail queue declarations. */
@@ -123,10 +619,58 @@ struct { \/* * Tail queue functions. */ +#if (defined(_KERNEL) && defined(INVARIANTS )) +/* +* QMD_TAILQ_CHECK_HEAD(TAILQ_HEAD *head, TAILQ_ENTRY NAME) +* +* If the tailq is non-empty, validates that the first element of the tailq +* points back at 'head.' +*/ +#define QMD_TAILQ_CHECK_HEAD (head, field) do { \ + if (!TAILQ_EMPTY(head) && \ + TAILQ_FIRST((head))->field.tqe_prev != \ + &TAILQ_FIRST((head))) \ + panic("Bad tailq head %p first->prev != head", (head)); \ +} while (0) + +/* +* QMD_TAILQ_CHECK_TAIL(TAILQ_HEAD *head, TAILQ_ENTRY NAME) +* +* Validates that the tail of the tailq is a pointer to pointer to NULL. +*/ +#define QMD_TAILQ_CHECK_TAIL (head, field) do { \ + if (*(head)->tqh_last != NULL) \ + panic("Bad tailq NEXT(%p->tqh_last) != NULL", (head)); \ +} while (0) + +/* +* QMD_TAILQ_CHECK_NEXT(TYPE *elm, TAILQ_ENTRY NAME) +* +* If an element follows 'elm' in the tailq, validates that the next element +* points back at 'elm.' +*/ +#define QMD_TAILQ_CHECK_NEXT (elm, field) do { \ + if (TAILQ_NEXT((elm), field) != NULL && \ + TAILQ_NEXT((elm), field)->field.tqe_prev != \ + &((elm)->field.tqe_next)) \ + panic("Bad link elm %p next->prev != elm", (elm)); \ +} while (0) + +/* +* QMD_TAILQ_CHECK_PREV(TYPE *elm, TAILQ_ENTRY NAME) +* +* Validates that the previous element (or head of the tailq) points to 'elm.' +*/ +#define QMD_TAILQ_CHECK_PREV (elm, field) do { \ + if (*(elm)->field.tqe_prev != (elm)) \ + panic("Bad link elm %p prev->next != elm", (elm)); \ +} while (0) +#else #define QMD_TAILQ_CHECK_HEAD (head, field)#define QMD_TAILQ_CHECK_TAIL (head, headname)#define QMD_TAILQ_CHECK_NEXT (elm, field)#define QMD_TAILQ_CHECK_PREV (elm, field) +#endif /* (_KERNEL && INVARIANTS) */ #define TAILQ_CONCAT (head1, head2, field) do { \ if (!TAILQ_EMPTY(head2)) { \
@@ -191,9 +735,8 @@ struct { \#define TAILQ_INSERT_AFTER (head, listelm, elm, field) do { \ QMD_TAILQ_CHECK_NEXT(listelm, field); \ - TAILQ_NEXT((elm), field) = TAILQ_NEXT((listelm), field); \ - if (TAILQ_NEXT((listelm), field) != NULL) \ - TAILQ_NEXT((elm), field)->field.tqe_prev = \ + if ((TAILQ_NEXT((elm), field) = TAILQ_NEXT((listelm), field)) != NULL)\ + TAILQ_NEXT((elm), field)->field.tqe_prev = \ &TAILQ_NEXT((elm), field); \ else { \ (head)->tqh_last = &TAILQ_NEXT((elm), field); \
@@ -217,8 +760,7 @@ struct { \#define TAILQ_INSERT_HEAD (head, elm, field) do { \ QMD_TAILQ_CHECK_HEAD(head, field); \ - TAILQ_NEXT((elm), field) = TAILQ_FIRST((head)); \ - if (TAILQ_FIRST((head)) != NULL) \ + if ((TAILQ_NEXT((elm), field) = TAILQ_FIRST((head))) != NULL) \ TAILQ_FIRST((head))->field.tqe_prev = \ &TAILQ_NEXT((elm), field); \ else \
@@ -250,21 +792,24 @@ struct { \* you may want to prefetch the last data element. */ #define TAILQ_LAST_FAST (head, type, field) \ - (TAILQ_EMPTY(head) ? NULL : __containerof((head)->tqh_last, \ - QUEUE_TYPEOF(type), field.tqe_next)) + (TAILQ_EMPTY(head) ? NULL : __containerof((head)->tqh_last, QUEUE_TYPEOF(type), field.tqe_next))#define TAILQ_NEXT (elm, field) ((elm)->field.tqe_next)#define TAILQ_PREV (elm, headname, field) \ (*(((struct headname *)((elm)->field.tqe_prev))->tqh_last)) +#define TAILQ_PREV_FAST (elm, head, type, field) \ + ((elm)->field.tqe_prev == &(head)->tqh_first ? NULL : \ + __containerof((elm)->field.tqe_prev, QUEUE_TYPEOF(type), field.tqe_next)) +#define TAILQ_REMOVE (head, elm, field) do { \ QMD_SAVELINK(oldnext, (elm)->field.tqe_next); \ QMD_SAVELINK(oldprev, (elm)->field.tqe_prev); \ QMD_TAILQ_CHECK_NEXT(elm, field); \ QMD_TAILQ_CHECK_PREV(elm, field); \ if ((TAILQ_NEXT((elm), field)) != NULL) \ - TAILQ_NEXT((elm), field)->field.tqe_prev = \ + TAILQ_NEXT((elm), field)->field.tqe_prev = \ (elm)->field.tqe_prev; \ else { \ (head)->tqh_last = (elm)->field.tqe_prev; \
@@ -277,26 +822,20 @@ struct { \ } while (0)#define TAILQ_SWAP (head1, head2, type, field) do { \ - QUEUE_TYPEOF(type) * swap_first = (head1)->tqh_first; \ - QUEUE_TYPEOF(type) * *swap_last = (head1)->tqh_last; \ + QUEUE_TYPEOF(type) *swap_first = (head1)->tqh_first; \ + QUEUE_TYPEOF(type) **swap_last = (head1)->tqh_last; \ (head1)->tqh_first = (head2)->tqh_first; \ (head1)->tqh_last = (head2)->tqh_last; \ (head2)->tqh_first = swap_first; \ (head2)->tqh_last = swap_last; \ - swap_first = (head1)->tqh_first; \ - if (swap_first != NULL) \ + if ((swap_first = (head1)->tqh_first) != NULL) \ swap_first->field.tqe_prev = &(head1)->tqh_first; \ else \ (head1)->tqh_last = &(head1)->tqh_first; \ - swap_first = (head2)->tqh_first; \ - if (swap_first != NULL) \ + if ((swap_first = (head2)->tqh_first) != NULL) \ swap_first->field.tqe_prev = &(head2)->tqh_first; \ else \ (head2)->tqh_last = &(head2)->tqh_first; \ } while (0) -#ifdef __cplusplus -} -#endif - -#endif /* _SYS_QUEUE_H_ */ +#endif /* !_SYS_QUEUE_H_ */
Signature · 1 line
2.25.4
PATCH v6 9/11 eal/windows: improve CPU and NUMA node detection
In reply to Dmitry Kozlyuk1. Map CPU cores to their respective NUMA nodes as reported by system.
2. Support systems with more than 64 cores (multiple processor groups).
3. Fix magic constants, styling issues, and compiler warnings.
4. Add EAL private function to map DPDK socket ID to NUMA node number.
--- lib/librte_eal/windows/eal_lcore.c | 185 +++++++++++++++++---------- lib/librte_eal/windows/eal_windows.h | 10 ++ 2 files changed, 124 insertions(+), 71 deletions(-)
lib/librte_eal/windows/eal_lcore.c +114 -71
@@ -3,103 +3,146 @@*/ #include <pthread.h> +#include <stdbool.h> #include <stdint.h> #include <rte_common.h> +#include <rte_debug.h> +#include <rte_lcore.h> +#include <rte_os.h> #include "eal_private.h" #include "eal_thread.h" #include "eal_windows.h" -/* global data structure that contains the CPU map */ -static struct _wcpu_map { -unsigned int total_procs; -unsigned int proc_sockets; -unsigned int proc_cores; -unsigned int reserved; -struct _win_lcore_map { -uint8_t socket_id; -uint8_t core_id; - } wlcore_map[RTE_MAX_LCORE ]; -} wcpu_map = {0 }; - -/* -* Create a map of all processors and associated cores on the system -*/ +/** Number of logical processors (cores) in a processor group (32 or 64). */ +#define EAL_PROCESSOR_GROUP_SIZE (sizeof(KAFFINITY) * CHAR_BIT) + +struct lcore_map { +uint8_t socket_id; +uint8_t core_id; +}; + +struct socket_map { +uint16_t node_id; +}; + +struct cpu_map { +unsigned int socket_count; +unsigned int lcore_count; +struct lcore_map lcores[RTE_MAX_LCORE ]; +struct socket_map sockets[RTE_MAX_NUMA_NODES ]; +}; + +static struct cpu_map cpu_map = {0 }; +void -eal_create_cpu_map () +eal_create_cpu_map (void ) { - wcpu_map.total_procs = -GetActiveProcessorCount (ALL_PROCESSOR_GROUPS ); - -LOGICAL_PROCESSOR_RELATIONSHIP lprocRel; -DWORD lprocInfoSize =0 ; -BOOL ht_enabled = FALSE; - -/* First get the processor package information */ - lprocRel = RelationProcessorPackage; -/* Determine the size of buffer we need (pass NULL) */ -GetLogicalProcessorInformationEx (lprocRel,NULL , &lprocInfoSize); - wcpu_map.proc_sockets = lprocInfoSize /48 ; - - lprocInfoSize =0 ; -/* Next get the processor core information */ - lprocRel = RelationProcessorCore; -GetLogicalProcessorInformationEx (lprocRel,NULL , &lprocInfoSize); - wcpu_map.proc_cores = lprocInfoSize /48 ; - -if (wcpu_map.total_procs > wcpu_map.proc_cores) - ht_enabled = TRUE; - -/* Distribute the socket and core ids appropriately -* across the logical cores. For now, split the cores -* equally across the sockets. -*/ -unsigned int lcore =0 ; -for (unsigned int socket =0 ; socket < - wcpu_map.proc_sockets; ++socket) { -for (unsigned int core =0 ; - core < (wcpu_map.proc_cores / wcpu_map.proc_sockets); - ++core) { - wcpu_map.wlcore_map[lcore] - .socket_id = socket; - wcpu_map.wlcore_map[lcore] - .core_id = core; - lcore++; -if (ht_enabled) { - wcpu_map.wlcore_map[lcore] - .socket_id = socket; - wcpu_map.wlcore_map[lcore] - .core_id = core; - lcore++; +SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *infos, *info; +DWORD infos_size; +bool full = false; + + infos_size =0 ; +if (!GetLogicalProcessorInformationEx ( + RelationNumaNode,NULL , &infos_size)) { +DWORD error =GetLastError (); +if (error !=ERROR_INSUFFICIENT_BUFFER ) { +rte_panic ("cannot get NUMA node info size, error %lu" , +GetLastError ()); + } + } + + infos =malloc (infos_size); +if (infos ==NULL ) { +rte_panic ("cannot allocate memory for NUMA node information" ); +return ; + } + +if (!GetLogicalProcessorInformationEx ( + RelationNumaNode, infos, &infos_size)) { +rte_panic ("cannot get NUMA node information, error %lu" , +GetLastError ()); + } + + info = infos; +while ((uint8_t *)info - (uint8_t *)infos < infos_size) { +unsigned int node_id = info->NumaNode.NodeNumber; +GROUP_AFFINITY *cores = &info->NumaNode.GroupMask; +struct lcore_map *lcore; +unsigned int i, socket_id; + +/* NUMA node may be reported multiple times if it includes +* cores from different processor groups, e. g. 80 cores +* of a physical processor comprise one NUMA node, but two +* processor groups, because group size is limited by 32/64. +*/ +for (socket_id =0 ; socket_id < cpu_map.socket_count; + socket_id++) { +if (cpu_map.sockets[socket_id].node_id == node_id) +break ; + } + +if (socket_id == cpu_map.socket_count) { +if (socket_id ==RTE_DIM (cpu_map.sockets)) { + full = true; + goto exit; } + + cpu_map.sockets[socket_id].node_id = node_id; + cpu_map.socket_count++; + } + +for (i =0 ; i <EAL_PROCESSOR_GROUP_SIZE ; i++) { +if ((cores->Mask & ((KAFFINITY )1 << i)) ==0 ) +continue ; + +if (cpu_map.lcore_count ==RTE_DIM (cpu_map.lcores)) { + full = true; + goto exit; + } + + lcore = &cpu_map.lcores[cpu_map.lcore_count]; + lcore->socket_id = socket_id; + lcore->core_id = + cores->Group *EAL_PROCESSOR_GROUP_SIZE + i; + cpu_map.lcore_count++; } + + info = (SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX *)( + (uint8_t *)info + info->Size); + } + +exit: +if (full) { +/* RTE_LOG() may not be available, but this is important. */ +fprintf (stderr,"Enumerated maximum of %u NUMA nodes and %u cores\n" , + cpu_map.socket_count, cpu_map.lcore_count); } + +free (infos); } -/* -* Check if a cpu is present by the presence of the cpu information for it -*/ int eal_cpu_detected (unsigned int lcore_id) { -return (lcore_id < wcpu_map.total_procs); +return lcore_id < cpu_map.lcore_count; } -/* -* Get CPU socket id for a logical core -*/ unsigned eal_cpu_socket_id (unsigned int lcore_id) { -return wcpu_map.wlcore_map[lcore_id].socket_id; +return cpu_map.lcores[lcore_id].socket_id; } -/* -* Get CPU socket id (NUMA node) for a logical core -*/ unsigned eal_cpu_core_id (unsigned int lcore_id) { -return wcpu_map.wlcore_map[lcore_id].core_id; +return cpu_map.lcores[lcore_id].core_id; +} + +unsigned int +eal_socket_numa_node (unsigned int socket_id) +{ +return cpu_map.sockets[socket_id].node_id; }
lib/librte_eal/windows/eal_windows.h +10 -0
@@ -26,4 +26,14 @@ void eal_create_cpu_map(void);*/ int eal_thread_create (pthread_t *thread); +/** +* Get system NUMA node number for a socket ID. +* +* @param socket_id +* Valid EAL socket ID. +* @return +* NUMA node number to use with Win32 API. +*/ +unsigned int eal_socket_numa_node (unsigned int socket_id); +#endif /* _EAL_WINDOWS_H_ */
Signature · 1 line
2.25.4
PATCH v6 10/11 eal/windows: initialize hugepage info
In reply to Dmitry KozlyukAdd hugepages discovery ("large pages" in Windows terminology) and update documentation for required privilege setup. Only 2MB hugepages are supported and their number is estimated roughly due to the lack or unstable status of suitable OS APIs. Assign myself as maintainer for the implementation file.
--- MAINTAINERS | 4 + config/meson.build | 2 + doc/guides/windows_gsg/build_dpdk.rst | 20 ----- doc/guides/windows_gsg/index.rst | 1 + doc/guides/windows_gsg/run_apps.rst | 47 +++++++++++ lib/librte_eal/windows/eal.c | 14 ++++ lib/librte_eal/windows/eal_hugepages.c | 108 +++++++++++++++++++++++++ lib/librte_eal/windows/meson.build | 1 + 8 files changed, 177 insertions(+), 20 deletions(-) create mode 100644 doc/guides/windows_gsg/run_apps.rst create mode 100644 lib/librte_eal/windows/eal_hugepages.c
MAINTAINERS +4 -0
@@ -336,6 +336,10 @@ F: lib/librte_eal/windows/ F: lib/librte_eal/rte_eal_exports.def F: doc/guides/windows_gsg/ +Windows memory allocation +M: Dmitry Kozlyuk <[email protected]> +F: lib/librte_eal/eal_hugepages.c + Core Libraries --------------
config/meson.build +2 -0
@@ -268,6 +268,8 @@ if is_windows if cc.get_id() == 'gcc' add_project_arguments('-D__USE_MINGW_ANSI_STDIO', language: 'c') endif + + add_project_link_arguments('-ladvapi32', language: 'c') endif if get_option('b_lto')
doc/guides/windows_gsg/build_dpdk.rst +0 -20
@@ -111,23 +111,3 @@ Depending on the distribution, paths in this file may need adjustments. meson --cross-file config/x86/meson_mingw.txt -Dexamples=helloworld build ninja -C build - - -Run the helloworld example -========================== - -Navigate to the examples in the build directory and run `dpdk-helloworld.exe`. - -.. code-block:: console - - cd C:\Users\me\dpdk\build\examples - dpdk-helloworld.exe - hello from core 1 - hello from core 3 - hello from core 0 - hello from core 2 - -Note for MinGW-w64: applications are linked to ``libwinpthread-1.dll`` -by default. To run the example, either add toolchain executables directory -to the PATH or copy the library to the working directory. -Alternatively, static linking may be used (mind the LGPLv2.1 license).
doc/guides/windows_gsg/index.rst +1 -0
@@ -12,3 +12,4 @@ Getting Started Guide for Windows intro build_dpdk + run_apps
doc/guides/windows_gsg/run_apps.rst new file +47 -0
47 lines not on this page · raw message
lib/librte_eal/windows/eal.c +14 -0
@@ -19,8 +19,11 @@#include <eal_private.h> #include <rte_trace_point.h> +#include "eal_hugepages.h" #include "eal_windows.h" +#define MEMSIZE_IF_NO_HUGE_PAGE (64ULL * 1024ULL * 1024ULL) +/* Allow the application to print its usage message too if set */ static rte_usage_hook_t rte_application_usage_hook;
@@ -276,6 +279,17 @@ rte_eal_init(int argc, char **argv)if (fctret <0 )exit (1 ); +if (!internal_config.no_hugetlbfs && (eal_hugepage_info_init () <0 )) { +rte_eal_init_alert ("Cannot get hugepage information" ); + rte_errno =EACCES ; +return -1 ; + } + +if (internal_config.memory ==0 && !internal_config.force_sockets) { +if (internal_config.no_hugetlbfs) + internal_config.memory =MEMSIZE_IF_NO_HUGE_PAGE ; + } +eal_thread_init_master (rte_config.master_lcore);RTE_LCORE_FOREACH_SLAVE (i) {
lib/librte_eal/windows/eal_hugepages.c new file +108 -0
108 lines not on this page · raw message
lib/librte_eal/windows/meson.build +1 -0
@@ -6,6 +6,7 @@ subdir('include') sources += files( 'eal.c', 'eal_debug.c', + 'eal_hugepages.c', 'eal_lcore.c', 'eal_log.c', 'eal_thread.c',
Signature · 1 line
2.25.4
PATCH v6 11/11 eal/windows: implement basic memory management
In reply to Dmitry KozlyukBasic memory management supports core libraries and PMDs operating in IOVA as PA mode. It uses a kernel-mode driver, virt2phys, to obtain IOVAs of hugepages allocated from user-mode. Multi-process mode is not implemented and is forcefully disabled at startup. Assign myself as a maintainer for Windows file and memory management implementation.
--- MAINTAINERS | 1 + config/meson.build | 12 +- doc/guides/windows_gsg/run_apps.rst | 54 +- lib/librte_eal/common/meson.build | 11 + lib/librte_eal/common/rte_malloc.c | 1 + lib/librte_eal/rte_eal_exports.def | 119 +++ lib/librte_eal/windows/eal.c | 63 +- lib/librte_eal/windows/eal_file.c | 123 +++ lib/librte_eal/windows/eal_memalloc.c | 441 +++++++++++ lib/librte_eal/windows/eal_memory.c | 710 ++++++++++++++++++ lib/librte_eal/windows/eal_mp.c | 103 +++ lib/librte_eal/windows/eal_windows.h | 75 ++ lib/librte_eal/windows/include/meson.build | 1 + lib/librte_eal/windows/include/rte_os.h | 17 + .../windows/include/rte_virt2phys.h | 34 + lib/librte_eal/windows/include/rte_windows.h | 2 + lib/librte_eal/windows/include/unistd.h | 3 + lib/librte_eal/windows/meson.build | 6 + 18 files changed, 1769 insertions(+), 7 deletions(-) create mode 100644 lib/librte_eal/windows/eal_file.c create mode 100644 lib/librte_eal/windows/eal_memalloc.c create mode 100644 lib/librte_eal/windows/eal_memory.c create mode 100644 lib/librte_eal/windows/eal_mp.c create mode 100644 lib/librte_eal/windows/include/rte_virt2phys.h
MAINTAINERS +1 -0
@@ -339,6 +339,7 @@ F: doc/guides/windows_gsg/ Windows memory allocation M: Dmitry Kozlyuk <[email protected]> F: lib/librte_eal/eal_hugepages.c +F: lib/librte_eal/eal_mem* Core Libraries
config/meson.build +9 -3
@@ -261,15 +261,21 @@ if is_freebsd endif if is_windows - # Minimum supported API is Windows 7. - add_project_arguments('-D_WIN32_WINNT=0x0601', language: 'c') + # VirtualAlloc2() is available since Windows 10 / Server 2016. + add_project_arguments('-D_WIN32_WINNT=0x0A00', language: 'c') # Use MinGW-w64 stdio, because DPDK assumes ANSI-compliant formatting. if cc.get_id() == 'gcc' add_project_arguments('-D__USE_MINGW_ANSI_STDIO', language: 'c') endif - add_project_link_arguments('-ladvapi32', language: 'c') + # Contrary to docs, VirtualAlloc2() is exported by mincore.lib + # in Windows SDK, while MinGW exports it by advapi32.a. + if is_ms_linker + add_project_link_arguments('-lmincore', language: 'c') + endif + + add_project_link_arguments('-ladvapi32', '-lsetupapi', language: 'c') endif if get_option('b_lto')
doc/guides/windows_gsg/run_apps.rst +51 -3
68 lines not on this page · raw message
lib/librte_eal/common/meson.build +11 -0
@@ -8,13 +8,24 @@ if is_windows 'eal_common_bus.c', 'eal_common_class.c', 'eal_common_devargs.c', + 'eal_common_dynmem.c', 'eal_common_errno.c', + 'eal_common_fbarray.c', 'eal_common_launch.c', 'eal_common_lcore.c', 'eal_common_log.c', + 'eal_common_mcfg.c', + 'eal_common_memalloc.c', + 'eal_common_memory.c', + 'eal_common_memzone.c', 'eal_common_options.c', + 'eal_common_string_fns.c', + 'eal_common_tailqs.c', 'eal_common_thread.c', 'eal_common_trace_points.c', + 'malloc_elem.c', + 'malloc_heap.c', + 'rte_malloc.c', ) subdir_done() endif
lib/librte_eal/common/rte_malloc.c +1 -0
@@ -20,6 +20,7 @@#include <rte_lcore.h> #include <rte_common.h> #include <rte_spinlock.h> +#include <rte_eal_trace.h> #include <rte_malloc.h>
lib/librte_eal/rte_eal_exports.def +119 -0
128 lines not on this page · raw message
lib/librte_eal/windows/eal.c +62 -1
87 lines not on this page · raw message
lib/librte_eal/windows/eal_file.c new file +123 -0
123 lines not on this page · raw message
lib/librte_eal/windows/eal_memalloc.c new file +441 -0
441 lines not on this page · raw message
lib/librte_eal/windows/eal_memory.c new file +710 -0
710 lines not on this page · raw message
lib/librte_eal/windows/eal_mp.c new file +103 -0
103 lines not on this page · raw message
lib/librte_eal/windows/eal_windows.h +75 -0
87 lines not on this page · raw message
lib/librte_eal/windows/include/meson.build +1 -0
@@ -5,5 +5,6 @@ includes += include_directories('.') headers += files( 'rte_os.h', + 'rte_virt2phys.h', 'rte_windows.h', )
lib/librte_eal/windows/include/rte_os.h +17 -0
@@ -14,6 +14,7 @@#include <stdarg.h> #include <stdio.h> #include <stdlib.h> +#include <string.h> #ifdef __cplusplusextern "C" {
@@ -36,6 +37,9 @@ extern "C" {#define strncasecmp (s1, s2, count) _strnicmp(s1, s2, count) +#define close _close +#define unlink _unlink +/* cpu_set macros implementation */ #define RTE_CPU_AND (dst, src1, src2) CPU_AND(dst, src1, src2)#define RTE_CPU_OR (dst, src1, src2) CPU_OR(dst, src1, src2)
@@ -46,6 +50,7 @@ extern "C" {typedef long long ssize_t ;#ifndef RTE_TOOLCHAIN_GCC +static inline int asprintf (char **buffer,const char *format, ...) {
@@ -72,6 +77,18 @@ asprintf(char **buffer, const char *format, ...) }return ret; } + +static inline const char * +eal_strerror (int code) +{ +static char buffer[128 ]; + +strerror_s (buffer,sizeof (buffer), code); +return buffer; +} + +#define strerror eal_strerror +#endif /* RTE_TOOLCHAIN_GCC */ #ifdef __cplusplus
lib/librte_eal/windows/include/rte_virt2phys.h new file +34 -0
@@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: BSD-3-Clause +* Copyright (c) 2020 Dmitry Kozlyuk +*/ + +/** +* @file virt2phys driver interface +*/ + +/** +* Driver device interface GUID {539c2135-793a-4926-afec-d3a1b61bbc8a}. +*/ +DEFINE_GUID (GUID_DEVINTERFACE_VIRT2PHYS , +0x539c2135 ,0x793a ,0x4926 , +0xaf ,0xec ,0xd3 ,0xa1 ,0xb6 ,0x1b ,0xbc ,0x8a ); + +/** +* Driver device type for IO control codes. +*/ +#define VIRT2PHYS_DEVTYPE 0x8000 + +/** +* Translate a valid non-paged virtual address to a physical address. +* +* Note: A physical address zero (0) is reported if input address +* is paged out or not mapped. However, if input is a valid mapping +* of I/O port 0x0000, output is also zero. There is no way +* to distinguish between these cases by return value only. +* +* Input: a non-paged virtual address (PVOID). +* +* Output: the corresponding physical address (LARGE_INTEGER). +*/ +#define IOCTL_VIRT2PHYS_TRANSLATE CTL_CODE( \ + VIRT2PHYS_DEVTYPE, 0x800, METHOD_BUFFERED, FILE_ANY_ACCESS)
lib/librte_eal/windows/include/rte_windows.h +2 -0
@@ -23,6 +23,8 @@#include <basetsd.h> #include <psapi.h> +#include <setupapi.h> +#include <winioctl.h> /* Have GUIDs defined. */ #ifndef INITGUID
lib/librte_eal/windows/include/unistd.h +3 -0
@@ -9,4 +9,7 @@* as Microsoft libc does not contain unistd.h. This may be removed * in future releases. */ + +#include <io.h> +#endif /* _UNISTD_H_ */
lib/librte_eal/windows/meson.build +6 -0
@@ -6,10 +6,16 @@ subdir('include') sources += files( 'eal.c', 'eal_debug.c', + 'eal_file.c', 'eal_hugepages.c', 'eal_lcore.c', 'eal_log.c', + 'eal_memalloc.c', + 'eal_memory.c', + 'eal_mp.c', 'eal_thread.c', 'fnmatch.c', 'getopt.c', ) + +dpdk_conf.set10('RTE_EAL_NUMA_AWARE_HUGEPAGES', true)
Signature · 1 line
2.25.4
v6 cover Windows basic memory management
In reply to Dmitry KozlyukThis patchset implements basic MM with the following features:
* Hugepages are dynamically allocated in user-mode.
* Only 2MB hugepages are supported.
* IOVA is always PA, obtained through kernel-mode driver.
* No 32-bit support (presumably not demanded).
* Ni multi-process support (it is forcefully disabled).
* No-huge mode for testing without IOVA is available.
Testing revealed Windows Server 2019 does not allow allocating hugepage memory at a reserved address, despite advertised API. So allocator has to temporary free the region to be allocated. This creates in inherent race condition. This issue is being discussed with Microsoft privately.
New EAL public functions for memory mapping are introduced to mitigate OS differences in DPDK libraries and applications: rte_mem_map, rte_mem_unmap, rte_mem_lock, rte_mem_page_size.
To support common MM routines, internal wrappers for low-level memory reservation and file management are introduced. These changes affect Linux and FreeBSD EAL. Shared code is placed unded /unix/ subdirectory (suggested by Thomas).
To avoid code duplication between Linux and Windows EAL, common code for EALs supporting dynamic memory allocation is extracted (discussed with Anatoly Burakov in v4 thread). This is a separate patch to ease the review, but it can be merged with the previous one.
EAL tracepoints save size_t values as long, which is invalid on Windows. New size_t emitter for tracepoints is introduced (suggested by Jerin Jacob to Fady Bader, see [1]). Also, to avoid workaround in every file using the tracepoints, stubs are added to Windows EAL.
Entire <sys/queue.h> is imported from FreeBSD, replacing existing partial import. There is already a license exception for this file. The file is imported as-is, so it causes a bunch of checkpatch warnings.
[1]: http://mails.dpdk.org/archives/dev/2020-May/168076.html
---
v7:
* Change EAL internal file management API (Neil Horman).
v6:
* Fix 32-bit build on x86 (CI).
* Fix Makefile build (Anatoly Burakov, Thomas Monjalon).
* Restore 32-bit common code (Anatoly Burakov).
* Fix error reporting in memory management (Anatoly Burakov).
* Add Doxygen comment for size_t tracepoint emitter (Jerin Jacob).
* Update MAINTAINERS for new files and new code (Thomas Monjalon).
* Rename rte_get_page_size to rte_mem_page_size.
* Mark DPDK-only wrappers internal, move them to separate file.
* Get rid of warnings in enabled common code with Clang on Windows.
v5:
* Fix allocation and deallocation on Windows Server (Fady Bader).
* Replace remaining VirtualFree with VirtualFreeEx (Ranjit Menon).
* Fix errors in eal_get_virtual_area (Anatoly Burakov).
* Fix error handling and documentation for rte_mem_lock (Anatoly Burakov).
* Extract common code for EALs w/dynamic allocation (Anatoly Burakov).
* Use POSIX value for rte_errno after rte_mem_unmap() on Windows.
* Add stubs to use tracing functions without workarounds.
v4:
* Rebase on ToT, drop patches merged into master.
* Rearrange patches to split Windows code (Jerin).
* Fix Linux and FreeBSD build with make (Ophir).
* Use int instead of enum to hold a set of flags (Anatoly).
* Rename eal_mem_reserve items and fix their description (Anatoly).
* Add eal_mem_set_dump() wrapper around madvise (Anatoly).
* Don't claim Windows Server 2016 support due to lack of API (Tal).
* Replace enum rte_page_sizes with a set of #defines (Jerin).
* Fix documentation, SPDX tags, logging (Thomas).
v3:
* Fix Linux build on and aarch64 and 32-bit x86 (reported by CI).
* Fix logic and error handling while allocating segments.
* Fix Unix rte_mem_map(): return NULL on failure.
* Fix some checkpatch.sh issues:
* Do not return positive errno, use DWORD for GetLastError().
* Make dpdk-kmods source files non-executable.
* Improve GSG for Windows Server (suggested by Ranjit Menon).
v2:
* Rebase on ToT. Move all new code shared between Linux and FreeBSD to /unix/ subdirectory, also factor out some existing code there.
* Improve description of Clang issue with rte_page_sizes on Windows. Restore -fstrict-enum for EAL. Check running, not target compiler.
* Use EAL prefix for private facilities instead if RTE.
* Improve documentation comments for new functions.
* Remove co-installer for virt2phys. Add a typecast for clarity.
* Document virt2phys in user guide, improve its own README.
* Explicitly and forcefully disable multi-process.
Dmitry Kozlyuk (11):
eal: replace rte_page_sizes with a set of constants
eal: introduce internal wrappers for file operations
eal: introduce memory management wrappers
eal/mem: extract common code for memseg list initialization eal/mem: extract common code for dynamic memory allocation
trace: add size_t field emitter
eal/windows: add tracing support stubs
eal/windows: replace sys/queue.h with a complete one from FreeBSD eal/windows: improve CPU and NUMA node detection eal/windows: initialize hugepage info
eal/windows: implement basic memory management
MAINTAINERS | 9 + config/meson.build | 12 +- doc/guides/rel_notes/release_20_08.rst | 2 + doc/guides/windows_gsg/build_dpdk.rst | 20 - doc/guides/windows_gsg/index.rst | 1 + doc/guides/windows_gsg/run_apps.rst | 95 +++ lib/librte_eal/common/eal_common_dynmem.c | 521 +++++++++++++ lib/librte_eal/common/eal_common_fbarray.c | 71 +- lib/librte_eal/common/eal_common_memory.c | 158 +++- lib/librte_eal/common/eal_common_thread.c | 5 +- lib/librte_eal/common/eal_private.h | 254 ++++++- lib/librte_eal/common/meson.build | 16 + lib/librte_eal/common/rte_malloc.c | 1 + lib/librte_eal/freebsd/Makefile | 5 + lib/librte_eal/freebsd/eal_memory.c | 98 +-- lib/librte_eal/include/rte_eal_memory.h | 93 +++ lib/librte_eal/include/rte_eal_trace.h | 8 +- lib/librte_eal/include/rte_memory.h | 23 +- lib/librte_eal/include/rte_trace_point.h | 3 + lib/librte_eal/linux/Makefile | 6 + lib/librte_eal/linux/eal_memalloc.c | 5 +- lib/librte_eal/linux/eal_memory.c | 614 +-------------- lib/librte_eal/meson.build | 4 + lib/librte_eal/rte_eal_exports.def | 119 +++ lib/librte_eal/rte_eal_version.map | 9 + lib/librte_eal/unix/eal_file.c | 80 ++ lib/librte_eal/unix/eal_unix_memory.c | 152 ++++ lib/librte_eal/unix/meson.build | 7 + lib/librte_eal/windows/eal.c | 107 +++ lib/librte_eal/windows/eal_file.c | 125 +++ lib/librte_eal/windows/eal_hugepages.c | 108 +++ lib/librte_eal/windows/eal_lcore.c | 185 +++-- lib/librte_eal/windows/eal_memalloc.c | 441 +++++++++++ lib/librte_eal/windows/eal_memory.c | 710 ++++++++++++++++++ lib/librte_eal/windows/eal_mp.c | 103 +++ lib/librte_eal/windows/eal_windows.h | 85 +++ lib/librte_eal/windows/include/meson.build | 1 + lib/librte_eal/windows/include/rte_os.h | 17 + .../windows/include/rte_virt2phys.h | 34 + lib/librte_eal/windows/include/rte_windows.h | 2 + lib/librte_eal/windows/include/sys/queue.h | 663 ++++++++++++++-- lib/librte_eal/windows/include/unistd.h | 3 + lib/librte_eal/windows/meson.build | 7 + lib/librte_mempool/rte_mempool_trace.h | 10 +- 44 files changed, 4053 insertions(+), 939 deletions(-) create mode 100644 doc/guides/windows_gsg/run_apps.rst create mode 100644 lib/librte_eal/common/eal_common_dynmem.c create mode 100644 lib/librte_eal/include/rte_eal_memory.h create mode 100644 lib/librte_eal/unix/eal_file.c create mode 100644 lib/librte_eal/unix/eal_unix_memory.c create mode 100644 lib/librte_eal/unix/meson.build create mode 100644 lib/librte_eal/windows/eal_file.c create mode 100644 lib/librte_eal/windows/eal_hugepages.c create mode 100644 lib/librte_eal/windows/eal_memalloc.c create mode 100644 lib/librte_eal/windows/eal_memory.c create mode 100644 lib/librte_eal/windows/eal_mp.c create mode 100644 lib/librte_eal/windows/include/rte_virt2phys.h
Signature · 1 line
2.25.4
PATCH v7 1/11 eal: replace rte_page_sizes with a set of constants
Clang on Windows follows MS ABI where enum values are limited to 2^31-1. Enum rte_page_sizes has members valued above this limit, which get wrapped to zero, resulting in compilation error (duplicate values in enum). Using MS ABI is mandatory for Windows EAL to call Win32 APIs.
Remove rte_page_sizes and replace its values with #define's. This enumeration is not used in public API, so there's no ABI breakage. Announce API changes for 20.08 in documentation.
--- doc/guides/rel_notes/release_20_08.rst | 2 ++ lib/librte_eal/include/rte_memory.h | 23 ++++++++++------------- 2 files changed, 12 insertions(+), 13 deletions(-)
doc/guides/rel_notes/release_20_08.rst +2 -0
@@ -85,6 +85,8 @@ API Changes Also, make sure to start the actual text at the margin. ========================================================= +* ``rte_page_sizes`` enumeration is replaced with ``RTE_PGSIZE_xxx`` defines. + ABI Changes -----------
lib/librte_eal/include/rte_memory.h +10 -13
@@ -24,19 +24,16 @@ extern "C" {#include <rte_config.h> #include <rte_fbarray.h> -__extension__ -enum rte_page_sizes { -RTE_PGSIZE_4K =1ULL <<12 , -RTE_PGSIZE_64K =1ULL <<16 , -RTE_PGSIZE_256K =1ULL <<18 , -RTE_PGSIZE_2M =1ULL <<21 , -RTE_PGSIZE_16M =1ULL <<24 , -RTE_PGSIZE_256M =1ULL <<28 , -RTE_PGSIZE_512M =1ULL <<29 , -RTE_PGSIZE_1G =1ULL <<30 , -RTE_PGSIZE_4G =1ULL <<32 , -RTE_PGSIZE_16G =1ULL <<34 , -}; +#define RTE_PGSIZE_4K (1ULL << 12) +#define RTE_PGSIZE_64K (1ULL << 16) +#define RTE_PGSIZE_256K (1ULL << 18) +#define RTE_PGSIZE_2M (1ULL << 21) +#define RTE_PGSIZE_16M (1ULL << 24) +#define RTE_PGSIZE_256M (1ULL << 28) +#define RTE_PGSIZE_512M (1ULL << 29) +#define RTE_PGSIZE_1G (1ULL << 30) +#define RTE_PGSIZE_4G (1ULL << 32) +#define RTE_PGSIZE_16G (1ULL << 34)#define SOCKET_ID_ANY -1/**< Any NUMA socket. */
Signature · 1 line
2.25.4
PATCH v7 2/11 eal: introduce internal wrappers for file operations
In reply to Dmitry KozlyukIntroduce OS-independent wrappers in order to support common EAL code on Unix and Windows:
* eal_file_open: open or create a file.
* eal_file_lock: lock or unlock an open file.
* eal_file_truncate: enforce a given size for an open file.
Implementation for Linux and FreeBSD is placed in "unix" subdirectory, which is intended for common code between the two. These thin wrappers require no special maintenance.
Common code supporting multi-process doesn't use the new wrappers, because it is inherently Unix-specific and would impose excessive requirements on the wrappers.
--- MAINTAINERS | 3 + lib/librte_eal/common/eal_common_fbarray.c | 31 ++++----- lib/librte_eal/common/eal_private.h | 73 ++++++++++++++++++++ lib/librte_eal/freebsd/Makefile | 4 ++ lib/librte_eal/linux/Makefile | 4 ++ lib/librte_eal/meson.build | 4 ++ lib/librte_eal/unix/eal_file.c | 80 ++++++++++++++++++++++ lib/librte_eal/unix/meson.build | 6 ++ 8 files changed, 186 insertions(+), 19 deletions(-) create mode 100644 lib/librte_eal/unix/eal_file.c create mode 100644 lib/librte_eal/unix/meson.build
MAINTAINERS +3 -0
@@ -323,6 +323,9 @@ FreeBSD UIO M: Bruce Richardson <[email protected]> F: kernel/freebsd/nic_uio/ +Unix shared files +F: lib/librte_eal/unix/ + Windows support M: Harini Ramakrishnan <[email protected]> M: Omar Cardona <[email protected]>
lib/librte_eal/common/eal_common_fbarray.c +12 -19
@@ -8,8 +8,8 @@#include <sys/mman.h> #include <stdint.h> #include <errno.h> -#include <sys/file.h> #include <string.h> +#include <unistd.h> #include <rte_common.h> #include <rte_log.h>
@@ -85,10 +85,8 @@ resize_and_map(int fd, void *addr, size_t len)char path[PATH_MAX ];void *map_addr; -if (ftruncate (fd, len)) { +if (eal_file_truncate (fd, len)) {RTE_LOG (ERR ,EAL ,"Cannot truncate %s\n" , path); -/* pass errno up the chain */ - rte_errno = errno;return -1 ; }
@@ -772,15 +770,15 @@ rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,* and see if we succeed. If we don't, someone else is using it * already. */ - fd =open (path,O_CREAT |O_RDWR ,0600 ); + fd =eal_file_open (path,EAL_OPEN_CREATE |EAL_OPEN_READWRITE );if (fd <0 ) {RTE_LOG (DEBUG ,EAL ,"%s(): couldn't open %s: %s\n" , - __func__, path,strerror (errno)); - rte_errno = errno; + __func__, path,rte_strerror (rte_errno)); goto fail; - }else if (flock (fd,LOCK_EX |LOCK_NB )) { + }else if (eal_file_lock ( + fd,EAL_FLOCK_EXCLUSIVE ,EAL_FLOCK_RETURN )) {RTE_LOG (DEBUG ,EAL ,"%s(): couldn't lock %s: %s\n" , - __func__, path,strerror (errno)); + __func__, path,rte_strerror (rte_errno)); rte_errno =EBUSY ; goto fail; }
@@ -789,10 +787,8 @@ rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,* still attach to it, but no other process could reinitialize * it. */ -if (flock (fd,LOCK_SH |LOCK_NB )) { - rte_errno = errno; +if (eal_file_lock (fd,EAL_FLOCK_SHARED ,EAL_FLOCK_RETURN )) goto fail; - }if (resize_and_map (fd, data, mmap_len)) goto fail;
@@ -888,17 +884,14 @@ rte_fbarray_attach(struct rte_fbarray *arr)eal_get_fbarray_path (path,sizeof (path), arr->name); - fd =open (path,O_RDWR ); + fd =eal_file_open (path,EAL_OPEN_READWRITE );if (fd <0 ) { - rte_errno = errno; goto fail; }/* lock the file, to let others know we're using it */ -if (flock (fd,LOCK_SH |LOCK_NB )) { - rte_errno = errno; +if (eal_file_lock (fd,EAL_FLOCK_SHARED ,EAL_FLOCK_RETURN )) goto fail; - }if (resize_and_map (fd, data, mmap_len)) goto fail;
@@ -1025,7 +1018,7 @@ rte_fbarray_destroy(struct rte_fbarray *arr)* has been detached by all other processes */ fd = tmp->fd; -if (flock (fd,LOCK_EX |LOCK_NB )) { +if (eal_file_lock (fd,EAL_FLOCK_EXCLUSIVE ,EAL_FLOCK_RETURN )) {RTE_LOG (DEBUG ,EAL ,"Cannot destroy fbarray - another process is using it\n" ); rte_errno =EBUSY ; ret =-1 ;
@@ -1042,7 +1035,7 @@ rte_fbarray_destroy(struct rte_fbarray *arr)* we're still holding an exclusive lock, so drop it to * shared. */ -flock (fd,LOCK_SH |LOCK_NB ); +eal_file_lock (fd,EAL_FLOCK_SHARED ,EAL_FLOCK_RETURN ); ret =-1 ; goto out;
lib/librte_eal/common/eal_private.h +73 -0
77 lines not on this page · raw message
lib/librte_eal/freebsd/Makefile +4 -0
@@ -7,6 +7,7 @@ LIB = librte_eal.a ARCH_DIR ?= $(RTE_ARCH) VPATH += $(RTE_SDK)/lib/librte_eal/$(ARCH_DIR) +VPATH += $(RTE_SDK)/lib/librte_eal/unix VPATH += $(RTE_SDK)/lib/librte_eal/common CFLAGS += -I$(SRCDIR)/include
@@ -74,6 +75,9 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_service.c SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_random.c SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_reciprocal.c +# from unix dir +SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += eal_file.c + # from arch dir SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_cpuflags.c SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_hypervisor.c
lib/librte_eal/linux/Makefile +4 -0
@@ -7,6 +7,7 @@ LIB = librte_eal.a ARCH_DIR ?= $(RTE_ARCH) VPATH += $(RTE_SDK)/lib/librte_eal/$(ARCH_DIR) +VPATH += $(RTE_SDK)/lib/librte_eal/unix VPATH += $(RTE_SDK)/lib/librte_eal/common CFLAGS += -I$(SRCDIR)/include
@@ -81,6 +82,9 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_service.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_random.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_reciprocal.c +# from unix dir +SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_file.c + # from arch dir SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_cpuflags.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_hypervisor.c
lib/librte_eal/meson.build +4 -0
@@ -6,6 +6,10 @@ subdir('include') subdir('common') +if not is_windows + subdir('unix') +endif + dpdk_conf.set('RTE_EXEC_ENV_' + exec_env.to_upper(), 1) subdir(exec_env)
lib/librte_eal/unix/eal_file.c new file +80 -0
80 lines not on this page · raw message
lib/librte_eal/unix/meson.build new file +6 -0
@@ -0,0 +1,6 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2020 Dmitry Kozlyuk + +sources += files( + 'eal_file.c', +)
Signature · 1 line
2.25.4
PATCH v7 3/11 eal: introduce memory management wrappers
In reply to Dmitry KozlyukIntroduce OS-independent wrappers for memory management operations used across DPDK and specifically in common code of EAL:
* rte_mem_map()
* rte_mem_unmap()
* rte_mem_page_size()
* rte_mem_lock()
Windows uses different APIs for memory mapping and reservation, while Unices reserve memory by mapping it. Introduce EAL private functions to support memory reservation in common code:
* eal_mem_reserve()
* eal_mem_free()
* eal_mem_set_dump()
Wrappers follow POSIX semantics limited to DPDK tasks, but their signatures deliberately differ from POSIX ones to be more safe and expressive. New symbols are internal. Being thin wrappers, they require no special maintenance.
--- lib/librte_eal/common/eal_common_fbarray.c | 40 +++--- lib/librte_eal/common/eal_common_memory.c | 61 ++++----- lib/librte_eal/common/eal_private.h | 78 ++++++++++- lib/librte_eal/freebsd/Makefile | 1 + lib/librte_eal/include/rte_eal_memory.h | 93 +++++++++++++ lib/librte_eal/linux/Makefile | 1 + lib/librte_eal/linux/eal_memalloc.c | 5 +- lib/librte_eal/rte_eal_version.map | 9 ++ lib/librte_eal/unix/eal_unix_memory.c | 152 +++++++++++++++++++++ lib/librte_eal/unix/meson.build | 1 + 10 files changed, 376 insertions(+), 65 deletions(-) create mode 100644 lib/librte_eal/include/rte_eal_memory.h create mode 100644 lib/librte_eal/unix/eal_unix_memory.c
lib/librte_eal/common/eal_common_fbarray.c +19 -21
@@ -5,15 +5,16 @@#include <fcntl.h> #include <inttypes.h> #include <limits.h> -#include <sys/mman.h> #include <stdint.h> #include <errno.h> #include <string.h> #include <unistd.h> #include <rte_common.h> -#include <rte_log.h> +#include <rte_eal_memory.h> #include <rte_errno.h> +#include <rte_log.h> +#include <rte_memory.h> #include <rte_spinlock.h> #include <rte_tailq.h>
@@ -90,12 +91,9 @@ resize_and_map(int fd, void *addr, size_t len)return -1 ; } - map_addr =mmap (addr, len,PROT_READ |PROT_WRITE , -MAP_SHARED |MAP_FIXED , fd,0 ); + map_addr =rte_mem_map (addr, len,RTE_PROT_READ |RTE_PROT_WRITE , +RTE_MAP_SHARED |RTE_MAP_FORCE_ADDRESS , fd,0 );if (map_addr != addr) { -RTE_LOG (ERR ,EAL ,"mmap() failed: %s\n" ,strerror (errno)); -/* pass errno up the chain */ - rte_errno = errno;return -1 ; }return 0 ;
@@ -733,7 +731,7 @@ rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,return -1 ; } - page_sz =sysconf (_SC_PAGESIZE); + page_sz =rte_mem_page_size ();if (page_sz == (size_t )-1 ) {free (ma);return -1 ;
@@ -754,11 +752,13 @@ rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,if (internal_config.no_shconf) {/* remap virtual area as writable */ -void *new_data =mmap (data, mmap_len,PROT_READ |PROT_WRITE , -MAP_FIXED |MAP_PRIVATE |MAP_ANONYMOUS , fd,0 ); -if (new_data ==MAP_FAILED ) { +static const int flags =RTE_MAP_FORCE_ADDRESS | +RTE_MAP_PRIVATE |RTE_MAP_ANONYMOUS ; +void *new_data =rte_mem_map (data, mmap_len, +RTE_PROT_READ |RTE_PROT_WRITE , flags, fd,0 ); +if (new_data ==NULL ) {RTE_LOG (DEBUG ,EAL ,"%s(): couldn't remap anonymous memory: %s\n" , - __func__,strerror (errno)); + __func__,rte_strerror (rte_errno)); goto fail; } }else {
@@ -820,7 +820,7 @@ rte_fbarray_init(struct rte_fbarray *arr, const char *name, unsigned int len,return 0 ; fail:if (data) -munmap (data, mmap_len); +rte_mem_unmap (data, mmap_len);if (fd >=0 )close (fd);free (ma);
@@ -858,7 +858,7 @@ rte_fbarray_attach(struct rte_fbarray *arr)return -1 ; } - page_sz =sysconf (_SC_PAGESIZE); + page_sz =rte_mem_page_size ();if (page_sz == (size_t )-1 ) {free (ma);return -1 ;
@@ -909,7 +909,7 @@ rte_fbarray_attach(struct rte_fbarray *arr)return 0 ; fail:if (data) -munmap (data, mmap_len); +rte_mem_unmap (data, mmap_len);if (fd >=0 )close (fd);free (ma);
@@ -937,8 +937,7 @@ rte_fbarray_detach(struct rte_fbarray *arr)* really do anything about it, things will blow up either way. */ -size_t page_sz =sysconf (_SC_PAGESIZE); - +size_t page_sz =rte_mem_page_size ();if (page_sz == (size_t )-1 )return -1 ;
@@ -957,7 +956,7 @@ rte_fbarray_detach(struct rte_fbarray *arr) goto out; } -munmap (arr->data, mmap_len); +rte_mem_unmap (arr->data, mmap_len);/* area is unmapped, close fd and remove the tailq entry */ if (tmp->fd >=0 )
@@ -992,8 +991,7 @@ rte_fbarray_destroy(struct rte_fbarray *arr)* really do anything about it, things will blow up either way. */ -size_t page_sz =sysconf (_SC_PAGESIZE); - +size_t page_sz =rte_mem_page_size ();if (page_sz == (size_t )-1 )return -1 ;
@@ -1042,7 +1040,7 @@ rte_fbarray_destroy(struct rte_fbarray *arr) }close (fd); } -munmap (arr->data, mmap_len); +rte_mem_unmap (arr->data, mmap_len);/* area is unmapped, remove the tailq entry */ TAILQ_REMOVE (&mem_area_tailq, tmp, next);
lib/librte_eal/common/eal_common_memory.c +23 -38
143 lines not on this page · raw message
lib/librte_eal/common/eal_private.h +75 -3
106 lines not on this page · raw message
lib/librte_eal/freebsd/Makefile +1 -0
@@ -77,6 +77,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_reciprocal.c # from unix dir SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += eal_file.c +SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += eal_unix_memory.c # from arch dir SRCS-$(CONFIG_RTE_EXEC_ENV_FREEBSD) += rte_cpuflags.c
lib/librte_eal/include/rte_eal_memory.h new file +93 -0
93 lines not on this page · raw message
lib/librte_eal/linux/Makefile +1 -0
@@ -84,6 +84,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_reciprocal.c # from unix dir SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_file.c +SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += eal_unix_memory.c # from arch dir SRCS-$(CONFIG_RTE_EXEC_ENV_LINUX) += rte_cpuflags.c
lib/librte_eal/linux/eal_memalloc.c +2 -3
@@ -630,7 +630,7 @@ alloc_seg(struct rte_memseg *ms, void *addr, int socket_id, mapped:munmap (addr, alloc_sz); unmapped: - flags =MAP_FIXED ; + flags =EAL_RESERVE_FORCE_ADDRESS ; new_addr =eal_get_virtual_area (addr, &alloc_sz, alloc_sz,0 , flags);if (new_addr != addr) {if (new_addr !=NULL )
@@ -687,8 +687,7 @@ free_seg(struct rte_memseg *ms, struct hugepage_info *hi,return -1 ; } -if (madvise (ms->addr, ms->len,MADV_DONTDUMP ) !=0 ) -RTE_LOG (DEBUG ,EAL ,"madvise failed: %s\n" ,strerror (errno)); +eal_mem_set_dump (ms->addr, ms->len, false); exit_early = false;
lib/librte_eal/rte_eal_version.map +9 -0
@@ -387,3 +387,12 @@ EXPERIMENTAL { rte_trace_regexp; rte_trace_save; }; + +INTERNAL { + global: + + rte_mem_lock; + rte_mem_map; + rte_mem_page_size; + rte_mem_unmap; +};
lib/librte_eal/unix/eal_unix_memory.c new file +152 -0
152 lines not on this page · raw message
lib/librte_eal/unix/meson.build +1 -0
@@ -3,4 +3,5 @@ sources += files( 'eal_file.c', + 'eal_unix_memory.c', )
Signature · 1 line
2.25.4