V1 event/dlb add Intel DLB PMD
PATCH 26/27 event/dlb: add dequeue, dequeue_burst, and variants
In reply to McDaniel, TimothyChange-Id I0c90ad10be82750dc4c7c4bcbeed9b12677a996d
Signed-off-by McDaniel, Timothy <[email protected]>
--- drivers/event/dlb/dlb.c | 879 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 879 insertions(+)
drivers/event/dlb/dlb.c +879 -0
@@ -826,6 +826,700 @@ dlb_hw_do_enqueue(struct dlb_port *qm_port,dlb_pp_write (qm_port->qe4, port_data); } +static __rte_always_inline int +dlb_recv_qe (struct dlb_port *qm_port,struct dlb_dequeue_qe *qe, +uint8_t *offset) +{ +uint8_t xor_mask[2 ][4 ] = { {0x0F ,0x0E ,0x0C ,0x08 }, + {0x00 ,0x01 ,0x03 ,0x07 } }; +uint8_t and_mask[4 ] = {0x0F ,0x0E ,0x0C ,0x08 }; +volatile struct dlb_dequeue_qe *cq_addr; +__m128i *qes = (__m128i *)qe; +uint64_t *cache_line_base; +uint8_t gen_bits; + + cq_addr = dlb_port[qm_port->id][PORT_TYPE (qm_port)].cq_base; + cq_addr = &cq_addr[qm_port->cq_idx]; + + cache_line_base = (void *)(((uintptr_t )cq_addr) & ~0x3F ); + *offset = ((uintptr_t )cq_addr &0x30 ) >>4 ; + +/* Load the next CQ cache line from memory. Pack these reads as tight +* as possible to reduce the chance that DLB invalidates the line while +* the CPU is reading it. Read the cache line backwards to ensure that +* if QE[N] (N > 0) is valid, then QEs[0:N-1] are too. +* +* (Valid QEs start at &qe[offset]) +*/ + qes[3 ] =_mm_load_si128 ((__m128i *)&cache_line_base[6 ]); + qes[2 ] =_mm_load_si128 ((__m128i *)&cache_line_base[4 ]); + qes[1 ] =_mm_load_si128 ((__m128i *)&cache_line_base[2 ]); + qes[0 ] =_mm_load_si128 ((__m128i *)&cache_line_base[0 ]); + +/* Evict the cache line ASAP */ +dlb_cldemote (cache_line_base); + +/* Extract and combine the gen bits */ + gen_bits = ((_mm_extract_epi8 (qes[0 ],15 ) &0x1 ) <<0 ) | + ((_mm_extract_epi8 (qes[1 ],15 ) &0x1 ) <<1 ) | + ((_mm_extract_epi8 (qes[2 ],15 ) &0x1 ) <<2 ) | + ((_mm_extract_epi8 (qes[3 ],15 ) &0x1 ) <<3 ); + +/* XOR the combined bits such that a 1 represents a valid QE */ + gen_bits ^= xor_mask[qm_port->gen_bit][*offset]; + +/* Mask off gen bits we don't care about */ + gen_bits &= and_mask[*offset]; + +return __builtin_popcount (gen_bits); +} + +static __rte_always_inline int +dlb_recv_qe_sparse (struct dlb_port *qm_port,struct dlb_dequeue_qe *qe) +{ +volatile struct dlb_dequeue_qe *cq_addr; +uint8_t xor_mask[2 ] = {0x0F ,0x00 }; +const uint8_t and_mask =0x0F ; +__m128i *qes = (__m128i *)qe; +uint8_t gen_bits, gen_bit; +uintptr_t addr[4 ]; +uint16_t idx; + + cq_addr = dlb_port[qm_port->id][PORT_TYPE (qm_port)].cq_base; + + idx = qm_port->cq_idx; + +/* Load the next 4 QEs */ + addr[0 ] = (uintptr_t )&cq_addr[idx]; + addr[1 ] = (uintptr_t )&cq_addr[(idx +4 ) & qm_port->cq_depth_mask]; + addr[2 ] = (uintptr_t )&cq_addr[(idx +8 ) & qm_port->cq_depth_mask]; + addr[3 ] = (uintptr_t )&cq_addr[(idx +12 ) & qm_port->cq_depth_mask]; + +/* Prefetch next batch of QEs (all CQs occupy minimum 8 cache lines) */ +rte_prefetch0 (&cq_addr[(idx +16 ) & qm_port->cq_depth_mask]); +rte_prefetch0 (&cq_addr[(idx +20 ) & qm_port->cq_depth_mask]); +rte_prefetch0 (&cq_addr[(idx +24 ) & qm_port->cq_depth_mask]); +rte_prefetch0 (&cq_addr[(idx +28 ) & qm_port->cq_depth_mask]); + +/* Correct the xor_mask for wrap-around QEs */ + gen_bit = qm_port->gen_bit; + xor_mask[gen_bit] ^= !!((idx +4 ) > qm_port->cq_depth_mask) <<1 ; + xor_mask[gen_bit] ^= !!((idx +8 ) > qm_port->cq_depth_mask) <<2 ; + xor_mask[gen_bit] ^= !!((idx +12 ) > qm_port->cq_depth_mask) <<3 ; + +/* Read the cache lines backwards to ensure that if QE[N] (N > 0) is +* valid, then QEs[0:N-1] are too. +*/ + qes[3 ] =_mm_load_si128 ((__m128i *)(void *)addr[3 ]); +rte_compiler_barrier (); + qes[2 ] =_mm_load_si128 ((__m128i *)(void *)addr[2 ]); +rte_compiler_barrier (); + qes[1 ] =_mm_load_si128 ((__m128i *)(void *)addr[1 ]); +rte_compiler_barrier (); + qes[0 ] =_mm_load_si128 ((__m128i *)(void *)addr[0 ]); + +/* Extract and combine the gen bits */ + gen_bits = ((_mm_extract_epi8 (qes[0 ],15 ) &0x1 ) <<0 ) | + ((_mm_extract_epi8 (qes[1 ],15 ) &0x1 ) <<1 ) | + ((_mm_extract_epi8 (qes[2 ],15 ) &0x1 ) <<2 ) | + ((_mm_extract_epi8 (qes[3 ],15 ) &0x1 ) <<3 ); + +/* XOR the combined bits such that a 1 represents a valid QE */ + gen_bits ^= xor_mask[gen_bit]; + +/* Mask off gen bits we don't care about */ + gen_bits &= and_mask; + +return __builtin_popcount (gen_bits); +} + +static inline void +dlb_inc_cq_idx (struct dlb_port *qm_port,int cnt) +{ +uint16_t idx = qm_port->cq_idx_unmasked + cnt; + + qm_port->cq_idx_unmasked = idx; + qm_port->cq_idx = idx & qm_port->cq_depth_mask; + qm_port->gen_bit = (~(idx >> qm_port->gen_bit_shift)) &0x1 ; +} + +static inline bool +dlb_cq_is_empty (struct dlb_port *qm_port) +{ +volatile struct dlb_dequeue_qe *qe_ptr; +struct dlb_dequeue_qe qe; + + qe_ptr = dlb_port[qm_port->id][PORT_TYPE (qm_port)].cq_base; + qe = qe_ptr[qm_port->cq_idx]; + +return (qe.cq_gen != qm_port->gen_bit); +} + +static inline void dlb_send_rsvd_token_int_arm (struct dlb_port *qm_port) +{ +struct dlb_cq_pop_qe *qe = qm_port->consume_qe; +struct process_local_port_data *port_data; + + qe->tokens =0 ; + +rte_memcpy (&qm_port->qe4[0 ], qe,sizeof (struct dlb_cq_pop_qe )); + +/* Set int_arm to true in qm_port->qe4[0] so consume_qe isn't +* affected. +*/ + qm_port->qe4[0 ].int_arm =1 ; + +/* Fill remaining 3 entries (1,2, and 3) with noop QEs */ + +memset (&qm_port->qe4[1 ], +0 , + (DLB_NUM_QES_PER_CACHE_LINE -1 ) * +sizeof (struct dlb_cq_pop_qe )); + +/* No store fence needed since no pointer is being sent, and CQ token +* pops can be safely reordered with other HCWs. +*/ + port_data = &dlb_port[qm_port->id][PORT_TYPE (qm_port)]; + +dlb_pp_write (qm_port->qe4, port_data); + +DLB_LOG_DBG ("dlb: send rsvd token int arm\n" ); +} + +static inline void dlb_issue_int_arm_hcw (struct dlb_port *qm_port) +{ +dlb_send_rsvd_token_int_arm (qm_port); + + qm_port->cq_rsvd_token_deficit =1 ; + qm_port->int_armed = true; + +/* The delayed token return needs to account for this token pop HCW */ + qm_port->issued_releases--; +} + +static inline int +dlb_block_on_cq_interrupt (struct dlb_hw_dev *handle, +struct dlb_port *qm_port, +struct dlb_eventdev_port *ev_port) +{ +struct process_local_port_data *port_data; +int ret; + +/* If the interrupt is not armed, either sleep-poll (see comment below) +* or arm the interrupt. +*/ +if (!qm_port->int_armed) { +/* The reserved token scheme requires setting the interrupt +* depth threshold equal to the number of reserved tokens. +* Until the port receives its set of reserved tokens it cannot +* block on the interrupt, since it will not fire until the CQ +* contains at least num-reserved-tokens QEs. In lieu of that, +* we sleep-poll the CQ, which gives a similar behavior (other +* threads can run) albeit with a potentially slower response +* time. +*/ +if (qm_port->use_rsvd_token_scheme && + qm_port->cq_rsvd_token_deficit) { +sched_yield (); +return 0 ; + } + +dlb_issue_int_arm_hcw (qm_port); + } + + port_data = &dlb_port[qm_port->id][PORT_TYPE (qm_port)]; + +/* Note: it's safe to access the per-process cq_base address here, +* since the PMD won't block on the CQ until after attempting at least +* one CQ dequeue. +*/ + ret =dlb_iface_block_on_cq_interrupt ( + handle, + qm_port->id, + !ev_port->qm_port.is_directed, + &port_data->cq_base[qm_port->cq_idx], + qm_port->gen_bit, + false); + +/* If the CQ int ioctl was unsuccessful, the interrupt remains armed */ + qm_port->int_armed = (ret !=0 ); + +return ret; +} + +static inline int +dlb_process_dequeue_qes (struct dlb_eventdev_port *ev_port, +struct dlb_port *qm_port, +struct rte_event *events, +struct dlb_dequeue_qe *qes, +int cnt) +{ +uint8_t *qid_mappings = qm_port->qid_mappings; +int i, num; + +RTE_SET_USED (ev_port);/* avoids unused variable error */ + +for (i =0 , num =0 ; i < cnt; i++) { +struct dlb_dequeue_qe *qe = &qes[i]; +int sched_type_map[4 ] = { + [DLB_SCHED_ATOMIC ] =RTE_SCHED_TYPE_ATOMIC , + [DLB_SCHED_UNORDERED ] =RTE_SCHED_TYPE_PARALLEL , + [DLB_SCHED_ORDERED ] =RTE_SCHED_TYPE_ORDERED , + [DLB_SCHED_DIRECTED ] =RTE_SCHED_TYPE_ATOMIC , + }; + +DLB_LOG_DBG ("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n" , + (long long )qe->data, qe->qid, + qe->u.event_type.major, + qe->u.event_type.sub, + qe->pp_id, qe->sched_type, qe->qid, qe->error); + +/* Fill in event information. +* Note that flow_id must be embedded in the data by +* the app, such as the mbuf RSS hash field if the data +* buffer is a mbuf. +*/ +if (unlikely (qe->error)) { +DLB_LOG_ERR ("QE error bit ON\n" ); +DLB_INC_STAT (ev_port->stats.traffic.rx_drop,1 ); +dlb_consume_qe_immediate (qm_port,1 ); +continue ;/* Ignore */ + } + + events[num].u64 = qe->data; + events[num].queue_id = qid_mappings[qe->qid]; + events[num].priority =DLB_TO_EV_PRIO ((uint8_t )qe->priority); + events[num].event_type = qe->u.event_type.major; + events[num].sub_event_type = qe->u.event_type.sub; + events[num].sched_type = sched_type_map[qe->sched_type]; +DLB_INC_STAT (ev_port->stats.rx_sched_cnt[qe->sched_type],1 ); + +DLB_INC_STAT (ev_port->stats.traffic.rx_ok,1 ); + + num++; + } + +return num; +} + +static inline int +dlb_process_dequeue_four_qes (struct dlb_eventdev_port *ev_port, +struct dlb_port *qm_port, +struct rte_event *events, +struct dlb_dequeue_qe *qes) +{ +int sched_type_map[] = { + [DLB_SCHED_ATOMIC ] =RTE_SCHED_TYPE_ATOMIC , + [DLB_SCHED_UNORDERED ] =RTE_SCHED_TYPE_PARALLEL , + [DLB_SCHED_ORDERED ] =RTE_SCHED_TYPE_ORDERED , + [DLB_SCHED_DIRECTED ] =RTE_SCHED_TYPE_ATOMIC , + }; +const int num_events =DLB_NUM_QES_PER_CACHE_LINE ; +uint8_t *qid_mappings = qm_port->qid_mappings; +__m128i sse_evt[2 ]; +#ifdef DLB_QUELL_STATS + (void )ev_port;/* avoids unused variable error */ +#endif +int i; + +/* In the unlikely case that any of the QE error bits are set, process +* them one at a time. +*/ +if (unlikely (qes[0 ].error || qes[1 ].error || + qes[2 ].error || qes[3 ].error)) +return dlb_process_dequeue_qes (ev_port, qm_port, events, + qes, num_events); + +for (i =0 ; i <DLB_NUM_QES_PER_CACHE_LINE ; i++) { +DLB_LOG_DBG ("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n" , + (long long )qes[i].data, qes[i].qid, + qes[i].u.event_type.major, + qes[i].u.event_type.sub, + qes[i].pp_id, qes[i].sched_type, qes[i].qid, + qes[i].error); + } + + events[0 ].u64 = qes[0 ].data; + events[1 ].u64 = qes[1 ].data; + events[2 ].u64 = qes[2 ].data; + events[3 ].u64 = qes[3 ].data; + +/* Construct the metadata portion of two struct rte_events +* in one 128b SSE register. Event metadata is constructed in the SSE +* registers like so: +* sse_evt[0][63:0]: event[0]'s metadata +* sse_evt[0][127:64]: event[1]'s metadata +* sse_evt[1][63:0]: event[2]'s metadata +* sse_evt[1][127:64]: event[3]'s metadata +*/ + sse_evt[0 ] =_mm_setzero_si128 (); + sse_evt[1 ] =_mm_setzero_si128 (); + +/* Convert the hardware queue ID to an event queue ID and store it in +* the metadata: +* sse_evt[0][47:40] = qid_mappings[qes[0].qid] +* sse_evt[0][111:104] = qid_mappings[qes[1].qid] +* sse_evt[1][47:40] = qid_mappings[qes[2].qid] +* sse_evt[1][111:104] = qid_mappings[qes[3].qid] +*/ +#define RTE_EVENT_QUEUE_ID_BYTE 5 + sse_evt[0 ] =_mm_insert_epi8 (sse_evt[0 ], + qid_mappings[qes[0 ].qid], +RTE_EVENT_QUEUE_ID_BYTE ); + sse_evt[0 ] =_mm_insert_epi8 (sse_evt[0 ], + qid_mappings[qes[1 ].qid], +RTE_EVENT_QUEUE_ID_BYTE +8 ); + sse_evt[1 ] =_mm_insert_epi8 (sse_evt[1 ], + qid_mappings[qes[2 ].qid], +RTE_EVENT_QUEUE_ID_BYTE ); + sse_evt[1 ] =_mm_insert_epi8 (sse_evt[1 ], + qid_mappings[qes[3 ].qid], +RTE_EVENT_QUEUE_ID_BYTE +8 ); + +/* Convert the hardware priority to an event priority and store it in +* the metadata: +* sse_evt[0][55:48] = DLB_TO_EV_PRIO(qes[0].priority) +* sse_evt[0][119:112] = DLB_TO_EV_PRIO(qes[1].priority) +* sse_evt[1][55:48] = DLB_TO_EV_PRIO(qes[2].priority) +* sse_evt[1][119:112] = DLB_TO_EV_PRIO(qes[3].priority) +*/ +#define RTE_EVENT_PRIO_BYTE 6 + sse_evt[0 ] =_mm_insert_epi8 (sse_evt[0 ], +DLB_TO_EV_PRIO ((uint8_t )qes[0 ].priority), +RTE_EVENT_PRIO_BYTE ); + sse_evt[0 ] =_mm_insert_epi8 (sse_evt[0 ], +DLB_TO_EV_PRIO ((uint8_t )qes[1 ].priority), +RTE_EVENT_PRIO_BYTE +8 ); + sse_evt[1 ] =_mm_insert_epi8 (sse_evt[1 ], +DLB_TO_EV_PRIO ((uint8_t )qes[2 ].priority), +RTE_EVENT_PRIO_BYTE ); + sse_evt[1 ] =_mm_insert_epi8 (sse_evt[1 ], +DLB_TO_EV_PRIO ((uint8_t )qes[3 ].priority), +RTE_EVENT_PRIO_BYTE +8 ); + +/* Write the event type and sub event type to the event metadata. Leave +* flow ID unspecified, since the hardware does not maintain it during +* scheduling: +* sse_evt[0][31:0] = qes[0].u.event_type.major << 28 | +* qes[0].u.event_type.sub << 20; +* sse_evt[0][95:64] = qes[1].u.event_type.major << 28 | +* qes[1].u.event_type.sub << 20; +* sse_evt[1][31:0] = qes[2].u.event_type.major << 28 | +* qes[2].u.event_type.sub << 20; +* sse_evt[1][95:64] = qes[3].u.event_type.major << 28 | +* qes[3].u.event_type.sub << 20; +*/ +#define RTE_EVENT_EV_TYPE_DW 0 +#define RTE_EVENT_EV_TYPE_SHIFT 28 +#define RTE_EVENT_SUB_EV_TYPE_SHIFT 20 + sse_evt[0 ] =_mm_insert_epi32 (sse_evt[0 ], + qes[0 ].u.event_type.major <<RTE_EVENT_EV_TYPE_SHIFT | + qes[0 ].u.event_type.sub <<RTE_EVENT_SUB_EV_TYPE_SHIFT , +RTE_EVENT_EV_TYPE_DW ); + sse_evt[0 ] =_mm_insert_epi32 (sse_evt[0 ], + qes[1 ].u.event_type.major <<RTE_EVENT_EV_TYPE_SHIFT | + qes[1 ].u.event_type.sub <<RTE_EVENT_SUB_EV_TYPE_SHIFT , +RTE_EVENT_EV_TYPE_DW +2 ); + sse_evt[1 ] =_mm_insert_epi32 (sse_evt[1 ], + qes[2 ].u.event_type.major <<RTE_EVENT_EV_TYPE_SHIFT | + qes[2 ].u.event_type.sub <<RTE_EVENT_SUB_EV_TYPE_SHIFT , +RTE_EVENT_EV_TYPE_DW ); + sse_evt[1 ] =_mm_insert_epi32 (sse_evt[1 ], + qes[3 ].u.event_type.major <<RTE_EVENT_EV_TYPE_SHIFT | + qes[3 ].u.event_type.sub <<RTE_EVENT_SUB_EV_TYPE_SHIFT , +RTE_EVENT_EV_TYPE_DW +2 ); + +/* Write the sched type to the event metadata. 'op' and 'rsvd' are not +* set: +* sse_evt[0][39:32] = sched_type_map[qes[0].sched_type] << 6 +* sse_evt[0][103:96] = sched_type_map[qes[1].sched_type] << 6 +* sse_evt[1][39:32] = sched_type_map[qes[2].sched_type] << 6 +* sse_evt[1][103:96] = sched_type_map[qes[3].sched_type] << 6 +*/ +#define RTE_EVENT_SCHED_TYPE_BYTE 4 +#define RTE_EVENT_SCHED_TYPE_SHIFT 6 + sse_evt[0 ] =_mm_insert_epi8 (sse_evt[0 ], + sched_type_map[qes[0 ].sched_type] <<RTE_EVENT_SCHED_TYPE_SHIFT , +RTE_EVENT_SCHED_TYPE_BYTE ); + sse_evt[0 ] =_mm_insert_epi8 (sse_evt[0 ], + sched_type_map[qes[1 ].sched_type] <<RTE_EVENT_SCHED_TYPE_SHIFT , +RTE_EVENT_SCHED_TYPE_BYTE +8 ); + sse_evt[1 ] =_mm_insert_epi8 (sse_evt[1 ], + sched_type_map[qes[2 ].sched_type] <<RTE_EVENT_SCHED_TYPE_SHIFT , +RTE_EVENT_SCHED_TYPE_BYTE ); + sse_evt[1 ] =_mm_insert_epi8 (sse_evt[1 ], + sched_type_map[qes[3 ].sched_type] <<RTE_EVENT_SCHED_TYPE_SHIFT , +RTE_EVENT_SCHED_TYPE_BYTE +8 ); + +/* Store the metadata to the event (use the double-precision +* _mm_storeh_pd because there is no integer function for storing the +* upper 64b): +* events[0].event = sse_evt[0][63:0] +* events[1].event = sse_evt[0][127:64] +* events[2].event = sse_evt[1][63:0] +* events[3].event = sse_evt[1][127:64] +*/ +_mm_storel_epi64 ((__m128i *)&events[0 ].event, sse_evt[0 ]); +_mm_storeh_pd ((double *)&events[1 ].event, (__m128d ) sse_evt[0 ]); +_mm_storel_epi64 ((__m128i *)&events[2 ].event, sse_evt[1 ]); +_mm_storeh_pd ((double *)&events[3 ].event, (__m128d ) sse_evt[1 ]); + +DLB_INC_STAT (ev_port->stats.rx_sched_cnt[qes[0 ].sched_type],1 ); +DLB_INC_STAT (ev_port->stats.rx_sched_cnt[qes[1 ].sched_type],1 ); +DLB_INC_STAT (ev_port->stats.rx_sched_cnt[qes[2 ].sched_type],1 ); +DLB_INC_STAT (ev_port->stats.rx_sched_cnt[qes[3 ].sched_type],1 ); + +DLB_INC_STAT (ev_port->stats.traffic.rx_ok, num_events); + +return num_events; +} + +static inline int +dlb_dequeue_wait (struct dlb_eventdev *dlb, +struct dlb_eventdev_port *ev_port, +struct dlb_port *qm_port, +uint64_t timeout, +uint64_t start_ticks) +{ +struct process_local_port_data *port_data; +uint64_t elapsed_ticks; + + port_data = &dlb_port[qm_port->id][PORT_TYPE (qm_port)]; + + elapsed_ticks =rte_get_timer_cycles () - start_ticks; + +/* Wait/poll time expired */ +if (elapsed_ticks >= timeout) { +/* Interrupts not supported by PF PMD */ +return 1 ; + }else if (dlb->umwait_allowed) { +volatile struct dlb_dequeue_qe *cq_base; + + cq_base = port_data->cq_base; + +/* Block on cache line write to CQ. Note: it's +* safe to access the per-process cq_base +* address here, since the PMD has already +* attempted at least one CQ dequeue. +*/ +dlb_umonitor (&cq_base[qm_port->cq_idx]); + +/* Avoid race condition. Check if still empty */ +if (dlb_cq_is_empty (qm_port)) { +dlb_umwait (RTE_LIBRTE_PMD_DLB_UMWAIT_CTL_STATE , + timeout + start_ticks); +DLB_INC_STAT (ev_port->stats.traffic.rx_umonitor_umwait, +1 ); + } + }else { +uint64_t poll_interval =RTE_LIBRTE_PMD_DLB_POLL_INTERVAL ; +uint64_t curr_ticks =rte_get_timer_cycles (); +uint64_t init_ticks = curr_ticks; + +while ((curr_ticks - start_ticks < timeout) && + (curr_ticks - init_ticks < poll_interval)) + curr_ticks =rte_get_timer_cycles (); + } + +return 0 ; +} + +static inline int16_t +dlb_hw_dequeue (struct dlb_eventdev *dlb, +struct dlb_eventdev_port *ev_port, +struct rte_event *events, +uint16_t max_num, +uint64_t dequeue_timeout_ticks) +{ +uint64_t timeout; +uint64_t start_ticks =0ULL ; +struct dlb_port *qm_port; +int num =0 ; + + qm_port = &ev_port->qm_port; + +/* We have a special implementation for waiting. Wait can be: +* 1) no waiting at all +* 2) busy poll only +* 3) wait for interrupt. If wakeup and poll time +* has expired, then return to caller +* 4) umonitor/umwait repeatedly up to poll time +*/ + +/* If configured for per dequeue wait, then use wait value provided +* to this API. Otherwise we must use the global +* value from eventdev config time. +*/ +if (!dlb->global_dequeue_wait) + timeout = dequeue_timeout_ticks; +else + timeout = dlb->global_dequeue_wait_ticks; + +if (timeout) + start_ticks =rte_get_timer_cycles (); + +while (num < max_num) { +struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE ]; +uint8_t offset; +int num_avail; + +/* Copy up to 4 QEs from the current cache line into qes */ + num_avail =dlb_recv_qe (qm_port, qes, &offset); + +/* But don't process more than the user requested */ + num_avail =RTE_MIN (num_avail, max_num - num); + +dlb_inc_cq_idx (qm_port, num_avail); + +if (num_avail ==DLB_NUM_QES_PER_CACHE_LINE ) + num +=dlb_process_dequeue_four_qes (ev_port, + qm_port, + &events[num], + &qes[offset]); +else if (num_avail) + num +=dlb_process_dequeue_qes (ev_port, + qm_port, + &events[num], + &qes[offset], + num_avail); +else if ((timeout ==0 ) || (num >0 )) +/* Not waiting in any form, or 1+ events received? */ +break ; +else if (dlb_dequeue_wait (dlb, ev_port, qm_port, + timeout, start_ticks)) +break ; + } + + qm_port->owed_tokens += num; + +if (num && qm_port->token_pop_mode ==AUTO_POP ) +dlb_consume_qe_immediate (qm_port, num); + + ev_port->outstanding_releases += num; + +return num; +} + +static inline int +dlb_process_dequeue_qe(struct dlb_eventdev_port *ev_port __rte_unused, +struct dlb_port *qm_port, +struct rte_event *event, +struct dlb_dequeue_qe *qe) +{ +int sched_type_map[4 ] = { + [DLB_SCHED_ATOMIC ] =RTE_SCHED_TYPE_ATOMIC , + [DLB_SCHED_UNORDERED ] =RTE_SCHED_TYPE_PARALLEL , + [DLB_SCHED_ORDERED ] =RTE_SCHED_TYPE_ORDERED , + [DLB_SCHED_DIRECTED ] =RTE_SCHED_TYPE_ATOMIC , + }; +uint8_t *qid_mappings = qm_port->qid_mappings; + +DLB_LOG_DBG ("dequeue success, data = 0x%llx, qid=%d, event_type=%d, subevent=%d\npp_id = %d, sched_type = %d, qid = %d, err=%d\n" , + (long long )qe->data, qe->qid, + qe->u.event_type.major, + qe->u.event_type.sub, + qe->pp_id, qe->sched_type, qe->qid, qe->error); + +/* Fill in event information. +* Note that flow_id must be embedded in the data by +* the app, such as the mbuf RSS hash field if the data +* buffer is a mbuf. +*/ +if (unlikely (qe->error)) { +DLB_LOG_ERR ("QE error bit ON\n" ); +DLB_INC_STAT (ev_port->stats.traffic.rx_drop,1 ); +dlb_consume_qe_immediate (qm_port,1 ); +return 0 ; + } + + event->u64 = qe->data; + event->queue_id = qid_mappings[qe->qid]; + event->priority =DLB_TO_EV_PRIO ((uint8_t )qe->priority); + event->event_type = qe->u.event_type.major; + event->sub_event_type = qe->u.event_type.sub; + event->sched_type = sched_type_map[qe->sched_type]; +DLB_INC_STAT (ev_port->stats.rx_sched_cnt[qe->sched_type],1 ); + +DLB_INC_STAT (ev_port->stats.traffic.rx_ok,1 ); + +return 1 ; +} + +static inline int16_t +dlb_hw_dequeue_sparse (struct dlb_eventdev *dlb, +struct dlb_eventdev_port *ev_port, +struct rte_event *events, +uint16_t max_num, +uint64_t dequeue_timeout_ticks) +{ +uint64_t timeout; +uint64_t start_ticks =0ULL ; +struct dlb_port *qm_port; +int num =0 ; + + qm_port = &ev_port->qm_port; + +/* We have a special implementation for waiting. Wait can be: +* 1) no waiting at all +* 2) busy poll only +* 3) wait for interrupt. If wakeup and poll time +* has expired, then return to caller +* 4) umonitor/umwait repeatedly up to poll time +*/ + +/* If configured for per dequeue wait, then use wait value provided +* to this API. Otherwise we must use the global +* value from eventdev config time. +*/ +if (!dlb->global_dequeue_wait) + timeout = dequeue_timeout_ticks; +else + timeout = dlb->global_dequeue_wait_ticks; + +if (timeout) + start_ticks =rte_get_timer_cycles (); + +while (num < max_num) { +struct dlb_dequeue_qe qes[DLB_NUM_QES_PER_CACHE_LINE ]; +int num_avail; + +/* Copy up to 4 QEs from the current cache line into qes */ + num_avail =dlb_recv_qe_sparse (qm_port, qes); + +/* But don't process more than the user requested */ + num_avail =RTE_MIN (num_avail, max_num - num); + +dlb_inc_cq_idx (qm_port, num_avail <<2 ); + +if (num_avail ==DLB_NUM_QES_PER_CACHE_LINE ) + num +=dlb_process_dequeue_four_qes (ev_port, + qm_port, + &events[num], + &qes[0 ]); +else if (num_avail) + num +=dlb_process_dequeue_qes (ev_port, + qm_port, + &events[num], + &qes[0 ], + num_avail); +else if ((timeout ==0 ) || (num >0 )) +/* Not waiting in any form, or 1+ events received? */ +break ; +else if (dlb_dequeue_wait (dlb, ev_port, qm_port, + timeout, start_ticks)) +break ; + } + + qm_port->owed_tokens += num; + +if (num && qm_port->token_pop_mode ==AUTO_POP ) +dlb_consume_qe_immediate (qm_port, num); + + ev_port->outstanding_releases += num; + +return num; +} + static int16_t dlb_hw_map_ldb_qid_to_port(struct dlb_hw_dev *handle, uint32_t qm_port_id,
@@ -1964,6 +2658,181 @@ dlb_eventdev_configure(const struct rte_eventdev *dev)return 0 ; } +static void +dlb_event_release (struct dlb_eventdev *dlb, +uint8_t port_id, +int n) +{ +struct process_local_port_data *port_data; +struct dlb_eventdev_port *ev_port; +struct dlb_port *qm_port; +int i, cnt; + +if (port_id > dlb->num_ports) { +DLB_LOG_ERR ("Invalid port id %d in dlb-event_release\n" , + port_id); + rte_errno = -EINVAL ; +return ; + } + + ev_port = &dlb->ev_ports[port_id]; + qm_port = &ev_port->qm_port; + port_data = &dlb_port[qm_port->id][PORT_TYPE (qm_port)]; + + cnt =0 ; + +if (qm_port->is_directed) { + cnt = n; + goto sw_credit_update; + } + +/* Since MOVDIR64B is weakly-ordered, use an SFENCE to ensure that +* application writes complete before enqueueing the release HCW. +*/ +rte_wmb (); + +for (i =0 ; i < n; i +=DLB_NUM_QES_PER_CACHE_LINE ) { +int j; + +/* Zero-out QEs */ + qm_port->qe4[0 ].cmd_byte =0 ; + qm_port->qe4[1 ].cmd_byte =0 ; + qm_port->qe4[2 ].cmd_byte =0 ; + qm_port->qe4[3 ].cmd_byte =0 ; + +for (j =0 ; j <DLB_NUM_QES_PER_CACHE_LINE && (i + j) < n; j++) + qm_port->qe4[j].cmd_byte =DLB_COMP_CMD_BYTE ; + + qm_port->issued_releases += j; + +if (j ==0 ) +break ; + +if (qm_port->token_pop_mode ==DELAYED_POP && j <4 && + qm_port->issued_releases >= qm_port->token_pop_thresh -1 ) { +dlb_construct_token_pop_qe (qm_port, j); + +/* Reset the releases counter for the next QE batch */ + qm_port->issued_releases -= qm_port->token_pop_thresh; + +/* When using delayed token pop mode, the initial token +* threshold is the full CQ depth. After the first +* token pop, we need to reset it to the dequeue_depth. +*/ + qm_port->token_pop_thresh = qm_port->dequeue_depth; + } + +dlb_hw_do_enqueue (qm_port, port_data); + + cnt += j; + } + +if (qm_port->token_pop_mode ==DELAYED_POP && + qm_port->issued_releases >= qm_port->token_pop_thresh -1 ) { +dlb_consume_qe_immediate (qm_port, qm_port->owed_tokens); + qm_port->issued_releases -= qm_port->token_pop_thresh; + qm_port->token_pop_thresh = qm_port->dequeue_depth; + } + +sw_credit_update: +/* each release returns one credit */ +if (!ev_port->outstanding_releases) +rte_panic ("%s: Outstanding releases underflowed.\n" , __func__); + ev_port->outstanding_releases -= cnt; + ev_port->inflight_credits += cnt; + +/* Replenish s/w credits if enough releases are performed */ +dlb_replenish_sw_credits (dlb, ev_port); +} + +static uint16_t +dlb_event_dequeue_burst (void *event_port,struct rte_event *ev,uint16_t num, +uint64_t wait) +{ +struct dlb_eventdev_port *ev_port = event_port; +struct dlb_port *qm_port = &ev_port->qm_port; +struct dlb_eventdev *dlb = ev_port->dlb; +struct process_local_port_data *port_data; +uint16_t cnt; + + rte_errno =0 ; + +RTE_ASSERT (ev_port->setup_done); +RTE_ASSERT (ev !=NULL ); + + port_data = &dlb_port[qm_port->id][PORT_TYPE (qm_port)]; + +if (!port_data->mmaped) +dlb_iface_port_mmap (qm_port); + +if (ev_port->implicit_release && ev_port->outstanding_releases >0 ) { +uint16_t out_rels = ev_port->outstanding_releases; + +dlb_event_release (dlb, ev_port->id, out_rels); + +DLB_INC_STAT (ev_port->stats.tx_implicit_rel, out_rels); + } + +if (qm_port->token_pop_mode ==DEFERRED_POP && + qm_port->owed_tokens) +dlb_consume_qe_immediate (qm_port, qm_port->owed_tokens); + + cnt =dlb_hw_dequeue (dlb, ev_port, ev, num, wait); + +DLB_INC_STAT (ev_port->stats.traffic.total_polls,1 ); +DLB_INC_STAT (ev_port->stats.traffic.zero_polls, ((cnt ==0 ) ?1 :0 )); +return cnt; +} + +static uint16_t +dlb_event_dequeue (void *event_port,struct rte_event *ev,uint64_t wait) +{ +return dlb_event_dequeue_burst (event_port, ev,1 , wait); +} + +static uint16_t +dlb_event_dequeue_burst_sparse (void *event_port,struct rte_event *ev, +uint16_t num,uint64_t wait) +{ +struct dlb_eventdev_port *ev_port = event_port; +struct dlb_port *qm_port = &ev_port->qm_port; +struct dlb_eventdev *dlb = ev_port->dlb; +struct process_local_port_data *port_data; +uint16_t cnt; + + rte_errno =0 ; + +RTE_ASSERT (ev_port->setup_done); +RTE_ASSERT (ev !=NULL ); + + port_data = &dlb_port[qm_port->id][PORT_TYPE (qm_port)]; + +if (!port_data->mmaped) +dlb_iface_port_mmap (qm_port); + +if (ev_port->implicit_release && ev_port->outstanding_releases >0 ) { +uint16_t out_rels = ev_port->outstanding_releases; + +dlb_event_release (dlb, ev_port->id, out_rels); + +DLB_INC_STAT (ev_port->stats.tx_implicit_rel, out_rels); + } + +if (qm_port->token_pop_mode ==DEFERRED_POP && + qm_port->owed_tokens) + cnt =dlb_hw_dequeue_sparse (dlb, ev_port, ev, num, wait); + +DLB_INC_STAT (ev_port->stats.traffic.total_polls,1 ); +DLB_INC_STAT (ev_port->stats.traffic.zero_polls, ((cnt ==0 ) ?1 :0 )); +return cnt; +} + +static uint16_t +dlb_event_dequeue_sparse (void *event_port,struct rte_event *ev,uint64_t wait) +{ +return dlb_event_dequeue_burst_sparse (event_port, ev,1 , wait); +} + static int dlb_event_queue_join_ldb(struct dlb_eventdev *dlb, struct dlb_eventdev_port *ev_port,
@@ -2907,6 +3776,8 @@ set_num_atm_inflights(const char *key __rte_unused,void dlb_entry_points_init (struct rte_eventdev *dev) { +struct dlb_eventdev *dlb; +static struct rte_eventdev_ops dlb_eventdev_entry_ops = { .dev_infos_get = dlb_eventdev_info_get, .dev_configure = dlb_eventdev_configure,
@@ -2937,7 +3808,15 @@ dlb_entry_points_init(struct rte_eventdev *dev) dev->enqueue_burst = dlb_event_enqueue_burst; dev->enqueue_new_burst = dlb_event_enqueue_new_burst; dev->enqueue_forward_burst = dlb_event_enqueue_forward_burst; + dev->dequeue = dlb_event_dequeue; + dev->dequeue_burst = dlb_event_dequeue_burst; + + dlb = dev->data->dev_private; +if (dlb->poll_mode ==DLB_CQ_POLL_MODE_SPARSE ) { + dev->dequeue = dlb_event_dequeue_sparse; + dev->dequeue_burst = dlb_event_dequeue_burst_sparse; + } } static void
Signature · 1 line
2.13.6
PATCH 27/27 event/dlb: add eventdev_stop and eventdev_close
In reply to McDaniel, TimothyChange-Id Iff64ba83ead496b3dedeaf323ee09bce1f631a6f
Signed-off-by McDaniel, Timothy <[email protected]>
--- drivers/event/dlb/dlb.c | 263 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 263 insertions(+)
drivers/event/dlb/dlb.c +263 -0
@@ -305,6 +305,23 @@ dlb_event_enqueue_forward_burst_delayed(void *event_port,const struct rte_event events[], uint16_t num); +void +dlb_free_qe_mem (struct dlb_port *qm_port) +{ +if (qm_port ==NULL ) +return ; + +if (qm_port->qe4) { +rte_free (qm_port->qe4); + qm_port->qe4 =NULL ; + } + +if (qm_port->consume_qe) { +rte_free (qm_port->consume_qe); + qm_port->consume_qe =NULL ; + } +} +int dlb_init_qe_mem (struct dlb_port *qm_port,char *mz_name) {
@@ -1922,6 +1939,250 @@ dlb_eventdev_start(struct rte_eventdev *dev)return 0 ; } +static void +dlb_flush_port (struct rte_eventdev *dev,int port_id) +{ +struct dlb_eventdev *dlb =dlb_pmd_priv (dev); +eventdev_stop_flush_t flush; +struct rte_event ev; +uint8_t dev_id; +void *arg; +int i; + + flush = dev->dev_ops->dev_stop_flush; + dev_id = dev->data->dev_id; + arg = dev->data->dev_stop_flush_arg; + +while (rte_event_dequeue_burst (dev_id, port_id, &ev,1 ,0 )) { +if (flush) +flush (dev_id, ev, arg); + +if (dlb->ev_ports[port_id].qm_port.is_directed) +continue ; + + ev.op =RTE_EVENT_OP_RELEASE ; + +rte_event_enqueue_burst (dev_id, port_id, &ev,1 ); + } + +/* Enqueue any additional outstanding releases */ + ev.op =RTE_EVENT_OP_RELEASE ; + +for (i = dlb->ev_ports[port_id].outstanding_releases; i >0 ; i--) +rte_event_enqueue_burst (dev_id, port_id, &ev,1 ); +} + +static uint32_t +dlb_get_ldb_queue_depth (struct dlb_eventdev *dlb, +struct dlb_eventdev_queue *queue) +{ +struct dlb_hw_dev *handle = &dlb->qm_instance; +struct dlb_get_ldb_queue_depth_args cfg; +struct dlb_cmd_response response; +int ret; + + cfg.queue_id = queue->qm_queue.id; + cfg.response = (uintptr_t )&response; + + ret =dlb_iface_get_ldb_queue_depth (handle, &cfg); +if (ret <0 ) { +DLB_LOG_ERR ("dlb: get_ldb_queue_depth ret=%d (driver status: %s)\n" , + ret, dlb_error_strings[response.status]); +return ret; + } + +return response.id; +} + +static uint32_t +dlb_get_dir_queue_depth (struct dlb_eventdev *dlb, +struct dlb_eventdev_queue *queue) +{ +struct dlb_hw_dev *handle = &dlb->qm_instance; +struct dlb_get_dir_queue_depth_args cfg; +struct dlb_cmd_response response; +int ret; + + cfg.queue_id = queue->qm_queue.id; + cfg.response = (uintptr_t )&response; + + ret =dlb_iface_get_dir_queue_depth (handle, &cfg); +if (ret <0 ) { +DLB_LOG_ERR ("dlb: get_dir_queue_depth ret=%d (driver status: %s)\n" , + ret, dlb_error_strings[response.status]); +return ret; + } + +return response.id; +} + +uint32_t +dlb_get_queue_depth (struct dlb_eventdev *dlb, +struct dlb_eventdev_queue *queue) +{ +if (queue->qm_queue.is_directed) +return dlb_get_dir_queue_depth (dlb, queue); +else +return dlb_get_ldb_queue_depth (dlb, queue); +} + +static bool +dlb_queue_is_empty (struct dlb_eventdev *dlb, +struct dlb_eventdev_queue *queue) +{ +return dlb_get_queue_depth (dlb, queue) ==0 ; +} + +static bool +dlb_linked_queues_empty (struct dlb_eventdev *dlb) +{ +int i; + +for (i =0 ; i < dlb->num_queues; i++) { +if (dlb->ev_queues[i].num_links ==0 ) +continue ; +if (!dlb_queue_is_empty (dlb, &dlb->ev_queues[i])) +return false; + } + +return true; +} + +static bool +dlb_queues_empty (struct dlb_eventdev *dlb) +{ +int i; + +for (i =0 ; i < dlb->num_queues; i++) { +if (!dlb_queue_is_empty (dlb, &dlb->ev_queues[i])) +return false; + } + +return true; +} + +void +dlb_drain (struct rte_eventdev *dev) +{ +struct dlb_eventdev *dlb =dlb_pmd_priv (dev); +struct dlb_eventdev_port *ev_port =NULL ; +uint8_t dev_id; +int i; + + dev_id = dev->data->dev_id; + +while (!dlb_linked_queues_empty (dlb)) { +/* Flush all the ev_ports, which will drain all their connected +* queues. +*/ +for (i =0 ; i < dlb->num_ports; i++) +dlb_flush_port (dev, i); + } + +/* The queues are empty, but there may be events left in the ports. */ +for (i =0 ; i < dlb->num_ports; i++) +dlb_flush_port (dev, i); + +/* If the domain's queues are empty, we're done. */ +if (dlb_queues_empty (dlb)) +return ; + +/* Else, there must be at least one unlinked load-balanced queue. +* Select a load-balanced port with which to drain the unlinked +* queue(s). +*/ +for (i =0 ; i < dlb->num_ports; i++) { + ev_port = &dlb->ev_ports[i]; + +if (!ev_port->qm_port.is_directed) +break ; + } + +if (i == dlb->num_ports) { +DLB_LOG_ERR ("internal error: no LDB ev_ports\n" ); +return ; + } + + rte_errno =0 ; +rte_event_port_unlink (dev_id, ev_port->id,NULL ,0 ); + +if (rte_errno) { +DLB_LOG_ERR ("internal error: failed to unlink ev_port %d\n" , + ev_port->id); +return ; + } + +for (i =0 ; i < dlb->num_queues; i++) { +uint8_t qid, prio; +int ret; + +if (dlb_queue_is_empty (dlb, &dlb->ev_queues[i])) +continue ; + + qid = i; + prio =0 ; + +/* Link the ev_port to the queue */ + ret =rte_event_port_link (dev_id, ev_port->id, &qid, &prio,1 ); +if (ret !=1 ) { +DLB_LOG_ERR ("internal error: failed to link ev_port %d to queue %d\n" , + ev_port->id, qid); +return ; + } + +/* Flush the queue */ +while (!dlb_queue_is_empty (dlb, &dlb->ev_queues[i])) +dlb_flush_port (dev, ev_port->id); + +/* Drain any extant events in the ev_port. */ +dlb_flush_port (dev, ev_port->id); + +/* Unlink the ev_port from the queue */ + ret =rte_event_port_unlink (dev_id, ev_port->id, &qid,1 ); +if (ret !=1 ) { +DLB_LOG_ERR ("internal error: failed to unlink ev_port %d to queue %d\n" , + ev_port->id, qid); +return ; + } + } +} + +static void +dlb_eventdev_stop (struct rte_eventdev *dev) +{ +struct dlb_eventdev *dlb =dlb_pmd_priv (dev); + +rte_spinlock_lock (&dlb->qm_instance.resource_lock); + +if (dlb->run_state ==DLB_RUN_STATE_STOPPED ) { +DLB_LOG_DBG ("Internal error: already stopped\n" ); +rte_spinlock_unlock (&dlb->qm_instance.resource_lock); +return ; + }else if (dlb->run_state !=DLB_RUN_STATE_STARTED ) { +DLB_LOG_ERR ("Internal error: bad state %d for dev_stop\n" , + (int )dlb->run_state); +rte_spinlock_unlock (&dlb->qm_instance.resource_lock); +return ; + } + + dlb->run_state =DLB_RUN_STATE_STOPPING ; + +rte_spinlock_unlock (&dlb->qm_instance.resource_lock); + +dlb_drain (dev); + + dlb->run_state =DLB_RUN_STATE_STOPPED ; +} + +static int +dlb_eventdev_close (struct rte_eventdev *dev) +{ +dlb_hw_reset_sched_domain (dev, false); + +return 0 ; +} + +static inline int dlb_check_enqueue_sw_credits (struct dlb_eventdev *dlb,struct dlb_eventdev_port *ev_port)
@@ -3782,6 +4043,8 @@ dlb_entry_points_init(struct rte_eventdev *dev) .dev_infos_get = dlb_eventdev_info_get, .dev_configure = dlb_eventdev_configure, .dev_start = dlb_eventdev_start, + .dev_stop = dlb_eventdev_stop, + .dev_close = dlb_eventdev_close, .queue_def_conf = dlb_eventdev_queue_default_conf_get, .queue_setup = dlb_eventdev_queue_setup, .queue_release = dlb_eventdev_queue_release,
Signature · 1 line
2.13.6