be2net: add support for ndo_busy_poll
Includes: - ndo_busy_poll implementation - Locking between napi and busy_poll - Fix rx_post_starvation (replenish rx-queues in out-of-mememory scenario) logic to accomodate busy_poll. v2 changes: [Eric D.'s comment] call alloc_pages() with GFP_ATOMIC even in ndo_busy_poll context as it is not allowed to sleep. Signed-off-by: Sathya Perla <sathya.perla@emulex.com> Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Sathya Perla
David S. Miller
parent
4d961a101e
commit
6384a4d0dc
2 changed files with 187 additions and 18 deletions
|
|
@ -199,6 +199,19 @@ struct be_eq_obj {
|
|||
u16 spurious_intr;
|
||||
struct napi_struct napi;
|
||||
struct be_adapter *adapter;
|
||||
|
||||
#ifdef CONFIG_NET_RX_BUSY_POLL
|
||||
#define BE_EQ_IDLE 0
|
||||
#define BE_EQ_NAPI 1 /* napi owns this EQ */
|
||||
#define BE_EQ_POLL 2 /* poll owns this EQ */
|
||||
#define BE_EQ_LOCKED (BE_EQ_NAPI | BE_EQ_POLL)
|
||||
#define BE_EQ_NAPI_YIELD 4 /* napi yielded this EQ */
|
||||
#define BE_EQ_POLL_YIELD 8 /* poll yielded this EQ */
|
||||
#define BE_EQ_YIELD (BE_EQ_NAPI_YIELD | BE_EQ_POLL_YIELD)
|
||||
#define BE_EQ_USER_PEND (BE_EQ_POLL | BE_EQ_POLL_YIELD)
|
||||
unsigned int state;
|
||||
spinlock_t lock; /* lock to serialize napi and busy-poll */
|
||||
#endif /* CONFIG_NET_RX_BUSY_POLL */
|
||||
} ____cacheline_aligned_in_smp;
|
||||
|
||||
struct be_aic_obj { /* Adaptive interrupt coalescing (AIC) info */
|
||||
|
|
@ -212,6 +225,11 @@ struct be_aic_obj { /* Adaptive interrupt coalescing (AIC) info */
|
|||
u64 tx_reqs_prev; /* Used to calculate TX pps */
|
||||
};
|
||||
|
||||
enum {
|
||||
NAPI_POLLING,
|
||||
BUSY_POLLING
|
||||
};
|
||||
|
||||
struct be_mcc_obj {
|
||||
struct be_queue_info q;
|
||||
struct be_queue_info cq;
|
||||
|
|
@ -561,6 +579,10 @@ extern const struct ethtool_ops be_ethtool_ops;
|
|||
for (i = 0, eqo = &adapter->eq_obj[i]; i < adapter->num_evt_qs; \
|
||||
i++, eqo++)
|
||||
|
||||
#define for_all_rx_queues_on_eq(adapter, eqo, rxo, i) \
|
||||
for (i = eqo->idx, rxo = &adapter->rx_obj[i]; i < adapter->num_rx_qs;\
|
||||
i += adapter->num_evt_qs, rxo += adapter->num_evt_qs)
|
||||
|
||||
#define is_mcc_eqo(eqo) (eqo->idx == 0)
|
||||
#define mcc_eqo(adapter) (&adapter->eq_obj[0])
|
||||
|
||||
|
|
@ -711,6 +733,106 @@ static inline int qnq_async_evt_rcvd(struct be_adapter *adapter)
|
|||
return adapter->flags & BE_FLAGS_QNQ_ASYNC_EVT_RCVD;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_NET_RX_BUSY_POLL
|
||||
static inline bool be_lock_napi(struct be_eq_obj *eqo)
|
||||
{
|
||||
bool status = true;
|
||||
|
||||
spin_lock(&eqo->lock); /* BH is already disabled */
|
||||
if (eqo->state & BE_EQ_LOCKED) {
|
||||
WARN_ON(eqo->state & BE_EQ_NAPI);
|
||||
eqo->state |= BE_EQ_NAPI_YIELD;
|
||||
status = false;
|
||||
} else {
|
||||
eqo->state = BE_EQ_NAPI;
|
||||
}
|
||||
spin_unlock(&eqo->lock);
|
||||
return status;
|
||||
}
|
||||
|
||||
static inline void be_unlock_napi(struct be_eq_obj *eqo)
|
||||
{
|
||||
spin_lock(&eqo->lock); /* BH is already disabled */
|
||||
|
||||
WARN_ON(eqo->state & (BE_EQ_POLL | BE_EQ_NAPI_YIELD));
|
||||
eqo->state = BE_EQ_IDLE;
|
||||
|
||||
spin_unlock(&eqo->lock);
|
||||
}
|
||||
|
||||
static inline bool be_lock_busy_poll(struct be_eq_obj *eqo)
|
||||
{
|
||||
bool status = true;
|
||||
|
||||
spin_lock_bh(&eqo->lock);
|
||||
if (eqo->state & BE_EQ_LOCKED) {
|
||||
eqo->state |= BE_EQ_POLL_YIELD;
|
||||
status = false;
|
||||
} else {
|
||||
eqo->state |= BE_EQ_POLL;
|
||||
}
|
||||
spin_unlock_bh(&eqo->lock);
|
||||
return status;
|
||||
}
|
||||
|
||||
static inline void be_unlock_busy_poll(struct be_eq_obj *eqo)
|
||||
{
|
||||
spin_lock_bh(&eqo->lock);
|
||||
|
||||
WARN_ON(eqo->state & (BE_EQ_NAPI));
|
||||
eqo->state = BE_EQ_IDLE;
|
||||
|
||||
spin_unlock_bh(&eqo->lock);
|
||||
}
|
||||
|
||||
static inline void be_enable_busy_poll(struct be_eq_obj *eqo)
|
||||
{
|
||||
spin_lock_init(&eqo->lock);
|
||||
eqo->state = BE_EQ_IDLE;
|
||||
}
|
||||
|
||||
static inline void be_disable_busy_poll(struct be_eq_obj *eqo)
|
||||
{
|
||||
local_bh_disable();
|
||||
|
||||
/* It's enough to just acquire napi lock on the eqo to stop
|
||||
* be_busy_poll() from processing any queueus.
|
||||
*/
|
||||
while (!be_lock_napi(eqo))
|
||||
mdelay(1);
|
||||
|
||||
local_bh_enable();
|
||||
}
|
||||
|
||||
#else /* CONFIG_NET_RX_BUSY_POLL */
|
||||
|
||||
static inline bool be_lock_napi(struct be_eq_obj *eqo)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
static inline void be_unlock_napi(struct be_eq_obj *eqo)
|
||||
{
|
||||
}
|
||||
|
||||
static inline bool be_lock_busy_poll(struct be_eq_obj *eqo)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
static inline void be_unlock_busy_poll(struct be_eq_obj *eqo)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void be_enable_busy_poll(struct be_eq_obj *eqo)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void be_disable_busy_poll(struct be_eq_obj *eqo)
|
||||
{
|
||||
}
|
||||
#endif /* CONFIG_NET_RX_BUSY_POLL */
|
||||
|
||||
void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm,
|
||||
u16 num_popped);
|
||||
void be_link_status_update(struct be_adapter *adapter, u8 link_status);
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@
|
|||
#include <asm/div64.h>
|
||||
#include <linux/aer.h>
|
||||
#include <linux/if_bridge.h>
|
||||
#include <net/busy_poll.h>
|
||||
|
||||
MODULE_VERSION(DRV_VER);
|
||||
MODULE_DEVICE_TABLE(pci, be_dev_ids);
|
||||
|
|
@ -1556,7 +1557,7 @@ static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb,
|
|||
}
|
||||
|
||||
/* Process the RX completion indicated by rxcp when GRO is disabled */
|
||||
static void be_rx_compl_process(struct be_rx_obj *rxo,
|
||||
static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi,
|
||||
struct be_rx_compl_info *rxcp)
|
||||
{
|
||||
struct be_adapter *adapter = rxo->adapter;
|
||||
|
|
@ -1581,7 +1582,7 @@ static void be_rx_compl_process(struct be_rx_obj *rxo,
|
|||
skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
|
||||
if (netdev->features & NETIF_F_RXHASH)
|
||||
skb->rxhash = rxcp->rss_hash;
|
||||
|
||||
skb_mark_napi_id(skb, napi);
|
||||
|
||||
if (rxcp->vlanf)
|
||||
__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
|
||||
|
|
@ -1639,6 +1640,7 @@ static void be_rx_compl_process_gro(struct be_rx_obj *rxo,
|
|||
skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
|
||||
if (adapter->netdev->features & NETIF_F_RXHASH)
|
||||
skb->rxhash = rxcp->rss_hash;
|
||||
skb_mark_napi_id(skb, napi);
|
||||
|
||||
if (rxcp->vlanf)
|
||||
__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
|
||||
|
|
@ -1819,6 +1821,8 @@ static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp)
|
|||
|
||||
if (posted) {
|
||||
atomic_add(posted, &rxq->used);
|
||||
if (rxo->rx_post_starved)
|
||||
rxo->rx_post_starved = false;
|
||||
be_rxq_notify(adapter, rxq->id, posted);
|
||||
} else if (atomic_read(&rxq->used) == 0) {
|
||||
/* Let be_worker replenish when memory is available */
|
||||
|
|
@ -2021,6 +2025,7 @@ static void be_evt_queues_destroy(struct be_adapter *adapter)
|
|||
if (eqo->q.created) {
|
||||
be_eq_clean(eqo);
|
||||
be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ);
|
||||
napi_hash_del(&eqo->napi);
|
||||
netif_napi_del(&eqo->napi);
|
||||
}
|
||||
be_queue_free(adapter, &eqo->q);
|
||||
|
|
@ -2040,6 +2045,7 @@ static int be_evt_queues_create(struct be_adapter *adapter)
|
|||
for_all_evt_queues(adapter, eqo, i) {
|
||||
netif_napi_add(adapter->netdev, &eqo->napi, be_poll,
|
||||
BE_NAPI_WEIGHT);
|
||||
napi_hash_add(&eqo->napi);
|
||||
aic = &adapter->aic_obj[i];
|
||||
eqo->adapter = adapter;
|
||||
eqo->tx_budget = BE_TX_BUDGET;
|
||||
|
|
@ -2262,7 +2268,7 @@ static inline bool do_gro(struct be_rx_compl_info *rxcp)
|
|||
}
|
||||
|
||||
static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
|
||||
int budget)
|
||||
int budget, int polling)
|
||||
{
|
||||
struct be_adapter *adapter = rxo->adapter;
|
||||
struct be_queue_info *rx_cq = &rxo->cq;
|
||||
|
|
@ -2293,10 +2299,12 @@ static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
|
|||
goto loop_continue;
|
||||
}
|
||||
|
||||
if (do_gro(rxcp))
|
||||
/* Don't do gro when we're busy_polling */
|
||||
if (do_gro(rxcp) && polling != BUSY_POLLING)
|
||||
be_rx_compl_process_gro(rxo, napi, rxcp);
|
||||
else
|
||||
be_rx_compl_process(rxo, rxcp);
|
||||
be_rx_compl_process(rxo, napi, rxcp);
|
||||
|
||||
loop_continue:
|
||||
be_rx_stats_update(rxo, rxcp);
|
||||
}
|
||||
|
|
@ -2304,7 +2312,11 @@ loop_continue:
|
|||
if (work_done) {
|
||||
be_cq_notify(adapter, rx_cq->id, true, work_done);
|
||||
|
||||
if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM)
|
||||
/* When an rx-obj gets into post_starved state, just
|
||||
* let be_worker do the posting.
|
||||
*/
|
||||
if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM &&
|
||||
!rxo->rx_post_starved)
|
||||
be_post_rx_frags(rxo, GFP_ATOMIC);
|
||||
}
|
||||
|
||||
|
|
@ -2349,6 +2361,7 @@ int be_poll(struct napi_struct *napi, int budget)
|
|||
struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
|
||||
struct be_adapter *adapter = eqo->adapter;
|
||||
int max_work = 0, work, i, num_evts;
|
||||
struct be_rx_obj *rxo;
|
||||
bool tx_done;
|
||||
|
||||
num_evts = events_get(eqo);
|
||||
|
|
@ -2361,13 +2374,18 @@ int be_poll(struct napi_struct *napi, int budget)
|
|||
max_work = budget;
|
||||
}
|
||||
|
||||
/* This loop will iterate twice for EQ0 in which
|
||||
* completions of the last RXQ (default one) are also processed
|
||||
* For other EQs the loop iterates only once
|
||||
*/
|
||||
for (i = eqo->idx; i < adapter->num_rx_qs; i += adapter->num_evt_qs) {
|
||||
work = be_process_rx(&adapter->rx_obj[i], napi, budget);
|
||||
max_work = max(work, max_work);
|
||||
if (be_lock_napi(eqo)) {
|
||||
/* This loop will iterate twice for EQ0 in which
|
||||
* completions of the last RXQ (default one) are also processed
|
||||
* For other EQs the loop iterates only once
|
||||
*/
|
||||
for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
|
||||
work = be_process_rx(rxo, napi, budget, NAPI_POLLING);
|
||||
max_work = max(work, max_work);
|
||||
}
|
||||
be_unlock_napi(eqo);
|
||||
} else {
|
||||
max_work = budget;
|
||||
}
|
||||
|
||||
if (is_mcc_eqo(eqo))
|
||||
|
|
@ -2383,6 +2401,28 @@ int be_poll(struct napi_struct *napi, int budget)
|
|||
return max_work;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_NET_RX_BUSY_POLL
|
||||
static int be_busy_poll(struct napi_struct *napi)
|
||||
{
|
||||
struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
|
||||
struct be_adapter *adapter = eqo->adapter;
|
||||
struct be_rx_obj *rxo;
|
||||
int i, work = 0;
|
||||
|
||||
if (!be_lock_busy_poll(eqo))
|
||||
return LL_FLUSH_BUSY;
|
||||
|
||||
for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
|
||||
work = be_process_rx(rxo, napi, 4, BUSY_POLLING);
|
||||
if (work)
|
||||
break;
|
||||
}
|
||||
|
||||
be_unlock_busy_poll(eqo);
|
||||
return work;
|
||||
}
|
||||
#endif
|
||||
|
||||
void be_detect_error(struct be_adapter *adapter)
|
||||
{
|
||||
u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0;
|
||||
|
|
@ -2614,9 +2654,11 @@ static int be_close(struct net_device *netdev)
|
|||
|
||||
be_roce_dev_close(adapter);
|
||||
|
||||
if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
|
||||
for_all_evt_queues(adapter, eqo, i)
|
||||
for_all_evt_queues(adapter, eqo, i) {
|
||||
if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
|
||||
napi_disable(&eqo->napi);
|
||||
be_disable_busy_poll(eqo);
|
||||
}
|
||||
adapter->flags &= ~BE_FLAGS_NAPI_ENABLED;
|
||||
}
|
||||
|
||||
|
|
@ -2727,6 +2769,7 @@ static int be_open(struct net_device *netdev)
|
|||
|
||||
for_all_evt_queues(adapter, eqo, i) {
|
||||
napi_enable(&eqo->napi);
|
||||
be_enable_busy_poll(eqo);
|
||||
be_eq_notify(adapter, eqo->q.id, true, false, 0);
|
||||
}
|
||||
adapter->flags |= BE_FLAGS_NAPI_ENABLED;
|
||||
|
|
@ -3989,6 +4032,9 @@ static const struct net_device_ops be_netdev_ops = {
|
|||
#endif
|
||||
.ndo_bridge_setlink = be_ndo_bridge_setlink,
|
||||
.ndo_bridge_getlink = be_ndo_bridge_getlink,
|
||||
#ifdef CONFIG_NET_RX_BUSY_POLL
|
||||
.ndo_busy_poll = be_busy_poll
|
||||
#endif
|
||||
};
|
||||
|
||||
static void be_netdev_init(struct net_device *netdev)
|
||||
|
|
@ -4376,10 +4422,11 @@ static void be_worker(struct work_struct *work)
|
|||
be_cmd_get_die_temperature(adapter);
|
||||
|
||||
for_all_rx_queues(adapter, rxo, i) {
|
||||
if (rxo->rx_post_starved) {
|
||||
rxo->rx_post_starved = false;
|
||||
/* Replenish RX-queues starved due to memory
|
||||
* allocation failures.
|
||||
*/
|
||||
if (rxo->rx_post_starved)
|
||||
be_post_rx_frags(rxo, GFP_KERNEL);
|
||||
}
|
||||
}
|
||||
|
||||
be_eqd_update(adapter);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue