mirrors
/
linux-stable
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
1
0
Fork 0
Code Issues Releases Wiki Activity

gve: Reduce alloc and copy costs in the GQ rx path 

Previously, even if just one of the many fragments of a 9k packet
required a copy, we'd copy the whole packet into a freshly-allocated
9k-sized linear SKB, and this led to performance issues.

By having a pool of pages to copy into, each fragment can be
independently handled, leading to a reduced incidence of
allocation and copy.

Signed-off-by: Shailend Chand <shailend@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Shailend Chand 2022-10-29 09:53:22 -07:00 committed by David S. Miller
parent d08b0f8f46
commit 82fd151d38
6 changed files with 333 additions and 261 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
drivers/net/ethernet/google/gve/gve.h
View File

@ -60,7 +60,8 @@ struct gve_rx_slot_page_info {
void *page_address;
u32 page_offset; /* offset to write to in page */
int pagecnt_bias; /* expected pagecnt if only the driver has a ref */
u8 can_flip;
u16 pad; /* adjustment for rx padding */
u8 can_flip; /* tracks if the networking stack is using the page */
};
/* A list of pages registered with the device during setup and used by a queue
@ -149,10 +150,17 @@ struct gve_rx_ctx {
/* head and tail of skb chain for the current packet or NULL if none */
struct sk_buff *skb_head;
struct sk_buff *skb_tail;
u16 total_expected_size;
u8 expected_frag_cnt;
u8 curr_frag_cnt;
u8 reuse_frags;
u32 total_size;
u8 frag_cnt;
bool drop_pkt;
};
struct gve_rx_cnts {
u32 ok_pkt_bytes;
u16 ok_pkt_cnt;
u16 total_pkt_cnt;
u16 cont_pkt_cnt;
u16 desc_err_pkt_cnt;
};
/* Contains datapath state used to represent an RX queue. */
@ -167,6 +175,10 @@ struct gve_rx_ring {
/* threshold for posting new buffs and descs */
u32 db_threshold;
u16 packet_buffer_size;
u32 qpl_copy_pool_mask;
u32 qpl_copy_pool_head;
struct gve_rx_slot_page_info *qpl_copy_pool;
};
/* DQO fields. */
@ -216,7 +228,9 @@ struct gve_rx_ring {
u64 rx_desc_err_dropped_pkt; /* free-running count of packets dropped by descriptor error */
u64 rx_cont_packet_cnt; /* free-running multi-fragment packets received */
u64 rx_frag_flip_cnt; /* free-running count of rx segments where page_flip was used */
u64 rx_frag_copy_cnt; /* free-running count of rx segments copied into skb linear portion */
u64 rx_frag_copy_cnt; /* free-running count of rx segments copied */
u64 rx_frag_alloc_cnt; /* free-running count of rx page allocations */
u32 q_num; /* queue index */
u32 ntfy_id; /* notification block index */
struct gve_queue_resources *q_resources; /* head and tail pointer idx */

2
drivers/net/ethernet/google/gve/gve_ethtool.c
View File

@ -45,6 +45,7 @@ static const char gve_gstrings_main_stats[][ETH_GSTRING_LEN] = {
static const char gve_gstrings_rx_stats[][ETH_GSTRING_LEN] = {
"rx_posted_desc[%u]", "rx_completed_desc[%u]", "rx_consumed_desc[%u]", "rx_bytes[%u]",
"rx_cont_packet_cnt[%u]", "rx_frag_flip_cnt[%u]", "rx_frag_copy_cnt[%u]",
"rx_frag_alloc_cnt[%u]",
"rx_dropped_pkt[%u]", "rx_copybreak_pkt[%u]", "rx_copied_pkt[%u]",
"rx_queue_drop_cnt[%u]", "rx_no_buffers_posted[%u]",
"rx_drops_packet_over_mru[%u]", "rx_drops_invalid_checksum[%u]",
@ -271,6 +272,7 @@ gve_get_ethtool_stats(struct net_device *netdev,
data[i++] = rx->rx_cont_packet_cnt;
data[i++] = rx->rx_frag_flip_cnt;
data[i++] = rx->rx_frag_copy_cnt;
data[i++] = rx->rx_frag_alloc_cnt;
/* rx dropped packets */
data[i++] = tmp_rx_skb_alloc_fail +
tmp_rx_buf_alloc_fail +

534
drivers/net/ethernet/google/gve/gve_rx.c
View File

@ -35,6 +35,12 @@ static void gve_rx_unfill_pages(struct gve_priv *priv, struct gve_rx_ring *rx)
rx->data.page_info[i].pagecnt_bias - 1);
gve_unassign_qpl(priv, rx->data.qpl->id);
rx->data.qpl = NULL;
for (i = 0; i < rx->qpl_copy_pool_mask + 1; i++) {
page_ref_sub(rx->qpl_copy_pool[i].page,
rx->qpl_copy_pool[i].pagecnt_bias - 1);
put_page(rx->qpl_copy_pool[i].page);
}
}
kvfree(rx->data.page_info);
rx->data.page_info = NULL;
@ -63,6 +69,10 @@ static void gve_rx_free_ring(struct gve_priv *priv, int idx)
dma_free_coherent(dev, bytes, rx->data.data_ring,
rx->data.data_bus);
rx->data.data_ring = NULL;
kvfree(rx->qpl_copy_pool);
rx->qpl_copy_pool = NULL;
netif_dbg(priv, drv, priv->dev, "freed rx ring %d\n", idx);
}
@ -101,6 +111,7 @@ static int gve_prefill_rx_pages(struct gve_rx_ring *rx)
u32 slots;
int err;
int i;
int j;
/* Allocate one page per Rx queue slot. Each page is split into two
* packet buffers, when possible we "page flip" between the two.
@ -135,7 +146,31 @@ static int gve_prefill_rx_pages(struct gve_rx_ring *rx)
goto alloc_err;
}
if (!rx->data.raw_addressing) {
for (j = 0; j < rx->qpl_copy_pool_mask + 1; j++) {
struct page *page = alloc_page(GFP_KERNEL);
if (!page)
goto alloc_err_qpl;
rx->qpl_copy_pool[j].page = page;
rx->qpl_copy_pool[j].page_offset = 0;
rx->qpl_copy_pool[j].page_address = page_address(page);
/* The page already has 1 ref. */
page_ref_add(page, INT_MAX - 1);
rx->qpl_copy_pool[j].pagecnt_bias = INT_MAX;
}
}
return slots;
alloc_err_qpl:
while (j--) {
page_ref_sub(rx->qpl_copy_pool[j].page,
rx->qpl_copy_pool[j].pagecnt_bias - 1);
put_page(rx->qpl_copy_pool[j].page);
}
alloc_err:
while (i--)
gve_rx_free_buffer(&priv->pdev->dev,
@ -146,12 +181,11 @@ alloc_err:
static void gve_rx_ctx_clear(struct gve_rx_ctx *ctx)
{
ctx->curr_frag_cnt = 0;
ctx->total_expected_size = 0;
ctx->expected_frag_cnt = 0;
ctx->skb_head = NULL;
ctx->skb_tail = NULL;
ctx->reuse_frags = false;
ctx->total_size = 0;
ctx->frag_cnt = 0;
ctx->drop_pkt = false;
}
static int gve_rx_alloc_ring(struct gve_priv *priv, int idx)
@ -181,10 +215,22 @@ static int gve_rx_alloc_ring(struct gve_priv *priv, int idx)
GFP_KERNEL);
if (!rx->data.data_ring)
return -ENOMEM;
rx->qpl_copy_pool_mask = min_t(u32, U32_MAX, slots * 2) - 1;
rx->qpl_copy_pool_head = 0;
rx->qpl_copy_pool = kvcalloc(rx->qpl_copy_pool_mask + 1,
sizeof(rx->qpl_copy_pool[0]),
GFP_KERNEL);
if (!rx->qpl_copy_pool) {
err = -ENOMEM;
goto abort_with_slots;
}
filled_pages = gve_prefill_rx_pages(rx);
if (filled_pages < 0) {
err = -ENOMEM;
goto abort_with_slots;
goto abort_with_copy_pool;
}
rx->fill_cnt = filled_pages;
/* Ensure data ring slots (packet buffers) are visible. */
@ -236,6 +282,9 @@ abort_with_q_resources:
rx->q_resources = NULL;
abort_filled:
gve_rx_unfill_pages(priv, rx);
abort_with_copy_pool:
kvfree(rx->qpl_copy_pool);
rx->qpl_copy_pool = NULL;
abort_with_slots:
bytes = sizeof(*rx->data.data_ring) * slots;
dma_free_coherent(hdev, bytes, rx->data.data_ring, rx->data.data_bus);
@ -292,30 +341,47 @@ static enum pkt_hash_types gve_rss_type(__be16 pkt_flags)
return PKT_HASH_TYPE_L2;
}
static u16 gve_rx_ctx_padding(struct gve_rx_ctx *ctx)
{
return (ctx->curr_frag_cnt == 0) ? GVE_RX_PAD : 0;
}
static struct sk_buff *gve_rx_add_frags(struct napi_struct *napi,
struct gve_rx_slot_page_info *page_info,
u16 packet_buffer_size, u16 len,
struct gve_rx_ctx *ctx)
{
u32 offset = page_info->page_offset + gve_rx_ctx_padding(ctx);
struct sk_buff *skb;
u32 offset = page_info->page_offset + page_info->pad;
struct sk_buff *skb = ctx->skb_tail;
int num_frags = 0;
if (!ctx->skb_head)
ctx->skb_head = napi_get_frags(napi);
if (!skb) {
skb = napi_get_frags(napi);
if (unlikely(!skb))
return NULL;
if (unlikely(!ctx->skb_head))
return NULL;
ctx->skb_head = skb;
ctx->skb_tail = skb;
} else {
num_frags = skb_shinfo(ctx->skb_tail)->nr_frags;
if (num_frags == MAX_SKB_FRAGS) {
skb = napi_alloc_skb(napi, 0);
if (!skb)
return NULL;
skb = ctx->skb_head;
skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page_info->page,
// We will never chain more than two SKBs: 2 * 16 * 2k > 64k
// which is why we do not need to chain by using skb->next
skb_shinfo(ctx->skb_tail)->frag_list = skb;
ctx->skb_tail = skb;
num_frags = 0;
}
}
if (skb != ctx->skb_head) {
ctx->skb_head->len += len;
ctx->skb_head->data_len += len;
ctx->skb_head->truesize += packet_buffer_size;
}
skb_add_rx_frag(skb, num_frags, page_info->page,
offset, len, packet_buffer_size);
return skb;
return ctx->skb_head;
}
static void gve_rx_flip_buff(struct gve_rx_slot_page_info *page_info, __be64 *slot_addr)
@ -363,6 +429,92 @@ gve_rx_raw_addressing(struct device *dev, struct net_device *netdev,
return skb;
}
static struct sk_buff *gve_rx_copy_to_pool(struct gve_rx_ring *rx,
struct gve_rx_slot_page_info *page_info,
u16 len, struct napi_struct *napi)
{
u32 pool_idx = rx->qpl_copy_pool_head & rx->qpl_copy_pool_mask;
void *src = page_info->page_address + page_info->page_offset;
struct gve_rx_slot_page_info *copy_page_info;
struct gve_rx_ctx *ctx = &rx->ctx;
bool alloc_page = false;
struct sk_buff *skb;
void *dst;
copy_page_info = &rx->qpl_copy_pool[pool_idx];
if (!copy_page_info->can_flip) {
int recycle = gve_rx_can_recycle_buffer(copy_page_info);
if (unlikely(recycle < 0)) {
gve_schedule_reset(rx->gve);
return NULL;
}
alloc_page = !recycle;
}
if (alloc_page) {
struct gve_rx_slot_page_info alloc_page_info;
struct page *page;
/* The least recently used page turned out to be
* still in use by the kernel. Ignoring it and moving
* on alleviates head-of-line blocking.
*/
rx->qpl_copy_pool_head++;
page = alloc_page(GFP_ATOMIC);
if (!page)
return NULL;
alloc_page_info.page = page;
alloc_page_info.page_offset = 0;
alloc_page_info.page_address = page_address(page);
alloc_page_info.pad = page_info->pad;
memcpy(alloc_page_info.page_address, src, page_info->pad + len);
skb = gve_rx_add_frags(napi, &alloc_page_info,
rx->packet_buffer_size,
len, ctx);
u64_stats_update_begin(&rx->statss);
rx->rx_frag_copy_cnt++;
rx->rx_frag_alloc_cnt++;
u64_stats_update_end(&rx->statss);
return skb;
}
dst = copy_page_info->page_address + copy_page_info->page_offset;
memcpy(dst, src, page_info->pad + len);
copy_page_info->pad = page_info->pad;
skb = gve_rx_add_frags(napi, copy_page_info,
rx->packet_buffer_size, len, ctx);
if (unlikely(!skb))
return NULL;
gve_dec_pagecnt_bias(copy_page_info);
copy_page_info->page_offset += rx->packet_buffer_size;
copy_page_info->page_offset &= (PAGE_SIZE - 1);
if (copy_page_info->can_flip) {
/* We have used both halves of this copy page, it
* is time for it to go to the back of the queue.
*/
copy_page_info->can_flip = false;
rx->qpl_copy_pool_head++;
prefetch(rx->qpl_copy_pool[rx->qpl_copy_pool_head & rx->qpl_copy_pool_mask].page);
} else {
copy_page_info->can_flip = true;
}
u64_stats_update_begin(&rx->statss);
rx->rx_frag_copy_cnt++;
u64_stats_update_end(&rx->statss);
return skb;
}
static struct sk_buff *
gve_rx_qpl(struct device *dev, struct net_device *netdev,
struct gve_rx_ring *rx, struct gve_rx_slot_page_info *page_info,
@ -377,7 +529,7 @@ gve_rx_qpl(struct device *dev, struct net_device *netdev,
* choice is to copy the data out of it so that we can return it to the
* device.
*/
if (ctx->reuse_frags) {
if (page_info->can_flip) {
skb = gve_rx_add_frags(napi, page_info, rx->packet_buffer_size, len, ctx);
/* No point in recycling if we didn't get the skb */
if (skb) {
@ -386,116 +538,23 @@ gve_rx_qpl(struct device *dev, struct net_device *netdev,
gve_rx_flip_buff(page_info, &data_slot->qpl_offset);
}
} else {
const u16 padding = gve_rx_ctx_padding(ctx);
skb = gve_rx_copy(netdev, napi, page_info, len, padding, ctx);
if (skb) {
u64_stats_update_begin(&rx->statss);
rx->rx_frag_copy_cnt++;
u64_stats_update_end(&rx->statss);
}
skb = gve_rx_copy_to_pool(rx, page_info, len, napi);
}
return skb;
}
#define GVE_PKTCONT_BIT_IS_SET(x) (GVE_RXF_PKT_CONT & (x))
static u16 gve_rx_get_fragment_size(struct gve_rx_ctx *ctx, struct gve_rx_desc *desc)
{
return be16_to_cpu(desc->len) - gve_rx_ctx_padding(ctx);
}
static bool gve_rx_ctx_init(struct gve_rx_ctx *ctx, struct gve_rx_ring *rx)
{
bool qpl_mode = !rx->data.raw_addressing, packet_size_error = false;
bool buffer_error = false, desc_error = false, seqno_error = false;
struct gve_rx_slot_page_info *page_info;
struct gve_priv *priv = rx->gve;
u32 idx = rx->cnt & rx->mask;
bool reuse_frags, can_flip;
struct gve_rx_desc *desc;
u16 packet_size = 0;
u16 n_frags = 0;
int recycle;
/** In QPL mode, we only flip buffers when all buffers containing the packet
* can be flipped. RDA can_flip decisions will be made later, per frag.
*/
can_flip = qpl_mode;
reuse_frags = can_flip;
do {
u16 frag_size;
n_frags++;
desc = &rx->desc.desc_ring[idx];
desc_error = unlikely(desc->flags_seq & GVE_RXF_ERR) || desc_error;
if (GVE_SEQNO(desc->flags_seq) != rx->desc.seqno) {
seqno_error = true;
netdev_warn(priv->dev,
"RX seqno error: want=%d, got=%d, dropping packet and scheduling reset.",
rx->desc.seqno, GVE_SEQNO(desc->flags_seq));
}
frag_size = be16_to_cpu(desc->len);
packet_size += frag_size;
if (frag_size > rx->packet_buffer_size) {
packet_size_error = true;
netdev_warn(priv->dev,
"RX fragment error: packet_buffer_size=%d, frag_size=%d, dropping packet.",
rx->packet_buffer_size, be16_to_cpu(desc->len));
}
page_info = &rx->data.page_info[idx];
if (can_flip) {
recycle = gve_rx_can_recycle_buffer(page_info);
reuse_frags = reuse_frags && recycle > 0;
buffer_error = buffer_error || unlikely(recycle < 0);
}
idx = (idx + 1) & rx->mask;
rx->desc.seqno = gve_next_seqno(rx->desc.seqno);
} while (GVE_PKTCONT_BIT_IS_SET(desc->flags_seq));
prefetch(rx->desc.desc_ring + idx);
ctx->curr_frag_cnt = 0;
ctx->total_expected_size = packet_size - GVE_RX_PAD;
ctx->expected_frag_cnt = n_frags;
ctx->skb_head = NULL;
ctx->reuse_frags = reuse_frags;
if (ctx->expected_frag_cnt > 1) {
u64_stats_update_begin(&rx->statss);
rx->rx_cont_packet_cnt++;
u64_stats_update_end(&rx->statss);
}
if (ctx->total_expected_size > priv->rx_copybreak && !ctx->reuse_frags && qpl_mode) {
u64_stats_update_begin(&rx->statss);
rx->rx_copied_pkt++;
u64_stats_update_end(&rx->statss);
}
if (unlikely(buffer_error || seqno_error || packet_size_error)) {
gve_schedule_reset(priv);
return false;
}
if (unlikely(desc_error)) {
u64_stats_update_begin(&rx->statss);
rx->rx_desc_err_dropped_pkt++;
u64_stats_update_end(&rx->statss);
return false;
}
return true;
}
static struct sk_buff *gve_rx_skb(struct gve_priv *priv, struct gve_rx_ring *rx,
struct gve_rx_slot_page_info *page_info, struct napi_struct *napi,
u16 len, union gve_rx_data_slot *data_slot)
u16 len, union gve_rx_data_slot *data_slot,
bool is_only_frag)
{
struct net_device *netdev = priv->dev;
struct gve_rx_ctx *ctx = &rx->ctx;
struct sk_buff *skb = NULL;
if (len <= priv->rx_copybreak && ctx->expected_frag_cnt == 1) {
if (len <= priv->rx_copybreak && is_only_frag) {
/* Just copy small packets */
skb = gve_rx_copy(netdev, napi, page_info, len, GVE_RX_PAD, ctx);
skb = gve_rx_copy(netdev, napi, page_info, len, GVE_RX_PAD);
if (skb) {
u64_stats_update_begin(&rx->statss);
rx->rx_copied_pkt++;
@ -504,29 +563,25 @@ static struct sk_buff *gve_rx_skb(struct gve_priv *priv, struct gve_rx_ring *rx,
u64_stats_update_end(&rx->statss);
}
} else {
if (rx->data.raw_addressing) {
int recycle = gve_rx_can_recycle_buffer(page_info);
int recycle = gve_rx_can_recycle_buffer(page_info);
if (unlikely(recycle < 0)) {
gve_schedule_reset(priv);
return NULL;
}
page_info->can_flip = recycle;
if (page_info->can_flip) {
u64_stats_update_begin(&rx->statss);
rx->rx_frag_flip_cnt++;
u64_stats_update_end(&rx->statss);
}
if (unlikely(recycle < 0)) {
gve_schedule_reset(priv);
return NULL;
}
page_info->can_flip = recycle;
if (page_info->can_flip) {
u64_stats_update_begin(&rx->statss);
rx->rx_frag_flip_cnt++;
u64_stats_update_end(&rx->statss);
}
if (rx->data.raw_addressing) {
skb = gve_rx_raw_addressing(&priv->pdev->dev, netdev,
page_info, len, napi,
data_slot,
rx->packet_buffer_size, ctx);
} else {
if (ctx->reuse_frags) {
u64_stats_update_begin(&rx->statss);
rx->rx_frag_flip_cnt++;
u64_stats_update_end(&rx->statss);
}
skb = gve_rx_qpl(&priv->pdev->dev, netdev, rx,
page_info, len, napi, data_slot);
}
@ -534,101 +589,113 @@ static struct sk_buff *gve_rx_skb(struct gve_priv *priv, struct gve_rx_ring *rx,
return skb;
}
static bool gve_rx(struct gve_rx_ring *rx, netdev_features_t feat,
u64 *packet_size_bytes, u32 *work_done)
#define GVE_PKTCONT_BIT_IS_SET(x) (GVE_RXF_PKT_CONT & (x))
static void gve_rx(struct gve_rx_ring *rx, netdev_features_t feat,
struct gve_rx_desc *desc, u32 idx,
struct gve_rx_cnts *cnts)
{
bool is_last_frag = !GVE_PKTCONT_BIT_IS_SET(desc->flags_seq);
struct gve_rx_slot_page_info *page_info;
u16 frag_size = be16_to_cpu(desc->len);
struct gve_rx_ctx *ctx = &rx->ctx;
union gve_rx_data_slot *data_slot;
struct gve_priv *priv = rx->gve;
struct gve_rx_desc *first_desc;
struct sk_buff *skb = NULL;
struct gve_rx_desc *desc;
struct napi_struct *napi;
dma_addr_t page_bus;
u32 work_cnt = 0;
void *va;
u32 idx;
u16 len;
idx = rx->cnt & rx->mask;
first_desc = &rx->desc.desc_ring[idx];
desc = first_desc;
napi = &priv->ntfy_blocks[rx->ntfy_id].napi;
struct napi_struct *napi = &priv->ntfy_blocks[rx->ntfy_id].napi;
bool is_first_frag = ctx->frag_cnt == 0;
if (unlikely(!gve_rx_ctx_init(ctx, rx)))
goto skb_alloc_fail;
bool is_only_frag = is_first_frag && is_last_frag;
while (ctx->curr_frag_cnt < ctx->expected_frag_cnt) {
/* Prefetch two packet buffers ahead, we will need it soon. */
page_info = &rx->data.page_info[(idx + 2) & rx->mask];
va = page_info->page_address + page_info->page_offset;
if (unlikely(ctx->drop_pkt))
goto finish_frag;
prefetch(page_info->page); /* Kernel page struct. */
prefetch(va); /* Packet header. */
prefetch(va + 64); /* Next cacheline too. */
if (desc->flags_seq & GVE_RXF_ERR) {
ctx->drop_pkt = true;
cnts->desc_err_pkt_cnt++;
napi_free_frags(napi);
goto finish_frag;
}
len = gve_rx_get_fragment_size(ctx, desc);
if (unlikely(frag_size > rx->packet_buffer_size)) {
netdev_warn(priv->dev, "Unexpected frag size %d, can't exceed %d, scheduling reset",
frag_size, rx->packet_buffer_size);
ctx->drop_pkt = true;
napi_free_frags(napi);
gve_schedule_reset(rx->gve);
goto finish_frag;
}
page_info = &rx->data.page_info[idx];
data_slot = &rx->data.data_ring[idx];
page_bus = rx->data.raw_addressing ?
be64_to_cpu(data_slot->addr) - page_info->page_offset :
rx->data.qpl->page_buses[idx];
dma_sync_single_for_cpu(&priv->pdev->dev, page_bus, PAGE_SIZE, DMA_FROM_DEVICE);
/* Prefetch two packet buffers ahead, we will need it soon. */
page_info = &rx->data.page_info[(idx + 2) & rx->mask];
va = page_info->page_address + page_info->page_offset;
prefetch(page_info->page); /* Kernel page struct. */
prefetch(va); /* Packet header. */
prefetch(va + 64); /* Next cacheline too. */
skb = gve_rx_skb(priv, rx, page_info, napi, len, data_slot);
if (!skb) {
u64_stats_update_begin(&rx->statss);
rx->rx_skb_alloc_fail++;
u64_stats_update_end(&rx->statss);
goto skb_alloc_fail;
page_info = &rx->data.page_info[idx];
data_slot = &rx->data.data_ring[idx];
page_bus = (rx->data.raw_addressing) ?
be64_to_cpu(data_slot->addr) - page_info->page_offset :
rx->data.qpl->page_buses[idx];
dma_sync_single_for_cpu(&priv->pdev->dev, page_bus,
PAGE_SIZE, DMA_FROM_DEVICE);
page_info->pad = is_first_frag ? GVE_RX_PAD : 0;
frag_size -= page_info->pad;
skb = gve_rx_skb(priv, rx, page_info, napi, frag_size,
data_slot, is_only_frag);
if (!skb) {
u64_stats_update_begin(&rx->statss);
rx->rx_skb_alloc_fail++;
u64_stats_update_end(&rx->statss);
napi_free_frags(napi);
ctx->drop_pkt = true;
goto finish_frag;
}
ctx->total_size += frag_size;
if (is_first_frag) {
if (likely(feat & NETIF_F_RXCSUM)) {
/* NIC passes up the partial sum */
if (desc->csum)
skb->ip_summed = CHECKSUM_COMPLETE;
else
skb->ip_summed = CHECKSUM_NONE;
skb->csum = csum_unfold(desc->csum);
}
ctx->curr_frag_cnt++;
rx->cnt++;
idx = rx->cnt & rx->mask;
work_cnt++;
desc = &rx->desc.desc_ring[idx];
/* parse flags & pass relevant info up */
if (likely(feat & NETIF_F_RXHASH) &&
gve_needs_rss(desc->flags_seq))
skb_set_hash(skb, be32_to_cpu(desc->rss_hash),
gve_rss_type(desc->flags_seq));
}
if (likely(feat & NETIF_F_RXCSUM)) {
/* NIC passes up the partial sum */
if (first_desc->csum)
skb->ip_summed = CHECKSUM_COMPLETE;
if (is_last_frag) {
skb_record_rx_queue(skb, rx->q_num);
if (skb_is_nonlinear(skb))
napi_gro_frags(napi);
else
skb->ip_summed = CHECKSUM_NONE;
skb->csum = csum_unfold(first_desc->csum);
napi_gro_receive(napi, skb);
goto finish_ok_pkt;
}
/* parse flags & pass relevant info up */
if (likely(feat & NETIF_F_RXHASH) &&
gve_needs_rss(first_desc->flags_seq))
skb_set_hash(skb, be32_to_cpu(first_desc->rss_hash),
gve_rss_type(first_desc->flags_seq));
goto finish_frag;
*packet_size_bytes = skb->len + (skb->protocol ? ETH_HLEN : 0);
*work_done = work_cnt;
skb_record_rx_queue(skb, rx->q_num);
if (skb_is_nonlinear(skb))
napi_gro_frags(napi);
else
napi_gro_receive(napi, skb);
gve_rx_ctx_clear(ctx);
return true;
skb_alloc_fail:
if (napi->skb)
napi_free_frags(napi);
*packet_size_bytes = 0;
*work_done = ctx->expected_frag_cnt;
while (ctx->curr_frag_cnt < ctx->expected_frag_cnt) {
rx->cnt++;
ctx->curr_frag_cnt++;
finish_ok_pkt:
cnts->ok_pkt_bytes += ctx->total_size;
cnts->ok_pkt_cnt++;
finish_frag:
ctx->frag_cnt++;
if (is_last_frag) {
cnts->total_pkt_cnt++;
cnts->cont_pkt_cnt += (ctx->frag_cnt > 1);
gve_rx_ctx_clear(ctx);
}
gve_rx_ctx_clear(ctx);
return false;
}
bool gve_rx_work_pending(struct gve_rx_ring *rx)
@ -704,36 +771,39 @@ static bool gve_rx_refill_buffers(struct gve_priv *priv, struct gve_rx_ring *rx)
static int gve_clean_rx_done(struct gve_rx_ring *rx, int budget,
netdev_features_t feat)
{
u32 work_done = 0, total_packet_cnt = 0, ok_packet_cnt = 0;
struct gve_rx_ctx *ctx = &rx->ctx;
struct gve_priv *priv = rx->gve;
struct gve_rx_cnts cnts = {0};
struct gve_rx_desc *next_desc;
u32 idx = rx->cnt & rx->mask;
struct gve_rx_desc *desc;
u64 bytes = 0;
u32 work_done = 0;
desc = &rx->desc.desc_ring[idx];
struct gve_rx_desc *desc = &rx->desc.desc_ring[idx];
// Exceed budget only if (and till) the inflight packet is consumed.
while ((GVE_SEQNO(desc->flags_seq) == rx->desc.seqno) &&
work_done < budget) {
u64 packet_size_bytes = 0;
u32 work_cnt = 0;
bool dropped;
(work_done < budget || ctx->frag_cnt)) {
next_desc = &rx->desc.desc_ring[(idx + 1) & rx->mask];
prefetch(next_desc);
netif_info(priv, rx_status, priv->dev,
"[%d] idx=%d desc=%p desc->flags_seq=0x%x\n",
rx->q_num, idx, desc, desc->flags_seq);
netif_info(priv, rx_status, priv->dev,
"[%d] seqno=%d rx->desc.seqno=%d\n",
rx->q_num, GVE_SEQNO(desc->flags_seq),
rx->desc.seqno);
gve_rx(rx, feat, desc, idx, &cnts);
dropped = !gve_rx(rx, feat, &packet_size_bytes, &work_cnt);
if (!dropped) {
bytes += packet_size_bytes;
ok_packet_cnt++;
}
total_packet_cnt++;
rx->cnt++;
idx = rx->cnt & rx->mask;
desc = &rx->desc.desc_ring[idx];
work_done += work_cnt;
rx->desc.seqno = gve_next_seqno(rx->desc.seqno);
work_done++;
}
// The device will only send whole packets.
if (unlikely(ctx->frag_cnt)) {
struct napi_struct *napi = &priv->ntfy_blocks[rx->ntfy_id].napi;
napi_free_frags(napi);
gve_rx_ctx_clear(&rx->ctx);
netdev_warn(priv->dev, "Unexpected seq number %d with incomplete packet, expected %d, scheduling reset",
GVE_SEQNO(desc->flags_seq), rx->desc.seqno);
gve_schedule_reset(rx->gve);
}
if (!work_done && rx->fill_cnt - rx->cnt > rx->db_threshold)
@ -741,8 +811,10 @@ static int gve_clean_rx_done(struct gve_rx_ring *rx, int budget,
if (work_done) {
u64_stats_update_begin(&rx->statss);
rx->rpackets += ok_packet_cnt;
rx->rbytes += bytes;
rx->rpackets += cnts.ok_pkt_cnt;
rx->rbytes += cnts.ok_pkt_bytes;
rx->rx_cont_packet_cnt += cnts.cont_pkt_cnt;
rx->rx_desc_err_dropped_pkt += cnts.desc_err_pkt_cnt;
u64_stats_update_end(&rx->statss);
}
@ -767,7 +839,7 @@ static int gve_clean_rx_done(struct gve_rx_ring *rx, int budget,
}
gve_rx_write_doorbell(priv, rx);
return total_packet_cnt;
return cnts.total_pkt_cnt;
}
int gve_rx_poll(struct gve_notify_block *block, int budget)

2
drivers/net/ethernet/google/gve/gve_rx_dqo.c
View File

@ -568,7 +568,7 @@ static int gve_rx_dqo(struct napi_struct *napi, struct gve_rx_ring *rx,
if (eop && buf_len <= priv->rx_copybreak) {
rx->ctx.skb_head = gve_rx_copy(priv->dev, napi,
&buf_state->page_info, buf_len, 0, NULL);
&buf_state->page_info, buf_len, 0);
if (unlikely(!rx->ctx.skb_head))
goto error;
rx->ctx.skb_tail = rx->ctx.skb_head;

28
drivers/net/ethernet/google/gve/gve_utils.c
View File

@ -50,34 +50,18 @@ void gve_rx_add_to_block(struct gve_priv *priv, int queue_idx)
struct sk_buff *gve_rx_copy(struct net_device *dev, struct napi_struct *napi,
struct gve_rx_slot_page_info *page_info, u16 len,
u16 padding, struct gve_rx_ctx *ctx)
u16 padding)
{
void *va = page_info->page_address + padding + page_info->page_offset;
int skb_linear_offset = 0;
bool set_protocol = false;
struct sk_buff *skb;
if (ctx) {
if (!ctx->skb_head)
ctx->skb_head = napi_alloc_skb(napi, ctx->total_expected_size);
skb = napi_alloc_skb(napi, len);
if (unlikely(!skb))
return NULL;
if (unlikely(!ctx->skb_head))
return NULL;
skb = ctx->skb_head;
skb_linear_offset = skb->len;
set_protocol = ctx->curr_frag_cnt == ctx->expected_frag_cnt - 1;
} else {
skb = napi_alloc_skb(napi, len);
if (unlikely(!skb))
return NULL;
set_protocol = true;
}
__skb_put(skb, len);
skb_copy_to_linear_data_offset(skb, skb_linear_offset, va, len);
if (set_protocol)
skb->protocol = eth_type_trans(skb, dev);
skb_copy_to_linear_data_offset(skb, 0, va, len);
skb->protocol = eth_type_trans(skb, dev);
return skb;
}

2
drivers/net/ethernet/google/gve/gve_utils.h
View File

@ -19,7 +19,7 @@ void gve_rx_add_to_block(struct gve_priv *priv, int queue_idx);
struct sk_buff *gve_rx_copy(struct net_device *dev, struct napi_struct *napi,
struct gve_rx_slot_page_info *page_info, u16 len,
u16 pad, struct gve_rx_ctx *ctx);
u16 pad);
/* Decrement pagecnt_bias. Set it back to INT_MAX if it reached zero. */
void gve_dec_pagecnt_bias(struct gve_rx_slot_page_info *page_info);