mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-25 11:55:37 +00:00
net: xdp: add xdp_update_skb_shared_info utility routine
Introduce xdp_update_skb_shared_info routine to update frags array metadata in skb_shared_info data structure converting to a skb from a xdp_buff or xdp_frame. According to the current skb_shared_info architecture in xdp_frame/xdp_buff and to the xdp frags support, there is no need to run skb_add_rx_frag() and reset frags array converting the buffer to a skb since the frag array will be in the same position for xdp_buff/xdp_frame and for the skb, we just need to update memory metadata. Introduce XDP_FLAGS_PF_MEMALLOC flag in xdp_buff_flags in order to mark the xdp_buff or xdp_frame as under memory-pressure if pages of the frags array are under memory pressure. Doing so we can avoid looping over all fragments in xdp_update_skb_shared_info routine. The driver is expected to set the flag constructing the xdp_buffer using xdp_buff_set_frag_pfmemalloc utility routine. Rely on xdp_update_skb_shared_info in __xdp_build_skb_from_frame routine converting the non-linear xdp_frame to a skb after performing a XDP_REDIRECT. Acked-by: Toke Hoiland-Jorgensen <toke@redhat.com> Acked-by: John Fastabend <john.fastabend@gmail.com> Acked-by: Jesper Dangaard Brouer <brouer@redhat.com> Signed-off-by: Lorenzo Bianconi <lorenzo@kernel.org> Link: https://lore.kernel.org/r/bfd23fb8a8d7438724f7819c567cdf99ffd6226f.1642758637.git.lorenzo@kernel.org Signed-off-by: Alexei Starovoitov <ast@kernel.org>
This commit is contained in:
parent
d094c9851a
commit
d65a1906b3
2 changed files with 44 additions and 1 deletions
|
@ -67,7 +67,10 @@ struct xdp_txq_info {
|
|||
};
|
||||
|
||||
enum xdp_buff_flags {
|
||||
XDP_FLAGS_HAS_FRAGS = BIT(0), /* non-linear xdp buff */
|
||||
XDP_FLAGS_HAS_FRAGS = BIT(0), /* non-linear xdp buff */
|
||||
XDP_FLAGS_FRAGS_PF_MEMALLOC = BIT(1), /* xdp paged memory is under
|
||||
* pressure
|
||||
*/
|
||||
};
|
||||
|
||||
struct xdp_buff {
|
||||
|
@ -96,6 +99,16 @@ static __always_inline void xdp_buff_clear_frags_flag(struct xdp_buff *xdp)
|
|||
xdp->flags &= ~XDP_FLAGS_HAS_FRAGS;
|
||||
}
|
||||
|
||||
static __always_inline bool xdp_buff_is_frag_pfmemalloc(struct xdp_buff *xdp)
|
||||
{
|
||||
return !!(xdp->flags & XDP_FLAGS_FRAGS_PF_MEMALLOC);
|
||||
}
|
||||
|
||||
static __always_inline void xdp_buff_set_frag_pfmemalloc(struct xdp_buff *xdp)
|
||||
{
|
||||
xdp->flags |= XDP_FLAGS_FRAGS_PF_MEMALLOC;
|
||||
}
|
||||
|
||||
static __always_inline void
|
||||
xdp_init_buff(struct xdp_buff *xdp, u32 frame_sz, struct xdp_rxq_info *rxq)
|
||||
{
|
||||
|
@ -151,6 +164,11 @@ static __always_inline bool xdp_frame_has_frags(struct xdp_frame *frame)
|
|||
return !!(frame->flags & XDP_FLAGS_HAS_FRAGS);
|
||||
}
|
||||
|
||||
static __always_inline bool xdp_frame_is_frag_pfmemalloc(struct xdp_frame *frame)
|
||||
{
|
||||
return !!(frame->flags & XDP_FLAGS_FRAGS_PF_MEMALLOC);
|
||||
}
|
||||
|
||||
#define XDP_BULK_QUEUE_SIZE 16
|
||||
struct xdp_frame_bulk {
|
||||
int count;
|
||||
|
@ -186,6 +204,19 @@ static inline void xdp_scrub_frame(struct xdp_frame *frame)
|
|||
frame->dev_rx = NULL;
|
||||
}
|
||||
|
||||
static inline void
|
||||
xdp_update_skb_shared_info(struct sk_buff *skb, u8 nr_frags,
|
||||
unsigned int size, unsigned int truesize,
|
||||
bool pfmemalloc)
|
||||
{
|
||||
skb_shinfo(skb)->nr_frags = nr_frags;
|
||||
|
||||
skb->len += size;
|
||||
skb->data_len += size;
|
||||
skb->truesize += truesize;
|
||||
skb->pfmemalloc |= pfmemalloc;
|
||||
}
|
||||
|
||||
/* Avoids inlining WARN macro in fast-path */
|
||||
void xdp_warn(const char *msg, const char *func, const int line);
|
||||
#define XDP_WARN(msg) xdp_warn(msg, __func__, __LINE__)
|
||||
|
|
|
@ -561,8 +561,14 @@ struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf,
|
|||
struct sk_buff *skb,
|
||||
struct net_device *dev)
|
||||
{
|
||||
struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf);
|
||||
unsigned int headroom, frame_size;
|
||||
void *hard_start;
|
||||
u8 nr_frags;
|
||||
|
||||
/* xdp frags frame */
|
||||
if (unlikely(xdp_frame_has_frags(xdpf)))
|
||||
nr_frags = sinfo->nr_frags;
|
||||
|
||||
/* Part of headroom was reserved to xdpf */
|
||||
headroom = sizeof(*xdpf) + xdpf->headroom;
|
||||
|
@ -582,6 +588,12 @@ struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf,
|
|||
if (xdpf->metasize)
|
||||
skb_metadata_set(skb, xdpf->metasize);
|
||||
|
||||
if (unlikely(xdp_frame_has_frags(xdpf)))
|
||||
xdp_update_skb_shared_info(skb, nr_frags,
|
||||
sinfo->xdp_frags_size,
|
||||
nr_frags * xdpf->frame_sz,
|
||||
xdp_frame_is_frag_pfmemalloc(xdpf));
|
||||
|
||||
/* Essential SKB info: protocol and skb->dev */
|
||||
skb->protocol = eth_type_trans(skb, dev);
|
||||
|
||||
|
|
Loading…
Reference in a new issue