mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-10-03 15:47:36 +00:00
net/tcp: optimise io_uring zc ubuf refcounting
io_uring keeps a reference to ubuf_info during submission, so if tcp_sendmsg_locked() sees msghdr::msg_ubuf in can be sure the buffer will be kept alive and doesn't need to additionally pin it. Signed-off-by: Pavel Begunkov <asml.silence@gmail.com> Reviewed-by: Eric Dumazet <edumazet@google.com> Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
parent
eea96a3e2c
commit
a753358472
1 changed files with 6 additions and 3 deletions
|
@ -1231,7 +1231,6 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
|
||||||
if ((flags & MSG_ZEROCOPY) && size) {
|
if ((flags & MSG_ZEROCOPY) && size) {
|
||||||
if (msg->msg_ubuf) {
|
if (msg->msg_ubuf) {
|
||||||
uarg = msg->msg_ubuf;
|
uarg = msg->msg_ubuf;
|
||||||
net_zcopy_get(uarg);
|
|
||||||
zc = sk->sk_route_caps & NETIF_F_SG;
|
zc = sk->sk_route_caps & NETIF_F_SG;
|
||||||
} else if (sock_flag(sk, SOCK_ZEROCOPY)) {
|
} else if (sock_flag(sk, SOCK_ZEROCOPY)) {
|
||||||
skb = tcp_write_queue_tail(sk);
|
skb = tcp_write_queue_tail(sk);
|
||||||
|
@ -1458,6 +1457,8 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
|
||||||
tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
|
tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
|
||||||
}
|
}
|
||||||
out_nopush:
|
out_nopush:
|
||||||
|
/* msg->msg_ubuf is pinned by the caller so we don't take extra refs */
|
||||||
|
if (uarg && !msg->msg_ubuf)
|
||||||
net_zcopy_put(uarg);
|
net_zcopy_put(uarg);
|
||||||
return copied + copied_syn;
|
return copied + copied_syn;
|
||||||
|
|
||||||
|
@ -1467,6 +1468,8 @@ int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
|
||||||
if (copied + copied_syn)
|
if (copied + copied_syn)
|
||||||
goto out;
|
goto out;
|
||||||
out_err:
|
out_err:
|
||||||
|
/* msg->msg_ubuf is pinned by the caller so we don't take extra refs */
|
||||||
|
if (uarg && !msg->msg_ubuf)
|
||||||
net_zcopy_put_abort(uarg, true);
|
net_zcopy_put_abort(uarg, true);
|
||||||
err = sk_stream_error(sk, flags, err);
|
err = sk_stream_error(sk, flags, err);
|
||||||
/* make sure we wake any epoll edge trigger waiter */
|
/* make sure we wake any epoll edge trigger waiter */
|
||||||
|
|
Loading…
Reference in a new issue