diff options
Diffstat (limited to 'net/core')
| -rw-r--r-- | net/core/dev.c | 47 | ||||
| -rw-r--r-- | net/core/filter.c | 5 | ||||
| -rw-r--r-- | net/core/gro.c | 31 | ||||
| -rw-r--r-- | net/core/net_namespace.c | 17 | ||||
| -rw-r--r-- | net/core/skbuff.c | 46 | ||||
| -rw-r--r-- | net/core/sock.c | 4 | ||||
| -rw-r--r-- | net/core/sock_reuseport.c | 9 |
7 files changed, 140 insertions, 19 deletions
diff --git a/net/core/dev.c b/net/core/dev.c index df1a840b0334..3c58db732761 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4207,6 +4207,18 @@ static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device if (unlikely(!skb)) goto out_null; + /* Only fraglist GSO, whose segments already carry valid checksums, + * can be handed to a device as CHECKSUM_UNNECESSARY. Any other GSO + * skb has no csum_start for the device to segment from. NONE means + * the same on transmit, but makes netif_needs_gso() send it to the + * GSO code, which turns TCP into CHECKSUM_PARTIAL for TSO and + * segments anything the device cannot offload in software. + */ + if (unlikely(skb->ip_summed == CHECKSUM_UNNECESSARY && + skb_is_gso(skb) && + !(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST))) + skb->ip_summed = CHECKSUM_NONE; + features = netif_skb_features(skb); skb = validate_xmit_vlan(skb, features); if (unlikely(!skb)) @@ -11101,6 +11113,14 @@ atomic_t dev_unreg_count = ATOMIC_INIT(0); static void net_set_todo(struct net_device *dev) { list_add_tail(&dev->todo_list, &net_todo_list); + atomic_inc(&dev_net(dev)->dev_unreg_pending); +} + +/* Paired with net_set_todo(). @net must not be used after this call. */ +static void net_unreg_done(struct net *net) +{ + if (atomic_dec_and_test(&net->dev_unreg_pending)) + wake_up(&netdev_unregistering_wq); } static netdev_features_t netdev_sync_upper_features(struct net_device *lower, @@ -12000,6 +12020,7 @@ void netdev_run_todo(void) if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) { netdev_WARN(dev, "run_todo but not unregistering\n"); list_del(&dev->todo_list); + net_unreg_done(dev_net(dev)); continue; } @@ -12016,8 +12037,11 @@ void netdev_run_todo(void) cnt = 0; while (!list_empty(&list)) { + struct net *net; + dev = netdev_wait_allrefs_any(&list); list_del(&dev->todo_list); + net = dev_net(dev); /* paranoia */ BUG_ON(netdev_refcnt_read(dev) != 1); @@ -12038,6 +12062,8 @@ void netdev_run_todo(void) /* Free network device */ kobject_put(&dev->dev.kobj); + + net_unreg_done(net); } if (cnt && atomic_sub_and_test(cnt, &dev_unreg_count)) wake_up(&netdev_unregistering_wq); @@ -13486,6 +13512,17 @@ static void __net_exit default_device_exit_net(struct net *net) } } +static bool __net_exit netns_dev_unreg_pending(const struct list_head *net_list) +{ + struct net *net; + + list_for_each_entry(net, net_list, exit_list) { + if (atomic_read(&net->dev_unreg_pending)) + return true; + } + return false; +} + static void __net_exit default_device_exit_batch(struct list_head *net_list) { /* At exit all network devices most be removed from a network @@ -13527,6 +13564,16 @@ static void __net_exit default_device_exit_batch(struct list_head *net_list) unregister_netdevice_many(&dev_kill_list); rtnl_unlock(); + /* Devices of these netns unregistered from another context + * (e.g. a veth whose peer was deleted) can still be waiting + * in netdev_run_todo() for their last references, while + * NETDEV_UNREGISTER is rebroadcast for them. Wait for them + * before pernet exit handlers tear down the state used by + * netdevice notifiers, and before the netns is freed. + * No new unregistration can start: all devices are unlisted. + */ + wait_event(netdev_unregistering_wq, !netns_dev_unreg_pending(net_list)); + rtnl_net_flush_workqueue(); } diff --git a/net/core/filter.c b/net/core/filter.c index ca204d3e9ddc..946b07a17fce 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -5652,11 +5652,12 @@ static int bpf_sol_tcp_setsockopt(struct sock *sk, int optname, case TCP_BPF_IW: if (val <= 0 || tp->data_segs_out > tp->syn_data) return -EINVAL; - tcp_snd_cwnd_set(tp, val); + tcp_snd_cwnd_set(tp, min_t(u32, val, tp->snd_cwnd_clamp)); break; case TCP_BPF_SNDCWND_CLAMP: if (val <= 0) return -EINVAL; + val = min_t(u32, val, TCP_MAX_SND_CWND); tp->snd_cwnd_clamp = val; WRITE_ONCE(tp->snd_ssthresh, val); break; @@ -6187,7 +6188,7 @@ static int bpf_sock_ops_get_syn(struct bpf_sock_ops_kern *bpf_sock, * not be available. It has to resort to * saved_syn (if it is saved). */ - saved_syn = inet_reqsk(sk)->saved_syn; + saved_syn = rcu_dereference(inet_reqsk(sk)->saved_syn); else saved_syn = tcp_sk(sk)->saved_syn; diff --git a/net/core/gro.c b/net/core/gro.c index 1b033855a20d..1241b23d66ef 100644 --- a/net/core/gro.c +++ b/net/core/gro.c @@ -238,6 +238,14 @@ int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb) NAPI_GRO_CB(skb)->flush)) return -E2BIG; + /* Fraglist members keep their own headers. Their checksum can not + * be right both for a forwarded packet and for a decapsulated one, + * if remote checksum offload adjusted it. + */ + if (unlikely(skb->remcsum_offload)) + return -EINVAL; + + skb_gro_frag0_invalidate(skb); if (!pskb_may_pull(skb, skb_gro_offset(skb))) { NAPI_GRO_CB(skb)->flush = 1; return -ENOMEM; @@ -374,6 +382,7 @@ static void gro_list_prepare(const struct list_head *head, } diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev; + diffs |= p->protocol != skb->protocol; diffs |= p->vlan_all ^ skb->vlan_all; diffs |= skb_metadata_differs(p, skb); if (maclen == ETH_HLEN) @@ -413,7 +422,8 @@ static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff) headlen = skb_headlen(skb); NAPI_GRO_CB(skb)->frag0 = skb->data; NAPI_GRO_CB(skb)->frag0_len = headlen; - if (headlen) + /* Shared fragments must be pulled into the linear head before parsing. */ + if (headlen || skb_has_shared_frag(skb)) return; pinfo = skb_shinfo(skb); @@ -450,12 +460,19 @@ static void gro_pull_from_frag0(struct sk_buff *skb, int grow) } } -static void gro_try_pull_from_frag0(struct sk_buff *skb) +static bool gro_try_pull_from_frag0(struct sk_buff *skb) { int grow = skb_gro_offset(skb) - skb_headlen(skb); - if (grow > 0) + if (grow <= 0) + return true; + + /* frag0 can only be used if no pull happened. */ + if (likely(NAPI_GRO_CB(skb)->frag0_len)) { gro_pull_from_frag0(skb, grow); + return true; + } + return pskb_may_pull(skb, skb_gro_offset(skb)); } static void gro_flush_oldest(struct gro_node *gro, struct list_head *head) @@ -557,13 +574,17 @@ found_ptype: if (NAPI_GRO_CB(skb)->flush) goto normal; + /* Must be called before setting NAPI_GRO_CB(skb)->{age|last}. + * A held packet must have its headers in the linear part. + */ + if (unlikely(!gro_try_pull_from_frag0(skb))) + goto normal; + if (unlikely(gro_list->count >= MAX_GRO_SKBS)) gro_flush_oldest(gro, &gro_list->list); else gro_list->count++; - /* Must be called before setting NAPI_GRO_CB(skb)->{age|last} */ - gro_try_pull_from_frag0(skb); NAPI_GRO_CB(skb)->age = jiffies; NAPI_GRO_CB(skb)->last = skb; if (!skb_is_gso(skb)) diff --git a/net/core/net_namespace.c b/net/core/net_namespace.c index 300322bcf8d7..217021eeab13 100644 --- a/net/core/net_namespace.c +++ b/net/core/net_namespace.c @@ -270,14 +270,19 @@ static void ops_undo_single(struct pernet_operations *ops, /* should be called with nsid_lock held */ static int alloc_netid(struct net *net, struct net *peer, int reqid) { - int min = 0, max = 0; + int ret; + u32 id; if (reqid >= 0) { - min = reqid; - max = reqid + 1; + id = reqid; + ret = idr_alloc_u32(&net->netns_ids, peer, &id, id, + GFP_ATOMIC); + if (ret) + return ret; + return id; } - return idr_alloc(&net->netns_ids, peer, min, max, GFP_ATOMIC); + return idr_alloc(&net->netns_ids, peer, 0, 0, GFP_ATOMIC); } /* This function is used by idr_for_each(). If net is equal to peer, the @@ -639,10 +644,10 @@ static void unhash_nsid(struct net *last) * use for_each_net_rcu() or net_rwsem. */ for_each_net(tmp) { - int id = 0; + unsigned long id = 0; spin_lock(&tmp->nsid_lock); - while ((peer = idr_get_next(&tmp->netns_ids, &id))) { + while ((peer = idr_get_next_ul(&tmp->netns_ids, &id))) { int curr_id = id; id++; diff --git a/net/core/skbuff.c b/net/core/skbuff.c index c41e9a3e6a04..1ac1e327ac4c 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -967,6 +967,7 @@ int skb_pp_cow_data(struct page_pool *pool, struct sk_buff **pskb, skb_reserve(nskb, headroom); skb_copy_header(nskb, skb); + nskb->mac_len = skb->mac_len; skb_mark_for_recycle(nskb); err = skb_copy_bits(skb, 0, nskb->data, size); @@ -1495,6 +1496,10 @@ void napi_skb_free_stolen_head(struct sk_buff *skb) skb_orphan(skb); skb->slow_gro = 0; } + if (skb->fclone != SKB_FCLONE_UNAVAILABLE) { + kfree_skbmem(skb); + return; + } napi_skb_cache_put(skb); } @@ -1908,6 +1913,7 @@ int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, struct ubuf_info *uarg, struct net_devmem_dmabuf_binding *binding) { + unsigned int orig_truesize = skb->truesize; int err, orig_len = skb->len; if (uarg->ops->link_skb) { @@ -1928,12 +1934,18 @@ int skb_zerocopy_iter_stream(struct sock *sk, struct sk_buff *skb, err = __zerocopy_sg_from_iter(msg, sk, skb, &msg->msg_iter, len, binding); if (err == -EFAULT || (err == -EMSGSIZE && skb->len == orig_len)) { + u32 delta_truesize = skb->truesize - orig_truesize; struct sock *save_sk = skb->sk; /* Streams do not free skb on error. Reset to prev state. */ iov_iter_revert(&msg->msg_iter, skb->len - orig_len); skb->sk = sk; - ___pskb_trim(skb, orig_len); + if (!___pskb_trim(skb, orig_len)) { + skb->truesize -= delta_truesize; + sk_wmem_queued_add(sk, -delta_truesize); + if (!skb_zcopy_pure(skb)) + sk_mem_uncharge(sk, delta_truesize); + } skb->sk = save_sk; return err; } @@ -6263,7 +6275,7 @@ void kfree_skb_partial(struct sk_buff *skb, bool head_stolen) { if (head_stolen) { skb_release_head_state(skb); - kmem_cache_free(net_hotdata.skbuff_cache, skb); + kfree_skbmem(skb); } else { __kfree_skb(skb); } @@ -6379,6 +6391,31 @@ bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, } EXPORT_SYMBOL(skb_try_coalesce); +static void skb_ext_scrub(struct sk_buff *skb) +{ +#if IS_ENABLED(CONFIG_INET_PSP) + struct psp_skb_ext *pse = skb_ext_find(skb, SKB_EXT_PSP); + + if (pse) { + struct skb_ext *ext = skb->extensions; + struct psp_skb_ext tmp; + + /* Nothing else to scrub if PSP is the only extension in ext. */ + if (ext->chunks == SKB_EXT_CHUNKSIZEOF(*ext) + + skb_ext_type_len[SKB_EXT_PSP]) + return; + + tmp = *pse; + skb_ext_reset(skb); + pse = skb_ext_add(skb, SKB_EXT_PSP); + if (pse) + memcpy(pse, &tmp, sizeof(*pse)); + return; + } +#endif + skb_ext_reset(skb); +} + /** * skb_scrub_packet - scrub an skb * @@ -6390,7 +6427,8 @@ EXPORT_SYMBOL(skb_try_coalesce); * operations. * skb_scrub_packet can also be used to clean a skb before injecting it in * another namespace (@xnet == true). We have to clear all information in the - * skb that could impact namespace isolation. + * skb that could impact namespace isolation. Note that PSP skb extension is + * preserved across packet scrubbing by design. */ void skb_scrub_packet(struct sk_buff *skb, bool xnet) { @@ -6398,7 +6436,7 @@ void skb_scrub_packet(struct sk_buff *skb, bool xnet) skb->skb_iif = 0; skb->ignore_df = 0; skb_dst_drop(skb); - skb_ext_reset(skb); + skb_ext_scrub(skb); nf_reset_ct(skb); nf_reset_trace(skb); diff --git a/net/core/sock.c b/net/core/sock.c index e2401d735c5a..ad414878dc5b 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -3427,7 +3427,7 @@ int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind) goto suppress_allocation; } - if (!allocated) + if (sk->sk_bypass_prot_mem) return 1; /* Under limit. */ @@ -3508,7 +3508,7 @@ suppress_allocation: trace_sock_exceed_buf_limit(sk, prot, allocated, kind); - if (allocated) + if (!sk->sk_bypass_prot_mem) sk_memory_allocated_sub(sk, amt); if (charged) diff --git a/net/core/sock_reuseport.c b/net/core/sock_reuseport.c index fccbdd0430b0..b9e5648fa4c5 100644 --- a/net/core/sock_reuseport.c +++ b/net/core/sock_reuseport.c @@ -6,6 +6,7 @@ * selecting the socket index from the array of available sockets. */ +#include <net/inet_connection_sock.h> #include <net/ip.h> #include <net/sock_reuseport.h> #include <linux/bpf.h> @@ -692,6 +693,14 @@ int reuseport_attach_prog(struct sock *sk, struct bpf_prog *prog) if (!sk->sk_reuseport) return -EINVAL; + /* Do not put a shutdown()ed TCP listener back into the + * listening section of a reuseport group while it is in + * TCP_CLOSE: requests can still migrate to it. + */ + if (sk_is_tcp(sk) && + inet_csk(sk)->unhashed_state == TCP_LISTEN) + return -EINVAL; + err = reuseport_alloc(sk, false); if (err) return err; |
