ich777/linux_media
1
0
Fork 0
mirror of https://github.com/tbsdtv/linux_media.git synced 2025-07-23 04:33:26 +02:00
Code Issues Packages Projects Releases Wiki Activity

tcp: Access &tcp_hashinfo via net.

Browse Source 
We will soon introduce an optional per-netns ehash.

This means we cannot use tcp_hashinfo directly in most places.

Instead, access it via net->ipv4.tcp_death_row.hashinfo.

The access will be valid only while initialising tcp_hashinfo
itself and creating/destroying each netns.

Signed-off-by: Kuniyuki Iwashima <kuniyu@amazon.com>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Kuniyuki Iwashima
2022-09-07 18:10:20 -07:00
committed by Jakub Kicinski
parent 429e42c1c5
commit 4461568aa4
17 changed files with 108 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c
View File

@@ -1069,8 +1069,7 @@ static void chtls_pass_accept_rpl(struct sk_buff *skb,
cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry); cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
} }
static void inet_inherit_port(struct inet_hashinfo *hash_info, static void inet_inherit_port(struct sock *lsk, struct sock *newsk)
struct sock *lsk, struct sock *newsk)
{ {
local_bh_disable(); local_bh_disable();
__inet_inherit_port(lsk, newsk); __inet_inherit_port(lsk, newsk);
@@ -1240,7 +1239,7 @@ static struct sock *chtls_recv_sock(struct sock *lsk,
ipv4.sysctl_tcp_window_scaling), ipv4.sysctl_tcp_window_scaling),
tp->window_clamp); tp->window_clamp);
neigh_release(n); neigh_release(n);
inet_inherit_port(&tcp_hashinfo, lsk, newsk); inet_inherit_port(lsk, newsk);
csk_set_flag(csk, CSK_CONN_INLINE); csk_set_flag(csk, CSK_CONN_INLINE);
bh_unlock_sock(newsk); /* tcp_create_openreq_child ->sk_clone_lock */ bh_unlock_sock(newsk); /* tcp_create_openreq_child ->sk_clone_lock */

5
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_rx.c
View File

@@ -461,6 +461,7 @@ static void resync_update_sn(struct mlx5e_rq *rq, struct sk_buff *skb)
{ {
struct ethhdr *eth = (struct ethhdr *)(skb->data); struct ethhdr *eth = (struct ethhdr *)(skb->data);
struct net_device *netdev = rq->netdev; struct net_device *netdev = rq->netdev;
struct net *net = dev_net(netdev);
struct sock *sk = NULL; struct sock *sk = NULL;
unsigned int datalen; unsigned int datalen;
struct iphdr *iph; struct iphdr *iph;
@@ -475,7 +476,7 @@ static void resync_update_sn(struct mlx5e_rq *rq, struct sk_buff *skb)
depth += sizeof(struct iphdr); depth += sizeof(struct iphdr);
th = (void *)iph + sizeof(struct iphdr); th = (void *)iph + sizeof(struct iphdr);
sk = inet_lookup_established(dev_net(netdev), &tcp_hashinfo, sk = inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
iph->saddr, th->source, iph->daddr, iph->saddr, th->source, iph->daddr,
th->dest, netdev->ifindex); th->dest, netdev->ifindex);
#if IS_ENABLED(CONFIG_IPV6) #if IS_ENABLED(CONFIG_IPV6)
@@ -485,7 +486,7 @@ static void resync_update_sn(struct mlx5e_rq *rq, struct sk_buff *skb)
depth += sizeof(struct ipv6hdr); depth += sizeof(struct ipv6hdr);
th = (void *)ipv6h + sizeof(struct ipv6hdr); th = (void *)ipv6h + sizeof(struct ipv6hdr);
sk = __inet6_lookup_established(dev_net(netdev), &tcp_hashinfo, sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
&ipv6h->saddr, th->source, &ipv6h->saddr, th->source,
&ipv6h->daddr, ntohs(th->dest), &ipv6h->daddr, ntohs(th->dest),
netdev->ifindex, 0); netdev->ifindex, 0);

5
drivers/net/ethernet/netronome/nfp/crypto/tls.c
View File

@@ -474,6 +474,7 @@ int nfp_net_tls_rx_resync_req(struct net_device *netdev,
{ {
struct nfp_net *nn = netdev_priv(netdev); struct nfp_net *nn = netdev_priv(netdev);
struct nfp_net_tls_offload_ctx *ntls; struct nfp_net_tls_offload_ctx *ntls;
struct net *net = dev_net(netdev);
struct ipv6hdr *ipv6h; struct ipv6hdr *ipv6h;
struct tcphdr *th; struct tcphdr *th;
struct iphdr *iph; struct iphdr *iph;
@@ -494,13 +495,13 @@ int nfp_net_tls_rx_resync_req(struct net_device *netdev,
switch (ipv6h->version) { switch (ipv6h->version) {
case 4: case 4:
sk = inet_lookup_established(dev_net(netdev), &tcp_hashinfo, sk = inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
iph->saddr, th->source, iph->daddr, iph->saddr, th->source, iph->daddr,
th->dest, netdev->ifindex); th->dest, netdev->ifindex);
break; break;
#if IS_ENABLED(CONFIG_IPV6) #if IS_ENABLED(CONFIG_IPV6)
case 6: case 6:
sk = __inet6_lookup_established(dev_net(netdev), &tcp_hashinfo, sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
&ipv6h->saddr, th->source, &ipv6h->saddr, th->source,
&ipv6h->daddr, ntohs(th->dest), &ipv6h->daddr, ntohs(th->dest),
netdev->ifindex, 0); netdev->ifindex, 0);

5
net/core/filter.c
View File

@@ -6373,6 +6373,7 @@ static const struct bpf_func_proto bpf_lwt_seg6_adjust_srh_proto = {
static struct sock *sk_lookup(struct net *net, struct bpf_sock_tuple *tuple, static struct sock *sk_lookup(struct net *net, struct bpf_sock_tuple *tuple,
int dif, int sdif, u8 family, u8 proto) int dif, int sdif, u8 family, u8 proto)
{ {
struct inet_hashinfo *hinfo = net->ipv4.tcp_death_row.hashinfo;
bool refcounted = false; bool refcounted = false;
struct sock *sk = NULL; struct sock *sk = NULL;
@@ -6381,7 +6382,7 @@ static struct sock *sk_lookup(struct net *net, struct bpf_sock_tuple *tuple,
__be32 dst4 = tuple->ipv4.daddr; __be32 dst4 = tuple->ipv4.daddr;
if (proto == IPPROTO_TCP) if (proto == IPPROTO_TCP)
sk = __inet_lookup(net, &tcp_hashinfo, NULL, 0, sk = __inet_lookup(net, hinfo, NULL, 0,
src4, tuple->ipv4.sport, src4, tuple->ipv4.sport,
dst4, tuple->ipv4.dport, dst4, tuple->ipv4.dport,
dif, sdif, &refcounted); dif, sdif, &refcounted);
@@ -6395,7 +6396,7 @@ static struct sock *sk_lookup(struct net *net, struct bpf_sock_tuple *tuple,
struct in6_addr *dst6 = (struct in6_addr *)&tuple->ipv6.daddr; struct in6_addr *dst6 = (struct in6_addr *)&tuple->ipv6.daddr;
if (proto == IPPROTO_TCP) if (proto == IPPROTO_TCP)
sk = __inet6_lookup(net, &tcp_hashinfo, NULL, 0, sk = __inet6_lookup(net, hinfo, NULL, 0,
src6, tuple->ipv6.sport, src6, tuple->ipv6.sport,
dst6, ntohs(tuple->ipv6.dport), dst6, ntohs(tuple->ipv6.dport),
dif, sdif, &refcounted); dif, sdif, &refcounted);

3
net/ipv4/esp4.c
View File

@@ -134,6 +134,7 @@ static void esp_free_tcp_sk(struct rcu_head *head)
static struct sock *esp_find_tcp_sk(struct xfrm_state *x) static struct sock *esp_find_tcp_sk(struct xfrm_state *x)
{ {
struct xfrm_encap_tmpl *encap = x->encap; struct xfrm_encap_tmpl *encap = x->encap;
struct net *net = xs_net(x);
struct esp_tcp_sk *esk; struct esp_tcp_sk *esk;
__be16 sport, dport; __be16 sport, dport;
struct sock *nsk; struct sock *nsk;
@@ -160,7 +161,7 @@ static struct sock *esp_find_tcp_sk(struct xfrm_state *x)
} }
spin_unlock_bh(&x->lock); spin_unlock_bh(&x->lock);
sk = inet_lookup_established(xs_net(x), &tcp_hashinfo, x->id.daddr.a4, sk = inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, x->id.daddr.a4,
dport, x->props.saddr.a4, sport, 0); dport, x->props.saddr.a4, sport, 0);
if (!sk) if (!sk)
return ERR_PTR(-ENOENT); return ERR_PTR(-ENOENT);

2
net/ipv4/inet_hashtables.c
View File

@@ -386,7 +386,7 @@ static inline struct sock *inet_lookup_run_bpf(struct net *net,
struct sock *sk, *reuse_sk; struct sock *sk, *reuse_sk;
bool no_reuseport; bool no_reuseport;
if (hashinfo != &tcp_hashinfo) if (hashinfo != net->ipv4.tcp_death_row.hashinfo)
return NULL; /* only TCP is supported */ return NULL; /* only TCP is supported */
no_reuseport = bpf_sk_lookup_run_v4(net, IPPROTO_TCP, saddr, sport, no_reuseport = bpf_sk_lookup_run_v4(net, IPPROTO_TCP, saddr, sport,

4
net/ipv4/netfilter/nf_socket_ipv4.c
View File

@@ -71,8 +71,8 @@ nf_socket_get_sock_v4(struct net *net, struct sk_buff *skb, const int doff,
{ {
switch (protocol) { switch (protocol) {
case IPPROTO_TCP: case IPPROTO_TCP:
return inet_lookup(net, &tcp_hashinfo, skb, doff, return inet_lookup(net, net->ipv4.tcp_death_row.hashinfo,
saddr, sport, daddr, dport, skb, doff, saddr, sport, daddr, dport,
in->ifindex); in->ifindex);
case IPPROTO_UDP: case IPPROTO_UDP:
return udp4_lib_lookup(net, saddr, sport, daddr, dport, return udp4_lib_lookup(net, saddr, sport, daddr, dport,

16
net/ipv4/netfilter/nf_tproxy_ipv4.c
View File

@@ -79,6 +79,7 @@ nf_tproxy_get_sock_v4(struct net *net, struct sk_buff *skb,
const struct net_device *in, const struct net_device *in,
const enum nf_tproxy_lookup_t lookup_type) const enum nf_tproxy_lookup_t lookup_type)
{ {
struct inet_hashinfo *hinfo = net->ipv4.tcp_death_row.hashinfo;
struct sock *sk; struct sock *sk;
switch (protocol) { switch (protocol) {
@@ -92,12 +93,10 @@ nf_tproxy_get_sock_v4(struct net *net, struct sk_buff *skb,
switch (lookup_type) { switch (lookup_type) {
case NF_TPROXY_LOOKUP_LISTENER: case NF_TPROXY_LOOKUP_LISTENER:
sk = inet_lookup_listener(net, &tcp_hashinfo, skb, sk = inet_lookup_listener(net, hinfo, skb,
ip_hdrlen(skb) + ip_hdrlen(skb) + __tcp_hdrlen(hp),
__tcp_hdrlen(hp), saddr, sport, daddr, dport,
saddr, sport, in->ifindex, 0);
daddr, dport,
in->ifindex, 0);
if (sk && !refcount_inc_not_zero(&sk->sk_refcnt)) if (sk && !refcount_inc_not_zero(&sk->sk_refcnt))
sk = NULL; sk = NULL;
@@ -108,9 +107,8 @@ nf_tproxy_get_sock_v4(struct net *net, struct sk_buff *skb,
*/ */
break; break;
case NF_TPROXY_LOOKUP_ESTABLISHED: case NF_TPROXY_LOOKUP_ESTABLISHED:
sk = inet_lookup_established(net, &tcp_hashinfo, sk = inet_lookup_established(net, hinfo, saddr, sport,
saddr, sport, daddr, dport, daddr, dport, in->ifindex);
in->ifindex);
break; break;
default: default:
BUG(); BUG();

18
net/ipv4/tcp_diag.c
View File

@@ -181,13 +181,21 @@ static size_t tcp_diag_get_aux_size(struct sock *sk, bool net_admin)
static void tcp_diag_dump(struct sk_buff *skb, struct netlink_callback *cb, static void tcp_diag_dump(struct sk_buff *skb, struct netlink_callback *cb,
const struct inet_diag_req_v2 *r) const struct inet_diag_req_v2 *r)
{ {
inet_diag_dump_icsk(&tcp_hashinfo, skb, cb, r); struct inet_hashinfo *hinfo;
hinfo = sock_net(cb->skb->sk)->ipv4.tcp_death_row.hashinfo;
inet_diag_dump_icsk(hinfo, skb, cb, r);
} }
static int tcp_diag_dump_one(struct netlink_callback *cb, static int tcp_diag_dump_one(struct netlink_callback *cb,
const struct inet_diag_req_v2 *req) const struct inet_diag_req_v2 *req)
{ {
return inet_diag_dump_one_icsk(&tcp_hashinfo, cb, req); struct inet_hashinfo *hinfo;
hinfo = sock_net(cb->skb->sk)->ipv4.tcp_death_row.hashinfo;
return inet_diag_dump_one_icsk(hinfo, cb, req);
} }
#ifdef CONFIG_INET_DIAG_DESTROY #ifdef CONFIG_INET_DIAG_DESTROY
@@ -195,9 +203,13 @@ static int tcp_diag_destroy(struct sk_buff *in_skb,
const struct inet_diag_req_v2 *req) const struct inet_diag_req_v2 *req)
{ {
struct net *net = sock_net(in_skb->sk); struct net *net = sock_net(in_skb->sk);
struct sock *sk = inet_diag_find_one_icsk(net, &tcp_hashinfo, req); struct inet_hashinfo *hinfo;
struct sock *sk;
int err; int err;
hinfo = net->ipv4.tcp_death_row.hashinfo;
sk = inet_diag_find_one_icsk(net, hinfo, req);
if (IS_ERR(sk)) if (IS_ERR(sk))
return PTR_ERR(sk); return PTR_ERR(sk);

72
net/ipv4/tcp_ipv4.c
View File

@@ -248,9 +248,11 @@ int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
if (!inet_opt || !inet_opt->opt.srr) if (!inet_opt || !inet_opt->opt.srr)
daddr = fl4->daddr; daddr = fl4->daddr;
tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
if (!inet->inet_saddr) { if (!inet->inet_saddr) {
if (inet_csk(sk)->icsk_bind2_hash) { if (inet_csk(sk)->icsk_bind2_hash) {
prev_addr_hashbucket = inet_bhashfn_portaddr(&tcp_hashinfo, prev_addr_hashbucket = inet_bhashfn_portaddr(tcp_death_row->hashinfo,
sk, net, inet->inet_num); sk, net, inet->inet_num);
prev_sk_rcv_saddr = sk->sk_rcv_saddr; prev_sk_rcv_saddr = sk->sk_rcv_saddr;
} }
@@ -292,7 +294,6 @@ int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
* complete initialization after this. * complete initialization after this.
*/ */
tcp_set_state(sk, TCP_SYN_SENT); tcp_set_state(sk, TCP_SYN_SENT);
tcp_death_row = &net->ipv4.tcp_death_row;
err = inet_hash_connect(tcp_death_row, sk); err = inet_hash_connect(tcp_death_row, sk);
if (err) if (err)
goto failure; goto failure;
@@ -494,9 +495,9 @@ int tcp_v4_err(struct sk_buff *skb, u32 info)
int err; int err;
struct net *net = dev_net(skb->dev); struct net *net = dev_net(skb->dev);
sk = __inet_lookup_established(net, &tcp_hashinfo, iph->daddr, sk = __inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
th->dest, iph->saddr, ntohs(th->source), iph->daddr, th->dest, iph->saddr,
inet_iif(skb), 0); ntohs(th->source), inet_iif(skb), 0);
if (!sk) { if (!sk) {
__ICMP_INC_STATS(net, ICMP_MIB_INERRORS); __ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
return -ENOENT; return -ENOENT;
@@ -759,8 +760,8 @@ static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb)
* Incoming packet is checked with md5 hash with finding key, * Incoming packet is checked with md5 hash with finding key,
* no RST generated if md5 hash doesn't match. * no RST generated if md5 hash doesn't match.
*/ */
sk1 = __inet_lookup_listener(net, &tcp_hashinfo, NULL, 0, sk1 = __inet_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo,
ip_hdr(skb)->saddr, NULL, 0, ip_hdr(skb)->saddr,
th->source, ip_hdr(skb)->daddr, th->source, ip_hdr(skb)->daddr,
ntohs(th->source), dif, sdif); ntohs(th->source), dif, sdif);
/* don't send rst if it can't find key */ /* don't send rst if it can't find key */
@@ -1728,6 +1729,7 @@ EXPORT_SYMBOL(tcp_v4_do_rcv);
int tcp_v4_early_demux(struct sk_buff *skb) int tcp_v4_early_demux(struct sk_buff *skb)
{ {
struct net *net = dev_net(skb->dev);
const struct iphdr *iph; const struct iphdr *iph;
const struct tcphdr *th; const struct tcphdr *th;
struct sock *sk; struct sock *sk;
@@ -1744,7 +1746,7 @@ int tcp_v4_early_demux(struct sk_buff *skb)
if (th->doff < sizeof(struct tcphdr) / 4) if (th->doff < sizeof(struct tcphdr) / 4)
return 0; return 0;
sk = __inet_lookup_established(dev_net(skb->dev), &tcp_hashinfo, sk = __inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
iph->saddr, th->source, iph->saddr, th->source,
iph->daddr, ntohs(th->dest), iph->daddr, ntohs(th->dest),
skb->skb_iif, inet_sdif(skb)); skb->skb_iif, inet_sdif(skb));
@@ -1970,7 +1972,8 @@ int tcp_v4_rcv(struct sk_buff *skb)
th = (const struct tcphdr *)skb->data; th = (const struct tcphdr *)skb->data;
iph = ip_hdr(skb); iph = ip_hdr(skb);
lookup: lookup:
sk = __inet_lookup_skb(&tcp_hashinfo, skb, __tcp_hdrlen(th), th->source, sk = __inet_lookup_skb(net->ipv4.tcp_death_row.hashinfo,
skb, __tcp_hdrlen(th), th->source,
th->dest, sdif, &refcounted); th->dest, sdif, &refcounted);
if (!sk) if (!sk)
goto no_tcp_socket; goto no_tcp_socket;
@@ -2152,9 +2155,9 @@ do_time_wait:
} }
switch (tcp_timewait_state_process(inet_twsk(sk), skb, th)) { switch (tcp_timewait_state_process(inet_twsk(sk), skb, th)) {
case TCP_TW_SYN: { case TCP_TW_SYN: {
struct sock *sk2 = inet_lookup_listener(dev_net(skb->dev), struct sock *sk2 = inet_lookup_listener(net,
&tcp_hashinfo, skb, net->ipv4.tcp_death_row.hashinfo,
__tcp_hdrlen(th), skb, __tcp_hdrlen(th),
iph->saddr, th->source, iph->saddr, th->source,
iph->daddr, th->dest, iph->daddr, th->dest,
inet_iif(skb), inet_iif(skb),
@@ -2304,15 +2307,16 @@ static bool seq_sk_match(struct seq_file *seq, const struct sock *sk)
*/ */
static void *listening_get_first(struct seq_file *seq) static void *listening_get_first(struct seq_file *seq)
{ {
struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
struct tcp_iter_state *st = seq->private; struct tcp_iter_state *st = seq->private;
st->offset = 0; st->offset = 0;
for (; st->bucket <= tcp_hashinfo.lhash2_mask; st->bucket++) { for (; st->bucket <= hinfo->lhash2_mask; st->bucket++) {
struct inet_listen_hashbucket *ilb2; struct inet_listen_hashbucket *ilb2;
struct hlist_nulls_node *node; struct hlist_nulls_node *node;
struct sock *sk; struct sock *sk;
ilb2 = &tcp_hashinfo.lhash2[st->bucket]; ilb2 = &hinfo->lhash2[st->bucket];
if (hlist_nulls_empty(&ilb2->nulls_head)) if (hlist_nulls_empty(&ilb2->nulls_head))
continue; continue;
@@ -2337,6 +2341,7 @@ static void *listening_get_next(struct seq_file *seq, void *cur)
struct tcp_iter_state *st = seq->private; struct tcp_iter_state *st = seq->private;
struct inet_listen_hashbucket *ilb2; struct inet_listen_hashbucket *ilb2;
struct hlist_nulls_node *node; struct hlist_nulls_node *node;
struct inet_hashinfo *hinfo;
struct sock *sk = cur; struct sock *sk = cur;
++st->num; ++st->num;
@@ -2348,7 +2353,8 @@ static void *listening_get_next(struct seq_file *seq, void *cur)
return sk; return sk;
} }
ilb2 = &tcp_hashinfo.lhash2[st->bucket]; hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
ilb2 = &hinfo->lhash2[st->bucket];
spin_unlock(&ilb2->lock); spin_unlock(&ilb2->lock);
++st->bucket; ++st->bucket;
return listening_get_first(seq); return listening_get_first(seq);
@@ -2370,9 +2376,10 @@ static void *listening_get_idx(struct seq_file *seq, loff_t *pos)
return rc; return rc;
} }
static inline bool empty_bucket(const struct tcp_iter_state *st) static inline bool empty_bucket(struct inet_hashinfo *hinfo,
const struct tcp_iter_state *st)
{ {
return hlist_nulls_empty(&tcp_hashinfo.ehash[st->bucket].chain); return hlist_nulls_empty(&hinfo->ehash[st->bucket].chain);
} }
/* /*
@@ -2381,20 +2388,21 @@ static inline bool empty_bucket(const struct tcp_iter_state *st)
*/ */
static void *established_get_first(struct seq_file *seq) static void *established_get_first(struct seq_file *seq)
{ {
struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
struct tcp_iter_state *st = seq->private; struct tcp_iter_state *st = seq->private;
st->offset = 0; st->offset = 0;
for (; st->bucket <= tcp_hashinfo.ehash_mask; ++st->bucket) { for (; st->bucket <= hinfo->ehash_mask; ++st->bucket) {
struct sock *sk; struct sock *sk;
struct hlist_nulls_node *node; struct hlist_nulls_node *node;
spinlock_t *lock = inet_ehash_lockp(&tcp_hashinfo, st->bucket); spinlock_t *lock = inet_ehash_lockp(hinfo, st->bucket);
/* Lockless fast path for the common case of empty buckets */ /* Lockless fast path for the common case of empty buckets */
if (empty_bucket(st)) if (empty_bucket(hinfo, st))
continue; continue;
spin_lock_bh(lock); spin_lock_bh(lock);
sk_nulls_for_each(sk, node, &tcp_hashinfo.ehash[st->bucket].chain) { sk_nulls_for_each(sk, node, &hinfo->ehash[st->bucket].chain) {
if (seq_sk_match(seq, sk)) if (seq_sk_match(seq, sk))
return sk; return sk;
} }
@@ -2406,6 +2414,7 @@ static void *established_get_first(struct seq_file *seq)
static void *established_get_next(struct seq_file *seq, void *cur) static void *established_get_next(struct seq_file *seq, void *cur)
{ {
struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
struct tcp_iter_state *st = seq->private; struct tcp_iter_state *st = seq->private;
struct hlist_nulls_node *node; struct hlist_nulls_node *node;
struct sock *sk = cur; struct sock *sk = cur;
@@ -2420,7 +2429,7 @@ static void *established_get_next(struct seq_file *seq, void *cur)
return sk; return sk;
} }
spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket)); spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
++st->bucket; ++st->bucket;
return established_get_first(seq); return established_get_first(seq);
} }
@@ -2458,6 +2467,7 @@ static void *tcp_get_idx(struct seq_file *seq, loff_t pos)
static void *tcp_seek_last_pos(struct seq_file *seq) static void *tcp_seek_last_pos(struct seq_file *seq)
{ {
struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
struct tcp_iter_state *st = seq->private; struct tcp_iter_state *st = seq->private;
int bucket = st->bucket; int bucket = st->bucket;
int offset = st->offset; int offset = st->offset;
@@ -2466,7 +2476,7 @@ static void *tcp_seek_last_pos(struct seq_file *seq)
switch (st->state) { switch (st->state) {
case TCP_SEQ_STATE_LISTENING: case TCP_SEQ_STATE_LISTENING:
if (st->bucket > tcp_hashinfo.lhash2_mask) if (st->bucket > hinfo->lhash2_mask)
break; break;
st->state = TCP_SEQ_STATE_LISTENING; st->state = TCP_SEQ_STATE_LISTENING;
rc = listening_get_first(seq); rc = listening_get_first(seq);
@@ -2478,7 +2488,7 @@ static void *tcp_seek_last_pos(struct seq_file *seq)
st->state = TCP_SEQ_STATE_ESTABLISHED; st->state = TCP_SEQ_STATE_ESTABLISHED;
fallthrough; fallthrough;
case TCP_SEQ_STATE_ESTABLISHED: case TCP_SEQ_STATE_ESTABLISHED:
if (st->bucket > tcp_hashinfo.ehash_mask) if (st->bucket > hinfo->ehash_mask)
break; break;
rc = established_get_first(seq); rc = established_get_first(seq);
while (offset-- && rc && bucket == st->bucket) while (offset-- && rc && bucket == st->bucket)
@@ -2546,16 +2556,17 @@ EXPORT_SYMBOL(tcp_seq_next);
void tcp_seq_stop(struct seq_file *seq, void *v) void tcp_seq_stop(struct seq_file *seq, void *v)
{ {
struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
struct tcp_iter_state *st = seq->private; struct tcp_iter_state *st = seq->private;
switch (st->state) { switch (st->state) {
case TCP_SEQ_STATE_LISTENING: case TCP_SEQ_STATE_LISTENING:
if (v != SEQ_START_TOKEN) if (v != SEQ_START_TOKEN)
spin_unlock(&tcp_hashinfo.lhash2[st->bucket].lock); spin_unlock(&hinfo->lhash2[st->bucket].lock);
break; break;
case TCP_SEQ_STATE_ESTABLISHED: case TCP_SEQ_STATE_ESTABLISHED:
if (v) if (v)
spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket)); spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
break; break;
} }
} }
@@ -2750,6 +2761,7 @@ static int bpf_iter_tcp_realloc_batch(struct bpf_tcp_iter_state *iter,
static unsigned int bpf_iter_tcp_listening_batch(struct seq_file *seq, static unsigned int bpf_iter_tcp_listening_batch(struct seq_file *seq,
struct sock *start_sk) struct sock *start_sk)
{ {
struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
struct bpf_tcp_iter_state *iter = seq->private; struct bpf_tcp_iter_state *iter = seq->private;
struct tcp_iter_state *st = &iter->state; struct tcp_iter_state *st = &iter->state;
struct hlist_nulls_node *node; struct hlist_nulls_node *node;
@@ -2769,7 +2781,7 @@ static unsigned int bpf_iter_tcp_listening_batch(struct seq_file *seq,
expected++; expected++;
} }
} }
spin_unlock(&tcp_hashinfo.lhash2[st->bucket].lock); spin_unlock(&hinfo->lhash2[st->bucket].lock);
return expected; return expected;
} }
@@ -2777,6 +2789,7 @@ static unsigned int bpf_iter_tcp_listening_batch(struct seq_file *seq,
static unsigned int bpf_iter_tcp_established_batch(struct seq_file *seq, static unsigned int bpf_iter_tcp_established_batch(struct seq_file *seq,
struct sock *start_sk) struct sock *start_sk)
{ {
struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
struct bpf_tcp_iter_state *iter = seq->private; struct bpf_tcp_iter_state *iter = seq->private;
struct tcp_iter_state *st = &iter->state; struct tcp_iter_state *st = &iter->state;
struct hlist_nulls_node *node; struct hlist_nulls_node *node;
@@ -2796,13 +2809,14 @@ static unsigned int bpf_iter_tcp_established_batch(struct seq_file *seq,
expected++; expected++;
} }
} }
spin_unlock_bh(inet_ehash_lockp(&tcp_hashinfo, st->bucket)); spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
return expected; return expected;
} }
static struct sock *bpf_iter_tcp_batch(struct seq_file *seq) static struct sock *bpf_iter_tcp_batch(struct seq_file *seq)
{ {
struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
struct bpf_tcp_iter_state *iter = seq->private; struct bpf_tcp_iter_state *iter = seq->private;
struct tcp_iter_state *st = &iter->state; struct tcp_iter_state *st = &iter->state;
unsigned int expected; unsigned int expected;
@@ -2818,7 +2832,7 @@ static struct sock *bpf_iter_tcp_batch(struct seq_file *seq)
st->offset = 0; st->offset = 0;
st->bucket++; st->bucket++;
if (st->state == TCP_SEQ_STATE_LISTENING && if (st->state == TCP_SEQ_STATE_LISTENING &&
st->bucket > tcp_hashinfo.lhash2_mask) { st->bucket > hinfo->lhash2_mask) {
st->state = TCP_SEQ_STATE_ESTABLISHED; st->state = TCP_SEQ_STATE_ESTABLISHED;
st->bucket = 0; st->bucket = 0;
} }

2
net/ipv4/tcp_minisocks.c
View File

@@ -319,7 +319,7 @@ void tcp_time_wait(struct sock *sk, int state, int timeo)
/* Linkage updates. /* Linkage updates.
* Note that access to tw after this point is illegal. * Note that access to tw after this point is illegal.
*/ */
inet_twsk_hashdance(tw, sk, &tcp_hashinfo); inet_twsk_hashdance(tw, sk, net->ipv4.tcp_death_row.hashinfo);
local_bh_enable(); local_bh_enable();
} else { } else {
/* Sorry, if we're out of memory, just CLOSE this /* Sorry, if we're out of memory, just CLOSE this

3
net/ipv6/esp6.c
View File

@@ -151,6 +151,7 @@ static void esp_free_tcp_sk(struct rcu_head *head)
static struct sock *esp6_find_tcp_sk(struct xfrm_state *x) static struct sock *esp6_find_tcp_sk(struct xfrm_state *x)
{ {
struct xfrm_encap_tmpl *encap = x->encap; struct xfrm_encap_tmpl *encap = x->encap;
struct net *net = xs_net(x);
struct esp_tcp_sk *esk; struct esp_tcp_sk *esk;
__be16 sport, dport; __be16 sport, dport;
struct sock *nsk; struct sock *nsk;
@@ -177,7 +178,7 @@ static struct sock *esp6_find_tcp_sk(struct xfrm_state *x)
} }
spin_unlock_bh(&x->lock); spin_unlock_bh(&x->lock);
sk = __inet6_lookup_established(xs_net(x), &tcp_hashinfo, &x->id.daddr.in6, sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, &x->id.daddr.in6,
dport, &x->props.saddr.in6, ntohs(sport), 0, 0); dport, &x->props.saddr.in6, ntohs(sport), 0, 0);
if (!sk) if (!sk)
return ERR_PTR(-ENOENT); return ERR_PTR(-ENOENT);

4
net/ipv6/inet6_hashtables.c
View File

@@ -21,8 +21,6 @@
#include <net/ip.h> #include <net/ip.h>
#include <net/sock_reuseport.h> #include <net/sock_reuseport.h>
extern struct inet_hashinfo tcp_hashinfo;
u32 inet6_ehashfn(const struct net *net, u32 inet6_ehashfn(const struct net *net,
const struct in6_addr *laddr, const u16 lport, const struct in6_addr *laddr, const u16 lport,
const struct in6_addr *faddr, const __be16 fport) const struct in6_addr *faddr, const __be16 fport)
@@ -169,7 +167,7 @@ static inline struct sock *inet6_lookup_run_bpf(struct net *net,
struct sock *sk, *reuse_sk; struct sock *sk, *reuse_sk;
bool no_reuseport; bool no_reuseport;
if (hashinfo != &tcp_hashinfo) if (hashinfo != net->ipv4.tcp_death_row.hashinfo)
return NULL; /* only TCP is supported */ return NULL; /* only TCP is supported */
no_reuseport = bpf_sk_lookup_run_v6(net, IPPROTO_TCP, saddr, sport, no_reuseport = bpf_sk_lookup_run_v6(net, IPPROTO_TCP, saddr, sport,

4
net/ipv6/netfilter/nf_socket_ipv6.c
View File

@@ -83,8 +83,8 @@ nf_socket_get_sock_v6(struct net *net, struct sk_buff *skb, int doff,
{ {
switch (protocol) { switch (protocol) {
case IPPROTO_TCP: case IPPROTO_TCP:
return inet6_lookup(net, &tcp_hashinfo, skb, doff, return inet6_lookup(net, net->ipv4.tcp_death_row.hashinfo,
saddr, sport, daddr, dport, skb, doff, saddr, sport, daddr, dport,
in->ifindex); in->ifindex);
case IPPROTO_UDP: case IPPROTO_UDP:
return udp6_lib_lookup(net, saddr, sport, daddr, dport, return udp6_lib_lookup(net, saddr, sport, daddr, dport,

8
net/ipv6/netfilter/nf_tproxy_ipv6.c
View File

@@ -80,6 +80,7 @@ nf_tproxy_get_sock_v6(struct net *net, struct sk_buff *skb, int thoff,
const struct net_device *in, const struct net_device *in,
const enum nf_tproxy_lookup_t lookup_type) const enum nf_tproxy_lookup_t lookup_type)
{ {
struct inet_hashinfo *hinfo = net->ipv4.tcp_death_row.hashinfo;
struct sock *sk; struct sock *sk;
switch (protocol) { switch (protocol) {
@@ -93,7 +94,7 @@ nf_tproxy_get_sock_v6(struct net *net, struct sk_buff *skb, int thoff,
switch (lookup_type) { switch (lookup_type) {
case NF_TPROXY_LOOKUP_LISTENER: case NF_TPROXY_LOOKUP_LISTENER:
sk = inet6_lookup_listener(net, &tcp_hashinfo, skb, sk = inet6_lookup_listener(net, hinfo, skb,
thoff + __tcp_hdrlen(hp), thoff + __tcp_hdrlen(hp),
saddr, sport, saddr, sport,
daddr, ntohs(dport), daddr, ntohs(dport),
@@ -108,9 +109,8 @@ nf_tproxy_get_sock_v6(struct net *net, struct sk_buff *skb, int thoff,
*/ */
break; break;
case NF_TPROXY_LOOKUP_ESTABLISHED: case NF_TPROXY_LOOKUP_ESTABLISHED:
sk = __inet6_lookup_established(net, &tcp_hashinfo, sk = __inet6_lookup_established(net, hinfo, saddr, sport, daddr,
saddr, sport, daddr, ntohs(dport), ntohs(dport), in->ifindex, 0);
in->ifindex, 0);
break; break;
default: default:
BUG(); BUG();

23
net/ipv6/tcp_ipv6.c
View File

@@ -287,12 +287,14 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
goto failure; goto failure;
} }
tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
if (!saddr) { if (!saddr) {
struct inet_bind_hashbucket *prev_addr_hashbucket = NULL; struct inet_bind_hashbucket *prev_addr_hashbucket = NULL;
struct in6_addr prev_v6_rcv_saddr; struct in6_addr prev_v6_rcv_saddr;
if (icsk->icsk_bind2_hash) { if (icsk->icsk_bind2_hash) {
prev_addr_hashbucket = inet_bhashfn_portaddr(&tcp_hashinfo, prev_addr_hashbucket = inet_bhashfn_portaddr(tcp_death_row->hashinfo,
sk, net, inet->inet_num); sk, net, inet->inet_num);
prev_v6_rcv_saddr = sk->sk_v6_rcv_saddr; prev_v6_rcv_saddr = sk->sk_v6_rcv_saddr;
} }
@@ -325,7 +327,6 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
inet->inet_dport = usin->sin6_port; inet->inet_dport = usin->sin6_port;
tcp_set_state(sk, TCP_SYN_SENT); tcp_set_state(sk, TCP_SYN_SENT);
tcp_death_row = &net->ipv4.tcp_death_row;
err = inet6_hash_connect(tcp_death_row, sk); err = inet6_hash_connect(tcp_death_row, sk);
if (err) if (err)
goto late_failure; goto late_failure;
@@ -402,7 +403,7 @@ static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
bool fatal; bool fatal;
int err; int err;
sk = __inet6_lookup_established(net, &tcp_hashinfo, sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
&hdr->daddr, th->dest, &hdr->daddr, th->dest,
&hdr->saddr, ntohs(th->source), &hdr->saddr, ntohs(th->source),
skb->dev->ifindex, inet6_sdif(skb)); skb->dev->ifindex, inet6_sdif(skb));
@@ -1035,11 +1036,10 @@ static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb)
* Incoming packet is checked with md5 hash with finding key, * Incoming packet is checked with md5 hash with finding key,
* no RST generated if md5 hash doesn't match. * no RST generated if md5 hash doesn't match.
*/ */
sk1 = inet6_lookup_listener(net, sk1 = inet6_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo,
&tcp_hashinfo, NULL, 0, NULL, 0, &ipv6h->saddr, th->source,
&ipv6h->saddr, &ipv6h->daddr, ntohs(th->source),
th->source, &ipv6h->daddr, dif, sdif);
ntohs(th->source), dif, sdif);
if (!sk1) if (!sk1)
goto out; goto out;
@@ -1637,7 +1637,7 @@ INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb)
hdr = ipv6_hdr(skb); hdr = ipv6_hdr(skb);
lookup: lookup:
sk = __inet6_lookup_skb(&tcp_hashinfo, skb, __tcp_hdrlen(th), sk = __inet6_lookup_skb(net->ipv4.tcp_death_row.hashinfo, skb, __tcp_hdrlen(th),
th->source, th->dest, inet6_iif(skb), sdif, th->source, th->dest, inet6_iif(skb), sdif,
&refcounted); &refcounted);
if (!sk) if (!sk)
@@ -1812,7 +1812,7 @@ do_time_wait:
{ {
struct sock *sk2; struct sock *sk2;
sk2 = inet6_lookup_listener(dev_net(skb->dev), &tcp_hashinfo, sk2 = inet6_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo,
skb, __tcp_hdrlen(th), skb, __tcp_hdrlen(th),
&ipv6_hdr(skb)->saddr, th->source, &ipv6_hdr(skb)->saddr, th->source,
&ipv6_hdr(skb)->daddr, &ipv6_hdr(skb)->daddr,
@@ -1845,6 +1845,7 @@ do_time_wait:
void tcp_v6_early_demux(struct sk_buff *skb) void tcp_v6_early_demux(struct sk_buff *skb)
{ {
struct net *net = dev_net(skb->dev);
const struct ipv6hdr *hdr; const struct ipv6hdr *hdr;
const struct tcphdr *th; const struct tcphdr *th;
struct sock *sk; struct sock *sk;
@@ -1862,7 +1863,7 @@ void tcp_v6_early_demux(struct sk_buff *skb)
return; return;
/* Note : We use inet6_iif() here, not tcp_v6_iif() */ /* Note : We use inet6_iif() here, not tcp_v6_iif() */
sk = __inet6_lookup_established(dev_net(skb->dev), &tcp_hashinfo, sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
&hdr->saddr, th->source, &hdr->saddr, th->source,
&hdr->daddr, ntohs(th->dest), &hdr->daddr, ntohs(th->dest),
inet6_iif(skb), inet6_sdif(skb)); inet6_iif(skb), inet6_sdif(skb));

7
net/mptcp/mptcp_diag.c
View File

@@ -81,15 +81,18 @@ static void mptcp_diag_dump_listeners(struct sk_buff *skb, struct netlink_callba
struct mptcp_diag_ctx *diag_ctx = (void *)cb->ctx; struct mptcp_diag_ctx *diag_ctx = (void *)cb->ctx;
struct nlattr *bc = cb_data->inet_diag_nla_bc; struct nlattr *bc = cb_data->inet_diag_nla_bc;
struct net *net = sock_net(skb->sk); struct net *net = sock_net(skb->sk);
struct inet_hashinfo *hinfo;
int i; int i;
for (i = diag_ctx->l_slot; i <= tcp_hashinfo.lhash2_mask; i++) { hinfo = net->ipv4.tcp_death_row.hashinfo;
for (i = diag_ctx->l_slot; i <= hinfo->lhash2_mask; i++) {
struct inet_listen_hashbucket *ilb; struct inet_listen_hashbucket *ilb;
struct hlist_nulls_node *node; struct hlist_nulls_node *node;
struct sock *sk; struct sock *sk;
int num = 0; int num = 0;
ilb = &tcp_hashinfo.lhash2[i]; ilb = &hinfo->lhash2[i];
rcu_read_lock(); rcu_read_lock();
spin_lock(&ilb->lock); spin_lock(&ilb->lock);