| 1 |
// SPDX-License-Identifier: GPL-2.0-or-later |
| 2 |
/* |
| 3 |
* INET An implementation of the TCP/IP protocol suite for the LINUX |
| 4 |
* operating system. INET is implemented using the BSD Socket |
| 5 |
* interface as the means of communication with the user level. |
| 6 |
* |
| 7 |
* Generic INET6 transport hashtables |
| 8 |
* |
| 9 |
* Authors: Lotsa people, from code originally in tcp, generalised here |
| 10 |
* by Arnaldo Carvalho de Melo <[email protected]> |
| 11 |
*/ |
| 12 |
|
| 13 |
#include <linux/module.h> |
| 14 |
#include <linux/random.h> |
| 15 |
|
| 16 |
#include <net/addrconf.h> |
| 17 |
#include <net/hotdata.h> |
| 18 |
#include <net/inet_connection_sock.h> |
| 19 |
#include <net/inet_hashtables.h> |
| 20 |
#include <net/inet6_hashtables.h> |
| 21 |
#include <net/secure_seq.h> |
| 22 |
#include <net/ip.h> |
| 23 |
#include <net/sock_reuseport.h> |
| 24 |
#include <net/tcp.h> |
| 25 |
|
| 26 |
u32 inet6_ehashfn(const struct net *net, |
| 27 |
const struct in6_addr *laddr, const u16 lport, |
| 28 |
const struct in6_addr *faddr, const __be16 fport) |
| 29 |
{ |
| 30 |
u32 lhash, fhash; |
| 31 |
|
| 32 |
net_get_random_once(&inet6_ehash_secret, sizeof(inet6_ehash_secret)); |
| 33 |
net_get_random_once(&tcp_ipv6_hash_secret, sizeof(tcp_ipv6_hash_secret)); |
| 34 |
|
| 35 |
lhash = (__force u32)laddr->s6_addr32[3]; |
| 36 |
fhash = __ipv6_addr_jhash(faddr, tcp_ipv6_hash_secret); |
| 37 |
|
| 38 |
return lport + __inet6_ehashfn(lhash, 0, fhash, fport, |
| 39 |
inet6_ehash_secret + net_hash_mix(net)); |
| 40 |
} |
| 41 |
EXPORT_SYMBOL_GPL(inet6_ehashfn); |
| 42 |
|
| 43 |
/* |
| 44 |
* Sockets in TCP_CLOSE state are _always_ taken out of the hash, so |
| 45 |
* we need not check it for TCP lookups anymore, thanks Alexey. -DaveM |
| 46 |
* |
| 47 |
* The sockhash lock must be held as a reader here. |
| 48 |
*/ |
| 49 |
struct sock *__inet6_lookup_established(const struct net *net, |
| 50 |
const struct in6_addr *saddr, |
| 51 |
const __be16 sport, |
| 52 |
const struct in6_addr *daddr, |
| 53 |
const u16 hnum, |
| 54 |
const int dif, const int sdif) |
| 55 |
{ |
| 56 |
const __portpair ports = INET_COMBINED_PORTS(sport, hnum); |
| 57 |
const struct hlist_nulls_node *node; |
| 58 |
struct inet_ehash_bucket *head; |
| 59 |
struct inet_hashinfo *hashinfo; |
| 60 |
unsigned int hash, slot; |
| 61 |
struct sock *sk; |
| 62 |
|
| 63 |
hashinfo = net->ipv4.tcp_death_row.hashinfo; |
| 64 |
hash = inet6_ehashfn(net, daddr, hnum, saddr, sport); |
| 65 |
slot = hash & hashinfo->ehash_mask; |
| 66 |
head = &hashinfo->ehash[slot]; |
| 67 |
begin: |
| 68 |
sk_nulls_for_each_rcu(sk, node, &head->chain) { |
| 69 |
if (sk->sk_hash != hash) |
| 70 |
continue; |
| 71 |
if (!inet6_match(net, sk, saddr, daddr, ports, dif, sdif)) |
| 72 |
continue; |
| 73 |
if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt))) |
| 74 |
goto out; |
| 75 |
|
| 76 |
if (unlikely(!inet6_match(net, sk, saddr, daddr, ports, dif, sdif))) { |
| 77 |
sock_gen_put(sk); |
| 78 |
goto begin; |
| 79 |
} |
| 80 |
goto found; |
| 81 |
} |
| 82 |
if (get_nulls_value(node) != slot) |
| 83 |
goto begin; |
| 84 |
out: |
| 85 |
sk = NULL; |
| 86 |
found: |
| 87 |
return sk; |
| 88 |
} |
| 89 |
EXPORT_SYMBOL(__inet6_lookup_established); |
| 90 |
|
| 91 |
static inline int compute_score(struct sock *sk, const struct net *net, |
| 92 |
const unsigned short hnum, |
| 93 |
const struct in6_addr *daddr, |
| 94 |
const int dif, const int sdif) |
| 95 |
{ |
| 96 |
int score = -1; |
| 97 |
|
| 98 |
if (net_eq(sock_net(sk), net) && |
| 99 |
READ_ONCE(inet_sk(sk)->inet_num) == hnum && |
| 100 |
sk->sk_family == PF_INET6) { |
| 101 |
if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr)) |
| 102 |
return -1; |
| 103 |
|
| 104 |
if (!inet_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif)) |
| 105 |
return -1; |
| 106 |
|
| 107 |
score = sk->sk_bound_dev_if ? 2 : 1; |
| 108 |
if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id()) |
| 109 |
score++; |
| 110 |
} |
| 111 |
return score; |
| 112 |
} |
| 113 |
|
| 114 |
/** |
| 115 |
* inet6_lookup_reuseport() - execute reuseport logic on AF_INET6 socket if necessary. |
| 116 |
* @net: network namespace. |
| 117 |
* @sk: AF_INET6 socket, must be in TCP_LISTEN state for TCP or TCP_CLOSE for UDP. |
| 118 |
* @skb: context for a potential SK_REUSEPORT program. |
| 119 |
* @doff: header offset. |
| 120 |
* @saddr: source address. |
| 121 |
* @sport: source port. |
| 122 |
* @daddr: destination address. |
| 123 |
* @hnum: destination port in host byte order. |
| 124 |
* @ehashfn: hash function used to generate the fallback hash. |
| 125 |
* |
| 126 |
* Return: NULL if sk doesn't have SO_REUSEPORT set, otherwise a pointer to |
| 127 |
* the selected sock or an error. |
| 128 |
*/ |
| 129 |
struct sock *inet6_lookup_reuseport(const struct net *net, struct sock *sk, |
| 130 |
struct sk_buff *skb, int doff, |
| 131 |
const struct in6_addr *saddr, |
| 132 |
__be16 sport, |
| 133 |
const struct in6_addr *daddr, |
| 134 |
unsigned short hnum, |
| 135 |
inet6_ehashfn_t *ehashfn) |
| 136 |
{ |
| 137 |
struct sock *reuse_sk = NULL; |
| 138 |
u32 phash; |
| 139 |
|
| 140 |
if (sk->sk_reuseport) { |
| 141 |
phash = INDIRECT_CALL_INET(ehashfn, udp6_ehashfn, inet6_ehashfn, |
| 142 |
net, daddr, hnum, saddr, sport); |
| 143 |
reuse_sk = reuseport_select_sock(sk, phash, skb, doff); |
| 144 |
} |
| 145 |
return reuse_sk; |
| 146 |
} |
| 147 |
EXPORT_SYMBOL_GPL(inet6_lookup_reuseport); |
| 148 |
|
| 149 |
/* called with rcu_read_lock() */ |
| 150 |
static struct sock *inet6_lhash2_lookup(const struct net *net, |
| 151 |
struct inet_listen_hashbucket *ilb2, |
| 152 |
struct sk_buff *skb, int doff, |
| 153 |
const struct in6_addr *saddr, |
| 154 |
const __be16 sport, const struct in6_addr *daddr, |
| 155 |
const unsigned short hnum, const int dif, const int sdif) |
| 156 |
{ |
| 157 |
struct sock *sk, *result = NULL; |
| 158 |
struct hlist_nulls_node *node; |
| 159 |
int score, hiscore = 0; |
| 160 |
|
| 161 |
sk_nulls_for_each_rcu(sk, node, &ilb2->nulls_head) { |
| 162 |
score = compute_score(sk, net, hnum, daddr, dif, sdif); |
| 163 |
if (score > hiscore) { |
| 164 |
result = inet6_lookup_reuseport(net, sk, skb, doff, |
| 165 |
saddr, sport, daddr, hnum, inet6_ehashfn); |
| 166 |
if (result) |
| 167 |
return result; |
| 168 |
|
| 169 |
result = sk; |
| 170 |
hiscore = score; |
| 171 |
} |
| 172 |
} |
| 173 |
|
| 174 |
return result; |
| 175 |
} |
| 176 |
|
| 177 |
struct sock *inet6_lookup_run_sk_lookup(const struct net *net, |
| 178 |
int protocol, |
| 179 |
struct sk_buff *skb, int doff, |
| 180 |
const struct in6_addr *saddr, |
| 181 |
const __be16 sport, |
| 182 |
const struct in6_addr *daddr, |
| 183 |
const u16 hnum, const int dif, |
| 184 |
inet6_ehashfn_t *ehashfn) |
| 185 |
{ |
| 186 |
struct sock *sk, *reuse_sk; |
| 187 |
bool no_reuseport; |
| 188 |
|
| 189 |
no_reuseport = bpf_sk_lookup_run_v6(net, protocol, saddr, sport, |
| 190 |
daddr, hnum, dif, &sk); |
| 191 |
if (no_reuseport || IS_ERR_OR_NULL(sk)) |
| 192 |
return sk; |
| 193 |
|
| 194 |
reuse_sk = inet6_lookup_reuseport(net, sk, skb, doff, |
| 195 |
saddr, sport, daddr, hnum, ehashfn); |
| 196 |
if (reuse_sk) |
| 197 |
sk = reuse_sk; |
| 198 |
return sk; |
| 199 |
} |
| 200 |
EXPORT_SYMBOL_GPL(inet6_lookup_run_sk_lookup); |
| 201 |
|
| 202 |
struct sock *inet6_lookup_listener(const struct net *net, |
| 203 |
struct sk_buff *skb, int doff, |
| 204 |
const struct in6_addr *saddr, |
| 205 |
const __be16 sport, |
| 206 |
const struct in6_addr *daddr, |
| 207 |
const unsigned short hnum, |
| 208 |
const int dif, const int sdif) |
| 209 |
{ |
| 210 |
struct inet_listen_hashbucket *ilb2; |
| 211 |
struct inet_hashinfo *hashinfo; |
| 212 |
struct sock *result = NULL; |
| 213 |
unsigned int hash2; |
| 214 |
|
| 215 |
/* Lookup redirect from BPF */ |
| 216 |
if (static_branch_unlikely(&bpf_sk_lookup_enabled)) { |
| 217 |
result = inet6_lookup_run_sk_lookup(net, IPPROTO_TCP, skb, doff, |
| 218 |
saddr, sport, daddr, hnum, dif, |
| 219 |
inet6_ehashfn); |
| 220 |
if (result) |
| 221 |
goto done; |
| 222 |
} |
| 223 |
|
| 224 |
hashinfo = net->ipv4.tcp_death_row.hashinfo; |
| 225 |
hash2 = ipv6_portaddr_hash(net, daddr, hnum); |
| 226 |
ilb2 = inet_lhash2_bucket(hashinfo, hash2); |
| 227 |
|
| 228 |
result = inet6_lhash2_lookup(net, ilb2, skb, doff, |
| 229 |
saddr, sport, daddr, hnum, |
| 230 |
dif, sdif); |
| 231 |
if (result) |
| 232 |
goto done; |
| 233 |
|
| 234 |
/* Lookup lhash2 with in6addr_any */ |
| 235 |
hash2 = ipv6_portaddr_hash(net, &in6addr_any, hnum); |
| 236 |
ilb2 = inet_lhash2_bucket(hashinfo, hash2); |
| 237 |
|
| 238 |
result = inet6_lhash2_lookup(net, ilb2, skb, doff, |
| 239 |
saddr, sport, &in6addr_any, hnum, |
| 240 |
dif, sdif); |
| 241 |
done: |
| 242 |
if (IS_ERR(result)) |
| 243 |
return NULL; |
| 244 |
return result; |
| 245 |
} |
| 246 |
EXPORT_SYMBOL_GPL(inet6_lookup_listener); |
| 247 |
|
| 248 |
struct sock *inet6_lookup(const struct net *net, |
| 249 |
struct sk_buff *skb, int doff, |
| 250 |
const struct in6_addr *saddr, const __be16 sport, |
| 251 |
const struct in6_addr *daddr, const __be16 dport, |
| 252 |
const int dif) |
| 253 |
{ |
| 254 |
struct sock *sk; |
| 255 |
bool refcounted; |
| 256 |
|
| 257 |
sk = __inet6_lookup(net, skb, doff, saddr, sport, daddr, |
| 258 |
ntohs(dport), dif, 0, &refcounted); |
| 259 |
if (sk && !refcounted && !refcount_inc_not_zero(&sk->sk_refcnt)) |
| 260 |
sk = NULL; |
| 261 |
return sk; |
| 262 |
} |
| 263 |
EXPORT_SYMBOL_GPL(inet6_lookup); |
| 264 |
|
| 265 |
static int __inet6_check_established(struct inet_timewait_death_row *death_row, |
| 266 |
struct sock *sk, const __u16 lport, |
| 267 |
struct inet_timewait_sock **twp, |
| 268 |
bool rcu_lookup, |
| 269 |
u32 hash) |
| 270 |
{ |
| 271 |
struct inet_hashinfo *hinfo = death_row->hashinfo; |
| 272 |
struct inet_sock *inet = inet_sk(sk); |
| 273 |
const struct in6_addr *daddr = &sk->sk_v6_rcv_saddr; |
| 274 |
const struct in6_addr *saddr = &sk->sk_v6_daddr; |
| 275 |
const int dif = sk->sk_bound_dev_if; |
| 276 |
struct net *net = sock_net(sk); |
| 277 |
const int sdif = l3mdev_master_ifindex_by_index(net, dif); |
| 278 |
const __portpair ports = INET_COMBINED_PORTS(inet->inet_dport, lport); |
| 279 |
struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash); |
| 280 |
struct inet_timewait_sock *tw = NULL; |
| 281 |
const struct hlist_nulls_node *node; |
| 282 |
struct sock *sk2; |
| 283 |
spinlock_t *lock; |
| 284 |
|
| 285 |
if (rcu_lookup) { |
| 286 |
sk_nulls_for_each(sk2, node, &head->chain) { |
| 287 |
if (sk2->sk_hash != hash || |
| 288 |
!inet6_match(net, sk2, saddr, daddr, |
| 289 |
ports, dif, sdif)) |
| 290 |
continue; |
| 291 |
if (sk2->sk_state == TCP_TIME_WAIT) |
| 292 |
break; |
| 293 |
return -EADDRNOTAVAIL; |
| 294 |
} |
| 295 |
return 0; |
| 296 |
} |
| 297 |
|
| 298 |
lock = inet_ehash_lockp(hinfo, hash); |
| 299 |
spin_lock(lock); |
| 300 |
|
| 301 |
sk_nulls_for_each(sk2, node, &head->chain) { |
| 302 |
if (sk2->sk_hash != hash) |
| 303 |
continue; |
| 304 |
|
| 305 |
if (likely(inet6_match(net, sk2, saddr, daddr, ports, |
| 306 |
dif, sdif))) { |
| 307 |
if (sk2->sk_state == TCP_TIME_WAIT) { |
| 308 |
tw = inet_twsk(sk2); |
| 309 |
if (tcp_twsk_unique(sk, sk2, twp)) |
| 310 |
break; |
| 311 |
} |
| 312 |
goto not_unique; |
| 313 |
} |
| 314 |
} |
| 315 |
|
| 316 |
/* Must record num and sport now. Otherwise we will see |
| 317 |
* in hash table socket with a funny identity. |
| 318 |
*/ |
| 319 |
inet->inet_num = lport; |
| 320 |
inet->inet_sport = htons(lport); |
| 321 |
sk->sk_hash = hash; |
| 322 |
WARN_ON(!sk_unhashed(sk)); |
| 323 |
__sk_nulls_add_node_rcu(sk, &head->chain); |
| 324 |
if (tw) { |
| 325 |
sk_nulls_del_node_init_rcu((struct sock *)tw); |
| 326 |
__NET_INC_STATS(net, LINUX_MIB_TIMEWAITRECYCLED); |
| 327 |
} |
| 328 |
spin_unlock(lock); |
| 329 |
sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); |
| 330 |
|
| 331 |
if (twp) { |
| 332 |
*twp = tw; |
| 333 |
} else if (tw) { |
| 334 |
/* Silly. Should hash-dance instead... */ |
| 335 |
inet_twsk_deschedule_put(tw); |
| 336 |
} |
| 337 |
return 0; |
| 338 |
|
| 339 |
not_unique: |
| 340 |
spin_unlock(lock); |
| 341 |
return -EADDRNOTAVAIL; |
| 342 |
} |
| 343 |
|
| 344 |
static u64 inet6_sk_port_offset(const struct sock *sk) |
| 345 |
{ |
| 346 |
const struct inet_sock *inet = inet_sk(sk); |
| 347 |
|
| 348 |
return secure_ipv6_port_ephemeral(sk->sk_v6_rcv_saddr.s6_addr32, |
| 349 |
sk->sk_v6_daddr.s6_addr32, |
| 350 |
inet->inet_dport); |
| 351 |
} |
| 352 |
|
| 353 |
int inet6_hash_connect(struct inet_timewait_death_row *death_row, |
| 354 |
struct sock *sk) |
| 355 |
{ |
| 356 |
const struct in6_addr *daddr = &sk->sk_v6_rcv_saddr; |
| 357 |
const struct in6_addr *saddr = &sk->sk_v6_daddr; |
| 358 |
const struct inet_sock *inet = inet_sk(sk); |
| 359 |
const struct net *net = sock_net(sk); |
| 360 |
u64 port_offset = 0; |
| 361 |
u32 hash_port0; |
| 362 |
|
| 363 |
if (!inet_sk(sk)->inet_num) |
| 364 |
port_offset = inet6_sk_port_offset(sk); |
| 365 |
|
| 366 |
hash_port0 = inet6_ehashfn(net, daddr, 0, saddr, inet->inet_dport); |
| 367 |
|
| 368 |
return __inet_hash_connect(death_row, sk, port_offset, hash_port0, |
| 369 |
__inet6_check_established); |
| 370 |
} |
| 371 |
EXPORT_SYMBOL_GPL(inet6_hash_connect); |