| 1 |
// SPDX-License-Identifier: GPL-2.0-or-later |
| 2 |
/* |
| 3 |
* IPV6 GSO/GRO offload support |
| 4 |
* Linux INET6 implementation |
| 5 |
*/ |
| 6 |
|
| 7 |
#include <linux/kernel.h> |
| 8 |
#include <linux/socket.h> |
| 9 |
#include <linux/netdevice.h> |
| 10 |
#include <linux/skbuff.h> |
| 11 |
#include <linux/printk.h> |
| 12 |
|
| 13 |
#include <net/protocol.h> |
| 14 |
#include <net/ipv6.h> |
| 15 |
#include <net/inet_common.h> |
| 16 |
#include <net/tcp.h> |
| 17 |
#include <net/udp.h> |
| 18 |
#include <net/gro.h> |
| 19 |
#include <net/gso.h> |
| 20 |
|
| 21 |
#include "ip6_offload.h" |
| 22 |
#include "tcpv6_offload.c" |
| 23 |
|
| 24 |
static int ipv6_gro_pull_exthdrs(struct sk_buff *skb, int off, int proto) |
| 25 |
{ |
| 26 |
const struct net_offload *ops = NULL; |
| 27 |
struct ipv6_opt_hdr *opth; |
| 28 |
|
| 29 |
for (;;) { |
| 30 |
int len; |
| 31 |
|
| 32 |
ops = rcu_dereference(inet6_offloads[proto]); |
| 33 |
|
| 34 |
if (unlikely(!ops)) |
| 35 |
break; |
| 36 |
|
| 37 |
if (!(ops->flags & INET6_PROTO_GSO_EXTHDR)) |
| 38 |
break; |
| 39 |
|
| 40 |
opth = skb_gro_header(skb, off + sizeof(*opth), off); |
| 41 |
if (unlikely(!opth)) |
| 42 |
break; |
| 43 |
|
| 44 |
len = ipv6_optlen(opth); |
| 45 |
|
| 46 |
opth = skb_gro_header(skb, off + len, off); |
| 47 |
if (unlikely(!opth)) |
| 48 |
break; |
| 49 |
proto = opth->nexthdr; |
| 50 |
|
| 51 |
off += len; |
| 52 |
} |
| 53 |
|
| 54 |
skb_gro_pull(skb, off - skb_gro_receive_network_offset(skb)); |
| 55 |
return proto; |
| 56 |
} |
| 57 |
|
| 58 |
static int ipv6_gso_pull_exthdrs(struct sk_buff *skb, int proto) |
| 59 |
{ |
| 60 |
const struct net_offload *ops = NULL; |
| 61 |
|
| 62 |
for (;;) { |
| 63 |
struct ipv6_opt_hdr *opth; |
| 64 |
int len; |
| 65 |
|
| 66 |
ops = rcu_dereference(inet6_offloads[proto]); |
| 67 |
|
| 68 |
if (unlikely(!ops)) |
| 69 |
break; |
| 70 |
|
| 71 |
if (!(ops->flags & INET6_PROTO_GSO_EXTHDR)) |
| 72 |
break; |
| 73 |
|
| 74 |
if (unlikely(!pskb_may_pull(skb, 8))) |
| 75 |
break; |
| 76 |
|
| 77 |
opth = (void *)skb->data; |
| 78 |
len = ipv6_optlen(opth); |
| 79 |
|
| 80 |
if (unlikely(!pskb_may_pull(skb, len))) |
| 81 |
break; |
| 82 |
|
| 83 |
opth = (void *)skb->data; |
| 84 |
proto = opth->nexthdr; |
| 85 |
__skb_pull(skb, len); |
| 86 |
} |
| 87 |
|
| 88 |
return proto; |
| 89 |
} |
| 90 |
|
| 91 |
static struct sk_buff *ipv6_gso_segment(struct sk_buff *skb, |
| 92 |
netdev_features_t features) |
| 93 |
{ |
| 94 |
struct sk_buff *segs = ERR_PTR(-EINVAL); |
| 95 |
struct ipv6hdr *ipv6h; |
| 96 |
const struct net_offload *ops; |
| 97 |
int proto; |
| 98 |
struct frag_hdr *fptr; |
| 99 |
unsigned int payload_len; |
| 100 |
u8 *prevhdr; |
| 101 |
int offset = 0; |
| 102 |
bool encap, udpfrag; |
| 103 |
int nhoff; |
| 104 |
bool gso_partial; |
| 105 |
|
| 106 |
skb_reset_network_header(skb); |
| 107 |
nhoff = skb_network_header(skb) - skb_mac_header(skb); |
| 108 |
if (unlikely(!pskb_may_pull(skb, sizeof(*ipv6h)))) |
| 109 |
goto out; |
| 110 |
|
| 111 |
encap = SKB_GSO_CB(skb)->encap_level > 0; |
| 112 |
if (encap) |
| 113 |
features &= skb->dev->hw_enc_features; |
| 114 |
SKB_GSO_CB(skb)->encap_level += sizeof(*ipv6h); |
| 115 |
|
| 116 |
ipv6h = ipv6_hdr(skb); |
| 117 |
__skb_pull(skb, sizeof(*ipv6h)); |
| 118 |
segs = ERR_PTR(-EPROTONOSUPPORT); |
| 119 |
|
| 120 |
proto = ipv6_gso_pull_exthdrs(skb, ipv6h->nexthdr); |
| 121 |
|
| 122 |
if (skb->encapsulation && |
| 123 |
skb_shinfo(skb)->gso_type & (SKB_GSO_IPXIP4 | SKB_GSO_IPXIP6)) |
| 124 |
udpfrag = proto == IPPROTO_UDP && encap && |
| 125 |
(skb_shinfo(skb)->gso_type & SKB_GSO_UDP); |
| 126 |
else |
| 127 |
udpfrag = proto == IPPROTO_UDP && !skb->encapsulation && |
| 128 |
(skb_shinfo(skb)->gso_type & SKB_GSO_UDP); |
| 129 |
|
| 130 |
ops = rcu_dereference(inet6_offloads[proto]); |
| 131 |
if (likely(ops && ops->callbacks.gso_segment)) { |
| 132 |
if (!skb_reset_transport_header_careful(skb)) |
| 133 |
goto out; |
| 134 |
|
| 135 |
segs = ops->callbacks.gso_segment(skb, features); |
| 136 |
if (!segs) |
| 137 |
skb->network_header = skb_mac_header(skb) + nhoff - skb->head; |
| 138 |
} |
| 139 |
|
| 140 |
if (IS_ERR_OR_NULL(segs)) |
| 141 |
goto out; |
| 142 |
|
| 143 |
gso_partial = !!(skb_shinfo(segs)->gso_type & SKB_GSO_PARTIAL); |
| 144 |
|
| 145 |
for (skb = segs; skb; skb = skb->next) { |
| 146 |
ipv6h = (struct ipv6hdr *)(skb_mac_header(skb) + nhoff); |
| 147 |
if (gso_partial && skb_is_gso(skb)) |
| 148 |
payload_len = skb_shinfo(skb)->gso_size + |
| 149 |
SKB_GSO_CB(skb)->data_offset + |
| 150 |
skb->head - (unsigned char *)(ipv6h + 1); |
| 151 |
else |
| 152 |
payload_len = skb->len - nhoff - sizeof(*ipv6h); |
| 153 |
ipv6h->payload_len = htons(payload_len); |
| 154 |
skb->network_header = (u8 *)ipv6h - skb->head; |
| 155 |
skb_reset_mac_len(skb); |
| 156 |
|
| 157 |
if (udpfrag) { |
| 158 |
int err = ip6_find_1stfragopt(skb, &prevhdr); |
| 159 |
if (err < 0) { |
| 160 |
kfree_skb_list(segs); |
| 161 |
return ERR_PTR(err); |
| 162 |
} |
| 163 |
fptr = (struct frag_hdr *)((u8 *)ipv6h + err); |
| 164 |
fptr->frag_off = htons(offset); |
| 165 |
if (skb->next) |
| 166 |
fptr->frag_off |= htons(IP6_MF); |
| 167 |
offset += (ntohs(ipv6h->payload_len) - |
| 168 |
sizeof(struct frag_hdr)); |
| 169 |
} |
| 170 |
if (encap) |
| 171 |
skb_reset_inner_headers(skb); |
| 172 |
} |
| 173 |
|
| 174 |
out: |
| 175 |
return segs; |
| 176 |
} |
| 177 |
|
| 178 |
/* Return the total length of all the extension hdrs, following the same |
| 179 |
* logic in ipv6_gso_pull_exthdrs() when parsing ext-hdrs. |
| 180 |
*/ |
| 181 |
static int ipv6_exthdrs_len(struct ipv6hdr *iph, |
| 182 |
const struct net_offload **opps) |
| 183 |
{ |
| 184 |
struct ipv6_opt_hdr *opth = (void *)iph; |
| 185 |
int len = 0, proto, optlen = sizeof(*iph); |
| 186 |
|
| 187 |
proto = iph->nexthdr; |
| 188 |
for (;;) { |
| 189 |
*opps = rcu_dereference(inet6_offloads[proto]); |
| 190 |
if (unlikely(!(*opps))) |
| 191 |
break; |
| 192 |
if (!((*opps)->flags & INET6_PROTO_GSO_EXTHDR)) |
| 193 |
break; |
| 194 |
|
| 195 |
opth = (void *)opth + optlen; |
| 196 |
optlen = ipv6_optlen(opth); |
| 197 |
len += optlen; |
| 198 |
proto = opth->nexthdr; |
| 199 |
} |
| 200 |
return len; |
| 201 |
} |
| 202 |
|
| 203 |
INDIRECT_CALLABLE_SCOPE struct sk_buff *ipv6_gro_receive(struct list_head *head, |
| 204 |
struct sk_buff *skb) |
| 205 |
{ |
| 206 |
const struct net_offload *ops; |
| 207 |
struct sk_buff *pp = NULL; |
| 208 |
struct sk_buff *p; |
| 209 |
struct ipv6hdr *iph; |
| 210 |
unsigned int nlen; |
| 211 |
unsigned int hlen; |
| 212 |
unsigned int off; |
| 213 |
u16 flush = 1; |
| 214 |
int proto; |
| 215 |
|
| 216 |
off = skb_gro_offset(skb); |
| 217 |
hlen = off + sizeof(*iph); |
| 218 |
iph = skb_gro_header(skb, hlen, off); |
| 219 |
if (unlikely(!iph)) |
| 220 |
goto out; |
| 221 |
|
| 222 |
NAPI_GRO_CB(skb)->network_offsets[NAPI_GRO_CB(skb)->encap_mark] = off; |
| 223 |
|
| 224 |
flush += ntohs(iph->payload_len) != skb->len - hlen; |
| 225 |
|
| 226 |
proto = iph->nexthdr; |
| 227 |
ops = rcu_dereference(inet6_offloads[proto]); |
| 228 |
if (!ops || !ops->callbacks.gro_receive) { |
| 229 |
proto = ipv6_gro_pull_exthdrs(skb, hlen, proto); |
| 230 |
|
| 231 |
ops = rcu_dereference(inet6_offloads[proto]); |
| 232 |
if (!ops || !ops->callbacks.gro_receive) |
| 233 |
goto out; |
| 234 |
|
| 235 |
iph = skb_gro_network_header(skb); |
| 236 |
} else { |
| 237 |
skb_gro_pull(skb, sizeof(*iph)); |
| 238 |
} |
| 239 |
|
| 240 |
skb_set_transport_header(skb, skb_gro_offset(skb)); |
| 241 |
|
| 242 |
NAPI_GRO_CB(skb)->proto = proto; |
| 243 |
|
| 244 |
flush--; |
| 245 |
nlen = skb_gro_offset(skb) - off; |
| 246 |
|
| 247 |
list_for_each_entry(p, head, list) { |
| 248 |
const struct ipv6hdr *iph2; |
| 249 |
__be32 first_word; /* <Version:4><Traffic_Class:8><Flow_Label:20> */ |
| 250 |
|
| 251 |
if (!NAPI_GRO_CB(p)->same_flow) |
| 252 |
continue; |
| 253 |
|
| 254 |
iph2 = (struct ipv6hdr *)(p->data + off); |
| 255 |
first_word = *(__be32 *)iph ^ *(__be32 *)iph2; |
| 256 |
|
| 257 |
/* All fields must match except length and Traffic Class. |
| 258 |
* XXX skbs on the gro_list have all been parsed and pulled |
| 259 |
* already so we don't need to compare nlen |
| 260 |
* (nlen != (sizeof(*iph2) + ipv6_exthdrs_len(iph2, &ops))) |
| 261 |
* memcmp() alone below is sufficient, right? |
| 262 |
*/ |
| 263 |
if ((first_word & htonl(0xF00FFFFF)) || |
| 264 |
!ipv6_addr_equal(&iph->saddr, &iph2->saddr) || |
| 265 |
!ipv6_addr_equal(&iph->daddr, &iph2->daddr) || |
| 266 |
iph->nexthdr != iph2->nexthdr) { |
| 267 |
not_same_flow: |
| 268 |
NAPI_GRO_CB(p)->same_flow = 0; |
| 269 |
continue; |
| 270 |
} |
| 271 |
if (unlikely(nlen > sizeof(struct ipv6hdr))) { |
| 272 |
if (memcmp(iph + 1, iph2 + 1, |
| 273 |
nlen - sizeof(struct ipv6hdr))) |
| 274 |
goto not_same_flow; |
| 275 |
} |
| 276 |
} |
| 277 |
|
| 278 |
NAPI_GRO_CB(skb)->flush |= flush; |
| 279 |
|
| 280 |
skb_gro_postpull_rcsum(skb, iph, nlen); |
| 281 |
|
| 282 |
if (unlikely(gro_recursion_inc_test(skb))) { |
| 283 |
flush = 1; |
| 284 |
goto out; |
| 285 |
} |
| 286 |
|
| 287 |
if (likely(proto == IPPROTO_TCP)) |
| 288 |
pp = tcp6_gro_receive(head, skb); |
| 289 |
#if IS_BUILTIN(CONFIG_IPV6) |
| 290 |
else if (likely(proto == IPPROTO_UDP)) |
| 291 |
pp = udp6_gro_receive(head, skb); |
| 292 |
#endif |
| 293 |
else |
| 294 |
pp = ops->callbacks.gro_receive(head, skb); |
| 295 |
out: |
| 296 |
skb_gro_flush_final(skb, pp, flush); |
| 297 |
|
| 298 |
return pp; |
| 299 |
} |
| 300 |
|
| 301 |
static struct sk_buff *sit_ip6ip6_gro_receive(struct list_head *head, |
| 302 |
struct sk_buff *skb) |
| 303 |
{ |
| 304 |
/* Common GRO receive for SIT and IP6IP6 */ |
| 305 |
|
| 306 |
if (NAPI_GRO_CB(skb)->encap_mark) { |
| 307 |
NAPI_GRO_CB(skb)->flush = 1; |
| 308 |
return NULL; |
| 309 |
} |
| 310 |
|
| 311 |
NAPI_GRO_CB(skb)->encap_mark = 1; |
| 312 |
|
| 313 |
return ipv6_gro_receive(head, skb); |
| 314 |
} |
| 315 |
|
| 316 |
static struct sk_buff *ip4ip6_gro_receive(struct list_head *head, |
| 317 |
struct sk_buff *skb) |
| 318 |
{ |
| 319 |
/* Common GRO receive for SIT and IP6IP6 */ |
| 320 |
|
| 321 |
if (NAPI_GRO_CB(skb)->encap_mark) { |
| 322 |
NAPI_GRO_CB(skb)->flush = 1; |
| 323 |
return NULL; |
| 324 |
} |
| 325 |
|
| 326 |
NAPI_GRO_CB(skb)->encap_mark = 1; |
| 327 |
|
| 328 |
return inet_gro_receive(head, skb); |
| 329 |
} |
| 330 |
|
| 331 |
INDIRECT_CALLABLE_SCOPE int ipv6_gro_complete(struct sk_buff *skb, int nhoff) |
| 332 |
{ |
| 333 |
const struct net_offload *ops; |
| 334 |
struct ipv6hdr *iph; |
| 335 |
int err = -ENOSYS; |
| 336 |
|
| 337 |
if (skb->encapsulation) { |
| 338 |
skb_set_inner_protocol(skb, cpu_to_be16(ETH_P_IPV6)); |
| 339 |
skb_set_inner_network_header(skb, nhoff); |
| 340 |
} |
| 341 |
|
| 342 |
iph = (struct ipv6hdr *)(skb->data + nhoff); |
| 343 |
ipv6_set_payload_len(iph, skb->len - nhoff - sizeof(*iph)); |
| 344 |
|
| 345 |
nhoff += sizeof(*iph) + ipv6_exthdrs_len(iph, &ops); |
| 346 |
|
| 347 |
if (likely(ops == &net_hotdata.tcpv6_offload)) |
| 348 |
return tcp6_gro_complete(skb, nhoff); |
| 349 |
#if IS_BUILTIN(CONFIG_IPV6) |
| 350 |
if (ops == &net_hotdata.udpv6_offload) |
| 351 |
return udp6_gro_complete(skb, nhoff); |
| 352 |
#endif |
| 353 |
|
| 354 |
if (WARN_ON(!ops || !ops->callbacks.gro_complete)) |
| 355 |
goto out; |
| 356 |
|
| 357 |
err = ops->callbacks.gro_complete(skb, nhoff); |
| 358 |
|
| 359 |
out: |
| 360 |
return err; |
| 361 |
} |
| 362 |
|
| 363 |
static int sit_gro_complete(struct sk_buff *skb, int nhoff) |
| 364 |
{ |
| 365 |
skb->encapsulation = 1; |
| 366 |
skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP4; |
| 367 |
return ipv6_gro_complete(skb, nhoff); |
| 368 |
} |
| 369 |
|
| 370 |
static int ip6ip6_gro_complete(struct sk_buff *skb, int nhoff) |
| 371 |
{ |
| 372 |
skb->encapsulation = 1; |
| 373 |
skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6; |
| 374 |
return ipv6_gro_complete(skb, nhoff); |
| 375 |
} |
| 376 |
|
| 377 |
static int ip4ip6_gro_complete(struct sk_buff *skb, int nhoff) |
| 378 |
{ |
| 379 |
skb->encapsulation = 1; |
| 380 |
skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6; |
| 381 |
return inet_gro_complete(skb, nhoff); |
| 382 |
} |
| 383 |
|
| 384 |
|
| 385 |
static struct sk_buff *sit_gso_segment(struct sk_buff *skb, |
| 386 |
netdev_features_t features) |
| 387 |
{ |
| 388 |
if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP4)) |
| 389 |
return ERR_PTR(-EINVAL); |
| 390 |
|
| 391 |
return ipv6_gso_segment(skb, features); |
| 392 |
} |
| 393 |
|
| 394 |
static struct sk_buff *ip4ip6_gso_segment(struct sk_buff *skb, |
| 395 |
netdev_features_t features) |
| 396 |
{ |
| 397 |
if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP6)) |
| 398 |
return ERR_PTR(-EINVAL); |
| 399 |
|
| 400 |
return inet_gso_segment(skb, features); |
| 401 |
} |
| 402 |
|
| 403 |
static struct sk_buff *ip6ip6_gso_segment(struct sk_buff *skb, |
| 404 |
netdev_features_t features) |
| 405 |
{ |
| 406 |
if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP6)) |
| 407 |
return ERR_PTR(-EINVAL); |
| 408 |
|
| 409 |
return ipv6_gso_segment(skb, features); |
| 410 |
} |
| 411 |
|
| 412 |
static const struct net_offload sit_offload = { |
| 413 |
.callbacks = { |
| 414 |
.gso_segment = sit_gso_segment, |
| 415 |
.gro_receive = sit_ip6ip6_gro_receive, |
| 416 |
.gro_complete = sit_gro_complete, |
| 417 |
}, |
| 418 |
}; |
| 419 |
|
| 420 |
static const struct net_offload ip4ip6_offload = { |
| 421 |
.callbacks = { |
| 422 |
.gso_segment = ip4ip6_gso_segment, |
| 423 |
.gro_receive = ip4ip6_gro_receive, |
| 424 |
.gro_complete = ip4ip6_gro_complete, |
| 425 |
}, |
| 426 |
}; |
| 427 |
|
| 428 |
static const struct net_offload ip6ip6_offload = { |
| 429 |
.callbacks = { |
| 430 |
.gso_segment = ip6ip6_gso_segment, |
| 431 |
.gro_receive = sit_ip6ip6_gro_receive, |
| 432 |
.gro_complete = ip6ip6_gro_complete, |
| 433 |
}, |
| 434 |
}; |
| 435 |
static int __init ipv6_offload_init(void) |
| 436 |
{ |
| 437 |
|
| 438 |
if (tcpv6_offload_init() < 0) |
| 439 |
pr_crit("%s: Cannot add TCP protocol offload\n", __func__); |
| 440 |
if (ipv6_exthdrs_offload_init() < 0) |
| 441 |
pr_crit("%s: Cannot add EXTHDRS protocol offload\n", __func__); |
| 442 |
|
| 443 |
net_hotdata.ipv6_packet_offload = (struct packet_offload) { |
| 444 |
.type = cpu_to_be16(ETH_P_IPV6), |
| 445 |
.callbacks = { |
| 446 |
.gso_segment = ipv6_gso_segment, |
| 447 |
.gro_receive = ipv6_gro_receive, |
| 448 |
.gro_complete = ipv6_gro_complete, |
| 449 |
}, |
| 450 |
}; |
| 451 |
dev_add_offload(&net_hotdata.ipv6_packet_offload); |
| 452 |
|
| 453 |
inet_add_offload(&sit_offload, IPPROTO_IPV6); |
| 454 |
inet6_add_offload(&ip6ip6_offload, IPPROTO_IPV6); |
| 455 |
inet6_add_offload(&ip4ip6_offload, IPPROTO_IPIP); |
| 456 |
|
| 457 |
return 0; |
| 458 |
} |
| 459 |
|
| 460 |
fs_initcall(ipv6_offload_init); |