| 1 |
// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) |
| 2 |
/* Copyright (c) 2018 Facebook */ |
| 3 |
|
| 4 |
#include <stdlib.h> |
| 5 |
#include <memory.h> |
| 6 |
#include <unistd.h> |
| 7 |
#include <arpa/inet.h> |
| 8 |
#include <linux/bpf.h> |
| 9 |
#include <linux/if_ether.h> |
| 10 |
#include <linux/pkt_cls.h> |
| 11 |
#include <linux/rtnetlink.h> |
| 12 |
#include <linux/netdev.h> |
| 13 |
#include <sys/socket.h> |
| 14 |
#include <errno.h> |
| 15 |
#include <time.h> |
| 16 |
|
| 17 |
#include "bpf.h" |
| 18 |
#include "libbpf.h" |
| 19 |
#include "libbpf_internal.h" |
| 20 |
#include "nlattr.h" |
| 21 |
|
| 22 |
#ifndef SOL_NETLINK |
| 23 |
#define SOL_NETLINK 270 |
| 24 |
#endif |
| 25 |
|
| 26 |
typedef int (*libbpf_dump_nlmsg_t)(void *cookie, void *msg, struct nlattr **tb); |
| 27 |
|
| 28 |
typedef int (*__dump_nlmsg_t)(struct nlmsghdr *nlmsg, libbpf_dump_nlmsg_t, |
| 29 |
void *cookie); |
| 30 |
|
| 31 |
struct xdp_link_info { |
| 32 |
__u32 prog_id; |
| 33 |
__u32 drv_prog_id; |
| 34 |
__u32 hw_prog_id; |
| 35 |
__u32 skb_prog_id; |
| 36 |
__u8 attach_mode; |
| 37 |
}; |
| 38 |
|
| 39 |
struct xdp_id_md { |
| 40 |
int ifindex; |
| 41 |
__u32 flags; |
| 42 |
struct xdp_link_info info; |
| 43 |
__u64 feature_flags; |
| 44 |
}; |
| 45 |
|
| 46 |
struct xdp_features_md { |
| 47 |
int ifindex; |
| 48 |
__u32 xdp_zc_max_segs; |
| 49 |
__u64 flags; |
| 50 |
}; |
| 51 |
|
| 52 |
static int libbpf_netlink_open(__u32 *nl_pid, int proto) |
| 53 |
{ |
| 54 |
struct sockaddr_nl sa; |
| 55 |
socklen_t addrlen; |
| 56 |
int one = 1, ret; |
| 57 |
int sock; |
| 58 |
|
| 59 |
memset(&sa, 0, sizeof(sa)); |
| 60 |
sa.nl_family = AF_NETLINK; |
| 61 |
|
| 62 |
sock = socket(AF_NETLINK, SOCK_RAW | SOCK_CLOEXEC, proto); |
| 63 |
if (sock < 0) |
| 64 |
return -errno; |
| 65 |
|
| 66 |
if (setsockopt(sock, SOL_NETLINK, NETLINK_EXT_ACK, |
| 67 |
&one, sizeof(one)) < 0) { |
| 68 |
pr_warn("Netlink error reporting not supported\n"); |
| 69 |
} |
| 70 |
|
| 71 |
if (bind(sock, (struct sockaddr *)&sa, sizeof(sa)) < 0) { |
| 72 |
ret = -errno; |
| 73 |
goto cleanup; |
| 74 |
} |
| 75 |
|
| 76 |
addrlen = sizeof(sa); |
| 77 |
if (getsockname(sock, (struct sockaddr *)&sa, &addrlen) < 0) { |
| 78 |
ret = -errno; |
| 79 |
goto cleanup; |
| 80 |
} |
| 81 |
|
| 82 |
if (addrlen != sizeof(sa)) { |
| 83 |
ret = -LIBBPF_ERRNO__INTERNAL; |
| 84 |
goto cleanup; |
| 85 |
} |
| 86 |
|
| 87 |
*nl_pid = sa.nl_pid; |
| 88 |
return sock; |
| 89 |
|
| 90 |
cleanup: |
| 91 |
close(sock); |
| 92 |
return ret; |
| 93 |
} |
| 94 |
|
| 95 |
static void libbpf_netlink_close(int sock) |
| 96 |
{ |
| 97 |
close(sock); |
| 98 |
} |
| 99 |
|
| 100 |
enum { |
| 101 |
NL_CONT, |
| 102 |
NL_NEXT, |
| 103 |
NL_DONE, |
| 104 |
}; |
| 105 |
|
| 106 |
static int netlink_recvmsg(int sock, struct msghdr *mhdr, int flags) |
| 107 |
{ |
| 108 |
int len; |
| 109 |
|
| 110 |
do { |
| 111 |
len = recvmsg(sock, mhdr, flags); |
| 112 |
} while (len < 0 && (errno == EINTR || errno == EAGAIN)); |
| 113 |
|
| 114 |
if (len < 0) |
| 115 |
return -errno; |
| 116 |
return len; |
| 117 |
} |
| 118 |
|
| 119 |
static int alloc_iov(struct iovec *iov, int len) |
| 120 |
{ |
| 121 |
void *nbuf; |
| 122 |
|
| 123 |
nbuf = realloc(iov->iov_base, len); |
| 124 |
if (!nbuf) |
| 125 |
return -ENOMEM; |
| 126 |
|
| 127 |
iov->iov_base = nbuf; |
| 128 |
iov->iov_len = len; |
| 129 |
return 0; |
| 130 |
} |
| 131 |
|
| 132 |
static int libbpf_netlink_recv(int sock, __u32 nl_pid, int seq, |
| 133 |
__dump_nlmsg_t _fn, libbpf_dump_nlmsg_t fn, |
| 134 |
void *cookie) |
| 135 |
{ |
| 136 |
struct iovec iov = {}; |
| 137 |
struct msghdr mhdr = { |
| 138 |
.msg_iov = &iov, |
| 139 |
.msg_iovlen = 1, |
| 140 |
}; |
| 141 |
bool multipart = true; |
| 142 |
struct nlmsgerr *err; |
| 143 |
struct nlmsghdr *nh; |
| 144 |
int len, ret; |
| 145 |
|
| 146 |
ret = alloc_iov(&iov, 8192); |
| 147 |
if (ret) |
| 148 |
goto done; |
| 149 |
|
| 150 |
while (multipart) { |
| 151 |
start: |
| 152 |
multipart = false; |
| 153 |
len = netlink_recvmsg(sock, &mhdr, MSG_PEEK | MSG_TRUNC); |
| 154 |
if (len < 0) { |
| 155 |
ret = len; |
| 156 |
goto done; |
| 157 |
} |
| 158 |
|
| 159 |
if (len > iov.iov_len) { |
| 160 |
ret = alloc_iov(&iov, len); |
| 161 |
if (ret) |
| 162 |
goto done; |
| 163 |
} |
| 164 |
|
| 165 |
len = netlink_recvmsg(sock, &mhdr, 0); |
| 166 |
if (len < 0) { |
| 167 |
ret = len; |
| 168 |
goto done; |
| 169 |
} |
| 170 |
|
| 171 |
if (len == 0) |
| 172 |
break; |
| 173 |
|
| 174 |
for (nh = (struct nlmsghdr *)iov.iov_base; NLMSG_OK(nh, len); |
| 175 |
nh = NLMSG_NEXT(nh, len)) { |
| 176 |
if (nh->nlmsg_pid != nl_pid) { |
| 177 |
ret = -LIBBPF_ERRNO__WRNGPID; |
| 178 |
goto done; |
| 179 |
} |
| 180 |
if (nh->nlmsg_seq != seq) { |
| 181 |
ret = -LIBBPF_ERRNO__INVSEQ; |
| 182 |
goto done; |
| 183 |
} |
| 184 |
if (nh->nlmsg_flags & NLM_F_MULTI) |
| 185 |
multipart = true; |
| 186 |
switch (nh->nlmsg_type) { |
| 187 |
case NLMSG_ERROR: |
| 188 |
err = (struct nlmsgerr *)NLMSG_DATA(nh); |
| 189 |
if (!err->error) |
| 190 |
continue; |
| 191 |
ret = err->error; |
| 192 |
libbpf_nla_dump_errormsg(nh); |
| 193 |
goto done; |
| 194 |
case NLMSG_DONE: |
| 195 |
ret = 0; |
| 196 |
goto done; |
| 197 |
default: |
| 198 |
break; |
| 199 |
} |
| 200 |
if (_fn) { |
| 201 |
ret = _fn(nh, fn, cookie); |
| 202 |
switch (ret) { |
| 203 |
case NL_CONT: |
| 204 |
break; |
| 205 |
case NL_NEXT: |
| 206 |
goto start; |
| 207 |
case NL_DONE: |
| 208 |
ret = 0; |
| 209 |
goto done; |
| 210 |
default: |
| 211 |
goto done; |
| 212 |
} |
| 213 |
} |
| 214 |
} |
| 215 |
if (len) |
| 216 |
pr_warn("Invalid message or trailing data in Netlink response: %d bytes left\n", len); |
| 217 |
} |
| 218 |
ret = 0; |
| 219 |
done: |
| 220 |
free(iov.iov_base); |
| 221 |
return ret; |
| 222 |
} |
| 223 |
|
| 224 |
static int libbpf_netlink_send_recv(struct libbpf_nla_req *req, |
| 225 |
int proto, __dump_nlmsg_t parse_msg, |
| 226 |
libbpf_dump_nlmsg_t parse_attr, |
| 227 |
void *cookie) |
| 228 |
{ |
| 229 |
__u32 nl_pid = 0; |
| 230 |
int sock, ret; |
| 231 |
|
| 232 |
sock = libbpf_netlink_open(&nl_pid, proto); |
| 233 |
if (sock < 0) |
| 234 |
return sock; |
| 235 |
|
| 236 |
req->nh.nlmsg_pid = 0; |
| 237 |
req->nh.nlmsg_seq = time(NULL); |
| 238 |
|
| 239 |
if (send(sock, req, req->nh.nlmsg_len, 0) < 0) { |
| 240 |
ret = -errno; |
| 241 |
goto out; |
| 242 |
} |
| 243 |
|
| 244 |
ret = libbpf_netlink_recv(sock, nl_pid, req->nh.nlmsg_seq, |
| 245 |
parse_msg, parse_attr, cookie); |
| 246 |
out: |
| 247 |
libbpf_netlink_close(sock); |
| 248 |
return ret; |
| 249 |
} |
| 250 |
|
| 251 |
static int parse_genl_family_id(struct nlmsghdr *nh, libbpf_dump_nlmsg_t fn, |
| 252 |
void *cookie) |
| 253 |
{ |
| 254 |
struct genlmsghdr *gnl = NLMSG_DATA(nh); |
| 255 |
struct nlattr *na = (struct nlattr *)((void *)gnl + GENL_HDRLEN); |
| 256 |
struct nlattr *tb[CTRL_ATTR_FAMILY_ID + 1]; |
| 257 |
__u16 *id = cookie; |
| 258 |
|
| 259 |
libbpf_nla_parse(tb, CTRL_ATTR_FAMILY_ID, na, |
| 260 |
NLMSG_PAYLOAD(nh, sizeof(*gnl)), NULL); |
| 261 |
if (!tb[CTRL_ATTR_FAMILY_ID]) |
| 262 |
return NL_CONT; |
| 263 |
|
| 264 |
*id = libbpf_nla_getattr_u16(tb[CTRL_ATTR_FAMILY_ID]); |
| 265 |
return NL_DONE; |
| 266 |
} |
| 267 |
|
| 268 |
static int libbpf_netlink_resolve_genl_family_id(const char *name, |
| 269 |
__u16 len, __u16 *id) |
| 270 |
{ |
| 271 |
struct libbpf_nla_req req = { |
| 272 |
.nh.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN), |
| 273 |
.nh.nlmsg_type = GENL_ID_CTRL, |
| 274 |
.nh.nlmsg_flags = NLM_F_REQUEST, |
| 275 |
.gnl.cmd = CTRL_CMD_GETFAMILY, |
| 276 |
.gnl.version = 2, |
| 277 |
}; |
| 278 |
int err; |
| 279 |
|
| 280 |
err = nlattr_add(&req, CTRL_ATTR_FAMILY_NAME, name, len); |
| 281 |
if (err < 0) |
| 282 |
return err; |
| 283 |
|
| 284 |
return libbpf_netlink_send_recv(&req, NETLINK_GENERIC, |
| 285 |
parse_genl_family_id, NULL, id); |
| 286 |
} |
| 287 |
|
| 288 |
static int __bpf_set_link_xdp_fd_replace(int ifindex, int fd, int old_fd, |
| 289 |
__u32 flags) |
| 290 |
{ |
| 291 |
struct nlattr *nla; |
| 292 |
int ret; |
| 293 |
struct libbpf_nla_req req; |
| 294 |
|
| 295 |
memset(&req, 0, sizeof(req)); |
| 296 |
req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(struct ifinfomsg)); |
| 297 |
req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; |
| 298 |
req.nh.nlmsg_type = RTM_SETLINK; |
| 299 |
req.ifinfo.ifi_family = AF_UNSPEC; |
| 300 |
req.ifinfo.ifi_index = ifindex; |
| 301 |
|
| 302 |
nla = nlattr_begin_nested(&req, IFLA_XDP); |
| 303 |
if (!nla) |
| 304 |
return -EMSGSIZE; |
| 305 |
ret = nlattr_add(&req, IFLA_XDP_FD, &fd, sizeof(fd)); |
| 306 |
if (ret < 0) |
| 307 |
return ret; |
| 308 |
if (flags) { |
| 309 |
ret = nlattr_add(&req, IFLA_XDP_FLAGS, &flags, sizeof(flags)); |
| 310 |
if (ret < 0) |
| 311 |
return ret; |
| 312 |
} |
| 313 |
if (flags & XDP_FLAGS_REPLACE) { |
| 314 |
ret = nlattr_add(&req, IFLA_XDP_EXPECTED_FD, &old_fd, |
| 315 |
sizeof(old_fd)); |
| 316 |
if (ret < 0) |
| 317 |
return ret; |
| 318 |
} |
| 319 |
nlattr_end_nested(&req, nla); |
| 320 |
|
| 321 |
return libbpf_netlink_send_recv(&req, NETLINK_ROUTE, NULL, NULL, NULL); |
| 322 |
} |
| 323 |
|
| 324 |
int bpf_xdp_attach(int ifindex, int prog_fd, __u32 flags, const struct bpf_xdp_attach_opts *opts) |
| 325 |
{ |
| 326 |
int old_prog_fd, err; |
| 327 |
|
| 328 |
if (!OPTS_VALID(opts, bpf_xdp_attach_opts)) |
| 329 |
return libbpf_err(-EINVAL); |
| 330 |
|
| 331 |
old_prog_fd = OPTS_GET(opts, old_prog_fd, 0); |
| 332 |
if (old_prog_fd) |
| 333 |
flags |= XDP_FLAGS_REPLACE; |
| 334 |
else |
| 335 |
old_prog_fd = -1; |
| 336 |
|
| 337 |
err = __bpf_set_link_xdp_fd_replace(ifindex, prog_fd, old_prog_fd, flags); |
| 338 |
return libbpf_err(err); |
| 339 |
} |
| 340 |
|
| 341 |
int bpf_xdp_detach(int ifindex, __u32 flags, const struct bpf_xdp_attach_opts *opts) |
| 342 |
{ |
| 343 |
return bpf_xdp_attach(ifindex, -1, flags, opts); |
| 344 |
} |
| 345 |
|
| 346 |
static int __dump_link_nlmsg(struct nlmsghdr *nlh, |
| 347 |
libbpf_dump_nlmsg_t dump_link_nlmsg, void *cookie) |
| 348 |
{ |
| 349 |
struct nlattr *tb[IFLA_MAX + 1], *attr; |
| 350 |
struct ifinfomsg *ifi = NLMSG_DATA(nlh); |
| 351 |
int len; |
| 352 |
|
| 353 |
len = nlh->nlmsg_len - NLMSG_LENGTH(sizeof(*ifi)); |
| 354 |
attr = (struct nlattr *) ((void *) ifi + NLMSG_ALIGN(sizeof(*ifi))); |
| 355 |
|
| 356 |
if (libbpf_nla_parse(tb, IFLA_MAX, attr, len, NULL) != 0) |
| 357 |
return -LIBBPF_ERRNO__NLPARSE; |
| 358 |
|
| 359 |
return dump_link_nlmsg(cookie, ifi, tb); |
| 360 |
} |
| 361 |
|
| 362 |
static int get_xdp_info(void *cookie, void *msg, struct nlattr **tb) |
| 363 |
{ |
| 364 |
struct nlattr *xdp_tb[IFLA_XDP_MAX + 1]; |
| 365 |
struct xdp_id_md *xdp_id = cookie; |
| 366 |
struct ifinfomsg *ifinfo = msg; |
| 367 |
int ret; |
| 368 |
|
| 369 |
if (xdp_id->ifindex && xdp_id->ifindex != ifinfo->ifi_index) |
| 370 |
return 0; |
| 371 |
|
| 372 |
if (!tb[IFLA_XDP]) |
| 373 |
return 0; |
| 374 |
|
| 375 |
ret = libbpf_nla_parse_nested(xdp_tb, IFLA_XDP_MAX, tb[IFLA_XDP], NULL); |
| 376 |
if (ret) |
| 377 |
return ret; |
| 378 |
|
| 379 |
if (!xdp_tb[IFLA_XDP_ATTACHED]) |
| 380 |
return 0; |
| 381 |
|
| 382 |
xdp_id->info.attach_mode = libbpf_nla_getattr_u8( |
| 383 |
xdp_tb[IFLA_XDP_ATTACHED]); |
| 384 |
|
| 385 |
if (xdp_id->info.attach_mode == XDP_ATTACHED_NONE) |
| 386 |
return 0; |
| 387 |
|
| 388 |
if (xdp_tb[IFLA_XDP_PROG_ID]) |
| 389 |
xdp_id->info.prog_id = libbpf_nla_getattr_u32( |
| 390 |
xdp_tb[IFLA_XDP_PROG_ID]); |
| 391 |
|
| 392 |
if (xdp_tb[IFLA_XDP_SKB_PROG_ID]) |
| 393 |
xdp_id->info.skb_prog_id = libbpf_nla_getattr_u32( |
| 394 |
xdp_tb[IFLA_XDP_SKB_PROG_ID]); |
| 395 |
|
| 396 |
if (xdp_tb[IFLA_XDP_DRV_PROG_ID]) |
| 397 |
xdp_id->info.drv_prog_id = libbpf_nla_getattr_u32( |
| 398 |
xdp_tb[IFLA_XDP_DRV_PROG_ID]); |
| 399 |
|
| 400 |
if (xdp_tb[IFLA_XDP_HW_PROG_ID]) |
| 401 |
xdp_id->info.hw_prog_id = libbpf_nla_getattr_u32( |
| 402 |
xdp_tb[IFLA_XDP_HW_PROG_ID]); |
| 403 |
|
| 404 |
return 0; |
| 405 |
} |
| 406 |
|
| 407 |
static int parse_xdp_features(struct nlmsghdr *nh, libbpf_dump_nlmsg_t fn, |
| 408 |
void *cookie) |
| 409 |
{ |
| 410 |
struct genlmsghdr *gnl = NLMSG_DATA(nh); |
| 411 |
struct nlattr *na = (struct nlattr *)((void *)gnl + GENL_HDRLEN); |
| 412 |
struct nlattr *tb[NETDEV_CMD_MAX + 1]; |
| 413 |
struct xdp_features_md *md = cookie; |
| 414 |
__u32 ifindex; |
| 415 |
|
| 416 |
libbpf_nla_parse(tb, NETDEV_CMD_MAX, na, |
| 417 |
NLMSG_PAYLOAD(nh, sizeof(*gnl)), NULL); |
| 418 |
|
| 419 |
if (!tb[NETDEV_A_DEV_IFINDEX] || !tb[NETDEV_A_DEV_XDP_FEATURES]) |
| 420 |
return NL_CONT; |
| 421 |
|
| 422 |
ifindex = libbpf_nla_getattr_u32(tb[NETDEV_A_DEV_IFINDEX]); |
| 423 |
if (ifindex != md->ifindex) |
| 424 |
return NL_CONT; |
| 425 |
|
| 426 |
md->flags = libbpf_nla_getattr_u64(tb[NETDEV_A_DEV_XDP_FEATURES]); |
| 427 |
if (tb[NETDEV_A_DEV_XDP_ZC_MAX_SEGS]) |
| 428 |
md->xdp_zc_max_segs = |
| 429 |
libbpf_nla_getattr_u32(tb[NETDEV_A_DEV_XDP_ZC_MAX_SEGS]); |
| 430 |
return NL_DONE; |
| 431 |
} |
| 432 |
|
| 433 |
int bpf_xdp_query(int ifindex, int xdp_flags, struct bpf_xdp_query_opts *opts) |
| 434 |
{ |
| 435 |
struct libbpf_nla_req req = { |
| 436 |
.nh.nlmsg_len = NLMSG_LENGTH(sizeof(struct ifinfomsg)), |
| 437 |
.nh.nlmsg_type = RTM_GETLINK, |
| 438 |
.nh.nlmsg_flags = NLM_F_DUMP | NLM_F_REQUEST, |
| 439 |
.ifinfo.ifi_family = AF_PACKET, |
| 440 |
}; |
| 441 |
struct xdp_id_md xdp_id = {}; |
| 442 |
struct xdp_features_md md = { |
| 443 |
.ifindex = ifindex, |
| 444 |
}; |
| 445 |
__u16 id; |
| 446 |
int err; |
| 447 |
|
| 448 |
if (!OPTS_VALID(opts, bpf_xdp_query_opts)) |
| 449 |
return libbpf_err(-EINVAL); |
| 450 |
|
| 451 |
if (xdp_flags & ~XDP_FLAGS_MASK) |
| 452 |
return libbpf_err(-EINVAL); |
| 453 |
|
| 454 |
/* Check whether the single {HW,DRV,SKB} mode is set */ |
| 455 |
xdp_flags &= XDP_FLAGS_SKB_MODE | XDP_FLAGS_DRV_MODE | XDP_FLAGS_HW_MODE; |
| 456 |
if (xdp_flags & (xdp_flags - 1)) |
| 457 |
return libbpf_err(-EINVAL); |
| 458 |
|
| 459 |
xdp_id.ifindex = ifindex; |
| 460 |
xdp_id.flags = xdp_flags; |
| 461 |
|
| 462 |
err = libbpf_netlink_send_recv(&req, NETLINK_ROUTE, __dump_link_nlmsg, |
| 463 |
get_xdp_info, &xdp_id); |
| 464 |
if (err) |
| 465 |
return libbpf_err(err); |
| 466 |
|
| 467 |
OPTS_SET(opts, prog_id, xdp_id.info.prog_id); |
| 468 |
OPTS_SET(opts, drv_prog_id, xdp_id.info.drv_prog_id); |
| 469 |
OPTS_SET(opts, hw_prog_id, xdp_id.info.hw_prog_id); |
| 470 |
OPTS_SET(opts, skb_prog_id, xdp_id.info.skb_prog_id); |
| 471 |
OPTS_SET(opts, attach_mode, xdp_id.info.attach_mode); |
| 472 |
|
| 473 |
if (!OPTS_HAS(opts, feature_flags)) |
| 474 |
return 0; |
| 475 |
|
| 476 |
err = libbpf_netlink_resolve_genl_family_id("netdev", sizeof("netdev"), &id); |
| 477 |
if (err < 0) { |
| 478 |
if (err == -ENOENT) { |
| 479 |
opts->feature_flags = 0; |
| 480 |
goto skip_feature_flags; |
| 481 |
} |
| 482 |
return libbpf_err(err); |
| 483 |
} |
| 484 |
|
| 485 |
memset(&req, 0, sizeof(req)); |
| 486 |
req.nh.nlmsg_len = NLMSG_LENGTH(GENL_HDRLEN); |
| 487 |
req.nh.nlmsg_flags = NLM_F_REQUEST; |
| 488 |
req.nh.nlmsg_type = id; |
| 489 |
req.gnl.cmd = NETDEV_CMD_DEV_GET; |
| 490 |
req.gnl.version = 2; |
| 491 |
|
| 492 |
err = nlattr_add(&req, NETDEV_A_DEV_IFINDEX, &ifindex, sizeof(ifindex)); |
| 493 |
if (err < 0) |
| 494 |
return libbpf_err(err); |
| 495 |
|
| 496 |
err = libbpf_netlink_send_recv(&req, NETLINK_GENERIC, |
| 497 |
parse_xdp_features, NULL, &md); |
| 498 |
if (err) |
| 499 |
return libbpf_err(err); |
| 500 |
|
| 501 |
OPTS_SET(opts, feature_flags, md.flags); |
| 502 |
OPTS_SET(opts, xdp_zc_max_segs, md.xdp_zc_max_segs); |
| 503 |
|
| 504 |
skip_feature_flags: |
| 505 |
return 0; |
| 506 |
} |
| 507 |
|
| 508 |
int bpf_xdp_query_id(int ifindex, int flags, __u32 *prog_id) |
| 509 |
{ |
| 510 |
LIBBPF_OPTS(bpf_xdp_query_opts, opts); |
| 511 |
int ret; |
| 512 |
|
| 513 |
ret = bpf_xdp_query(ifindex, flags, &opts); |
| 514 |
if (ret) |
| 515 |
return libbpf_err(ret); |
| 516 |
|
| 517 |
flags &= XDP_FLAGS_MODES; |
| 518 |
|
| 519 |
if (opts.attach_mode != XDP_ATTACHED_MULTI && !flags) |
| 520 |
*prog_id = opts.prog_id; |
| 521 |
else if (flags & XDP_FLAGS_DRV_MODE) |
| 522 |
*prog_id = opts.drv_prog_id; |
| 523 |
else if (flags & XDP_FLAGS_HW_MODE) |
| 524 |
*prog_id = opts.hw_prog_id; |
| 525 |
else if (flags & XDP_FLAGS_SKB_MODE) |
| 526 |
*prog_id = opts.skb_prog_id; |
| 527 |
else |
| 528 |
*prog_id = 0; |
| 529 |
|
| 530 |
return 0; |
| 531 |
} |
| 532 |
|
| 533 |
|
| 534 |
typedef int (*qdisc_config_t)(struct libbpf_nla_req *req, const struct bpf_tc_hook *hook); |
| 535 |
|
| 536 |
static int clsact_config(struct libbpf_nla_req *req, const struct bpf_tc_hook *hook) |
| 537 |
{ |
| 538 |
req->tc.tcm_parent = TC_H_CLSACT; |
| 539 |
req->tc.tcm_handle = TC_H_MAKE(TC_H_CLSACT, 0); |
| 540 |
|
| 541 |
return nlattr_add(req, TCA_KIND, "clsact", sizeof("clsact")); |
| 542 |
} |
| 543 |
|
| 544 |
static int qdisc_config(struct libbpf_nla_req *req, const struct bpf_tc_hook *hook) |
| 545 |
{ |
| 546 |
const char *qdisc = OPTS_GET(hook, qdisc, NULL); |
| 547 |
|
| 548 |
req->tc.tcm_parent = OPTS_GET(hook, parent, TC_H_ROOT); |
| 549 |
req->tc.tcm_handle = OPTS_GET(hook, handle, 0); |
| 550 |
|
| 551 |
return nlattr_add(req, TCA_KIND, qdisc, strlen(qdisc) + 1); |
| 552 |
} |
| 553 |
|
| 554 |
static int attach_point_to_config(struct bpf_tc_hook *hook, |
| 555 |
qdisc_config_t *config) |
| 556 |
{ |
| 557 |
switch (OPTS_GET(hook, attach_point, 0)) { |
| 558 |
case BPF_TC_INGRESS: |
| 559 |
case BPF_TC_EGRESS: |
| 560 |
case BPF_TC_INGRESS | BPF_TC_EGRESS: |
| 561 |
if (OPTS_GET(hook, parent, 0)) |
| 562 |
return -EINVAL; |
| 563 |
*config = &clsact_config; |
| 564 |
return 0; |
| 565 |
case BPF_TC_CUSTOM: |
| 566 |
return -EOPNOTSUPP; |
| 567 |
case BPF_TC_QDISC: |
| 568 |
*config = &qdisc_config; |
| 569 |
return 0; |
| 570 |
default: |
| 571 |
return -EINVAL; |
| 572 |
} |
| 573 |
} |
| 574 |
|
| 575 |
static int tc_get_tcm_parent(enum bpf_tc_attach_point attach_point, |
| 576 |
__u32 *parent) |
| 577 |
{ |
| 578 |
switch (attach_point) { |
| 579 |
case BPF_TC_INGRESS: |
| 580 |
case BPF_TC_EGRESS: |
| 581 |
if (*parent) |
| 582 |
return -EINVAL; |
| 583 |
*parent = TC_H_MAKE(TC_H_CLSACT, |
| 584 |
attach_point == BPF_TC_INGRESS ? |
| 585 |
TC_H_MIN_INGRESS : TC_H_MIN_EGRESS); |
| 586 |
break; |
| 587 |
case BPF_TC_CUSTOM: |
| 588 |
if (!*parent) |
| 589 |
return -EINVAL; |
| 590 |
break; |
| 591 |
default: |
| 592 |
return -EINVAL; |
| 593 |
} |
| 594 |
return 0; |
| 595 |
} |
| 596 |
|
| 597 |
static int tc_qdisc_modify(struct bpf_tc_hook *hook, int cmd, int flags) |
| 598 |
{ |
| 599 |
qdisc_config_t config; |
| 600 |
int ret; |
| 601 |
struct libbpf_nla_req req; |
| 602 |
|
| 603 |
ret = attach_point_to_config(hook, &config); |
| 604 |
if (ret < 0) |
| 605 |
return ret; |
| 606 |
|
| 607 |
memset(&req, 0, sizeof(req)); |
| 608 |
req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(struct tcmsg)); |
| 609 |
req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK | flags; |
| 610 |
req.nh.nlmsg_type = cmd; |
| 611 |
req.tc.tcm_family = AF_UNSPEC; |
| 612 |
req.tc.tcm_ifindex = OPTS_GET(hook, ifindex, 0); |
| 613 |
|
| 614 |
ret = config(&req, hook); |
| 615 |
if (ret < 0) |
| 616 |
return ret; |
| 617 |
|
| 618 |
return libbpf_netlink_send_recv(&req, NETLINK_ROUTE, NULL, NULL, NULL); |
| 619 |
} |
| 620 |
|
| 621 |
static int tc_qdisc_create_excl(struct bpf_tc_hook *hook) |
| 622 |
{ |
| 623 |
return tc_qdisc_modify(hook, RTM_NEWQDISC, NLM_F_CREATE | NLM_F_EXCL); |
| 624 |
} |
| 625 |
|
| 626 |
static int tc_qdisc_delete(struct bpf_tc_hook *hook) |
| 627 |
{ |
| 628 |
return tc_qdisc_modify(hook, RTM_DELQDISC, 0); |
| 629 |
} |
| 630 |
|
| 631 |
int bpf_tc_hook_create(struct bpf_tc_hook *hook) |
| 632 |
{ |
| 633 |
int ret; |
| 634 |
|
| 635 |
if (!hook || !OPTS_VALID(hook, bpf_tc_hook) || |
| 636 |
OPTS_GET(hook, ifindex, 0) <= 0) |
| 637 |
return libbpf_err(-EINVAL); |
| 638 |
|
| 639 |
ret = tc_qdisc_create_excl(hook); |
| 640 |
return libbpf_err(ret); |
| 641 |
} |
| 642 |
|
| 643 |
static int __bpf_tc_detach(const struct bpf_tc_hook *hook, |
| 644 |
const struct bpf_tc_opts *opts, |
| 645 |
const bool flush); |
| 646 |
|
| 647 |
int bpf_tc_hook_destroy(struct bpf_tc_hook *hook) |
| 648 |
{ |
| 649 |
if (!hook || !OPTS_VALID(hook, bpf_tc_hook) || |
| 650 |
OPTS_GET(hook, ifindex, 0) <= 0) |
| 651 |
return libbpf_err(-EINVAL); |
| 652 |
|
| 653 |
switch (OPTS_GET(hook, attach_point, 0)) { |
| 654 |
case BPF_TC_INGRESS: |
| 655 |
case BPF_TC_EGRESS: |
| 656 |
return libbpf_err(__bpf_tc_detach(hook, NULL, true)); |
| 657 |
case BPF_TC_QDISC: |
| 658 |
case BPF_TC_INGRESS | BPF_TC_EGRESS: |
| 659 |
return libbpf_err(tc_qdisc_delete(hook)); |
| 660 |
case BPF_TC_CUSTOM: |
| 661 |
return libbpf_err(-EOPNOTSUPP); |
| 662 |
default: |
| 663 |
return libbpf_err(-EINVAL); |
| 664 |
} |
| 665 |
} |
| 666 |
|
| 667 |
struct bpf_cb_ctx { |
| 668 |
struct bpf_tc_opts *opts; |
| 669 |
bool processed; |
| 670 |
}; |
| 671 |
|
| 672 |
static int __get_tc_info(void *cookie, struct tcmsg *tc, struct nlattr **tb, |
| 673 |
bool unicast) |
| 674 |
{ |
| 675 |
struct nlattr *tbb[TCA_BPF_MAX + 1]; |
| 676 |
struct bpf_cb_ctx *info = cookie; |
| 677 |
|
| 678 |
if (!info || !info->opts) |
| 679 |
return -EINVAL; |
| 680 |
if (unicast && info->processed) |
| 681 |
return -EINVAL; |
| 682 |
if (!tb[TCA_OPTIONS]) |
| 683 |
return NL_CONT; |
| 684 |
|
| 685 |
libbpf_nla_parse_nested(tbb, TCA_BPF_MAX, tb[TCA_OPTIONS], NULL); |
| 686 |
if (!tbb[TCA_BPF_ID]) |
| 687 |
return -EINVAL; |
| 688 |
|
| 689 |
OPTS_SET(info->opts, prog_id, libbpf_nla_getattr_u32(tbb[TCA_BPF_ID])); |
| 690 |
OPTS_SET(info->opts, handle, tc->tcm_handle); |
| 691 |
OPTS_SET(info->opts, priority, TC_H_MAJ(tc->tcm_info) >> 16); |
| 692 |
|
| 693 |
info->processed = true; |
| 694 |
return unicast ? NL_NEXT : NL_DONE; |
| 695 |
} |
| 696 |
|
| 697 |
static int get_tc_info(struct nlmsghdr *nh, libbpf_dump_nlmsg_t fn, |
| 698 |
void *cookie) |
| 699 |
{ |
| 700 |
struct tcmsg *tc = NLMSG_DATA(nh); |
| 701 |
struct nlattr *tb[TCA_MAX + 1]; |
| 702 |
|
| 703 |
libbpf_nla_parse(tb, TCA_MAX, |
| 704 |
(struct nlattr *)((void *)tc + NLMSG_ALIGN(sizeof(*tc))), |
| 705 |
NLMSG_PAYLOAD(nh, sizeof(*tc)), NULL); |
| 706 |
if (!tb[TCA_KIND]) |
| 707 |
return NL_CONT; |
| 708 |
return __get_tc_info(cookie, tc, tb, nh->nlmsg_flags & NLM_F_ECHO); |
| 709 |
} |
| 710 |
|
| 711 |
static int tc_add_fd_and_name(struct libbpf_nla_req *req, int fd) |
| 712 |
{ |
| 713 |
struct bpf_prog_info info; |
| 714 |
__u32 info_len = sizeof(info); |
| 715 |
char name[256]; |
| 716 |
int len, ret; |
| 717 |
|
| 718 |
memset(&info, 0, info_len); |
| 719 |
ret = bpf_prog_get_info_by_fd(fd, &info, &info_len); |
| 720 |
if (ret < 0) |
| 721 |
return ret; |
| 722 |
|
| 723 |
ret = nlattr_add(req, TCA_BPF_FD, &fd, sizeof(fd)); |
| 724 |
if (ret < 0) |
| 725 |
return ret; |
| 726 |
len = snprintf(name, sizeof(name), "%s:[%u]", info.name, info.id); |
| 727 |
if (len < 0) |
| 728 |
return -errno; |
| 729 |
if (len >= sizeof(name)) |
| 730 |
return -ENAMETOOLONG; |
| 731 |
return nlattr_add(req, TCA_BPF_NAME, name, len + 1); |
| 732 |
} |
| 733 |
|
| 734 |
int bpf_tc_attach(const struct bpf_tc_hook *hook, struct bpf_tc_opts *opts) |
| 735 |
{ |
| 736 |
__u32 protocol, bpf_flags, handle, priority, parent, prog_id, flags; |
| 737 |
int ret, ifindex, attach_point, prog_fd; |
| 738 |
struct bpf_cb_ctx info = {}; |
| 739 |
struct libbpf_nla_req req; |
| 740 |
struct nlattr *nla; |
| 741 |
|
| 742 |
if (!hook || !opts || |
| 743 |
!OPTS_VALID(hook, bpf_tc_hook) || |
| 744 |
!OPTS_VALID(opts, bpf_tc_opts)) |
| 745 |
return libbpf_err(-EINVAL); |
| 746 |
|
| 747 |
ifindex = OPTS_GET(hook, ifindex, 0); |
| 748 |
parent = OPTS_GET(hook, parent, 0); |
| 749 |
attach_point = OPTS_GET(hook, attach_point, 0); |
| 750 |
|
| 751 |
handle = OPTS_GET(opts, handle, 0); |
| 752 |
priority = OPTS_GET(opts, priority, 0); |
| 753 |
prog_fd = OPTS_GET(opts, prog_fd, 0); |
| 754 |
prog_id = OPTS_GET(opts, prog_id, 0); |
| 755 |
flags = OPTS_GET(opts, flags, 0); |
| 756 |
|
| 757 |
if (ifindex <= 0 || !prog_fd || prog_id) |
| 758 |
return libbpf_err(-EINVAL); |
| 759 |
if (priority > UINT16_MAX) |
| 760 |
return libbpf_err(-EINVAL); |
| 761 |
if (flags & ~BPF_TC_F_REPLACE) |
| 762 |
return libbpf_err(-EINVAL); |
| 763 |
|
| 764 |
flags = (flags & BPF_TC_F_REPLACE) ? NLM_F_REPLACE : NLM_F_EXCL; |
| 765 |
protocol = ETH_P_ALL; |
| 766 |
|
| 767 |
memset(&req, 0, sizeof(req)); |
| 768 |
req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(struct tcmsg)); |
| 769 |
req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_CREATE | |
| 770 |
NLM_F_ECHO | flags; |
| 771 |
req.nh.nlmsg_type = RTM_NEWTFILTER; |
| 772 |
req.tc.tcm_family = AF_UNSPEC; |
| 773 |
req.tc.tcm_ifindex = ifindex; |
| 774 |
req.tc.tcm_handle = handle; |
| 775 |
req.tc.tcm_info = TC_H_MAKE(priority << 16, htons(protocol)); |
| 776 |
|
| 777 |
ret = tc_get_tcm_parent(attach_point, &parent); |
| 778 |
if (ret < 0) |
| 779 |
return libbpf_err(ret); |
| 780 |
req.tc.tcm_parent = parent; |
| 781 |
|
| 782 |
ret = nlattr_add(&req, TCA_KIND, "bpf", sizeof("bpf")); |
| 783 |
if (ret < 0) |
| 784 |
return libbpf_err(ret); |
| 785 |
nla = nlattr_begin_nested(&req, TCA_OPTIONS); |
| 786 |
if (!nla) |
| 787 |
return libbpf_err(-EMSGSIZE); |
| 788 |
ret = tc_add_fd_and_name(&req, prog_fd); |
| 789 |
if (ret < 0) |
| 790 |
return libbpf_err(ret); |
| 791 |
bpf_flags = TCA_BPF_FLAG_ACT_DIRECT; |
| 792 |
ret = nlattr_add(&req, TCA_BPF_FLAGS, &bpf_flags, sizeof(bpf_flags)); |
| 793 |
if (ret < 0) |
| 794 |
return libbpf_err(ret); |
| 795 |
nlattr_end_nested(&req, nla); |
| 796 |
|
| 797 |
info.opts = opts; |
| 798 |
|
| 799 |
ret = libbpf_netlink_send_recv(&req, NETLINK_ROUTE, get_tc_info, NULL, |
| 800 |
&info); |
| 801 |
if (ret < 0) |
| 802 |
return libbpf_err(ret); |
| 803 |
if (!info.processed) |
| 804 |
return libbpf_err(-ENOENT); |
| 805 |
return ret; |
| 806 |
} |
| 807 |
|
| 808 |
static int __bpf_tc_detach(const struct bpf_tc_hook *hook, |
| 809 |
const struct bpf_tc_opts *opts, |
| 810 |
const bool flush) |
| 811 |
{ |
| 812 |
__u32 protocol = 0, handle, priority, parent, prog_id, flags; |
| 813 |
int ret, ifindex, attach_point, prog_fd; |
| 814 |
struct libbpf_nla_req req; |
| 815 |
|
| 816 |
if (!hook || |
| 817 |
!OPTS_VALID(hook, bpf_tc_hook) || |
| 818 |
!OPTS_VALID(opts, bpf_tc_opts)) |
| 819 |
return -EINVAL; |
| 820 |
|
| 821 |
ifindex = OPTS_GET(hook, ifindex, 0); |
| 822 |
parent = OPTS_GET(hook, parent, 0); |
| 823 |
attach_point = OPTS_GET(hook, attach_point, 0); |
| 824 |
|
| 825 |
handle = OPTS_GET(opts, handle, 0); |
| 826 |
priority = OPTS_GET(opts, priority, 0); |
| 827 |
prog_fd = OPTS_GET(opts, prog_fd, 0); |
| 828 |
prog_id = OPTS_GET(opts, prog_id, 0); |
| 829 |
flags = OPTS_GET(opts, flags, 0); |
| 830 |
|
| 831 |
if (ifindex <= 0 || flags || prog_fd || prog_id) |
| 832 |
return -EINVAL; |
| 833 |
if (priority > UINT16_MAX) |
| 834 |
return -EINVAL; |
| 835 |
if (!flush) { |
| 836 |
if (!handle || !priority) |
| 837 |
return -EINVAL; |
| 838 |
protocol = ETH_P_ALL; |
| 839 |
} else { |
| 840 |
if (handle || priority) |
| 841 |
return -EINVAL; |
| 842 |
} |
| 843 |
|
| 844 |
memset(&req, 0, sizeof(req)); |
| 845 |
req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(struct tcmsg)); |
| 846 |
req.nh.nlmsg_flags = NLM_F_REQUEST | NLM_F_ACK; |
| 847 |
req.nh.nlmsg_type = RTM_DELTFILTER; |
| 848 |
req.tc.tcm_family = AF_UNSPEC; |
| 849 |
req.tc.tcm_ifindex = ifindex; |
| 850 |
if (!flush) { |
| 851 |
req.tc.tcm_handle = handle; |
| 852 |
req.tc.tcm_info = TC_H_MAKE(priority << 16, htons(protocol)); |
| 853 |
} |
| 854 |
|
| 855 |
ret = tc_get_tcm_parent(attach_point, &parent); |
| 856 |
if (ret < 0) |
| 857 |
return ret; |
| 858 |
req.tc.tcm_parent = parent; |
| 859 |
|
| 860 |
if (!flush) { |
| 861 |
ret = nlattr_add(&req, TCA_KIND, "bpf", sizeof("bpf")); |
| 862 |
if (ret < 0) |
| 863 |
return ret; |
| 864 |
} |
| 865 |
|
| 866 |
return libbpf_netlink_send_recv(&req, NETLINK_ROUTE, NULL, NULL, NULL); |
| 867 |
} |
| 868 |
|
| 869 |
int bpf_tc_detach(const struct bpf_tc_hook *hook, |
| 870 |
const struct bpf_tc_opts *opts) |
| 871 |
{ |
| 872 |
int ret; |
| 873 |
|
| 874 |
if (!opts) |
| 875 |
return libbpf_err(-EINVAL); |
| 876 |
|
| 877 |
ret = __bpf_tc_detach(hook, opts, false); |
| 878 |
return libbpf_err(ret); |
| 879 |
} |
| 880 |
|
| 881 |
int bpf_tc_query(const struct bpf_tc_hook *hook, struct bpf_tc_opts *opts) |
| 882 |
{ |
| 883 |
__u32 protocol, handle, priority, parent, prog_id, flags; |
| 884 |
int ret, ifindex, attach_point, prog_fd; |
| 885 |
struct bpf_cb_ctx info = {}; |
| 886 |
struct libbpf_nla_req req; |
| 887 |
|
| 888 |
if (!hook || !opts || |
| 889 |
!OPTS_VALID(hook, bpf_tc_hook) || |
| 890 |
!OPTS_VALID(opts, bpf_tc_opts)) |
| 891 |
return libbpf_err(-EINVAL); |
| 892 |
|
| 893 |
ifindex = OPTS_GET(hook, ifindex, 0); |
| 894 |
parent = OPTS_GET(hook, parent, 0); |
| 895 |
attach_point = OPTS_GET(hook, attach_point, 0); |
| 896 |
|
| 897 |
handle = OPTS_GET(opts, handle, 0); |
| 898 |
priority = OPTS_GET(opts, priority, 0); |
| 899 |
prog_fd = OPTS_GET(opts, prog_fd, 0); |
| 900 |
prog_id = OPTS_GET(opts, prog_id, 0); |
| 901 |
flags = OPTS_GET(opts, flags, 0); |
| 902 |
|
| 903 |
if (ifindex <= 0 || flags || prog_fd || prog_id || |
| 904 |
!handle || !priority) |
| 905 |
return libbpf_err(-EINVAL); |
| 906 |
if (priority > UINT16_MAX) |
| 907 |
return libbpf_err(-EINVAL); |
| 908 |
|
| 909 |
protocol = ETH_P_ALL; |
| 910 |
|
| 911 |
memset(&req, 0, sizeof(req)); |
| 912 |
req.nh.nlmsg_len = NLMSG_LENGTH(sizeof(struct tcmsg)); |
| 913 |
req.nh.nlmsg_flags = NLM_F_REQUEST; |
| 914 |
req.nh.nlmsg_type = RTM_GETTFILTER; |
| 915 |
req.tc.tcm_family = AF_UNSPEC; |
| 916 |
req.tc.tcm_ifindex = ifindex; |
| 917 |
req.tc.tcm_handle = handle; |
| 918 |
req.tc.tcm_info = TC_H_MAKE(priority << 16, htons(protocol)); |
| 919 |
|
| 920 |
ret = tc_get_tcm_parent(attach_point, &parent); |
| 921 |
if (ret < 0) |
| 922 |
return libbpf_err(ret); |
| 923 |
req.tc.tcm_parent = parent; |
| 924 |
|
| 925 |
ret = nlattr_add(&req, TCA_KIND, "bpf", sizeof("bpf")); |
| 926 |
if (ret < 0) |
| 927 |
return libbpf_err(ret); |
| 928 |
|
| 929 |
info.opts = opts; |
| 930 |
|
| 931 |
ret = libbpf_netlink_send_recv(&req, NETLINK_ROUTE, get_tc_info, NULL, |
| 932 |
&info); |
| 933 |
if (ret < 0) |
| 934 |
return libbpf_err(ret); |
| 935 |
if (!info.processed) |
| 936 |
return libbpf_err(-ENOENT); |
| 937 |
return ret; |
| 938 |
} |