| 1 |
/* SPDX-License-Identifier: GPL-2.0 */ |
| 2 |
#ifndef __PACKET_INTERNAL_H__ |
| 3 |
#define __PACKET_INTERNAL_H__ |
| 4 |
|
| 5 |
#include <linux/refcount.h> |
| 6 |
|
| 7 |
struct packet_mclist { |
| 8 |
struct packet_mclist *next; |
| 9 |
int ifindex; |
| 10 |
int count; |
| 11 |
unsigned short type; |
| 12 |
unsigned short alen; |
| 13 |
unsigned char addr[MAX_ADDR_LEN]; |
| 14 |
struct list_head remove_list; |
| 15 |
}; |
| 16 |
|
| 17 |
/* kbdq - kernel block descriptor queue */ |
| 18 |
struct tpacket_kbdq_core { |
| 19 |
struct pgv *pkbdq; |
| 20 |
unsigned int feature_req_word; |
| 21 |
unsigned int hdrlen; |
| 22 |
unsigned char reset_pending_on_curr_blk; |
| 23 |
unsigned short kactive_blk_num; |
| 24 |
unsigned short blk_sizeof_priv; |
| 25 |
|
| 26 |
unsigned short version; |
| 27 |
|
| 28 |
char *pkblk_start; |
| 29 |
char *pkblk_end; |
| 30 |
int kblk_size; |
| 31 |
unsigned int max_frame_len; |
| 32 |
unsigned int knum_blocks; |
| 33 |
uint64_t knxt_seq_num; |
| 34 |
char *prev; |
| 35 |
char *nxt_offset; |
| 36 |
|
| 37 |
struct sk_buff *skb; |
| 38 |
|
| 39 |
rwlock_t blk_fill_in_prog_lock; |
| 40 |
|
| 41 |
/* Default is set to 8ms */ |
| 42 |
#define DEFAULT_PRB_RETIRE_TOV (8) |
| 43 |
|
| 44 |
ktime_t interval_ktime; |
| 45 |
|
| 46 |
/* timer to retire an outstanding block */ |
| 47 |
struct hrtimer retire_blk_timer; |
| 48 |
}; |
| 49 |
|
| 50 |
struct pgv { |
| 51 |
char *buffer; |
| 52 |
}; |
| 53 |
|
| 54 |
struct packet_ring_buffer { |
| 55 |
struct pgv *pg_vec; |
| 56 |
|
| 57 |
unsigned int head; |
| 58 |
unsigned int frames_per_block; |
| 59 |
unsigned int frame_size; |
| 60 |
unsigned int frame_max; |
| 61 |
|
| 62 |
unsigned int pg_vec_order; |
| 63 |
unsigned int pg_vec_pages; |
| 64 |
unsigned int pg_vec_len; |
| 65 |
|
| 66 |
unsigned int __percpu *pending_refcnt; |
| 67 |
|
| 68 |
union { |
| 69 |
unsigned long *rx_owner_map; |
| 70 |
struct tpacket_kbdq_core prb_bdqc; |
| 71 |
}; |
| 72 |
}; |
| 73 |
|
| 74 |
extern struct mutex fanout_mutex; |
| 75 |
#define PACKET_FANOUT_MAX (1 << 16) |
| 76 |
|
| 77 |
struct packet_fanout { |
| 78 |
possible_net_t net; |
| 79 |
unsigned int num_members; |
| 80 |
u32 max_num_members; |
| 81 |
u16 id; |
| 82 |
u8 type; |
| 83 |
u8 flags; |
| 84 |
union { |
| 85 |
atomic_t rr_cur; |
| 86 |
struct bpf_prog __rcu *bpf_prog; |
| 87 |
}; |
| 88 |
struct list_head list; |
| 89 |
spinlock_t lock; |
| 90 |
refcount_t sk_ref; |
| 91 |
struct packet_type prot_hook ____cacheline_aligned_in_smp; |
| 92 |
struct sock __rcu *arr[] __counted_by(max_num_members); |
| 93 |
}; |
| 94 |
|
| 95 |
struct packet_rollover { |
| 96 |
int sock; |
| 97 |
atomic_long_t num; |
| 98 |
atomic_long_t num_huge; |
| 99 |
atomic_long_t num_failed; |
| 100 |
#define ROLLOVER_HLEN (L1_CACHE_BYTES / sizeof(u32)) |
| 101 |
u32 history[ROLLOVER_HLEN] ____cacheline_aligned; |
| 102 |
} ____cacheline_aligned_in_smp; |
| 103 |
|
| 104 |
struct packet_sock { |
| 105 |
/* struct sock has to be the first member of packet_sock */ |
| 106 |
struct sock sk; |
| 107 |
struct packet_fanout *fanout; |
| 108 |
union tpacket_stats_u stats; |
| 109 |
struct packet_ring_buffer rx_ring; |
| 110 |
struct packet_ring_buffer tx_ring; |
| 111 |
int copy_thresh; |
| 112 |
spinlock_t bind_lock; |
| 113 |
struct mutex pg_vec_lock; |
| 114 |
unsigned long flags; |
| 115 |
int ifindex; /* bound device */ |
| 116 |
u8 vnet_hdr_sz; |
| 117 |
__be16 num; |
| 118 |
struct packet_rollover *rollover; |
| 119 |
struct packet_mclist *mclist; |
| 120 |
atomic_long_t mapped; |
| 121 |
enum tpacket_versions tp_version; |
| 122 |
unsigned int tp_hdrlen; |
| 123 |
unsigned int tp_reserve; |
| 124 |
unsigned int tp_tstamp; |
| 125 |
struct completion skb_completion; |
| 126 |
struct net_device __rcu *cached_dev; |
| 127 |
struct packet_type prot_hook ____cacheline_aligned_in_smp; |
| 128 |
atomic_t tp_drops ____cacheline_aligned_in_smp; |
| 129 |
}; |
| 130 |
|
| 131 |
#define pkt_sk(ptr) container_of_const(ptr, struct packet_sock, sk) |
| 132 |
|
| 133 |
enum packet_sock_flags { |
| 134 |
PACKET_SOCK_ORIGDEV, |
| 135 |
PACKET_SOCK_AUXDATA, |
| 136 |
PACKET_SOCK_TX_HAS_OFF, |
| 137 |
PACKET_SOCK_TP_LOSS, |
| 138 |
PACKET_SOCK_RUNNING, |
| 139 |
PACKET_SOCK_PRESSURE, |
| 140 |
PACKET_SOCK_QDISC_BYPASS, |
| 141 |
}; |
| 142 |
|
| 143 |
static inline void packet_sock_flag_set(struct packet_sock *po, |
| 144 |
enum packet_sock_flags flag, |
| 145 |
bool val) |
| 146 |
{ |
| 147 |
if (val) |
| 148 |
set_bit(flag, &po->flags); |
| 149 |
else |
| 150 |
clear_bit(flag, &po->flags); |
| 151 |
} |
| 152 |
|
| 153 |
static inline bool packet_sock_flag(const struct packet_sock *po, |
| 154 |
enum packet_sock_flags flag) |
| 155 |
{ |
| 156 |
return test_bit(flag, &po->flags); |
| 157 |
} |
| 158 |
|
| 159 |
#endif |