| 1 |
/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ |
| 2 |
/* |
| 3 |
* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. |
| 4 |
*/ |
| 5 |
|
| 6 |
#ifndef MLX5_VFIO_CMD_H |
| 7 |
#define MLX5_VFIO_CMD_H |
| 8 |
|
| 9 |
#include <linux/kernel.h> |
| 10 |
#include <linux/vfio_pci_core.h> |
| 11 |
#include <linux/mlx5/driver.h> |
| 12 |
#include <linux/mlx5/vport.h> |
| 13 |
#include <linux/mlx5/cq.h> |
| 14 |
#include <linux/mlx5/qp.h> |
| 15 |
|
| 16 |
enum mlx5_vf_migf_state { |
| 17 |
MLX5_MIGF_STATE_ERROR = 1, |
| 18 |
MLX5_MIGF_STATE_PRE_COPY_ERROR, |
| 19 |
MLX5_MIGF_STATE_PRE_COPY, |
| 20 |
MLX5_MIGF_STATE_SAVE_STOP_COPY_CHUNK, |
| 21 |
MLX5_MIGF_STATE_COMPLETE, |
| 22 |
}; |
| 23 |
|
| 24 |
enum mlx5_vf_load_state { |
| 25 |
MLX5_VF_LOAD_STATE_READ_HEADER, |
| 26 |
MLX5_VF_LOAD_STATE_PREP_HEADER_DATA, |
| 27 |
MLX5_VF_LOAD_STATE_READ_HEADER_DATA, |
| 28 |
MLX5_VF_LOAD_STATE_PREP_IMAGE, |
| 29 |
MLX5_VF_LOAD_STATE_READ_IMAGE, |
| 30 |
MLX5_VF_LOAD_STATE_LOAD_IMAGE, |
| 31 |
}; |
| 32 |
|
| 33 |
struct mlx5_vf_migration_tag_stop_copy_data { |
| 34 |
__le64 stop_copy_size; |
| 35 |
}; |
| 36 |
|
| 37 |
enum mlx5_vf_migf_header_flags { |
| 38 |
MLX5_MIGF_HEADER_FLAGS_TAG_MANDATORY = 0, |
| 39 |
MLX5_MIGF_HEADER_FLAGS_TAG_OPTIONAL = 1 << 0, |
| 40 |
}; |
| 41 |
|
| 42 |
enum mlx5_vf_migf_header_tag { |
| 43 |
MLX5_MIGF_HEADER_TAG_FW_DATA = 0, |
| 44 |
MLX5_MIGF_HEADER_TAG_STOP_COPY_SIZE = 1 << 0, |
| 45 |
}; |
| 46 |
|
| 47 |
struct mlx5_vf_migration_header { |
| 48 |
__le64 record_size; |
| 49 |
/* For future use in case we may need to change the kernel protocol */ |
| 50 |
__le32 flags; /* Use mlx5_vf_migf_header_flags */ |
| 51 |
__le32 tag; /* Use mlx5_vf_migf_header_tag */ |
| 52 |
__u8 data[]; /* Its size is given in the record_size */ |
| 53 |
}; |
| 54 |
|
| 55 |
struct mlx5_vhca_data_buffer { |
| 56 |
struct page **page_list; |
| 57 |
struct dma_iova_state state; |
| 58 |
loff_t start_pos; |
| 59 |
u64 length; |
| 60 |
u32 npages; |
| 61 |
u32 mkey; |
| 62 |
u32 *mkey_in; |
| 63 |
enum dma_data_direction dma_dir; |
| 64 |
u8 stop_copy_chunk_num; |
| 65 |
struct list_head buf_elm; |
| 66 |
struct mlx5_vf_migration_file *migf; |
| 67 |
}; |
| 68 |
|
| 69 |
struct mlx5vf_async_data { |
| 70 |
struct mlx5_async_work cb_work; |
| 71 |
struct work_struct work; |
| 72 |
struct mlx5_vhca_data_buffer *buf; |
| 73 |
struct mlx5_vhca_data_buffer *header_buf; |
| 74 |
int status; |
| 75 |
u8 stop_copy_chunk:1; |
| 76 |
void *out; |
| 77 |
}; |
| 78 |
|
| 79 |
struct mlx5vf_save_work_data { |
| 80 |
struct mlx5_vf_migration_file *migf; |
| 81 |
size_t next_required_umem_size; |
| 82 |
struct work_struct work; |
| 83 |
u8 chunk_num; |
| 84 |
}; |
| 85 |
|
| 86 |
#define MAX_NUM_CHUNKS 2 |
| 87 |
|
| 88 |
struct mlx5_vf_migration_file { |
| 89 |
struct file *filp; |
| 90 |
struct mutex lock; |
| 91 |
enum mlx5_vf_migf_state state; |
| 92 |
|
| 93 |
enum mlx5_vf_load_state load_state; |
| 94 |
u32 pdn; |
| 95 |
loff_t max_pos; |
| 96 |
u64 record_size; |
| 97 |
u32 record_tag; |
| 98 |
u64 stop_copy_prep_size; |
| 99 |
u64 pre_copy_initial_bytes; |
| 100 |
size_t next_required_umem_size; |
| 101 |
u8 num_ready_chunks; |
| 102 |
/* Upon chunk mode preserve another set of buffers for stop_copy phase */ |
| 103 |
struct mlx5_vhca_data_buffer *buf[MAX_NUM_CHUNKS]; |
| 104 |
struct mlx5_vhca_data_buffer *buf_header[MAX_NUM_CHUNKS]; |
| 105 |
struct mlx5vf_save_work_data save_data[MAX_NUM_CHUNKS]; |
| 106 |
spinlock_t list_lock; |
| 107 |
struct list_head buf_list; |
| 108 |
struct list_head avail_list; |
| 109 |
struct mlx5vf_pci_core_device *mvdev; |
| 110 |
wait_queue_head_t poll_wait; |
| 111 |
struct completion save_comp; |
| 112 |
struct mlx5_async_ctx async_ctx; |
| 113 |
struct mlx5vf_async_data async_data; |
| 114 |
}; |
| 115 |
|
| 116 |
struct mlx5_vhca_cq_buf { |
| 117 |
struct mlx5_frag_buf_ctrl fbc; |
| 118 |
struct mlx5_frag_buf frag_buf; |
| 119 |
int cqe_size; |
| 120 |
int nent; |
| 121 |
}; |
| 122 |
|
| 123 |
struct mlx5_vhca_cq { |
| 124 |
struct mlx5_vhca_cq_buf buf; |
| 125 |
struct mlx5_db db; |
| 126 |
struct mlx5_core_cq mcq; |
| 127 |
size_t ncqe; |
| 128 |
}; |
| 129 |
|
| 130 |
struct mlx5_vhca_recv_buf { |
| 131 |
u32 npages; |
| 132 |
struct page **page_list; |
| 133 |
struct dma_iova_state state; |
| 134 |
u32 next_rq_offset; |
| 135 |
u32 *mkey_in; |
| 136 |
u32 mkey; |
| 137 |
}; |
| 138 |
|
| 139 |
struct mlx5_vhca_qp { |
| 140 |
struct mlx5_frag_buf buf; |
| 141 |
struct mlx5_db db; |
| 142 |
struct mlx5_vhca_recv_buf recv_buf; |
| 143 |
u32 tracked_page_size; |
| 144 |
u32 max_msg_size; |
| 145 |
u32 qpn; |
| 146 |
struct { |
| 147 |
unsigned int pc; |
| 148 |
unsigned int cc; |
| 149 |
unsigned int wqe_cnt; |
| 150 |
__be32 *db; |
| 151 |
struct mlx5_frag_buf_ctrl fbc; |
| 152 |
} rq; |
| 153 |
}; |
| 154 |
|
| 155 |
struct mlx5_vhca_page_tracker { |
| 156 |
u32 id; |
| 157 |
u32 pdn; |
| 158 |
u8 is_err:1; |
| 159 |
u8 object_changed:1; |
| 160 |
struct mlx5_uars_page *uar; |
| 161 |
struct mlx5_vhca_cq cq; |
| 162 |
struct mlx5_vhca_qp *host_qp; |
| 163 |
struct mlx5_vhca_qp *fw_qp; |
| 164 |
struct mlx5_nb nb; |
| 165 |
int status; |
| 166 |
}; |
| 167 |
|
| 168 |
struct mlx5vf_pci_core_device { |
| 169 |
struct vfio_pci_core_device core_device; |
| 170 |
int vf_id; |
| 171 |
u16 vhca_id; |
| 172 |
u8 migrate_cap:1; |
| 173 |
u8 deferred_reset:1; |
| 174 |
u8 mdev_detach:1; |
| 175 |
u8 log_active:1; |
| 176 |
u8 chunk_mode:1; |
| 177 |
struct completion tracker_comp; |
| 178 |
/* protect migration state */ |
| 179 |
struct mutex state_mutex; |
| 180 |
enum vfio_device_mig_state mig_state; |
| 181 |
/* protect the reset_done flow */ |
| 182 |
spinlock_t reset_lock; |
| 183 |
struct mlx5_vf_migration_file *resuming_migf; |
| 184 |
struct mlx5_vf_migration_file *saving_migf; |
| 185 |
struct mlx5_vhca_page_tracker tracker; |
| 186 |
struct workqueue_struct *cb_wq; |
| 187 |
struct notifier_block nb; |
| 188 |
struct mlx5_core_dev *mdev; |
| 189 |
}; |
| 190 |
|
| 191 |
enum { |
| 192 |
MLX5VF_QUERY_INC = (1UL << 0), |
| 193 |
MLX5VF_QUERY_FINAL = (1UL << 1), |
| 194 |
MLX5VF_QUERY_CLEANUP = (1UL << 2), |
| 195 |
}; |
| 196 |
|
| 197 |
int mlx5vf_cmd_suspend_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod); |
| 198 |
int mlx5vf_cmd_resume_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod); |
| 199 |
int mlx5vf_cmd_query_vhca_migration_state(struct mlx5vf_pci_core_device *mvdev, |
| 200 |
size_t *state_size, u64 *total_size, |
| 201 |
u8 query_flags); |
| 202 |
void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev, |
| 203 |
const struct vfio_migration_ops *mig_ops, |
| 204 |
const struct vfio_log_ops *log_ops); |
| 205 |
void mlx5vf_cmd_remove_migratable(struct mlx5vf_pci_core_device *mvdev); |
| 206 |
void mlx5vf_cmd_close_migratable(struct mlx5vf_pci_core_device *mvdev); |
| 207 |
int mlx5vf_cmd_save_vhca_state(struct mlx5vf_pci_core_device *mvdev, |
| 208 |
struct mlx5_vf_migration_file *migf, |
| 209 |
struct mlx5_vhca_data_buffer *buf, bool inc, |
| 210 |
bool track); |
| 211 |
int mlx5vf_cmd_load_vhca_state(struct mlx5vf_pci_core_device *mvdev, |
| 212 |
struct mlx5_vf_migration_file *migf, |
| 213 |
struct mlx5_vhca_data_buffer *buf); |
| 214 |
int mlx5vf_cmd_alloc_pd(struct mlx5_vf_migration_file *migf); |
| 215 |
void mlx5vf_cmd_dealloc_pd(struct mlx5_vf_migration_file *migf); |
| 216 |
void mlx5fv_cmd_clean_migf_resources(struct mlx5_vf_migration_file *migf); |
| 217 |
struct mlx5_vhca_data_buffer * |
| 218 |
mlx5vf_alloc_data_buffer(struct mlx5_vf_migration_file *migf, u32 npages, |
| 219 |
enum dma_data_direction dma_dir); |
| 220 |
void mlx5vf_free_data_buffer(struct mlx5_vhca_data_buffer *buf); |
| 221 |
struct mlx5_vhca_data_buffer * |
| 222 |
mlx5vf_get_data_buffer(struct mlx5_vf_migration_file *migf, u32 npages, |
| 223 |
enum dma_data_direction dma_dir); |
| 224 |
void mlx5vf_put_data_buffer(struct mlx5_vhca_data_buffer *buf); |
| 225 |
static inline struct page * |
| 226 |
mlx5vf_get_migration_page(struct mlx5_vhca_data_buffer *buf, |
| 227 |
unsigned long offset) |
| 228 |
{ |
| 229 |
int page_entry = offset / PAGE_SIZE; |
| 230 |
|
| 231 |
if (page_entry >= buf->npages) |
| 232 |
return NULL; |
| 233 |
|
| 234 |
return buf->page_list[page_entry]; |
| 235 |
} |
| 236 |
void mlx5vf_state_mutex_unlock(struct mlx5vf_pci_core_device *mvdev); |
| 237 |
void mlx5vf_disable_fds(struct mlx5vf_pci_core_device *mvdev, |
| 238 |
enum mlx5_vf_migf_state *last_save_state); |
| 239 |
void mlx5vf_mig_file_cleanup_cb(struct work_struct *_work); |
| 240 |
void mlx5vf_mig_file_set_save_work(struct mlx5_vf_migration_file *migf, |
| 241 |
u8 chunk_num, size_t next_required_umem_size); |
| 242 |
int mlx5vf_start_page_tracker(struct vfio_device *vdev, |
| 243 |
struct rb_root_cached *ranges, u32 nnodes, u64 *page_size); |
| 244 |
int mlx5vf_stop_page_tracker(struct vfio_device *vdev); |
| 245 |
int mlx5vf_tracker_read_and_clear(struct vfio_device *vdev, unsigned long iova, |
| 246 |
unsigned long length, struct iova_bitmap *dirty); |
| 247 |
#endif /* MLX5_VFIO_CMD_H */ |