Commit | Line | Data |
---|---|---|
c9b9dcb4 AL |
1 | // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
2 | // Copyright (c) 2019 Mellanox Technologies | |
3 | ||
4 | #include <linux/mlx5/driver.h> | |
5 | #include <linux/mlx5/device.h> | |
6 | ||
7 | #include "mlx5_core.h" | |
8 | #include "lib/mlx5.h" | |
9 | ||
10 | struct mlx5_dm { | |
11 | /* protect access to icm bitmask */ | |
12 | spinlock_t lock; | |
13 | unsigned long *steering_sw_icm_alloc_blocks; | |
14 | unsigned long *header_modify_sw_icm_alloc_blocks; | |
66765836 | 15 | unsigned long *header_modify_pattern_sw_icm_alloc_blocks; |
c9b9dcb4 AL |
16 | }; |
17 | ||
18 | struct mlx5_dm *mlx5_dm_create(struct mlx5_core_dev *dev) | |
19 | { | |
66765836 | 20 | u64 header_modify_pattern_icm_blocks = 0; |
c9b9dcb4 AL |
21 | u64 header_modify_icm_blocks = 0; |
22 | u64 steering_icm_blocks = 0; | |
23 | struct mlx5_dm *dm; | |
66765836 | 24 | bool support_v2; |
c9b9dcb4 AL |
25 | |
26 | if (!(MLX5_CAP_GEN_64(dev, general_obj_types) & MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM)) | |
586ee9e8 | 27 | return NULL; |
c9b9dcb4 AL |
28 | |
29 | dm = kzalloc(sizeof(*dm), GFP_KERNEL); | |
30 | if (!dm) | |
31 | return ERR_PTR(-ENOMEM); | |
32 | ||
33 | spin_lock_init(&dm->lock); | |
34 | ||
35 | if (MLX5_CAP64_DEV_MEM(dev, steering_sw_icm_start_address)) { | |
36 | steering_icm_blocks = | |
37 | BIT(MLX5_CAP_DEV_MEM(dev, log_steering_sw_icm_size) - | |
38 | MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)); | |
39 | ||
40 | dm->steering_sw_icm_alloc_blocks = | |
41 | kcalloc(BITS_TO_LONGS(steering_icm_blocks), | |
42 | sizeof(unsigned long), GFP_KERNEL); | |
43 | if (!dm->steering_sw_icm_alloc_blocks) | |
44 | goto err_steering; | |
45 | } | |
46 | ||
47 | if (MLX5_CAP64_DEV_MEM(dev, header_modify_sw_icm_start_address)) { | |
48 | header_modify_icm_blocks = | |
49 | BIT(MLX5_CAP_DEV_MEM(dev, log_header_modify_sw_icm_size) - | |
50 | MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)); | |
51 | ||
52 | dm->header_modify_sw_icm_alloc_blocks = | |
53 | kcalloc(BITS_TO_LONGS(header_modify_icm_blocks), | |
54 | sizeof(unsigned long), GFP_KERNEL); | |
55 | if (!dm->header_modify_sw_icm_alloc_blocks) | |
56 | goto err_modify_hdr; | |
57 | } | |
58 | ||
66765836 YK |
59 | support_v2 = MLX5_CAP_FLOWTABLE_NIC_RX(dev, sw_owner_v2) && |
60 | MLX5_CAP_FLOWTABLE_NIC_TX(dev, sw_owner_v2) && | |
61 | MLX5_CAP64_DEV_MEM(dev, header_modify_pattern_sw_icm_start_address); | |
62 | ||
63 | if (support_v2) { | |
64 | header_modify_pattern_icm_blocks = | |
65 | BIT(MLX5_CAP_DEV_MEM(dev, log_header_modify_pattern_sw_icm_size) - | |
66 | MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)); | |
67 | ||
68 | dm->header_modify_pattern_sw_icm_alloc_blocks = | |
69 | kcalloc(BITS_TO_LONGS(header_modify_pattern_icm_blocks), | |
70 | sizeof(unsigned long), GFP_KERNEL); | |
71 | if (!dm->header_modify_pattern_sw_icm_alloc_blocks) | |
72 | goto err_pattern; | |
73 | } | |
74 | ||
c9b9dcb4 AL |
75 | return dm; |
76 | ||
66765836 YK |
77 | err_pattern: |
78 | kfree(dm->header_modify_sw_icm_alloc_blocks); | |
79 | ||
c9b9dcb4 AL |
80 | err_modify_hdr: |
81 | kfree(dm->steering_sw_icm_alloc_blocks); | |
82 | ||
83 | err_steering: | |
84 | kfree(dm); | |
85 | ||
86 | return ERR_PTR(-ENOMEM); | |
87 | } | |
88 | ||
89 | void mlx5_dm_cleanup(struct mlx5_core_dev *dev) | |
90 | { | |
91 | struct mlx5_dm *dm = dev->dm; | |
92 | ||
93 | if (!dev->dm) | |
94 | return; | |
95 | ||
96 | if (dm->steering_sw_icm_alloc_blocks) { | |
97 | WARN_ON(!bitmap_empty(dm->steering_sw_icm_alloc_blocks, | |
98 | BIT(MLX5_CAP_DEV_MEM(dev, log_steering_sw_icm_size) - | |
99 | MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)))); | |
100 | kfree(dm->steering_sw_icm_alloc_blocks); | |
101 | } | |
102 | ||
103 | if (dm->header_modify_sw_icm_alloc_blocks) { | |
104 | WARN_ON(!bitmap_empty(dm->header_modify_sw_icm_alloc_blocks, | |
105 | BIT(MLX5_CAP_DEV_MEM(dev, | |
106 | log_header_modify_sw_icm_size) - | |
107 | MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)))); | |
108 | kfree(dm->header_modify_sw_icm_alloc_blocks); | |
109 | } | |
110 | ||
66765836 YK |
111 | if (dm->header_modify_pattern_sw_icm_alloc_blocks) { |
112 | WARN_ON(!bitmap_empty(dm->header_modify_pattern_sw_icm_alloc_blocks, | |
113 | BIT(MLX5_CAP_DEV_MEM(dev, | |
114 | log_header_modify_pattern_sw_icm_size) - | |
115 | MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)))); | |
116 | kfree(dm->header_modify_pattern_sw_icm_alloc_blocks); | |
117 | } | |
118 | ||
c9b9dcb4 AL |
119 | kfree(dm); |
120 | } | |
121 | ||
122 | int mlx5_dm_sw_icm_alloc(struct mlx5_core_dev *dev, enum mlx5_sw_icm_type type, | |
dff8e2d1 ES |
123 | u64 length, u32 log_alignment, u16 uid, |
124 | phys_addr_t *addr, u32 *obj_id) | |
c9b9dcb4 AL |
125 | { |
126 | u32 num_blocks = DIV_ROUND_UP_ULL(length, MLX5_SW_ICM_BLOCK_SIZE(dev)); | |
127 | u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {}; | |
128 | u32 in[MLX5_ST_SZ_DW(create_sw_icm_in)] = {}; | |
129 | struct mlx5_dm *dm = dev->dm; | |
130 | unsigned long *block_map; | |
131 | u64 icm_start_addr; | |
132 | u32 log_icm_size; | |
dff8e2d1 | 133 | u64 align_mask; |
c9b9dcb4 AL |
134 | u32 max_blocks; |
135 | u64 block_idx; | |
136 | void *sw_icm; | |
137 | int ret; | |
138 | ||
139 | if (!dev->dm) | |
140 | return -EOPNOTSUPP; | |
141 | ||
142 | if (!length || (length & (length - 1)) || | |
143 | length & (MLX5_SW_ICM_BLOCK_SIZE(dev) - 1)) | |
144 | return -EINVAL; | |
145 | ||
146 | MLX5_SET(general_obj_in_cmd_hdr, in, opcode, | |
147 | MLX5_CMD_OP_CREATE_GENERAL_OBJECT); | |
148 | MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_OBJ_TYPE_SW_ICM); | |
149 | MLX5_SET(general_obj_in_cmd_hdr, in, uid, uid); | |
150 | ||
151 | switch (type) { | |
152 | case MLX5_SW_ICM_TYPE_STEERING: | |
153 | icm_start_addr = MLX5_CAP64_DEV_MEM(dev, steering_sw_icm_start_address); | |
154 | log_icm_size = MLX5_CAP_DEV_MEM(dev, log_steering_sw_icm_size); | |
155 | block_map = dm->steering_sw_icm_alloc_blocks; | |
156 | break; | |
157 | case MLX5_SW_ICM_TYPE_HEADER_MODIFY: | |
158 | icm_start_addr = MLX5_CAP64_DEV_MEM(dev, header_modify_sw_icm_start_address); | |
159 | log_icm_size = MLX5_CAP_DEV_MEM(dev, | |
160 | log_header_modify_sw_icm_size); | |
161 | block_map = dm->header_modify_sw_icm_alloc_blocks; | |
162 | break; | |
66765836 YK |
163 | case MLX5_SW_ICM_TYPE_HEADER_MODIFY_PATTERN: |
164 | icm_start_addr = MLX5_CAP64_DEV_MEM(dev, | |
165 | header_modify_pattern_sw_icm_start_address); | |
166 | log_icm_size = MLX5_CAP_DEV_MEM(dev, | |
167 | log_header_modify_pattern_sw_icm_size); | |
168 | block_map = dm->header_modify_pattern_sw_icm_alloc_blocks; | |
169 | break; | |
c9b9dcb4 AL |
170 | default: |
171 | return -EINVAL; | |
172 | } | |
173 | ||
174 | if (!block_map) | |
175 | return -EOPNOTSUPP; | |
176 | ||
177 | max_blocks = BIT(log_icm_size - MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)); | |
dff8e2d1 ES |
178 | |
179 | if (log_alignment < MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)) | |
180 | log_alignment = MLX5_LOG_SW_ICM_BLOCK_SIZE(dev); | |
181 | align_mask = BIT(log_alignment - MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)) - 1; | |
182 | ||
c9b9dcb4 | 183 | spin_lock(&dm->lock); |
dff8e2d1 ES |
184 | block_idx = bitmap_find_next_zero_area(block_map, max_blocks, 0, |
185 | num_blocks, align_mask); | |
c9b9dcb4 AL |
186 | |
187 | if (block_idx < max_blocks) | |
188 | bitmap_set(block_map, | |
189 | block_idx, num_blocks); | |
190 | ||
191 | spin_unlock(&dm->lock); | |
192 | ||
193 | if (block_idx >= max_blocks) | |
194 | return -ENOMEM; | |
195 | ||
196 | sw_icm = MLX5_ADDR_OF(create_sw_icm_in, in, sw_icm); | |
197 | icm_start_addr += block_idx << MLX5_LOG_SW_ICM_BLOCK_SIZE(dev); | |
198 | MLX5_SET64(sw_icm, sw_icm, sw_icm_start_addr, | |
199 | icm_start_addr); | |
200 | MLX5_SET(sw_icm, sw_icm, log_sw_icm_size, ilog2(length)); | |
201 | ||
202 | ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
203 | if (ret) { | |
204 | spin_lock(&dm->lock); | |
205 | bitmap_clear(block_map, | |
206 | block_idx, num_blocks); | |
207 | spin_unlock(&dm->lock); | |
208 | ||
209 | return ret; | |
210 | } | |
211 | ||
212 | *addr = icm_start_addr; | |
213 | *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); | |
214 | ||
215 | return 0; | |
216 | } | |
217 | EXPORT_SYMBOL_GPL(mlx5_dm_sw_icm_alloc); | |
218 | ||
219 | int mlx5_dm_sw_icm_dealloc(struct mlx5_core_dev *dev, enum mlx5_sw_icm_type type, | |
220 | u64 length, u16 uid, phys_addr_t addr, u32 obj_id) | |
221 | { | |
222 | u32 num_blocks = DIV_ROUND_UP_ULL(length, MLX5_SW_ICM_BLOCK_SIZE(dev)); | |
223 | u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {}; | |
224 | u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {}; | |
225 | struct mlx5_dm *dm = dev->dm; | |
226 | unsigned long *block_map; | |
227 | u64 icm_start_addr; | |
228 | u64 start_idx; | |
229 | int err; | |
230 | ||
231 | if (!dev->dm) | |
232 | return -EOPNOTSUPP; | |
233 | ||
234 | switch (type) { | |
235 | case MLX5_SW_ICM_TYPE_STEERING: | |
236 | icm_start_addr = MLX5_CAP64_DEV_MEM(dev, steering_sw_icm_start_address); | |
237 | block_map = dm->steering_sw_icm_alloc_blocks; | |
238 | break; | |
239 | case MLX5_SW_ICM_TYPE_HEADER_MODIFY: | |
240 | icm_start_addr = MLX5_CAP64_DEV_MEM(dev, header_modify_sw_icm_start_address); | |
241 | block_map = dm->header_modify_sw_icm_alloc_blocks; | |
242 | break; | |
66765836 YK |
243 | case MLX5_SW_ICM_TYPE_HEADER_MODIFY_PATTERN: |
244 | icm_start_addr = MLX5_CAP64_DEV_MEM(dev, | |
245 | header_modify_pattern_sw_icm_start_address); | |
246 | block_map = dm->header_modify_pattern_sw_icm_alloc_blocks; | |
247 | break; | |
c9b9dcb4 AL |
248 | default: |
249 | return -EINVAL; | |
250 | } | |
251 | ||
252 | MLX5_SET(general_obj_in_cmd_hdr, in, opcode, | |
253 | MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); | |
254 | MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_OBJ_TYPE_SW_ICM); | |
255 | MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id); | |
256 | MLX5_SET(general_obj_in_cmd_hdr, in, uid, uid); | |
257 | ||
258 | err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); | |
259 | if (err) | |
260 | return err; | |
261 | ||
262 | start_idx = (addr - icm_start_addr) >> MLX5_LOG_SW_ICM_BLOCK_SIZE(dev); | |
263 | spin_lock(&dm->lock); | |
264 | bitmap_clear(block_map, | |
265 | start_idx, num_blocks); | |
266 | spin_unlock(&dm->lock); | |
267 | ||
268 | return 0; | |
269 | } | |
270 | EXPORT_SYMBOL_GPL(mlx5_dm_sw_icm_dealloc); |