Commit | Line | Data |
---|---|---|
684b89bc SO |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * ISM driver for s390. | |
4 | * | |
5 | * Copyright IBM Corp. 2018 | |
6 | */ | |
7 | #define KMSG_COMPONENT "ism" | |
8 | #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt | |
9 | ||
10 | #include <linux/module.h> | |
11 | #include <linux/types.h> | |
12 | #include <linux/interrupt.h> | |
13 | #include <linux/device.h> | |
684b89bc | 14 | #include <linux/err.h> |
201091eb UB |
15 | #include <linux/ctype.h> |
16 | #include <linux/processor.h> | |
684b89bc SO |
17 | |
18 | #include "ism.h" | |
19 | ||
20 | MODULE_DESCRIPTION("ISM driver for s390"); | |
21 | MODULE_LICENSE("GPL"); | |
22 | ||
23 | #define PCI_DEVICE_ID_IBM_ISM 0x04ED | |
24 | #define DRV_NAME "ism" | |
25 | ||
26 | static const struct pci_device_id ism_device_table[] = { | |
27 | { PCI_VDEVICE(IBM, PCI_DEVICE_ID_IBM_ISM), 0 }, | |
28 | { 0, } | |
29 | }; | |
30 | MODULE_DEVICE_TABLE(pci, ism_device_table); | |
31 | ||
32 | static debug_info_t *ism_debug_info; | |
89e7d2ba SR |
33 | |
34 | #define NO_CLIENT 0xff /* must be >= MAX_CLIENTS */ | |
35 | static struct ism_client *clients[MAX_CLIENTS]; /* use an array rather than */ | |
36 | /* a list for fast mapping */ | |
37 | static u8 max_client; | |
76631ffa | 38 | static DEFINE_MUTEX(clients_lock); |
b3bf7602 | 39 | static bool ism_v2_capable; |
89e7d2ba SR |
40 | struct ism_dev_list { |
41 | struct list_head list; | |
42 | struct mutex mutex; /* protects ism device list */ | |
43 | }; | |
44 | ||
45 | static struct ism_dev_list ism_dev_list = { | |
46 | .list = LIST_HEAD_INIT(ism_dev_list.list), | |
47 | .mutex = __MUTEX_INITIALIZER(ism_dev_list.mutex), | |
48 | }; | |
49 | ||
6b5c13b5 NS |
50 | static void ism_setup_forwarding(struct ism_client *client, struct ism_dev *ism) |
51 | { | |
52 | unsigned long flags; | |
53 | ||
54 | spin_lock_irqsave(&ism->lock, flags); | |
55 | ism->subs[client->id] = client; | |
56 | spin_unlock_irqrestore(&ism->lock, flags); | |
57 | } | |
58 | ||
89e7d2ba SR |
59 | int ism_register_client(struct ism_client *client) |
60 | { | |
61 | struct ism_dev *ism; | |
89e7d2ba SR |
62 | int i, rc = -ENOSPC; |
63 | ||
64 | mutex_lock(&ism_dev_list.mutex); | |
76631ffa | 65 | mutex_lock(&clients_lock); |
89e7d2ba SR |
66 | for (i = 0; i < MAX_CLIENTS; ++i) { |
67 | if (!clients[i]) { | |
68 | clients[i] = client; | |
69 | client->id = i; | |
70 | if (i == max_client) | |
71 | max_client++; | |
72 | rc = 0; | |
73 | break; | |
74 | } | |
75 | } | |
76631ffa NS |
76 | mutex_unlock(&clients_lock); |
77 | ||
89e7d2ba SR |
78 | if (i < MAX_CLIENTS) { |
79 | /* initialize with all devices that we got so far */ | |
80 | list_for_each_entry(ism, &ism_dev_list.list, list) { | |
81 | ism->priv[i] = NULL; | |
82 | client->add(ism); | |
6b5c13b5 | 83 | ism_setup_forwarding(client, ism); |
89e7d2ba SR |
84 | } |
85 | } | |
86 | mutex_unlock(&ism_dev_list.mutex); | |
87 | ||
88 | return rc; | |
89 | } | |
90 | EXPORT_SYMBOL_GPL(ism_register_client); | |
91 | ||
92 | int ism_unregister_client(struct ism_client *client) | |
93 | { | |
94 | struct ism_dev *ism; | |
95 | unsigned long flags; | |
96 | int rc = 0; | |
97 | ||
98 | mutex_lock(&ism_dev_list.mutex); | |
89e7d2ba | 99 | list_for_each_entry(ism, &ism_dev_list.list, list) { |
6b5c13b5 NS |
100 | spin_lock_irqsave(&ism->lock, flags); |
101 | /* Stop forwarding IRQs and events */ | |
102 | ism->subs[client->id] = NULL; | |
89e7d2ba SR |
103 | for (int i = 0; i < ISM_NR_DMBS; ++i) { |
104 | if (ism->sba_client_arr[i] == client->id) { | |
266deeea NS |
105 | WARN(1, "%s: attempt to unregister '%s' with registered dmb(s)\n", |
106 | __func__, client->name); | |
89e7d2ba | 107 | rc = -EBUSY; |
266deeea | 108 | goto err_reg_dmb; |
89e7d2ba SR |
109 | } |
110 | } | |
6b5c13b5 | 111 | spin_unlock_irqrestore(&ism->lock, flags); |
89e7d2ba | 112 | } |
89e7d2ba SR |
113 | mutex_unlock(&ism_dev_list.mutex); |
114 | ||
266deeea NS |
115 | mutex_lock(&clients_lock); |
116 | clients[client->id] = NULL; | |
117 | if (client->id + 1 == max_client) | |
118 | max_client--; | |
119 | mutex_unlock(&clients_lock); | |
120 | return rc; | |
121 | ||
122 | err_reg_dmb: | |
123 | spin_unlock_irqrestore(&ism->lock, flags); | |
124 | mutex_unlock(&ism_dev_list.mutex); | |
89e7d2ba SR |
125 | return rc; |
126 | } | |
127 | EXPORT_SYMBOL_GPL(ism_unregister_client); | |
684b89bc SO |
128 | |
129 | static int ism_cmd(struct ism_dev *ism, void *cmd) | |
130 | { | |
131 | struct ism_req_hdr *req = cmd; | |
132 | struct ism_resp_hdr *resp = cmd; | |
133 | ||
c475f177 SO |
134 | __ism_write_cmd(ism, req + 1, sizeof(*req), req->len - sizeof(*req)); |
135 | __ism_write_cmd(ism, req, 0, sizeof(*req)); | |
684b89bc SO |
136 | |
137 | WRITE_ONCE(resp->ret, ISM_ERROR); | |
138 | ||
c475f177 | 139 | __ism_read_cmd(ism, resp, 0, sizeof(*resp)); |
684b89bc SO |
140 | if (resp->ret) { |
141 | debug_text_event(ism_debug_info, 0, "cmd failure"); | |
142 | debug_event(ism_debug_info, 0, resp, sizeof(*resp)); | |
143 | goto out; | |
144 | } | |
c475f177 | 145 | __ism_read_cmd(ism, resp + 1, sizeof(*resp), resp->len - sizeof(*resp)); |
684b89bc SO |
146 | out: |
147 | return resp->ret; | |
148 | } | |
149 | ||
150 | static int ism_cmd_simple(struct ism_dev *ism, u32 cmd_code) | |
151 | { | |
152 | union ism_cmd_simple cmd; | |
153 | ||
154 | memset(&cmd, 0, sizeof(cmd)); | |
155 | cmd.request.hdr.cmd = cmd_code; | |
156 | cmd.request.hdr.len = sizeof(cmd.request); | |
157 | ||
158 | return ism_cmd(ism, &cmd); | |
159 | } | |
160 | ||
161 | static int query_info(struct ism_dev *ism) | |
162 | { | |
163 | union ism_qi cmd; | |
164 | ||
165 | memset(&cmd, 0, sizeof(cmd)); | |
166 | cmd.request.hdr.cmd = ISM_QUERY_INFO; | |
167 | cmd.request.hdr.len = sizeof(cmd.request); | |
168 | ||
169 | if (ism_cmd(ism, &cmd)) | |
170 | goto out; | |
171 | ||
172 | debug_text_event(ism_debug_info, 3, "query info"); | |
173 | debug_event(ism_debug_info, 3, &cmd.response, sizeof(cmd.response)); | |
174 | out: | |
175 | return 0; | |
176 | } | |
177 | ||
178 | static int register_sba(struct ism_dev *ism) | |
179 | { | |
180 | union ism_reg_sba cmd; | |
181 | dma_addr_t dma_handle; | |
182 | struct ism_sba *sba; | |
183 | ||
750afb08 LC |
184 | sba = dma_alloc_coherent(&ism->pdev->dev, PAGE_SIZE, &dma_handle, |
185 | GFP_KERNEL); | |
684b89bc SO |
186 | if (!sba) |
187 | return -ENOMEM; | |
188 | ||
189 | memset(&cmd, 0, sizeof(cmd)); | |
190 | cmd.request.hdr.cmd = ISM_REG_SBA; | |
191 | cmd.request.hdr.len = sizeof(cmd.request); | |
192 | cmd.request.sba = dma_handle; | |
193 | ||
194 | if (ism_cmd(ism, &cmd)) { | |
195 | dma_free_coherent(&ism->pdev->dev, PAGE_SIZE, sba, dma_handle); | |
196 | return -EIO; | |
197 | } | |
198 | ||
199 | ism->sba = sba; | |
200 | ism->sba_dma_addr = dma_handle; | |
201 | ||
202 | return 0; | |
203 | } | |
204 | ||
205 | static int register_ieq(struct ism_dev *ism) | |
206 | { | |
207 | union ism_reg_ieq cmd; | |
208 | dma_addr_t dma_handle; | |
209 | struct ism_eq *ieq; | |
210 | ||
750afb08 LC |
211 | ieq = dma_alloc_coherent(&ism->pdev->dev, PAGE_SIZE, &dma_handle, |
212 | GFP_KERNEL); | |
684b89bc SO |
213 | if (!ieq) |
214 | return -ENOMEM; | |
215 | ||
216 | memset(&cmd, 0, sizeof(cmd)); | |
217 | cmd.request.hdr.cmd = ISM_REG_IEQ; | |
218 | cmd.request.hdr.len = sizeof(cmd.request); | |
219 | cmd.request.ieq = dma_handle; | |
220 | cmd.request.len = sizeof(*ieq); | |
221 | ||
222 | if (ism_cmd(ism, &cmd)) { | |
223 | dma_free_coherent(&ism->pdev->dev, PAGE_SIZE, ieq, dma_handle); | |
224 | return -EIO; | |
225 | } | |
226 | ||
227 | ism->ieq = ieq; | |
228 | ism->ieq_idx = -1; | |
229 | ism->ieq_dma_addr = dma_handle; | |
230 | ||
231 | return 0; | |
232 | } | |
233 | ||
234 | static int unregister_sba(struct ism_dev *ism) | |
235 | { | |
0ff06c44 SO |
236 | int ret; |
237 | ||
684b89bc SO |
238 | if (!ism->sba) |
239 | return 0; | |
240 | ||
0ff06c44 SO |
241 | ret = ism_cmd_simple(ism, ISM_UNREG_SBA); |
242 | if (ret && ret != ISM_ERROR) | |
684b89bc SO |
243 | return -EIO; |
244 | ||
245 | dma_free_coherent(&ism->pdev->dev, PAGE_SIZE, | |
246 | ism->sba, ism->sba_dma_addr); | |
247 | ||
248 | ism->sba = NULL; | |
249 | ism->sba_dma_addr = 0; | |
250 | ||
251 | return 0; | |
252 | } | |
253 | ||
254 | static int unregister_ieq(struct ism_dev *ism) | |
255 | { | |
0ff06c44 SO |
256 | int ret; |
257 | ||
684b89bc SO |
258 | if (!ism->ieq) |
259 | return 0; | |
260 | ||
0ff06c44 SO |
261 | ret = ism_cmd_simple(ism, ISM_UNREG_IEQ); |
262 | if (ret && ret != ISM_ERROR) | |
684b89bc SO |
263 | return -EIO; |
264 | ||
265 | dma_free_coherent(&ism->pdev->dev, PAGE_SIZE, | |
266 | ism->ieq, ism->ieq_dma_addr); | |
267 | ||
268 | ism->ieq = NULL; | |
269 | ism->ieq_dma_addr = 0; | |
270 | ||
271 | return 0; | |
272 | } | |
273 | ||
274 | static int ism_read_local_gid(struct ism_dev *ism) | |
275 | { | |
276 | union ism_read_gid cmd; | |
277 | int ret; | |
278 | ||
279 | memset(&cmd, 0, sizeof(cmd)); | |
280 | cmd.request.hdr.cmd = ISM_READ_GID; | |
281 | cmd.request.hdr.len = sizeof(cmd.request); | |
282 | ||
283 | ret = ism_cmd(ism, &cmd); | |
284 | if (ret) | |
285 | goto out; | |
286 | ||
89e7d2ba | 287 | ism->local_gid = cmd.response.gid; |
684b89bc SO |
288 | out: |
289 | return ret; | |
290 | } | |
291 | ||
1baedb13 | 292 | static void ism_free_dmb(struct ism_dev *ism, struct ism_dmb *dmb) |
684b89bc SO |
293 | { |
294 | clear_bit(dmb->sba_idx, ism->sba_bitmap); | |
83781384 GB |
295 | dma_unmap_page(&ism->pdev->dev, dmb->dma_addr, dmb->dmb_len, |
296 | DMA_FROM_DEVICE); | |
297 | folio_put(virt_to_folio(dmb->cpu_addr)); | |
684b89bc SO |
298 | } |
299 | ||
1baedb13 | 300 | static int ism_alloc_dmb(struct ism_dev *ism, struct ism_dmb *dmb) |
684b89bc | 301 | { |
83781384 | 302 | struct folio *folio; |
684b89bc | 303 | unsigned long bit; |
83781384 | 304 | int rc; |
684b89bc SO |
305 | |
306 | if (PAGE_ALIGN(dmb->dmb_len) > dma_get_max_seg_size(&ism->pdev->dev)) | |
307 | return -EINVAL; | |
308 | ||
309 | if (!dmb->sba_idx) { | |
310 | bit = find_next_zero_bit(ism->sba_bitmap, ISM_NR_DMBS, | |
311 | ISM_DMB_BIT_OFFSET); | |
312 | if (bit == ISM_NR_DMBS) | |
211e5b7a | 313 | return -ENOSPC; |
684b89bc SO |
314 | |
315 | dmb->sba_idx = bit; | |
316 | } | |
317 | if (dmb->sba_idx < ISM_DMB_BIT_OFFSET || | |
318 | test_and_set_bit(dmb->sba_idx, ism->sba_bitmap)) | |
319 | return -EINVAL; | |
320 | ||
83781384 GB |
321 | folio = folio_alloc(GFP_KERNEL | __GFP_NOWARN | __GFP_NOMEMALLOC | |
322 | __GFP_NORETRY, get_order(dmb->dmb_len)); | |
684b89bc | 323 | |
83781384 GB |
324 | if (!folio) { |
325 | rc = -ENOMEM; | |
326 | goto out_bit; | |
327 | } | |
328 | ||
329 | dmb->cpu_addr = folio_address(folio); | |
330 | dmb->dma_addr = dma_map_page(&ism->pdev->dev, | |
331 | virt_to_page(dmb->cpu_addr), 0, | |
332 | dmb->dmb_len, DMA_FROM_DEVICE); | |
333 | if (dma_mapping_error(&ism->pdev->dev, dmb->dma_addr)) { | |
334 | rc = -ENOMEM; | |
335 | goto out_free; | |
336 | } | |
337 | ||
338 | return 0; | |
339 | ||
340 | out_free: | |
341 | kfree(dmb->cpu_addr); | |
342 | out_bit: | |
343 | clear_bit(dmb->sba_idx, ism->sba_bitmap); | |
344 | return rc; | |
684b89bc SO |
345 | } |
346 | ||
9de4df7b SR |
347 | int ism_register_dmb(struct ism_dev *ism, struct ism_dmb *dmb, |
348 | struct ism_client *client) | |
684b89bc | 349 | { |
684b89bc | 350 | union ism_reg_dmb cmd; |
6b5c13b5 | 351 | unsigned long flags; |
684b89bc SO |
352 | int ret; |
353 | ||
354 | ret = ism_alloc_dmb(ism, dmb); | |
355 | if (ret) | |
356 | goto out; | |
357 | ||
358 | memset(&cmd, 0, sizeof(cmd)); | |
359 | cmd.request.hdr.cmd = ISM_REG_DMB; | |
360 | cmd.request.hdr.len = sizeof(cmd.request); | |
361 | ||
362 | cmd.request.dmb = dmb->dma_addr; | |
363 | cmd.request.dmb_len = dmb->dmb_len; | |
364 | cmd.request.sba_idx = dmb->sba_idx; | |
365 | cmd.request.vlan_valid = dmb->vlan_valid; | |
366 | cmd.request.vlan_id = dmb->vlan_id; | |
367 | cmd.request.rgid = dmb->rgid; | |
368 | ||
369 | ret = ism_cmd(ism, &cmd); | |
370 | if (ret) { | |
371 | ism_free_dmb(ism, dmb); | |
372 | goto out; | |
373 | } | |
374 | dmb->dmb_tok = cmd.response.dmb_tok; | |
6b5c13b5 | 375 | spin_lock_irqsave(&ism->lock, flags); |
9de4df7b | 376 | ism->sba_client_arr[dmb->sba_idx - ISM_DMB_BIT_OFFSET] = client->id; |
6b5c13b5 | 377 | spin_unlock_irqrestore(&ism->lock, flags); |
684b89bc SO |
378 | out: |
379 | return ret; | |
380 | } | |
9de4df7b | 381 | EXPORT_SYMBOL_GPL(ism_register_dmb); |
684b89bc | 382 | |
9de4df7b | 383 | int ism_unregister_dmb(struct ism_dev *ism, struct ism_dmb *dmb) |
684b89bc | 384 | { |
684b89bc | 385 | union ism_unreg_dmb cmd; |
6b5c13b5 | 386 | unsigned long flags; |
684b89bc SO |
387 | int ret; |
388 | ||
389 | memset(&cmd, 0, sizeof(cmd)); | |
390 | cmd.request.hdr.cmd = ISM_UNREG_DMB; | |
391 | cmd.request.hdr.len = sizeof(cmd.request); | |
392 | ||
393 | cmd.request.dmb_tok = dmb->dmb_tok; | |
394 | ||
6b5c13b5 | 395 | spin_lock_irqsave(&ism->lock, flags); |
9de4df7b | 396 | ism->sba_client_arr[dmb->sba_idx - ISM_DMB_BIT_OFFSET] = NO_CLIENT; |
6b5c13b5 | 397 | spin_unlock_irqrestore(&ism->lock, flags); |
9de4df7b | 398 | |
684b89bc | 399 | ret = ism_cmd(ism, &cmd); |
0ff06c44 | 400 | if (ret && ret != ISM_ERROR) |
684b89bc SO |
401 | goto out; |
402 | ||
403 | ism_free_dmb(ism, dmb); | |
404 | out: | |
405 | return ret; | |
406 | } | |
9de4df7b | 407 | EXPORT_SYMBOL_GPL(ism_unregister_dmb); |
684b89bc | 408 | |
9de4df7b | 409 | static int ism_add_vlan_id(struct ism_dev *ism, u64 vlan_id) |
684b89bc | 410 | { |
684b89bc SO |
411 | union ism_set_vlan_id cmd; |
412 | ||
413 | memset(&cmd, 0, sizeof(cmd)); | |
414 | cmd.request.hdr.cmd = ISM_ADD_VLAN_ID; | |
415 | cmd.request.hdr.len = sizeof(cmd.request); | |
416 | ||
417 | cmd.request.vlan_id = vlan_id; | |
418 | ||
419 | return ism_cmd(ism, &cmd); | |
420 | } | |
421 | ||
9de4df7b | 422 | static int ism_del_vlan_id(struct ism_dev *ism, u64 vlan_id) |
684b89bc | 423 | { |
684b89bc SO |
424 | union ism_set_vlan_id cmd; |
425 | ||
426 | memset(&cmd, 0, sizeof(cmd)); | |
427 | cmd.request.hdr.cmd = ISM_DEL_VLAN_ID; | |
428 | cmd.request.hdr.len = sizeof(cmd.request); | |
429 | ||
430 | cmd.request.vlan_id = vlan_id; | |
431 | ||
432 | return ism_cmd(ism, &cmd); | |
433 | } | |
434 | ||
684b89bc SO |
435 | static unsigned int max_bytes(unsigned int start, unsigned int len, |
436 | unsigned int boundary) | |
437 | { | |
438 | return min(boundary - (start & (boundary - 1)), len); | |
439 | } | |
440 | ||
820f2100 SR |
441 | int ism_move(struct ism_dev *ism, u64 dmb_tok, unsigned int idx, bool sf, |
442 | unsigned int offset, void *data, unsigned int size) | |
684b89bc | 443 | { |
684b89bc SO |
444 | unsigned int bytes; |
445 | u64 dmb_req; | |
446 | int ret; | |
447 | ||
448 | while (size) { | |
449 | bytes = max_bytes(offset, size, PAGE_SIZE); | |
450 | dmb_req = ISM_CREATE_REQ(dmb_tok, idx, size == bytes ? sf : 0, | |
451 | offset); | |
452 | ||
453 | ret = __ism_move(ism, dmb_req, data, bytes); | |
454 | if (ret) | |
455 | return ret; | |
456 | ||
457 | size -= bytes; | |
458 | data += bytes; | |
459 | offset += bytes; | |
460 | } | |
461 | ||
462 | return 0; | |
463 | } | |
820f2100 | 464 | EXPORT_SYMBOL_GPL(ism_move); |
684b89bc SO |
465 | |
466 | static void ism_handle_event(struct ism_dev *ism) | |
467 | { | |
89e7d2ba | 468 | struct ism_event *entry; |
6b5c13b5 | 469 | struct ism_client *clt; |
89e7d2ba | 470 | int i; |
684b89bc SO |
471 | |
472 | while ((ism->ieq_idx + 1) != READ_ONCE(ism->ieq->header.idx)) { | |
473 | if (++(ism->ieq_idx) == ARRAY_SIZE(ism->ieq->entry)) | |
474 | ism->ieq_idx = 0; | |
475 | ||
476 | entry = &ism->ieq->entry[ism->ieq_idx]; | |
477 | debug_event(ism_debug_info, 2, entry, sizeof(*entry)); | |
6b5c13b5 NS |
478 | for (i = 0; i < max_client; ++i) { |
479 | clt = ism->subs[i]; | |
480 | if (clt) | |
481 | clt->handle_event(ism, entry); | |
482 | } | |
684b89bc SO |
483 | } |
484 | } | |
485 | ||
486 | static irqreturn_t ism_handle_irq(int irq, void *data) | |
487 | { | |
488 | struct ism_dev *ism = data; | |
489 | unsigned long bit, end; | |
490 | unsigned long *bv; | |
8b2fed8e | 491 | u16 dmbemask; |
6b5c13b5 | 492 | u8 client_id; |
684b89bc SO |
493 | |
494 | bv = (void *) &ism->sba->dmb_bits[ISM_DMB_WORD_OFFSET]; | |
495 | end = sizeof(ism->sba->dmb_bits) * BITS_PER_BYTE - ISM_DMB_BIT_OFFSET; | |
496 | ||
497 | spin_lock(&ism->lock); | |
498 | ism->sba->s = 0; | |
499 | barrier(); | |
500 | for (bit = 0;;) { | |
501 | bit = find_next_bit_inv(bv, end, bit); | |
502 | if (bit >= end) | |
503 | break; | |
504 | ||
505 | clear_bit_inv(bit, bv); | |
8b2fed8e | 506 | dmbemask = ism->sba->dmbe_mask[bit + ISM_DMB_BIT_OFFSET]; |
007b6568 | 507 | ism->sba->dmbe_mask[bit + ISM_DMB_BIT_OFFSET] = 0; |
684b89bc | 508 | barrier(); |
6b5c13b5 NS |
509 | client_id = ism->sba_client_arr[bit]; |
510 | if (unlikely(client_id == NO_CLIENT || !ism->subs[client_id])) | |
511 | continue; | |
512 | ism->subs[client_id]->handle_irq(ism, bit + ISM_DMB_BIT_OFFSET, dmbemask); | |
684b89bc SO |
513 | } |
514 | ||
515 | if (ism->sba->e) { | |
516 | ism->sba->e = 0; | |
517 | barrier(); | |
518 | ism_handle_event(ism); | |
519 | } | |
520 | spin_unlock(&ism->lock); | |
521 | return IRQ_HANDLED; | |
522 | } | |
523 | ||
684b89bc SO |
524 | static int ism_dev_init(struct ism_dev *ism) |
525 | { | |
526 | struct pci_dev *pdev = ism->pdev; | |
89e7d2ba | 527 | int i, ret; |
684b89bc SO |
528 | |
529 | ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI); | |
530 | if (ret <= 0) | |
531 | goto out; | |
532 | ||
9de4df7b SR |
533 | ism->sba_client_arr = kzalloc(ISM_NR_DMBS, GFP_KERNEL); |
534 | if (!ism->sba_client_arr) | |
535 | goto free_vectors; | |
536 | memset(ism->sba_client_arr, NO_CLIENT, ISM_NR_DMBS); | |
537 | ||
684b89bc SO |
538 | ret = request_irq(pci_irq_vector(pdev, 0), ism_handle_irq, 0, |
539 | pci_name(pdev), ism); | |
540 | if (ret) | |
9de4df7b | 541 | goto free_client_arr; |
684b89bc SO |
542 | |
543 | ret = register_sba(ism); | |
544 | if (ret) | |
545 | goto free_irq; | |
546 | ||
547 | ret = register_ieq(ism); | |
548 | if (ret) | |
549 | goto unreg_sba; | |
550 | ||
551 | ret = ism_read_local_gid(ism); | |
552 | if (ret) | |
553 | goto unreg_ieq; | |
554 | ||
9de4df7b | 555 | if (!ism_add_vlan_id(ism, ISM_RESERVED_VLANID)) |
201091eb | 556 | /* hardware is V2 capable */ |
b3bf7602 WG |
557 | ism_v2_capable = true; |
558 | else | |
559 | ism_v2_capable = false; | |
201091eb | 560 | |
76631ffa NS |
561 | mutex_lock(&ism_dev_list.mutex); |
562 | mutex_lock(&clients_lock); | |
563 | for (i = 0; i < max_client; ++i) { | |
89e7d2ba | 564 | if (clients[i]) { |
76631ffa NS |
565 | clients[i]->add(ism); |
566 | ism_setup_forwarding(clients[i], ism); | |
89e7d2ba | 567 | } |
76631ffa NS |
568 | } |
569 | mutex_unlock(&clients_lock); | |
89e7d2ba | 570 | |
89e7d2ba SR |
571 | list_add(&ism->list, &ism_dev_list.list); |
572 | mutex_unlock(&ism_dev_list.mutex); | |
573 | ||
684b89bc SO |
574 | query_info(ism); |
575 | return 0; | |
576 | ||
577 | unreg_ieq: | |
578 | unregister_ieq(ism); | |
579 | unreg_sba: | |
580 | unregister_sba(ism); | |
581 | free_irq: | |
582 | free_irq(pci_irq_vector(pdev, 0), ism); | |
9de4df7b SR |
583 | free_client_arr: |
584 | kfree(ism->sba_client_arr); | |
684b89bc SO |
585 | free_vectors: |
586 | pci_free_irq_vectors(pdev); | |
587 | out: | |
588 | return ret; | |
589 | } | |
590 | ||
591 | static int ism_probe(struct pci_dev *pdev, const struct pci_device_id *id) | |
592 | { | |
593 | struct ism_dev *ism; | |
594 | int ret; | |
595 | ||
596 | ism = kzalloc(sizeof(*ism), GFP_KERNEL); | |
597 | if (!ism) | |
598 | return -ENOMEM; | |
599 | ||
600 | spin_lock_init(&ism->lock); | |
601 | dev_set_drvdata(&pdev->dev, ism); | |
602 | ism->pdev = pdev; | |
8c81ba20 SR |
603 | ism->dev.parent = &pdev->dev; |
604 | device_initialize(&ism->dev); | |
605 | dev_set_name(&ism->dev, dev_name(&pdev->dev)); | |
606 | ret = device_add(&ism->dev); | |
607 | if (ret) | |
608 | goto err_dev; | |
684b89bc SO |
609 | |
610 | ret = pci_enable_device_mem(pdev); | |
611 | if (ret) | |
612 | goto err; | |
613 | ||
614 | ret = pci_request_mem_regions(pdev, DRV_NAME); | |
615 | if (ret) | |
616 | goto err_disable; | |
617 | ||
657d42cf | 618 | ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); |
684b89bc | 619 | if (ret) |
c475f177 | 620 | goto err_resource; |
684b89bc | 621 | |
a6f44cf9 | 622 | dma_set_seg_boundary(&pdev->dev, SZ_1M - 1); |
b0da3498 | 623 | dma_set_max_seg_size(&pdev->dev, SZ_1M); |
684b89bc SO |
624 | pci_set_master(pdev); |
625 | ||
684b89bc SO |
626 | ret = ism_dev_init(ism); |
627 | if (ret) | |
8c81ba20 | 628 | goto err_resource; |
684b89bc SO |
629 | |
630 | return 0; | |
631 | ||
684b89bc SO |
632 | err_resource: |
633 | pci_release_mem_regions(pdev); | |
634 | err_disable: | |
635 | pci_disable_device(pdev); | |
636 | err: | |
8c81ba20 SR |
637 | device_del(&ism->dev); |
638 | err_dev: | |
684b89bc | 639 | dev_set_drvdata(&pdev->dev, NULL); |
8c81ba20 SR |
640 | kfree(ism); |
641 | ||
684b89bc SO |
642 | return ret; |
643 | } | |
644 | ||
645 | static void ism_dev_exit(struct ism_dev *ism) | |
646 | { | |
647 | struct pci_dev *pdev = ism->pdev; | |
89e7d2ba SR |
648 | unsigned long flags; |
649 | int i; | |
650 | ||
76631ffa | 651 | spin_lock_irqsave(&ism->lock, flags); |
89e7d2ba | 652 | for (i = 0; i < max_client; ++i) |
76631ffa NS |
653 | ism->subs[i] = NULL; |
654 | spin_unlock_irqrestore(&ism->lock, flags); | |
89e7d2ba | 655 | |
76631ffa NS |
656 | mutex_lock(&ism_dev_list.mutex); |
657 | mutex_lock(&clients_lock); | |
658 | for (i = 0; i < max_client; ++i) { | |
659 | if (clients[i]) | |
660 | clients[i]->remove(ism); | |
661 | } | |
662 | mutex_unlock(&clients_lock); | |
684b89bc | 663 | |
b3bf7602 | 664 | if (ism_v2_capable) |
9de4df7b | 665 | ism_del_vlan_id(ism, ISM_RESERVED_VLANID); |
684b89bc SO |
666 | unregister_ieq(ism); |
667 | unregister_sba(ism); | |
668 | free_irq(pci_irq_vector(pdev, 0), ism); | |
9de4df7b | 669 | kfree(ism->sba_client_arr); |
684b89bc | 670 | pci_free_irq_vectors(pdev); |
89e7d2ba | 671 | list_del_init(&ism->list); |
76631ffa | 672 | mutex_unlock(&ism_dev_list.mutex); |
684b89bc SO |
673 | } |
674 | ||
675 | static void ism_remove(struct pci_dev *pdev) | |
676 | { | |
677 | struct ism_dev *ism = dev_get_drvdata(&pdev->dev); | |
678 | ||
679 | ism_dev_exit(ism); | |
680 | ||
684b89bc SO |
681 | pci_release_mem_regions(pdev); |
682 | pci_disable_device(pdev); | |
89e7d2ba | 683 | device_del(&ism->dev); |
684b89bc SO |
684 | dev_set_drvdata(&pdev->dev, NULL); |
685 | kfree(ism); | |
686 | } | |
687 | ||
684b89bc SO |
688 | static struct pci_driver ism_driver = { |
689 | .name = DRV_NAME, | |
690 | .id_table = ism_device_table, | |
691 | .probe = ism_probe, | |
692 | .remove = ism_remove, | |
684b89bc SO |
693 | }; |
694 | ||
695 | static int __init ism_init(void) | |
696 | { | |
697 | int ret; | |
698 | ||
699 | ism_debug_info = debug_register("ism", 2, 1, 16); | |
700 | if (!ism_debug_info) | |
701 | return -ENODEV; | |
702 | ||
89e7d2ba SR |
703 | memset(clients, 0, sizeof(clients)); |
704 | max_client = 0; | |
684b89bc SO |
705 | debug_register_view(ism_debug_info, &debug_hex_ascii_view); |
706 | ret = pci_register_driver(&ism_driver); | |
707 | if (ret) | |
708 | debug_unregister(ism_debug_info); | |
709 | ||
710 | return ret; | |
711 | } | |
712 | ||
713 | static void __exit ism_exit(void) | |
714 | { | |
715 | pci_unregister_driver(&ism_driver); | |
716 | debug_unregister(ism_debug_info); | |
717 | } | |
718 | ||
719 | module_init(ism_init); | |
720 | module_exit(ism_exit); | |
820f2100 SR |
721 | |
722 | /*************************** SMC-D Implementation *****************************/ | |
723 | ||
724 | #if IS_ENABLED(CONFIG_SMC) | |
d565fa43 GB |
725 | static int ism_query_rgid(struct ism_dev *ism, u64 rgid, u32 vid_valid, |
726 | u32 vid) | |
727 | { | |
728 | union ism_query_rgid cmd; | |
729 | ||
730 | memset(&cmd, 0, sizeof(cmd)); | |
731 | cmd.request.hdr.cmd = ISM_QUERY_RGID; | |
732 | cmd.request.hdr.len = sizeof(cmd.request); | |
733 | ||
734 | cmd.request.rgid = rgid; | |
735 | cmd.request.vlan_valid = vid_valid; | |
736 | cmd.request.vlan_id = vid; | |
737 | ||
738 | return ism_cmd(ism, &cmd); | |
739 | } | |
740 | ||
b40584d1 WG |
741 | static int smcd_query_rgid(struct smcd_dev *smcd, struct smcd_gid *rgid, |
742 | u32 vid_valid, u32 vid) | |
820f2100 | 743 | { |
b40584d1 | 744 | return ism_query_rgid(smcd->priv, rgid->gid, vid_valid, vid); |
820f2100 SR |
745 | } |
746 | ||
747 | static int smcd_register_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb, | |
748 | struct ism_client *client) | |
749 | { | |
750 | return ism_register_dmb(smcd->priv, (struct ism_dmb *)dmb, client); | |
751 | } | |
752 | ||
753 | static int smcd_unregister_dmb(struct smcd_dev *smcd, struct smcd_dmb *dmb) | |
754 | { | |
755 | return ism_unregister_dmb(smcd->priv, (struct ism_dmb *)dmb); | |
756 | } | |
757 | ||
758 | static int smcd_add_vlan_id(struct smcd_dev *smcd, u64 vlan_id) | |
759 | { | |
760 | return ism_add_vlan_id(smcd->priv, vlan_id); | |
761 | } | |
762 | ||
763 | static int smcd_del_vlan_id(struct smcd_dev *smcd, u64 vlan_id) | |
764 | { | |
765 | return ism_del_vlan_id(smcd->priv, vlan_id); | |
766 | } | |
767 | ||
768 | static int smcd_set_vlan_required(struct smcd_dev *smcd) | |
769 | { | |
770 | return ism_cmd_simple(smcd->priv, ISM_SET_VLAN); | |
771 | } | |
772 | ||
773 | static int smcd_reset_vlan_required(struct smcd_dev *smcd) | |
774 | { | |
775 | return ism_cmd_simple(smcd->priv, ISM_RESET_VLAN); | |
776 | } | |
777 | ||
d565fa43 GB |
778 | static int ism_signal_ieq(struct ism_dev *ism, u64 rgid, u32 trigger_irq, |
779 | u32 event_code, u64 info) | |
780 | { | |
781 | union ism_sig_ieq cmd; | |
782 | ||
783 | memset(&cmd, 0, sizeof(cmd)); | |
784 | cmd.request.hdr.cmd = ISM_SIGNAL_IEQ; | |
785 | cmd.request.hdr.len = sizeof(cmd.request); | |
786 | ||
787 | cmd.request.rgid = rgid; | |
788 | cmd.request.trigger_irq = trigger_irq; | |
789 | cmd.request.event_code = event_code; | |
790 | cmd.request.info = info; | |
791 | ||
792 | return ism_cmd(ism, &cmd); | |
793 | } | |
794 | ||
b40584d1 WG |
795 | static int smcd_signal_ieq(struct smcd_dev *smcd, struct smcd_gid *rgid, |
796 | u32 trigger_irq, u32 event_code, u64 info) | |
820f2100 | 797 | { |
b40584d1 WG |
798 | return ism_signal_ieq(smcd->priv, rgid->gid, |
799 | trigger_irq, event_code, info); | |
820f2100 SR |
800 | } |
801 | ||
802 | static int smcd_move(struct smcd_dev *smcd, u64 dmb_tok, unsigned int idx, | |
803 | bool sf, unsigned int offset, void *data, | |
804 | unsigned int size) | |
805 | { | |
806 | return ism_move(smcd->priv, dmb_tok, idx, sf, offset, data, size); | |
807 | } | |
808 | ||
f947568e SR |
809 | static int smcd_supports_v2(void) |
810 | { | |
b3bf7602 | 811 | return ism_v2_capable; |
f947568e SR |
812 | } |
813 | ||
d565fa43 GB |
814 | static u64 ism_get_local_gid(struct ism_dev *ism) |
815 | { | |
816 | return ism->local_gid; | |
817 | } | |
818 | ||
b40584d1 WG |
819 | static void smcd_get_local_gid(struct smcd_dev *smcd, |
820 | struct smcd_gid *smcd_gid) | |
820f2100 | 821 | { |
b40584d1 WG |
822 | smcd_gid->gid = ism_get_local_gid(smcd->priv); |
823 | smcd_gid->gid_ext = 0; | |
820f2100 SR |
824 | } |
825 | ||
d565fa43 GB |
826 | static u16 ism_get_chid(struct ism_dev *ism) |
827 | { | |
828 | if (!ism || !ism->pdev) | |
829 | return 0; | |
830 | ||
831 | return to_zpci(ism->pdev)->pchid; | |
832 | } | |
833 | ||
820f2100 SR |
834 | static u16 smcd_get_chid(struct smcd_dev *smcd) |
835 | { | |
836 | return ism_get_chid(smcd->priv); | |
837 | } | |
838 | ||
839 | static inline struct device *smcd_get_dev(struct smcd_dev *dev) | |
840 | { | |
841 | struct ism_dev *ism = dev->priv; | |
842 | ||
843 | return &ism->dev; | |
844 | } | |
845 | ||
846 | static const struct smcd_ops ism_ops = { | |
847 | .query_remote_gid = smcd_query_rgid, | |
848 | .register_dmb = smcd_register_dmb, | |
849 | .unregister_dmb = smcd_unregister_dmb, | |
850 | .add_vlan_id = smcd_add_vlan_id, | |
851 | .del_vlan_id = smcd_del_vlan_id, | |
852 | .set_vlan_required = smcd_set_vlan_required, | |
853 | .reset_vlan_required = smcd_reset_vlan_required, | |
854 | .signal_event = smcd_signal_ieq, | |
855 | .move_data = smcd_move, | |
f947568e | 856 | .supports_v2 = smcd_supports_v2, |
820f2100 SR |
857 | .get_local_gid = smcd_get_local_gid, |
858 | .get_chid = smcd_get_chid, | |
8c81ba20 | 859 | .get_dev = smcd_get_dev, |
820f2100 SR |
860 | }; |
861 | ||
862 | const struct smcd_ops *ism_get_smcd_ops(void) | |
863 | { | |
864 | return &ism_ops; | |
865 | } | |
866 | EXPORT_SYMBOL_GPL(ism_get_smcd_ops); | |
867 | #endif |