Commit | Line | Data |
---|---|---|
3bbfe987 SH |
1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* | |
3 | * Arm Firmware Framework for ARMv8-A(FFA) interface driver | |
4 | * | |
5 | * The Arm FFA specification[1] describes a software architecture to | |
6 | * leverages the virtualization extension to isolate software images | |
7 | * provided by an ecosystem of vendors from each other and describes | |
8 | * interfaces that standardize communication between the various software | |
9 | * images including communication between images in the Secure world and | |
10 | * Normal world. Any Hypervisor could use the FFA interfaces to enable | |
11 | * communication between VMs it manages. | |
12 | * | |
13 | * The Hypervisor a.k.a Partition managers in FFA terminology can assign | |
14 | * system resources(Memory regions, Devices, CPU cycles) to the partitions | |
15 | * and manage isolation amongst them. | |
16 | * | |
17 | * [1] https://developer.arm.com/docs/den0077/latest | |
18 | * | |
19 | * Copyright (C) 2021 ARM Ltd. | |
20 | */ | |
21 | ||
22 | #define DRIVER_NAME "ARM FF-A" | |
23 | #define pr_fmt(fmt) DRIVER_NAME ": " fmt | |
24 | ||
897e9e60 | 25 | #include <linux/acpi.h> |
3bbfe987 SH |
26 | #include <linux/arm_ffa.h> |
27 | #include <linux/bitfield.h> | |
897e9e60 | 28 | #include <linux/cpuhotplug.h> |
d0c0bce8 | 29 | #include <linux/device.h> |
e0573444 | 30 | #include <linux/hashtable.h> |
897e9e60 | 31 | #include <linux/interrupt.h> |
3bbfe987 | 32 | #include <linux/io.h> |
d0c0bce8 | 33 | #include <linux/kernel.h> |
3bbfe987 | 34 | #include <linux/module.h> |
cc2195fe | 35 | #include <linux/mm.h> |
e0573444 | 36 | #include <linux/mutex.h> |
897e9e60 | 37 | #include <linux/of_irq.h> |
cc2195fe | 38 | #include <linux/scatterlist.h> |
3bbfe987 | 39 | #include <linux/slab.h> |
897e9e60 | 40 | #include <linux/smp.h> |
d0c0bce8 | 41 | #include <linux/uuid.h> |
0184450b | 42 | #include <linux/xarray.h> |
3bbfe987 SH |
43 | |
44 | #include "common.h" | |
45 | ||
bcefd1bf | 46 | #define FFA_DRIVER_VERSION FFA_VERSION_1_1 |
3bbfe987 SH |
47 | #define FFA_MIN_VERSION FFA_VERSION_1_0 |
48 | ||
49 | #define SENDER_ID_MASK GENMASK(31, 16) | |
50 | #define RECEIVER_ID_MASK GENMASK(15, 0) | |
51 | #define SENDER_ID(x) ((u16)(FIELD_GET(SENDER_ID_MASK, (x)))) | |
52 | #define RECEIVER_ID(x) ((u16)(FIELD_GET(RECEIVER_ID_MASK, (x)))) | |
53 | #define PACK_TARGET_INFO(s, r) \ | |
54 | (FIELD_PREP(SENDER_ID_MASK, (s)) | FIELD_PREP(RECEIVER_ID_MASK, (r))) | |
55 | ||
3bbfe987 SH |
56 | /* |
57 | * Keeping RX TX buffer size as 4K for now | |
58 | * 64K may be preferred to keep it min a page in 64K PAGE_SIZE config | |
59 | */ | |
60 | #define RXTX_BUFFER_SIZE SZ_4K | |
61 | ||
e0573444 SH |
62 | #define FFA_MAX_NOTIFICATIONS 64 |
63 | ||
3bbfe987 SH |
64 | static ffa_fn *invoke_ffa_fn; |
65 | ||
66 | static const int ffa_linux_errmap[] = { | |
67 | /* better than switch case as long as return value is continuous */ | |
68 | 0, /* FFA_RET_SUCCESS */ | |
69 | -EOPNOTSUPP, /* FFA_RET_NOT_SUPPORTED */ | |
70 | -EINVAL, /* FFA_RET_INVALID_PARAMETERS */ | |
71 | -ENOMEM, /* FFA_RET_NO_MEMORY */ | |
72 | -EBUSY, /* FFA_RET_BUSY */ | |
73 | -EINTR, /* FFA_RET_INTERRUPTED */ | |
74 | -EACCES, /* FFA_RET_DENIED */ | |
75 | -EAGAIN, /* FFA_RET_RETRY */ | |
76 | -ECANCELED, /* FFA_RET_ABORTED */ | |
1609626c | 77 | -ENODATA, /* FFA_RET_NO_DATA */ |
3bbfe987 SH |
78 | }; |
79 | ||
80 | static inline int ffa_to_linux_errno(int errno) | |
81 | { | |
dd925db6 SH |
82 | int err_idx = -errno; |
83 | ||
84 | if (err_idx >= 0 && err_idx < ARRAY_SIZE(ffa_linux_errmap)) | |
85 | return ffa_linux_errmap[err_idx]; | |
3bbfe987 SH |
86 | return -EINVAL; |
87 | } | |
88 | ||
897e9e60 SH |
89 | struct ffa_pcpu_irq { |
90 | struct ffa_drv_info *info; | |
91 | }; | |
92 | ||
3bbfe987 SH |
93 | struct ffa_drv_info { |
94 | u32 version; | |
95 | u16 vm_id; | |
96 | struct mutex rx_lock; /* lock to protect Rx buffer */ | |
97 | struct mutex tx_lock; /* lock to protect Tx buffer */ | |
98 | void *rx_buffer; | |
99 | void *tx_buffer; | |
e57fba91 | 100 | bool mem_ops_native; |
192e88cf | 101 | bool bitmap_created; |
f4bfcaee | 102 | bool notif_enabled; |
897e9e60 SH |
103 | unsigned int sched_recv_irq; |
104 | unsigned int cpuhp_state; | |
105 | struct ffa_pcpu_irq __percpu *irq_pcpu; | |
106 | struct workqueue_struct *notif_pcpu_wq; | |
1b6bf41b | 107 | struct work_struct notif_pcpu_work; |
897e9e60 | 108 | struct work_struct irq_work; |
0184450b SH |
109 | struct xarray partition_info; |
110 | unsigned int partition_count; | |
e0573444 SH |
111 | DECLARE_HASHTABLE(notifier_hash, ilog2(FFA_MAX_NOTIFICATIONS)); |
112 | struct mutex notify_lock; /* lock to protect notifier hashtable */ | |
3bbfe987 SH |
113 | }; |
114 | ||
115 | static struct ffa_drv_info *drv_info; | |
116 | ||
8e3f9da6 SH |
117 | /* |
118 | * The driver must be able to support all the versions from the earliest | |
119 | * supported FFA_MIN_VERSION to the latest supported FFA_DRIVER_VERSION. | |
120 | * The specification states that if firmware supports a FFA implementation | |
121 | * that is incompatible with and at a greater version number than specified | |
122 | * by the caller(FFA_DRIVER_VERSION passed as parameter to FFA_VERSION), | |
123 | * it must return the NOT_SUPPORTED error code. | |
124 | */ | |
125 | static u32 ffa_compatible_version_find(u32 version) | |
126 | { | |
229d58e3 WD |
127 | u16 major = FFA_MAJOR_VERSION(version), minor = FFA_MINOR_VERSION(version); |
128 | u16 drv_major = FFA_MAJOR_VERSION(FFA_DRIVER_VERSION); | |
129 | u16 drv_minor = FFA_MINOR_VERSION(FFA_DRIVER_VERSION); | |
8e3f9da6 SH |
130 | |
131 | if ((major < drv_major) || (major == drv_major && minor <= drv_minor)) | |
132 | return version; | |
133 | ||
134 | pr_info("Firmware version higher than driver version, downgrading\n"); | |
135 | return FFA_DRIVER_VERSION; | |
136 | } | |
137 | ||
3bbfe987 SH |
138 | static int ffa_version_check(u32 *version) |
139 | { | |
140 | ffa_value_t ver; | |
141 | ||
142 | invoke_ffa_fn((ffa_value_t){ | |
143 | .a0 = FFA_VERSION, .a1 = FFA_DRIVER_VERSION, | |
144 | }, &ver); | |
145 | ||
146 | if (ver.a0 == FFA_RET_NOT_SUPPORTED) { | |
147 | pr_info("FFA_VERSION returned not supported\n"); | |
148 | return -EOPNOTSUPP; | |
149 | } | |
150 | ||
8e3f9da6 SH |
151 | if (ver.a0 < FFA_MIN_VERSION) { |
152 | pr_err("Incompatible v%d.%d! Earliest supported v%d.%d\n", | |
229d58e3 WD |
153 | FFA_MAJOR_VERSION(ver.a0), FFA_MINOR_VERSION(ver.a0), |
154 | FFA_MAJOR_VERSION(FFA_MIN_VERSION), | |
155 | FFA_MINOR_VERSION(FFA_MIN_VERSION)); | |
3bbfe987 SH |
156 | return -EINVAL; |
157 | } | |
158 | ||
229d58e3 WD |
159 | pr_info("Driver version %d.%d\n", FFA_MAJOR_VERSION(FFA_DRIVER_VERSION), |
160 | FFA_MINOR_VERSION(FFA_DRIVER_VERSION)); | |
161 | pr_info("Firmware version %d.%d found\n", FFA_MAJOR_VERSION(ver.a0), | |
162 | FFA_MINOR_VERSION(ver.a0)); | |
8e3f9da6 SH |
163 | *version = ffa_compatible_version_find(ver.a0); |
164 | ||
3bbfe987 SH |
165 | return 0; |
166 | } | |
167 | ||
d0c0bce8 SH |
168 | static int ffa_rx_release(void) |
169 | { | |
170 | ffa_value_t ret; | |
171 | ||
172 | invoke_ffa_fn((ffa_value_t){ | |
173 | .a0 = FFA_RX_RELEASE, | |
174 | }, &ret); | |
175 | ||
176 | if (ret.a0 == FFA_ERROR) | |
177 | return ffa_to_linux_errno((int)ret.a2); | |
178 | ||
179 | /* check for ret.a0 == FFA_RX_RELEASE ? */ | |
180 | ||
181 | return 0; | |
182 | } | |
183 | ||
3bbfe987 SH |
184 | static int ffa_rxtx_map(phys_addr_t tx_buf, phys_addr_t rx_buf, u32 pg_cnt) |
185 | { | |
186 | ffa_value_t ret; | |
187 | ||
188 | invoke_ffa_fn((ffa_value_t){ | |
189 | .a0 = FFA_FN_NATIVE(RXTX_MAP), | |
190 | .a1 = tx_buf, .a2 = rx_buf, .a3 = pg_cnt, | |
191 | }, &ret); | |
192 | ||
193 | if (ret.a0 == FFA_ERROR) | |
194 | return ffa_to_linux_errno((int)ret.a2); | |
195 | ||
196 | return 0; | |
197 | } | |
198 | ||
199 | static int ffa_rxtx_unmap(u16 vm_id) | |
200 | { | |
201 | ffa_value_t ret; | |
202 | ||
203 | invoke_ffa_fn((ffa_value_t){ | |
204 | .a0 = FFA_RXTX_UNMAP, .a1 = PACK_TARGET_INFO(vm_id, 0), | |
205 | }, &ret); | |
206 | ||
207 | if (ret.a0 == FFA_ERROR) | |
208 | return ffa_to_linux_errno((int)ret.a2); | |
209 | ||
210 | return 0; | |
211 | } | |
212 | ||
bb1be749 SH |
213 | #define PARTITION_INFO_GET_RETURN_COUNT_ONLY BIT(0) |
214 | ||
d0c0bce8 SH |
215 | /* buffer must be sizeof(struct ffa_partition_info) * num_partitions */ |
216 | static int | |
217 | __ffa_partition_info_get(u32 uuid0, u32 uuid1, u32 uuid2, u32 uuid3, | |
218 | struct ffa_partition_info *buffer, int num_partitions) | |
219 | { | |
bb1be749 | 220 | int idx, count, flags = 0, sz, buf_sz; |
d0c0bce8 SH |
221 | ffa_value_t partition_info; |
222 | ||
c6e04536 SH |
223 | if (drv_info->version > FFA_VERSION_1_0 && |
224 | (!buffer || !num_partitions)) /* Just get the count for now */ | |
bb1be749 SH |
225 | flags = PARTITION_INFO_GET_RETURN_COUNT_ONLY; |
226 | ||
d0c0bce8 SH |
227 | mutex_lock(&drv_info->rx_lock); |
228 | invoke_ffa_fn((ffa_value_t){ | |
229 | .a0 = FFA_PARTITION_INFO_GET, | |
230 | .a1 = uuid0, .a2 = uuid1, .a3 = uuid2, .a4 = uuid3, | |
bb1be749 | 231 | .a5 = flags, |
d0c0bce8 SH |
232 | }, &partition_info); |
233 | ||
234 | if (partition_info.a0 == FFA_ERROR) { | |
235 | mutex_unlock(&drv_info->rx_lock); | |
236 | return ffa_to_linux_errno((int)partition_info.a2); | |
237 | } | |
238 | ||
239 | count = partition_info.a2; | |
240 | ||
bb1be749 SH |
241 | if (drv_info->version > FFA_VERSION_1_0) { |
242 | buf_sz = sz = partition_info.a3; | |
243 | if (sz > sizeof(*buffer)) | |
244 | buf_sz = sizeof(*buffer); | |
245 | } else { | |
246 | /* FFA_VERSION_1_0 lacks size in the response */ | |
247 | buf_sz = sz = 8; | |
248 | } | |
249 | ||
d0c0bce8 | 250 | if (buffer && count <= num_partitions) |
bb1be749 SH |
251 | for (idx = 0; idx < count; idx++) |
252 | memcpy(buffer + idx, drv_info->rx_buffer + idx * sz, | |
253 | buf_sz); | |
d0c0bce8 SH |
254 | |
255 | ffa_rx_release(); | |
256 | ||
257 | mutex_unlock(&drv_info->rx_lock); | |
258 | ||
259 | return count; | |
260 | } | |
261 | ||
262 | /* buffer is allocated and caller must free the same if returned count > 0 */ | |
263 | static int | |
264 | ffa_partition_probe(const uuid_t *uuid, struct ffa_partition_info **buffer) | |
265 | { | |
266 | int count; | |
267 | u32 uuid0_4[4]; | |
268 | struct ffa_partition_info *pbuf; | |
269 | ||
270 | export_uuid((u8 *)uuid0_4, uuid); | |
271 | count = __ffa_partition_info_get(uuid0_4[0], uuid0_4[1], uuid0_4[2], | |
272 | uuid0_4[3], NULL, 0); | |
273 | if (count <= 0) | |
274 | return count; | |
275 | ||
276 | pbuf = kcalloc(count, sizeof(*pbuf), GFP_KERNEL); | |
277 | if (!pbuf) | |
278 | return -ENOMEM; | |
279 | ||
280 | count = __ffa_partition_info_get(uuid0_4[0], uuid0_4[1], uuid0_4[2], | |
281 | uuid0_4[3], pbuf, count); | |
282 | if (count <= 0) | |
283 | kfree(pbuf); | |
284 | else | |
285 | *buffer = pbuf; | |
286 | ||
287 | return count; | |
288 | } | |
289 | ||
3bbfe987 SH |
290 | #define VM_ID_MASK GENMASK(15, 0) |
291 | static int ffa_id_get(u16 *vm_id) | |
292 | { | |
293 | ffa_value_t id; | |
294 | ||
295 | invoke_ffa_fn((ffa_value_t){ | |
296 | .a0 = FFA_ID_GET, | |
297 | }, &id); | |
298 | ||
299 | if (id.a0 == FFA_ERROR) | |
300 | return ffa_to_linux_errno((int)id.a2); | |
301 | ||
302 | *vm_id = FIELD_GET(VM_ID_MASK, (id.a2)); | |
303 | ||
304 | return 0; | |
305 | } | |
306 | ||
d0c0bce8 SH |
307 | static int ffa_msg_send_direct_req(u16 src_id, u16 dst_id, bool mode_32bit, |
308 | struct ffa_send_direct_data *data) | |
309 | { | |
310 | u32 req_id, resp_id, src_dst_ids = PACK_TARGET_INFO(src_id, dst_id); | |
311 | ffa_value_t ret; | |
312 | ||
313 | if (mode_32bit) { | |
314 | req_id = FFA_MSG_SEND_DIRECT_REQ; | |
315 | resp_id = FFA_MSG_SEND_DIRECT_RESP; | |
316 | } else { | |
317 | req_id = FFA_FN_NATIVE(MSG_SEND_DIRECT_REQ); | |
318 | resp_id = FFA_FN_NATIVE(MSG_SEND_DIRECT_RESP); | |
319 | } | |
320 | ||
321 | invoke_ffa_fn((ffa_value_t){ | |
322 | .a0 = req_id, .a1 = src_dst_ids, .a2 = 0, | |
323 | .a3 = data->data0, .a4 = data->data1, .a5 = data->data2, | |
324 | .a6 = data->data3, .a7 = data->data4, | |
325 | }, &ret); | |
326 | ||
327 | while (ret.a0 == FFA_INTERRUPT) | |
328 | invoke_ffa_fn((ffa_value_t){ | |
329 | .a0 = FFA_RUN, .a1 = ret.a1, | |
330 | }, &ret); | |
331 | ||
332 | if (ret.a0 == FFA_ERROR) | |
333 | return ffa_to_linux_errno((int)ret.a2); | |
334 | ||
335 | if (ret.a0 == resp_id) { | |
336 | data->data0 = ret.a3; | |
337 | data->data1 = ret.a4; | |
338 | data->data2 = ret.a5; | |
339 | data->data3 = ret.a6; | |
340 | data->data4 = ret.a7; | |
341 | return 0; | |
342 | } | |
343 | ||
344 | return -EINVAL; | |
345 | } | |
346 | ||
cc2195fe SH |
347 | static int ffa_mem_first_frag(u32 func_id, phys_addr_t buf, u32 buf_sz, |
348 | u32 frag_len, u32 len, u64 *handle) | |
349 | { | |
350 | ffa_value_t ret; | |
351 | ||
352 | invoke_ffa_fn((ffa_value_t){ | |
353 | .a0 = func_id, .a1 = len, .a2 = frag_len, | |
354 | .a3 = buf, .a4 = buf_sz, | |
355 | }, &ret); | |
356 | ||
357 | while (ret.a0 == FFA_MEM_OP_PAUSE) | |
358 | invoke_ffa_fn((ffa_value_t){ | |
359 | .a0 = FFA_MEM_OP_RESUME, | |
360 | .a1 = ret.a1, .a2 = ret.a2, | |
361 | }, &ret); | |
362 | ||
363 | if (ret.a0 == FFA_ERROR) | |
364 | return ffa_to_linux_errno((int)ret.a2); | |
365 | ||
987756f6 MB |
366 | if (ret.a0 == FFA_SUCCESS) { |
367 | if (handle) | |
368 | *handle = PACK_HANDLE(ret.a2, ret.a3); | |
369 | } else if (ret.a0 == FFA_MEM_FRAG_RX) { | |
370 | if (handle) | |
371 | *handle = PACK_HANDLE(ret.a1, ret.a2); | |
372 | } else { | |
cc2195fe | 373 | return -EOPNOTSUPP; |
987756f6 | 374 | } |
cc2195fe SH |
375 | |
376 | return frag_len; | |
377 | } | |
378 | ||
379 | static int ffa_mem_next_frag(u64 handle, u32 frag_len) | |
380 | { | |
381 | ffa_value_t ret; | |
382 | ||
383 | invoke_ffa_fn((ffa_value_t){ | |
384 | .a0 = FFA_MEM_FRAG_TX, | |
385 | .a1 = HANDLE_LOW(handle), .a2 = HANDLE_HIGH(handle), | |
386 | .a3 = frag_len, | |
387 | }, &ret); | |
388 | ||
389 | while (ret.a0 == FFA_MEM_OP_PAUSE) | |
390 | invoke_ffa_fn((ffa_value_t){ | |
391 | .a0 = FFA_MEM_OP_RESUME, | |
392 | .a1 = ret.a1, .a2 = ret.a2, | |
393 | }, &ret); | |
394 | ||
395 | if (ret.a0 == FFA_ERROR) | |
396 | return ffa_to_linux_errno((int)ret.a2); | |
397 | ||
987756f6 MB |
398 | if (ret.a0 == FFA_MEM_FRAG_RX) |
399 | return ret.a3; | |
400 | else if (ret.a0 == FFA_SUCCESS) | |
401 | return 0; | |
cc2195fe | 402 | |
987756f6 | 403 | return -EOPNOTSUPP; |
cc2195fe SH |
404 | } |
405 | ||
406 | static int | |
407 | ffa_transmit_fragment(u32 func_id, phys_addr_t buf, u32 buf_sz, u32 frag_len, | |
408 | u32 len, u64 *handle, bool first) | |
409 | { | |
410 | if (!first) | |
411 | return ffa_mem_next_frag(*handle, frag_len); | |
412 | ||
413 | return ffa_mem_first_frag(func_id, buf, buf_sz, frag_len, len, handle); | |
414 | } | |
415 | ||
416 | static u32 ffa_get_num_pages_sg(struct scatterlist *sg) | |
417 | { | |
418 | u32 num_pages = 0; | |
419 | ||
420 | do { | |
421 | num_pages += sg->length / FFA_PAGE_SIZE; | |
422 | } while ((sg = sg_next(sg))); | |
423 | ||
424 | return num_pages; | |
425 | } | |
426 | ||
11358053 | 427 | static u16 ffa_memory_attributes_get(u32 func_id) |
9dda1178 SH |
428 | { |
429 | /* | |
430 | * For the memory lend or donate operation, if the receiver is a PE or | |
431 | * a proxy endpoint, the owner/sender must not specify the attributes | |
432 | */ | |
433 | if (func_id == FFA_FN_NATIVE(MEM_LEND) || | |
434 | func_id == FFA_MEM_LEND) | |
435 | return 0; | |
436 | ||
437 | return FFA_MEM_NORMAL | FFA_MEM_WRITE_BACK | FFA_MEM_INNER_SHAREABLE; | |
438 | } | |
439 | ||
cc2195fe SH |
440 | static int |
441 | ffa_setup_and_transmit(u32 func_id, void *buffer, u32 max_fragsize, | |
442 | struct ffa_mem_ops_args *args) | |
443 | { | |
444 | int rc = 0; | |
445 | bool first = true; | |
c9b21ef0 | 446 | u32 composite_offset; |
cc2195fe | 447 | phys_addr_t addr = 0; |
c9b21ef0 | 448 | struct ffa_mem_region *mem_region = buffer; |
cc2195fe SH |
449 | struct ffa_composite_mem_region *composite; |
450 | struct ffa_mem_region_addr_range *constituents; | |
451 | struct ffa_mem_region_attributes *ep_mem_access; | |
cc2195fe SH |
452 | u32 idx, frag_len, length, buf_sz = 0, num_entries = sg_nents(args->sg); |
453 | ||
454 | mem_region->tag = args->tag; | |
455 | mem_region->flags = args->flags; | |
456 | mem_region->sender_id = drv_info->vm_id; | |
9dda1178 | 457 | mem_region->attributes = ffa_memory_attributes_get(func_id); |
e4607b84 SH |
458 | ep_mem_access = buffer + |
459 | ffa_mem_desc_offset(buffer, 0, drv_info->version); | |
460 | composite_offset = ffa_mem_desc_offset(buffer, args->nattrs, | |
461 | drv_info->version); | |
cc2195fe SH |
462 | |
463 | for (idx = 0; idx < args->nattrs; idx++, ep_mem_access++) { | |
464 | ep_mem_access->receiver = args->attrs[idx].receiver; | |
465 | ep_mem_access->attrs = args->attrs[idx].attrs; | |
c9b21ef0 | 466 | ep_mem_access->composite_off = composite_offset; |
111a833d SH |
467 | ep_mem_access->flag = 0; |
468 | ep_mem_access->reserved = 0; | |
cc2195fe | 469 | } |
3aa0519a | 470 | mem_region->handle = 0; |
cc2195fe | 471 | mem_region->ep_count = args->nattrs; |
11358053 SH |
472 | if (drv_info->version <= FFA_VERSION_1_0) { |
473 | mem_region->ep_mem_size = 0; | |
474 | } else { | |
475 | mem_region->ep_mem_size = sizeof(*ep_mem_access); | |
476 | mem_region->ep_mem_offset = sizeof(*mem_region); | |
477 | memset(mem_region->reserved, 0, 12); | |
478 | } | |
cc2195fe | 479 | |
c9b21ef0 | 480 | composite = buffer + composite_offset; |
cc2195fe SH |
481 | composite->total_pg_cnt = ffa_get_num_pages_sg(args->sg); |
482 | composite->addr_range_cnt = num_entries; | |
111a833d | 483 | composite->reserved = 0; |
cc2195fe | 484 | |
c9b21ef0 SH |
485 | length = composite_offset + CONSTITUENTS_OFFSET(num_entries); |
486 | frag_len = composite_offset + CONSTITUENTS_OFFSET(0); | |
cc2195fe SH |
487 | if (frag_len > max_fragsize) |
488 | return -ENXIO; | |
489 | ||
490 | if (!args->use_txbuf) { | |
491 | addr = virt_to_phys(buffer); | |
492 | buf_sz = max_fragsize / FFA_PAGE_SIZE; | |
493 | } | |
494 | ||
495 | constituents = buffer + frag_len; | |
496 | idx = 0; | |
497 | do { | |
498 | if (frag_len == max_fragsize) { | |
499 | rc = ffa_transmit_fragment(func_id, addr, buf_sz, | |
500 | frag_len, length, | |
501 | &args->g_handle, first); | |
502 | if (rc < 0) | |
503 | return -ENXIO; | |
504 | ||
505 | first = false; | |
506 | idx = 0; | |
507 | frag_len = 0; | |
508 | constituents = buffer; | |
509 | } | |
510 | ||
511 | if ((void *)constituents - buffer > max_fragsize) { | |
512 | pr_err("Memory Region Fragment > Tx Buffer size\n"); | |
513 | return -EFAULT; | |
514 | } | |
515 | ||
516 | constituents->address = sg_phys(args->sg); | |
517 | constituents->pg_cnt = args->sg->length / FFA_PAGE_SIZE; | |
111a833d | 518 | constituents->reserved = 0; |
cc2195fe SH |
519 | constituents++; |
520 | frag_len += sizeof(struct ffa_mem_region_addr_range); | |
521 | } while ((args->sg = sg_next(args->sg))); | |
522 | ||
523 | return ffa_transmit_fragment(func_id, addr, buf_sz, frag_len, | |
524 | length, &args->g_handle, first); | |
525 | } | |
526 | ||
527 | static int ffa_memory_ops(u32 func_id, struct ffa_mem_ops_args *args) | |
528 | { | |
529 | int ret; | |
530 | void *buffer; | |
531 | ||
532 | if (!args->use_txbuf) { | |
533 | buffer = alloc_pages_exact(RXTX_BUFFER_SIZE, GFP_KERNEL); | |
534 | if (!buffer) | |
535 | return -ENOMEM; | |
536 | } else { | |
537 | buffer = drv_info->tx_buffer; | |
538 | mutex_lock(&drv_info->tx_lock); | |
539 | } | |
540 | ||
541 | ret = ffa_setup_and_transmit(func_id, buffer, RXTX_BUFFER_SIZE, args); | |
542 | ||
543 | if (args->use_txbuf) | |
544 | mutex_unlock(&drv_info->tx_lock); | |
545 | else | |
546 | free_pages_exact(buffer, RXTX_BUFFER_SIZE); | |
547 | ||
548 | return ret < 0 ? ret : 0; | |
549 | } | |
550 | ||
551 | static int ffa_memory_reclaim(u64 g_handle, u32 flags) | |
552 | { | |
553 | ffa_value_t ret; | |
554 | ||
555 | invoke_ffa_fn((ffa_value_t){ | |
556 | .a0 = FFA_MEM_RECLAIM, | |
557 | .a1 = HANDLE_LOW(g_handle), .a2 = HANDLE_HIGH(g_handle), | |
558 | .a3 = flags, | |
559 | }, &ret); | |
560 | ||
561 | if (ret.a0 == FFA_ERROR) | |
562 | return ffa_to_linux_errno((int)ret.a2); | |
563 | ||
564 | return 0; | |
565 | } | |
566 | ||
cb1f4c2c SH |
567 | static int ffa_features(u32 func_feat_id, u32 input_props, |
568 | u32 *if_props_1, u32 *if_props_2) | |
569 | { | |
570 | ffa_value_t id; | |
571 | ||
572 | if (!ARM_SMCCC_IS_FAST_CALL(func_feat_id) && input_props) { | |
573 | pr_err("%s: Invalid Parameters: %x, %x", __func__, | |
574 | func_feat_id, input_props); | |
575 | return ffa_to_linux_errno(FFA_RET_INVALID_PARAMETERS); | |
576 | } | |
577 | ||
578 | invoke_ffa_fn((ffa_value_t){ | |
579 | .a0 = FFA_FEATURES, .a1 = func_feat_id, .a2 = input_props, | |
580 | }, &id); | |
581 | ||
582 | if (id.a0 == FFA_ERROR) | |
583 | return ffa_to_linux_errno((int)id.a2); | |
584 | ||
585 | if (if_props_1) | |
586 | *if_props_1 = id.a2; | |
587 | if (if_props_2) | |
588 | *if_props_2 = id.a3; | |
589 | ||
590 | return 0; | |
591 | } | |
592 | ||
192e88cf SH |
593 | static int ffa_notification_bitmap_create(void) |
594 | { | |
595 | ffa_value_t ret; | |
596 | u16 vcpu_count = nr_cpu_ids; | |
597 | ||
598 | invoke_ffa_fn((ffa_value_t){ | |
599 | .a0 = FFA_NOTIFICATION_BITMAP_CREATE, | |
600 | .a1 = drv_info->vm_id, .a2 = vcpu_count, | |
601 | }, &ret); | |
602 | ||
603 | if (ret.a0 == FFA_ERROR) | |
604 | return ffa_to_linux_errno((int)ret.a2); | |
605 | ||
606 | return 0; | |
607 | } | |
608 | ||
609 | static int ffa_notification_bitmap_destroy(void) | |
610 | { | |
611 | ffa_value_t ret; | |
612 | ||
613 | invoke_ffa_fn((ffa_value_t){ | |
614 | .a0 = FFA_NOTIFICATION_BITMAP_DESTROY, | |
615 | .a1 = drv_info->vm_id, | |
616 | }, &ret); | |
617 | ||
618 | if (ret.a0 == FFA_ERROR) | |
619 | return ffa_to_linux_errno((int)ret.a2); | |
620 | ||
621 | return 0; | |
622 | } | |
623 | ||
933db703 SH |
624 | #define NOTIFICATION_LOW_MASK GENMASK(31, 0) |
625 | #define NOTIFICATION_HIGH_MASK GENMASK(63, 32) | |
626 | #define NOTIFICATION_BITMAP_HIGH(x) \ | |
627 | ((u32)(FIELD_GET(NOTIFICATION_HIGH_MASK, (x)))) | |
628 | #define NOTIFICATION_BITMAP_LOW(x) \ | |
629 | ((u32)(FIELD_GET(NOTIFICATION_LOW_MASK, (x)))) | |
faa19623 SH |
630 | #define PACK_NOTIFICATION_BITMAP(low, high) \ |
631 | (FIELD_PREP(NOTIFICATION_LOW_MASK, (low)) | \ | |
632 | FIELD_PREP(NOTIFICATION_HIGH_MASK, (high))) | |
633 | ||
634 | #define RECEIVER_VCPU_MASK GENMASK(31, 16) | |
635 | #define PACK_NOTIFICATION_GET_RECEIVER_INFO(vcpu_r, r) \ | |
636 | (FIELD_PREP(RECEIVER_VCPU_MASK, (vcpu_r)) | \ | |
637 | FIELD_PREP(RECEIVER_ID_MASK, (r))) | |
933db703 | 638 | |
3522be48 SH |
639 | #define NOTIFICATION_INFO_GET_MORE_PEND_MASK BIT(0) |
640 | #define NOTIFICATION_INFO_GET_ID_COUNT GENMASK(11, 7) | |
641 | #define ID_LIST_MASK_64 GENMASK(51, 12) | |
642 | #define ID_LIST_MASK_32 GENMASK(31, 12) | |
643 | #define MAX_IDS_64 20 | |
644 | #define MAX_IDS_32 10 | |
645 | ||
e0573444 | 646 | #define PER_VCPU_NOTIFICATION_FLAG BIT(0) |
1b6bf41b SH |
647 | #define SECURE_PARTITION_BITMAP BIT(0) |
648 | #define NON_SECURE_VM_BITMAP BIT(1) | |
649 | #define SPM_FRAMEWORK_BITMAP BIT(2) | |
650 | #define NS_HYP_FRAMEWORK_BITMAP BIT(3) | |
e0573444 | 651 | |
933db703 SH |
652 | static int ffa_notification_bind_common(u16 dst_id, u64 bitmap, |
653 | u32 flags, bool is_bind) | |
654 | { | |
655 | ffa_value_t ret; | |
656 | u32 func, src_dst_ids = PACK_TARGET_INFO(dst_id, drv_info->vm_id); | |
657 | ||
658 | func = is_bind ? FFA_NOTIFICATION_BIND : FFA_NOTIFICATION_UNBIND; | |
659 | ||
660 | invoke_ffa_fn((ffa_value_t){ | |
661 | .a0 = func, .a1 = src_dst_ids, .a2 = flags, | |
662 | .a3 = NOTIFICATION_BITMAP_LOW(bitmap), | |
663 | .a4 = NOTIFICATION_BITMAP_HIGH(bitmap), | |
664 | }, &ret); | |
665 | ||
666 | if (ret.a0 == FFA_ERROR) | |
667 | return ffa_to_linux_errno((int)ret.a2); | |
668 | else if (ret.a0 != FFA_SUCCESS) | |
669 | return -EINVAL; | |
670 | ||
671 | return 0; | |
672 | } | |
673 | ||
47561777 SH |
674 | static |
675 | int ffa_notification_set(u16 src_id, u16 dst_id, u32 flags, u64 bitmap) | |
676 | { | |
677 | ffa_value_t ret; | |
678 | u32 src_dst_ids = PACK_TARGET_INFO(dst_id, src_id); | |
679 | ||
680 | invoke_ffa_fn((ffa_value_t) { | |
681 | .a0 = FFA_NOTIFICATION_SET, .a1 = src_dst_ids, .a2 = flags, | |
682 | .a3 = NOTIFICATION_BITMAP_LOW(bitmap), | |
683 | .a4 = NOTIFICATION_BITMAP_HIGH(bitmap), | |
684 | }, &ret); | |
685 | ||
686 | if (ret.a0 == FFA_ERROR) | |
687 | return ffa_to_linux_errno((int)ret.a2); | |
688 | else if (ret.a0 != FFA_SUCCESS) | |
689 | return -EINVAL; | |
690 | ||
691 | return 0; | |
692 | } | |
693 | ||
faa19623 SH |
694 | struct ffa_notify_bitmaps { |
695 | u64 sp_map; | |
696 | u64 vm_map; | |
697 | u64 arch_map; | |
698 | }; | |
699 | ||
700 | static int ffa_notification_get(u32 flags, struct ffa_notify_bitmaps *notify) | |
701 | { | |
702 | ffa_value_t ret; | |
703 | u16 src_id = drv_info->vm_id; | |
704 | u16 cpu_id = smp_processor_id(); | |
705 | u32 rec_vcpu_ids = PACK_NOTIFICATION_GET_RECEIVER_INFO(cpu_id, src_id); | |
706 | ||
707 | invoke_ffa_fn((ffa_value_t){ | |
708 | .a0 = FFA_NOTIFICATION_GET, .a1 = rec_vcpu_ids, .a2 = flags, | |
709 | }, &ret); | |
710 | ||
711 | if (ret.a0 == FFA_ERROR) | |
712 | return ffa_to_linux_errno((int)ret.a2); | |
713 | else if (ret.a0 != FFA_SUCCESS) | |
714 | return -EINVAL; /* Something else went wrong. */ | |
715 | ||
716 | notify->sp_map = PACK_NOTIFICATION_BITMAP(ret.a2, ret.a3); | |
717 | notify->vm_map = PACK_NOTIFICATION_BITMAP(ret.a4, ret.a5); | |
718 | notify->arch_map = PACK_NOTIFICATION_BITMAP(ret.a6, ret.a7); | |
719 | ||
720 | return 0; | |
721 | } | |
722 | ||
0184450b SH |
723 | struct ffa_dev_part_info { |
724 | ffa_sched_recv_cb callback; | |
725 | void *cb_data; | |
726 | rwlock_t rw_lock; | |
727 | }; | |
728 | ||
729 | static void __do_sched_recv_cb(u16 part_id, u16 vcpu, bool is_per_vcpu) | |
3522be48 | 730 | { |
0184450b SH |
731 | struct ffa_dev_part_info *partition; |
732 | ffa_sched_recv_cb callback; | |
733 | void *cb_data; | |
734 | ||
735 | partition = xa_load(&drv_info->partition_info, part_id); | |
736 | read_lock(&partition->rw_lock); | |
737 | callback = partition->callback; | |
738 | cb_data = partition->cb_data; | |
739 | read_unlock(&partition->rw_lock); | |
740 | ||
741 | if (callback) | |
742 | callback(vcpu, is_per_vcpu, cb_data); | |
3522be48 SH |
743 | } |
744 | ||
745 | static void ffa_notification_info_get(void) | |
746 | { | |
747 | int idx, list, max_ids, lists_cnt, ids_processed, ids_count[MAX_IDS_64]; | |
748 | bool is_64b_resp; | |
749 | ffa_value_t ret; | |
750 | u64 id_list; | |
751 | ||
752 | do { | |
753 | invoke_ffa_fn((ffa_value_t){ | |
754 | .a0 = FFA_FN_NATIVE(NOTIFICATION_INFO_GET), | |
755 | }, &ret); | |
756 | ||
757 | if (ret.a0 != FFA_FN_NATIVE(SUCCESS) && ret.a0 != FFA_SUCCESS) { | |
758 | if (ret.a2 != FFA_RET_NO_DATA) | |
759 | pr_err("Notification Info fetch failed: 0x%lx (0x%lx)", | |
760 | ret.a0, ret.a2); | |
761 | return; | |
762 | } | |
763 | ||
764 | is_64b_resp = (ret.a0 == FFA_FN64_SUCCESS); | |
765 | ||
766 | ids_processed = 0; | |
767 | lists_cnt = FIELD_GET(NOTIFICATION_INFO_GET_ID_COUNT, ret.a2); | |
768 | if (is_64b_resp) { | |
769 | max_ids = MAX_IDS_64; | |
770 | id_list = FIELD_GET(ID_LIST_MASK_64, ret.a2); | |
771 | } else { | |
772 | max_ids = MAX_IDS_32; | |
773 | id_list = FIELD_GET(ID_LIST_MASK_32, ret.a2); | |
774 | } | |
775 | ||
776 | for (idx = 0; idx < lists_cnt; idx++, id_list >>= 2) | |
777 | ids_count[idx] = (id_list & 0x3) + 1; | |
778 | ||
779 | /* Process IDs */ | |
780 | for (list = 0; list < lists_cnt; list++) { | |
781 | u16 vcpu_id, part_id, *packed_id_list = (u16 *)&ret.a3; | |
782 | ||
783 | if (ids_processed >= max_ids - 1) | |
784 | break; | |
785 | ||
f1ed48ef | 786 | part_id = packed_id_list[ids_processed++]; |
3522be48 SH |
787 | |
788 | if (!ids_count[list]) { /* Global Notification */ | |
789 | __do_sched_recv_cb(part_id, 0, false); | |
790 | continue; | |
791 | } | |
792 | ||
793 | /* Per vCPU Notification */ | |
794 | for (idx = 0; idx < ids_count[list]; idx++) { | |
795 | if (ids_processed >= max_ids - 1) | |
796 | break; | |
797 | ||
f1ed48ef | 798 | vcpu_id = packed_id_list[ids_processed++]; |
3522be48 SH |
799 | |
800 | __do_sched_recv_cb(part_id, vcpu_id, true); | |
801 | } | |
802 | } | |
803 | } while (ret.a2 & NOTIFICATION_INFO_GET_MORE_PEND_MASK); | |
804 | } | |
805 | ||
fe2ddb6b SH |
806 | static int ffa_run(struct ffa_device *dev, u16 vcpu) |
807 | { | |
808 | ffa_value_t ret; | |
809 | u32 target = dev->vm_id << 16 | vcpu; | |
810 | ||
811 | invoke_ffa_fn((ffa_value_t){ .a0 = FFA_RUN, .a1 = target, }, &ret); | |
812 | ||
813 | while (ret.a0 == FFA_INTERRUPT) | |
814 | invoke_ffa_fn((ffa_value_t){ .a0 = FFA_RUN, .a1 = ret.a1, }, | |
815 | &ret); | |
816 | ||
817 | if (ret.a0 == FFA_ERROR) | |
818 | return ffa_to_linux_errno((int)ret.a2); | |
819 | ||
820 | return 0; | |
821 | } | |
822 | ||
e57fba91 SH |
823 | static void ffa_set_up_mem_ops_native_flag(void) |
824 | { | |
825 | if (!ffa_features(FFA_FN_NATIVE(MEM_LEND), 0, NULL, NULL) || | |
826 | !ffa_features(FFA_FN_NATIVE(MEM_SHARE), 0, NULL, NULL)) | |
827 | drv_info->mem_ops_native = true; | |
828 | } | |
829 | ||
d0c0bce8 SH |
830 | static u32 ffa_api_version_get(void) |
831 | { | |
832 | return drv_info->version; | |
833 | } | |
834 | ||
835 | static int ffa_partition_info_get(const char *uuid_str, | |
836 | struct ffa_partition_info *buffer) | |
837 | { | |
838 | int count; | |
839 | uuid_t uuid; | |
840 | struct ffa_partition_info *pbuf; | |
841 | ||
842 | if (uuid_parse(uuid_str, &uuid)) { | |
843 | pr_err("invalid uuid (%s)\n", uuid_str); | |
844 | return -ENODEV; | |
845 | } | |
846 | ||
f3c45c04 | 847 | count = ffa_partition_probe(&uuid, &pbuf); |
d0c0bce8 SH |
848 | if (count <= 0) |
849 | return -ENOENT; | |
850 | ||
851 | memcpy(buffer, pbuf, sizeof(*pbuf) * count); | |
852 | kfree(pbuf); | |
853 | return 0; | |
854 | } | |
855 | ||
106b11b1 SH |
856 | static void ffa_mode_32bit_set(struct ffa_device *dev) |
857 | { | |
2d698e8b | 858 | dev->mode_32bit = true; |
106b11b1 SH |
859 | } |
860 | ||
d0c0bce8 SH |
861 | static int ffa_sync_send_receive(struct ffa_device *dev, |
862 | struct ffa_send_direct_data *data) | |
863 | { | |
864 | return ffa_msg_send_direct_req(drv_info->vm_id, dev->vm_id, | |
865 | dev->mode_32bit, data); | |
866 | } | |
867 | ||
8c3812c8 | 868 | static int ffa_memory_share(struct ffa_mem_ops_args *args) |
cc2195fe | 869 | { |
e57fba91 SH |
870 | if (drv_info->mem_ops_native) |
871 | return ffa_memory_ops(FFA_FN_NATIVE(MEM_SHARE), args); | |
cc2195fe | 872 | |
e57fba91 | 873 | return ffa_memory_ops(FFA_MEM_SHARE, args); |
cc2195fe SH |
874 | } |
875 | ||
8c3812c8 | 876 | static int ffa_memory_lend(struct ffa_mem_ops_args *args) |
82a8daae MB |
877 | { |
878 | /* Note that upon a successful MEM_LEND request the caller | |
879 | * must ensure that the memory region specified is not accessed | |
880 | * until a successful MEM_RECALIM call has been made. | |
881 | * On systems with a hypervisor present this will been enforced, | |
882 | * however on systems without a hypervisor the responsibility | |
883 | * falls to the calling kernel driver to prevent access. | |
884 | */ | |
e57fba91 SH |
885 | if (drv_info->mem_ops_native) |
886 | return ffa_memory_ops(FFA_FN_NATIVE(MEM_LEND), args); | |
82a8daae | 887 | |
e57fba91 | 888 | return ffa_memory_ops(FFA_MEM_LEND, args); |
82a8daae MB |
889 | } |
890 | ||
e0573444 SH |
891 | #define FFA_SECURE_PARTITION_ID_FLAG BIT(15) |
892 | ||
f4bfcaee SH |
893 | #define ffa_notifications_disabled() (!drv_info->notif_enabled) |
894 | ||
e0573444 SH |
895 | enum notify_type { |
896 | NON_SECURE_VM, | |
897 | SECURE_PARTITION, | |
898 | FRAMEWORK, | |
899 | }; | |
900 | ||
901 | struct notifier_cb_info { | |
902 | struct hlist_node hnode; | |
903 | ffa_notifier_cb cb; | |
904 | void *cb_data; | |
905 | enum notify_type type; | |
906 | }; | |
907 | ||
0184450b SH |
908 | static int ffa_sched_recv_cb_update(u16 part_id, ffa_sched_recv_cb callback, |
909 | void *cb_data, bool is_registration) | |
910 | { | |
911 | struct ffa_dev_part_info *partition; | |
912 | bool cb_valid; | |
913 | ||
f4bfcaee SH |
914 | if (ffa_notifications_disabled()) |
915 | return -EOPNOTSUPP; | |
916 | ||
0184450b SH |
917 | partition = xa_load(&drv_info->partition_info, part_id); |
918 | write_lock(&partition->rw_lock); | |
919 | ||
920 | cb_valid = !!partition->callback; | |
921 | if (!(is_registration ^ cb_valid)) { | |
922 | write_unlock(&partition->rw_lock); | |
923 | return -EINVAL; | |
924 | } | |
925 | ||
926 | partition->callback = callback; | |
927 | partition->cb_data = cb_data; | |
928 | ||
929 | write_unlock(&partition->rw_lock); | |
930 | return 0; | |
931 | } | |
932 | ||
933 | static int ffa_sched_recv_cb_register(struct ffa_device *dev, | |
934 | ffa_sched_recv_cb cb, void *cb_data) | |
935 | { | |
936 | return ffa_sched_recv_cb_update(dev->vm_id, cb, cb_data, true); | |
937 | } | |
938 | ||
939 | static int ffa_sched_recv_cb_unregister(struct ffa_device *dev) | |
940 | { | |
941 | return ffa_sched_recv_cb_update(dev->vm_id, NULL, NULL, false); | |
942 | } | |
943 | ||
e0573444 SH |
944 | static int ffa_notification_bind(u16 dst_id, u64 bitmap, u32 flags) |
945 | { | |
946 | return ffa_notification_bind_common(dst_id, bitmap, flags, true); | |
947 | } | |
948 | ||
949 | static int ffa_notification_unbind(u16 dst_id, u64 bitmap) | |
950 | { | |
951 | return ffa_notification_bind_common(dst_id, bitmap, 0, false); | |
952 | } | |
953 | ||
954 | /* Should be called while the notify_lock is taken */ | |
955 | static struct notifier_cb_info * | |
956 | notifier_hash_node_get(u16 notify_id, enum notify_type type) | |
957 | { | |
958 | struct notifier_cb_info *node; | |
959 | ||
960 | hash_for_each_possible(drv_info->notifier_hash, node, hnode, notify_id) | |
961 | if (type == node->type) | |
962 | return node; | |
963 | ||
964 | return NULL; | |
965 | } | |
966 | ||
967 | static int | |
968 | update_notifier_cb(int notify_id, enum notify_type type, ffa_notifier_cb cb, | |
969 | void *cb_data, bool is_registration) | |
970 | { | |
971 | struct notifier_cb_info *cb_info = NULL; | |
972 | bool cb_found; | |
973 | ||
974 | cb_info = notifier_hash_node_get(notify_id, type); | |
975 | cb_found = !!cb_info; | |
976 | ||
977 | if (!(is_registration ^ cb_found)) | |
978 | return -EINVAL; | |
979 | ||
980 | if (is_registration) { | |
981 | cb_info = kzalloc(sizeof(*cb_info), GFP_KERNEL); | |
982 | if (!cb_info) | |
983 | return -ENOMEM; | |
984 | ||
985 | cb_info->type = type; | |
986 | cb_info->cb = cb; | |
987 | cb_info->cb_data = cb_data; | |
988 | ||
989 | hash_add(drv_info->notifier_hash, &cb_info->hnode, notify_id); | |
990 | } else { | |
991 | hash_del(&cb_info->hnode); | |
992 | } | |
993 | ||
994 | return 0; | |
995 | } | |
996 | ||
997 | static enum notify_type ffa_notify_type_get(u16 vm_id) | |
998 | { | |
999 | if (vm_id & FFA_SECURE_PARTITION_ID_FLAG) | |
1000 | return SECURE_PARTITION; | |
1001 | else | |
1002 | return NON_SECURE_VM; | |
1003 | } | |
1004 | ||
1005 | static int ffa_notify_relinquish(struct ffa_device *dev, int notify_id) | |
1006 | { | |
1007 | int rc; | |
1008 | enum notify_type type = ffa_notify_type_get(dev->vm_id); | |
1009 | ||
f4bfcaee SH |
1010 | if (ffa_notifications_disabled()) |
1011 | return -EOPNOTSUPP; | |
1012 | ||
e0573444 SH |
1013 | if (notify_id >= FFA_MAX_NOTIFICATIONS) |
1014 | return -EINVAL; | |
1015 | ||
1016 | mutex_lock(&drv_info->notify_lock); | |
1017 | ||
1018 | rc = update_notifier_cb(notify_id, type, NULL, NULL, false); | |
1019 | if (rc) { | |
1020 | pr_err("Could not unregister notification callback\n"); | |
1021 | mutex_unlock(&drv_info->notify_lock); | |
1022 | return rc; | |
1023 | } | |
1024 | ||
1025 | rc = ffa_notification_unbind(dev->vm_id, BIT(notify_id)); | |
1026 | ||
1027 | mutex_unlock(&drv_info->notify_lock); | |
1028 | ||
1029 | return rc; | |
1030 | } | |
1031 | ||
1032 | static int ffa_notify_request(struct ffa_device *dev, bool is_per_vcpu, | |
1033 | ffa_notifier_cb cb, void *cb_data, int notify_id) | |
1034 | { | |
1035 | int rc; | |
1036 | u32 flags = 0; | |
1037 | enum notify_type type = ffa_notify_type_get(dev->vm_id); | |
1038 | ||
f4bfcaee SH |
1039 | if (ffa_notifications_disabled()) |
1040 | return -EOPNOTSUPP; | |
1041 | ||
e0573444 SH |
1042 | if (notify_id >= FFA_MAX_NOTIFICATIONS) |
1043 | return -EINVAL; | |
1044 | ||
1045 | mutex_lock(&drv_info->notify_lock); | |
1046 | ||
1047 | if (is_per_vcpu) | |
1048 | flags = PER_VCPU_NOTIFICATION_FLAG; | |
1049 | ||
1050 | rc = ffa_notification_bind(dev->vm_id, BIT(notify_id), flags); | |
1051 | if (rc) { | |
1052 | mutex_unlock(&drv_info->notify_lock); | |
1053 | return rc; | |
1054 | } | |
1055 | ||
1056 | rc = update_notifier_cb(notify_id, type, cb, cb_data, true); | |
1057 | if (rc) { | |
1058 | pr_err("Failed to register callback for %d - %d\n", | |
1059 | notify_id, rc); | |
1060 | ffa_notification_unbind(dev->vm_id, BIT(notify_id)); | |
1061 | } | |
1062 | mutex_unlock(&drv_info->notify_lock); | |
1063 | ||
1064 | return rc; | |
1065 | } | |
1066 | ||
e5adb3b2 SH |
1067 | static int ffa_notify_send(struct ffa_device *dev, int notify_id, |
1068 | bool is_per_vcpu, u16 vcpu) | |
1069 | { | |
1070 | u32 flags = 0; | |
1071 | ||
f4bfcaee SH |
1072 | if (ffa_notifications_disabled()) |
1073 | return -EOPNOTSUPP; | |
1074 | ||
e5adb3b2 SH |
1075 | if (is_per_vcpu) |
1076 | flags |= (PER_VCPU_NOTIFICATION_FLAG | vcpu << 16); | |
1077 | ||
1078 | return ffa_notification_set(dev->vm_id, drv_info->vm_id, flags, | |
1079 | BIT(notify_id)); | |
1080 | } | |
1081 | ||
1b6bf41b SH |
1082 | static void handle_notif_callbacks(u64 bitmap, enum notify_type type) |
1083 | { | |
1084 | int notify_id; | |
1085 | struct notifier_cb_info *cb_info = NULL; | |
1086 | ||
1087 | for (notify_id = 0; notify_id <= FFA_MAX_NOTIFICATIONS && bitmap; | |
1088 | notify_id++, bitmap >>= 1) { | |
1089 | if (!(bitmap & 1)) | |
1090 | continue; | |
1091 | ||
1092 | mutex_lock(&drv_info->notify_lock); | |
1093 | cb_info = notifier_hash_node_get(notify_id, type); | |
1094 | mutex_unlock(&drv_info->notify_lock); | |
1095 | ||
1096 | if (cb_info && cb_info->cb) | |
1097 | cb_info->cb(notify_id, cb_info->cb_data); | |
1098 | } | |
1099 | } | |
1100 | ||
1101 | static void notif_pcpu_irq_work_fn(struct work_struct *work) | |
1102 | { | |
1103 | int rc; | |
1104 | struct ffa_notify_bitmaps bitmaps; | |
1105 | ||
1106 | rc = ffa_notification_get(SECURE_PARTITION_BITMAP | | |
1107 | SPM_FRAMEWORK_BITMAP, &bitmaps); | |
1108 | if (rc) { | |
1109 | pr_err("Failed to retrieve notifications with %d!\n", rc); | |
1110 | return; | |
1111 | } | |
1112 | ||
1113 | handle_notif_callbacks(bitmaps.vm_map, NON_SECURE_VM); | |
1114 | handle_notif_callbacks(bitmaps.sp_map, SECURE_PARTITION); | |
1115 | handle_notif_callbacks(bitmaps.arch_map, FRAMEWORK); | |
1116 | } | |
1117 | ||
1118 | static void | |
1119 | ffa_self_notif_handle(u16 vcpu, bool is_per_vcpu, void *cb_data) | |
1120 | { | |
1121 | struct ffa_drv_info *info = cb_data; | |
1122 | ||
1123 | if (!is_per_vcpu) | |
1124 | notif_pcpu_irq_work_fn(&info->notif_pcpu_work); | |
1125 | else | |
1126 | queue_work_on(vcpu, info->notif_pcpu_wq, | |
1127 | &info->notif_pcpu_work); | |
1128 | } | |
1129 | ||
5b0c6328 | 1130 | static const struct ffa_info_ops ffa_drv_info_ops = { |
d0c0bce8 SH |
1131 | .api_version_get = ffa_api_version_get, |
1132 | .partition_info_get = ffa_partition_info_get, | |
5b0c6328 SH |
1133 | }; |
1134 | ||
1135 | static const struct ffa_msg_ops ffa_drv_msg_ops = { | |
d0c0bce8 SH |
1136 | .mode_32bit_set = ffa_mode_32bit_set, |
1137 | .sync_send_receive = ffa_sync_send_receive, | |
5b0c6328 SH |
1138 | }; |
1139 | ||
1140 | static const struct ffa_mem_ops ffa_drv_mem_ops = { | |
cc2195fe SH |
1141 | .memory_reclaim = ffa_memory_reclaim, |
1142 | .memory_share = ffa_memory_share, | |
82a8daae | 1143 | .memory_lend = ffa_memory_lend, |
d0c0bce8 SH |
1144 | }; |
1145 | ||
fe2ddb6b SH |
1146 | static const struct ffa_cpu_ops ffa_drv_cpu_ops = { |
1147 | .run = ffa_run, | |
1148 | }; | |
1149 | ||
0184450b SH |
1150 | static const struct ffa_notifier_ops ffa_drv_notifier_ops = { |
1151 | .sched_recv_cb_register = ffa_sched_recv_cb_register, | |
1152 | .sched_recv_cb_unregister = ffa_sched_recv_cb_unregister, | |
e0573444 SH |
1153 | .notify_request = ffa_notify_request, |
1154 | .notify_relinquish = ffa_notify_relinquish, | |
e5adb3b2 | 1155 | .notify_send = ffa_notify_send, |
0184450b SH |
1156 | }; |
1157 | ||
5b0c6328 SH |
1158 | static const struct ffa_ops ffa_drv_ops = { |
1159 | .info_ops = &ffa_drv_info_ops, | |
1160 | .msg_ops = &ffa_drv_msg_ops, | |
1161 | .mem_ops = &ffa_drv_mem_ops, | |
fe2ddb6b | 1162 | .cpu_ops = &ffa_drv_cpu_ops, |
0184450b | 1163 | .notifier_ops = &ffa_drv_notifier_ops, |
5b0c6328 SH |
1164 | }; |
1165 | ||
d0c0bce8 SH |
1166 | void ffa_device_match_uuid(struct ffa_device *ffa_dev, const uuid_t *uuid) |
1167 | { | |
1168 | int count, idx; | |
1169 | struct ffa_partition_info *pbuf, *tpbuf; | |
1170 | ||
bb1be749 SH |
1171 | /* |
1172 | * FF-A v1.1 provides UUID for each partition as part of the discovery | |
1173 | * API, the discovered UUID must be populated in the device's UUID and | |
1174 | * there is no need to copy the same from the driver table. | |
1175 | */ | |
1176 | if (drv_info->version > FFA_VERSION_1_0) | |
1177 | return; | |
1178 | ||
d0c0bce8 SH |
1179 | count = ffa_partition_probe(uuid, &pbuf); |
1180 | if (count <= 0) | |
1181 | return; | |
1182 | ||
1183 | for (idx = 0, tpbuf = pbuf; idx < count; idx++, tpbuf++) | |
1184 | if (tpbuf->id == ffa_dev->vm_id) | |
1185 | uuid_copy(&ffa_dev->uuid, uuid); | |
1186 | kfree(pbuf); | |
1187 | } | |
1188 | ||
1189 | static void ffa_setup_partitions(void) | |
1190 | { | |
1191 | int count, idx; | |
bb1be749 | 1192 | uuid_t uuid; |
d0c0bce8 | 1193 | struct ffa_device *ffa_dev; |
0184450b | 1194 | struct ffa_dev_part_info *info; |
d0c0bce8 SH |
1195 | struct ffa_partition_info *pbuf, *tpbuf; |
1196 | ||
1197 | count = ffa_partition_probe(&uuid_null, &pbuf); | |
1198 | if (count <= 0) { | |
1199 | pr_info("%s: No partitions found, error %d\n", __func__, count); | |
1200 | return; | |
1201 | } | |
1202 | ||
0184450b | 1203 | xa_init(&drv_info->partition_info); |
d0c0bce8 | 1204 | for (idx = 0, tpbuf = pbuf; idx < count; idx++, tpbuf++) { |
bb1be749 SH |
1205 | import_uuid(&uuid, (u8 *)tpbuf->uuid); |
1206 | ||
1207 | /* Note that if the UUID will be uuid_null, that will require | |
d0c0bce8 | 1208 | * ffa_device_match() to find the UUID of this partition id |
bb1be749 SH |
1209 | * with help of ffa_device_match_uuid(). FF-A v1.1 and above |
1210 | * provides UUID here for each partition as part of the | |
1211 | * discovery API and the same is passed. | |
d0c0bce8 | 1212 | */ |
5b0c6328 | 1213 | ffa_dev = ffa_device_register(&uuid, tpbuf->id, &ffa_drv_ops); |
d0c0bce8 SH |
1214 | if (!ffa_dev) { |
1215 | pr_err("%s: failed to register partition ID 0x%x\n", | |
1216 | __func__, tpbuf->id); | |
1217 | continue; | |
1218 | } | |
106b11b1 SH |
1219 | |
1220 | if (drv_info->version > FFA_VERSION_1_0 && | |
1221 | !(tpbuf->properties & FFA_PARTITION_AARCH64_EXEC)) | |
2d698e8b | 1222 | ffa_mode_32bit_set(ffa_dev); |
0184450b SH |
1223 | |
1224 | info = kzalloc(sizeof(*info), GFP_KERNEL); | |
1225 | if (!info) { | |
1226 | ffa_device_unregister(ffa_dev); | |
1227 | continue; | |
1228 | } | |
59b2e242 | 1229 | rwlock_init(&info->rw_lock); |
0184450b | 1230 | xa_store(&drv_info->partition_info, tpbuf->id, info, GFP_KERNEL); |
d0c0bce8 | 1231 | } |
0184450b SH |
1232 | drv_info->partition_count = count; |
1233 | ||
d0c0bce8 | 1234 | kfree(pbuf); |
1b6bf41b SH |
1235 | |
1236 | /* Allocate for the host */ | |
1237 | info = kzalloc(sizeof(*info), GFP_KERNEL); | |
1238 | if (!info) | |
1239 | return; | |
1240 | xa_store(&drv_info->partition_info, drv_info->vm_id, info, GFP_KERNEL); | |
1241 | drv_info->partition_count++; | |
d0c0bce8 SH |
1242 | } |
1243 | ||
0184450b SH |
1244 | static void ffa_partitions_cleanup(void) |
1245 | { | |
1246 | struct ffa_dev_part_info **info; | |
1247 | int idx, count = drv_info->partition_count; | |
1248 | ||
1249 | if (!count) | |
1250 | return; | |
1251 | ||
05857a1e | 1252 | info = kcalloc(count, sizeof(*info), GFP_KERNEL); |
0184450b SH |
1253 | if (!info) |
1254 | return; | |
1255 | ||
1256 | xa_extract(&drv_info->partition_info, (void **)info, 0, VM_ID_MASK, | |
1257 | count, XA_PRESENT); | |
1258 | ||
1259 | for (idx = 0; idx < count; idx++) | |
1260 | kfree(info[idx]); | |
1261 | kfree(info); | |
1262 | ||
1263 | drv_info->partition_count = 0; | |
1264 | xa_destroy(&drv_info->partition_info); | |
1265 | } | |
1266 | ||
897e9e60 SH |
1267 | /* FFA FEATURE IDs */ |
1268 | #define FFA_FEAT_NOTIFICATION_PENDING_INT (1) | |
1269 | #define FFA_FEAT_SCHEDULE_RECEIVER_INT (2) | |
1270 | #define FFA_FEAT_MANAGED_EXIT_INT (3) | |
1271 | ||
1272 | static irqreturn_t irq_handler(int irq, void *irq_data) | |
192e88cf | 1273 | { |
897e9e60 SH |
1274 | struct ffa_pcpu_irq *pcpu = irq_data; |
1275 | struct ffa_drv_info *info = pcpu->info; | |
192e88cf | 1276 | |
897e9e60 SH |
1277 | queue_work(info->notif_pcpu_wq, &info->irq_work); |
1278 | ||
1279 | return IRQ_HANDLED; | |
1280 | } | |
1281 | ||
1282 | static void ffa_sched_recv_irq_work_fn(struct work_struct *work) | |
1283 | { | |
1284 | ffa_notification_info_get(); | |
1285 | } | |
1286 | ||
1287 | static int ffa_sched_recv_irq_map(void) | |
1288 | { | |
1289 | int ret, irq, sr_intid; | |
1290 | ||
1291 | /* The returned sr_intid is assumed to be SGI donated to NS world */ | |
1292 | ret = ffa_features(FFA_FEAT_SCHEDULE_RECEIVER_INT, 0, &sr_intid, NULL); | |
1293 | if (ret < 0) { | |
1294 | if (ret != -EOPNOTSUPP) | |
1295 | pr_err("Failed to retrieve scheduler Rx interrupt\n"); | |
1296 | return ret; | |
192e88cf SH |
1297 | } |
1298 | ||
897e9e60 SH |
1299 | if (acpi_disabled) { |
1300 | struct of_phandle_args oirq = {}; | |
1301 | struct device_node *gic; | |
1302 | ||
1303 | /* Only GICv3 supported currently with the device tree */ | |
1304 | gic = of_find_compatible_node(NULL, NULL, "arm,gic-v3"); | |
1305 | if (!gic) | |
1306 | return -ENXIO; | |
1307 | ||
1308 | oirq.np = gic; | |
1309 | oirq.args_count = 1; | |
1310 | oirq.args[0] = sr_intid; | |
1311 | irq = irq_create_of_mapping(&oirq); | |
1312 | of_node_put(gic); | |
1313 | #ifdef CONFIG_ACPI | |
1314 | } else { | |
1315 | irq = acpi_register_gsi(NULL, sr_intid, ACPI_EDGE_SENSITIVE, | |
1316 | ACPI_ACTIVE_HIGH); | |
1317 | #endif | |
1318 | } | |
1319 | ||
1320 | if (irq <= 0) { | |
1321 | pr_err("Failed to create IRQ mapping!\n"); | |
1322 | return -ENODATA; | |
1323 | } | |
1324 | ||
1325 | return irq; | |
1326 | } | |
1327 | ||
1328 | static void ffa_sched_recv_irq_unmap(void) | |
1329 | { | |
6d67cbe6 | 1330 | if (drv_info->sched_recv_irq) { |
897e9e60 | 1331 | irq_dispose_mapping(drv_info->sched_recv_irq); |
6d67cbe6 SH |
1332 | drv_info->sched_recv_irq = 0; |
1333 | } | |
897e9e60 SH |
1334 | } |
1335 | ||
1336 | static int ffa_cpuhp_pcpu_irq_enable(unsigned int cpu) | |
1337 | { | |
1338 | enable_percpu_irq(drv_info->sched_recv_irq, IRQ_TYPE_NONE); | |
1339 | return 0; | |
1340 | } | |
1341 | ||
1342 | static int ffa_cpuhp_pcpu_irq_disable(unsigned int cpu) | |
1343 | { | |
1344 | disable_percpu_irq(drv_info->sched_recv_irq); | |
1345 | return 0; | |
1346 | } | |
1347 | ||
1348 | static void ffa_uninit_pcpu_irq(void) | |
1349 | { | |
6d67cbe6 | 1350 | if (drv_info->cpuhp_state) { |
897e9e60 | 1351 | cpuhp_remove_state(drv_info->cpuhp_state); |
6d67cbe6 SH |
1352 | drv_info->cpuhp_state = 0; |
1353 | } | |
897e9e60 | 1354 | |
6d67cbe6 | 1355 | if (drv_info->notif_pcpu_wq) { |
897e9e60 | 1356 | destroy_workqueue(drv_info->notif_pcpu_wq); |
6d67cbe6 SH |
1357 | drv_info->notif_pcpu_wq = NULL; |
1358 | } | |
897e9e60 SH |
1359 | |
1360 | if (drv_info->sched_recv_irq) | |
1361 | free_percpu_irq(drv_info->sched_recv_irq, drv_info->irq_pcpu); | |
1362 | ||
6d67cbe6 | 1363 | if (drv_info->irq_pcpu) { |
897e9e60 | 1364 | free_percpu(drv_info->irq_pcpu); |
6d67cbe6 SH |
1365 | drv_info->irq_pcpu = NULL; |
1366 | } | |
897e9e60 SH |
1367 | } |
1368 | ||
1369 | static int ffa_init_pcpu_irq(unsigned int irq) | |
1370 | { | |
1371 | struct ffa_pcpu_irq __percpu *irq_pcpu; | |
1372 | int ret, cpu; | |
1373 | ||
1374 | irq_pcpu = alloc_percpu(struct ffa_pcpu_irq); | |
1375 | if (!irq_pcpu) | |
1376 | return -ENOMEM; | |
1377 | ||
1378 | for_each_present_cpu(cpu) | |
1379 | per_cpu_ptr(irq_pcpu, cpu)->info = drv_info; | |
1380 | ||
1381 | drv_info->irq_pcpu = irq_pcpu; | |
1382 | ||
1383 | ret = request_percpu_irq(irq, irq_handler, "ARM-FFA", irq_pcpu); | |
192e88cf | 1384 | if (ret) { |
897e9e60 | 1385 | pr_err("Error registering notification IRQ %d: %d\n", irq, ret); |
192e88cf SH |
1386 | return ret; |
1387 | } | |
192e88cf | 1388 | |
897e9e60 | 1389 | INIT_WORK(&drv_info->irq_work, ffa_sched_recv_irq_work_fn); |
1b6bf41b | 1390 | INIT_WORK(&drv_info->notif_pcpu_work, notif_pcpu_irq_work_fn); |
897e9e60 SH |
1391 | drv_info->notif_pcpu_wq = create_workqueue("ffa_pcpu_irq_notification"); |
1392 | if (!drv_info->notif_pcpu_wq) | |
1393 | return -EINVAL; | |
1394 | ||
1395 | ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "ffa/pcpu-irq:starting", | |
1396 | ffa_cpuhp_pcpu_irq_enable, | |
1397 | ffa_cpuhp_pcpu_irq_disable); | |
1398 | ||
1399 | if (ret < 0) | |
1400 | return ret; | |
1401 | ||
1402 | drv_info->cpuhp_state = ret; | |
192e88cf SH |
1403 | return 0; |
1404 | } | |
1405 | ||
1406 | static void ffa_notifications_cleanup(void) | |
1407 | { | |
897e9e60 SH |
1408 | ffa_uninit_pcpu_irq(); |
1409 | ffa_sched_recv_irq_unmap(); | |
1410 | ||
192e88cf SH |
1411 | if (drv_info->bitmap_created) { |
1412 | ffa_notification_bitmap_destroy(); | |
1413 | drv_info->bitmap_created = false; | |
1414 | } | |
f4bfcaee | 1415 | drv_info->notif_enabled = false; |
192e88cf SH |
1416 | } |
1417 | ||
95520fc0 | 1418 | static void ffa_notifications_setup(void) |
897e9e60 SH |
1419 | { |
1420 | int ret, irq; | |
1421 | ||
1422 | ret = ffa_features(FFA_NOTIFICATION_BITMAP_CREATE, 0, NULL, NULL); | |
1423 | if (ret) { | |
95520fc0 SH |
1424 | pr_info("Notifications not supported, continuing with it ..\n"); |
1425 | return; | |
897e9e60 SH |
1426 | } |
1427 | ||
1428 | ret = ffa_notification_bitmap_create(); | |
1429 | if (ret) { | |
95520fc0 SH |
1430 | pr_info("Notification bitmap create error %d\n", ret); |
1431 | return; | |
897e9e60 SH |
1432 | } |
1433 | drv_info->bitmap_created = true; | |
1434 | ||
1435 | irq = ffa_sched_recv_irq_map(); | |
1436 | if (irq <= 0) { | |
1437 | ret = irq; | |
1438 | goto cleanup; | |
1439 | } | |
1440 | ||
1441 | drv_info->sched_recv_irq = irq; | |
1442 | ||
1443 | ret = ffa_init_pcpu_irq(irq); | |
1444 | if (ret) | |
1445 | goto cleanup; | |
1446 | ||
e0573444 SH |
1447 | hash_init(drv_info->notifier_hash); |
1448 | mutex_init(&drv_info->notify_lock); | |
1449 | ||
f4bfcaee | 1450 | drv_info->notif_enabled = true; |
6f47023f | 1451 | return; |
897e9e60 | 1452 | cleanup: |
95520fc0 | 1453 | pr_info("Notification setup failed %d, not enabled\n", ret); |
897e9e60 | 1454 | ffa_notifications_cleanup(); |
897e9e60 SH |
1455 | } |
1456 | ||
3bbfe987 SH |
1457 | static int __init ffa_init(void) |
1458 | { | |
1459 | int ret; | |
1460 | ||
1461 | ret = ffa_transport_init(&invoke_ffa_fn); | |
1462 | if (ret) | |
1463 | return ret; | |
1464 | ||
1465 | ret = arm_ffa_bus_init(); | |
1466 | if (ret) | |
1467 | return ret; | |
1468 | ||
1469 | drv_info = kzalloc(sizeof(*drv_info), GFP_KERNEL); | |
1470 | if (!drv_info) { | |
1471 | ret = -ENOMEM; | |
1472 | goto ffa_bus_exit; | |
1473 | } | |
1474 | ||
1475 | ret = ffa_version_check(&drv_info->version); | |
1476 | if (ret) | |
1477 | goto free_drv_info; | |
1478 | ||
1479 | if (ffa_id_get(&drv_info->vm_id)) { | |
1480 | pr_err("failed to obtain VM id for self\n"); | |
1481 | ret = -ENODEV; | |
1482 | goto free_drv_info; | |
1483 | } | |
1484 | ||
1485 | drv_info->rx_buffer = alloc_pages_exact(RXTX_BUFFER_SIZE, GFP_KERNEL); | |
1486 | if (!drv_info->rx_buffer) { | |
1487 | ret = -ENOMEM; | |
1488 | goto free_pages; | |
1489 | } | |
1490 | ||
1491 | drv_info->tx_buffer = alloc_pages_exact(RXTX_BUFFER_SIZE, GFP_KERNEL); | |
1492 | if (!drv_info->tx_buffer) { | |
1493 | ret = -ENOMEM; | |
1494 | goto free_pages; | |
1495 | } | |
1496 | ||
1497 | ret = ffa_rxtx_map(virt_to_phys(drv_info->tx_buffer), | |
1498 | virt_to_phys(drv_info->rx_buffer), | |
1499 | RXTX_BUFFER_SIZE / FFA_PAGE_SIZE); | |
1500 | if (ret) { | |
1501 | pr_err("failed to register FFA RxTx buffers\n"); | |
1502 | goto free_pages; | |
1503 | } | |
1504 | ||
1505 | mutex_init(&drv_info->rx_lock); | |
1506 | mutex_init(&drv_info->tx_lock); | |
1507 | ||
e57fba91 SH |
1508 | ffa_set_up_mem_ops_native_flag(); |
1509 | ||
95520fc0 | 1510 | ffa_notifications_setup(); |
897e9e60 | 1511 | |
6f47023f SH |
1512 | ffa_setup_partitions(); |
1513 | ||
1514 | ret = ffa_sched_recv_cb_update(drv_info->vm_id, ffa_self_notif_handle, | |
1515 | drv_info, true); | |
1516 | if (ret) | |
1517 | pr_info("Failed to register driver sched callback %d\n", ret); | |
1518 | ||
897e9e60 | 1519 | return 0; |
3bbfe987 SH |
1520 | free_pages: |
1521 | if (drv_info->tx_buffer) | |
1522 | free_pages_exact(drv_info->tx_buffer, RXTX_BUFFER_SIZE); | |
1523 | free_pages_exact(drv_info->rx_buffer, RXTX_BUFFER_SIZE); | |
1524 | free_drv_info: | |
1525 | kfree(drv_info); | |
1526 | ffa_bus_exit: | |
1527 | arm_ffa_bus_exit(); | |
1528 | return ret; | |
1529 | } | |
1530 | subsys_initcall(ffa_init); | |
1531 | ||
1532 | static void __exit ffa_exit(void) | |
1533 | { | |
192e88cf | 1534 | ffa_notifications_cleanup(); |
0184450b | 1535 | ffa_partitions_cleanup(); |
3bbfe987 SH |
1536 | ffa_rxtx_unmap(drv_info->vm_id); |
1537 | free_pages_exact(drv_info->tx_buffer, RXTX_BUFFER_SIZE); | |
1538 | free_pages_exact(drv_info->rx_buffer, RXTX_BUFFER_SIZE); | |
0184450b | 1539 | xa_destroy(&drv_info->partition_info); |
3bbfe987 SH |
1540 | kfree(drv_info); |
1541 | arm_ffa_bus_exit(); | |
1542 | } | |
1543 | module_exit(ffa_exit); | |
1544 | ||
1545 | MODULE_ALIAS("arm-ffa"); | |
1546 | MODULE_AUTHOR("Sudeep Holla <sudeep.holla@arm.com>"); | |
1547 | MODULE_DESCRIPTION("Arm FF-A interface driver"); | |
1548 | MODULE_LICENSE("GPL v2"); |