1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2017-2018 The Linux Foundation. All rights reserved. */
4 #include <linux/completion.h>
5 #include <linux/circ_buf.h>
6 #include <linux/list.h>
9 #include "a6xx_gmu.xml.h"
12 #define HFI_MSG_ID(val) [val] = #val
14 static const char * const a6xx_hfi_msg_id[] = {
15 HFI_MSG_ID(HFI_H2F_MSG_INIT),
16 HFI_MSG_ID(HFI_H2F_MSG_FW_VERSION),
17 HFI_MSG_ID(HFI_H2F_MSG_BW_TABLE),
18 HFI_MSG_ID(HFI_H2F_MSG_PERF_TABLE),
19 HFI_MSG_ID(HFI_H2F_MSG_TEST),
20 HFI_MSG_ID(HFI_H2F_MSG_START),
21 HFI_MSG_ID(HFI_H2F_MSG_CORE_FW_START),
22 HFI_MSG_ID(HFI_H2F_MSG_GX_BW_PERF_VOTE),
23 HFI_MSG_ID(HFI_H2F_MSG_PREPARE_SLUMBER),
26 static int a6xx_hfi_queue_read(struct a6xx_gmu *gmu,
27 struct a6xx_hfi_queue *queue, u32 *data, u32 dwords)
29 struct a6xx_hfi_queue_header *header = queue->header;
30 u32 i, hdr, index = header->read_index;
32 if (header->read_index == header->write_index) {
33 header->rx_request = 1;
37 hdr = queue->data[index];
40 * If we are to assume that the GMU firmware is in fact a rational actor
41 * and is programmed to not send us a larger response than we expect
42 * then we can also assume that if the header size is unexpectedly large
43 * that it is due to memory corruption and/or hardware failure. In this
44 * case the only reasonable course of action is to BUG() to help harden
48 BUG_ON(HFI_HEADER_SIZE(hdr) > dwords);
50 for (i = 0; i < HFI_HEADER_SIZE(hdr); i++) {
51 data[i] = queue->data[index];
52 index = (index + 1) % header->size;
56 index = ALIGN(index, 4) % header->size;
58 header->read_index = index;
59 return HFI_HEADER_SIZE(hdr);
62 static int a6xx_hfi_queue_write(struct a6xx_gmu *gmu,
63 struct a6xx_hfi_queue *queue, u32 *data, u32 dwords)
65 struct a6xx_hfi_queue_header *header = queue->header;
66 u32 i, space, index = header->write_index;
68 spin_lock(&queue->lock);
70 space = CIRC_SPACE(header->write_index, header->read_index,
74 spin_unlock(&queue->lock);
78 for (i = 0; i < dwords; i++) {
79 queue->data[index] = data[i];
80 index = (index + 1) % header->size;
83 /* Cookify any non used data at the end of the write buffer */
85 for (; index % 4; index = (index + 1) % header->size)
86 queue->data[index] = 0xfafafafa;
89 header->write_index = index;
90 spin_unlock(&queue->lock);
92 gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 0x01);
96 static int a6xx_hfi_wait_for_ack(struct a6xx_gmu *gmu, u32 id, u32 seqnum,
97 u32 *payload, u32 payload_size)
99 struct a6xx_hfi_queue *queue = &gmu->queues[HFI_RESPONSE_QUEUE];
103 /* Wait for a response */
104 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO, val,
105 val & A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ, 100, 5000);
108 DRM_DEV_ERROR(gmu->dev,
109 "Message %s id %d timed out waiting for response\n",
110 a6xx_hfi_msg_id[id], seqnum);
114 /* Clear the interrupt */
115 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR,
116 A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ);
119 struct a6xx_hfi_msg_response resp;
121 /* Get the next packet */
122 ret = a6xx_hfi_queue_read(gmu, queue, (u32 *) &resp,
125 /* If the queue is empty our response never made it */
127 DRM_DEV_ERROR(gmu->dev,
128 "The HFI response queue is unexpectedly empty\n");
133 if (HFI_HEADER_ID(resp.header) == HFI_F2H_MSG_ERROR) {
134 struct a6xx_hfi_msg_error *error =
135 (struct a6xx_hfi_msg_error *) &resp;
137 DRM_DEV_ERROR(gmu->dev, "GMU firmware error %d\n",
142 if (seqnum != HFI_HEADER_SEQNUM(resp.ret_header)) {
143 DRM_DEV_ERROR(gmu->dev,
144 "Unexpected message id %d on the response queue\n",
145 HFI_HEADER_SEQNUM(resp.ret_header));
150 DRM_DEV_ERROR(gmu->dev,
151 "Message %s id %d returned error %d\n",
152 a6xx_hfi_msg_id[id], seqnum, resp.error);
156 /* All is well, copy over the buffer */
157 if (payload && payload_size)
158 memcpy(payload, resp.payload,
159 min_t(u32, payload_size, sizeof(resp.payload)));
165 static int a6xx_hfi_send_msg(struct a6xx_gmu *gmu, int id,
166 void *data, u32 size, u32 *payload, u32 payload_size)
168 struct a6xx_hfi_queue *queue = &gmu->queues[HFI_COMMAND_QUEUE];
169 int ret, dwords = size >> 2;
172 seqnum = atomic_inc_return(&queue->seqnum) % 0xfff;
174 /* First dword of the message is the message header - fill it in */
175 *((u32 *) data) = (seqnum << 20) | (HFI_MSG_CMD << 16) |
178 ret = a6xx_hfi_queue_write(gmu, queue, data, dwords);
180 DRM_DEV_ERROR(gmu->dev, "Unable to send message %s id %d\n",
181 a6xx_hfi_msg_id[id], seqnum);
185 return a6xx_hfi_wait_for_ack(gmu, id, seqnum, payload, payload_size);
188 static int a6xx_hfi_send_gmu_init(struct a6xx_gmu *gmu, int boot_state)
190 struct a6xx_hfi_msg_gmu_init_cmd msg = { 0 };
192 msg.dbg_buffer_addr = (u32) gmu->debug.iova;
193 msg.dbg_buffer_size = (u32) gmu->debug.size;
194 msg.boot_state = boot_state;
196 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_INIT, &msg, sizeof(msg),
200 static int a6xx_hfi_get_fw_version(struct a6xx_gmu *gmu, u32 *version)
202 struct a6xx_hfi_msg_fw_version msg = { 0 };
204 /* Currently supporting version 1.1 */
205 msg.supported_version = (1 << 28) | (1 << 16);
207 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_FW_VERSION, &msg, sizeof(msg),
208 version, sizeof(*version));
211 static int a6xx_hfi_send_perf_table_v1(struct a6xx_gmu *gmu)
213 struct a6xx_hfi_msg_perf_table_v1 msg = { 0 };
216 msg.num_gpu_levels = gmu->nr_gpu_freqs;
217 msg.num_gmu_levels = gmu->nr_gmu_freqs;
219 for (i = 0; i < gmu->nr_gpu_freqs; i++) {
220 msg.gx_votes[i].vote = gmu->gx_arc_votes[i];
221 msg.gx_votes[i].freq = gmu->gpu_freqs[i] / 1000;
224 for (i = 0; i < gmu->nr_gmu_freqs; i++) {
225 msg.cx_votes[i].vote = gmu->cx_arc_votes[i];
226 msg.cx_votes[i].freq = gmu->gmu_freqs[i] / 1000;
229 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_PERF_TABLE, &msg, sizeof(msg),
233 static int a6xx_hfi_send_perf_table(struct a6xx_gmu *gmu)
235 struct a6xx_hfi_msg_perf_table msg = { 0 };
238 msg.num_gpu_levels = gmu->nr_gpu_freqs;
239 msg.num_gmu_levels = gmu->nr_gmu_freqs;
241 for (i = 0; i < gmu->nr_gpu_freqs; i++) {
242 msg.gx_votes[i].vote = gmu->gx_arc_votes[i];
243 msg.gx_votes[i].acd = 0xffffffff;
244 msg.gx_votes[i].freq = gmu->gpu_freqs[i] / 1000;
247 for (i = 0; i < gmu->nr_gmu_freqs; i++) {
248 msg.cx_votes[i].vote = gmu->cx_arc_votes[i];
249 msg.cx_votes[i].freq = gmu->gmu_freqs[i] / 1000;
252 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_PERF_TABLE, &msg, sizeof(msg),
256 static void a618_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
258 /* Send a single "off" entry since the 618 GMU doesn't do bus scaling */
259 msg->bw_level_num = 1;
261 msg->ddr_cmds_num = 3;
262 msg->ddr_wait_bitmask = 0x01;
264 msg->ddr_cmds_addrs[0] = 0x50000;
265 msg->ddr_cmds_addrs[1] = 0x5003c;
266 msg->ddr_cmds_addrs[2] = 0x5000c;
268 msg->ddr_cmds_data[0][0] = 0x40000000;
269 msg->ddr_cmds_data[0][1] = 0x40000000;
270 msg->ddr_cmds_data[0][2] = 0x40000000;
273 * These are the CX (CNOC) votes - these are used by the GMU but the
274 * votes are known and fixed for the target
276 msg->cnoc_cmds_num = 1;
277 msg->cnoc_wait_bitmask = 0x01;
279 msg->cnoc_cmds_addrs[0] = 0x5007c;
280 msg->cnoc_cmds_data[0][0] = 0x40000000;
281 msg->cnoc_cmds_data[1][0] = 0x60000001;
284 static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
286 /* Send a single "off" entry since the 630 GMU doesn't do bus scaling */
287 msg->bw_level_num = 1;
289 msg->ddr_cmds_num = 3;
290 msg->ddr_wait_bitmask = 0x07;
292 msg->ddr_cmds_addrs[0] = 0x50000;
293 msg->ddr_cmds_addrs[1] = 0x5005c;
294 msg->ddr_cmds_addrs[2] = 0x5000c;
296 msg->ddr_cmds_data[0][0] = 0x40000000;
297 msg->ddr_cmds_data[0][1] = 0x40000000;
298 msg->ddr_cmds_data[0][2] = 0x40000000;
301 * These are the CX (CNOC) votes. This is used but the values for the
302 * sdm845 GMU are known and fixed so we can hard code them.
305 msg->cnoc_cmds_num = 3;
306 msg->cnoc_wait_bitmask = 0x05;
308 msg->cnoc_cmds_addrs[0] = 0x50034;
309 msg->cnoc_cmds_addrs[1] = 0x5007c;
310 msg->cnoc_cmds_addrs[2] = 0x5004c;
312 msg->cnoc_cmds_data[0][0] = 0x40000000;
313 msg->cnoc_cmds_data[0][1] = 0x00000000;
314 msg->cnoc_cmds_data[0][2] = 0x40000000;
316 msg->cnoc_cmds_data[1][0] = 0x60000001;
317 msg->cnoc_cmds_data[1][1] = 0x20000001;
318 msg->cnoc_cmds_data[1][2] = 0x60000001;
322 static int a6xx_hfi_send_bw_table(struct a6xx_gmu *gmu)
324 struct a6xx_hfi_msg_bw_table msg = { 0 };
325 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
326 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
328 if (adreno_is_a618(adreno_gpu))
329 a618_build_bw_table(&msg);
331 a6xx_build_bw_table(&msg);
333 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_BW_TABLE, &msg, sizeof(msg),
337 static int a6xx_hfi_send_test(struct a6xx_gmu *gmu)
339 struct a6xx_hfi_msg_test msg = { 0 };
341 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_TEST, &msg, sizeof(msg),
345 static int a6xx_hfi_send_start(struct a6xx_gmu *gmu)
347 struct a6xx_hfi_msg_start msg = { 0 };
349 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_START, &msg, sizeof(msg),
353 static int a6xx_hfi_send_core_fw_start(struct a6xx_gmu *gmu)
355 struct a6xx_hfi_msg_core_fw_start msg = { 0 };
357 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_CORE_FW_START, &msg,
358 sizeof(msg), NULL, 0);
361 int a6xx_hfi_set_freq(struct a6xx_gmu *gmu, int index)
363 struct a6xx_hfi_gx_bw_perf_vote_cmd msg = { 0 };
365 msg.ack_type = 1; /* blocking */
367 msg.bw = 0; /* TODO: bus scaling */
369 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_GX_BW_PERF_VOTE, &msg,
370 sizeof(msg), NULL, 0);
373 int a6xx_hfi_send_prep_slumber(struct a6xx_gmu *gmu)
375 struct a6xx_hfi_prep_slumber_cmd msg = { 0 };
377 /* TODO: should freq and bw fields be non-zero ? */
379 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_PREPARE_SLUMBER, &msg,
380 sizeof(msg), NULL, 0);
383 static int a6xx_hfi_start_v1(struct a6xx_gmu *gmu, int boot_state)
387 ret = a6xx_hfi_send_gmu_init(gmu, boot_state);
391 ret = a6xx_hfi_get_fw_version(gmu, NULL);
396 * We have to get exchange version numbers per the sequence but at this
397 * point th kernel driver doesn't need to know the exact version of
401 ret = a6xx_hfi_send_perf_table_v1(gmu);
405 ret = a6xx_hfi_send_bw_table(gmu);
410 * Let the GMU know that there won't be any more HFI messages until next
413 a6xx_hfi_send_test(gmu);
418 int a6xx_hfi_start(struct a6xx_gmu *gmu, int boot_state)
423 return a6xx_hfi_start_v1(gmu, boot_state);
426 ret = a6xx_hfi_send_perf_table(gmu);
430 ret = a6xx_hfi_send_bw_table(gmu);
434 ret = a6xx_hfi_send_core_fw_start(gmu);
439 * Downstream driver sends this in its "a6xx_hw_init" equivalent,
440 * but seems to be no harm in sending it here
442 ret = a6xx_hfi_send_start(gmu);
449 void a6xx_hfi_stop(struct a6xx_gmu *gmu)
453 for (i = 0; i < ARRAY_SIZE(gmu->queues); i++) {
454 struct a6xx_hfi_queue *queue = &gmu->queues[i];
459 if (queue->header->read_index != queue->header->write_index)
460 DRM_DEV_ERROR(gmu->dev, "HFI queue %d is not empty\n", i);
462 queue->header->read_index = 0;
463 queue->header->write_index = 0;
467 static void a6xx_hfi_queue_init(struct a6xx_hfi_queue *queue,
468 struct a6xx_hfi_queue_header *header, void *virt, u64 iova,
471 spin_lock_init(&queue->lock);
472 queue->header = header;
474 atomic_set(&queue->seqnum, 0);
476 /* Set up the shared memory header */
478 header->type = 10 << 8 | id;
480 header->size = SZ_4K >> 2;
481 header->msg_size = 0;
483 header->rx_watermark = 1;
484 header->tx_watermark = 1;
485 header->rx_request = 1;
486 header->tx_request = 0;
487 header->read_index = 0;
488 header->write_index = 0;
491 void a6xx_hfi_init(struct a6xx_gmu *gmu)
493 struct a6xx_gmu_bo *hfi = &gmu->hfi;
494 struct a6xx_hfi_queue_table_header *table = hfi->virt;
495 struct a6xx_hfi_queue_header *headers = hfi->virt + sizeof(*table);
500 * The table size is the size of the table header plus all of the queue
503 table_size = sizeof(*table);
504 table_size += (ARRAY_SIZE(gmu->queues) *
505 sizeof(struct a6xx_hfi_queue_header));
508 table->size = table_size;
509 /* First queue header is located immediately after the table header */
510 table->qhdr0_offset = sizeof(*table) >> 2;
511 table->qhdr_size = sizeof(struct a6xx_hfi_queue_header) >> 2;
512 table->num_queues = ARRAY_SIZE(gmu->queues);
513 table->active_queues = ARRAY_SIZE(gmu->queues);
517 a6xx_hfi_queue_init(&gmu->queues[0], &headers[0], hfi->virt + offset,
518 hfi->iova + offset, 0);
520 /* GMU response queue */
522 a6xx_hfi_queue_init(&gmu->queues[1], &headers[1], hfi->virt + offset,
523 hfi->iova + offset, gmu->legacy ? 4 : 1);