1 /* Copyright (c) 2016-2018, The Linux Foundation. All rights reserved.
3 * This program is free software; you can redistribute it and/or modify
4 * it under the terms of the GNU General Public License version 2 and
5 * only version 2 as published by the Free Software Foundation.
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
13 #define pr_fmt(fmt) "[drm:%s:%d] " fmt, __func__, __LINE__
15 #include <linux/debugfs.h>
16 #include <linux/errno.h>
17 #include <linux/mutex.h>
18 #include <linux/sort.h>
19 #include <linux/clk.h>
20 #include <linux/bitmap.h>
23 #include "dpu_trace.h"
25 #include "dpu_core_perf.h"
28 * enum dpu_perf_mode - performance tuning mode
29 * @DPU_PERF_MODE_NORMAL: performance controlled by user mode client
30 * @DPU_PERF_MODE_MINIMUM: performance bounded by minimum setting
31 * @DPU_PERF_MODE_FIXED: performance bounded by fixed setting
35 DPU_PERF_MODE_MINIMUM,
40 static struct dpu_kms *_dpu_crtc_get_kms(struct drm_crtc *crtc)
42 struct msm_drm_private *priv;
44 if (!crtc->dev || !crtc->dev->dev_private) {
45 DPU_ERROR("invalid device\n");
49 priv = crtc->dev->dev_private;
50 if (!priv || !priv->kms) {
51 DPU_ERROR("invalid kms\n");
55 return to_dpu_kms(priv->kms);
58 static bool _dpu_core_video_mode_intf_connected(struct drm_crtc *crtc)
60 struct drm_crtc *tmp_crtc;
62 drm_for_each_crtc(tmp_crtc, crtc->dev) {
63 if ((dpu_crtc_get_intf_mode(tmp_crtc) == INTF_MODE_VIDEO) &&
65 DPU_DEBUG("video interface connected crtc:%d\n",
74 static void _dpu_core_perf_calc_crtc(struct dpu_kms *kms,
75 struct drm_crtc *crtc,
76 struct drm_crtc_state *state,
77 struct dpu_core_perf_params *perf)
79 struct dpu_crtc_state *dpu_cstate;
82 if (!kms || !kms->catalog || !crtc || !state || !perf) {
83 DPU_ERROR("invalid parameters\n");
87 dpu_cstate = to_dpu_crtc_state(state);
88 memset(perf, 0, sizeof(struct dpu_core_perf_params));
90 if (!dpu_cstate->bw_control) {
91 for (i = 0; i < DPU_CORE_PERF_DATA_BUS_ID_MAX; i++) {
92 perf->bw_ctl[i] = kms->catalog->perf.max_bw_high *
94 perf->max_per_pipe_ib[i] = perf->bw_ctl[i];
96 perf->core_clk_rate = kms->perf.max_core_clk_rate;
97 } else if (kms->perf.perf_tune.mode == DPU_PERF_MODE_MINIMUM) {
98 for (i = 0; i < DPU_CORE_PERF_DATA_BUS_ID_MAX; i++) {
100 perf->max_per_pipe_ib[i] = 0;
102 perf->core_clk_rate = 0;
103 } else if (kms->perf.perf_tune.mode == DPU_PERF_MODE_FIXED) {
104 for (i = 0; i < DPU_CORE_PERF_DATA_BUS_ID_MAX; i++) {
105 perf->bw_ctl[i] = kms->perf.fix_core_ab_vote;
106 perf->max_per_pipe_ib[i] = kms->perf.fix_core_ib_vote;
108 perf->core_clk_rate = kms->perf.fix_core_clk_rate;
112 "crtc=%d clk_rate=%llu core_ib=%llu core_ab=%llu llcc_ib=%llu llcc_ab=%llu mem_ib=%llu mem_ab=%llu\n",
113 crtc->base.id, perf->core_clk_rate,
114 perf->max_per_pipe_ib[DPU_CORE_PERF_DATA_BUS_ID_MNOC],
115 perf->bw_ctl[DPU_CORE_PERF_DATA_BUS_ID_MNOC],
116 perf->max_per_pipe_ib[DPU_CORE_PERF_DATA_BUS_ID_LLCC],
117 perf->bw_ctl[DPU_CORE_PERF_DATA_BUS_ID_LLCC],
118 perf->max_per_pipe_ib[DPU_CORE_PERF_DATA_BUS_ID_EBI],
119 perf->bw_ctl[DPU_CORE_PERF_DATA_BUS_ID_EBI]);
122 int dpu_core_perf_crtc_check(struct drm_crtc *crtc,
123 struct drm_crtc_state *state)
126 u64 bw_sum_of_intfs = 0;
127 enum dpu_crtc_client_type curr_client_type;
129 struct dpu_crtc_state *dpu_cstate;
130 struct drm_crtc *tmp_crtc;
134 if (!crtc || !state) {
135 DPU_ERROR("invalid crtc\n");
139 kms = _dpu_crtc_get_kms(crtc);
140 if (!kms || !kms->catalog) {
141 DPU_ERROR("invalid parameters\n");
145 /* we only need bandwidth check on real-time clients (interfaces) */
146 if (dpu_crtc_get_client_type(crtc) == NRT_CLIENT)
149 dpu_cstate = to_dpu_crtc_state(state);
151 /* obtain new values */
152 _dpu_core_perf_calc_crtc(kms, crtc, state, &dpu_cstate->new_perf);
154 for (i = DPU_CORE_PERF_DATA_BUS_ID_MNOC;
155 i < DPU_CORE_PERF_DATA_BUS_ID_MAX; i++) {
156 bw_sum_of_intfs = dpu_cstate->new_perf.bw_ctl[i];
157 curr_client_type = dpu_crtc_get_client_type(crtc);
159 drm_for_each_crtc(tmp_crtc, crtc->dev) {
160 if (tmp_crtc->enabled &&
161 (dpu_crtc_get_client_type(tmp_crtc) ==
163 (tmp_crtc != crtc)) {
164 struct dpu_crtc_state *tmp_cstate =
165 to_dpu_crtc_state(tmp_crtc->state);
167 DPU_DEBUG("crtc:%d bw:%llu ctrl:%d\n",
169 tmp_cstate->new_perf.bw_ctl[i],
170 tmp_cstate->bw_control);
172 * For bw check only use the bw if the
173 * atomic property has been already set
175 if (tmp_cstate->bw_control)
177 tmp_cstate->new_perf.bw_ctl[i];
181 /* convert bandwidth to kb */
182 bw = DIV_ROUND_UP_ULL(bw_sum_of_intfs, 1000);
183 DPU_DEBUG("calculated bandwidth=%uk\n", bw);
185 is_video_mode = dpu_crtc_get_intf_mode(crtc) == INTF_MODE_VIDEO;
186 threshold = (is_video_mode ||
187 _dpu_core_video_mode_intf_connected(crtc)) ?
188 kms->catalog->perf.max_bw_low :
189 kms->catalog->perf.max_bw_high;
191 DPU_DEBUG("final threshold bw limit = %d\n", threshold);
193 if (!dpu_cstate->bw_control) {
194 DPU_DEBUG("bypass bandwidth check\n");
195 } else if (!threshold) {
196 DPU_ERROR("no bandwidth limits specified\n");
198 } else if (bw > threshold) {
199 DPU_ERROR("exceeds bandwidth: %ukb > %ukb\n", bw,
208 static int _dpu_core_perf_crtc_update_bus(struct dpu_kms *kms,
209 struct drm_crtc *crtc, u32 bus_id)
211 struct dpu_core_perf_params perf = { { 0 } };
212 enum dpu_crtc_client_type curr_client_type
213 = dpu_crtc_get_client_type(crtc);
214 struct drm_crtc *tmp_crtc;
215 struct dpu_crtc_state *dpu_cstate;
218 drm_for_each_crtc(tmp_crtc, crtc->dev) {
219 if (tmp_crtc->enabled &&
221 dpu_crtc_get_client_type(tmp_crtc)) {
222 dpu_cstate = to_dpu_crtc_state(tmp_crtc->state);
224 perf.max_per_pipe_ib[bus_id] =
225 max(perf.max_per_pipe_ib[bus_id],
226 dpu_cstate->new_perf.max_per_pipe_ib[bus_id]);
228 DPU_DEBUG("crtc=%d bus_id=%d bw=%llu\n",
229 tmp_crtc->base.id, bus_id,
230 dpu_cstate->new_perf.bw_ctl[bus_id]);
237 * @dpu_core_perf_crtc_release_bw() - request zero bandwidth
238 * @crtc - pointer to a crtc
240 * Function checks a state variable for the crtc, if all pending commit
241 * requests are done, meaning no more bandwidth is needed, release
244 void dpu_core_perf_crtc_release_bw(struct drm_crtc *crtc)
246 struct drm_crtc *tmp_crtc;
247 struct dpu_crtc *dpu_crtc;
248 struct dpu_crtc_state *dpu_cstate;
253 DPU_ERROR("invalid crtc\n");
257 kms = _dpu_crtc_get_kms(crtc);
258 if (!kms || !kms->catalog) {
259 DPU_ERROR("invalid kms\n");
263 dpu_crtc = to_dpu_crtc(crtc);
264 dpu_cstate = to_dpu_crtc_state(crtc->state);
266 /* only do this for command mode rt client */
267 if (dpu_crtc_get_intf_mode(crtc) != INTF_MODE_CMD)
271 * If video interface present, cmd panel bandwidth cannot be
274 if (dpu_crtc_get_intf_mode(crtc) == INTF_MODE_CMD)
275 drm_for_each_crtc(tmp_crtc, crtc->dev) {
276 if (tmp_crtc->enabled &&
277 dpu_crtc_get_intf_mode(tmp_crtc) ==
282 /* Release the bandwidth */
283 if (kms->perf.enable_bw_release) {
284 trace_dpu_cmd_release_bw(crtc->base.id);
285 DPU_DEBUG("Release BW crtc=%d\n", crtc->base.id);
286 for (i = 0; i < DPU_CORE_PERF_DATA_BUS_ID_MAX; i++) {
287 dpu_crtc->cur_perf.bw_ctl[i] = 0;
288 _dpu_core_perf_crtc_update_bus(kms, crtc, i);
293 static int _dpu_core_perf_set_core_clk_rate(struct dpu_kms *kms, u64 rate)
295 struct dss_clk *core_clk = kms->perf.core_clk;
297 if (core_clk->max_rate && (rate > core_clk->max_rate))
298 rate = core_clk->max_rate;
300 core_clk->rate = rate;
301 return msm_dss_clk_set_rate(core_clk, 1);
304 static u64 _dpu_core_perf_get_core_clk_rate(struct dpu_kms *kms)
306 u64 clk_rate = kms->perf.perf_tune.min_core_clk;
307 struct drm_crtc *crtc;
308 struct dpu_crtc_state *dpu_cstate;
310 drm_for_each_crtc(crtc, kms->dev) {
312 dpu_cstate = to_dpu_crtc_state(crtc->state);
313 clk_rate = max(dpu_cstate->new_perf.core_clk_rate,
315 clk_rate = clk_round_rate(kms->perf.core_clk->clk,
320 if (kms->perf.perf_tune.mode == DPU_PERF_MODE_FIXED)
321 clk_rate = kms->perf.fix_core_clk_rate;
323 DPU_DEBUG("clk:%llu\n", clk_rate);
328 int dpu_core_perf_crtc_update(struct drm_crtc *crtc,
329 int params_changed, bool stop_req)
331 struct dpu_core_perf_params *new, *old;
332 int update_bus = 0, update_clk = 0;
334 struct dpu_crtc *dpu_crtc;
335 struct dpu_crtc_state *dpu_cstate;
337 struct msm_drm_private *priv;
342 DPU_ERROR("invalid crtc\n");
346 kms = _dpu_crtc_get_kms(crtc);
347 if (!kms || !kms->catalog) {
348 DPU_ERROR("invalid kms\n");
351 priv = kms->dev->dev_private;
353 dpu_crtc = to_dpu_crtc(crtc);
354 dpu_cstate = to_dpu_crtc_state(crtc->state);
356 DPU_DEBUG("crtc:%d stop_req:%d core_clk:%llu\n",
357 crtc->base.id, stop_req, kms->perf.core_clk_rate);
359 old = &dpu_crtc->cur_perf;
360 new = &dpu_cstate->new_perf;
362 if (crtc->enabled && !stop_req) {
363 for (i = 0; i < DPU_CORE_PERF_DATA_BUS_ID_MAX; i++) {
365 * cases for bus bandwidth update.
366 * 1. new bandwidth vote - "ab or ib vote" is higher
367 * than current vote for update request.
368 * 2. new bandwidth vote - "ab or ib vote" is lower
369 * than current vote at end of commit or stop.
371 if ((params_changed && ((new->bw_ctl[i] >
373 (new->max_per_pipe_ib[i] >
374 old->max_per_pipe_ib[i]))) ||
375 (!params_changed && ((new->bw_ctl[i] <
377 (new->max_per_pipe_ib[i] <
378 old->max_per_pipe_ib[i])))) {
380 "crtc=%d p=%d new_bw=%llu,old_bw=%llu\n",
381 crtc->base.id, params_changed,
382 new->bw_ctl[i], old->bw_ctl[i]);
383 old->bw_ctl[i] = new->bw_ctl[i];
384 old->max_per_pipe_ib[i] =
385 new->max_per_pipe_ib[i];
386 update_bus |= BIT(i);
390 if ((params_changed &&
391 (new->core_clk_rate > old->core_clk_rate)) ||
393 (new->core_clk_rate < old->core_clk_rate))) {
394 old->core_clk_rate = new->core_clk_rate;
398 DPU_DEBUG("crtc=%d disable\n", crtc->base.id);
399 memset(old, 0, sizeof(*old));
400 memset(new, 0, sizeof(*new));
404 trace_dpu_perf_crtc_update(crtc->base.id,
405 new->bw_ctl[DPU_CORE_PERF_DATA_BUS_ID_MNOC],
406 new->bw_ctl[DPU_CORE_PERF_DATA_BUS_ID_LLCC],
407 new->bw_ctl[DPU_CORE_PERF_DATA_BUS_ID_EBI],
408 new->core_clk_rate, stop_req,
409 update_bus, update_clk);
411 for (i = 0; i < DPU_CORE_PERF_DATA_BUS_ID_MAX; i++) {
412 if (update_bus & BIT(i)) {
413 ret = _dpu_core_perf_crtc_update_bus(kms, crtc, i);
415 DPU_ERROR("crtc-%d: failed to update bw vote for bus-%d\n",
423 * Update the clock after bandwidth vote to ensure
424 * bandwidth is available before clock rate is increased.
427 clk_rate = _dpu_core_perf_get_core_clk_rate(kms);
429 trace_dpu_core_perf_update_clk(kms->dev, stop_req, clk_rate);
431 ret = _dpu_core_perf_set_core_clk_rate(kms, clk_rate);
433 DPU_ERROR("failed to set %s clock rate %llu\n",
434 kms->perf.core_clk->clk_name, clk_rate);
438 kms->perf.core_clk_rate = clk_rate;
439 DPU_DEBUG("update clk rate = %lld HZ\n", clk_rate);
444 #ifdef CONFIG_DEBUG_FS
446 static ssize_t _dpu_core_perf_mode_write(struct file *file,
447 const char __user *user_buf, size_t count, loff_t *ppos)
449 struct dpu_core_perf *perf = file->private_data;
450 struct dpu_perf_cfg *cfg = &perf->catalog->perf;
454 ret = kstrtouint_from_user(user_buf, count, 0, &perf_mode);
458 if (perf_mode >= DPU_PERF_MODE_MAX)
461 if (perf_mode == DPU_PERF_MODE_FIXED) {
462 DRM_INFO("fix performance mode\n");
463 } else if (perf_mode == DPU_PERF_MODE_MINIMUM) {
464 /* run the driver with max clk and BW vote */
465 perf->perf_tune.min_core_clk = perf->max_core_clk_rate;
466 perf->perf_tune.min_bus_vote =
467 (u64) cfg->max_bw_high * 1000;
468 DRM_INFO("minimum performance mode\n");
469 } else if (perf_mode == DPU_PERF_MODE_NORMAL) {
470 /* reset the perf tune params to 0 */
471 perf->perf_tune.min_core_clk = 0;
472 perf->perf_tune.min_bus_vote = 0;
473 DRM_INFO("normal performance mode\n");
475 perf->perf_tune.mode = perf_mode;
480 static ssize_t _dpu_core_perf_mode_read(struct file *file,
481 char __user *buff, size_t count, loff_t *ppos)
483 struct dpu_core_perf *perf = file->private_data;
487 len = scnprintf(buf, sizeof(buf),
488 "mode %d min_mdp_clk %llu min_bus_vote %llu\n",
489 perf->perf_tune.mode,
490 perf->perf_tune.min_core_clk,
491 perf->perf_tune.min_bus_vote);
493 return simple_read_from_buffer(buff, count, ppos, buf, len);
496 static const struct file_operations dpu_core_perf_mode_fops = {
498 .read = _dpu_core_perf_mode_read,
499 .write = _dpu_core_perf_mode_write,
502 int dpu_core_perf_debugfs_init(struct dpu_kms *dpu_kms, struct dentry *parent)
504 struct dpu_core_perf *perf = &dpu_kms->perf;
505 struct dpu_mdss_cfg *catalog = perf->catalog;
506 struct dentry *entry;
508 entry = debugfs_create_dir("core_perf", parent);
509 if (IS_ERR_OR_NULL(entry))
512 debugfs_create_u64("max_core_clk_rate", 0600, entry,
513 &perf->max_core_clk_rate);
514 debugfs_create_u64("core_clk_rate", 0600, entry,
515 &perf->core_clk_rate);
516 debugfs_create_u32("enable_bw_release", 0600, entry,
517 (u32 *)&perf->enable_bw_release);
518 debugfs_create_u32("threshold_low", 0600, entry,
519 (u32 *)&catalog->perf.max_bw_low);
520 debugfs_create_u32("threshold_high", 0600, entry,
521 (u32 *)&catalog->perf.max_bw_high);
522 debugfs_create_u32("min_core_ib", 0600, entry,
523 (u32 *)&catalog->perf.min_core_ib);
524 debugfs_create_u32("min_llcc_ib", 0600, entry,
525 (u32 *)&catalog->perf.min_llcc_ib);
526 debugfs_create_u32("min_dram_ib", 0600, entry,
527 (u32 *)&catalog->perf.min_dram_ib);
528 debugfs_create_file("perf_mode", 0600, entry,
529 (u32 *)perf, &dpu_core_perf_mode_fops);
530 debugfs_create_u64("fix_core_clk_rate", 0600, entry,
531 &perf->fix_core_clk_rate);
532 debugfs_create_u64("fix_core_ib_vote", 0600, entry,
533 &perf->fix_core_ib_vote);
534 debugfs_create_u64("fix_core_ab_vote", 0600, entry,
535 &perf->fix_core_ab_vote);
541 void dpu_core_perf_destroy(struct dpu_core_perf *perf)
544 DPU_ERROR("invalid parameters\n");
548 perf->max_core_clk_rate = 0;
549 perf->core_clk = NULL;
550 perf->catalog = NULL;
554 int dpu_core_perf_init(struct dpu_core_perf *perf,
555 struct drm_device *dev,
556 struct dpu_mdss_cfg *catalog,
557 struct dss_clk *core_clk)
560 perf->catalog = catalog;
561 perf->core_clk = core_clk;
563 perf->max_core_clk_rate = core_clk->max_rate;
564 if (!perf->max_core_clk_rate) {
565 DPU_DEBUG("optional max core clk rate, use default\n");
566 perf->max_core_clk_rate = DPU_PERF_DEFAULT_MAX_CORE_CLK_RATE;