Commit | Line | Data |
---|---|---|
9b9d8632 JL |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Copyright (c) 2014-2020, NVIDIA CORPORATION. All rights reserved. | |
4 | */ | |
5 | ||
6 | #include <linux/kernel.h> | |
7 | #include <linux/io.h> | |
8 | #include <linux/clk.h> | |
9 | #include <linux/delay.h> | |
10 | #include <linux/of.h> | |
11 | ||
12 | #include <soc/tegra/mc.h> | |
13 | ||
14 | #include "tegra210-emc.h" | |
15 | #include "tegra210-mc.h" | |
16 | ||
17 | /* | |
18 | * Enable flags for specifying verbosity. | |
19 | */ | |
20 | #define INFO (1 << 0) | |
21 | #define STEPS (1 << 1) | |
22 | #define SUB_STEPS (1 << 2) | |
23 | #define PRELOCK (1 << 3) | |
24 | #define PRELOCK_STEPS (1 << 4) | |
25 | #define ACTIVE_EN (1 << 5) | |
26 | #define PRAMP_UP (1 << 6) | |
27 | #define PRAMP_DN (1 << 7) | |
28 | #define EMA_WRITES (1 << 10) | |
29 | #define EMA_UPDATES (1 << 11) | |
30 | #define PER_TRAIN (1 << 16) | |
31 | #define CC_PRINT (1 << 17) | |
32 | #define CCFIFO (1 << 29) | |
33 | #define REGS (1 << 30) | |
34 | #define REG_LISTS (1 << 31) | |
35 | ||
36 | #define emc_dbg(emc, flags, ...) dev_dbg(emc->dev, __VA_ARGS__) | |
37 | ||
38 | #define DVFS_CLOCK_CHANGE_VERSION 21021 | |
39 | #define EMC_PRELOCK_VERSION 2101 | |
40 | ||
41 | enum { | |
42 | DVFS_SEQUENCE = 1, | |
43 | WRITE_TRAINING_SEQUENCE = 2, | |
44 | PERIODIC_TRAINING_SEQUENCE = 3, | |
45 | DVFS_PT1 = 10, | |
46 | DVFS_UPDATE = 11, | |
47 | TRAINING_PT1 = 12, | |
48 | TRAINING_UPDATE = 13, | |
49 | PERIODIC_TRAINING_UPDATE = 14 | |
50 | }; | |
51 | ||
52 | /* | |
53 | * PTFV defines - basically just indexes into the per table PTFV array. | |
54 | */ | |
55 | #define PTFV_DQSOSC_MOVAVG_C0D0U0_INDEX 0 | |
56 | #define PTFV_DQSOSC_MOVAVG_C0D0U1_INDEX 1 | |
57 | #define PTFV_DQSOSC_MOVAVG_C0D1U0_INDEX 2 | |
58 | #define PTFV_DQSOSC_MOVAVG_C0D1U1_INDEX 3 | |
59 | #define PTFV_DQSOSC_MOVAVG_C1D0U0_INDEX 4 | |
60 | #define PTFV_DQSOSC_MOVAVG_C1D0U1_INDEX 5 | |
61 | #define PTFV_DQSOSC_MOVAVG_C1D1U0_INDEX 6 | |
62 | #define PTFV_DQSOSC_MOVAVG_C1D1U1_INDEX 7 | |
63 | #define PTFV_DVFS_SAMPLES_INDEX 9 | |
64 | #define PTFV_MOVAVG_WEIGHT_INDEX 10 | |
65 | #define PTFV_CONFIG_CTRL_INDEX 11 | |
66 | ||
67 | #define PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA (1 << 0) | |
68 | ||
69 | /* | |
70 | * Do arithmetic in fixed point. | |
71 | */ | |
72 | #define MOVAVG_PRECISION_FACTOR 100 | |
73 | ||
74 | /* | |
75 | * The division portion of the average operation. | |
76 | */ | |
77 | #define __AVERAGE_PTFV(dev) \ | |
78 | ({ next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] = \ | |
79 | next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \ | |
80 | next->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; }) | |
81 | ||
82 | /* | |
83 | * Convert val to fixed point and add it to the temporary average. | |
84 | */ | |
85 | #define __INCREMENT_PTFV(dev, val) \ | |
86 | ({ next->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] += \ | |
87 | ((val) * MOVAVG_PRECISION_FACTOR); }) | |
88 | ||
89 | /* | |
90 | * Convert a moving average back to integral form and return the value. | |
91 | */ | |
92 | #define __MOVAVG_AC(timing, dev) \ | |
93 | ((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX] / \ | |
94 | MOVAVG_PRECISION_FACTOR) | |
95 | ||
96 | /* Weighted update. */ | |
97 | #define __WEIGHTED_UPDATE_PTFV(dev, nval) \ | |
98 | do { \ | |
99 | int w = PTFV_MOVAVG_WEIGHT_INDEX; \ | |
100 | int dqs = PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX; \ | |
101 | \ | |
102 | next->ptfv_list[dqs] = \ | |
103 | ((nval * MOVAVG_PRECISION_FACTOR) + \ | |
104 | (next->ptfv_list[dqs] * \ | |
105 | next->ptfv_list[w])) / \ | |
106 | (next->ptfv_list[w] + 1); \ | |
107 | \ | |
108 | emc_dbg(emc, EMA_UPDATES, "%s: (s=%lu) EMA: %u\n", \ | |
109 | __stringify(dev), nval, next->ptfv_list[dqs]); \ | |
110 | } while (0) | |
111 | ||
112 | /* Access a particular average. */ | |
113 | #define __MOVAVG(timing, dev) \ | |
114 | ((timing)->ptfv_list[PTFV_DQSOSC_MOVAVG_ ## dev ## _INDEX]) | |
115 | ||
116 | static u32 update_clock_tree_delay(struct tegra210_emc *emc, int type) | |
117 | { | |
118 | bool periodic_training_update = type == PERIODIC_TRAINING_UPDATE; | |
119 | struct tegra210_emc_timing *last = emc->last; | |
120 | struct tegra210_emc_timing *next = emc->next; | |
121 | u32 last_timing_rate_mhz = last->rate / 1000; | |
122 | u32 next_timing_rate_mhz = next->rate / 1000; | |
123 | bool dvfs_update = type == DVFS_UPDATE; | |
124 | s32 tdel = 0, tmdel = 0, adel = 0; | |
125 | bool dvfs_pt1 = type == DVFS_PT1; | |
126 | unsigned long cval = 0; | |
127 | u32 temp[2][2], value; | |
128 | unsigned int i; | |
129 | ||
130 | /* | |
131 | * Dev0 MSB. | |
132 | */ | |
133 | if (dvfs_pt1 || periodic_training_update) { | |
134 | value = tegra210_emc_mrr_read(emc, 2, 19); | |
135 | ||
136 | for (i = 0; i < emc->num_channels; i++) { | |
137 | temp[i][0] = (value & 0x00ff) << 8; | |
138 | temp[i][1] = (value & 0xff00) << 0; | |
139 | value >>= 16; | |
140 | } | |
141 | ||
142 | /* | |
143 | * Dev0 LSB. | |
144 | */ | |
145 | value = tegra210_emc_mrr_read(emc, 2, 18); | |
146 | ||
147 | for (i = 0; i < emc->num_channels; i++) { | |
148 | temp[i][0] |= (value & 0x00ff) >> 0; | |
149 | temp[i][1] |= (value & 0xff00) >> 8; | |
150 | value >>= 16; | |
151 | } | |
152 | } | |
153 | ||
154 | if (dvfs_pt1 || periodic_training_update) { | |
155 | cval = tegra210_emc_actual_osc_clocks(last->run_clocks); | |
156 | cval *= 1000000; | |
157 | cval /= last_timing_rate_mhz * 2 * temp[0][0]; | |
158 | } | |
159 | ||
160 | if (dvfs_pt1) | |
161 | __INCREMENT_PTFV(C0D0U0, cval); | |
162 | else if (dvfs_update) | |
163 | __AVERAGE_PTFV(C0D0U0); | |
164 | else if (periodic_training_update) | |
165 | __WEIGHTED_UPDATE_PTFV(C0D0U0, cval); | |
166 | ||
167 | if (dvfs_update || periodic_training_update) { | |
168 | tdel = next->current_dram_clktree[C0D0U0] - | |
169 | __MOVAVG_AC(next, C0D0U0); | |
170 | tmdel = (tdel < 0) ? -1 * tdel : tdel; | |
171 | adel = tmdel; | |
172 | ||
173 | if (tmdel * 128 * next_timing_rate_mhz / 1000000 > | |
174 | next->tree_margin) | |
175 | next->current_dram_clktree[C0D0U0] = | |
176 | __MOVAVG_AC(next, C0D0U0); | |
177 | } | |
178 | ||
179 | if (dvfs_pt1 || periodic_training_update) { | |
180 | cval = tegra210_emc_actual_osc_clocks(last->run_clocks); | |
181 | cval *= 1000000; | |
182 | cval /= last_timing_rate_mhz * 2 * temp[0][1]; | |
183 | } | |
184 | ||
185 | if (dvfs_pt1) | |
186 | __INCREMENT_PTFV(C0D0U1, cval); | |
187 | else if (dvfs_update) | |
188 | __AVERAGE_PTFV(C0D0U1); | |
189 | else if (periodic_training_update) | |
190 | __WEIGHTED_UPDATE_PTFV(C0D0U1, cval); | |
191 | ||
192 | if (dvfs_update || periodic_training_update) { | |
193 | tdel = next->current_dram_clktree[C0D0U1] - | |
194 | __MOVAVG_AC(next, C0D0U1); | |
195 | tmdel = (tdel < 0) ? -1 * tdel : tdel; | |
196 | ||
197 | if (tmdel > adel) | |
198 | adel = tmdel; | |
199 | ||
200 | if (tmdel * 128 * next_timing_rate_mhz / 1000000 > | |
201 | next->tree_margin) | |
202 | next->current_dram_clktree[C0D0U1] = | |
203 | __MOVAVG_AC(next, C0D0U1); | |
204 | } | |
205 | ||
206 | if (emc->num_channels > 1) { | |
207 | if (dvfs_pt1 || periodic_training_update) { | |
208 | cval = tegra210_emc_actual_osc_clocks(last->run_clocks); | |
209 | cval *= 1000000; | |
210 | cval /= last_timing_rate_mhz * 2 * temp[1][0]; | |
211 | } | |
212 | ||
213 | if (dvfs_pt1) | |
214 | __INCREMENT_PTFV(C1D0U0, cval); | |
215 | else if (dvfs_update) | |
216 | __AVERAGE_PTFV(C1D0U0); | |
217 | else if (periodic_training_update) | |
218 | __WEIGHTED_UPDATE_PTFV(C1D0U0, cval); | |
219 | ||
220 | if (dvfs_update || periodic_training_update) { | |
221 | tdel = next->current_dram_clktree[C1D0U0] - | |
222 | __MOVAVG_AC(next, C1D0U0); | |
223 | tmdel = (tdel < 0) ? -1 * tdel : tdel; | |
224 | ||
225 | if (tmdel > adel) | |
226 | adel = tmdel; | |
227 | ||
228 | if (tmdel * 128 * next_timing_rate_mhz / 1000000 > | |
229 | next->tree_margin) | |
230 | next->current_dram_clktree[C1D0U0] = | |
231 | __MOVAVG_AC(next, C1D0U0); | |
232 | } | |
233 | ||
234 | if (dvfs_pt1 || periodic_training_update) { | |
235 | cval = tegra210_emc_actual_osc_clocks(last->run_clocks); | |
236 | cval *= 1000000; | |
237 | cval /= last_timing_rate_mhz * 2 * temp[1][1]; | |
238 | } | |
239 | ||
240 | if (dvfs_pt1) | |
241 | __INCREMENT_PTFV(C1D0U1, cval); | |
242 | else if (dvfs_update) | |
243 | __AVERAGE_PTFV(C1D0U1); | |
244 | else if (periodic_training_update) | |
245 | __WEIGHTED_UPDATE_PTFV(C1D0U1, cval); | |
246 | ||
247 | if (dvfs_update || periodic_training_update) { | |
248 | tdel = next->current_dram_clktree[C1D0U1] - | |
249 | __MOVAVG_AC(next, C1D0U1); | |
250 | tmdel = (tdel < 0) ? -1 * tdel : tdel; | |
251 | ||
252 | if (tmdel > adel) | |
253 | adel = tmdel; | |
254 | ||
255 | if (tmdel * 128 * next_timing_rate_mhz / 1000000 > | |
256 | next->tree_margin) | |
257 | next->current_dram_clktree[C1D0U1] = | |
258 | __MOVAVG_AC(next, C1D0U1); | |
259 | } | |
260 | } | |
261 | ||
262 | if (emc->num_devices < 2) | |
263 | goto done; | |
264 | ||
265 | /* | |
266 | * Dev1 MSB. | |
267 | */ | |
268 | if (dvfs_pt1 || periodic_training_update) { | |
269 | value = tegra210_emc_mrr_read(emc, 1, 19); | |
270 | ||
271 | for (i = 0; i < emc->num_channels; i++) { | |
272 | temp[i][0] = (value & 0x00ff) << 8; | |
273 | temp[i][1] = (value & 0xff00) << 0; | |
274 | value >>= 16; | |
275 | } | |
276 | ||
277 | /* | |
278 | * Dev1 LSB. | |
279 | */ | |
be4c5c6e | 280 | value = tegra210_emc_mrr_read(emc, 1, 18); |
9b9d8632 JL |
281 | |
282 | for (i = 0; i < emc->num_channels; i++) { | |
283 | temp[i][0] |= (value & 0x00ff) >> 0; | |
284 | temp[i][1] |= (value & 0xff00) >> 8; | |
285 | value >>= 16; | |
286 | } | |
287 | } | |
288 | ||
289 | if (dvfs_pt1 || periodic_training_update) { | |
290 | cval = tegra210_emc_actual_osc_clocks(last->run_clocks); | |
291 | cval *= 1000000; | |
292 | cval /= last_timing_rate_mhz * 2 * temp[0][0]; | |
293 | } | |
294 | ||
295 | if (dvfs_pt1) | |
296 | __INCREMENT_PTFV(C0D1U0, cval); | |
297 | else if (dvfs_update) | |
298 | __AVERAGE_PTFV(C0D1U0); | |
299 | else if (periodic_training_update) | |
300 | __WEIGHTED_UPDATE_PTFV(C0D1U0, cval); | |
301 | ||
302 | if (dvfs_update || periodic_training_update) { | |
303 | tdel = next->current_dram_clktree[C0D1U0] - | |
304 | __MOVAVG_AC(next, C0D1U0); | |
305 | tmdel = (tdel < 0) ? -1 * tdel : tdel; | |
306 | ||
307 | if (tmdel > adel) | |
308 | adel = tmdel; | |
309 | ||
310 | if (tmdel * 128 * next_timing_rate_mhz / 1000000 > | |
311 | next->tree_margin) | |
312 | next->current_dram_clktree[C0D1U0] = | |
313 | __MOVAVG_AC(next, C0D1U0); | |
314 | } | |
315 | ||
316 | if (dvfs_pt1 || periodic_training_update) { | |
317 | cval = tegra210_emc_actual_osc_clocks(last->run_clocks); | |
318 | cval *= 1000000; | |
319 | cval /= last_timing_rate_mhz * 2 * temp[0][1]; | |
320 | } | |
321 | ||
322 | if (dvfs_pt1) | |
323 | __INCREMENT_PTFV(C0D1U1, cval); | |
324 | else if (dvfs_update) | |
325 | __AVERAGE_PTFV(C0D1U1); | |
326 | else if (periodic_training_update) | |
327 | __WEIGHTED_UPDATE_PTFV(C0D1U1, cval); | |
328 | ||
329 | if (dvfs_update || periodic_training_update) { | |
330 | tdel = next->current_dram_clktree[C0D1U1] - | |
331 | __MOVAVG_AC(next, C0D1U1); | |
332 | tmdel = (tdel < 0) ? -1 * tdel : tdel; | |
333 | ||
334 | if (tmdel > adel) | |
335 | adel = tmdel; | |
336 | ||
337 | if (tmdel * 128 * next_timing_rate_mhz / 1000000 > | |
338 | next->tree_margin) | |
339 | next->current_dram_clktree[C0D1U1] = | |
340 | __MOVAVG_AC(next, C0D1U1); | |
341 | } | |
342 | ||
343 | if (emc->num_channels > 1) { | |
344 | if (dvfs_pt1 || periodic_training_update) { | |
345 | cval = tegra210_emc_actual_osc_clocks(last->run_clocks); | |
346 | cval *= 1000000; | |
347 | cval /= last_timing_rate_mhz * 2 * temp[1][0]; | |
348 | } | |
349 | ||
350 | if (dvfs_pt1) | |
351 | __INCREMENT_PTFV(C1D1U0, cval); | |
352 | else if (dvfs_update) | |
353 | __AVERAGE_PTFV(C1D1U0); | |
354 | else if (periodic_training_update) | |
355 | __WEIGHTED_UPDATE_PTFV(C1D1U0, cval); | |
356 | ||
357 | if (dvfs_update || periodic_training_update) { | |
358 | tdel = next->current_dram_clktree[C1D1U0] - | |
359 | __MOVAVG_AC(next, C1D1U0); | |
360 | tmdel = (tdel < 0) ? -1 * tdel : tdel; | |
361 | ||
362 | if (tmdel > adel) | |
363 | adel = tmdel; | |
364 | ||
365 | if (tmdel * 128 * next_timing_rate_mhz / 1000000 > | |
366 | next->tree_margin) | |
367 | next->current_dram_clktree[C1D1U0] = | |
368 | __MOVAVG_AC(next, C1D1U0); | |
369 | } | |
370 | ||
371 | if (dvfs_pt1 || periodic_training_update) { | |
372 | cval = tegra210_emc_actual_osc_clocks(last->run_clocks); | |
373 | cval *= 1000000; | |
374 | cval /= last_timing_rate_mhz * 2 * temp[1][1]; | |
375 | } | |
376 | ||
377 | if (dvfs_pt1) | |
378 | __INCREMENT_PTFV(C1D1U1, cval); | |
379 | else if (dvfs_update) | |
380 | __AVERAGE_PTFV(C1D1U1); | |
381 | else if (periodic_training_update) | |
382 | __WEIGHTED_UPDATE_PTFV(C1D1U1, cval); | |
383 | ||
384 | if (dvfs_update || periodic_training_update) { | |
385 | tdel = next->current_dram_clktree[C1D1U1] - | |
386 | __MOVAVG_AC(next, C1D1U1); | |
387 | tmdel = (tdel < 0) ? -1 * tdel : tdel; | |
388 | ||
389 | if (tmdel > adel) | |
390 | adel = tmdel; | |
391 | ||
392 | if (tmdel * 128 * next_timing_rate_mhz / 1000000 > | |
393 | next->tree_margin) | |
394 | next->current_dram_clktree[C1D1U1] = | |
395 | __MOVAVG_AC(next, C1D1U1); | |
396 | } | |
397 | } | |
398 | ||
399 | done: | |
400 | return adel; | |
401 | } | |
402 | ||
403 | static u32 periodic_compensation_handler(struct tegra210_emc *emc, u32 type, | |
404 | struct tegra210_emc_timing *last, | |
405 | struct tegra210_emc_timing *next) | |
406 | { | |
407 | #define __COPY_EMA(nt, lt, dev) \ | |
408 | ({ __MOVAVG(nt, dev) = __MOVAVG(lt, dev) * \ | |
409 | (nt)->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; }) | |
410 | ||
411 | u32 i, adel = 0, samples = next->ptfv_list[PTFV_DVFS_SAMPLES_INDEX]; | |
412 | u32 delay; | |
413 | ||
414 | delay = tegra210_emc_actual_osc_clocks(last->run_clocks); | |
415 | delay *= 1000; | |
416 | delay = 2 + (delay / last->rate); | |
417 | ||
418 | if (!next->periodic_training) | |
419 | return 0; | |
420 | ||
421 | if (type == DVFS_SEQUENCE) { | |
422 | if (last->periodic_training && | |
423 | (next->ptfv_list[PTFV_CONFIG_CTRL_INDEX] & | |
424 | PTFV_CONFIG_CTRL_USE_PREVIOUS_EMA)) { | |
425 | /* | |
426 | * If the previous frequency was using periodic | |
427 | * calibration then we can reuse the previous | |
428 | * frequencies EMA data. | |
429 | */ | |
430 | __COPY_EMA(next, last, C0D0U0); | |
431 | __COPY_EMA(next, last, C0D0U1); | |
432 | __COPY_EMA(next, last, C1D0U0); | |
433 | __COPY_EMA(next, last, C1D0U1); | |
434 | __COPY_EMA(next, last, C0D1U0); | |
435 | __COPY_EMA(next, last, C0D1U1); | |
436 | __COPY_EMA(next, last, C1D1U0); | |
437 | __COPY_EMA(next, last, C1D1U1); | |
438 | } else { | |
439 | /* Reset the EMA.*/ | |
440 | __MOVAVG(next, C0D0U0) = 0; | |
441 | __MOVAVG(next, C0D0U1) = 0; | |
442 | __MOVAVG(next, C1D0U0) = 0; | |
443 | __MOVAVG(next, C1D0U1) = 0; | |
444 | __MOVAVG(next, C0D1U0) = 0; | |
445 | __MOVAVG(next, C0D1U1) = 0; | |
446 | __MOVAVG(next, C1D1U0) = 0; | |
447 | __MOVAVG(next, C1D1U1) = 0; | |
448 | ||
449 | for (i = 0; i < samples; i++) { | |
450 | tegra210_emc_start_periodic_compensation(emc); | |
451 | udelay(delay); | |
452 | ||
453 | /* | |
454 | * Generate next sample of data. | |
455 | */ | |
456 | adel = update_clock_tree_delay(emc, DVFS_PT1); | |
457 | } | |
458 | } | |
459 | ||
460 | /* | |
461 | * Seems like it should be part of the | |
462 | * 'if (last_timing->periodic_training)' conditional | |
463 | * since is already done for the else clause. | |
464 | */ | |
465 | adel = update_clock_tree_delay(emc, DVFS_UPDATE); | |
466 | } | |
467 | ||
468 | if (type == PERIODIC_TRAINING_SEQUENCE) { | |
469 | tegra210_emc_start_periodic_compensation(emc); | |
470 | udelay(delay); | |
471 | ||
472 | adel = update_clock_tree_delay(emc, PERIODIC_TRAINING_UPDATE); | |
473 | } | |
474 | ||
475 | return adel; | |
476 | } | |
477 | ||
478 | static u32 tegra210_emc_r21021_periodic_compensation(struct tegra210_emc *emc) | |
479 | { | |
480 | u32 emc_cfg, emc_cfg_o, emc_cfg_update, del, value; | |
d71b90e3 | 481 | static const u32 list[] = { |
9b9d8632 JL |
482 | EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0, |
483 | EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1, | |
484 | EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2, | |
485 | EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3, | |
486 | EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0, | |
487 | EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1, | |
488 | EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2, | |
489 | EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3, | |
490 | EMC_DATA_BRLSHFT_0, | |
491 | EMC_DATA_BRLSHFT_1 | |
492 | }; | |
493 | struct tegra210_emc_timing *last = emc->last; | |
494 | unsigned int items = ARRAY_SIZE(list), i; | |
495 | unsigned long delay; | |
496 | ||
497 | if (last->periodic_training) { | |
498 | emc_dbg(emc, PER_TRAIN, "Periodic training starting\n"); | |
499 | ||
500 | value = emc_readl(emc, EMC_DBG); | |
501 | emc_cfg_o = emc_readl(emc, EMC_CFG); | |
502 | emc_cfg = emc_cfg_o & ~(EMC_CFG_DYN_SELF_REF | | |
503 | EMC_CFG_DRAM_ACPD | | |
9b9d8632 JL |
504 | EMC_CFG_DRAM_CLKSTOP_PD); |
505 | ||
506 | ||
507 | /* | |
508 | * 1. Power optimizations should be off. | |
509 | */ | |
510 | emc_writel(emc, emc_cfg, EMC_CFG); | |
511 | ||
512 | /* Does emc_timing_update() for above changes. */ | |
513 | tegra210_emc_dll_disable(emc); | |
514 | ||
515 | for (i = 0; i < emc->num_channels; i++) | |
516 | tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, | |
517 | EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK, | |
518 | 0); | |
519 | ||
520 | for (i = 0; i < emc->num_channels; i++) | |
521 | tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, | |
522 | EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, | |
523 | 0); | |
524 | ||
525 | emc_cfg_update = value = emc_readl(emc, EMC_CFG_UPDATE); | |
526 | value &= ~EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_MASK; | |
527 | value |= (2 << EMC_CFG_UPDATE_UPDATE_DLL_IN_UPDATE_SHIFT); | |
528 | emc_writel(emc, value, EMC_CFG_UPDATE); | |
529 | ||
530 | /* | |
531 | * 2. osc kick off - this assumes training and dvfs have set | |
532 | * correct MR23. | |
533 | */ | |
534 | tegra210_emc_start_periodic_compensation(emc); | |
535 | ||
536 | /* | |
537 | * 3. Let dram capture its clock tree delays. | |
538 | */ | |
539 | delay = tegra210_emc_actual_osc_clocks(last->run_clocks); | |
540 | delay *= 1000; | |
541 | delay /= last->rate + 1; | |
542 | udelay(delay); | |
543 | ||
544 | /* | |
545 | * 4. Check delta wrt previous values (save value if margin | |
546 | * exceeds what is set in table). | |
547 | */ | |
548 | del = periodic_compensation_handler(emc, | |
549 | PERIODIC_TRAINING_SEQUENCE, | |
550 | last, last); | |
551 | ||
552 | /* | |
553 | * 5. Apply compensation w.r.t. trained values (if clock tree | |
554 | * has drifted more than the set margin). | |
555 | */ | |
556 | if (last->tree_margin < ((del * 128 * (last->rate / 1000)) / 1000000)) { | |
557 | for (i = 0; i < items; i++) { | |
558 | value = tegra210_emc_compensate(last, list[i]); | |
559 | emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n", | |
560 | list[i], value); | |
561 | emc_writel(emc, value, list[i]); | |
562 | } | |
563 | } | |
564 | ||
565 | emc_writel(emc, emc_cfg_o, EMC_CFG); | |
566 | ||
567 | /* | |
568 | * 6. Timing update actally applies the new trimmers. | |
569 | */ | |
570 | tegra210_emc_timing_update(emc); | |
571 | ||
572 | /* 6.1. Restore the UPDATE_DLL_IN_UPDATE field. */ | |
573 | emc_writel(emc, emc_cfg_update, EMC_CFG_UPDATE); | |
574 | ||
575 | /* 6.2. Restore the DLL. */ | |
576 | tegra210_emc_dll_enable(emc); | |
577 | } | |
578 | ||
579 | return 0; | |
580 | } | |
581 | ||
582 | /* | |
583 | * Do the clock change sequence. | |
584 | */ | |
585 | static void tegra210_emc_r21021_set_clock(struct tegra210_emc *emc, u32 clksrc) | |
586 | { | |
587 | /* state variables */ | |
588 | static bool fsp_for_next_freq; | |
589 | /* constant configuration parameters */ | |
590 | const bool save_restore_clkstop_pd = true; | |
591 | const u32 zqcal_before_cc_cutoff = 2400; | |
592 | const bool cya_allow_ref_cc = false; | |
593 | const bool cya_issue_pc_ref = false; | |
594 | const bool opt_cc_short_zcal = true; | |
595 | const bool ref_b4_sref_en = false; | |
596 | const u32 tZQCAL_lpddr4 = 1000000; | |
597 | const bool opt_short_zcal = true; | |
598 | const bool opt_do_sw_qrst = true; | |
599 | const u32 opt_dvfs_mode = MAN_SR; | |
600 | /* | |
601 | * This is the timing table for the source frequency. It does _not_ | |
602 | * necessarily correspond to the actual timing values in the EMC at the | |
603 | * moment. If the boot BCT differs from the table then this can happen. | |
604 | * However, we need it for accessing the dram_timings (which are not | |
605 | * really registers) array for the current frequency. | |
606 | */ | |
607 | struct tegra210_emc_timing *fake, *last = emc->last, *next = emc->next; | |
608 | u32 tRTM, RP_war, R2P_war, TRPab_war, deltaTWATM, W2P_war, tRPST; | |
609 | u32 mr13_flip_fspwr, mr13_flip_fspop, ramp_up_wait, ramp_down_wait; | |
610 | u32 zq_wait_long, zq_latch_dvfs_wait_time, tZQCAL_lpddr4_fc_adj; | |
611 | u32 emc_auto_cal_config, auto_cal_en, emc_cfg, emc_sel_dpd_ctrl; | |
612 | u32 tFC_lpddr4 = 1000 * next->dram_timings[T_FC_LPDDR4]; | |
613 | u32 bg_reg_mode_change, enable_bglp_reg, enable_bg_reg; | |
614 | bool opt_zcal_en_cc = false, is_lpddr3 = false; | |
615 | bool compensate_trimmer_applicable = false; | |
616 | u32 emc_dbg, emc_cfg_pipe_clk, emc_pin; | |
617 | u32 src_clk_period, dst_clk_period; /* in picoseconds */ | |
618 | bool shared_zq_resistor = false; | |
619 | u32 value, dram_type; | |
620 | u32 opt_dll_mode = 0; | |
621 | unsigned long delay; | |
622 | unsigned int i; | |
623 | ||
624 | emc_dbg(emc, INFO, "Running clock change.\n"); | |
625 | ||
626 | /* XXX fake == last */ | |
627 | fake = tegra210_emc_find_timing(emc, last->rate * 1000UL); | |
628 | fsp_for_next_freq = !fsp_for_next_freq; | |
629 | ||
630 | value = emc_readl(emc, EMC_FBIO_CFG5) & EMC_FBIO_CFG5_DRAM_TYPE_MASK; | |
631 | dram_type = value >> EMC_FBIO_CFG5_DRAM_TYPE_SHIFT; | |
632 | ||
633 | if (last->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX] & BIT(31)) | |
634 | shared_zq_resistor = true; | |
635 | ||
636 | if ((next->burst_regs[EMC_ZCAL_INTERVAL_INDEX] != 0 && | |
637 | last->burst_regs[EMC_ZCAL_INTERVAL_INDEX] == 0) || | |
638 | dram_type == DRAM_TYPE_LPDDR4) | |
639 | opt_zcal_en_cc = true; | |
640 | ||
641 | if (dram_type == DRAM_TYPE_DDR3) | |
642 | opt_dll_mode = tegra210_emc_get_dll_state(next); | |
643 | ||
644 | if ((next->burst_regs[EMC_FBIO_CFG5_INDEX] & BIT(25)) && | |
645 | (dram_type == DRAM_TYPE_LPDDR2)) | |
646 | is_lpddr3 = true; | |
647 | ||
648 | emc_readl(emc, EMC_CFG); | |
649 | emc_readl(emc, EMC_AUTO_CAL_CONFIG); | |
650 | ||
651 | src_clk_period = 1000000000 / last->rate; | |
652 | dst_clk_period = 1000000000 / next->rate; | |
653 | ||
654 | if (dst_clk_period <= zqcal_before_cc_cutoff) | |
655 | tZQCAL_lpddr4_fc_adj = tZQCAL_lpddr4 - tFC_lpddr4; | |
656 | else | |
657 | tZQCAL_lpddr4_fc_adj = tZQCAL_lpddr4; | |
658 | ||
659 | tZQCAL_lpddr4_fc_adj /= dst_clk_period; | |
660 | ||
661 | emc_dbg = emc_readl(emc, EMC_DBG); | |
662 | emc_pin = emc_readl(emc, EMC_PIN); | |
663 | emc_cfg_pipe_clk = emc_readl(emc, EMC_CFG_PIPE_CLK); | |
664 | ||
665 | emc_cfg = next->burst_regs[EMC_CFG_INDEX]; | |
666 | emc_cfg &= ~(EMC_CFG_DYN_SELF_REF | EMC_CFG_DRAM_ACPD | | |
667 | EMC_CFG_DRAM_CLKSTOP_SR | EMC_CFG_DRAM_CLKSTOP_PD); | |
668 | emc_sel_dpd_ctrl = next->emc_sel_dpd_ctrl; | |
669 | emc_sel_dpd_ctrl &= ~(EMC_SEL_DPD_CTRL_CLK_SEL_DPD_EN | | |
670 | EMC_SEL_DPD_CTRL_CA_SEL_DPD_EN | | |
671 | EMC_SEL_DPD_CTRL_RESET_SEL_DPD_EN | | |
672 | EMC_SEL_DPD_CTRL_ODT_SEL_DPD_EN | | |
673 | EMC_SEL_DPD_CTRL_DATA_SEL_DPD_EN); | |
674 | ||
675 | emc_dbg(emc, INFO, "Clock change version: %d\n", | |
676 | DVFS_CLOCK_CHANGE_VERSION); | |
677 | emc_dbg(emc, INFO, "DRAM type = %d\n", dram_type); | |
678 | emc_dbg(emc, INFO, "DRAM dev #: %u\n", emc->num_devices); | |
679 | emc_dbg(emc, INFO, "Next EMC clksrc: 0x%08x\n", clksrc); | |
680 | emc_dbg(emc, INFO, "DLL clksrc: 0x%08x\n", next->dll_clk_src); | |
681 | emc_dbg(emc, INFO, "last rate: %u, next rate %u\n", last->rate, | |
682 | next->rate); | |
683 | emc_dbg(emc, INFO, "last period: %u, next period: %u\n", | |
684 | src_clk_period, dst_clk_period); | |
685 | emc_dbg(emc, INFO, " shared_zq_resistor: %d\n", !!shared_zq_resistor); | |
686 | emc_dbg(emc, INFO, " num_channels: %u\n", emc->num_channels); | |
687 | emc_dbg(emc, INFO, " opt_dll_mode: %d\n", opt_dll_mode); | |
688 | ||
689 | /* | |
690 | * Step 1: | |
691 | * Pre DVFS SW sequence. | |
692 | */ | |
693 | emc_dbg(emc, STEPS, "Step 1\n"); | |
694 | emc_dbg(emc, STEPS, "Step 1.1: Disable DLL temporarily.\n"); | |
695 | ||
696 | value = emc_readl(emc, EMC_CFG_DIG_DLL); | |
697 | value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN; | |
698 | emc_writel(emc, value, EMC_CFG_DIG_DLL); | |
699 | ||
700 | tegra210_emc_timing_update(emc); | |
701 | ||
702 | for (i = 0; i < emc->num_channels; i++) | |
703 | tegra210_emc_wait_for_update(emc, i, EMC_CFG_DIG_DLL, | |
704 | EMC_CFG_DIG_DLL_CFG_DLL_EN, 0); | |
705 | ||
706 | emc_dbg(emc, STEPS, "Step 1.2: Disable AUTOCAL temporarily.\n"); | |
707 | ||
708 | emc_auto_cal_config = next->emc_auto_cal_config; | |
709 | auto_cal_en = emc_auto_cal_config & EMC_AUTO_CAL_CONFIG_AUTO_CAL_ENABLE; | |
710 | emc_auto_cal_config &= ~EMC_AUTO_CAL_CONFIG_AUTO_CAL_START; | |
711 | emc_auto_cal_config |= EMC_AUTO_CAL_CONFIG_AUTO_CAL_MEASURE_STALL; | |
712 | emc_auto_cal_config |= EMC_AUTO_CAL_CONFIG_AUTO_CAL_UPDATE_STALL; | |
713 | emc_auto_cal_config |= auto_cal_en; | |
714 | emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG); | |
715 | emc_readl(emc, EMC_AUTO_CAL_CONFIG); /* Flush write. */ | |
716 | ||
717 | emc_dbg(emc, STEPS, "Step 1.3: Disable other power features.\n"); | |
718 | ||
719 | tegra210_emc_set_shadow_bypass(emc, ACTIVE); | |
720 | emc_writel(emc, emc_cfg, EMC_CFG); | |
721 | emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL); | |
722 | tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); | |
723 | ||
724 | if (next->periodic_training) { | |
725 | tegra210_emc_reset_dram_clktree_values(next); | |
726 | ||
727 | for (i = 0; i < emc->num_channels; i++) | |
728 | tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, | |
729 | EMC_EMC_STATUS_DRAM_IN_POWERDOWN_MASK, | |
730 | 0); | |
731 | ||
732 | for (i = 0; i < emc->num_channels; i++) | |
733 | tegra210_emc_wait_for_update(emc, i, EMC_EMC_STATUS, | |
734 | EMC_EMC_STATUS_DRAM_IN_SELF_REFRESH_MASK, | |
735 | 0); | |
736 | ||
737 | tegra210_emc_start_periodic_compensation(emc); | |
738 | ||
739 | delay = 1000 * tegra210_emc_actual_osc_clocks(last->run_clocks); | |
740 | udelay((delay / last->rate) + 2); | |
741 | ||
742 | value = periodic_compensation_handler(emc, DVFS_SEQUENCE, fake, | |
743 | next); | |
744 | value = (value * 128 * next->rate / 1000) / 1000000; | |
745 | ||
746 | if (next->periodic_training && value > next->tree_margin) | |
747 | compensate_trimmer_applicable = true; | |
748 | } | |
749 | ||
750 | emc_writel(emc, EMC_INTSTATUS_CLKCHANGE_COMPLETE, EMC_INTSTATUS); | |
751 | tegra210_emc_set_shadow_bypass(emc, ACTIVE); | |
752 | emc_writel(emc, emc_cfg, EMC_CFG); | |
753 | emc_writel(emc, emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL); | |
754 | emc_writel(emc, emc_cfg_pipe_clk | EMC_CFG_PIPE_CLK_CLK_ALWAYS_ON, | |
755 | EMC_CFG_PIPE_CLK); | |
756 | emc_writel(emc, next->emc_fdpd_ctrl_cmd_no_ramp & | |
757 | ~EMC_FDPD_CTRL_CMD_NO_RAMP_CMD_DPD_NO_RAMP_ENABLE, | |
758 | EMC_FDPD_CTRL_CMD_NO_RAMP); | |
759 | ||
760 | bg_reg_mode_change = | |
761 | ((next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & | |
762 | EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD) ^ | |
763 | (last->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & | |
764 | EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD)) || | |
765 | ((next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & | |
766 | EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) ^ | |
767 | (last->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & | |
768 | EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD)); | |
769 | enable_bglp_reg = | |
770 | (next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & | |
771 | EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD) == 0; | |
772 | enable_bg_reg = | |
773 | (next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & | |
774 | EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD) == 0; | |
775 | ||
776 | if (bg_reg_mode_change) { | |
777 | if (enable_bg_reg) | |
778 | emc_writel(emc, last->burst_regs | |
779 | [EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & | |
780 | ~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD, | |
781 | EMC_PMACRO_BG_BIAS_CTRL_0); | |
782 | ||
783 | if (enable_bglp_reg) | |
784 | emc_writel(emc, last->burst_regs | |
785 | [EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & | |
786 | ~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD, | |
787 | EMC_PMACRO_BG_BIAS_CTRL_0); | |
788 | } | |
789 | ||
790 | /* Check if we need to turn on VREF generator. */ | |
791 | if ((((last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & | |
792 | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF) == 0) && | |
793 | ((next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & | |
794 | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF) == 1)) || | |
795 | (((last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & | |
796 | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) == 0) && | |
797 | ((next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX] & | |
798 | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) != 0))) { | |
799 | u32 pad_tx_ctrl = | |
800 | next->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX]; | |
801 | u32 last_pad_tx_ctrl = | |
802 | last->burst_regs[EMC_PMACRO_DATA_PAD_TX_CTRL_INDEX]; | |
803 | u32 next_dq_e_ivref, next_dqs_e_ivref; | |
804 | ||
805 | next_dqs_e_ivref = pad_tx_ctrl & | |
806 | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF; | |
807 | next_dq_e_ivref = pad_tx_ctrl & | |
808 | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF; | |
809 | value = (last_pad_tx_ctrl & | |
810 | ~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_E_IVREF & | |
811 | ~EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQS_E_IVREF) | | |
812 | next_dq_e_ivref | next_dqs_e_ivref; | |
813 | emc_writel(emc, value, EMC_PMACRO_DATA_PAD_TX_CTRL); | |
814 | udelay(1); | |
815 | } else if (bg_reg_mode_change) { | |
816 | udelay(1); | |
817 | } | |
818 | ||
819 | tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); | |
820 | ||
821 | /* | |
822 | * Step 2: | |
823 | * Prelock the DLL. | |
824 | */ | |
825 | emc_dbg(emc, STEPS, "Step 2\n"); | |
826 | ||
827 | if (next->burst_regs[EMC_CFG_DIG_DLL_INDEX] & | |
828 | EMC_CFG_DIG_DLL_CFG_DLL_EN) { | |
829 | emc_dbg(emc, INFO, "Prelock enabled for target frequency.\n"); | |
830 | value = tegra210_emc_dll_prelock(emc, clksrc); | |
831 | emc_dbg(emc, INFO, "DLL out: 0x%03x\n", value); | |
832 | } else { | |
833 | emc_dbg(emc, INFO, "Disabling DLL for target frequency.\n"); | |
834 | tegra210_emc_dll_disable(emc); | |
835 | } | |
836 | ||
837 | /* | |
838 | * Step 3: | |
839 | * Prepare autocal for the clock change. | |
840 | */ | |
841 | emc_dbg(emc, STEPS, "Step 3\n"); | |
842 | ||
843 | tegra210_emc_set_shadow_bypass(emc, ACTIVE); | |
844 | emc_writel(emc, next->emc_auto_cal_config2, EMC_AUTO_CAL_CONFIG2); | |
845 | emc_writel(emc, next->emc_auto_cal_config3, EMC_AUTO_CAL_CONFIG3); | |
846 | emc_writel(emc, next->emc_auto_cal_config4, EMC_AUTO_CAL_CONFIG4); | |
847 | emc_writel(emc, next->emc_auto_cal_config5, EMC_AUTO_CAL_CONFIG5); | |
848 | emc_writel(emc, next->emc_auto_cal_config6, EMC_AUTO_CAL_CONFIG6); | |
849 | emc_writel(emc, next->emc_auto_cal_config7, EMC_AUTO_CAL_CONFIG7); | |
850 | emc_writel(emc, next->emc_auto_cal_config8, EMC_AUTO_CAL_CONFIG8); | |
851 | tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); | |
852 | ||
853 | emc_auto_cal_config |= (EMC_AUTO_CAL_CONFIG_AUTO_CAL_COMPUTE_START | | |
854 | auto_cal_en); | |
855 | emc_writel(emc, emc_auto_cal_config, EMC_AUTO_CAL_CONFIG); | |
856 | ||
857 | /* | |
858 | * Step 4: | |
859 | * Update EMC_CFG. (??) | |
860 | */ | |
861 | emc_dbg(emc, STEPS, "Step 4\n"); | |
862 | ||
863 | if (src_clk_period > 50000 && dram_type == DRAM_TYPE_LPDDR4) | |
864 | ccfifo_writel(emc, 1, EMC_SELF_REF, 0); | |
865 | else | |
866 | emc_writel(emc, next->emc_cfg_2, EMC_CFG_2); | |
867 | ||
868 | /* | |
869 | * Step 5: | |
870 | * Prepare reference variables for ZQCAL regs. | |
871 | */ | |
872 | emc_dbg(emc, STEPS, "Step 5\n"); | |
873 | ||
874 | if (dram_type == DRAM_TYPE_LPDDR4) | |
875 | zq_wait_long = max((u32)1, div_o3(1000000, dst_clk_period)); | |
876 | else if (dram_type == DRAM_TYPE_LPDDR2 || is_lpddr3) | |
877 | zq_wait_long = max(next->min_mrs_wait, | |
878 | div_o3(360000, dst_clk_period)) + 4; | |
879 | else if (dram_type == DRAM_TYPE_DDR3) | |
880 | zq_wait_long = max((u32)256, | |
881 | div_o3(320000, dst_clk_period) + 2); | |
882 | else | |
883 | zq_wait_long = 0; | |
884 | ||
885 | /* | |
886 | * Step 6: | |
887 | * Training code - removed. | |
888 | */ | |
889 | emc_dbg(emc, STEPS, "Step 6\n"); | |
890 | ||
891 | /* | |
892 | * Step 7: | |
893 | * Program FSP reference registers and send MRWs to new FSPWR. | |
894 | */ | |
895 | emc_dbg(emc, STEPS, "Step 7\n"); | |
896 | emc_dbg(emc, SUB_STEPS, "Step 7.1: Bug 200024907 - Patch RP R2P"); | |
897 | ||
898 | /* WAR 200024907 */ | |
899 | if (dram_type == DRAM_TYPE_LPDDR4) { | |
900 | u32 nRTP = 16; | |
901 | ||
902 | if (src_clk_period >= 1000000 / 1866) /* 535.91 ps */ | |
903 | nRTP = 14; | |
904 | ||
905 | if (src_clk_period >= 1000000 / 1600) /* 625.00 ps */ | |
906 | nRTP = 12; | |
907 | ||
908 | if (src_clk_period >= 1000000 / 1333) /* 750.19 ps */ | |
909 | nRTP = 10; | |
910 | ||
911 | if (src_clk_period >= 1000000 / 1066) /* 938.09 ps */ | |
912 | nRTP = 8; | |
913 | ||
914 | deltaTWATM = max_t(u32, div_o3(7500, src_clk_period), 8); | |
915 | ||
916 | /* | |
917 | * Originally there was a + .5 in the tRPST calculation. | |
918 | * However since we can't do FP in the kernel and the tRTM | |
919 | * computation was in a floating point ceiling function, adding | |
920 | * one to tRTP should be ok. There is no other source of non | |
921 | * integer values, so the result was always going to be | |
922 | * something for the form: f_ceil(N + .5) = N + 1; | |
923 | */ | |
924 | tRPST = (last->emc_mrw & 0x80) >> 7; | |
925 | tRTM = fake->dram_timings[RL] + div_o3(3600, src_clk_period) + | |
926 | max_t(u32, div_o3(7500, src_clk_period), 8) + tRPST + | |
927 | 1 + nRTP; | |
928 | ||
929 | emc_dbg(emc, INFO, "tRTM = %u, EMC_RP = %u\n", tRTM, | |
930 | next->burst_regs[EMC_RP_INDEX]); | |
931 | ||
932 | if (last->burst_regs[EMC_RP_INDEX] < tRTM) { | |
933 | if (tRTM > (last->burst_regs[EMC_R2P_INDEX] + | |
934 | last->burst_regs[EMC_RP_INDEX])) { | |
935 | R2P_war = tRTM - last->burst_regs[EMC_RP_INDEX]; | |
936 | RP_war = last->burst_regs[EMC_RP_INDEX]; | |
937 | TRPab_war = last->burst_regs[EMC_TRPAB_INDEX]; | |
938 | ||
939 | if (R2P_war > 63) { | |
940 | RP_war = R2P_war + | |
941 | last->burst_regs[EMC_RP_INDEX] - 63; | |
942 | ||
943 | if (TRPab_war < RP_war) | |
944 | TRPab_war = RP_war; | |
945 | ||
946 | R2P_war = 63; | |
947 | } | |
948 | } else { | |
949 | R2P_war = last->burst_regs[EMC_R2P_INDEX]; | |
950 | RP_war = last->burst_regs[EMC_RP_INDEX]; | |
951 | TRPab_war = last->burst_regs[EMC_TRPAB_INDEX]; | |
952 | } | |
953 | ||
954 | if (RP_war < deltaTWATM) { | |
955 | W2P_war = last->burst_regs[EMC_W2P_INDEX] | |
956 | + deltaTWATM - RP_war; | |
957 | if (W2P_war > 63) { | |
958 | RP_war = RP_war + W2P_war - 63; | |
959 | if (TRPab_war < RP_war) | |
960 | TRPab_war = RP_war; | |
961 | W2P_war = 63; | |
962 | } | |
963 | } else { | |
964 | W2P_war = last->burst_regs[ | |
965 | EMC_W2P_INDEX]; | |
966 | } | |
967 | ||
968 | if ((last->burst_regs[EMC_W2P_INDEX] ^ W2P_war) || | |
969 | (last->burst_regs[EMC_R2P_INDEX] ^ R2P_war) || | |
970 | (last->burst_regs[EMC_RP_INDEX] ^ RP_war) || | |
971 | (last->burst_regs[EMC_TRPAB_INDEX] ^ TRPab_war)) { | |
972 | emc_writel(emc, RP_war, EMC_RP); | |
973 | emc_writel(emc, R2P_war, EMC_R2P); | |
974 | emc_writel(emc, W2P_war, EMC_W2P); | |
975 | emc_writel(emc, TRPab_war, EMC_TRPAB); | |
976 | } | |
977 | ||
978 | tegra210_emc_timing_update(emc); | |
979 | } else { | |
980 | emc_dbg(emc, INFO, "Skipped WAR\n"); | |
981 | } | |
982 | } | |
983 | ||
984 | if (!fsp_for_next_freq) { | |
985 | mr13_flip_fspwr = (next->emc_mrw3 & 0xffffff3f) | 0x80; | |
986 | mr13_flip_fspop = (next->emc_mrw3 & 0xffffff3f) | 0x00; | |
987 | } else { | |
988 | mr13_flip_fspwr = (next->emc_mrw3 & 0xffffff3f) | 0x40; | |
989 | mr13_flip_fspop = (next->emc_mrw3 & 0xffffff3f) | 0xc0; | |
990 | } | |
991 | ||
992 | if (dram_type == DRAM_TYPE_LPDDR4) { | |
993 | emc_writel(emc, mr13_flip_fspwr, EMC_MRW3); | |
994 | emc_writel(emc, next->emc_mrw, EMC_MRW); | |
995 | emc_writel(emc, next->emc_mrw2, EMC_MRW2); | |
996 | } | |
997 | ||
998 | /* | |
999 | * Step 8: | |
1000 | * Program the shadow registers. | |
1001 | */ | |
1002 | emc_dbg(emc, STEPS, "Step 8\n"); | |
1003 | emc_dbg(emc, SUB_STEPS, "Writing burst_regs\n"); | |
1004 | ||
1005 | for (i = 0; i < next->num_burst; i++) { | |
1006 | const u16 *offsets = emc->offsets->burst; | |
1007 | u16 offset; | |
1008 | ||
1009 | if (!offsets[i]) | |
1010 | continue; | |
1011 | ||
1012 | value = next->burst_regs[i]; | |
1013 | offset = offsets[i]; | |
1014 | ||
1015 | if (dram_type != DRAM_TYPE_LPDDR4 && | |
1016 | (offset == EMC_MRW6 || offset == EMC_MRW7 || | |
1017 | offset == EMC_MRW8 || offset == EMC_MRW9 || | |
1018 | offset == EMC_MRW10 || offset == EMC_MRW11 || | |
1019 | offset == EMC_MRW12 || offset == EMC_MRW13 || | |
1020 | offset == EMC_MRW14 || offset == EMC_MRW15 || | |
1021 | offset == EMC_TRAINING_CTRL)) | |
1022 | continue; | |
1023 | ||
1024 | /* Pain... And suffering. */ | |
1025 | if (offset == EMC_CFG) { | |
1026 | value &= ~EMC_CFG_DRAM_ACPD; | |
1027 | value &= ~EMC_CFG_DYN_SELF_REF; | |
1028 | ||
1029 | if (dram_type == DRAM_TYPE_LPDDR4) { | |
1030 | value &= ~EMC_CFG_DRAM_CLKSTOP_SR; | |
1031 | value &= ~EMC_CFG_DRAM_CLKSTOP_PD; | |
1032 | } | |
1033 | } else if (offset == EMC_MRS_WAIT_CNT && | |
1034 | dram_type == DRAM_TYPE_LPDDR2 && | |
1035 | opt_zcal_en_cc && !opt_cc_short_zcal && | |
1036 | opt_short_zcal) { | |
1037 | value = (value & ~(EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK << | |
1038 | EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT)) | | |
1039 | ((zq_wait_long & EMC_MRS_WAIT_CNT_SHORT_WAIT_MASK) << | |
1040 | EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT); | |
1041 | } else if (offset == EMC_ZCAL_WAIT_CNT && | |
1042 | dram_type == DRAM_TYPE_DDR3 && opt_zcal_en_cc && | |
1043 | !opt_cc_short_zcal && opt_short_zcal) { | |
1044 | value = (value & ~(EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK << | |
1045 | EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_SHIFT)) | | |
4a661364 | 1046 | ((zq_wait_long & EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK) << |
9b9d8632 JL |
1047 | EMC_MRS_WAIT_CNT_SHORT_WAIT_SHIFT); |
1048 | } else if (offset == EMC_ZCAL_INTERVAL && opt_zcal_en_cc) { | |
1049 | value = 0; /* EMC_ZCAL_INTERVAL reset value. */ | |
1050 | } else if (offset == EMC_PMACRO_AUTOCAL_CFG_COMMON) { | |
1051 | value |= EMC_PMACRO_AUTOCAL_CFG_COMMON_E_CAL_BYPASS_DVFS; | |
1052 | } else if (offset == EMC_PMACRO_DATA_PAD_TX_CTRL) { | |
1053 | value &= ~(EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSP_TX_E_DCC | | |
1054 | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQSN_TX_E_DCC | | |
1055 | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_DQ_TX_E_DCC | | |
1056 | EMC_PMACRO_DATA_PAD_TX_CTRL_DATA_CMD_TX_E_DCC); | |
1057 | } else if (offset == EMC_PMACRO_CMD_PAD_TX_CTRL) { | |
1058 | value |= EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_DRVFORCEON; | |
1059 | value &= ~(EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSP_TX_E_DCC | | |
1060 | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQSN_TX_E_DCC | | |
1061 | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_DQ_TX_E_DCC | | |
1062 | EMC_PMACRO_CMD_PAD_TX_CTRL_CMD_CMD_TX_E_DCC); | |
1063 | } else if (offset == EMC_PMACRO_BRICK_CTRL_RFU1) { | |
1064 | value &= 0xf800f800; | |
1065 | } else if (offset == EMC_PMACRO_COMMON_PAD_TX_CTRL) { | |
1066 | value &= 0xfffffff0; | |
1067 | } | |
1068 | ||
1069 | emc_writel(emc, value, offset); | |
1070 | } | |
1071 | ||
0553d7b2 TR |
1072 | /* SW addition: do EMC refresh adjustment here. */ |
1073 | tegra210_emc_adjust_timing(emc, next); | |
1074 | ||
9b9d8632 JL |
1075 | if (dram_type == DRAM_TYPE_LPDDR4) { |
1076 | value = (23 << EMC_MRW_MRW_MA_SHIFT) | | |
1077 | (next->run_clocks & EMC_MRW_MRW_OP_MASK); | |
1078 | emc_writel(emc, value, EMC_MRW); | |
1079 | } | |
1080 | ||
1081 | /* Per channel burst registers. */ | |
1082 | emc_dbg(emc, SUB_STEPS, "Writing burst_regs_per_ch\n"); | |
1083 | ||
1084 | for (i = 0; i < next->num_burst_per_ch; i++) { | |
1085 | const struct tegra210_emc_per_channel_regs *burst = | |
1086 | emc->offsets->burst_per_channel; | |
1087 | ||
1088 | if (!burst[i].offset) | |
1089 | continue; | |
1090 | ||
1091 | if (dram_type != DRAM_TYPE_LPDDR4 && | |
1092 | (burst[i].offset == EMC_MRW6 || | |
1093 | burst[i].offset == EMC_MRW7 || | |
1094 | burst[i].offset == EMC_MRW8 || | |
1095 | burst[i].offset == EMC_MRW9 || | |
1096 | burst[i].offset == EMC_MRW10 || | |
1097 | burst[i].offset == EMC_MRW11 || | |
1098 | burst[i].offset == EMC_MRW12 || | |
1099 | burst[i].offset == EMC_MRW13 || | |
1100 | burst[i].offset == EMC_MRW14 || | |
1101 | burst[i].offset == EMC_MRW15)) | |
1102 | continue; | |
1103 | ||
1104 | /* Filter out second channel if not in DUAL_CHANNEL mode. */ | |
1105 | if (emc->num_channels < 2 && burst[i].bank >= 1) | |
1106 | continue; | |
1107 | ||
1108 | emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, | |
1109 | next->burst_reg_per_ch[i], burst[i].offset); | |
1110 | emc_channel_writel(emc, burst[i].bank, | |
1111 | next->burst_reg_per_ch[i], | |
1112 | burst[i].offset); | |
1113 | } | |
1114 | ||
1115 | /* Vref regs. */ | |
1116 | emc_dbg(emc, SUB_STEPS, "Writing vref_regs\n"); | |
1117 | ||
1118 | for (i = 0; i < next->vref_num; i++) { | |
1119 | const struct tegra210_emc_per_channel_regs *vref = | |
1120 | emc->offsets->vref_per_channel; | |
1121 | ||
1122 | if (!vref[i].offset) | |
1123 | continue; | |
1124 | ||
1125 | if (emc->num_channels < 2 && vref[i].bank >= 1) | |
1126 | continue; | |
1127 | ||
1128 | emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, | |
1129 | next->vref_perch_regs[i], vref[i].offset); | |
1130 | emc_channel_writel(emc, vref[i].bank, next->vref_perch_regs[i], | |
1131 | vref[i].offset); | |
1132 | } | |
1133 | ||
1134 | /* Trimmers. */ | |
1135 | emc_dbg(emc, SUB_STEPS, "Writing trim_regs\n"); | |
1136 | ||
1137 | for (i = 0; i < next->num_trim; i++) { | |
1138 | const u16 *offsets = emc->offsets->trim; | |
1139 | ||
1140 | if (!offsets[i]) | |
1141 | continue; | |
1142 | ||
1143 | if (compensate_trimmer_applicable && | |
1144 | (offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 || | |
1145 | offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 || | |
1146 | offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 || | |
1147 | offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 || | |
1148 | offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 || | |
1149 | offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 || | |
1150 | offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 || | |
1151 | offsets[i] == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 || | |
1152 | offsets[i] == EMC_DATA_BRLSHFT_0 || | |
1153 | offsets[i] == EMC_DATA_BRLSHFT_1)) { | |
1154 | value = tegra210_emc_compensate(next, offsets[i]); | |
1155 | emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, | |
1156 | value, offsets[i]); | |
1157 | emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n", | |
1158 | (u32)(u64)offsets[i], value); | |
1159 | emc_writel(emc, value, offsets[i]); | |
1160 | } else { | |
1161 | emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, | |
1162 | next->trim_regs[i], offsets[i]); | |
1163 | emc_writel(emc, next->trim_regs[i], offsets[i]); | |
1164 | } | |
1165 | } | |
1166 | ||
1167 | /* Per channel trimmers. */ | |
1168 | emc_dbg(emc, SUB_STEPS, "Writing trim_regs_per_ch\n"); | |
1169 | ||
1170 | for (i = 0; i < next->num_trim_per_ch; i++) { | |
1171 | const struct tegra210_emc_per_channel_regs *trim = | |
1172 | &emc->offsets->trim_per_channel[0]; | |
1173 | unsigned int offset; | |
1174 | ||
1175 | if (!trim[i].offset) | |
1176 | continue; | |
1177 | ||
1178 | if (emc->num_channels < 2 && trim[i].bank >= 1) | |
1179 | continue; | |
1180 | ||
1181 | offset = trim[i].offset; | |
1182 | ||
1183 | if (compensate_trimmer_applicable && | |
1184 | (offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_0 || | |
1185 | offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_1 || | |
1186 | offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_2 || | |
1187 | offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK0_3 || | |
1188 | offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_0 || | |
1189 | offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_1 || | |
1190 | offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_2 || | |
1191 | offset == EMC_PMACRO_OB_DDLL_LONG_DQ_RANK1_3 || | |
1192 | offset == EMC_DATA_BRLSHFT_0 || | |
1193 | offset == EMC_DATA_BRLSHFT_1)) { | |
1194 | value = tegra210_emc_compensate(next, offset); | |
1195 | emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, | |
1196 | value, offset); | |
1197 | emc_dbg(emc, EMA_WRITES, "0x%08x <= 0x%08x\n", offset, | |
1198 | value); | |
1199 | emc_channel_writel(emc, trim[i].bank, value, offset); | |
1200 | } else { | |
1201 | emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, | |
1202 | next->trim_perch_regs[i], offset); | |
1203 | emc_channel_writel(emc, trim[i].bank, | |
1204 | next->trim_perch_regs[i], offset); | |
1205 | } | |
1206 | } | |
1207 | ||
1208 | emc_dbg(emc, SUB_STEPS, "Writing burst_mc_regs\n"); | |
1209 | ||
1210 | for (i = 0; i < next->num_mc_regs; i++) { | |
1211 | const u16 *offsets = emc->offsets->burst_mc; | |
1212 | u32 *values = next->burst_mc_regs; | |
1213 | ||
1214 | emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, | |
1215 | values[i], offsets[i]); | |
1216 | mc_writel(emc->mc, values[i], offsets[i]); | |
1217 | } | |
1218 | ||
1219 | /* Registers to be programmed on the faster clock. */ | |
1220 | if (next->rate < last->rate) { | |
1221 | const u16 *la = emc->offsets->la_scale; | |
1222 | ||
1223 | emc_dbg(emc, SUB_STEPS, "Writing la_scale_regs\n"); | |
1224 | ||
1225 | for (i = 0; i < next->num_up_down; i++) { | |
1226 | emc_dbg(emc, REG_LISTS, "(%u) 0x%08x => 0x%08x\n", i, | |
1227 | next->la_scale_regs[i], la[i]); | |
1228 | mc_writel(emc->mc, next->la_scale_regs[i], la[i]); | |
1229 | } | |
1230 | } | |
1231 | ||
1232 | /* Flush all the burst register writes. */ | |
1233 | mc_readl(emc->mc, MC_EMEM_ADR_CFG); | |
1234 | ||
1235 | /* | |
1236 | * Step 9: | |
1237 | * LPDDR4 section A. | |
1238 | */ | |
1239 | emc_dbg(emc, STEPS, "Step 9\n"); | |
1240 | ||
1241 | value = next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX]; | |
1242 | value &= ~EMC_ZCAL_WAIT_CNT_ZCAL_WAIT_CNT_MASK; | |
1243 | ||
1244 | if (dram_type == DRAM_TYPE_LPDDR4) { | |
1245 | emc_writel(emc, 0, EMC_ZCAL_INTERVAL); | |
1246 | emc_writel(emc, value, EMC_ZCAL_WAIT_CNT); | |
1247 | ||
1248 | value = emc_dbg | (EMC_DBG_WRITE_MUX_ACTIVE | | |
1249 | EMC_DBG_WRITE_ACTIVE_ONLY); | |
1250 | ||
1251 | emc_writel(emc, value, EMC_DBG); | |
1252 | emc_writel(emc, 0, EMC_ZCAL_INTERVAL); | |
1253 | emc_writel(emc, emc_dbg, EMC_DBG); | |
1254 | } | |
1255 | ||
1256 | /* | |
1257 | * Step 10: | |
1258 | * LPDDR4 and DDR3 common section. | |
1259 | */ | |
1260 | emc_dbg(emc, STEPS, "Step 10\n"); | |
1261 | ||
1262 | if (opt_dvfs_mode == MAN_SR || dram_type == DRAM_TYPE_LPDDR4) { | |
1263 | if (dram_type == DRAM_TYPE_LPDDR4) | |
1264 | ccfifo_writel(emc, 0x101, EMC_SELF_REF, 0); | |
1265 | else | |
1266 | ccfifo_writel(emc, 0x1, EMC_SELF_REF, 0); | |
1267 | ||
1268 | if (dram_type == DRAM_TYPE_LPDDR4 && | |
1269 | dst_clk_period <= zqcal_before_cc_cutoff) { | |
1270 | ccfifo_writel(emc, mr13_flip_fspwr ^ 0x40, EMC_MRW3, 0); | |
1271 | ccfifo_writel(emc, (next->burst_regs[EMC_MRW6_INDEX] & | |
1272 | 0xFFFF3F3F) | | |
1273 | (last->burst_regs[EMC_MRW6_INDEX] & | |
1274 | 0x0000C0C0), EMC_MRW6, 0); | |
1275 | ccfifo_writel(emc, (next->burst_regs[EMC_MRW14_INDEX] & | |
1276 | 0xFFFF0707) | | |
1277 | (last->burst_regs[EMC_MRW14_INDEX] & | |
1278 | 0x00003838), EMC_MRW14, 0); | |
1279 | ||
1280 | if (emc->num_devices > 1) { | |
1281 | ccfifo_writel(emc, | |
1282 | (next->burst_regs[EMC_MRW7_INDEX] & | |
1283 | 0xFFFF3F3F) | | |
1284 | (last->burst_regs[EMC_MRW7_INDEX] & | |
1285 | 0x0000C0C0), EMC_MRW7, 0); | |
1286 | ccfifo_writel(emc, | |
1287 | (next->burst_regs[EMC_MRW15_INDEX] & | |
1288 | 0xFFFF0707) | | |
1289 | (last->burst_regs[EMC_MRW15_INDEX] & | |
1290 | 0x00003838), EMC_MRW15, 0); | |
1291 | } | |
1292 | ||
1293 | if (opt_zcal_en_cc) { | |
1294 | if (emc->num_devices < 2) | |
1295 | ccfifo_writel(emc, | |
1296 | 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | |
1297 | | EMC_ZQ_CAL_ZQ_CAL_CMD, | |
1298 | EMC_ZQ_CAL, 0); | |
1299 | else if (shared_zq_resistor) | |
1300 | ccfifo_writel(emc, | |
1301 | 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | |
1302 | | EMC_ZQ_CAL_ZQ_CAL_CMD, | |
1303 | EMC_ZQ_CAL, 0); | |
1304 | else | |
1305 | ccfifo_writel(emc, | |
1306 | EMC_ZQ_CAL_ZQ_CAL_CMD, | |
1307 | EMC_ZQ_CAL, 0); | |
1308 | } | |
1309 | } | |
1310 | } | |
1311 | ||
1312 | if (dram_type == DRAM_TYPE_LPDDR4) { | |
1313 | value = (1000 * fake->dram_timings[T_RP]) / src_clk_period; | |
1314 | ccfifo_writel(emc, mr13_flip_fspop | 0x8, EMC_MRW3, value); | |
1315 | ccfifo_writel(emc, 0, 0, tFC_lpddr4 / src_clk_period); | |
1316 | } | |
1317 | ||
1318 | if (dram_type == DRAM_TYPE_LPDDR4 || opt_dvfs_mode != MAN_SR) { | |
1319 | delay = 30; | |
1320 | ||
1321 | if (cya_allow_ref_cc) { | |
1322 | delay += (1000 * fake->dram_timings[T_RP]) / | |
1323 | src_clk_period; | |
1324 | delay += 4000 * fake->dram_timings[T_RFC]; | |
1325 | } | |
1326 | ||
1327 | ccfifo_writel(emc, emc_pin & ~(EMC_PIN_PIN_CKE_PER_DEV | | |
1328 | EMC_PIN_PIN_CKEB | | |
1329 | EMC_PIN_PIN_CKE), | |
1330 | EMC_PIN, delay); | |
1331 | } | |
1332 | ||
1333 | /* calculate reference delay multiplier */ | |
1334 | value = 1; | |
1335 | ||
1336 | if (ref_b4_sref_en) | |
1337 | value++; | |
1338 | ||
1339 | if (cya_allow_ref_cc) | |
1340 | value++; | |
1341 | ||
1342 | if (cya_issue_pc_ref) | |
1343 | value++; | |
1344 | ||
1345 | if (dram_type != DRAM_TYPE_LPDDR4) { | |
1346 | delay = ((1000 * fake->dram_timings[T_RP] / src_clk_period) + | |
1347 | (1000 * fake->dram_timings[T_RFC] / src_clk_period)); | |
1348 | delay = value * delay + 20; | |
1349 | } else { | |
1350 | delay = 0; | |
1351 | } | |
1352 | ||
1353 | /* | |
1354 | * Step 11: | |
1355 | * Ramp down. | |
1356 | */ | |
1357 | emc_dbg(emc, STEPS, "Step 11\n"); | |
1358 | ||
1359 | ccfifo_writel(emc, 0x0, EMC_CFG_SYNC, delay); | |
1360 | ||
1361 | value = emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE | EMC_DBG_WRITE_ACTIVE_ONLY; | |
1362 | ccfifo_writel(emc, value, EMC_DBG, 0); | |
1363 | ||
1364 | ramp_down_wait = tegra210_emc_dvfs_power_ramp_down(emc, src_clk_period, | |
1365 | 0); | |
1366 | ||
1367 | /* | |
1368 | * Step 12: | |
1369 | * And finally - trigger the clock change. | |
1370 | */ | |
1371 | emc_dbg(emc, STEPS, "Step 12\n"); | |
1372 | ||
1373 | ccfifo_writel(emc, 1, EMC_STALL_THEN_EXE_AFTER_CLKCHANGE, 0); | |
1374 | value &= ~EMC_DBG_WRITE_ACTIVE_ONLY; | |
1375 | ccfifo_writel(emc, value, EMC_DBG, 0); | |
1376 | ||
1377 | /* | |
1378 | * Step 13: | |
1379 | * Ramp up. | |
1380 | */ | |
1381 | emc_dbg(emc, STEPS, "Step 13\n"); | |
1382 | ||
1383 | ramp_up_wait = tegra210_emc_dvfs_power_ramp_up(emc, dst_clk_period, 0); | |
1384 | ccfifo_writel(emc, emc_dbg, EMC_DBG, 0); | |
1385 | ||
1386 | /* | |
1387 | * Step 14: | |
1388 | * Bringup CKE pins. | |
1389 | */ | |
1390 | emc_dbg(emc, STEPS, "Step 14\n"); | |
1391 | ||
1392 | if (dram_type == DRAM_TYPE_LPDDR4) { | |
1393 | value = emc_pin | EMC_PIN_PIN_CKE; | |
1394 | ||
1395 | if (emc->num_devices <= 1) | |
1396 | value &= ~(EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE_PER_DEV); | |
1397 | else | |
1398 | value |= EMC_PIN_PIN_CKEB | EMC_PIN_PIN_CKE_PER_DEV; | |
1399 | ||
1400 | ccfifo_writel(emc, value, EMC_PIN, 0); | |
1401 | } | |
1402 | ||
1403 | /* | |
1404 | * Step 15: (two step 15s ??) | |
1405 | * Calculate zqlatch wait time; has dependency on ramping times. | |
1406 | */ | |
1407 | emc_dbg(emc, STEPS, "Step 15\n"); | |
1408 | ||
1409 | if (dst_clk_period <= zqcal_before_cc_cutoff) { | |
1410 | s32 t = (s32)(ramp_up_wait + ramp_down_wait) / | |
1411 | (s32)dst_clk_period; | |
1412 | zq_latch_dvfs_wait_time = (s32)tZQCAL_lpddr4_fc_adj - t; | |
1413 | } else { | |
1414 | zq_latch_dvfs_wait_time = tZQCAL_lpddr4_fc_adj - | |
1415 | div_o3(1000 * next->dram_timings[T_PDEX], | |
1416 | dst_clk_period); | |
1417 | } | |
1418 | ||
1419 | emc_dbg(emc, INFO, "tZQCAL_lpddr4_fc_adj = %u\n", tZQCAL_lpddr4_fc_adj); | |
1420 | emc_dbg(emc, INFO, "dst_clk_period = %u\n", | |
1421 | dst_clk_period); | |
1422 | emc_dbg(emc, INFO, "next->dram_timings[T_PDEX] = %u\n", | |
1423 | next->dram_timings[T_PDEX]); | |
1424 | emc_dbg(emc, INFO, "zq_latch_dvfs_wait_time = %d\n", | |
1425 | max_t(s32, 0, zq_latch_dvfs_wait_time)); | |
1426 | ||
1427 | if (dram_type == DRAM_TYPE_LPDDR4 && opt_zcal_en_cc) { | |
1428 | delay = div_o3(1000 * next->dram_timings[T_PDEX], | |
1429 | dst_clk_period); | |
1430 | ||
1431 | if (emc->num_devices < 2) { | |
1432 | if (dst_clk_period > zqcal_before_cc_cutoff) | |
1433 | ccfifo_writel(emc, | |
1434 | 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | | |
1435 | EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL, | |
1436 | delay); | |
1437 | ||
1438 | value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000; | |
1439 | ccfifo_writel(emc, value, EMC_MRW3, delay); | |
1440 | ccfifo_writel(emc, 0, EMC_SELF_REF, 0); | |
1441 | ccfifo_writel(emc, 0, EMC_REF, 0); | |
1442 | ccfifo_writel(emc, 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | | |
1443 | EMC_ZQ_CAL_ZQ_LATCH_CMD, | |
1444 | EMC_ZQ_CAL, | |
1445 | max_t(s32, 0, zq_latch_dvfs_wait_time)); | |
1446 | } else if (shared_zq_resistor) { | |
1447 | if (dst_clk_period > zqcal_before_cc_cutoff) | |
1448 | ccfifo_writel(emc, | |
1449 | 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | | |
1450 | EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL, | |
1451 | delay); | |
1452 | ||
1453 | ccfifo_writel(emc, 2UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | | |
1454 | EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL, | |
1455 | max_t(s32, 0, zq_latch_dvfs_wait_time) + | |
1456 | delay); | |
1457 | ccfifo_writel(emc, 1UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | | |
1458 | EMC_ZQ_CAL_ZQ_LATCH_CMD, | |
1459 | EMC_ZQ_CAL, 0); | |
1460 | ||
1461 | value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000; | |
1462 | ccfifo_writel(emc, value, EMC_MRW3, 0); | |
1463 | ccfifo_writel(emc, 0, EMC_SELF_REF, 0); | |
1464 | ccfifo_writel(emc, 0, EMC_REF, 0); | |
1465 | ||
1466 | ccfifo_writel(emc, 1UL << EMC_ZQ_CAL_DEV_SEL_SHIFT | | |
1467 | EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL, | |
1468 | tZQCAL_lpddr4 / dst_clk_period); | |
1469 | } else { | |
1470 | if (dst_clk_period > zqcal_before_cc_cutoff) | |
1471 | ccfifo_writel(emc, EMC_ZQ_CAL_ZQ_CAL_CMD, | |
1472 | EMC_ZQ_CAL, delay); | |
1473 | ||
1474 | value = (mr13_flip_fspop & 0xfffffff7) | 0x0c000000; | |
1475 | ccfifo_writel(emc, value, EMC_MRW3, delay); | |
1476 | ccfifo_writel(emc, 0, EMC_SELF_REF, 0); | |
1477 | ccfifo_writel(emc, 0, EMC_REF, 0); | |
1478 | ||
1479 | ccfifo_writel(emc, EMC_ZQ_CAL_ZQ_LATCH_CMD, EMC_ZQ_CAL, | |
1480 | max_t(s32, 0, zq_latch_dvfs_wait_time)); | |
1481 | } | |
1482 | } | |
1483 | ||
1484 | /* WAR: delay for zqlatch */ | |
1485 | ccfifo_writel(emc, 0, 0, 10); | |
1486 | ||
1487 | /* | |
1488 | * Step 16: | |
1489 | * LPDDR4 Conditional Training Kickoff. Removed. | |
1490 | */ | |
1491 | ||
1492 | /* | |
1493 | * Step 17: | |
1494 | * MANSR exit self refresh. | |
1495 | */ | |
1496 | emc_dbg(emc, STEPS, "Step 17\n"); | |
1497 | ||
1498 | if (opt_dvfs_mode == MAN_SR && dram_type != DRAM_TYPE_LPDDR4) | |
1499 | ccfifo_writel(emc, 0, EMC_SELF_REF, 0); | |
1500 | ||
1501 | /* | |
1502 | * Step 18: | |
1503 | * Send MRWs to LPDDR3/DDR3. | |
1504 | */ | |
1505 | emc_dbg(emc, STEPS, "Step 18\n"); | |
1506 | ||
1507 | if (dram_type == DRAM_TYPE_LPDDR2) { | |
1508 | ccfifo_writel(emc, next->emc_mrw2, EMC_MRW2, 0); | |
1509 | ccfifo_writel(emc, next->emc_mrw, EMC_MRW, 0); | |
1510 | if (is_lpddr3) | |
1511 | ccfifo_writel(emc, next->emc_mrw4, EMC_MRW4, 0); | |
1512 | } else if (dram_type == DRAM_TYPE_DDR3) { | |
1513 | if (opt_dll_mode) | |
1514 | ccfifo_writel(emc, next->emc_emrs & | |
1515 | ~EMC_EMRS_USE_EMRS_LONG_CNT, EMC_EMRS, 0); | |
1516 | ccfifo_writel(emc, next->emc_emrs2 & | |
1517 | ~EMC_EMRS2_USE_EMRS2_LONG_CNT, EMC_EMRS2, 0); | |
1518 | ccfifo_writel(emc, next->emc_mrs | | |
1519 | EMC_EMRS_USE_EMRS_LONG_CNT, EMC_MRS, 0); | |
1520 | } | |
1521 | ||
1522 | /* | |
1523 | * Step 19: | |
1524 | * ZQCAL for LPDDR3/DDR3 | |
1525 | */ | |
1526 | emc_dbg(emc, STEPS, "Step 19\n"); | |
1527 | ||
1528 | if (opt_zcal_en_cc) { | |
1529 | if (dram_type == DRAM_TYPE_LPDDR2) { | |
1530 | value = opt_cc_short_zcal ? 90000 : 360000; | |
1531 | value = div_o3(value, dst_clk_period); | |
1532 | value = value << | |
1533 | EMC_MRS_WAIT_CNT2_MRS_EXT2_WAIT_CNT_SHIFT | | |
1534 | value << | |
1535 | EMC_MRS_WAIT_CNT2_MRS_EXT1_WAIT_CNT_SHIFT; | |
1536 | ccfifo_writel(emc, value, EMC_MRS_WAIT_CNT2, 0); | |
1537 | ||
1538 | value = opt_cc_short_zcal ? 0x56 : 0xab; | |
1539 | ccfifo_writel(emc, 2 << EMC_MRW_MRW_DEV_SELECTN_SHIFT | | |
1540 | EMC_MRW_USE_MRW_EXT_CNT | | |
1541 | 10 << EMC_MRW_MRW_MA_SHIFT | | |
1542 | value << EMC_MRW_MRW_OP_SHIFT, | |
1543 | EMC_MRW, 0); | |
1544 | ||
1545 | if (emc->num_devices > 1) { | |
1546 | value = 1 << EMC_MRW_MRW_DEV_SELECTN_SHIFT | | |
1547 | EMC_MRW_USE_MRW_EXT_CNT | | |
1548 | 10 << EMC_MRW_MRW_MA_SHIFT | | |
1549 | value << EMC_MRW_MRW_OP_SHIFT; | |
1550 | ccfifo_writel(emc, value, EMC_MRW, 0); | |
1551 | } | |
1552 | } else if (dram_type == DRAM_TYPE_DDR3) { | |
1553 | value = opt_cc_short_zcal ? 0 : EMC_ZQ_CAL_LONG; | |
1554 | ||
1555 | ccfifo_writel(emc, value | | |
1556 | 2 << EMC_ZQ_CAL_DEV_SEL_SHIFT | | |
1557 | EMC_ZQ_CAL_ZQ_CAL_CMD, EMC_ZQ_CAL, | |
1558 | 0); | |
1559 | ||
1560 | if (emc->num_devices > 1) { | |
1561 | value = value | 1 << EMC_ZQ_CAL_DEV_SEL_SHIFT | | |
1562 | EMC_ZQ_CAL_ZQ_CAL_CMD; | |
1563 | ccfifo_writel(emc, value, EMC_ZQ_CAL, 0); | |
1564 | } | |
1565 | } | |
1566 | } | |
1567 | ||
1568 | if (bg_reg_mode_change) { | |
1569 | tegra210_emc_set_shadow_bypass(emc, ACTIVE); | |
1570 | ||
1571 | if (ramp_up_wait <= 1250000) | |
1572 | delay = (1250000 - ramp_up_wait) / dst_clk_period; | |
1573 | else | |
1574 | delay = 0; | |
1575 | ||
1576 | ccfifo_writel(emc, | |
1577 | next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX], | |
1578 | EMC_PMACRO_BG_BIAS_CTRL_0, delay); | |
1579 | tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); | |
1580 | } | |
1581 | ||
1582 | /* | |
1583 | * Step 20: | |
1584 | * Issue ref and optional QRST. | |
1585 | */ | |
1586 | emc_dbg(emc, STEPS, "Step 20\n"); | |
1587 | ||
1588 | if (dram_type != DRAM_TYPE_LPDDR4) | |
1589 | ccfifo_writel(emc, 0, EMC_REF, 0); | |
1590 | ||
1591 | if (opt_do_sw_qrst) { | |
1592 | ccfifo_writel(emc, 1, EMC_ISSUE_QRST, 0); | |
1593 | ccfifo_writel(emc, 0, EMC_ISSUE_QRST, 2); | |
1594 | } | |
1595 | ||
1596 | /* | |
1597 | * Step 21: | |
1598 | * Restore ZCAL and ZCAL interval. | |
1599 | */ | |
1600 | emc_dbg(emc, STEPS, "Step 21\n"); | |
1601 | ||
1602 | if (save_restore_clkstop_pd || opt_zcal_en_cc) { | |
1603 | ccfifo_writel(emc, emc_dbg | EMC_DBG_WRITE_MUX_ACTIVE, | |
1604 | EMC_DBG, 0); | |
1605 | if (opt_zcal_en_cc && dram_type != DRAM_TYPE_LPDDR4) | |
1606 | ccfifo_writel(emc, next->burst_regs[EMC_ZCAL_INTERVAL_INDEX], | |
1607 | EMC_ZCAL_INTERVAL, 0); | |
1608 | ||
1609 | if (save_restore_clkstop_pd) | |
1610 | ccfifo_writel(emc, next->burst_regs[EMC_CFG_INDEX] & | |
1611 | ~EMC_CFG_DYN_SELF_REF, | |
1612 | EMC_CFG, 0); | |
1613 | ccfifo_writel(emc, emc_dbg, EMC_DBG, 0); | |
1614 | } | |
1615 | ||
1616 | /* | |
1617 | * Step 22: | |
1618 | * Restore EMC_CFG_PIPE_CLK. | |
1619 | */ | |
1620 | emc_dbg(emc, STEPS, "Step 22\n"); | |
1621 | ||
1622 | ccfifo_writel(emc, emc_cfg_pipe_clk, EMC_CFG_PIPE_CLK, 0); | |
1623 | ||
1624 | if (bg_reg_mode_change) { | |
1625 | if (enable_bg_reg) | |
1626 | emc_writel(emc, | |
1627 | next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & | |
1628 | ~EMC_PMACRO_BG_BIAS_CTRL_0_BGLP_E_PWRD, | |
1629 | EMC_PMACRO_BG_BIAS_CTRL_0); | |
1630 | else | |
1631 | emc_writel(emc, | |
1632 | next->burst_regs[EMC_PMACRO_BG_BIAS_CTRL_0_INDEX] & | |
1633 | ~EMC_PMACRO_BG_BIAS_CTRL_0_BG_E_PWRD, | |
1634 | EMC_PMACRO_BG_BIAS_CTRL_0); | |
1635 | } | |
1636 | ||
1637 | /* | |
1638 | * Step 23: | |
1639 | */ | |
1640 | emc_dbg(emc, STEPS, "Step 23\n"); | |
1641 | ||
1642 | value = emc_readl(emc, EMC_CFG_DIG_DLL); | |
1643 | value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC; | |
1644 | value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK; | |
1645 | value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK; | |
1646 | value &= ~EMC_CFG_DIG_DLL_CFG_DLL_EN; | |
1647 | value = (value & ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK) | | |
1648 | (2 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT); | |
1649 | emc_writel(emc, value, EMC_CFG_DIG_DLL); | |
1650 | ||
1651 | tegra210_emc_do_clock_change(emc, clksrc); | |
1652 | ||
1653 | /* | |
1654 | * Step 24: | |
1655 | * Save training results. Removed. | |
1656 | */ | |
1657 | ||
1658 | /* | |
1659 | * Step 25: | |
1660 | * Program MC updown registers. | |
1661 | */ | |
1662 | emc_dbg(emc, STEPS, "Step 25\n"); | |
1663 | ||
1664 | if (next->rate > last->rate) { | |
1665 | for (i = 0; i < next->num_up_down; i++) | |
1666 | mc_writel(emc->mc, next->la_scale_regs[i], | |
1667 | emc->offsets->la_scale[i]); | |
1668 | ||
1669 | tegra210_emc_timing_update(emc); | |
1670 | } | |
1671 | ||
1672 | /* | |
1673 | * Step 26: | |
1674 | * Restore ZCAL registers. | |
1675 | */ | |
1676 | emc_dbg(emc, STEPS, "Step 26\n"); | |
1677 | ||
1678 | if (dram_type == DRAM_TYPE_LPDDR4) { | |
1679 | tegra210_emc_set_shadow_bypass(emc, ACTIVE); | |
1680 | emc_writel(emc, next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX], | |
1681 | EMC_ZCAL_WAIT_CNT); | |
1682 | emc_writel(emc, next->burst_regs[EMC_ZCAL_INTERVAL_INDEX], | |
1683 | EMC_ZCAL_INTERVAL); | |
1684 | tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); | |
1685 | } | |
1686 | ||
1687 | if (dram_type != DRAM_TYPE_LPDDR4 && opt_zcal_en_cc && | |
1688 | !opt_short_zcal && opt_cc_short_zcal) { | |
1689 | udelay(2); | |
1690 | ||
1691 | tegra210_emc_set_shadow_bypass(emc, ACTIVE); | |
1692 | if (dram_type == DRAM_TYPE_LPDDR2) | |
1693 | emc_writel(emc, next->burst_regs[EMC_MRS_WAIT_CNT_INDEX], | |
1694 | EMC_MRS_WAIT_CNT); | |
1695 | else if (dram_type == DRAM_TYPE_DDR3) | |
1696 | emc_writel(emc, next->burst_regs[EMC_ZCAL_WAIT_CNT_INDEX], | |
1697 | EMC_ZCAL_WAIT_CNT); | |
1698 | tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); | |
1699 | } | |
1700 | ||
1701 | /* | |
1702 | * Step 27: | |
1703 | * Restore EMC_CFG, FDPD registers. | |
1704 | */ | |
1705 | emc_dbg(emc, STEPS, "Step 27\n"); | |
1706 | ||
1707 | tegra210_emc_set_shadow_bypass(emc, ACTIVE); | |
1708 | emc_writel(emc, next->burst_regs[EMC_CFG_INDEX], EMC_CFG); | |
1709 | tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); | |
1710 | emc_writel(emc, next->emc_fdpd_ctrl_cmd_no_ramp, | |
1711 | EMC_FDPD_CTRL_CMD_NO_RAMP); | |
1712 | emc_writel(emc, next->emc_sel_dpd_ctrl, EMC_SEL_DPD_CTRL); | |
1713 | ||
1714 | /* | |
1715 | * Step 28: | |
1716 | * Training recover. Removed. | |
1717 | */ | |
1718 | emc_dbg(emc, STEPS, "Step 28\n"); | |
1719 | ||
1720 | tegra210_emc_set_shadow_bypass(emc, ACTIVE); | |
1721 | emc_writel(emc, | |
1722 | next->burst_regs[EMC_PMACRO_AUTOCAL_CFG_COMMON_INDEX], | |
1723 | EMC_PMACRO_AUTOCAL_CFG_COMMON); | |
1724 | tegra210_emc_set_shadow_bypass(emc, ASSEMBLY); | |
1725 | ||
1726 | /* | |
1727 | * Step 29: | |
1728 | * Power fix WAR. | |
1729 | */ | |
1730 | emc_dbg(emc, STEPS, "Step 29\n"); | |
1731 | ||
1732 | emc_writel(emc, EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE0 | | |
1733 | EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE1 | | |
1734 | EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE2 | | |
1735 | EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE3 | | |
1736 | EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE4 | | |
1737 | EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE5 | | |
1738 | EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE6 | | |
1739 | EMC_PMACRO_CFG_PM_GLOBAL_0_DISABLE_CFG_BYTE7, | |
1740 | EMC_PMACRO_CFG_PM_GLOBAL_0); | |
1741 | emc_writel(emc, EMC_PMACRO_TRAINING_CTRL_0_CH0_TRAINING_E_WRPTR, | |
1742 | EMC_PMACRO_TRAINING_CTRL_0); | |
1743 | emc_writel(emc, EMC_PMACRO_TRAINING_CTRL_1_CH1_TRAINING_E_WRPTR, | |
1744 | EMC_PMACRO_TRAINING_CTRL_1); | |
1745 | emc_writel(emc, 0, EMC_PMACRO_CFG_PM_GLOBAL_0); | |
1746 | ||
1747 | /* | |
1748 | * Step 30: | |
1749 | * Re-enable autocal. | |
1750 | */ | |
1751 | emc_dbg(emc, STEPS, "Step 30: Re-enable DLL and AUTOCAL\n"); | |
1752 | ||
1753 | if (next->burst_regs[EMC_CFG_DIG_DLL_INDEX] & EMC_CFG_DIG_DLL_CFG_DLL_EN) { | |
1754 | value = emc_readl(emc, EMC_CFG_DIG_DLL); | |
1755 | value |= EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_TRAFFIC; | |
1756 | value |= EMC_CFG_DIG_DLL_CFG_DLL_EN; | |
1757 | value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_RW_UNTIL_LOCK; | |
1758 | value &= ~EMC_CFG_DIG_DLL_CFG_DLL_STALL_ALL_UNTIL_LOCK; | |
1759 | value = (value & ~EMC_CFG_DIG_DLL_CFG_DLL_MODE_MASK) | | |
1760 | (2 << EMC_CFG_DIG_DLL_CFG_DLL_MODE_SHIFT); | |
1761 | emc_writel(emc, value, EMC_CFG_DIG_DLL); | |
1762 | tegra210_emc_timing_update(emc); | |
1763 | } | |
1764 | ||
1765 | emc_writel(emc, next->emc_auto_cal_config, EMC_AUTO_CAL_CONFIG); | |
1766 | ||
1767 | /* Done! Yay. */ | |
1768 | } | |
1769 | ||
1770 | const struct tegra210_emc_sequence tegra210_emc_r21021 = { | |
1771 | .revision = 0x7, | |
1772 | .set_clock = tegra210_emc_r21021_set_clock, | |
1773 | .periodic_compensation = tegra210_emc_r21021_periodic_compensation, | |
1774 | }; |