ASoC: SOF: Intel: hda-stream: use SOF helpers for consistency
[linux-block.git] / sound / soc / sof / intel / hda-stream.c
1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
2 //
3 // This file is provided under a dual BSD/GPLv2 license.  When using or
4 // redistributing this file, you may do so under either license.
5 //
6 // Copyright(c) 2018 Intel Corporation. All rights reserved.
7 //
8 // Authors: Liam Girdwood <liam.r.girdwood@linux.intel.com>
9 //          Ranjani Sridharan <ranjani.sridharan@linux.intel.com>
10 //          Rander Wang <rander.wang@intel.com>
11 //          Keyon Jie <yang.jie@linux.intel.com>
12 //
13
14 /*
15  * Hardware interface for generic Intel audio DSP HDA IP
16  */
17
18 #include <linux/pm_runtime.h>
19 #include <sound/hdaudio_ext.h>
20 #include <sound/hda_register.h>
21 #include <sound/sof.h>
22 #include <trace/events/sof_intel.h>
23 #include "../ops.h"
24 #include "../sof-audio.h"
25 #include "hda.h"
26
27 #define HDA_LTRP_GB_VALUE_US    95
28
29 static inline const char *hda_hstream_direction_str(struct hdac_stream *hstream)
30 {
31         if (hstream->direction == SNDRV_PCM_STREAM_PLAYBACK)
32                 return "Playback";
33         else
34                 return "Capture";
35 }
36
37 static char *hda_hstream_dbg_get_stream_info_str(struct hdac_stream *hstream)
38 {
39         struct snd_soc_pcm_runtime *rtd;
40
41         if (hstream->substream)
42                 rtd = asoc_substream_to_rtd(hstream->substream);
43         else if (hstream->cstream)
44                 rtd = hstream->cstream->private_data;
45         else
46                 /* Non audio DMA user, like dma-trace */
47                 return kasprintf(GFP_KERNEL, "-- (%s, stream_tag: %u)",
48                                  hda_hstream_direction_str(hstream),
49                                  hstream->stream_tag);
50
51         return kasprintf(GFP_KERNEL, "dai_link \"%s\" (%s, stream_tag: %u)",
52                          rtd->dai_link->name, hda_hstream_direction_str(hstream),
53                          hstream->stream_tag);
54 }
55
56 /*
57  * set up one of BDL entries for a stream
58  */
59 static int hda_setup_bdle(struct snd_sof_dev *sdev,
60                           struct snd_dma_buffer *dmab,
61                           struct hdac_stream *hstream,
62                           struct sof_intel_dsp_bdl **bdlp,
63                           int offset, int size, int ioc)
64 {
65         struct hdac_bus *bus = sof_to_bus(sdev);
66         struct sof_intel_dsp_bdl *bdl = *bdlp;
67
68         while (size > 0) {
69                 dma_addr_t addr;
70                 int chunk;
71
72                 if (hstream->frags >= HDA_DSP_MAX_BDL_ENTRIES) {
73                         dev_err(sdev->dev, "error: stream frags exceeded\n");
74                         return -EINVAL;
75                 }
76
77                 addr = snd_sgbuf_get_addr(dmab, offset);
78                 /* program BDL addr */
79                 bdl->addr_l = cpu_to_le32(lower_32_bits(addr));
80                 bdl->addr_h = cpu_to_le32(upper_32_bits(addr));
81                 /* program BDL size */
82                 chunk = snd_sgbuf_get_chunk_size(dmab, offset, size);
83                 /* one BDLE should not cross 4K boundary */
84                 if (bus->align_bdle_4k) {
85                         u32 remain = 0x1000 - (offset & 0xfff);
86
87                         if (chunk > remain)
88                                 chunk = remain;
89                 }
90                 bdl->size = cpu_to_le32(chunk);
91                 /* only program IOC when the whole segment is processed */
92                 size -= chunk;
93                 bdl->ioc = (size || !ioc) ? 0 : cpu_to_le32(0x01);
94                 bdl++;
95                 hstream->frags++;
96                 offset += chunk;
97         }
98
99         *bdlp = bdl;
100         return offset;
101 }
102
103 /*
104  * set up Buffer Descriptor List (BDL) for host memory transfer
105  * BDL describes the location of the individual buffers and is little endian.
106  */
107 int hda_dsp_stream_setup_bdl(struct snd_sof_dev *sdev,
108                              struct snd_dma_buffer *dmab,
109                              struct hdac_stream *hstream)
110 {
111         struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata;
112         struct sof_intel_dsp_bdl *bdl;
113         int i, offset, period_bytes, periods;
114         int remain, ioc;
115
116         period_bytes = hstream->period_bytes;
117         dev_dbg(sdev->dev, "period_bytes:0x%x\n", period_bytes);
118         if (!period_bytes)
119                 period_bytes = hstream->bufsize;
120
121         periods = hstream->bufsize / period_bytes;
122
123         dev_dbg(sdev->dev, "periods:%d\n", periods);
124
125         remain = hstream->bufsize % period_bytes;
126         if (remain)
127                 periods++;
128
129         /* program the initial BDL entries */
130         bdl = (struct sof_intel_dsp_bdl *)hstream->bdl.area;
131         offset = 0;
132         hstream->frags = 0;
133
134         /*
135          * set IOC if don't use position IPC
136          * and period_wakeup needed.
137          */
138         ioc = hda->no_ipc_position ?
139               !hstream->no_period_wakeup : 0;
140
141         for (i = 0; i < periods; i++) {
142                 if (i == (periods - 1) && remain)
143                         /* set the last small entry */
144                         offset = hda_setup_bdle(sdev, dmab,
145                                                 hstream, &bdl, offset,
146                                                 remain, 0);
147                 else
148                         offset = hda_setup_bdle(sdev, dmab,
149                                                 hstream, &bdl, offset,
150                                                 period_bytes, ioc);
151         }
152
153         return offset;
154 }
155
156 int hda_dsp_stream_spib_config(struct snd_sof_dev *sdev,
157                                struct hdac_ext_stream *hext_stream,
158                                int enable, u32 size)
159 {
160         struct hdac_stream *hstream = &hext_stream->hstream;
161         u32 mask;
162
163         if (!sdev->bar[HDA_DSP_SPIB_BAR]) {
164                 dev_err(sdev->dev, "error: address of spib capability is NULL\n");
165                 return -EINVAL;
166         }
167
168         mask = (1 << hstream->index);
169
170         /* enable/disable SPIB for the stream */
171         snd_sof_dsp_update_bits(sdev, HDA_DSP_SPIB_BAR,
172                                 SOF_HDA_ADSP_REG_CL_SPBFIFO_SPBFCCTL, mask,
173                                 enable << hstream->index);
174
175         /* set the SPIB value */
176         sof_io_write(sdev, hstream->spib_addr, size);
177
178         return 0;
179 }
180
181 /* get next unused stream */
182 struct hdac_ext_stream *
183 hda_dsp_stream_get(struct snd_sof_dev *sdev, int direction, u32 flags)
184 {
185         struct hdac_bus *bus = sof_to_bus(sdev);
186         struct sof_intel_hda_stream *hda_stream;
187         struct hdac_ext_stream *hext_stream = NULL;
188         struct hdac_stream *s;
189
190         spin_lock_irq(&bus->reg_lock);
191
192         /* get an unused stream */
193         list_for_each_entry(s, &bus->stream_list, list) {
194                 if (s->direction == direction && !s->opened) {
195                         hext_stream = stream_to_hdac_ext_stream(s);
196                         hda_stream = container_of(hext_stream,
197                                                   struct sof_intel_hda_stream,
198                                                   hext_stream);
199                         /* check if the host DMA channel is reserved */
200                         if (hda_stream->host_reserved)
201                                 continue;
202
203                         s->opened = true;
204                         break;
205                 }
206         }
207
208         spin_unlock_irq(&bus->reg_lock);
209
210         /* stream found ? */
211         if (!hext_stream) {
212                 dev_err(sdev->dev, "error: no free %s streams\n",
213                         direction == SNDRV_PCM_STREAM_PLAYBACK ?
214                         "playback" : "capture");
215                 return hext_stream;
216         }
217
218         hda_stream->flags = flags;
219
220         /*
221          * Prevent DMI Link L1 entry for streams that don't support it.
222          * Workaround to address a known issue with host DMA that results
223          * in xruns during pause/release in capture scenarios.
224          */
225         if (!(flags & SOF_HDA_STREAM_DMI_L1_COMPATIBLE))
226                 snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR,
227                                         HDA_VS_INTEL_EM2,
228                                         HDA_VS_INTEL_EM2_L1SEN, 0);
229
230         return hext_stream;
231 }
232
233 /* free a stream */
234 int hda_dsp_stream_put(struct snd_sof_dev *sdev, int direction, int stream_tag)
235 {
236         struct hdac_bus *bus = sof_to_bus(sdev);
237         struct sof_intel_hda_stream *hda_stream;
238         struct hdac_ext_stream *hext_stream;
239         struct hdac_stream *s;
240         bool dmi_l1_enable = true;
241         bool found = false;
242
243         spin_lock_irq(&bus->reg_lock);
244
245         /*
246          * close stream matching the stream tag and check if there are any open streams
247          * that are DMI L1 incompatible.
248          */
249         list_for_each_entry(s, &bus->stream_list, list) {
250                 hext_stream = stream_to_hdac_ext_stream(s);
251                 hda_stream = container_of(hext_stream, struct sof_intel_hda_stream, hext_stream);
252
253                 if (!s->opened)
254                         continue;
255
256                 if (s->direction == direction && s->stream_tag == stream_tag) {
257                         s->opened = false;
258                         found = true;
259                 } else if (!(hda_stream->flags & SOF_HDA_STREAM_DMI_L1_COMPATIBLE)) {
260                         dmi_l1_enable = false;
261                 }
262         }
263
264         spin_unlock_irq(&bus->reg_lock);
265
266         /* Enable DMI L1 if permitted */
267         if (dmi_l1_enable)
268                 snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, HDA_VS_INTEL_EM2,
269                                         HDA_VS_INTEL_EM2_L1SEN, HDA_VS_INTEL_EM2_L1SEN);
270
271         if (!found) {
272                 dev_err(sdev->dev, "%s: stream_tag %d not opened!\n",
273                         __func__, stream_tag);
274                 return -ENODEV;
275         }
276
277         return 0;
278 }
279
280 static int hda_dsp_stream_reset(struct snd_sof_dev *sdev, struct hdac_stream *hstream)
281 {
282         int sd_offset = SOF_STREAM_SD_OFFSET(hstream);
283         int timeout = HDA_DSP_STREAM_RESET_TIMEOUT;
284         u32 val;
285
286         /* enter stream reset */
287         snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, sd_offset, SOF_STREAM_SD_OFFSET_CRST,
288                                 SOF_STREAM_SD_OFFSET_CRST);
289         do {
290                 val = snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR, sd_offset);
291                 if (val & SOF_STREAM_SD_OFFSET_CRST)
292                         break;
293         } while (--timeout);
294         if (timeout == 0) {
295                 dev_err(sdev->dev, "timeout waiting for stream reset\n");
296                 return -ETIMEDOUT;
297         }
298
299         timeout = HDA_DSP_STREAM_RESET_TIMEOUT;
300
301         /* exit stream reset and wait to read a zero before reading any other register */
302         snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, sd_offset, SOF_STREAM_SD_OFFSET_CRST, 0x0);
303
304         /* wait for hardware to report that stream is out of reset */
305         udelay(3);
306         do {
307                 val = snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR, sd_offset);
308                 if ((val & SOF_STREAM_SD_OFFSET_CRST) == 0)
309                         break;
310         } while (--timeout);
311         if (timeout == 0) {
312                 dev_err(sdev->dev, "timeout waiting for stream to exit reset\n");
313                 return -ETIMEDOUT;
314         }
315
316         return 0;
317 }
318
319 int hda_dsp_stream_trigger(struct snd_sof_dev *sdev,
320                            struct hdac_ext_stream *hext_stream, int cmd)
321 {
322         struct hdac_stream *hstream = &hext_stream->hstream;
323         int sd_offset = SOF_STREAM_SD_OFFSET(hstream);
324         u32 dma_start = SOF_HDA_SD_CTL_DMA_START;
325         int ret = 0;
326         u32 run;
327
328         /* cmd must be for audio stream */
329         switch (cmd) {
330         case SNDRV_PCM_TRIGGER_PAUSE_RELEASE:
331         case SNDRV_PCM_TRIGGER_START:
332                 snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, SOF_HDA_INTCTL,
333                                         1 << hstream->index,
334                                         1 << hstream->index);
335
336                 snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR,
337                                         sd_offset,
338                                         SOF_HDA_SD_CTL_DMA_START |
339                                         SOF_HDA_CL_DMA_SD_INT_MASK,
340                                         SOF_HDA_SD_CTL_DMA_START |
341                                         SOF_HDA_CL_DMA_SD_INT_MASK);
342
343                 ret = snd_sof_dsp_read_poll_timeout(sdev,
344                                         HDA_DSP_HDA_BAR,
345                                         sd_offset, run,
346                                         ((run & dma_start) == dma_start),
347                                         HDA_DSP_REG_POLL_INTERVAL_US,
348                                         HDA_DSP_STREAM_RUN_TIMEOUT);
349
350                 if (ret >= 0)
351                         hstream->running = true;
352
353                 break;
354         case SNDRV_PCM_TRIGGER_SUSPEND:
355         case SNDRV_PCM_TRIGGER_PAUSE_PUSH:
356         case SNDRV_PCM_TRIGGER_STOP:
357                 snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR,
358                                         sd_offset,
359                                         SOF_HDA_SD_CTL_DMA_START |
360                                         SOF_HDA_CL_DMA_SD_INT_MASK, 0x0);
361
362                 ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_HDA_BAR,
363                                                 sd_offset, run,
364                                                 !(run & dma_start),
365                                                 HDA_DSP_REG_POLL_INTERVAL_US,
366                                                 HDA_DSP_STREAM_RUN_TIMEOUT);
367
368                 if (ret >= 0) {
369                         snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR,
370                                           sd_offset + SOF_HDA_ADSP_REG_CL_SD_STS,
371                                           SOF_HDA_CL_DMA_SD_INT_MASK);
372
373                         hstream->running = false;
374                         snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR,
375                                                 SOF_HDA_INTCTL,
376                                                 1 << hstream->index, 0x0);
377                 }
378                 break;
379         default:
380                 dev_err(sdev->dev, "error: unknown command: %d\n", cmd);
381                 return -EINVAL;
382         }
383
384         if (ret < 0) {
385                 char *stream_name = hda_hstream_dbg_get_stream_info_str(hstream);
386
387                 dev_err(sdev->dev,
388                         "%s: cmd %d on %s: timeout on STREAM_SD_OFFSET read\n",
389                         __func__, cmd, stream_name ? stream_name : "unknown stream");
390                 kfree(stream_name);
391         }
392
393         return ret;
394 }
395
396 /* minimal recommended programming for ICCMAX stream */
397 int hda_dsp_iccmax_stream_hw_params(struct snd_sof_dev *sdev, struct hdac_ext_stream *hext_stream,
398                                     struct snd_dma_buffer *dmab,
399                                     struct snd_pcm_hw_params *params)
400 {
401         struct hdac_bus *bus = sof_to_bus(sdev);
402         struct hdac_stream *hstream = &hext_stream->hstream;
403         int sd_offset = SOF_STREAM_SD_OFFSET(hstream);
404         int ret;
405         u32 mask = 0x1 << hstream->index;
406
407         if (!hext_stream) {
408                 dev_err(sdev->dev, "error: no stream available\n");
409                 return -ENODEV;
410         }
411
412         if (!dmab) {
413                 dev_err(sdev->dev, "error: no dma buffer allocated!\n");
414                 return -ENODEV;
415         }
416
417         if (hstream->posbuf)
418                 *hstream->posbuf = 0;
419
420         /* reset BDL address */
421         snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR,
422                           sd_offset + SOF_HDA_ADSP_REG_CL_SD_BDLPL,
423                           0x0);
424         snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR,
425                           sd_offset + SOF_HDA_ADSP_REG_CL_SD_BDLPU,
426                           0x0);
427
428         hstream->frags = 0;
429
430         ret = hda_dsp_stream_setup_bdl(sdev, dmab, hstream);
431         if (ret < 0) {
432                 dev_err(sdev->dev, "error: set up of BDL failed\n");
433                 return ret;
434         }
435
436         /* program BDL address */
437         snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR,
438                           sd_offset + SOF_HDA_ADSP_REG_CL_SD_BDLPL,
439                           (u32)hstream->bdl.addr);
440         snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR,
441                           sd_offset + SOF_HDA_ADSP_REG_CL_SD_BDLPU,
442                           upper_32_bits(hstream->bdl.addr));
443
444         /* program cyclic buffer length */
445         snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR,
446                           sd_offset + SOF_HDA_ADSP_REG_CL_SD_CBL,
447                           hstream->bufsize);
448
449         /* program last valid index */
450         snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR,
451                                 sd_offset + SOF_HDA_ADSP_REG_CL_SD_LVI,
452                                 0xffff, (hstream->frags - 1));
453
454         /* decouple host and link DMA, enable DSP features */
455         snd_sof_dsp_update_bits(sdev, HDA_DSP_PP_BAR, SOF_HDA_REG_PP_PPCTL,
456                                 mask, mask);
457
458         /* Follow HW recommendation to set the guardband value to 95us during FW boot */
459         snd_hdac_chip_updateb(bus, VS_LTRP, HDA_VS_INTEL_LTRP_GB_MASK, HDA_LTRP_GB_VALUE_US);
460
461         /* start DMA */
462         snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, sd_offset,
463                                 SOF_HDA_SD_CTL_DMA_START, SOF_HDA_SD_CTL_DMA_START);
464
465         return 0;
466 }
467
468 /*
469  * prepare for common hdac registers settings, for both code loader
470  * and normal stream.
471  */
472 int hda_dsp_stream_hw_params(struct snd_sof_dev *sdev,
473                              struct hdac_ext_stream *hext_stream,
474                              struct snd_dma_buffer *dmab,
475                              struct snd_pcm_hw_params *params)
476 {
477         const struct sof_intel_dsp_desc *chip = get_chip_info(sdev->pdata);
478         struct hdac_bus *bus = sof_to_bus(sdev);
479         struct hdac_stream *hstream = &hext_stream->hstream;
480         int sd_offset = SOF_STREAM_SD_OFFSET(hstream);
481         int ret;
482         u32 dma_start = SOF_HDA_SD_CTL_DMA_START;
483         u32 mask;
484         u32 run;
485
486         if (!hext_stream) {
487                 dev_err(sdev->dev, "error: no stream available\n");
488                 return -ENODEV;
489         }
490
491         if (!dmab) {
492                 dev_err(sdev->dev, "error: no dma buffer allocated!\n");
493                 return -ENODEV;
494         }
495
496         /* decouple host and link DMA */
497         mask = 0x1 << hstream->index;
498         snd_sof_dsp_update_bits(sdev, HDA_DSP_PP_BAR, SOF_HDA_REG_PP_PPCTL,
499                                 mask, mask);
500
501         /* clear stream status */
502         snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, sd_offset,
503                                 SOF_HDA_CL_DMA_SD_INT_MASK |
504                                 SOF_HDA_SD_CTL_DMA_START, 0);
505
506         ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_HDA_BAR,
507                                             sd_offset, run,
508                                             !(run & dma_start),
509                                             HDA_DSP_REG_POLL_INTERVAL_US,
510                                             HDA_DSP_STREAM_RUN_TIMEOUT);
511
512         if (ret < 0) {
513                 char *stream_name = hda_hstream_dbg_get_stream_info_str(hstream);
514
515                 dev_err(sdev->dev,
516                         "%s: on %s: timeout on STREAM_SD_OFFSET read1\n",
517                         __func__, stream_name ? stream_name : "unknown stream");
518                 kfree(stream_name);
519                 return ret;
520         }
521
522         snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR,
523                                 sd_offset + SOF_HDA_ADSP_REG_CL_SD_STS,
524                                 SOF_HDA_CL_DMA_SD_INT_MASK,
525                                 SOF_HDA_CL_DMA_SD_INT_MASK);
526
527         /* stream reset */
528         ret = hda_dsp_stream_reset(sdev, hstream);
529         if (ret < 0)
530                 return ret;
531
532         if (hstream->posbuf)
533                 *hstream->posbuf = 0;
534
535         /* reset BDL address */
536         snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR,
537                           sd_offset + SOF_HDA_ADSP_REG_CL_SD_BDLPL,
538                           0x0);
539         snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR,
540                           sd_offset + SOF_HDA_ADSP_REG_CL_SD_BDLPU,
541                           0x0);
542
543         /* clear stream status */
544         snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, sd_offset,
545                                 SOF_HDA_CL_DMA_SD_INT_MASK |
546                                 SOF_HDA_SD_CTL_DMA_START, 0);
547
548         ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_HDA_BAR,
549                                             sd_offset, run,
550                                             !(run & dma_start),
551                                             HDA_DSP_REG_POLL_INTERVAL_US,
552                                             HDA_DSP_STREAM_RUN_TIMEOUT);
553
554         if (ret < 0) {
555                 char *stream_name = hda_hstream_dbg_get_stream_info_str(hstream);
556
557                 dev_err(sdev->dev,
558                         "%s: on %s: timeout on STREAM_SD_OFFSET read1\n",
559                         __func__, stream_name ? stream_name : "unknown stream");
560                 kfree(stream_name);
561                 return ret;
562         }
563
564         snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR,
565                                 sd_offset + SOF_HDA_ADSP_REG_CL_SD_STS,
566                                 SOF_HDA_CL_DMA_SD_INT_MASK,
567                                 SOF_HDA_CL_DMA_SD_INT_MASK);
568
569         hstream->frags = 0;
570
571         ret = hda_dsp_stream_setup_bdl(sdev, dmab, hstream);
572         if (ret < 0) {
573                 dev_err(sdev->dev, "error: set up of BDL failed\n");
574                 return ret;
575         }
576
577         /* program stream tag to set up stream descriptor for DMA */
578         snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, sd_offset,
579                                 SOF_HDA_CL_SD_CTL_STREAM_TAG_MASK,
580                                 hstream->stream_tag <<
581                                 SOF_HDA_CL_SD_CTL_STREAM_TAG_SHIFT);
582
583         /* program cyclic buffer length */
584         snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR,
585                           sd_offset + SOF_HDA_ADSP_REG_CL_SD_CBL,
586                           hstream->bufsize);
587
588         /*
589          * Recommended hardware programming sequence for HDAudio DMA format
590          * on earlier platforms - this is not needed on newer platforms
591          *
592          * 1. Put DMA into coupled mode by clearing PPCTL.PROCEN bit
593          *    for corresponding stream index before the time of writing
594          *    format to SDxFMT register.
595          * 2. Write SDxFMT
596          * 3. Set PPCTL.PROCEN bit for corresponding stream index to
597          *    enable decoupled mode
598          */
599
600         if (chip->quirks & SOF_INTEL_PROCEN_FMT_QUIRK) {
601                 /* couple host and link DMA, disable DSP features */
602                 snd_sof_dsp_update_bits(sdev, HDA_DSP_PP_BAR, SOF_HDA_REG_PP_PPCTL,
603                                         mask, 0);
604         }
605
606         /* program stream format */
607         snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR,
608                                 sd_offset +
609                                 SOF_HDA_ADSP_REG_CL_SD_FORMAT,
610                                 0xffff, hstream->format_val);
611
612         if (chip->quirks & SOF_INTEL_PROCEN_FMT_QUIRK) {
613                 /* decouple host and link DMA, enable DSP features */
614                 snd_sof_dsp_update_bits(sdev, HDA_DSP_PP_BAR, SOF_HDA_REG_PP_PPCTL,
615                                         mask, mask);
616         }
617
618         /* program last valid index */
619         snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR,
620                                 sd_offset + SOF_HDA_ADSP_REG_CL_SD_LVI,
621                                 0xffff, (hstream->frags - 1));
622
623         /* program BDL address */
624         snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR,
625                           sd_offset + SOF_HDA_ADSP_REG_CL_SD_BDLPL,
626                           (u32)hstream->bdl.addr);
627         snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR,
628                           sd_offset + SOF_HDA_ADSP_REG_CL_SD_BDLPU,
629                           upper_32_bits(hstream->bdl.addr));
630
631         /* enable position buffer, if needed */
632         if (bus->use_posbuf && bus->posbuf.addr &&
633             !(snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR, SOF_HDA_ADSP_DPLBASE)
634               & SOF_HDA_ADSP_DPLBASE_ENABLE)) {
635                 snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR, SOF_HDA_ADSP_DPUBASE,
636                                   upper_32_bits(bus->posbuf.addr));
637                 snd_sof_dsp_write(sdev, HDA_DSP_HDA_BAR, SOF_HDA_ADSP_DPLBASE,
638                                   (u32)bus->posbuf.addr |
639                                   SOF_HDA_ADSP_DPLBASE_ENABLE);
640         }
641
642         /* set interrupt enable bits */
643         snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, sd_offset,
644                                 SOF_HDA_CL_DMA_SD_INT_MASK,
645                                 SOF_HDA_CL_DMA_SD_INT_MASK);
646
647         /* read FIFO size */
648         if (hstream->direction == SNDRV_PCM_STREAM_PLAYBACK) {
649                 hstream->fifo_size =
650                         snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR,
651                                          sd_offset +
652                                          SOF_HDA_ADSP_REG_CL_SD_FIFOSIZE);
653                 hstream->fifo_size &= 0xffff;
654                 hstream->fifo_size += 1;
655         } else {
656                 hstream->fifo_size = 0;
657         }
658
659         return ret;
660 }
661
662 int hda_dsp_stream_hw_free(struct snd_sof_dev *sdev,
663                            struct snd_pcm_substream *substream)
664 {
665         struct hdac_stream *hstream = substream->runtime->private_data;
666         struct hdac_ext_stream *hext_stream = container_of(hstream,
667                                                          struct hdac_ext_stream,
668                                                          hstream);
669         struct hdac_bus *bus = sof_to_bus(sdev);
670         u32 mask = 0x1 << hstream->index;
671         int ret;
672
673         ret = hda_dsp_stream_reset(sdev, hstream);
674         if (ret < 0)
675                 return ret;
676
677         spin_lock_irq(&bus->reg_lock);
678         /* couple host and link DMA if link DMA channel is idle */
679         if (!hext_stream->link_locked)
680                 snd_sof_dsp_update_bits(sdev, HDA_DSP_PP_BAR,
681                                         SOF_HDA_REG_PP_PPCTL, mask, 0);
682         spin_unlock_irq(&bus->reg_lock);
683
684         hda_dsp_stream_spib_config(sdev, hext_stream, HDA_DSP_SPIB_DISABLE, 0);
685
686         hstream->substream = NULL;
687
688         return 0;
689 }
690
691 bool hda_dsp_check_stream_irq(struct snd_sof_dev *sdev)
692 {
693         struct hdac_bus *bus = sof_to_bus(sdev);
694         bool ret = false;
695         u32 status;
696
697         /* The function can be called at irq thread, so use spin_lock_irq */
698         spin_lock_irq(&bus->reg_lock);
699
700         status = snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR, SOF_HDA_INTSTS);
701
702         trace_sof_intel_hda_dsp_check_stream_irq(sdev, status);
703
704         /* if Register inaccessible, ignore it.*/
705         if (status != 0xffffffff)
706                 ret = true;
707
708         spin_unlock_irq(&bus->reg_lock);
709
710         return ret;
711 }
712
713 static void
714 hda_dsp_compr_bytes_transferred(struct hdac_stream *hstream, int direction)
715 {
716         u64 buffer_size = hstream->bufsize;
717         u64 prev_pos, pos, num_bytes;
718
719         div64_u64_rem(hstream->curr_pos, buffer_size, &prev_pos);
720         pos = hda_dsp_stream_get_position(hstream, direction, false);
721
722         if (pos < prev_pos)
723                 num_bytes = (buffer_size - prev_pos) +  pos;
724         else
725                 num_bytes = pos - prev_pos;
726
727         hstream->curr_pos += num_bytes;
728 }
729
730 static bool hda_dsp_stream_check(struct hdac_bus *bus, u32 status)
731 {
732         struct sof_intel_hda_dev *sof_hda = bus_to_sof_hda(bus);
733         struct hdac_stream *s;
734         bool active = false;
735         u32 sd_status;
736
737         list_for_each_entry(s, &bus->stream_list, list) {
738                 if (status & BIT(s->index) && s->opened) {
739                         sd_status = snd_hdac_stream_readb(s, SD_STS);
740
741                         trace_sof_intel_hda_dsp_stream_status(bus->dev, s, sd_status);
742
743                         snd_hdac_stream_writeb(s, SD_STS, sd_status);
744
745                         active = true;
746                         if ((!s->substream && !s->cstream) ||
747                             !s->running ||
748                             (sd_status & SOF_HDA_CL_DMA_SD_INT_COMPLETE) == 0)
749                                 continue;
750
751                         /* Inform ALSA only in case not do that with IPC */
752                         if (s->substream && sof_hda->no_ipc_position) {
753                                 snd_sof_pcm_period_elapsed(s->substream);
754                         } else if (s->cstream) {
755                                 hda_dsp_compr_bytes_transferred(s, s->cstream->direction);
756                                 snd_compr_fragment_elapsed(s->cstream);
757                         }
758                 }
759         }
760
761         return active;
762 }
763
764 irqreturn_t hda_dsp_stream_threaded_handler(int irq, void *context)
765 {
766         struct snd_sof_dev *sdev = context;
767         struct hdac_bus *bus = sof_to_bus(sdev);
768 #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
769         u32 rirb_status;
770 #endif
771         bool active;
772         u32 status;
773         int i;
774
775         /*
776          * Loop 10 times to handle missed interrupts caused by
777          * unsolicited responses from the codec
778          */
779         for (i = 0, active = true; i < 10 && active; i++) {
780                 spin_lock_irq(&bus->reg_lock);
781
782                 status = snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR, SOF_HDA_INTSTS);
783
784                 /* check streams */
785                 active = hda_dsp_stream_check(bus, status);
786
787                 /* check and clear RIRB interrupt */
788 #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
789                 if (status & AZX_INT_CTRL_EN) {
790                         rirb_status = snd_hdac_chip_readb(bus, RIRBSTS);
791                         if (rirb_status & RIRB_INT_MASK) {
792                                 /*
793                                  * Clearing the interrupt status here ensures
794                                  * that no interrupt gets masked after the RIRB
795                                  * wp is read in snd_hdac_bus_update_rirb.
796                                  */
797                                 snd_hdac_chip_writeb(bus, RIRBSTS,
798                                                      RIRB_INT_MASK);
799                                 active = true;
800                                 if (rirb_status & RIRB_INT_RESPONSE)
801                                         snd_hdac_bus_update_rirb(bus);
802                         }
803                 }
804 #endif
805                 spin_unlock_irq(&bus->reg_lock);
806         }
807
808         return IRQ_HANDLED;
809 }
810
811 int hda_dsp_stream_init(struct snd_sof_dev *sdev)
812 {
813         struct hdac_bus *bus = sof_to_bus(sdev);
814         struct hdac_ext_stream *hext_stream;
815         struct hdac_stream *hstream;
816         struct pci_dev *pci = to_pci_dev(sdev->dev);
817         struct sof_intel_hda_dev *sof_hda = bus_to_sof_hda(bus);
818         int sd_offset;
819         int i, num_playback, num_capture, num_total, ret;
820         u32 gcap;
821
822         gcap = snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR, SOF_HDA_GCAP);
823         dev_dbg(sdev->dev, "hda global caps = 0x%x\n", gcap);
824
825         /* get stream count from GCAP */
826         num_capture = (gcap >> 8) & 0x0f;
827         num_playback = (gcap >> 12) & 0x0f;
828         num_total = num_playback + num_capture;
829
830         dev_dbg(sdev->dev, "detected %d playback and %d capture streams\n",
831                 num_playback, num_capture);
832
833         if (num_playback >= SOF_HDA_PLAYBACK_STREAMS) {
834                 dev_err(sdev->dev, "error: too many playback streams %d\n",
835                         num_playback);
836                 return -EINVAL;
837         }
838
839         if (num_capture >= SOF_HDA_CAPTURE_STREAMS) {
840                 dev_err(sdev->dev, "error: too many capture streams %d\n",
841                         num_playback);
842                 return -EINVAL;
843         }
844
845         /*
846          * mem alloc for the position buffer
847          * TODO: check position buffer update
848          */
849         ret = snd_dma_alloc_pages(SNDRV_DMA_TYPE_DEV, &pci->dev,
850                                   SOF_HDA_DPIB_ENTRY_SIZE * num_total,
851                                   &bus->posbuf);
852         if (ret < 0) {
853                 dev_err(sdev->dev, "error: posbuffer dma alloc failed\n");
854                 return -ENOMEM;
855         }
856
857 #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
858         /* mem alloc for the CORB/RIRB ringbuffers */
859         ret = snd_dma_alloc_pages(SNDRV_DMA_TYPE_DEV, &pci->dev,
860                                   PAGE_SIZE, &bus->rb);
861         if (ret < 0) {
862                 dev_err(sdev->dev, "error: RB alloc failed\n");
863                 return -ENOMEM;
864         }
865 #endif
866
867         /* create capture streams */
868         for (i = 0; i < num_capture; i++) {
869                 struct sof_intel_hda_stream *hda_stream;
870
871                 hda_stream = devm_kzalloc(sdev->dev, sizeof(*hda_stream),
872                                           GFP_KERNEL);
873                 if (!hda_stream)
874                         return -ENOMEM;
875
876                 hda_stream->sdev = sdev;
877
878                 hext_stream = &hda_stream->hext_stream;
879
880                 hext_stream->pphc_addr = sdev->bar[HDA_DSP_PP_BAR] +
881                         SOF_HDA_PPHC_BASE + SOF_HDA_PPHC_INTERVAL * i;
882
883                 hext_stream->pplc_addr = sdev->bar[HDA_DSP_PP_BAR] +
884                         SOF_HDA_PPLC_BASE + SOF_HDA_PPLC_MULTI * num_total +
885                         SOF_HDA_PPLC_INTERVAL * i;
886
887                 hstream = &hext_stream->hstream;
888
889                 /* do we support SPIB */
890                 if (sdev->bar[HDA_DSP_SPIB_BAR]) {
891                         hstream->spib_addr = sdev->bar[HDA_DSP_SPIB_BAR] +
892                                 SOF_HDA_SPIB_BASE + SOF_HDA_SPIB_INTERVAL * i +
893                                 SOF_HDA_SPIB_SPIB;
894
895                         hstream->fifo_addr = sdev->bar[HDA_DSP_SPIB_BAR] +
896                                 SOF_HDA_SPIB_BASE + SOF_HDA_SPIB_INTERVAL * i +
897                                 SOF_HDA_SPIB_MAXFIFO;
898                 }
899
900                 hstream->bus = bus;
901                 hstream->sd_int_sta_mask = 1 << i;
902                 hstream->index = i;
903                 sd_offset = SOF_STREAM_SD_OFFSET(hstream);
904                 hstream->sd_addr = sdev->bar[HDA_DSP_HDA_BAR] + sd_offset;
905                 hstream->stream_tag = i + 1;
906                 hstream->opened = false;
907                 hstream->running = false;
908                 hstream->direction = SNDRV_PCM_STREAM_CAPTURE;
909
910                 /* memory alloc for stream BDL */
911                 ret = snd_dma_alloc_pages(SNDRV_DMA_TYPE_DEV, &pci->dev,
912                                           HDA_DSP_BDL_SIZE, &hstream->bdl);
913                 if (ret < 0) {
914                         dev_err(sdev->dev, "error: stream bdl dma alloc failed\n");
915                         return -ENOMEM;
916                 }
917                 hstream->posbuf = (__le32 *)(bus->posbuf.area +
918                         (hstream->index) * 8);
919
920                 list_add_tail(&hstream->list, &bus->stream_list);
921         }
922
923         /* create playback streams */
924         for (i = num_capture; i < num_total; i++) {
925                 struct sof_intel_hda_stream *hda_stream;
926
927                 hda_stream = devm_kzalloc(sdev->dev, sizeof(*hda_stream),
928                                           GFP_KERNEL);
929                 if (!hda_stream)
930                         return -ENOMEM;
931
932                 hda_stream->sdev = sdev;
933
934                 hext_stream = &hda_stream->hext_stream;
935
936                 /* we always have DSP support */
937                 hext_stream->pphc_addr = sdev->bar[HDA_DSP_PP_BAR] +
938                         SOF_HDA_PPHC_BASE + SOF_HDA_PPHC_INTERVAL * i;
939
940                 hext_stream->pplc_addr = sdev->bar[HDA_DSP_PP_BAR] +
941                         SOF_HDA_PPLC_BASE + SOF_HDA_PPLC_MULTI * num_total +
942                         SOF_HDA_PPLC_INTERVAL * i;
943
944                 hstream = &hext_stream->hstream;
945
946                 /* do we support SPIB */
947                 if (sdev->bar[HDA_DSP_SPIB_BAR]) {
948                         hstream->spib_addr = sdev->bar[HDA_DSP_SPIB_BAR] +
949                                 SOF_HDA_SPIB_BASE + SOF_HDA_SPIB_INTERVAL * i +
950                                 SOF_HDA_SPIB_SPIB;
951
952                         hstream->fifo_addr = sdev->bar[HDA_DSP_SPIB_BAR] +
953                                 SOF_HDA_SPIB_BASE + SOF_HDA_SPIB_INTERVAL * i +
954                                 SOF_HDA_SPIB_MAXFIFO;
955                 }
956
957                 hstream->bus = bus;
958                 hstream->sd_int_sta_mask = 1 << i;
959                 hstream->index = i;
960                 sd_offset = SOF_STREAM_SD_OFFSET(hstream);
961                 hstream->sd_addr = sdev->bar[HDA_DSP_HDA_BAR] + sd_offset;
962                 hstream->stream_tag = i - num_capture + 1;
963                 hstream->opened = false;
964                 hstream->running = false;
965                 hstream->direction = SNDRV_PCM_STREAM_PLAYBACK;
966
967                 /* mem alloc for stream BDL */
968                 ret = snd_dma_alloc_pages(SNDRV_DMA_TYPE_DEV, &pci->dev,
969                                           HDA_DSP_BDL_SIZE, &hstream->bdl);
970                 if (ret < 0) {
971                         dev_err(sdev->dev, "error: stream bdl dma alloc failed\n");
972                         return -ENOMEM;
973                 }
974
975                 hstream->posbuf = (__le32 *)(bus->posbuf.area +
976                         (hstream->index) * 8);
977
978                 list_add_tail(&hstream->list, &bus->stream_list);
979         }
980
981         /* store total stream count (playback + capture) from GCAP */
982         sof_hda->stream_max = num_total;
983
984         return 0;
985 }
986
987 void hda_dsp_stream_free(struct snd_sof_dev *sdev)
988 {
989         struct hdac_bus *bus = sof_to_bus(sdev);
990         struct hdac_stream *s, *_s;
991         struct hdac_ext_stream *hext_stream;
992         struct sof_intel_hda_stream *hda_stream;
993
994         /* free position buffer */
995         if (bus->posbuf.area)
996                 snd_dma_free_pages(&bus->posbuf);
997
998 #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA)
999         /* free position buffer */
1000         if (bus->rb.area)
1001                 snd_dma_free_pages(&bus->rb);
1002 #endif
1003
1004         list_for_each_entry_safe(s, _s, &bus->stream_list, list) {
1005                 /* TODO: decouple */
1006
1007                 /* free bdl buffer */
1008                 if (s->bdl.area)
1009                         snd_dma_free_pages(&s->bdl);
1010                 list_del(&s->list);
1011                 hext_stream = stream_to_hdac_ext_stream(s);
1012                 hda_stream = container_of(hext_stream, struct sof_intel_hda_stream,
1013                                           hext_stream);
1014                 devm_kfree(sdev->dev, hda_stream);
1015         }
1016 }
1017
1018 snd_pcm_uframes_t hda_dsp_stream_get_position(struct hdac_stream *hstream,
1019                                               int direction, bool can_sleep)
1020 {
1021         struct hdac_ext_stream *hext_stream = stream_to_hdac_ext_stream(hstream);
1022         struct sof_intel_hda_stream *hda_stream = hstream_to_sof_hda_stream(hext_stream);
1023         struct snd_sof_dev *sdev = hda_stream->sdev;
1024         snd_pcm_uframes_t pos;
1025
1026         switch (sof_hda_position_quirk) {
1027         case SOF_HDA_POSITION_QUIRK_USE_SKYLAKE_LEGACY:
1028                 /*
1029                  * This legacy code, inherited from the Skylake driver,
1030                  * mixes DPIB registers and DPIB DDR updates and
1031                  * does not seem to follow any known hardware recommendations.
1032                  * It's not clear e.g. why there is a different flow
1033                  * for capture and playback, the only information that matters is
1034                  * what traffic class is used, and on all SOF-enabled platforms
1035                  * only VC0 is supported so the work-around was likely not necessary
1036                  * and quite possibly wrong.
1037                  */
1038
1039                 /* DPIB/posbuf position mode:
1040                  * For Playback, Use DPIB register from HDA space which
1041                  * reflects the actual data transferred.
1042                  * For Capture, Use the position buffer for pointer, as DPIB
1043                  * is not accurate enough, its update may be completed
1044                  * earlier than the data written to DDR.
1045                  */
1046                 if (direction == SNDRV_PCM_STREAM_PLAYBACK) {
1047                         pos = snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR,
1048                                                AZX_REG_VS_SDXDPIB_XBASE +
1049                                                (AZX_REG_VS_SDXDPIB_XINTERVAL *
1050                                                 hstream->index));
1051                 } else {
1052                         /*
1053                          * For capture stream, we need more workaround to fix the
1054                          * position incorrect issue:
1055                          *
1056                          * 1. Wait at least 20us before reading position buffer after
1057                          * the interrupt generated(IOC), to make sure position update
1058                          * happens on frame boundary i.e. 20.833uSec for 48KHz.
1059                          * 2. Perform a dummy Read to DPIB register to flush DMA
1060                          * position value.
1061                          * 3. Read the DMA Position from posbuf. Now the readback
1062                          * value should be >= period boundary.
1063                          */
1064                         if (can_sleep)
1065                                 usleep_range(20, 21);
1066
1067                         snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR,
1068                                          AZX_REG_VS_SDXDPIB_XBASE +
1069                                          (AZX_REG_VS_SDXDPIB_XINTERVAL *
1070                                           hstream->index));
1071                         pos = snd_hdac_stream_get_pos_posbuf(hstream);
1072                 }
1073                 break;
1074         case SOF_HDA_POSITION_QUIRK_USE_DPIB_REGISTERS:
1075                 /*
1076                  * In case VC1 traffic is disabled this is the recommended option
1077                  */
1078                 pos = snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR,
1079                                        AZX_REG_VS_SDXDPIB_XBASE +
1080                                        (AZX_REG_VS_SDXDPIB_XINTERVAL *
1081                                         hstream->index));
1082                 break;
1083         case SOF_HDA_POSITION_QUIRK_USE_DPIB_DDR_UPDATE:
1084                 /*
1085                  * This is the recommended option when VC1 is enabled.
1086                  * While this isn't needed for SOF platforms it's added for
1087                  * consistency and debug.
1088                  */
1089                 pos = snd_hdac_stream_get_pos_posbuf(hstream);
1090                 break;
1091         default:
1092                 dev_err_once(sdev->dev, "hda_position_quirk value %d not supported\n",
1093                              sof_hda_position_quirk);
1094                 pos = 0;
1095                 break;
1096         }
1097
1098         if (pos >= hstream->bufsize)
1099                 pos = 0;
1100
1101         return pos;
1102 }