projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
spi: bcm63xx-hsspi: Add prepend mode support
[linux-2.6-block.git] / drivers / spi / spi.c
CommitLineData
b445bfcb 1// SPDX-License-Identifier: GPL-2.0-or-later
787f4889
MB		 2// SPI init/core code
3//
4// Copyright (C) 2005 David Brownell
5// Copyright (C) 2008 Secret Lab Technologies Ltd.
8ae12a0d 6
8ae12a0d
DB		 7#include <linux/kernel.h>
8#include <linux/device.h>
9#include <linux/init.h>
10#include <linux/cache.h>
99adef31
MB		 11#include <linux/dma-mapping.h>
12#include <linux/dmaengine.h>
94040828 13#include <linux/mutex.h>
2b7a32f7 14#include <linux/of_device.h>
d57a4282 15#include <linux/of_irq.h>
86be408b 16#include <linux/clk/clk-conf.h>
5a0e3ad6 17#include <linux/slab.h>
e0626e38 18#include <linux/mod_devicetable.h>
8ae12a0d 19#include <linux/spi/spi.h>
b5932f5c 20#include <linux/spi/spi-mem.h>
f3186dd8 21#include <linux/gpio/consumer.h>
3ae22e8c 22#include <linux/pm_runtime.h>
f48c767c 23#include <linux/pm_domain.h>
826cf175 24#include <linux/property.h>
025ed130 25#include <linux/export.h>
8bd75c77 26#include <linux/sched/rt.h>
ae7e81c0 27#include <uapi/linux/sched/types.h>
ffbbdd21
LW		 28#include <linux/delay.h>
29#include <linux/kthread.h>
64bee4d2
MW		 30#include <linux/ioport.h>
31#include <linux/acpi.h>
b1b8153c 32#include <linux/highmem.h>
9b61e302 33#include <linux/idr.h>
8a2e487e 34#include <linux/platform_data/x86/apple.h>
44ea6281 35#include <linux/ptp_clock_kernel.h>
6598b91b 36#include <linux/percpu.h>
8ae12a0d 37
56ec1978
MB		 38#define CREATE_TRACE_POINTS
39#include <trace/events/spi.h>
ca1438dc
AB		 40EXPORT_TRACEPOINT_SYMBOL(spi_transfer_start);
41EXPORT_TRACEPOINT_SYMBOL(spi_transfer_stop);
9b61e302 42
46336966
BB		 43#include "internals.h"
44
9b61e302 45static DEFINE_IDR(spi_master_idr);
56ec1978 46
8ae12a0d
DB		 47static void spidev_release(struct device *dev)
48{
0ffa0285 49 struct spi_device *spi = to_spi_device(dev);
8ae12a0d 50
8caab75f 51 spi_controller_put(spi->controller);
5039563e 52 kfree(spi->driver_override);
6598b91b 53 free_percpu(spi->pcpu_statistics);
07a389fe 54 kfree(spi);
8ae12a0d
DB		 55}
56
57static ssize_t
58modalias_show(struct device *dev, struct device_attribute *a, char *buf)
59{
60 const struct spi_device *spi = to_spi_device(dev);
8c4ff6d0
ZR		 61 int len;
62
63 len = acpi_device_modalias(dev, buf, PAGE_SIZE - 1);
64 if (len != -ENODEV)
65 return len;
8ae12a0d 66
d8e328b3 67 return sprintf(buf, "%s%s\n", SPI_MODULE_PREFIX, spi->modalias);
8ae12a0d 68}
aa7da564 69static DEVICE_ATTR_RO(modalias);
8ae12a0d 70
5039563e
TP		 71static ssize_t driver_override_store(struct device *dev,
72 struct device_attribute *a,
73 const char *buf, size_t count)
74{
75 struct spi_device *spi = to_spi_device(dev);
19368f0f 76 int ret;
5039563e 77
19368f0f
KK		 78 ret = driver_set_override(dev, &spi->driver_override, buf, count);
79 if (ret)
80 return ret;
5039563e
TP		 81
82 return count;
83}
84
85static ssize_t driver_override_show(struct device *dev,
86 struct device_attribute *a, char *buf)
87{
88 const struct spi_device *spi = to_spi_device(dev);
89 ssize_t len;
90
91 device_lock(dev);
92 len = snprintf(buf, PAGE_SIZE, "%s\n", spi->driver_override ? : "");
93 device_unlock(dev);
94 return len;
95}
96static DEVICE_ATTR_RW(driver_override);
97
d501cc4c 98static struct spi_statistics __percpu *spi_alloc_pcpu_stats(struct device *dev)
6598b91b
DJ		 99{
100 struct spi_statistics __percpu *pcpu_stats;
101
102 if (dev)
103 pcpu_stats = devm_alloc_percpu(dev, struct spi_statistics);
104 else
105 pcpu_stats = alloc_percpu_gfp(struct spi_statistics, GFP_KERNEL);
106
107 if (pcpu_stats) {
108 int cpu;
109
110 for_each_possible_cpu(cpu) {
111 struct spi_statistics *stat;
112
113 stat = per_cpu_ptr(pcpu_stats, cpu);
114 u64_stats_init(&stat->syncp);
115 }
116 }
117 return pcpu_stats;
118}
119
120#define spi_pcpu_stats_totalize(ret, in, field) \
121do { \
122 int i; \
123 ret = 0; \
124 for_each_possible_cpu(i) { \
125 const struct spi_statistics *pcpu_stats; \
126 u64 inc; \
127 unsigned int start; \
128 pcpu_stats = per_cpu_ptr(in, i); \
129 do { \
93cc2559 130 start = u64_stats_fetch_begin( \
6598b91b
DJ		 131 &pcpu_stats->syncp); \
132 inc = u64_stats_read(&pcpu_stats->field); \
93cc2559 133 } while (u64_stats_fetch_retry( \
6598b91b
DJ		 134 &pcpu_stats->syncp, start)); \
135 ret += inc; \
136 } \
137} while (0)
138
eca2ebc7 139#define SPI_STATISTICS_ATTRS(field, file) \
8caab75f
GU		 140static ssize_t spi_controller_##field##_show(struct device *dev, \
141 struct device_attribute *attr, \
142 char *buf) \
eca2ebc7 143{ \
8caab75f
GU		 144 struct spi_controller *ctlr = container_of(dev, \
145 struct spi_controller, dev); \
6598b91b 146 return spi_statistics_##field##_show(ctlr->pcpu_statistics, buf); \
eca2ebc7 147} \
8caab75f 148static struct device_attribute dev_attr_spi_controller_##field = { \
ad25c92e 149 .attr = { .name = file, .mode = 0444 }, \
8caab75f 150 .show = spi_controller_##field##_show, \
eca2ebc7
MS		 151}; \
152static ssize_t spi_device_##field##_show(struct device *dev, \
153 struct device_attribute *attr, \
154 char *buf) \
155{ \
d1eba93b 156 struct spi_device *spi = to_spi_device(dev); \
6598b91b 157 return spi_statistics_##field##_show(spi->pcpu_statistics, buf); \
eca2ebc7
MS		 158} \
159static struct device_attribute dev_attr_spi_device_##field = { \
ad25c92e 160 .attr = { .name = file, .mode = 0444 }, \
eca2ebc7
MS		 161 .show = spi_device_##field##_show, \
162}
163
6598b91b 164#define SPI_STATISTICS_SHOW_NAME(name, file, field) \
d501cc4c 165static ssize_t spi_statistics_##name##_show(struct spi_statistics __percpu *stat, \
eca2ebc7
MS		 166 char *buf) \
167{ \
eca2ebc7 168 ssize_t len; \
6598b91b
DJ		 169 u64 val; \
170 spi_pcpu_stats_totalize(val, stat, field); \
171 len = sysfs_emit(buf, "%llu\n", val); \
eca2ebc7
MS		 172 return len; \
173} \
174SPI_STATISTICS_ATTRS(name, file)
175
6598b91b 176#define SPI_STATISTICS_SHOW(field) \
eca2ebc7 177 SPI_STATISTICS_SHOW_NAME(field, __stringify(field), \
6598b91b 178 field)
eca2ebc7 179
6598b91b
DJ		 180SPI_STATISTICS_SHOW(messages);
181SPI_STATISTICS_SHOW(transfers);
182SPI_STATISTICS_SHOW(errors);
183SPI_STATISTICS_SHOW(timedout);
eca2ebc7 184
6598b91b
DJ		 185SPI_STATISTICS_SHOW(spi_sync);
186SPI_STATISTICS_SHOW(spi_sync_immediate);
187SPI_STATISTICS_SHOW(spi_async);
eca2ebc7 188
6598b91b
DJ		 189SPI_STATISTICS_SHOW(bytes);
190SPI_STATISTICS_SHOW(bytes_rx);
191SPI_STATISTICS_SHOW(bytes_tx);
eca2ebc7 192
6b7bc061
MS		 193#define SPI_STATISTICS_TRANSFER_BYTES_HISTO(index, number) \
194 SPI_STATISTICS_SHOW_NAME(transfer_bytes_histo##index, \
195 "transfer_bytes_histo_" number, \
6598b91b 196 transfer_bytes_histo[index])
6b7bc061
MS		 197SPI_STATISTICS_TRANSFER_BYTES_HISTO(0, "0-1");
198SPI_STATISTICS_TRANSFER_BYTES_HISTO(1, "2-3");
199SPI_STATISTICS_TRANSFER_BYTES_HISTO(2, "4-7");
200SPI_STATISTICS_TRANSFER_BYTES_HISTO(3, "8-15");
201SPI_STATISTICS_TRANSFER_BYTES_HISTO(4, "16-31");
202SPI_STATISTICS_TRANSFER_BYTES_HISTO(5, "32-63");
203SPI_STATISTICS_TRANSFER_BYTES_HISTO(6, "64-127");
204SPI_STATISTICS_TRANSFER_BYTES_HISTO(7, "128-255");
205SPI_STATISTICS_TRANSFER_BYTES_HISTO(8, "256-511");
206SPI_STATISTICS_TRANSFER_BYTES_HISTO(9, "512-1023");
207SPI_STATISTICS_TRANSFER_BYTES_HISTO(10, "1024-2047");
208SPI_STATISTICS_TRANSFER_BYTES_HISTO(11, "2048-4095");
209SPI_STATISTICS_TRANSFER_BYTES_HISTO(12, "4096-8191");
210SPI_STATISTICS_TRANSFER_BYTES_HISTO(13, "8192-16383");
211SPI_STATISTICS_TRANSFER_BYTES_HISTO(14, "16384-32767");
212SPI_STATISTICS_TRANSFER_BYTES_HISTO(15, "32768-65535");
213SPI_STATISTICS_TRANSFER_BYTES_HISTO(16, "65536+");
214
6598b91b 215SPI_STATISTICS_SHOW(transfers_split_maxsize);
d9f12122 216
aa7da564
GKH		 217static struct attribute *spi_dev_attrs[] = {
218 &dev_attr_modalias.attr,
5039563e 219 &dev_attr_driver_override.attr,
aa7da564 220 NULL,
8ae12a0d 221};
eca2ebc7
MS		 222
223static const struct attribute_group spi_dev_group = {
224 .attrs = spi_dev_attrs,
225};
226
227static struct attribute *spi_device_statistics_attrs[] = {
228 &dev_attr_spi_device_messages.attr,
229 &dev_attr_spi_device_transfers.attr,
230 &dev_attr_spi_device_errors.attr,
231 &dev_attr_spi_device_timedout.attr,
232 &dev_attr_spi_device_spi_sync.attr,
233 &dev_attr_spi_device_spi_sync_immediate.attr,
234 &dev_attr_spi_device_spi_async.attr,
235 &dev_attr_spi_device_bytes.attr,
236 &dev_attr_spi_device_bytes_rx.attr,
237 &dev_attr_spi_device_bytes_tx.attr,
6b7bc061
MS		 238 &dev_attr_spi_device_transfer_bytes_histo0.attr,
239 &dev_attr_spi_device_transfer_bytes_histo1.attr,
240 &dev_attr_spi_device_transfer_bytes_histo2.attr,
241 &dev_attr_spi_device_transfer_bytes_histo3.attr,
242 &dev_attr_spi_device_transfer_bytes_histo4.attr,
243 &dev_attr_spi_device_transfer_bytes_histo5.attr,
244 &dev_attr_spi_device_transfer_bytes_histo6.attr,
245 &dev_attr_spi_device_transfer_bytes_histo7.attr,
246 &dev_attr_spi_device_transfer_bytes_histo8.attr,
247 &dev_attr_spi_device_transfer_bytes_histo9.attr,
248 &dev_attr_spi_device_transfer_bytes_histo10.attr,
249 &dev_attr_spi_device_transfer_bytes_histo11.attr,
250 &dev_attr_spi_device_transfer_bytes_histo12.attr,
251 &dev_attr_spi_device_transfer_bytes_histo13.attr,
252 &dev_attr_spi_device_transfer_bytes_histo14.attr,
253 &dev_attr_spi_device_transfer_bytes_histo15.attr,
254 &dev_attr_spi_device_transfer_bytes_histo16.attr,
d9f12122 255 &dev_attr_spi_device_transfers_split_maxsize.attr,
eca2ebc7
MS		 256 NULL,
257};
258
259static const struct attribute_group spi_device_statistics_group = {
260 .name = "statistics",
261 .attrs = spi_device_statistics_attrs,
262};
263
264static const struct attribute_group *spi_dev_groups[] = {
265 &spi_dev_group,
266 &spi_device_statistics_group,
267 NULL,
268};
269
8caab75f
GU		 270static struct attribute *spi_controller_statistics_attrs[] = {
271 &dev_attr_spi_controller_messages.attr,
272 &dev_attr_spi_controller_transfers.attr,
273 &dev_attr_spi_controller_errors.attr,
274 &dev_attr_spi_controller_timedout.attr,
275 &dev_attr_spi_controller_spi_sync.attr,
276 &dev_attr_spi_controller_spi_sync_immediate.attr,
277 &dev_attr_spi_controller_spi_async.attr,
278 &dev_attr_spi_controller_bytes.attr,
279 &dev_attr_spi_controller_bytes_rx.attr,
280 &dev_attr_spi_controller_bytes_tx.attr,
281 &dev_attr_spi_controller_transfer_bytes_histo0.attr,
282 &dev_attr_spi_controller_transfer_bytes_histo1.attr,
283 &dev_attr_spi_controller_transfer_bytes_histo2.attr,
284 &dev_attr_spi_controller_transfer_bytes_histo3.attr,
285 &dev_attr_spi_controller_transfer_bytes_histo4.attr,
286 &dev_attr_spi_controller_transfer_bytes_histo5.attr,
287 &dev_attr_spi_controller_transfer_bytes_histo6.attr,
288 &dev_attr_spi_controller_transfer_bytes_histo7.attr,
289 &dev_attr_spi_controller_transfer_bytes_histo8.attr,
290 &dev_attr_spi_controller_transfer_bytes_histo9.attr,
291 &dev_attr_spi_controller_transfer_bytes_histo10.attr,
292 &dev_attr_spi_controller_transfer_bytes_histo11.attr,
293 &dev_attr_spi_controller_transfer_bytes_histo12.attr,
294 &dev_attr_spi_controller_transfer_bytes_histo13.attr,
295 &dev_attr_spi_controller_transfer_bytes_histo14.attr,
296 &dev_attr_spi_controller_transfer_bytes_histo15.attr,
297 &dev_attr_spi_controller_transfer_bytes_histo16.attr,
298 &dev_attr_spi_controller_transfers_split_maxsize.attr,
eca2ebc7
MS		 299 NULL,
300};
301
8caab75f 302static const struct attribute_group spi_controller_statistics_group = {
eca2ebc7 303 .name = "statistics",
8caab75f 304 .attrs = spi_controller_statistics_attrs,
eca2ebc7
MS		 305};
306
307static const struct attribute_group *spi_master_groups[] = {
8caab75f 308 &spi_controller_statistics_group,
eca2ebc7
MS		 309 NULL,
310};
311
d501cc4c 312static void spi_statistics_add_transfer_stats(struct spi_statistics __percpu *pcpu_stats,
da21fde0
UKK		 313 struct spi_transfer *xfer,
314 struct spi_controller *ctlr)
eca2ebc7 315{
6b7bc061 316 int l2len = min(fls(xfer->len), SPI_STATISTICS_HISTO_SIZE) - 1;
67b9d641 317 struct spi_statistics *stats;
6b7bc061
MS		 318
319 if (l2len < 0)
320 l2len = 0;
eca2ebc7 321
67b9d641
DJ		 322 get_cpu();
323 stats = this_cpu_ptr(pcpu_stats);
6598b91b 324 u64_stats_update_begin(&stats->syncp);
eca2ebc7 325
6598b91b
DJ		 326 u64_stats_inc(&stats->transfers);
327 u64_stats_inc(&stats->transfer_bytes_histo[l2len]);
eca2ebc7 328
6598b91b 329 u64_stats_add(&stats->bytes, xfer->len);
eca2ebc7 330 if ((xfer->tx_buf) &&
8caab75f 331 (xfer->tx_buf != ctlr->dummy_tx))
6598b91b 332 u64_stats_add(&stats->bytes_tx, xfer->len);
eca2ebc7 333 if ((xfer->rx_buf) &&
8caab75f 334 (xfer->rx_buf != ctlr->dummy_rx))
6598b91b 335 u64_stats_add(&stats->bytes_rx, xfer->len);
eca2ebc7 336
6598b91b 337 u64_stats_update_end(&stats->syncp);
67b9d641 338 put_cpu();
eca2ebc7 339}
8ae12a0d 340
350de7ce
AS		 341/*
342 * modalias support makes "modprobe $MODALIAS" new-style hotplug work,
8ae12a0d
DB		 343 * and the sysfs version makes coldplug work too.
344 */
3f076575 345static const struct spi_device_id *spi_match_id(const struct spi_device_id *id, const char *name)
75368bf6
AV		 346{
347 while (id->name[0]) {
3f076575 348 if (!strcmp(name, id->name))
75368bf6
AV		 349 return id;
350 id++;
351 }
352 return NULL;
353}
354
355const struct spi_device_id *spi_get_device_id(const struct spi_device *sdev)
356{
357 const struct spi_driver *sdrv = to_spi_driver(sdev->dev.driver);
358
3f076575 359 return spi_match_id(sdrv->id_table, sdev->modalias);
75368bf6
AV		 360}
361EXPORT_SYMBOL_GPL(spi_get_device_id);
362
aea672d0
AS		 363const void *spi_get_device_match_data(const struct spi_device *sdev)
364{
365 const void *match;
366
367 match = device_get_match_data(&sdev->dev);
368 if (match)
369 return match;
370
371 return (const void *)spi_get_device_id(sdev)->driver_data;
372}
373EXPORT_SYMBOL_GPL(spi_get_device_match_data);
374
8ae12a0d
DB		 375static int spi_match_device(struct device *dev, struct device_driver *drv)
376{
377 const struct spi_device *spi = to_spi_device(dev);
75368bf6
AV		 378 const struct spi_driver *sdrv = to_spi_driver(drv);
379
5039563e
TP		 380 /* Check override first, and if set, only use the named driver */
381 if (spi->driver_override)
382 return strcmp(spi->driver_override, drv->name) == 0;
383
2b7a32f7
SA		 384 /* Attempt an OF style match */
385 if (of_driver_match_device(dev, drv))
386 return 1;
387
64bee4d2
MW		 388 /* Then try ACPI */
389 if (acpi_driver_match_device(dev, drv))
390 return 1;
391
75368bf6 392 if (sdrv->id_table)
3f076575 393 return !!spi_match_id(sdrv->id_table, spi->modalias);
8ae12a0d 394
35f74fca 395 return strcmp(spi->modalias, drv->name) == 0;
8ae12a0d
DB		 396}
397
7eff2e7a 398static int spi_uevent(struct device *dev, struct kobj_uevent_env *env)
8ae12a0d
DB		 399{
400 const struct spi_device *spi = to_spi_device(dev);
8c4ff6d0
ZR		 401 int rc;
402
403 rc = acpi_device_uevent_modalias(dev, env);
404 if (rc != -ENODEV)
405 return rc;
8ae12a0d 406
2856670f 407 return add_uevent_var(env, "MODALIAS=%s%s", SPI_MODULE_PREFIX, spi->modalias);
8ae12a0d
DB		 408}
409
9db34ee6 410static int spi_probe(struct device *dev)
b885244e
DB		 411{
412 const struct spi_driver *sdrv = to_spi_driver(dev->driver);
44af7927 413 struct spi_device *spi = to_spi_device(dev);
33cf00e5
MW		 414 int ret;
415
86be408b
SN		 416 ret = of_clk_set_defaults(dev->of_node, false);
417 if (ret)
418 return ret;
419
44af7927
JH		 420 if (dev->of_node) {
421 spi->irq = of_irq_get(dev->of_node, 0);
422 if (spi->irq == -EPROBE_DEFER)
423 return -EPROBE_DEFER;
424 if (spi->irq < 0)
425 spi->irq = 0;
426 }
427
676e7c25 428 ret = dev_pm_domain_attach(dev, true);
71f277a7
UH		 429 if (ret)
430 return ret;
431
440408db
UKK		 432 if (sdrv->probe) {
433 ret = sdrv->probe(spi);
434 if (ret)
435 dev_pm_domain_detach(dev, true);
436 }
b885244e 437
33cf00e5 438 return ret;
b885244e
DB		 439}
440
fc7a6209 441static void spi_remove(struct device *dev)
b885244e
DB		 442{
443 const struct spi_driver *sdrv = to_spi_driver(dev->driver);
33cf00e5 444
a0386bba
UKK		 445 if (sdrv->remove)
446 sdrv->remove(to_spi_device(dev));
7795d475 447
676e7c25 448 dev_pm_domain_detach(dev, true);
b885244e
DB		 449}
450
9db34ee6 451static void spi_shutdown(struct device *dev)
b885244e 452{
a6f483b2
MS		 453 if (dev->driver) {
454 const struct spi_driver *sdrv = to_spi_driver(dev->driver);
b885244e 455
a6f483b2
MS		 456 if (sdrv->shutdown)
457 sdrv->shutdown(to_spi_device(dev));
458 }
b885244e
DB		 459}
460
9db34ee6
UKK		 461struct bus_type spi_bus_type = {
462 .name = "spi",
463 .dev_groups = spi_dev_groups,
464 .match = spi_match_device,
465 .uevent = spi_uevent,
466 .probe = spi_probe,
467 .remove = spi_remove,
468 .shutdown = spi_shutdown,
469};
470EXPORT_SYMBOL_GPL(spi_bus_type);
471
33e34dc6 472/**
ca5d2485 473 * __spi_register_driver - register a SPI driver
88c9321d 474 * @owner: owner module of the driver to register
33e34dc6
DB		 475 * @sdrv: the driver to register
476 * Context: can sleep
97d56dc6
JMC		 477 *
478 * Return: zero on success, else a negative error code.
33e34dc6 479 */
ca5d2485 480int __spi_register_driver(struct module *owner, struct spi_driver *sdrv)
b885244e 481{
ca5d2485 482 sdrv->driver.owner = owner;
b885244e 483 sdrv->driver.bus = &spi_bus_type;
5fa6863b
MB		 484
485 /*
486 * For Really Good Reasons we use spi: modaliases not of:
487 * modaliases for DT so module autoloading won't work if we
488 * don't have a spi_device_id as well as a compatible string.
489 */
490 if (sdrv->driver.of_match_table) {
491 const struct of_device_id *of_id;
492
493 for (of_id = sdrv->driver.of_match_table; of_id->compatible[0];
494 of_id++) {
495 const char *of_name;
496
497 /* Strip off any vendor prefix */
498 of_name = strnchr(of_id->compatible,
499 sizeof(of_id->compatible), ',');
500 if (of_name)
501 of_name++;
502 else
503 of_name = of_id->compatible;
504
505 if (sdrv->id_table) {
506 const struct spi_device_id *spi_id;
507
3f076575 508 spi_id = spi_match_id(sdrv->id_table, of_name);
b79332ef 509 if (spi_id)
5fa6863b
MB		 510 continue;
511 } else {
512 if (strcmp(sdrv->driver.name, of_name) == 0)
513 continue;
514 }
515
516 pr_warn("SPI driver %s has no spi_device_id for %s\n",
517 sdrv->driver.name, of_id->compatible);
518 }
519 }
520
b885244e
DB		 521 return driver_register(&sdrv->driver);
522}
ca5d2485 523EXPORT_SYMBOL_GPL(__spi_register_driver);
b885244e 524
8ae12a0d
DB		 525/*-------------------------------------------------------------------------*/
526
350de7ce
AS		 527/*
528 * SPI devices should normally not be created by SPI device drivers; that
8caab75f 529 * would make them board-specific. Similarly with SPI controller drivers.
8ae12a0d
DB		 530 * Device registration normally goes into like arch/.../mach.../board-YYY.c
531 * with other readonly (flashable) information about mainboard devices.
532 */
533
534struct boardinfo {
535 struct list_head list;
2b9603a0 536 struct spi_board_info board_info;
8ae12a0d
DB		 537};
538
539static LIST_HEAD(board_list);
8caab75f 540static LIST_HEAD(spi_controller_list);
2b9603a0
FT		 541
542/*
be73e323 543 * Used to protect add/del operation for board_info list and
350de7ce
AS		 544 * spi_controller list, and their matching process also used
545 * to protect object of type struct idr.
2b9603a0 546 */
94040828 547static DEFINE_MUTEX(board_lock);
8ae12a0d 548
dc87c98e
GL		 549/**
550 * spi_alloc_device - Allocate a new SPI device
8caab75f 551 * @ctlr: Controller to which device is connected
dc87c98e
GL		 552 * Context: can sleep
553 *
554 * Allows a driver to allocate and initialize a spi_device without
555 * registering it immediately. This allows a driver to directly
556 * fill the spi_device with device parameters before calling
557 * spi_add_device() on it.
558 *
559 * Caller is responsible to call spi_add_device() on the returned
8caab75f 560 * spi_device structure to add it to the SPI controller. If the caller
dc87c98e
GL		 561 * needs to discard the spi_device without adding it, then it should
562 * call spi_dev_put() on it.
563 *
97d56dc6 564 * Return: a pointer to the new device, or NULL.
dc87c98e 565 */
e3dc1399 566struct spi_device *spi_alloc_device(struct spi_controller *ctlr)
dc87c98e
GL		 567{
568 struct spi_device *spi;
dc87c98e 569
8caab75f 570 if (!spi_controller_get(ctlr))
dc87c98e
GL		 571 return NULL;
572
5fe5f05e 573 spi = kzalloc(sizeof(*spi), GFP_KERNEL);
dc87c98e 574 if (!spi) {
8caab75f 575 spi_controller_put(ctlr);
dc87c98e
GL		 576 return NULL;
577 }
578
6598b91b
DJ		 579 spi->pcpu_statistics = spi_alloc_pcpu_stats(NULL);
580 if (!spi->pcpu_statistics) {
581 kfree(spi);
582 spi_controller_put(ctlr);
583 return NULL;
584 }
585
8caab75f
GU		 586 spi->master = spi->controller = ctlr;
587 spi->dev.parent = &ctlr->dev;
dc87c98e
GL		 588 spi->dev.bus = &spi_bus_type;
589 spi->dev.release = spidev_release;
ea235786 590 spi->mode = ctlr->buswidth_override_bits;
eca2ebc7 591
dc87c98e
GL		 592 device_initialize(&spi->dev);
593 return spi;
594}
e3dc1399 595EXPORT_SYMBOL_GPL(spi_alloc_device);
dc87c98e 596
e13ac47b
JN		 597static void spi_dev_set_name(struct spi_device *spi)
598{
599 struct acpi_device *adev = ACPI_COMPANION(&spi->dev);
600
601 if (adev) {
602 dev_set_name(&spi->dev, "spi-%s", acpi_dev_name(adev));
603 return;
604 }
605
8caab75f 606 dev_set_name(&spi->dev, "%s.%u", dev_name(&spi->controller->dev),
303feb3c 607 spi_get_chipselect(spi, 0));
e13ac47b
JN		 608}
609
b6fb8d3a
MW		 610static int spi_dev_check(struct device *dev, void *data)
611{
612 struct spi_device *spi = to_spi_device(dev);
613 struct spi_device *new_spi = data;
614
8caab75f 615 if (spi->controller == new_spi->controller &&
303feb3c 616 spi_get_chipselect(spi, 0) == spi_get_chipselect(new_spi, 0))
b6fb8d3a
MW		 617 return -EBUSY;
618 return 0;
619}
620
c7299fea
SK		 621static void spi_cleanup(struct spi_device *spi)
622{
623 if (spi->controller->cleanup)
624 spi->controller->cleanup(spi);
625}
626
0c79378c 627static int __spi_add_device(struct spi_device *spi)
dc87c98e 628{
8caab75f
GU		 629 struct spi_controller *ctlr = spi->controller;
630 struct device *dev = ctlr->dev.parent;
dc87c98e
GL		 631 int status;
632
6bfb15f3
UKK		 633 /*
634 * We need to make sure there's no other device with this
635 * chipselect **BEFORE** we call setup(), else we'll trash
636 * its configuration.
637 */
b6fb8d3a
MW		 638 status = bus_for_each_dev(&spi_bus_type, NULL, spi, spi_dev_check);
639 if (status) {
e48880e0 640 dev_err(dev, "chipselect %d already in use\n",
303feb3c 641 spi_get_chipselect(spi, 0));
0c79378c 642 return status;
e48880e0
DB		 643 }
644
ddf75be4
LW		 645 /* Controller may unregister concurrently */
646 if (IS_ENABLED(CONFIG_SPI_DYNAMIC) &&
647 !device_is_registered(&ctlr->dev)) {
0c79378c 648 return -ENODEV;
ddf75be4
LW		 649 }
650
f3186dd8 651 if (ctlr->cs_gpiods)
303feb3c 652 spi_set_csgpiod(spi, 0, ctlr->cs_gpiods[spi_get_chipselect(spi, 0)]);
74317984 653
350de7ce
AS		 654 /*
655 * Drivers may modify this initial i/o setup, but will
e48880e0
DB		 656 * normally rely on the device being setup. Devices
657 * using SPI_CS_HIGH can't coexist well otherwise...
658 */
7d077197 659 status = spi_setup(spi);
dc87c98e 660 if (status < 0) {
eb288a1f
LW		 661 dev_err(dev, "can't setup %s, status %d\n",
662 dev_name(&spi->dev), status);
0c79378c 663 return status;
dc87c98e
GL		 664 }
665
e48880e0 666 /* Device may be bound to an active driver when this returns */
dc87c98e 667 status = device_add(&spi->dev);
c7299fea 668 if (status < 0) {
eb288a1f
LW		 669 dev_err(dev, "can't add %s, status %d\n",
670 dev_name(&spi->dev), status);
c7299fea
SK		 671 spi_cleanup(spi);
672 } else {
35f74fca 673 dev_dbg(dev, "registered child %s\n", dev_name(&spi->dev));
c7299fea 674 }
dc87c98e 675
0c79378c
SR		 676 return status;
677}
678
679/**
680 * spi_add_device - Add spi_device allocated with spi_alloc_device
681 * @spi: spi_device to register
682 *
683 * Companion function to spi_alloc_device. Devices allocated with
684 * spi_alloc_device can be added onto the spi bus with this function.
685 *
686 * Return: 0 on success; negative errno on failure
687 */
e3dc1399 688int spi_add_device(struct spi_device *spi)
0c79378c
SR		 689{
690 struct spi_controller *ctlr = spi->controller;
691 struct device *dev = ctlr->dev.parent;
692 int status;
693
694 /* Chipselects are numbered 0..max; validate. */
303feb3c
AKM		 695 if (spi_get_chipselect(spi, 0) >= ctlr->num_chipselect) {
696 dev_err(dev, "cs%d >= max %d\n", spi_get_chipselect(spi, 0),
0c79378c
SR		 697 ctlr->num_chipselect);
698 return -EINVAL;
699 }
700
701 /* Set the bus ID string */
702 spi_dev_set_name(spi);
703
6098475d 704 mutex_lock(&ctlr->add_lock);
0c79378c 705 status = __spi_add_device(spi);
6098475d 706 mutex_unlock(&ctlr->add_lock);
e48880e0 707 return status;
dc87c98e 708}
e3dc1399 709EXPORT_SYMBOL_GPL(spi_add_device);
8ae12a0d 710
0c79378c
SR		 711static int spi_add_device_locked(struct spi_device *spi)
712{
713 struct spi_controller *ctlr = spi->controller;
714 struct device *dev = ctlr->dev.parent;
715
716 /* Chipselects are numbered 0..max; validate. */
303feb3c
AKM		 717 if (spi_get_chipselect(spi, 0) >= ctlr->num_chipselect) {
718 dev_err(dev, "cs%d >= max %d\n", spi_get_chipselect(spi, 0),
0c79378c
SR		 719 ctlr->num_chipselect);
720 return -EINVAL;
721 }
722
723 /* Set the bus ID string */
724 spi_dev_set_name(spi);
725
6098475d 726 WARN_ON(!mutex_is_locked(&ctlr->add_lock));
0c79378c
SR		 727 return __spi_add_device(spi);
728}
729
33e34dc6
DB		 730/**
731 * spi_new_device - instantiate one new SPI device
8caab75f 732 * @ctlr: Controller to which device is connected
33e34dc6
DB		 733 * @chip: Describes the SPI device
734 * Context: can sleep
735 *
736 * On typical mainboards, this is purely internal; and it's not needed
8ae12a0d
DB		 737 * after board init creates the hard-wired devices. Some development
738 * platforms may not be able to use spi_register_board_info though, and
739 * this is exported so that for example a USB or parport based adapter
740 * driver could add devices (which it would learn about out-of-band).
082c8cb4 741 *
97d56dc6 742 * Return: the new device, or NULL.
8ae12a0d 743 */
8caab75f 744struct spi_device *spi_new_device(struct spi_controller *ctlr,
e9d5a461 745 struct spi_board_info *chip)
8ae12a0d
DB		 746{
747 struct spi_device *proxy;
8ae12a0d
DB		 748 int status;
749
350de7ce
AS		 750 /*
751 * NOTE: caller did any chip->bus_num checks necessary.
082c8cb4
DB		 752 *
753 * Also, unless we change the return value convention to use
754 * error-or-pointer (not NULL-or-pointer), troubleshootability
755 * suggests syslogged diagnostics are best here (ugh).
756 */
757
8caab75f 758 proxy = spi_alloc_device(ctlr);
dc87c98e 759 if (!proxy)
8ae12a0d
DB		 760 return NULL;
761
102eb975
GL		 762 WARN_ON(strlen(chip->modalias) >= sizeof(proxy->modalias));
763
303feb3c 764 spi_set_chipselect(proxy, 0, chip->chip_select);
8ae12a0d 765 proxy->max_speed_hz = chip->max_speed_hz;
980a01c9 766 proxy->mode = chip->mode;
8ae12a0d 767 proxy->irq = chip->irq;
51e99de5 768 strscpy(proxy->modalias, chip->modalias, sizeof(proxy->modalias));
8ae12a0d
DB		 769 proxy->dev.platform_data = (void *) chip->platform_data;
770 proxy->controller_data = chip->controller_data;
771 proxy->controller_state = NULL;
8ae12a0d 772
47afc77b
HK		 773 if (chip->swnode) {
774 status = device_add_software_node(&proxy->dev, chip->swnode);
826cf175 775 if (status) {
9d902c2a 776 dev_err(&ctlr->dev, "failed to add software node to '%s': %d\n",
826cf175
DT		 777 chip->modalias, status);
778 goto err_dev_put;
779 }
8ae12a0d
DB		 780 }
781
826cf175
DT		 782 status = spi_add_device(proxy);
783 if (status < 0)
df41a5da 784 goto err_dev_put;
826cf175 785
8ae12a0d 786 return proxy;
826cf175 787
826cf175 788err_dev_put:
df41a5da 789 device_remove_software_node(&proxy->dev);
826cf175
DT		 790 spi_dev_put(proxy);
791 return NULL;
8ae12a0d
DB		 792}
793EXPORT_SYMBOL_GPL(spi_new_device);
794
3b1884c2
GU		 795/**
796 * spi_unregister_device - unregister a single SPI device
797 * @spi: spi_device to unregister
798 *
799 * Start making the passed SPI device vanish. Normally this would be handled
8caab75f 800 * by spi_unregister_controller().
3b1884c2
GU		 801 */
802void spi_unregister_device(struct spi_device *spi)
803{
bd6c1644
GU		 804 if (!spi)
805 return;
806
8324147f 807 if (spi->dev.of_node) {
bd6c1644 808 of_node_clear_flag(spi->dev.of_node, OF_POPULATED);
8324147f
JH		 809 of_node_put(spi->dev.of_node);
810 }
7f24467f
OP		 811 if (ACPI_COMPANION(&spi->dev))
812 acpi_device_clear_enumerated(ACPI_COMPANION(&spi->dev));
47afc77b 813 device_remove_software_node(&spi->dev);
27e7db56
SK		 814 device_del(&spi->dev);
815 spi_cleanup(spi);
816 put_device(&spi->dev);
3b1884c2
GU		 817}
818EXPORT_SYMBOL_GPL(spi_unregister_device);
819
8caab75f
GU		 820static void spi_match_controller_to_boardinfo(struct spi_controller *ctlr,
821 struct spi_board_info *bi)
2b9603a0
FT		 822{
823 struct spi_device *dev;
824
8caab75f 825 if (ctlr->bus_num != bi->bus_num)
2b9603a0
FT		 826 return;
827
8caab75f 828 dev = spi_new_device(ctlr, bi);
2b9603a0 829 if (!dev)
8caab75f 830 dev_err(ctlr->dev.parent, "can't create new device for %s\n",
2b9603a0
FT		 831 bi->modalias);
832}
833
33e34dc6
DB		 834/**
835 * spi_register_board_info - register SPI devices for a given board
836 * @info: array of chip descriptors
837 * @n: how many descriptors are provided
838 * Context: can sleep
839 *
8ae12a0d
DB		 840 * Board-specific early init code calls this (probably during arch_initcall)
841 * with segments of the SPI device table. Any device nodes are created later,
842 * after the relevant parent SPI controller (bus_num) is defined. We keep
843 * this table of devices forever, so that reloading a controller driver will
844 * not make Linux forget about these hard-wired devices.
845 *
846 * Other code can also call this, e.g. a particular add-on board might provide
847 * SPI devices through its expansion connector, so code initializing that board
848 * would naturally declare its SPI devices.
849 *
850 * The board info passed can safely be __initdata ... but be careful of
851 * any embedded pointers (platform_data, etc), they're copied as-is.
97d56dc6
JMC		 852 *
853 * Return: zero on success, else a negative error code.
8ae12a0d 854 */
fd4a319b 855int spi_register_board_info(struct spi_board_info const *info, unsigned n)
8ae12a0d 856{
2b9603a0
FT		 857 struct boardinfo *bi;
858 int i;
8ae12a0d 859
c7908a37 860 if (!n)
f974cf57 861 return 0;
c7908a37 862
f9bdb7fd 863 bi = kcalloc(n, sizeof(*bi), GFP_KERNEL);
8ae12a0d
DB		 864 if (!bi)
865 return -ENOMEM;
8ae12a0d 866
2b9603a0 867 for (i = 0; i < n; i++, bi++, info++) {
8caab75f 868 struct spi_controller *ctlr;
8ae12a0d 869
2b9603a0 870 memcpy(&bi->board_info, info, sizeof(*info));
826cf175 871
2b9603a0
FT		 872 mutex_lock(&board_lock);
873 list_add_tail(&bi->list, &board_list);
8caab75f
GU		 874 list_for_each_entry(ctlr, &spi_controller_list, list)
875 spi_match_controller_to_boardinfo(ctlr,
876 &bi->board_info);
2b9603a0 877 mutex_unlock(&board_lock);
8ae12a0d 878 }
2b9603a0
FT		 879
880 return 0;
8ae12a0d
DB		 881}
882
883/*-------------------------------------------------------------------------*/
884
fb51601b
UKK		 885/* Core methods for SPI resource management */
886
887/**
888 * spi_res_alloc - allocate a spi resource that is life-cycle managed
889 * during the processing of a spi_message while using
890 * spi_transfer_one
891 * @spi: the spi device for which we allocate memory
892 * @release: the release code to execute for this resource
893 * @size: size to alloc and return
894 * @gfp: GFP allocation flags
895 *
896 * Return: the pointer to the allocated data
897 *
898 * This may get enhanced in the future to allocate from a memory pool
899 * of the @spi_device or @spi_controller to avoid repeated allocations.
900 */
da21fde0
UKK		 901static void *spi_res_alloc(struct spi_device *spi, spi_res_release_t release,
902 size_t size, gfp_t gfp)
fb51601b
UKK		 903{
904 struct spi_res *sres;
905
906 sres = kzalloc(sizeof(*sres) + size, gfp);
907 if (!sres)
908 return NULL;
909
910 INIT_LIST_HEAD(&sres->entry);
911 sres->release = release;
912
913 return sres->data;
914}
fb51601b
UKK		 915
916/**
917 * spi_res_free - free an spi resource
918 * @res: pointer to the custom data of a resource
fb51601b 919 */
da21fde0 920static void spi_res_free(void *res)
fb51601b
UKK		 921{
922 struct spi_res *sres = container_of(res, struct spi_res, data);
923
924 if (!res)
925 return;
926
927 WARN_ON(!list_empty(&sres->entry));
928 kfree(sres);
929}
fb51601b
UKK		 930
931/**
932 * spi_res_add - add a spi_res to the spi_message
933 * @message: the spi message
934 * @res: the spi_resource
935 */
da21fde0 936static void spi_res_add(struct spi_message *message, void *res)
fb51601b
UKK		 937{
938 struct spi_res *sres = container_of(res, struct spi_res, data);
939
940 WARN_ON(!list_empty(&sres->entry));
941 list_add_tail(&sres->entry, &message->resources);
942}
fb51601b
UKK		 943
944/**
945 * spi_res_release - release all spi resources for this message
946 * @ctlr: the @spi_controller
947 * @message: the @spi_message
948 */
da21fde0 949static void spi_res_release(struct spi_controller *ctlr, struct spi_message *message)
fb51601b
UKK		 950{
951 struct spi_res *res, *tmp;
952
953 list_for_each_entry_safe_reverse(res, tmp, &message->resources, entry) {
954 if (res->release)
955 res->release(ctlr, message, res->data);
956
957 list_del(&res->entry);
958
959 kfree(res);
960 }
961}
fb51601b
UKK		 962
963/*-------------------------------------------------------------------------*/
964
d347b4aa 965static void spi_set_cs(struct spi_device *spi, bool enable, bool force)
b158935f 966{
86527bcb 967 bool activate = enable;
25093bde 968
d40f0b6f
DA		 969 /*
970 * Avoid calling into the driver (or doing delays) if the chip select
971 * isn't actually changing from the last time this was called.
972 */
303feb3c
AKM		 973 if (!force && ((enable && spi->controller->last_cs == spi_get_chipselect(spi, 0)) ||
974 (!enable && spi->controller->last_cs != spi_get_chipselect(spi, 0))) &&
d40f0b6f
DA		 975 (spi->controller->last_cs_mode_high == (spi->mode & SPI_CS_HIGH)))
976 return;
977
5cb4e1f3
AS		 978 trace_spi_set_cs(spi, activate);
979
303feb3c 980 spi->controller->last_cs = enable ? spi_get_chipselect(spi, 0) : -1;
d40f0b6f
DA		 981 spi->controller->last_cs_mode_high = spi->mode & SPI_CS_HIGH;
982
303feb3c 983 if ((spi_get_csgpiod(spi, 0) || !spi->controller->set_cs_timing) && !activate)
95c07247 984 spi_delay_exec(&spi->cs_hold, NULL);
25093bde 985
b158935f
MB		 986 if (spi->mode & SPI_CS_HIGH)
987 enable = !enable;
988
303feb3c 989 if (spi_get_csgpiod(spi, 0)) {
f3186dd8 990 if (!(spi->mode & SPI_NO_CS)) {
f48dc6b9
LW		 991 /*
992 * Historically ACPI has no means of the GPIO polarity and
993 * thus the SPISerialBus() resource defines it on the per-chip
994 * basis. In order to avoid a chain of negations, the GPIO
995 * polarity is considered being Active High. Even for the cases
996 * when _DSD() is involved (in the updated versions of ACPI)
997 * the GPIO CS polarity must be defined Active High to avoid
998 * ambiguity. That's why we use enable, that takes SPI_CS_HIGH
999 * into account.
1000 */
1001 if (has_acpi_companion(&spi->dev))
303feb3c 1002 gpiod_set_value_cansleep(spi_get_csgpiod(spi, 0), !enable);
f48dc6b9
LW		 1003 else
1004 /* Polarity handled by GPIO library */
303feb3c 1005 gpiod_set_value_cansleep(spi_get_csgpiod(spi, 0), activate);
f3186dd8 1006 }
8eee6b9d 1007 /* Some SPI masters need both GPIO CS & slave_select */
8caab75f
GU		 1008 if ((spi->controller->flags & SPI_MASTER_GPIO_SS) &&
1009 spi->controller->set_cs)
1010 spi->controller->set_cs(spi, !enable);
1011 } else if (spi->controller->set_cs) {
1012 spi->controller->set_cs(spi, !enable);
8eee6b9d 1013 }
25093bde 1014
303feb3c 1015 if (spi_get_csgpiod(spi, 0) || !spi->controller->set_cs_timing) {
95c07247
HM		 1016 if (activate)
1017 spi_delay_exec(&spi->cs_setup, NULL);
1018 else
8c33ebfe 1019 spi_delay_exec(&spi->cs_inactive, NULL);
25093bde 1020 }
b158935f
MB		 1021}
1022
2de440f5 1023#ifdef CONFIG_HAS_DMA
0c17ba73
VW		 1024static int spi_map_buf_attrs(struct spi_controller *ctlr, struct device *dev,
1025 struct sg_table *sgt, void *buf, size_t len,
1026 enum dma_data_direction dir, unsigned long attrs)
6ad45a27
MB		 1027{
1028 const bool vmalloced_buf = is_vmalloc_addr(buf);
df88e91b 1029 unsigned int max_seg_size = dma_get_max_seg_size(dev);
b1b8153c
V		 1030#ifdef CONFIG_HIGHMEM
1031 const bool kmap_buf = ((unsigned long)buf >= PKMAP_BASE &&
1032 (unsigned long)buf < (PKMAP_BASE +
1033 (LAST_PKMAP * PAGE_SIZE)));
1034#else
1035 const bool kmap_buf = false;
1036#endif
65598c13
AG		 1037 int desc_len;
1038 int sgs;
6ad45a27 1039 struct page *vm_page;
8dd4a016 1040 struct scatterlist *sg;
6ad45a27
MB		 1041 void *sg_buf;
1042 size_t min;
1043 int i, ret;
1044
b1b8153c 1045 if (vmalloced_buf || kmap_buf) {
ebc4cb43 1046 desc_len = min_t(unsigned long, max_seg_size, PAGE_SIZE);
65598c13 1047 sgs = DIV_ROUND_UP(len + offset_in_page(buf), desc_len);
0569a88f 1048 } else if (virt_addr_valid(buf)) {
ebc4cb43 1049 desc_len = min_t(size_t, max_seg_size, ctlr->max_dma_len);
65598c13 1050 sgs = DIV_ROUND_UP(len, desc_len);
0569a88f
V		 1051 } else {
1052 return -EINVAL;
65598c13
AG		 1053 }
1054
6ad45a27
MB		 1055 ret = sg_alloc_table(sgt, sgs, GFP_KERNEL);
1056 if (ret != 0)
1057 return ret;
1058
8dd4a016 1059 sg = &sgt->sgl[0];
6ad45a27 1060 for (i = 0; i < sgs; i++) {
6ad45a27 1061
b1b8153c 1062 if (vmalloced_buf || kmap_buf) {
ce99319a
MC		 1063 /*
1064 * Next scatterlist entry size is the minimum between
1065 * the desc_len and the remaining buffer length that
1066 * fits in a page.
1067 */
1068 min = min_t(size_t, desc_len,
1069 min_t(size_t, len,
1070 PAGE_SIZE - offset_in_page(buf)));
b1b8153c
V		 1071 if (vmalloced_buf)
1072 vm_page = vmalloc_to_page(buf);
1073 else
1074 vm_page = kmap_to_page(buf);
6ad45a27
MB		 1075 if (!vm_page) {
1076 sg_free_table(sgt);
1077 return -ENOMEM;
1078 }
8dd4a016 1079 sg_set_page(sg, vm_page,
c1aefbdd 1080 min, offset_in_page(buf));
6ad45a27 1081 } else {
65598c13 1082 min = min_t(size_t, len, desc_len);
6ad45a27 1083 sg_buf = buf;
8dd4a016 1084 sg_set_buf(sg, sg_buf, min);
6ad45a27
MB		 1085 }
1086
6ad45a27
MB		 1087 buf += min;
1088 len -= min;
8dd4a016 1089 sg = sg_next(sg);
6ad45a27
MB		 1090 }
1091
0c17ba73 1092 ret = dma_map_sgtable(dev, sgt, dir, attrs);
6ad45a27
MB		 1093 if (ret < 0) {
1094 sg_free_table(sgt);
1095 return ret;
1096 }
1097
6ad45a27
MB		 1098 return 0;
1099}
1100
0c17ba73
VW		 1101int spi_map_buf(struct spi_controller *ctlr, struct device *dev,
1102 struct sg_table *sgt, void *buf, size_t len,
1103 enum dma_data_direction dir)
1104{
1105 return spi_map_buf_attrs(ctlr, dev, sgt, buf, len, dir, 0);
1106}
1107
1108static void spi_unmap_buf_attrs(struct spi_controller *ctlr,
1109 struct device *dev, struct sg_table *sgt,
1110 enum dma_data_direction dir,
1111 unsigned long attrs)
6ad45a27
MB		 1112{
1113 if (sgt->orig_nents) {
0c17ba73 1114 dma_unmap_sgtable(dev, sgt, dir, attrs);
6ad45a27 1115 sg_free_table(sgt);
8e9204cd
MS		 1116 sgt->orig_nents = 0;
1117 sgt->nents = 0;
6ad45a27
MB		 1118 }
1119}
1120
0c17ba73
VW		 1121void spi_unmap_buf(struct spi_controller *ctlr, struct device *dev,
1122 struct sg_table *sgt, enum dma_data_direction dir)
1123{
1124 spi_unmap_buf_attrs(ctlr, dev, sgt, dir, 0);
1125}
1126
8caab75f 1127static int __spi_map_msg(struct spi_controller *ctlr, struct spi_message *msg)
99adef31 1128{
99adef31
MB		 1129 struct device *tx_dev, *rx_dev;
1130 struct spi_transfer *xfer;
6ad45a27 1131 int ret;
3a2eba9b 1132
8caab75f 1133 if (!ctlr->can_dma)
99adef31
MB		 1134 return 0;
1135
8caab75f
GU		 1136 if (ctlr->dma_tx)
1137 tx_dev = ctlr->dma_tx->device->dev;
b470e10e
VK		 1138 else if (ctlr->dma_map_dev)
1139 tx_dev = ctlr->dma_map_dev;
c37f45b5 1140 else
8caab75f 1141 tx_dev = ctlr->dev.parent;
c37f45b5 1142
8caab75f
GU		 1143 if (ctlr->dma_rx)
1144 rx_dev = ctlr->dma_rx->device->dev;
b470e10e
VK		 1145 else if (ctlr->dma_map_dev)
1146 rx_dev = ctlr->dma_map_dev;
c37f45b5 1147 else
8caab75f 1148 rx_dev = ctlr->dev.parent;
99adef31
MB		 1149
1150 list_for_each_entry(xfer, &msg->transfers, transfer_list) {
0c17ba73
VW		 1151 /* The sync is done before each transfer. */
1152 unsigned long attrs = DMA_ATTR_SKIP_CPU_SYNC;
1153
8caab75f 1154 if (!ctlr->can_dma(ctlr, msg->spi, xfer))
99adef31
MB		 1155 continue;
1156
1157 if (xfer->tx_buf != NULL) {
0c17ba73
VW		 1158 ret = spi_map_buf_attrs(ctlr, tx_dev, &xfer->tx_sg,
1159 (void *)xfer->tx_buf,
1160 xfer->len, DMA_TO_DEVICE,
1161 attrs);
6ad45a27
MB		 1162 if (ret != 0)
1163 return ret;
99adef31
MB		 1164 }
1165
1166 if (xfer->rx_buf != NULL) {
0c17ba73
VW		 1167 ret = spi_map_buf_attrs(ctlr, rx_dev, &xfer->rx_sg,
1168 xfer->rx_buf, xfer->len,
1169 DMA_FROM_DEVICE, attrs);
6ad45a27 1170 if (ret != 0) {
0c17ba73
VW		 1171 spi_unmap_buf_attrs(ctlr, tx_dev,
1172 &xfer->tx_sg, DMA_TO_DEVICE,
1173 attrs);
1174
6ad45a27 1175 return ret;
99adef31
MB		 1176 }
1177 }
1178 }
1179
f25723dc
VW		 1180 ctlr->cur_rx_dma_dev = rx_dev;
1181 ctlr->cur_tx_dma_dev = tx_dev;
8caab75f 1182 ctlr->cur_msg_mapped = true;
99adef31
MB		 1183
1184 return 0;
1185}
1186
8caab75f 1187static int __spi_unmap_msg(struct spi_controller *ctlr, struct spi_message *msg)
99adef31 1188{
f25723dc
VW		 1189 struct device *rx_dev = ctlr->cur_rx_dma_dev;
1190 struct device *tx_dev = ctlr->cur_tx_dma_dev;
99adef31 1191 struct spi_transfer *xfer;
99adef31 1192
8caab75f 1193 if (!ctlr->cur_msg_mapped || !ctlr->can_dma)
99adef31
MB		 1194 return 0;
1195
99adef31 1196 list_for_each_entry(xfer, &msg->transfers, transfer_list) {
0c17ba73
VW		 1197 /* The sync has already been done after each transfer. */
1198 unsigned long attrs = DMA_ATTR_SKIP_CPU_SYNC;
1199
8caab75f 1200 if (!ctlr->can_dma(ctlr, msg->spi, xfer))
99adef31
MB		 1201 continue;
1202
0c17ba73
VW		 1203 spi_unmap_buf_attrs(ctlr, rx_dev, &xfer->rx_sg,
1204 DMA_FROM_DEVICE, attrs);
1205 spi_unmap_buf_attrs(ctlr, tx_dev, &xfer->tx_sg,
1206 DMA_TO_DEVICE, attrs);
99adef31
MB		 1207 }
1208
809b1b04
RG		 1209 ctlr->cur_msg_mapped = false;
1210
99adef31
MB		 1211 return 0;
1212}
0c17ba73
VW		 1213
1214static void spi_dma_sync_for_device(struct spi_controller *ctlr,
1215 struct spi_transfer *xfer)
1216{
1217 struct device *rx_dev = ctlr->cur_rx_dma_dev;
1218 struct device *tx_dev = ctlr->cur_tx_dma_dev;
1219
1220 if (!ctlr->cur_msg_mapped)
1221 return;
1222
1223 if (xfer->tx_sg.orig_nents)
1224 dma_sync_sgtable_for_device(tx_dev, &xfer->tx_sg, DMA_TO_DEVICE);
1225 if (xfer->rx_sg.orig_nents)
1226 dma_sync_sgtable_for_device(rx_dev, &xfer->rx_sg, DMA_FROM_DEVICE);
1227}
1228
1229static void spi_dma_sync_for_cpu(struct spi_controller *ctlr,
1230 struct spi_transfer *xfer)
1231{
1232 struct device *rx_dev = ctlr->cur_rx_dma_dev;
1233 struct device *tx_dev = ctlr->cur_tx_dma_dev;
1234
1235 if (!ctlr->cur_msg_mapped)
1236 return;
1237
1238 if (xfer->rx_sg.orig_nents)
1239 dma_sync_sgtable_for_cpu(rx_dev, &xfer->rx_sg, DMA_FROM_DEVICE);
1240 if (xfer->tx_sg.orig_nents)
1241 dma_sync_sgtable_for_cpu(tx_dev, &xfer->tx_sg, DMA_TO_DEVICE);
1242}
2de440f5 1243#else /* !CONFIG_HAS_DMA */
8caab75f 1244static inline int __spi_map_msg(struct spi_controller *ctlr,
2de440f5
GU		 1245 struct spi_message *msg)
1246{
1247 return 0;
1248}
1249
8caab75f 1250static inline int __spi_unmap_msg(struct spi_controller *ctlr,
4b786458 1251 struct spi_message *msg)
2de440f5
GU		 1252{
1253 return 0;
1254}
0c17ba73
VW		 1255
1256static void spi_dma_sync_for_device(struct spi_controller *ctrl,
1257 struct spi_transfer *xfer)
1258{
1259}
1260
1261static void spi_dma_sync_for_cpu(struct spi_controller *ctrl,
1262 struct spi_transfer *xfer)
1263{
1264}
2de440f5
GU		 1265#endif /* !CONFIG_HAS_DMA */
1266
8caab75f 1267static inline int spi_unmap_msg(struct spi_controller *ctlr,
4b786458
MS		 1268 struct spi_message *msg)
1269{
1270 struct spi_transfer *xfer;
1271
1272 list_for_each_entry(xfer, &msg->transfers, transfer_list) {
1273 /*
1274 * Restore the original value of tx_buf or rx_buf if they are
1275 * NULL.
1276 */
8caab75f 1277 if (xfer->tx_buf == ctlr->dummy_tx)
4b786458 1278 xfer->tx_buf = NULL;
8caab75f 1279 if (xfer->rx_buf == ctlr->dummy_rx)
4b786458
MS		 1280 xfer->rx_buf = NULL;
1281 }
1282
8caab75f 1283 return __spi_unmap_msg(ctlr, msg);
4b786458
MS		 1284}
1285
8caab75f 1286static int spi_map_msg(struct spi_controller *ctlr, struct spi_message *msg)
2de440f5
GU		 1287{
1288 struct spi_transfer *xfer;
1289 void *tmp;
1290 unsigned int max_tx, max_rx;
1291
aee67fe8 1292 if ((ctlr->flags & (SPI_CONTROLLER_MUST_RX | SPI_CONTROLLER_MUST_TX))
1293 && !(msg->spi->mode & SPI_3WIRE)) {
2de440f5
GU		 1294 max_tx = 0;
1295 max_rx = 0;
1296
1297 list_for_each_entry(xfer, &msg->transfers, transfer_list) {
8caab75f 1298 if ((ctlr->flags & SPI_CONTROLLER_MUST_TX) &&
2de440f5
GU		 1299 !xfer->tx_buf)
1300 max_tx = max(xfer->len, max_tx);
8caab75f 1301 if ((ctlr->flags & SPI_CONTROLLER_MUST_RX) &&
2de440f5
GU		 1302 !xfer->rx_buf)
1303 max_rx = max(xfer->len, max_rx);
1304 }
1305
1306 if (max_tx) {
8caab75f 1307 tmp = krealloc(ctlr->dummy_tx, max_tx,
b00bab9d 1308 GFP_KERNEL | GFP_DMA | __GFP_ZERO);
2de440f5
GU		 1309 if (!tmp)
1310 return -ENOMEM;
8caab75f 1311 ctlr->dummy_tx = tmp;
2de440f5
GU		 1312 }
1313
1314 if (max_rx) {
8caab75f 1315 tmp = krealloc(ctlr->dummy_rx, max_rx,
2de440f5
GU		 1316 GFP_KERNEL | GFP_DMA);
1317 if (!tmp)
1318 return -ENOMEM;
8caab75f 1319 ctlr->dummy_rx = tmp;
2de440f5
GU		 1320 }
1321
1322 if (max_tx || max_rx) {
1323 list_for_each_entry(xfer, &msg->transfers,
1324 transfer_list) {
5442dcaa
CL		 1325 if (!xfer->len)
1326 continue;
2de440f5 1327 if (!xfer->tx_buf)
8caab75f 1328 xfer->tx_buf = ctlr->dummy_tx;
2de440f5 1329 if (!xfer->rx_buf)
8caab75f 1330 xfer->rx_buf = ctlr->dummy_rx;
2de440f5
GU		 1331 }
1332 }
1333 }
1334
8caab75f 1335 return __spi_map_msg(ctlr, msg);
2de440f5 1336}
99adef31 1337
810923f3
LR		 1338static int spi_transfer_wait(struct spi_controller *ctlr,
1339 struct spi_message *msg,
1340 struct spi_transfer *xfer)
1341{
d501cc4c
DJ		 1342 struct spi_statistics __percpu *statm = ctlr->pcpu_statistics;
1343 struct spi_statistics __percpu *stats = msg->spi->pcpu_statistics;
6170d077 1344 u32 speed_hz = xfer->speed_hz;
49686df5 1345 unsigned long long ms;
810923f3
LR		 1346
1347 if (spi_controller_is_slave(ctlr)) {
1348 if (wait_for_completion_interruptible(&ctlr->xfer_completion)) {
1349 dev_dbg(&msg->spi->dev, "SPI transfer interrupted\n");
1350 return -EINTR;
1351 }
1352 } else {
6170d077
XY		 1353 if (!speed_hz)
1354 speed_hz = 100000;
1355
86b8bff7
AS		 1356 /*
1357 * For each byte we wait for 8 cycles of the SPI clock.
1358 * Since speed is defined in Hz and we want milliseconds,
1359 * use respective multiplier, but before the division,
1360 * otherwise we may get 0 for short transfers.
1361 */
1362 ms = 8LL * MSEC_PER_SEC * xfer->len;
6170d077 1363 do_div(ms, speed_hz);
810923f3 1364
86b8bff7
AS		 1365 /*
1366 * Increase it twice and add 200 ms tolerance, use
1367 * predefined maximum in case of overflow.
1368 */
1369 ms += ms + 200;
810923f3
LR		 1370 if (ms > UINT_MAX)
1371 ms = UINT_MAX;
1372
1373 ms = wait_for_completion_timeout(&ctlr->xfer_completion,
1374 msecs_to_jiffies(ms));
1375
1376 if (ms == 0) {
1377 SPI_STATISTICS_INCREMENT_FIELD(statm, timedout);
1378 SPI_STATISTICS_INCREMENT_FIELD(stats, timedout);
1379 dev_err(&msg->spi->dev,
1380 "SPI transfer timed out\n");
1381 return -ETIMEDOUT;
1382 }
1383 }
1384
1385 return 0;
1386}
1387
0ff2de8b
MS		 1388static void _spi_transfer_delay_ns(u32 ns)
1389{
1390 if (!ns)
1391 return;
86b8bff7 1392 if (ns <= NSEC_PER_USEC) {
0ff2de8b
MS		 1393 ndelay(ns);
1394 } else {
86b8bff7 1395 u32 us = DIV_ROUND_UP(ns, NSEC_PER_USEC);
0ff2de8b
MS		 1396
1397 if (us <= 10)
1398 udelay(us);
1399 else
1400 usleep_range(us, us + DIV_ROUND_UP(us, 10));
1401 }
1402}
1403
3984d39b 1404int spi_delay_to_ns(struct spi_delay *_delay, struct spi_transfer *xfer)
0ff2de8b 1405{
b2c98153
AA		 1406 u32 delay = _delay->value;
1407 u32 unit = _delay->unit;
d5864e5b 1408 u32 hz;
0ff2de8b 1409
b2c98153
AA		 1410 if (!delay)
1411 return 0;
0ff2de8b
MS		 1412
1413 switch (unit) {
1414 case SPI_DELAY_UNIT_USECS:
86b8bff7 1415 delay *= NSEC_PER_USEC;
0ff2de8b 1416 break;
86b8bff7
AS		 1417 case SPI_DELAY_UNIT_NSECS:
1418 /* Nothing to do here */
0ff2de8b 1419 break;
d5864e5b 1420 case SPI_DELAY_UNIT_SCK:
95c8222f 1421 /* Clock cycles need to be obtained from spi_transfer */
b2c98153
AA		 1422 if (!xfer)
1423 return -EINVAL;
86b8bff7
AS		 1424 /*
1425 * If there is unknown effective speed, approximate it
1426 * by underestimating with half of the requested hz.
d5864e5b
MS		 1427 */
1428 hz = xfer->effective_speed_hz ?: xfer->speed_hz / 2;
b2c98153
AA		 1429 if (!hz)
1430 return -EINVAL;
86b8bff7
AS		 1431
1432 /* Convert delay to nanoseconds */
1433 delay *= DIV_ROUND_UP(NSEC_PER_SEC, hz);
d5864e5b 1434 break;
0ff2de8b 1435 default:
b2c98153
AA		 1436 return -EINVAL;
1437 }
1438
1439 return delay;
1440}
3984d39b 1441EXPORT_SYMBOL_GPL(spi_delay_to_ns);
b2c98153
AA		 1442
1443int spi_delay_exec(struct spi_delay *_delay, struct spi_transfer *xfer)
1444{
1445 int delay;
1446
8fede89f
MB		 1447 might_sleep();
1448
b2c98153
AA		 1449 if (!_delay)
1450 return -EINVAL;
1451
3984d39b 1452 delay = spi_delay_to_ns(_delay, xfer);
b2c98153
AA		 1453 if (delay < 0)
1454 return delay;
1455
1456 _spi_transfer_delay_ns(delay);
1457
1458 return 0;
1459}
1460EXPORT_SYMBOL_GPL(spi_delay_exec);
1461
0ff2de8b
MS		 1462static void _spi_transfer_cs_change_delay(struct spi_message *msg,
1463 struct spi_transfer *xfer)
1464{
86b8bff7 1465 u32 default_delay_ns = 10 * NSEC_PER_USEC;
329f0dac
AA		 1466 u32 delay = xfer->cs_change_delay.value;
1467 u32 unit = xfer->cs_change_delay.unit;
1468 int ret;
0ff2de8b 1469
95c8222f 1470 /* Return early on "fast" mode - for everything but USECS */
6b3f236a
AA		 1471 if (!delay) {
1472 if (unit == SPI_DELAY_UNIT_USECS)
86b8bff7 1473 _spi_transfer_delay_ns(default_delay_ns);
0ff2de8b 1474 return;
6b3f236a 1475 }
0ff2de8b 1476
329f0dac
AA		 1477 ret = spi_delay_exec(&xfer->cs_change_delay, xfer);
1478 if (ret) {
0ff2de8b 1479 dev_err_once(&msg->spi->dev,
86b8bff7
AS		 1480 "Use of unsupported delay unit %i, using default of %luus\n",
1481 unit, default_delay_ns / NSEC_PER_USEC);
1482 _spi_transfer_delay_ns(default_delay_ns);
0ff2de8b 1483 }
0ff2de8b
MS		 1484}
1485
6e80133a
WZ		 1486void spi_transfer_cs_change_delay_exec(struct spi_message *msg,
1487 struct spi_transfer *xfer)
1488{
1489 _spi_transfer_cs_change_delay(msg, xfer);
1490}
1491EXPORT_SYMBOL_GPL(spi_transfer_cs_change_delay_exec);
1492
b158935f
MB		 1493/*
1494 * spi_transfer_one_message - Default implementation of transfer_one_message()
1495 *
1496 * This is a standard implementation of transfer_one_message() for
8ba811a7 1497 * drivers which implement a transfer_one() operation. It provides
b158935f
MB		 1498 * standard handling of delays and chip select management.
1499 */
8caab75f 1500static int spi_transfer_one_message(struct spi_controller *ctlr,
b158935f
MB		 1501 struct spi_message *msg)
1502{
1503 struct spi_transfer *xfer;
b158935f
MB		 1504 bool keep_cs = false;
1505 int ret = 0;
d501cc4c
DJ		 1506 struct spi_statistics __percpu *statm = ctlr->pcpu_statistics;
1507 struct spi_statistics __percpu *stats = msg->spi->pcpu_statistics;
b158935f 1508
5e0531f6
CL		 1509 xfer = list_first_entry(&msg->transfers, struct spi_transfer, transfer_list);
1510 spi_set_cs(msg->spi, !xfer->cs_off, false);
b158935f 1511
eca2ebc7
MS		 1512 SPI_STATISTICS_INCREMENT_FIELD(statm, messages);
1513 SPI_STATISTICS_INCREMENT_FIELD(stats, messages);
1514
b158935f
MB		 1515 list_for_each_entry(xfer, &msg->transfers, transfer_list) {
1516 trace_spi_transfer_start(msg, xfer);
1517
8caab75f
GU		 1518 spi_statistics_add_transfer_stats(statm, xfer, ctlr);
1519 spi_statistics_add_transfer_stats(stats, xfer, ctlr);
eca2ebc7 1520
b42faeee
VO		 1521 if (!ctlr->ptp_sts_supported) {
1522 xfer->ptp_sts_word_pre = 0;
1523 ptp_read_system_prets(xfer->ptp_sts);
1524 }
1525
b3063203 1526 if ((xfer->tx_buf || xfer->rx_buf) && xfer->len) {
8caab75f 1527 reinit_completion(&ctlr->xfer_completion);
b158935f 1528
809b1b04 1529fallback_pio:
0c17ba73 1530 spi_dma_sync_for_device(ctlr, xfer);
8caab75f 1531 ret = ctlr->transfer_one(ctlr, msg->spi, xfer);
38ec10f6 1532 if (ret < 0) {
0c17ba73
VW		 1533 spi_dma_sync_for_cpu(ctlr, xfer);
1534
809b1b04
RG		 1535 if (ctlr->cur_msg_mapped &&
1536 (xfer->error & SPI_TRANS_FAIL_NO_START)) {
1537 __spi_unmap_msg(ctlr, msg);
1538 ctlr->fallback = true;
1539 xfer->error &= ~SPI_TRANS_FAIL_NO_START;
1540 goto fallback_pio;
1541 }
1542
eca2ebc7
MS		 1543 SPI_STATISTICS_INCREMENT_FIELD(statm,
1544 errors);
1545 SPI_STATISTICS_INCREMENT_FIELD(stats,
1546 errors);
38ec10f6
MB		 1547 dev_err(&msg->spi->dev,
1548 "SPI transfer failed: %d\n", ret);
1549 goto out;
1550 }
b158935f 1551
d57e7960
MB		 1552 if (ret > 0) {
1553 ret = spi_transfer_wait(ctlr, msg, xfer);
1554 if (ret < 0)
1555 msg->status = ret;
1556 }
0c17ba73
VW		 1557
1558 spi_dma_sync_for_cpu(ctlr, xfer);
38ec10f6
MB		 1559 } else {
1560 if (xfer->len)
1561 dev_err(&msg->spi->dev,
1562 "Bufferless transfer has length %u\n",
1563 xfer->len);
13a42798 1564 }
b158935f 1565
b42faeee
VO		 1566 if (!ctlr->ptp_sts_supported) {
1567 ptp_read_system_postts(xfer->ptp_sts);
1568 xfer->ptp_sts_word_post = xfer->len;
1569 }
1570
b158935f
MB		 1571 trace_spi_transfer_stop(msg, xfer);
1572
1573 if (msg->status != -EINPROGRESS)
1574 goto out;
1575
bebcfd27 1576 spi_transfer_delay_exec(xfer);
b158935f
MB		 1577
1578 if (xfer->cs_change) {
1579 if (list_is_last(&xfer->transfer_list,
1580 &msg->transfers)) {
1581 keep_cs = true;
1582 } else {
5e0531f6
CL		 1583 if (!xfer->cs_off)
1584 spi_set_cs(msg->spi, false, false);
0ff2de8b 1585 _spi_transfer_cs_change_delay(msg, xfer);
5e0531f6
CL		 1586 if (!list_next_entry(xfer, transfer_list)->cs_off)
1587 spi_set_cs(msg->spi, true, false);
b158935f 1588 }
5e0531f6
CL		 1589 } else if (!list_is_last(&xfer->transfer_list, &msg->transfers) &&
1590 xfer->cs_off != list_next_entry(xfer, transfer_list)->cs_off) {
1591 spi_set_cs(msg->spi, xfer->cs_off, false);
b158935f
MB		 1592 }
1593
1594 msg->actual_length += xfer->len;
1595 }
1596
1597out:
1598 if (ret != 0 || !keep_cs)
d347b4aa 1599 spi_set_cs(msg->spi, false, false);
b158935f
MB		 1600
1601 if (msg->status == -EINPROGRESS)
1602 msg->status = ret;
1603
8caab75f
GU		 1604 if (msg->status && ctlr->handle_err)
1605 ctlr->handle_err(ctlr, msg);
b716c4ff 1606
0ed56252
MB		 1607 spi_finalize_current_message(ctlr);
1608
b158935f
MB		 1609 return ret;
1610}
1611
1612/**
1613 * spi_finalize_current_transfer - report completion of a transfer
8caab75f 1614 * @ctlr: the controller reporting completion
b158935f
MB		 1615 *
1616 * Called by SPI drivers using the core transfer_one_message()
1617 * implementation to notify it that the current interrupt driven
9e8f4882 1618 * transfer has finished and the next one may be scheduled.
b158935f 1619 */
8caab75f 1620void spi_finalize_current_transfer(struct spi_controller *ctlr)
b158935f 1621{
8caab75f 1622 complete(&ctlr->xfer_completion);
b158935f
MB		 1623}
1624EXPORT_SYMBOL_GPL(spi_finalize_current_transfer);
1625
e1268597
MB		 1626static void spi_idle_runtime_pm(struct spi_controller *ctlr)
1627{
1628 if (ctlr->auto_runtime_pm) {
1629 pm_runtime_mark_last_busy(ctlr->dev.parent);
1630 pm_runtime_put_autosuspend(ctlr->dev.parent);
1631 }
1632}
1633
ae7d2346
DJ		 1634static int __spi_pump_transfer_message(struct spi_controller *ctlr,
1635 struct spi_message *msg, bool was_busy)
1636{
1637 struct spi_transfer *xfer;
1638 int ret;
1639
1640 if (!was_busy && ctlr->auto_runtime_pm) {
1641 ret = pm_runtime_get_sync(ctlr->dev.parent);
1642 if (ret < 0) {
1643 pm_runtime_put_noidle(ctlr->dev.parent);
1644 dev_err(&ctlr->dev, "Failed to power device: %d\n",
1645 ret);
1646 return ret;
1647 }
1648 }
1649
1650 if (!was_busy)
1651 trace_spi_controller_busy(ctlr);
1652
1653 if (!was_busy && ctlr->prepare_transfer_hardware) {
1654 ret = ctlr->prepare_transfer_hardware(ctlr);
1655 if (ret) {
1656 dev_err(&ctlr->dev,
1657 "failed to prepare transfer hardware: %d\n",
1658 ret);
1659
1660 if (ctlr->auto_runtime_pm)
1661 pm_runtime_put(ctlr->dev.parent);
1662
1663 msg->status = ret;
1664 spi_finalize_current_message(ctlr);
1665
1666 return ret;
1667 }
1668 }
1669
1670 trace_spi_message_start(msg);
1671
8d699ff9
VW		 1672 ret = spi_split_transfers_maxsize(ctlr, msg,
1673 spi_max_transfer_size(msg->spi),
1674 GFP_KERNEL | GFP_DMA);
1675 if (ret) {
1676 msg->status = ret;
1677 spi_finalize_current_message(ctlr);
1678 return ret;
1679 }
1680
ae7d2346
DJ		 1681 if (ctlr->prepare_message) {
1682 ret = ctlr->prepare_message(ctlr, msg);
1683 if (ret) {
1684 dev_err(&ctlr->dev, "failed to prepare message: %d\n",
1685 ret);
1686 msg->status = ret;
1687 spi_finalize_current_message(ctlr);
1688 return ret;
1689 }
1690 msg->prepared = true;
1691 }
1692
1693 ret = spi_map_msg(ctlr, msg);
1694 if (ret) {
1695 msg->status = ret;
1696 spi_finalize_current_message(ctlr);
1697 return ret;
1698 }
1699
1700 if (!ctlr->ptp_sts_supported && !ctlr->transfer_one) {
1701 list_for_each_entry(xfer, &msg->transfers, transfer_list) {
1702 xfer->ptp_sts_word_pre = 0;
1703 ptp_read_system_prets(xfer->ptp_sts);
1704 }
1705 }
1706
dc302905
DJ		 1707 /*
1708 * Drivers implementation of transfer_one_message() must arrange for
1709 * spi_finalize_current_message() to get called. Most drivers will do
1710 * this in the calling context, but some don't. For those cases, a
1711 * completion is used to guarantee that this function does not return
1712 * until spi_finalize_current_message() is done accessing
1713 * ctlr->cur_msg.
1714 * Use of the following two flags enable to opportunistically skip the
1715 * use of the completion since its use involves expensive spin locks.
1716 * In case of a race with the context that calls
1717 * spi_finalize_current_message() the completion will always be used,
1718 * due to strict ordering of these flags using barriers.
1719 */
1720 WRITE_ONCE(ctlr->cur_msg_incomplete, true);
1721 WRITE_ONCE(ctlr->cur_msg_need_completion, false);
69fa9590 1722 reinit_completion(&ctlr->cur_msg_completion);
95c8222f 1723 smp_wmb(); /* Make these available to spi_finalize_current_message() */
dc302905 1724
ae7d2346
DJ		 1725 ret = ctlr->transfer_one_message(ctlr, msg);
1726 if (ret) {
1727 dev_err(&ctlr->dev,
1728 "failed to transfer one message from queue\n");
1729 return ret;
1730 }
1731
31d4c1bd
DJ		 1732 WRITE_ONCE(ctlr->cur_msg_need_completion, true);
1733 smp_mb(); /* See spi_finalize_current_message()... */
1734 if (READ_ONCE(ctlr->cur_msg_incomplete))
1735 wait_for_completion(&ctlr->cur_msg_completion);
1736
ae7d2346
DJ		 1737 return 0;
1738}
1739
ffbbdd21 1740/**
fc9e0f71 1741 * __spi_pump_messages - function which processes spi message queue
8caab75f 1742 * @ctlr: controller to process queue for
fc9e0f71 1743 * @in_kthread: true if we are in the context of the message pump thread
ffbbdd21
LW		 1744 *
1745 * This function checks if there is any spi message in the queue that
1746 * needs processing and if so call out to the driver to initialize hardware
1747 * and transfer each message.
1748 *
0461a414
MB		 1749 * Note that it is called both from the kthread itself and also from
1750 * inside spi_sync(); the queue extraction handling at the top of the
1751 * function should deal with this safely.
ffbbdd21 1752 */
8caab75f 1753static void __spi_pump_messages(struct spi_controller *ctlr, bool in_kthread)
ffbbdd21 1754{
d1c44c93 1755 struct spi_message *msg;
ffbbdd21 1756 bool was_busy = false;
d1c44c93 1757 unsigned long flags;
ffbbdd21
LW		 1758 int ret;
1759
c1038165
DJ		 1760 /* Take the IO mutex */
1761 mutex_lock(&ctlr->io_mutex);
1762
983aee5d 1763 /* Lock queue */
8caab75f 1764 spin_lock_irqsave(&ctlr->queue_lock, flags);
983aee5d
MB		 1765
1766 /* Make sure we are not already running a message */
8711a2ab 1767 if (ctlr->cur_msg)
c1038165 1768 goto out_unlock;
983aee5d
MB		 1769
1770 /* Check if the queue is idle */
8caab75f 1771 if (list_empty(&ctlr->queue) || !ctlr->running) {
8711a2ab 1772 if (!ctlr->busy)
c1038165 1773 goto out_unlock;
fc9e0f71 1774
e1268597 1775 /* Defer any non-atomic teardown to the thread */
f0125f1a 1776 if (!in_kthread) {
e1268597
MB		 1777 if (!ctlr->dummy_rx && !ctlr->dummy_tx &&
1778 !ctlr->unprepare_transfer_hardware) {
1779 spi_idle_runtime_pm(ctlr);
1780 ctlr->busy = false;
ae7d2346 1781 ctlr->queue_empty = true;
e1268597
MB		 1782 trace_spi_controller_idle(ctlr);
1783 } else {
1784 kthread_queue_work(ctlr->kworker,
1785 &ctlr->pump_messages);
1786 }
c1038165 1787 goto out_unlock;
f0125f1a
MB		 1788 }
1789
1790 ctlr->busy = false;
f0125f1a
MB		 1791 spin_unlock_irqrestore(&ctlr->queue_lock, flags);
1792
1793 kfree(ctlr->dummy_rx);
1794 ctlr->dummy_rx = NULL;
1795 kfree(ctlr->dummy_tx);
1796 ctlr->dummy_tx = NULL;
1797 if (ctlr->unprepare_transfer_hardware &&
1798 ctlr->unprepare_transfer_hardware(ctlr))
1799 dev_err(&ctlr->dev,
1800 "failed to unprepare transfer hardware\n");
e1268597 1801 spi_idle_runtime_pm(ctlr);
f0125f1a
MB		 1802 trace_spi_controller_idle(ctlr);
1803
1804 spin_lock_irqsave(&ctlr->queue_lock, flags);
ae7d2346 1805 ctlr->queue_empty = true;
c1038165 1806 goto out_unlock;
ffbbdd21 1807 }
ffbbdd21 1808
ffbbdd21 1809 /* Extract head of queue */
d1c44c93
VO		 1810 msg = list_first_entry(&ctlr->queue, struct spi_message, queue);
1811 ctlr->cur_msg = msg;
ffbbdd21 1812
d1c44c93 1813 list_del_init(&msg->queue);
8caab75f 1814 if (ctlr->busy)
ffbbdd21
LW		 1815 was_busy = true;
1816 else
8caab75f
GU		 1817 ctlr->busy = true;
1818 spin_unlock_irqrestore(&ctlr->queue_lock, flags);
ffbbdd21 1819
ae7d2346 1820 ret = __spi_pump_transfer_message(ctlr, msg, was_busy);
9c9c9da7 1821 kthread_queue_work(ctlr->kworker, &ctlr->pump_messages);
c191543e 1822
69fa9590
DJ		 1823 ctlr->cur_msg = NULL;
1824 ctlr->fallback = false;
1825
8caab75f 1826 mutex_unlock(&ctlr->io_mutex);
62826970
MB		 1827
1828 /* Prod the scheduler in case transfer_one() was busy waiting */
49023d2e
JH		 1829 if (!ret)
1830 cond_resched();
c1038165
DJ		 1831 return;
1832
1833out_unlock:
8711a2ab 1834 spin_unlock_irqrestore(&ctlr->queue_lock, flags);
c1038165 1835 mutex_unlock(&ctlr->io_mutex);
ffbbdd21
LW		 1836}
1837
fc9e0f71
MB		 1838/**
1839 * spi_pump_messages - kthread work function which processes spi message queue
8caab75f 1840 * @work: pointer to kthread work struct contained in the controller struct
fc9e0f71
MB		 1841 */
1842static void spi_pump_messages(struct kthread_work *work)
1843{
8caab75f
GU		 1844 struct spi_controller *ctlr =
1845 container_of(work, struct spi_controller, pump_messages);
fc9e0f71 1846
8caab75f 1847 __spi_pump_messages(ctlr, true);
fc9e0f71
MB		 1848}
1849
b42faeee 1850/**
350de7ce 1851 * spi_take_timestamp_pre - helper to collect the beginning of the TX timestamp
b42faeee
VO		 1852 * @ctlr: Pointer to the spi_controller structure of the driver
1853 * @xfer: Pointer to the transfer being timestamped
862dd2a9 1854 * @progress: How many words (not bytes) have been transferred so far
b42faeee
VO		 1855 * @irqs_off: If true, will disable IRQs and preemption for the duration of the
1856 * transfer, for less jitter in time measurement. Only compatible
1857 * with PIO drivers. If true, must follow up with
1858 * spi_take_timestamp_post or otherwise system will crash.
1859 * WARNING: for fully predictable results, the CPU frequency must
1860 * also be under control (governor).
350de7ce
AS		 1861 *
1862 * This is a helper for drivers to collect the beginning of the TX timestamp
1863 * for the requested byte from the SPI transfer. The frequency with which this
1864 * function must be called (once per word, once for the whole transfer, once
1865 * per batch of words etc) is arbitrary as long as the @tx buffer offset is
1866 * greater than or equal to the requested byte at the time of the call. The
1867 * timestamp is only taken once, at the first such call. It is assumed that
1868 * the driver advances its @tx buffer pointer monotonically.
b42faeee
VO		 1869 */
1870void spi_take_timestamp_pre(struct spi_controller *ctlr,
1871 struct spi_transfer *xfer,
862dd2a9 1872 size_t progress, bool irqs_off)
b42faeee 1873{
b42faeee
VO		 1874 if (!xfer->ptp_sts)
1875 return;
1876
6a726824 1877 if (xfer->timestamped)
b42faeee
VO		 1878 return;
1879
6a726824 1880 if (progress > xfer->ptp_sts_word_pre)
b42faeee
VO		 1881 return;
1882
1883 /* Capture the resolution of the timestamp */
862dd2a9 1884 xfer->ptp_sts_word_pre = progress;
b42faeee 1885
b42faeee
VO		 1886 if (irqs_off) {
1887 local_irq_save(ctlr->irq_flags);
1888 preempt_disable();
1889 }
1890
1891 ptp_read_system_prets(xfer->ptp_sts);
1892}
1893EXPORT_SYMBOL_GPL(spi_take_timestamp_pre);
1894
1895/**
350de7ce 1896 * spi_take_timestamp_post - helper to collect the end of the TX timestamp
b42faeee
VO		 1897 * @ctlr: Pointer to the spi_controller structure of the driver
1898 * @xfer: Pointer to the transfer being timestamped
862dd2a9 1899 * @progress: How many words (not bytes) have been transferred so far
b42faeee 1900 * @irqs_off: If true, will re-enable IRQs and preemption for the local CPU.
350de7ce
AS		 1901 *
1902 * This is a helper for drivers to collect the end of the TX timestamp for
1903 * the requested byte from the SPI transfer. Can be called with an arbitrary
1904 * frequency: only the first call where @tx exceeds or is equal to the
1905 * requested word will be timestamped.
b42faeee
VO		 1906 */
1907void spi_take_timestamp_post(struct spi_controller *ctlr,
1908 struct spi_transfer *xfer,
862dd2a9 1909 size_t progress, bool irqs_off)
b42faeee 1910{
b42faeee
VO		 1911 if (!xfer->ptp_sts)
1912 return;
1913
6a726824 1914 if (xfer->timestamped)
b42faeee
VO		 1915 return;
1916
862dd2a9 1917 if (progress < xfer->ptp_sts_word_post)
b42faeee
VO		 1918 return;
1919
1920 ptp_read_system_postts(xfer->ptp_sts);
1921
1922 if (irqs_off) {
1923 local_irq_restore(ctlr->irq_flags);
1924 preempt_enable();
1925 }
1926
1927 /* Capture the resolution of the timestamp */
862dd2a9 1928 xfer->ptp_sts_word_post = progress;
b42faeee 1929
6a726824 1930 xfer->timestamped = true;
b42faeee
VO		 1931}
1932EXPORT_SYMBOL_GPL(spi_take_timestamp_post);
1933
924b5867
DA		 1934/**
1935 * spi_set_thread_rt - set the controller to pump at realtime priority
1936 * @ctlr: controller to boost priority of
1937 *
1938 * This can be called because the controller requested realtime priority
1939 * (by setting the ->rt value before calling spi_register_controller()) or
1940 * because a device on the bus said that its transfers needed realtime
1941 * priority.
1942 *
1943 * NOTE: at the moment if any device on a bus says it needs realtime then
1944 * the thread will be at realtime priority for all transfers on that
1945 * controller. If this eventually becomes a problem we may see if we can
1946 * find a way to boost the priority only temporarily during relevant
1947 * transfers.
1948 */
1949static void spi_set_thread_rt(struct spi_controller *ctlr)
ffbbdd21 1950{
924b5867
DA		 1951 dev_info(&ctlr->dev,
1952 "will run message pump with realtime priority\n");
6d2b84a4 1953 sched_set_fifo(ctlr->kworker->task);
924b5867
DA		 1954}
1955
1956static int spi_init_queue(struct spi_controller *ctlr)
1957{
8caab75f
GU		 1958 ctlr->running = false;
1959 ctlr->busy = false;
ae7d2346 1960 ctlr->queue_empty = true;
ffbbdd21 1961
60a883d1
MS		 1962 ctlr->kworker = kthread_create_worker(0, dev_name(&ctlr->dev));
1963 if (IS_ERR(ctlr->kworker)) {
1964 dev_err(&ctlr->dev, "failed to create message pump kworker\n");
1965 return PTR_ERR(ctlr->kworker);
ffbbdd21 1966 }
60a883d1 1967
8caab75f 1968 kthread_init_work(&ctlr->pump_messages, spi_pump_messages);
f0125f1a 1969
ffbbdd21 1970 /*
8caab75f 1971 * Controller config will indicate if this controller should run the
ffbbdd21
LW		 1972 * message pump with high (realtime) priority to reduce the transfer
1973 * latency on the bus by minimising the delay between a transfer
1974 * request and the scheduling of the message pump thread. Without this
1975 * setting the message pump thread will remain at default priority.
1976 */
924b5867
DA		 1977 if (ctlr->rt)
1978 spi_set_thread_rt(ctlr);
ffbbdd21
LW		 1979
1980 return 0;
1981}
1982
1983/**
1984 * spi_get_next_queued_message() - called by driver to check for queued
1985 * messages
8caab75f 1986 * @ctlr: the controller to check for queued messages
ffbbdd21
LW		 1987 *
1988 * If there are more messages in the queue, the next message is returned from
1989 * this call.
97d56dc6
JMC		 1990 *
1991 * Return: the next message in the queue, else NULL if the queue is empty.
ffbbdd21 1992 */
8caab75f 1993struct spi_message *spi_get_next_queued_message(struct spi_controller *ctlr)
ffbbdd21
LW		 1994{
1995 struct spi_message *next;
1996 unsigned long flags;
1997
95c8222f 1998 /* Get a pointer to the next message, if any */
8caab75f
GU		 1999 spin_lock_irqsave(&ctlr->queue_lock, flags);
2000 next = list_first_entry_or_null(&ctlr->queue, struct spi_message,
1cfd97f9 2001 queue);
8caab75f 2002 spin_unlock_irqrestore(&ctlr->queue_lock, flags);
ffbbdd21
LW		 2003
2004 return next;
2005}
2006EXPORT_SYMBOL_GPL(spi_get_next_queued_message);
2007
2008/**
2009 * spi_finalize_current_message() - the current message is complete
8caab75f 2010 * @ctlr: the controller to return the message to
ffbbdd21
LW		 2011 *
2012 * Called by the driver to notify the core that the message in the front of the
2013 * queue is complete and can be removed from the queue.
2014 */
8caab75f 2015void spi_finalize_current_message(struct spi_controller *ctlr)
ffbbdd21 2016{
b42faeee 2017 struct spi_transfer *xfer;
ffbbdd21 2018 struct spi_message *mesg;
2841a5fc 2019 int ret;
ffbbdd21 2020
8caab75f 2021 mesg = ctlr->cur_msg;
ffbbdd21 2022
b42faeee
VO		 2023 if (!ctlr->ptp_sts_supported && !ctlr->transfer_one) {
2024 list_for_each_entry(xfer, &mesg->transfers, transfer_list) {
2025 ptp_read_system_postts(xfer->ptp_sts);
2026 xfer->ptp_sts_word_post = xfer->len;
2027 }
2028 }
2029
6a726824
VO		 2030 if (unlikely(ctlr->ptp_sts_supported))
2031 list_for_each_entry(xfer, &mesg->transfers, transfer_list)
2032 WARN_ON_ONCE(xfer->ptp_sts && !xfer->timestamped);
f971a207 2033
8caab75f 2034 spi_unmap_msg(ctlr, mesg);
99adef31 2035
350de7ce
AS		 2036 /*
2037 * In the prepare_messages callback the SPI bus has the opportunity
2038 * to split a transfer to smaller chunks.
2039 *
2040 * Release the split transfers here since spi_map_msg() is done on
2041 * the split transfers.
b59a7ca1
GW		 2042 */
2043 spi_res_release(ctlr, mesg);
2044
1714582a 2045 if (mesg->prepared && ctlr->unprepare_message) {
8caab75f 2046 ret = ctlr->unprepare_message(ctlr, mesg);
2841a5fc 2047 if (ret) {
8caab75f
GU		 2048 dev_err(&ctlr->dev, "failed to unprepare message: %d\n",
2049 ret);
2841a5fc
MB		 2050 }
2051 }
391949b6 2052
1714582a
DJ		 2053 mesg->prepared = false;
2054
dc302905
DJ		 2055 WRITE_ONCE(ctlr->cur_msg_incomplete, false);
2056 smp_mb(); /* See __spi_pump_transfer_message()... */
2057 if (READ_ONCE(ctlr->cur_msg_need_completion))
2058 complete(&ctlr->cur_msg_completion);
8e76ef88
MS		 2059
2060 trace_spi_message_done(mesg);
2841a5fc 2061
ffbbdd21
LW		 2062 mesg->state = NULL;
2063 if (mesg->complete)
2064 mesg->complete(mesg->context);
2065}
2066EXPORT_SYMBOL_GPL(spi_finalize_current_message);
2067
8caab75f 2068static int spi_start_queue(struct spi_controller *ctlr)
ffbbdd21
LW		 2069{
2070 unsigned long flags;
2071
8caab75f 2072 spin_lock_irqsave(&ctlr->queue_lock, flags);
ffbbdd21 2073
8caab75f
GU		 2074 if (ctlr->running || ctlr->busy) {
2075 spin_unlock_irqrestore(&ctlr->queue_lock, flags);
ffbbdd21
LW		 2076 return -EBUSY;
2077 }
2078
8caab75f
GU		 2079 ctlr->running = true;
2080 ctlr->cur_msg = NULL;
2081 spin_unlock_irqrestore(&ctlr->queue_lock, flags);
ffbbdd21 2082
60a883d1 2083 kthread_queue_work(ctlr->kworker, &ctlr->pump_messages);
ffbbdd21
LW		 2084
2085 return 0;
2086}
2087
8caab75f 2088static int spi_stop_queue(struct spi_controller *ctlr)
ffbbdd21
LW		 2089{
2090 unsigned long flags;
2091 unsigned limit = 500;
2092 int ret = 0;
2093
8caab75f 2094 spin_lock_irqsave(&ctlr->queue_lock, flags);
ffbbdd21
LW		 2095
2096 /*
2097 * This is a bit lame, but is optimized for the common execution path.
8caab75f 2098 * A wait_queue on the ctlr->busy could be used, but then the common
ffbbdd21
LW		 2099 * execution path (pump_messages) would be required to call wake_up or
2100 * friends on every SPI message. Do this instead.
2101 */
8caab75f
GU		 2102 while ((!list_empty(&ctlr->queue) || ctlr->busy) && limit--) {
2103 spin_unlock_irqrestore(&ctlr->queue_lock, flags);
f97b26b0 2104 usleep_range(10000, 11000);
8caab75f 2105 spin_lock_irqsave(&ctlr->queue_lock, flags);
ffbbdd21
LW		 2106 }
2107
8caab75f 2108 if (!list_empty(&ctlr->queue) || ctlr->busy)
ffbbdd21
LW		 2109 ret = -EBUSY;
2110 else
8caab75f 2111 ctlr->running = false;
ffbbdd21 2112
8caab75f 2113 spin_unlock_irqrestore(&ctlr->queue_lock, flags);
ffbbdd21
LW		 2114
2115 if (ret) {
8caab75f 2116 dev_warn(&ctlr->dev, "could not stop message queue\n");
ffbbdd21
LW		 2117 return ret;
2118 }
2119 return ret;
2120}
2121
8caab75f 2122static int spi_destroy_queue(struct spi_controller *ctlr)
ffbbdd21
LW		 2123{
2124 int ret;
2125
8caab75f 2126 ret = spi_stop_queue(ctlr);
ffbbdd21
LW		 2127
2128 /*
3989144f 2129 * kthread_flush_worker will block until all work is done.
ffbbdd21
LW		 2130 * If the reason that stop_queue timed out is that the work will never
2131 * finish, then it does no good to call flush/stop thread, so
2132 * return anyway.
2133 */
2134 if (ret) {
8caab75f 2135 dev_err(&ctlr->dev, "problem destroying queue\n");
ffbbdd21
LW		 2136 return ret;
2137 }
2138
60a883d1 2139 kthread_destroy_worker(ctlr->kworker);
ffbbdd21
LW		 2140
2141 return 0;
2142}
2143
0461a414
MB		 2144static int __spi_queued_transfer(struct spi_device *spi,
2145 struct spi_message *msg,
2146 bool need_pump)
ffbbdd21 2147{
8caab75f 2148 struct spi_controller *ctlr = spi->controller;
ffbbdd21
LW		 2149 unsigned long flags;
2150
8caab75f 2151 spin_lock_irqsave(&ctlr->queue_lock, flags);
ffbbdd21 2152
8caab75f
GU		 2153 if (!ctlr->running) {
2154 spin_unlock_irqrestore(&ctlr->queue_lock, flags);
ffbbdd21
LW		 2155 return -ESHUTDOWN;
2156 }
2157 msg->actual_length = 0;
2158 msg->status = -EINPROGRESS;
2159
8caab75f 2160 list_add_tail(&msg->queue, &ctlr->queue);
ae7d2346 2161 ctlr->queue_empty = false;
f0125f1a 2162 if (!ctlr->busy && need_pump)
60a883d1 2163 kthread_queue_work(ctlr->kworker, &ctlr->pump_messages);
ffbbdd21 2164
8caab75f 2165 spin_unlock_irqrestore(&ctlr->queue_lock, flags);
ffbbdd21
LW		 2166 return 0;
2167}
2168
0461a414
MB		 2169/**
2170 * spi_queued_transfer - transfer function for queued transfers
2171 * @spi: spi device which is requesting transfer
2172 * @msg: spi message which is to handled is queued to driver queue
97d56dc6
JMC		 2173 *
2174 * Return: zero on success, else a negative error code.
0461a414
MB		 2175 */
2176static int spi_queued_transfer(struct spi_device *spi, struct spi_message *msg)
2177{
2178 return __spi_queued_transfer(spi, msg, true);
2179}
2180
8caab75f 2181static int spi_controller_initialize_queue(struct spi_controller *ctlr)
ffbbdd21
LW		 2182{
2183 int ret;
2184
8caab75f
GU		 2185 ctlr->transfer = spi_queued_transfer;
2186 if (!ctlr->transfer_one_message)
2187 ctlr->transfer_one_message = spi_transfer_one_message;
ffbbdd21
LW		 2188
2189 /* Initialize and start queue */
8caab75f 2190 ret = spi_init_queue(ctlr);
ffbbdd21 2191 if (ret) {
8caab75f 2192 dev_err(&ctlr->dev, "problem initializing queue\n");
ffbbdd21
LW		 2193 goto err_init_queue;
2194 }
8caab75f
GU		 2195 ctlr->queued = true;
2196 ret = spi_start_queue(ctlr);
ffbbdd21 2197 if (ret) {
8caab75f 2198 dev_err(&ctlr->dev, "problem starting queue\n");
ffbbdd21
LW		 2199 goto err_start_queue;
2200 }
2201
2202 return 0;
2203
2204err_start_queue:
8caab75f 2205 spi_destroy_queue(ctlr);
c3676d5c 2206err_init_queue:
ffbbdd21
LW		 2207 return ret;
2208}
2209
988f259b
BB		 2210/**
2211 * spi_flush_queue - Send all pending messages in the queue from the callers'
2212 * context
2213 * @ctlr: controller to process queue for
2214 *
2215 * This should be used when one wants to ensure all pending messages have been
2216 * sent before doing something. Is used by the spi-mem code to make sure SPI
2217 * memory operations do not preempt regular SPI transfers that have been queued
2218 * before the spi-mem operation.
2219 */
2220void spi_flush_queue(struct spi_controller *ctlr)
2221{
2222 if (ctlr->transfer == spi_queued_transfer)
2223 __spi_pump_messages(ctlr, false);
2224}
2225
ffbbdd21
LW		 2226/*-------------------------------------------------------------------------*/
2227
7cb94361 2228#if defined(CONFIG_OF)
f276aacf
JG		 2229static void of_spi_parse_dt_cs_delay(struct device_node *nc,
2230 struct spi_delay *delay, const char *prop)
2231{
2232 u32 value;
2233
2234 if (!of_property_read_u32(nc, prop, &value)) {
2235 if (value > U16_MAX) {
2236 delay->value = DIV_ROUND_UP(value, 1000);
2237 delay->unit = SPI_DELAY_UNIT_USECS;
2238 } else {
2239 delay->value = value;
2240 delay->unit = SPI_DELAY_UNIT_NSECS;
2241 }
2242 }
2243}
2244
8caab75f 2245static int of_spi_parse_dt(struct spi_controller *ctlr, struct spi_device *spi,
c2e51ac3 2246 struct device_node *nc)
aff5e3f8 2247{
aff5e3f8 2248 u32 value;
c2e51ac3 2249 int rc;
aff5e3f8 2250
aff5e3f8 2251 /* Mode (clock phase/polarity/etc.) */
e0bcb680 2252 if (of_property_read_bool(nc, "spi-cpha"))
aff5e3f8 2253 spi->mode |= SPI_CPHA;
e0bcb680 2254 if (of_property_read_bool(nc, "spi-cpol"))
aff5e3f8 2255 spi->mode |= SPI_CPOL;
e0bcb680 2256 if (of_property_read_bool(nc, "spi-3wire"))
aff5e3f8 2257 spi->mode |= SPI_3WIRE;
e0bcb680 2258 if (of_property_read_bool(nc, "spi-lsb-first"))
aff5e3f8 2259 spi->mode |= SPI_LSB_FIRST;
3e5ec1db 2260 if (of_property_read_bool(nc, "spi-cs-high"))
f3186dd8
LW		 2261 spi->mode |= SPI_CS_HIGH;
2262
aff5e3f8
PA		 2263 /* Device DUAL/QUAD mode */
2264 if (!of_property_read_u32(nc, "spi-tx-bus-width", &value)) {
2265 switch (value) {
d962608c
DB		 2266 case 0:
2267 spi->mode |= SPI_NO_TX;
2268 break;
aff5e3f8
PA		 2269 case 1:
2270 break;
2271 case 2:
2272 spi->mode |= SPI_TX_DUAL;
2273 break;
2274 case 4:
2275 spi->mode |= SPI_TX_QUAD;
2276 break;
6b03061f
YNG		 2277 case 8:
2278 spi->mode |= SPI_TX_OCTAL;
2279 break;
aff5e3f8 2280 default:
8caab75f 2281 dev_warn(&ctlr->dev,
aff5e3f8
PA		 2282 "spi-tx-bus-width %d not supported\n",
2283 value);
2284 break;
2285 }
2286 }
2287
2288 if (!of_property_read_u32(nc, "spi-rx-bus-width", &value)) {
2289 switch (value) {
d962608c
DB		 2290 case 0:
2291 spi->mode |= SPI_NO_RX;
2292 break;
aff5e3f8
PA		 2293 case 1:
2294 break;
2295 case 2:
2296 spi->mode |= SPI_RX_DUAL;
2297 break;
2298 case 4:
2299 spi->mode |= SPI_RX_QUAD;
2300 break;
6b03061f
YNG		 2301 case 8:
2302 spi->mode |= SPI_RX_OCTAL;
2303 break;
aff5e3f8 2304 default:
8caab75f 2305 dev_warn(&ctlr->dev,
aff5e3f8
PA		 2306 "spi-rx-bus-width %d not supported\n",
2307 value);
2308 break;
2309 }
2310 }
2311
8caab75f 2312 if (spi_controller_is_slave(ctlr)) {
194276b0 2313 if (!of_node_name_eq(nc, "slave")) {
25c56c88
RH		 2314 dev_err(&ctlr->dev, "%pOF is not called 'slave'\n",
2315 nc);
6c364062
GU		 2316 return -EINVAL;
2317 }
2318 return 0;
2319 }
2320
2321 /* Device address */
2322 rc = of_property_read_u32(nc, "reg", &value);
2323 if (rc) {
25c56c88
RH		 2324 dev_err(&ctlr->dev, "%pOF has no valid 'reg' property (%d)\n",
2325 nc, rc);
6c364062
GU		 2326 return rc;
2327 }
303feb3c 2328 spi_set_chipselect(spi, 0, value);
6c364062 2329
aff5e3f8 2330 /* Device speed */
671c3bf5
CG		 2331 if (!of_property_read_u32(nc, "spi-max-frequency", &value))
2332 spi->max_speed_hz = value;
aff5e3f8 2333
f276aacf
JG		 2334 /* Device CS delays */
2335 of_spi_parse_dt_cs_delay(nc, &spi->cs_setup, "spi-cs-setup-delay-ns");
5827b31d
JG		 2336 of_spi_parse_dt_cs_delay(nc, &spi->cs_hold, "spi-cs-hold-delay-ns");
2337 of_spi_parse_dt_cs_delay(nc, &spi->cs_inactive, "spi-cs-inactive-delay-ns");
33a2fde5 2338
c2e51ac3
GU		 2339 return 0;
2340}
2341
2342static struct spi_device *
8caab75f 2343of_register_spi_device(struct spi_controller *ctlr, struct device_node *nc)
c2e51ac3
GU		 2344{
2345 struct spi_device *spi;
2346 int rc;
2347
2348 /* Alloc an spi_device */
8caab75f 2349 spi = spi_alloc_device(ctlr);
c2e51ac3 2350 if (!spi) {
25c56c88 2351 dev_err(&ctlr->dev, "spi_device alloc error for %pOF\n", nc);
c2e51ac3
GU		 2352 rc = -ENOMEM;
2353 goto err_out;
2354 }
2355
2356 /* Select device driver */
2357 rc = of_modalias_node(nc, spi->modalias,
2358 sizeof(spi->modalias));
2359 if (rc < 0) {
25c56c88 2360 dev_err(&ctlr->dev, "cannot find modalias for %pOF\n", nc);
c2e51ac3
GU		 2361 goto err_out;
2362 }
2363
8caab75f 2364 rc = of_spi_parse_dt(ctlr, spi, nc);
c2e51ac3
GU		 2365 if (rc)
2366 goto err_out;
2367
aff5e3f8
PA		 2368 /* Store a pointer to the node in the device structure */
2369 of_node_get(nc);
2370 spi->dev.of_node = nc;
0e793ba7 2371 spi->dev.fwnode = of_fwnode_handle(nc);
aff5e3f8
PA		 2372
2373 /* Register the new device */
aff5e3f8
PA		 2374 rc = spi_add_device(spi);
2375 if (rc) {
25c56c88 2376 dev_err(&ctlr->dev, "spi_device register error %pOF\n", nc);
8324147f 2377 goto err_of_node_put;
aff5e3f8
PA		 2378 }
2379
2380 return spi;
2381
8324147f
JH		 2382err_of_node_put:
2383 of_node_put(nc);
aff5e3f8
PA		 2384err_out:
2385 spi_dev_put(spi);
2386 return ERR_PTR(rc);
2387}
2388
d57a4282
GL		 2389/**
2390 * of_register_spi_devices() - Register child devices onto the SPI bus
8caab75f 2391 * @ctlr: Pointer to spi_controller device
d57a4282 2392 *
6c364062
GU		 2393 * Registers an spi_device for each child node of controller node which
2394 * represents a valid SPI slave.
d57a4282 2395 */
8caab75f 2396static void of_register_spi_devices(struct spi_controller *ctlr)
d57a4282
GL		 2397{
2398 struct spi_device *spi;
2399 struct device_node *nc;
d57a4282 2400
8caab75f 2401 if (!ctlr->dev.of_node)
d57a4282
GL		 2402 return;
2403
8caab75f 2404 for_each_available_child_of_node(ctlr->dev.of_node, nc) {
bd6c1644
GU		 2405 if (of_node_test_and_set_flag(nc, OF_POPULATED))
2406 continue;
8caab75f 2407 spi = of_register_spi_device(ctlr, nc);
e0af98a7 2408 if (IS_ERR(spi)) {
8caab75f 2409 dev_warn(&ctlr->dev,
25c56c88 2410 "Failed to create SPI device for %pOF\n", nc);
e0af98a7
RR		 2411 of_node_clear_flag(nc, OF_POPULATED);
2412 }
d57a4282
GL		 2413 }
2414}
2415#else
8caab75f 2416static void of_register_spi_devices(struct spi_controller *ctlr) { }
d57a4282
GL		 2417#endif
2418
0c79378c
SR		 2419/**
2420 * spi_new_ancillary_device() - Register ancillary SPI device
2421 * @spi: Pointer to the main SPI device registering the ancillary device
2422 * @chip_select: Chip Select of the ancillary device
2423 *
2424 * Register an ancillary SPI device; for example some chips have a chip-select
2425 * for normal device usage and another one for setup/firmware upload.
2426 *
2427 * This may only be called from main SPI device's probe routine.
2428 *
2429 * Return: 0 on success; negative errno on failure
2430 */
2431struct spi_device *spi_new_ancillary_device(struct spi_device *spi,
2432 u8 chip_select)
2433{
2434 struct spi_device *ancillary;
2435 int rc = 0;
2436
2437 /* Alloc an spi_device */
2438 ancillary = spi_alloc_device(spi->controller);
2439 if (!ancillary) {
2440 rc = -ENOMEM;
2441 goto err_out;
2442 }
2443
51e99de5 2444 strscpy(ancillary->modalias, "dummy", sizeof(ancillary->modalias));
0c79378c
SR		 2445
2446 /* Use provided chip-select for ancillary device */
303feb3c 2447 spi_set_chipselect(ancillary, 0, chip_select);
0c79378c
SR		 2448
2449 /* Take over SPI mode/speed from SPI main device */
2450 ancillary->max_speed_hz = spi->max_speed_hz;
b01d5506 2451 ancillary->mode = spi->mode;
0c79378c
SR		 2452
2453 /* Register the new device */
2454 rc = spi_add_device_locked(ancillary);
2455 if (rc) {
2456 dev_err(&spi->dev, "failed to register ancillary device\n");
2457 goto err_out;
2458 }
2459
2460 return ancillary;
2461
2462err_out:
2463 spi_dev_put(ancillary);
2464 return ERR_PTR(rc);
2465}
2466EXPORT_SYMBOL_GPL(spi_new_ancillary_device);
2467
64bee4d2 2468#ifdef CONFIG_ACPI
4c3c5954
AB		 2469struct acpi_spi_lookup {
2470 struct spi_controller *ctlr;
2471 u32 max_speed_hz;
2472 u32 mode;
2473 int irq;
2474 u8 bits_per_word;
2475 u8 chip_select;
87e59b36
SB		 2476 int n;
2477 int index;
4c3c5954
AB		 2478};
2479
e612af7a
SB		 2480static int acpi_spi_count(struct acpi_resource *ares, void *data)
2481{
2482 struct acpi_resource_spi_serialbus *sb;
2483 int *count = data;
2484
2485 if (ares->type != ACPI_RESOURCE_TYPE_SERIAL_BUS)
2486 return 1;
2487
2488 sb = &ares->data.spi_serial_bus;
2489 if (sb->type != ACPI_RESOURCE_SERIAL_TYPE_SPI)
2490 return 1;
2491
2492 *count = *count + 1;
2493
2494 return 1;
2495}
2496
2497/**
2498 * acpi_spi_count_resources - Count the number of SpiSerialBus resources
2499 * @adev: ACPI device
2500 *
2501 * Returns the number of SpiSerialBus resources in the ACPI-device's
2502 * resource-list; or a negative error code.
2503 */
2504int acpi_spi_count_resources(struct acpi_device *adev)
2505{
2506 LIST_HEAD(r);
2507 int count = 0;
2508 int ret;
2509
2510 ret = acpi_dev_get_resources(adev, &r, acpi_spi_count, &count);
2511 if (ret < 0)
2512 return ret;
2513
2514 acpi_dev_free_resource_list(&r);
2515
2516 return count;
2517}
2518EXPORT_SYMBOL_GPL(acpi_spi_count_resources);
2519
4c3c5954
AB		 2520static void acpi_spi_parse_apple_properties(struct acpi_device *dev,
2521 struct acpi_spi_lookup *lookup)
8a2e487e 2522{
8a2e487e
LW		 2523 const union acpi_object *obj;
2524
2525 if (!x86_apple_machine)
2526 return;
2527
2528 if (!acpi_dev_get_property(dev, "spiSclkPeriod", ACPI_TYPE_BUFFER, &obj)
2529 && obj->buffer.length >= 4)
4c3c5954 2530 lookup->max_speed_hz = NSEC_PER_SEC / *(u32 *)obj->buffer.pointer;
8a2e487e
LW		 2531
2532 if (!acpi_dev_get_property(dev, "spiWordSize", ACPI_TYPE_BUFFER, &obj)
2533 && obj->buffer.length == 8)
4c3c5954 2534 lookup->bits_per_word = *(u64 *)obj->buffer.pointer;
8a2e487e
LW		 2535
2536 if (!acpi_dev_get_property(dev, "spiBitOrder", ACPI_TYPE_BUFFER, &obj)
2537 && obj->buffer.length == 8 && !*(u64 *)obj->buffer.pointer)
4c3c5954 2538 lookup->mode |= SPI_LSB_FIRST;
8a2e487e
LW		 2539
2540 if (!acpi_dev_get_property(dev, "spiSPO", ACPI_TYPE_BUFFER, &obj)
2541 && obj->buffer.length == 8 && *(u64 *)obj->buffer.pointer)
4c3c5954 2542 lookup->mode |= SPI_CPOL;
8a2e487e
LW		 2543
2544 if (!acpi_dev_get_property(dev, "spiSPH", ACPI_TYPE_BUFFER, &obj)
2545 && obj->buffer.length == 8 && *(u64 *)obj->buffer.pointer)
4c3c5954 2546 lookup->mode |= SPI_CPHA;
8a2e487e
LW		 2547}
2548
87e59b36
SB		 2549static struct spi_controller *acpi_spi_find_controller_by_adev(struct acpi_device *adev);
2550
64bee4d2
MW		 2551static int acpi_spi_add_resource(struct acpi_resource *ares, void *data)
2552{
4c3c5954
AB		 2553 struct acpi_spi_lookup *lookup = data;
2554 struct spi_controller *ctlr = lookup->ctlr;
64bee4d2
MW		 2555
2556 if (ares->type == ACPI_RESOURCE_TYPE_SERIAL_BUS) {
2557 struct acpi_resource_spi_serialbus *sb;
4c3c5954
AB		 2558 acpi_handle parent_handle;
2559 acpi_status status;
64bee4d2
MW		 2560
2561 sb = &ares->data.spi_serial_bus;
2562 if (sb->type == ACPI_RESOURCE_SERIAL_TYPE_SPI) {
4c3c5954 2563
87e59b36
SB		 2564 if (lookup->index != -1 && lookup->n++ != lookup->index)
2565 return 1;
2566
4c3c5954
AB		 2567 status = acpi_get_handle(NULL,
2568 sb->resource_source.string_ptr,
2569 &parent_handle);
2570
87e59b36 2571 if (ACPI_FAILURE(status))
4c3c5954
AB		 2572 return -ENODEV;
2573
87e59b36
SB		 2574 if (ctlr) {
2575 if (ACPI_HANDLE(ctlr->dev.parent) != parent_handle)
2576 return -ENODEV;
2577 } else {
2578 struct acpi_device *adev;
2579
ac2a3fee
RW		 2580 adev = acpi_fetch_acpi_dev(parent_handle);
2581 if (!adev)
87e59b36
SB		 2582 return -ENODEV;
2583
2584 ctlr = acpi_spi_find_controller_by_adev(adev);
2585 if (!ctlr)
9c22ec4a 2586 return -EPROBE_DEFER;
87e59b36
SB		 2587
2588 lookup->ctlr = ctlr;
2589 }
2590
a0a90718
MW		 2591 /*
2592 * ACPI DeviceSelection numbering is handled by the
2593 * host controller driver in Windows and can vary
2594 * from driver to driver. In Linux we always expect
2595 * 0 .. max - 1 so we need to ask the driver to
2596 * translate between the two schemes.
2597 */
8caab75f
GU		 2598 if (ctlr->fw_translate_cs) {
2599 int cs = ctlr->fw_translate_cs(ctlr,
a0a90718
MW		 2600 sb->device_selection);
2601 if (cs < 0)
2602 return cs;
4c3c5954 2603 lookup->chip_select = cs;
a0a90718 2604 } else {
4c3c5954 2605 lookup->chip_select = sb->device_selection;
a0a90718
MW		 2606 }
2607
4c3c5954 2608 lookup->max_speed_hz = sb->connection_speed;
0dadde34 2609 lookup->bits_per_word = sb->data_bit_length;
64bee4d2
MW		 2610
2611 if (sb->clock_phase == ACPI_SPI_SECOND_PHASE)
4c3c5954 2612 lookup->mode |= SPI_CPHA;
64bee4d2 2613 if (sb->clock_polarity == ACPI_SPI_START_HIGH)
4c3c5954 2614 lookup->mode |= SPI_CPOL;
64bee4d2 2615 if (sb->device_polarity == ACPI_SPI_ACTIVE_HIGH)
4c3c5954 2616 lookup->mode |= SPI_CS_HIGH;
64bee4d2 2617 }
4c3c5954 2618 } else if (lookup->irq < 0) {
64bee4d2
MW		 2619 struct resource r;
2620
2621 if (acpi_dev_resource_interrupt(ares, 0, &r))
4c3c5954 2622 lookup->irq = r.start;
64bee4d2
MW		 2623 }
2624
2625 /* Always tell the ACPI core to skip this resource */
2626 return 1;
2627}
2628
000bee0e
SB		 2629/**
2630 * acpi_spi_device_alloc - Allocate a spi device, and fill it in with ACPI information
2631 * @ctlr: controller to which the spi device belongs
2632 * @adev: ACPI Device for the spi device
87e59b36 2633 * @index: Index of the spi resource inside the ACPI Node
000bee0e
SB		 2634 *
2635 * This should be used to allocate a new spi device from and ACPI Node.
2636 * The caller is responsible for calling spi_add_device to register the spi device.
2637 *
87e59b36
SB		 2638 * If ctlr is set to NULL, the Controller for the spi device will be looked up
2639 * using the resource.
2640 * If index is set to -1, index is not used.
2641 * Note: If index is -1, ctlr must be set.
2642 *
000bee0e
SB		 2643 * Return: a pointer to the new device, or ERR_PTR on error.
2644 */
2645struct spi_device *acpi_spi_device_alloc(struct spi_controller *ctlr,
87e59b36
SB		 2646 struct acpi_device *adev,
2647 int index)
64bee4d2 2648{
4c3c5954 2649 acpi_handle parent_handle = NULL;
64bee4d2 2650 struct list_head resource_list;
b28944c6 2651 struct acpi_spi_lookup lookup = {};
64bee4d2
MW		 2652 struct spi_device *spi;
2653 int ret;
2654
87e59b36
SB		 2655 if (!ctlr && index == -1)
2656 return ERR_PTR(-EINVAL);
2657
4c3c5954 2658 lookup.ctlr = ctlr;
4c3c5954 2659 lookup.irq = -1;
87e59b36
SB		 2660 lookup.index = index;
2661 lookup.n = 0;
64bee4d2
MW		 2662
2663 INIT_LIST_HEAD(&resource_list);
2664 ret = acpi_dev_get_resources(adev, &resource_list,
4c3c5954 2665 acpi_spi_add_resource, &lookup);
64bee4d2
MW		 2666 acpi_dev_free_resource_list(&resource_list);
2667
4c3c5954 2668 if (ret < 0)
95c8222f 2669 /* Found SPI in _CRS but it points to another controller */
b6747f4f 2670 return ERR_PTR(ret);
8a2e487e 2671
4c3c5954 2672 if (!lookup.max_speed_hz &&
10e92724 2673 ACPI_SUCCESS(acpi_get_parent(adev->handle, &parent_handle)) &&
87e59b36 2674 ACPI_HANDLE(lookup.ctlr->dev.parent) == parent_handle) {
4c3c5954
AB		 2675 /* Apple does not use _CRS but nested devices for SPI slaves */
2676 acpi_spi_parse_apple_properties(adev, &lookup);
2677 }
2678
2679 if (!lookup.max_speed_hz)
000bee0e 2680 return ERR_PTR(-ENODEV);
4c3c5954 2681
87e59b36 2682 spi = spi_alloc_device(lookup.ctlr);
4c3c5954 2683 if (!spi) {
87e59b36 2684 dev_err(&lookup.ctlr->dev, "failed to allocate SPI device for %s\n",
4c3c5954 2685 dev_name(&adev->dev));
000bee0e 2686 return ERR_PTR(-ENOMEM);
64bee4d2
MW		 2687 }
2688
4c3c5954
AB		 2689 ACPI_COMPANION_SET(&spi->dev, adev);
2690 spi->max_speed_hz = lookup.max_speed_hz;
ea235786 2691 spi->mode |= lookup.mode;
4c3c5954
AB		 2692 spi->irq = lookup.irq;
2693 spi->bits_per_word = lookup.bits_per_word;
303feb3c 2694 spi_set_chipselect(spi, 0, lookup.chip_select);
4c3c5954 2695
000bee0e
SB		 2696 return spi;
2697}
2698EXPORT_SYMBOL_GPL(acpi_spi_device_alloc);
2699
2700static acpi_status acpi_register_spi_device(struct spi_controller *ctlr,
2701 struct acpi_device *adev)
2702{
2703 struct spi_device *spi;
2704
2705 if (acpi_bus_get_status(adev) || !adev->status.present ||
2706 acpi_device_enumerated(adev))
2707 return AE_OK;
2708
87e59b36 2709 spi = acpi_spi_device_alloc(ctlr, adev, -1);
000bee0e
SB		 2710 if (IS_ERR(spi)) {
2711 if (PTR_ERR(spi) == -ENOMEM)
2712 return AE_NO_MEMORY;
2713 else
2714 return AE_OK;
2715 }
2716
0c6543f6
DD		 2717 acpi_set_modalias(adev, acpi_device_hid(adev), spi->modalias,
2718 sizeof(spi->modalias));
2719
33ada67d
CR		 2720 if (spi->irq < 0)
2721 spi->irq = acpi_dev_gpio_irq_get(adev, 0);
2722
7f24467f
OP		 2723 acpi_device_set_enumerated(adev);
2724
33cf00e5 2725 adev->power.flags.ignore_parent = true;
64bee4d2 2726 if (spi_add_device(spi)) {
33cf00e5 2727 adev->power.flags.ignore_parent = false;
8caab75f 2728 dev_err(&ctlr->dev, "failed to add SPI device %s from ACPI\n",
64bee4d2
MW		 2729 dev_name(&adev->dev));
2730 spi_dev_put(spi);
2731 }
2732
2733 return AE_OK;
2734}
2735
7f24467f
OP		 2736static acpi_status acpi_spi_add_device(acpi_handle handle, u32 level,
2737 void *data, void **return_value)
2738{
7030c428 2739 struct acpi_device *adev = acpi_fetch_acpi_dev(handle);
8caab75f 2740 struct spi_controller *ctlr = data;
7f24467f 2741
7030c428 2742 if (!adev)
7f24467f
OP		 2743 return AE_OK;
2744
8caab75f 2745 return acpi_register_spi_device(ctlr, adev);
7f24467f
OP		 2746}
2747
4c3c5954
AB		 2748#define SPI_ACPI_ENUMERATE_MAX_DEPTH 32
2749
8caab75f 2750static void acpi_register_spi_devices(struct spi_controller *ctlr)
64bee4d2
MW		 2751{
2752 acpi_status status;
2753 acpi_handle handle;
2754
8caab75f 2755 handle = ACPI_HANDLE(ctlr->dev.parent);
64bee4d2
MW		 2756 if (!handle)
2757 return;
2758
4c3c5954
AB		 2759 status = acpi_walk_namespace(ACPI_TYPE_DEVICE, ACPI_ROOT_OBJECT,
2760 SPI_ACPI_ENUMERATE_MAX_DEPTH,
8caab75f 2761 acpi_spi_add_device, NULL, ctlr, NULL);
64bee4d2 2762 if (ACPI_FAILURE(status))
8caab75f 2763 dev_warn(&ctlr->dev, "failed to enumerate SPI slaves\n");
64bee4d2
MW		 2764}
2765#else
8caab75f 2766static inline void acpi_register_spi_devices(struct spi_controller *ctlr) {}
64bee4d2
MW		 2767#endif /* CONFIG_ACPI */
2768
8caab75f 2769static void spi_controller_release(struct device *dev)
8ae12a0d 2770{
8caab75f 2771 struct spi_controller *ctlr;
8ae12a0d 2772
8caab75f
GU		 2773 ctlr = container_of(dev, struct spi_controller, dev);
2774 kfree(ctlr);
8ae12a0d
DB		 2775}
2776
2777static struct class spi_master_class = {
2778 .name = "spi_master",
2779 .owner = THIS_MODULE,
8caab75f 2780 .dev_release = spi_controller_release,
eca2ebc7 2781 .dev_groups = spi_master_groups,
8ae12a0d
DB		 2782};
2783
6c364062
GU		 2784#ifdef CONFIG_SPI_SLAVE
2785/**
2786 * spi_slave_abort - abort the ongoing transfer request on an SPI slave
2787 * controller
2788 * @spi: device used for the current transfer
2789 */
2790int spi_slave_abort(struct spi_device *spi)
2791{
8caab75f 2792 struct spi_controller *ctlr = spi->controller;
6c364062 2793
8caab75f
GU		 2794 if (spi_controller_is_slave(ctlr) && ctlr->slave_abort)
2795 return ctlr->slave_abort(ctlr);
6c364062
GU		 2796
2797 return -ENOTSUPP;
2798}
2799EXPORT_SYMBOL_GPL(spi_slave_abort);
2800
b8d3b056
YY		 2801int spi_target_abort(struct spi_device *spi)
2802{
2803 struct spi_controller *ctlr = spi->controller;
2804
2805 if (spi_controller_is_target(ctlr) && ctlr->target_abort)
2806 return ctlr->target_abort(ctlr);
2807
2808 return -ENOTSUPP;
2809}
2810EXPORT_SYMBOL_GPL(spi_target_abort);
2811
cc8b4659
GU		 2812static ssize_t slave_show(struct device *dev, struct device_attribute *attr,
2813 char *buf)
6c364062 2814{
8caab75f
GU		 2815 struct spi_controller *ctlr = container_of(dev, struct spi_controller,
2816 dev);
6c364062
GU		 2817 struct device *child;
2818
c21b0837 2819 child = device_find_any_child(&ctlr->dev);
6c364062
GU		 2820 return sprintf(buf, "%s\n",
2821 child ? to_spi_device(child)->modalias : NULL);
2822}
2823
cc8b4659
GU		 2824static ssize_t slave_store(struct device *dev, struct device_attribute *attr,
2825 const char *buf, size_t count)
6c364062 2826{
8caab75f
GU		 2827 struct spi_controller *ctlr = container_of(dev, struct spi_controller,
2828 dev);
6c364062
GU		 2829 struct spi_device *spi;
2830 struct device *child;
2831 char name[32];
2832 int rc;
2833
2834 rc = sscanf(buf, "%31s", name);
2835 if (rc != 1 || !name[0])
2836 return -EINVAL;
2837
c21b0837 2838 child = device_find_any_child(&ctlr->dev);
6c364062
GU		 2839 if (child) {
2840 /* Remove registered slave */
2841 device_unregister(child);
2842 put_device(child);
2843 }
2844
2845 if (strcmp(name, "(null)")) {
2846 /* Register new slave */
2847 spi = spi_alloc_device(ctlr);
2848 if (!spi)
2849 return -ENOMEM;
2850
51e99de5 2851 strscpy(spi->modalias, name, sizeof(spi->modalias));
6c364062
GU		 2852
2853 rc = spi_add_device(spi);
2854 if (rc) {
2855 spi_dev_put(spi);
2856 return rc;
2857 }
2858 }
2859
2860 return count;
2861}
2862
cc8b4659 2863static DEVICE_ATTR_RW(slave);
6c364062
GU		 2864
2865static struct attribute *spi_slave_attrs[] = {
2866 &dev_attr_slave.attr,
2867 NULL,
2868};
2869
2870static const struct attribute_group spi_slave_group = {
2871 .attrs = spi_slave_attrs,
2872};
2873
2874static const struct attribute_group *spi_slave_groups[] = {
8caab75f 2875 &spi_controller_statistics_group,
6c364062
GU		 2876 &spi_slave_group,
2877 NULL,
2878};
2879
2880static struct class spi_slave_class = {
2881 .name = "spi_slave",
2882 .owner = THIS_MODULE,
8caab75f 2883 .dev_release = spi_controller_release,
6c364062
GU		 2884 .dev_groups = spi_slave_groups,
2885};
2886#else
2887extern struct class spi_slave_class; /* dummy */
2888#endif
8ae12a0d
DB		 2889
2890/**
6c364062 2891 * __spi_alloc_controller - allocate an SPI master or slave controller
8ae12a0d 2892 * @dev: the controller, possibly using the platform_bus
33e34dc6 2893 * @size: how much zeroed driver-private data to allocate; the pointer to this
229e6af1
LW		 2894 * memory is in the driver_data field of the returned device, accessible
2895 * with spi_controller_get_devdata(); the memory is cacheline aligned;
2896 * drivers granting DMA access to portions of their private data need to
2897 * round up @size using ALIGN(size, dma_get_cache_alignment()).
6c364062
GU		 2898 * @slave: flag indicating whether to allocate an SPI master (false) or SPI
2899 * slave (true) controller
33e34dc6 2900 * Context: can sleep
8ae12a0d 2901 *
6c364062 2902 * This call is used only by SPI controller drivers, which are the
8ae12a0d 2903 * only ones directly touching chip registers. It's how they allocate
8caab75f 2904 * an spi_controller structure, prior to calling spi_register_controller().
8ae12a0d 2905 *
97d56dc6 2906 * This must be called from context that can sleep.
8ae12a0d 2907 *
6c364062 2908 * The caller is responsible for assigning the bus number and initializing the
8caab75f
GU		 2909 * controller's methods before calling spi_register_controller(); and (after
2910 * errors adding the device) calling spi_controller_put() to prevent a memory
2911 * leak.
97d56dc6 2912 *
6c364062 2913 * Return: the SPI controller structure on success, else NULL.
8ae12a0d 2914 */
8caab75f
GU		 2915struct spi_controller *__spi_alloc_controller(struct device *dev,
2916 unsigned int size, bool slave)
8ae12a0d 2917{
8caab75f 2918 struct spi_controller *ctlr;
229e6af1 2919 size_t ctlr_size = ALIGN(sizeof(*ctlr), dma_get_cache_alignment());
8ae12a0d 2920
0c868461
DB		 2921 if (!dev)
2922 return NULL;
2923
229e6af1 2924 ctlr = kzalloc(size + ctlr_size, GFP_KERNEL);
8caab75f 2925 if (!ctlr)
8ae12a0d
DB		 2926 return NULL;
2927
8caab75f 2928 device_initialize(&ctlr->dev);
16a8e2fb
UKK		 2929 INIT_LIST_HEAD(&ctlr->queue);
2930 spin_lock_init(&ctlr->queue_lock);
2931 spin_lock_init(&ctlr->bus_lock_spinlock);
2932 mutex_init(&ctlr->bus_lock_mutex);
2933 mutex_init(&ctlr->io_mutex);
2934 mutex_init(&ctlr->add_lock);
8caab75f
GU		 2935 ctlr->bus_num = -1;
2936 ctlr->num_chipselect = 1;
2937 ctlr->slave = slave;
6c364062 2938 if (IS_ENABLED(CONFIG_SPI_SLAVE) && slave)
8caab75f 2939 ctlr->dev.class = &spi_slave_class;
6c364062 2940 else
8caab75f
GU		 2941 ctlr->dev.class = &spi_master_class;
2942 ctlr->dev.parent = dev;
2943 pm_suspend_ignore_children(&ctlr->dev, true);
229e6af1 2944 spi_controller_set_devdata(ctlr, (void *)ctlr + ctlr_size);
8ae12a0d 2945
8caab75f 2946 return ctlr;
8ae12a0d 2947}
6c364062 2948EXPORT_SYMBOL_GPL(__spi_alloc_controller);
8ae12a0d 2949
5e844cc3
LW		 2950static void devm_spi_release_controller(struct device *dev, void *ctlr)
2951{
2952 spi_controller_put(*(struct spi_controller **)ctlr);
2953}
2954
2955/**
2956 * __devm_spi_alloc_controller - resource-managed __spi_alloc_controller()
2957 * @dev: physical device of SPI controller
2958 * @size: how much zeroed driver-private data to allocate
2959 * @slave: whether to allocate an SPI master (false) or SPI slave (true)
2960 * Context: can sleep
2961 *
2962 * Allocate an SPI controller and automatically release a reference on it
2963 * when @dev is unbound from its driver. Drivers are thus relieved from
2964 * having to call spi_controller_put().
2965 *
2966 * The arguments to this function are identical to __spi_alloc_controller().
2967 *
2968 * Return: the SPI controller structure on success, else NULL.
2969 */
2970struct spi_controller *__devm_spi_alloc_controller(struct device *dev,
2971 unsigned int size,
2972 bool slave)
2973{
2974 struct spi_controller **ptr, *ctlr;
2975
2976 ptr = devres_alloc(devm_spi_release_controller, sizeof(*ptr),
2977 GFP_KERNEL);
2978 if (!ptr)
2979 return NULL;
2980
2981 ctlr = __spi_alloc_controller(dev, size, slave);
2982 if (ctlr) {
794aaf01 2983 ctlr->devm_allocated = true;
5e844cc3
LW		 2984 *ptr = ctlr;
2985 devres_add(dev, ptr);
2986 } else {
2987 devres_free(ptr);
2988 }
2989
2990 return ctlr;
2991}
2992EXPORT_SYMBOL_GPL(__devm_spi_alloc_controller);
2993
f3186dd8
LW		 2994/**
2995 * spi_get_gpio_descs() - grab chip select GPIOs for the master
2996 * @ctlr: The SPI master to grab GPIO descriptors for
2997 */
2998static int spi_get_gpio_descs(struct spi_controller *ctlr)
2999{
3000 int nb, i;
3001 struct gpio_desc **cs;
3002 struct device *dev = &ctlr->dev;
7d93aecd
GU		 3003 unsigned long native_cs_mask = 0;
3004 unsigned int num_cs_gpios = 0;
f3186dd8
LW		 3005
3006 nb = gpiod_count(dev, "cs");
31ed8ebc
AS		 3007 if (nb < 0) {
3008 /* No GPIOs at all is fine, else return the error */
3009 if (nb == -ENOENT)
3010 return 0;
f3186dd8 3011 return nb;
31ed8ebc
AS		 3012 }
3013
3014 ctlr->num_chipselect = max_t(int, nb, ctlr->num_chipselect);
f3186dd8
LW		 3015
3016 cs = devm_kcalloc(dev, ctlr->num_chipselect, sizeof(*cs),
3017 GFP_KERNEL);
3018 if (!cs)
3019 return -ENOMEM;
3020 ctlr->cs_gpiods = cs;
3021
3022 for (i = 0; i < nb; i++) {
3023 /*
3024 * Most chipselects are active low, the inverted
3025 * semantics are handled by special quirks in gpiolib,
3026 * so initializing them GPIOD_OUT_LOW here means
3027 * "unasserted", in most cases this will drive the physical
3028 * line high.
3029 */
3030 cs[i] = devm_gpiod_get_index_optional(dev, "cs", i,
3031 GPIOD_OUT_LOW);
1723fdec
GU		 3032 if (IS_ERR(cs[i]))
3033 return PTR_ERR(cs[i]);
f3186dd8
LW		 3034
3035 if (cs[i]) {
3036 /*
3037 * If we find a CS GPIO, name it after the device and
3038 * chip select line.
3039 */
3040 char *gpioname;
3041
3042 gpioname = devm_kasprintf(dev, GFP_KERNEL, "%s CS%d",
3043 dev_name(dev), i);
3044 if (!gpioname)
3045 return -ENOMEM;
3046 gpiod_set_consumer_name(cs[i], gpioname);
7d93aecd
GU		 3047 num_cs_gpios++;
3048 continue;
f3186dd8 3049 }
7d93aecd
GU		 3050
3051 if (ctlr->max_native_cs && i >= ctlr->max_native_cs) {
3052 dev_err(dev, "Invalid native chip select %d\n", i);
3053 return -EINVAL;
f3186dd8 3054 }
7d93aecd
GU		 3055 native_cs_mask |= BIT(i);
3056 }
3057
f60d7270 3058 ctlr->unused_native_cs = ffs(~native_cs_mask) - 1;
dbaca8e5
AS		 3059
3060 if ((ctlr->flags & SPI_MASTER_GPIO_SS) && num_cs_gpios &&
3061 ctlr->max_native_cs && ctlr->unused_native_cs >= ctlr->max_native_cs) {
7d93aecd
GU		 3062 dev_err(dev, "No unused native chip select available\n");
3063 return -EINVAL;
f3186dd8
LW		 3064 }
3065
3066 return 0;
3067}
3068
bdf3a3b5
BB		 3069static int spi_controller_check_ops(struct spi_controller *ctlr)
3070{
3071 /*
b5932f5c
BB		 3072 * The controller may implement only the high-level SPI-memory like
3073 * operations if it does not support regular SPI transfers, and this is
3074 * valid use case.
3075 * If ->mem_ops is NULL, we request that at least one of the
3076 * ->transfer_xxx() method be implemented.
bdf3a3b5 3077 */
b5932f5c
BB		 3078 if (ctlr->mem_ops) {
3079 if (!ctlr->mem_ops->exec_op)
3080 return -EINVAL;
3081 } else if (!ctlr->transfer && !ctlr->transfer_one &&
3082 !ctlr->transfer_one_message) {
bdf3a3b5 3083 return -EINVAL;
b5932f5c 3084 }
bdf3a3b5
BB		 3085
3086 return 0;
3087}
3088
8ae12a0d 3089/**
8caab75f
GU		 3090 * spi_register_controller - register SPI master or slave controller
3091 * @ctlr: initialized master, originally from spi_alloc_master() or
3092 * spi_alloc_slave()
33e34dc6 3093 * Context: can sleep
8ae12a0d 3094 *
8caab75f 3095 * SPI controllers connect to their drivers using some non-SPI bus,
8ae12a0d 3096 * such as the platform bus. The final stage of probe() in that code
8caab75f 3097 * includes calling spi_register_controller() to hook up to this SPI bus glue.
8ae12a0d
DB		 3098 *
3099 * SPI controllers use board specific (often SOC specific) bus numbers,
3100 * and board-specific addressing for SPI devices combines those numbers
3101 * with chip select numbers. Since SPI does not directly support dynamic
3102 * device identification, boards need configuration tables telling which
3103 * chip is at which address.
3104 *
3105 * This must be called from context that can sleep. It returns zero on
8caab75f 3106 * success, else a negative error code (dropping the controller's refcount).
0c868461 3107 * After a successful return, the caller is responsible for calling
8caab75f 3108 * spi_unregister_controller().
97d56dc6
JMC		 3109 *
3110 * Return: zero on success, else a negative error code.
8ae12a0d 3111 */
8caab75f 3112int spi_register_controller(struct spi_controller *ctlr)
8ae12a0d 3113{
8caab75f 3114 struct device *dev = ctlr->dev.parent;
2b9603a0 3115 struct boardinfo *bi;
b93318a2 3116 int status;
42bdd706 3117 int id, first_dynamic;
8ae12a0d 3118
0c868461
DB		 3119 if (!dev)
3120 return -ENODEV;
3121
bdf3a3b5
BB		 3122 /*
3123 * Make sure all necessary hooks are implemented before registering
3124 * the SPI controller.
3125 */
3126 status = spi_controller_check_ops(ctlr);
3127 if (status)
3128 return status;
3129
04b2d03a 3130 if (ctlr->bus_num >= 0) {
95c8222f 3131 /* Devices with a fixed bus num must check-in with the num */
04b2d03a
GU		 3132 mutex_lock(&board_lock);
3133 id = idr_alloc(&spi_master_idr, ctlr, ctlr->bus_num,
3134 ctlr->bus_num + 1, GFP_KERNEL);
3135 mutex_unlock(&board_lock);
3136 if (WARN(id < 0, "couldn't get idr"))
3137 return id == -ENOSPC ? -EBUSY : id;
3138 ctlr->bus_num = id;
3139 } else if (ctlr->dev.of_node) {
95c8222f 3140 /* Allocate dynamic bus number using Linux idr */
9b61e302
SM		 3141 id = of_alias_get_id(ctlr->dev.of_node, "spi");
3142 if (id >= 0) {
3143 ctlr->bus_num = id;
3144 mutex_lock(&board_lock);
3145 id = idr_alloc(&spi_master_idr, ctlr, ctlr->bus_num,
3146 ctlr->bus_num + 1, GFP_KERNEL);
3147 mutex_unlock(&board_lock);
3148 if (WARN(id < 0, "couldn't get idr"))
3149 return id == -ENOSPC ? -EBUSY : id;
3150 }
3151 }
8caab75f 3152 if (ctlr->bus_num < 0) {
42bdd706
LS		 3153 first_dynamic = of_alias_get_highest_id("spi");
3154 if (first_dynamic < 0)
3155 first_dynamic = 0;
3156 else
3157 first_dynamic++;
3158
9a9a047a 3159 mutex_lock(&board_lock);
42bdd706
LS		 3160 id = idr_alloc(&spi_master_idr, ctlr, first_dynamic,
3161 0, GFP_KERNEL);
9a9a047a
SM		 3162 mutex_unlock(&board_lock);
3163 if (WARN(id < 0, "couldn't get idr"))
3164 return id;
3165 ctlr->bus_num = id;
8ae12a0d 3166 }
8caab75f
GU		 3167 ctlr->bus_lock_flag = 0;
3168 init_completion(&ctlr->xfer_completion);
69fa9590 3169 init_completion(&ctlr->cur_msg_completion);
8caab75f
GU		 3170 if (!ctlr->max_dma_len)
3171 ctlr->max_dma_len = INT_MAX;
cf32b71e 3172
350de7ce
AS		 3173 /*
3174 * Register the device, then userspace will see it.
3175 * Registration fails if the bus ID is in use.
8ae12a0d 3176 */
8caab75f 3177 dev_set_name(&ctlr->dev, "spi%u", ctlr->bus_num);
0a919ae4 3178
f48dc6b9
LW		 3179 if (!spi_controller_is_slave(ctlr) && ctlr->use_gpio_descriptors) {
3180 status = spi_get_gpio_descs(ctlr);
3181 if (status)
3182 goto free_bus_id;
3183 /*
3184 * A controller using GPIO descriptors always
3185 * supports SPI_CS_HIGH if need be.
3186 */
3187 ctlr->mode_bits |= SPI_CS_HIGH;
0a919ae4
AS		 3188 }
3189
f9481b08
TA		 3190 /*
3191 * Even if it's just one always-selected device, there must
3192 * be at least one chipselect.
3193 */
f9981d4f
AK		 3194 if (!ctlr->num_chipselect) {
3195 status = -EINVAL;
3196 goto free_bus_id;
3197 }
f9481b08 3198
95c8222f 3199 /* Setting last_cs to -1 means no chip selected */
6bb477df
YZ		 3200 ctlr->last_cs = -1;
3201
8caab75f 3202 status = device_add(&ctlr->dev);
f9981d4f
AK		 3203 if (status < 0)
3204 goto free_bus_id;
9b61e302 3205 dev_dbg(dev, "registered %s %s\n",
8caab75f 3206 spi_controller_is_slave(ctlr) ? "slave" : "master",
9b61e302 3207 dev_name(&ctlr->dev));
8ae12a0d 3208
b5932f5c
BB		 3209 /*
3210 * If we're using a queued driver, start the queue. Note that we don't
3211 * need the queueing logic if the driver is only supporting high-level
3212 * memory operations.
3213 */
3214 if (ctlr->transfer) {
8caab75f 3215 dev_info(dev, "controller is unqueued, this is deprecated\n");
b5932f5c 3216 } else if (ctlr->transfer_one || ctlr->transfer_one_message) {
8caab75f 3217 status = spi_controller_initialize_queue(ctlr);
ffbbdd21 3218 if (status) {
8caab75f 3219 device_del(&ctlr->dev);
f9981d4f 3220 goto free_bus_id;
ffbbdd21
LW		 3221 }
3222 }
95c8222f 3223 /* Add statistics */
6598b91b
DJ		 3224 ctlr->pcpu_statistics = spi_alloc_pcpu_stats(dev);
3225 if (!ctlr->pcpu_statistics) {
3226 dev_err(dev, "Error allocating per-cpu statistics\n");
d52b095b 3227 status = -ENOMEM;
6598b91b
DJ		 3228 goto destroy_queue;
3229 }
ffbbdd21 3230
2b9603a0 3231 mutex_lock(&board_lock);
8caab75f 3232 list_add_tail(&ctlr->list, &spi_controller_list);
2b9603a0 3233 list_for_each_entry(bi, &board_list, list)
8caab75f 3234 spi_match_controller_to_boardinfo(ctlr, &bi->board_info);
2b9603a0
FT		 3235 mutex_unlock(&board_lock);
3236
64bee4d2 3237 /* Register devices from the device tree and ACPI */
8caab75f
GU		 3238 of_register_spi_devices(ctlr);
3239 acpi_register_spi_devices(ctlr);
f9981d4f
AK		 3240 return status;
3241
6598b91b
DJ		 3242destroy_queue:
3243 spi_destroy_queue(ctlr);
f9981d4f
AK		 3244free_bus_id:
3245 mutex_lock(&board_lock);
3246 idr_remove(&spi_master_idr, ctlr->bus_num);
3247 mutex_unlock(&board_lock);
8ae12a0d
DB		 3248 return status;
3249}
8caab75f 3250EXPORT_SYMBOL_GPL(spi_register_controller);
8ae12a0d 3251
43cc5a0a 3252static void devm_spi_unregister(struct device *dev, void *res)
666d5b4c 3253{
43cc5a0a 3254 spi_unregister_controller(*(struct spi_controller **)res);
666d5b4c
MB		 3255}
3256
3257/**
8caab75f
GU		 3258 * devm_spi_register_controller - register managed SPI master or slave
3259 * controller
3260 * @dev: device managing SPI controller
3261 * @ctlr: initialized controller, originally from spi_alloc_master() or
3262 * spi_alloc_slave()
666d5b4c
MB		 3263 * Context: can sleep
3264 *
8caab75f 3265 * Register a SPI device as with spi_register_controller() which will
68b892f1 3266 * automatically be unregistered and freed.
97d56dc6
JMC		 3267 *
3268 * Return: zero on success, else a negative error code.
666d5b4c 3269 */
8caab75f
GU		 3270int devm_spi_register_controller(struct device *dev,
3271 struct spi_controller *ctlr)
666d5b4c 3272{
43cc5a0a 3273 struct spi_controller **ptr;
666d5b4c
MB		 3274 int ret;
3275
43cc5a0a
YY		 3276 ptr = devres_alloc(devm_spi_unregister, sizeof(*ptr), GFP_KERNEL);
3277 if (!ptr)
3278 return -ENOMEM;
3279
8caab75f 3280 ret = spi_register_controller(ctlr);
43cc5a0a
YY		 3281 if (!ret) {
3282 *ptr = ctlr;
3283 devres_add(dev, ptr);
3284 } else {
3285 devres_free(ptr);
3286 }
666d5b4c 3287
43cc5a0a 3288 return ret;
666d5b4c 3289}
8caab75f 3290EXPORT_SYMBOL_GPL(devm_spi_register_controller);
666d5b4c 3291
34860089 3292static int __unregister(struct device *dev, void *null)
8ae12a0d 3293{
34860089 3294 spi_unregister_device(to_spi_device(dev));
8ae12a0d
DB		 3295 return 0;
3296}
3297
3298/**
8caab75f
GU		 3299 * spi_unregister_controller - unregister SPI master or slave controller
3300 * @ctlr: the controller being unregistered
33e34dc6 3301 * Context: can sleep
8ae12a0d 3302 *
8caab75f 3303 * This call is used only by SPI controller drivers, which are the
8ae12a0d
DB		 3304 * only ones directly touching chip registers.
3305 *
3306 * This must be called from context that can sleep.
68b892f1
JH		 3307 *
3308 * Note that this function also drops a reference to the controller.
8ae12a0d 3309 */
8caab75f 3310void spi_unregister_controller(struct spi_controller *ctlr)
8ae12a0d 3311{
9b61e302 3312 struct spi_controller *found;
67f7b278 3313 int id = ctlr->bus_num;
89fc9a1a 3314
ddf75be4
LW		 3315 /* Prevent addition of new devices, unregister existing ones */
3316 if (IS_ENABLED(CONFIG_SPI_DYNAMIC))
6098475d 3317 mutex_lock(&ctlr->add_lock);
ddf75be4 3318
84855678
LW		 3319 device_for_each_child(&ctlr->dev, NULL, __unregister);
3320
9b61e302
SM		 3321 /* First make sure that this controller was ever added */
3322 mutex_lock(&board_lock);
67f7b278 3323 found = idr_find(&spi_master_idr, id);
9b61e302 3324 mutex_unlock(&board_lock);
8caab75f
GU		 3325 if (ctlr->queued) {
3326 if (spi_destroy_queue(ctlr))
3327 dev_err(&ctlr->dev, "queue remove failed\n");
ffbbdd21 3328 }
2b9603a0 3329 mutex_lock(&board_lock);
8caab75f 3330 list_del(&ctlr->list);
2b9603a0
FT		 3331 mutex_unlock(&board_lock);
3332
5e844cc3
LW		 3333 device_del(&ctlr->dev);
3334
95c8222f 3335 /* Free bus id */
9b61e302 3336 mutex_lock(&board_lock);
613bd1ea
JN		 3337 if (found == ctlr)
3338 idr_remove(&spi_master_idr, id);
9b61e302 3339 mutex_unlock(&board_lock);
ddf75be4
LW		 3340
3341 if (IS_ENABLED(CONFIG_SPI_DYNAMIC))
6098475d 3342 mutex_unlock(&ctlr->add_lock);
6c53b45c
MW		 3343
3344 /* Release the last reference on the controller if its driver
3345 * has not yet been converted to devm_spi_alloc_master/slave().
3346 */
3347 if (!ctlr->devm_allocated)
3348 put_device(&ctlr->dev);
8ae12a0d 3349}
8caab75f 3350EXPORT_SYMBOL_GPL(spi_unregister_controller);
8ae12a0d 3351
8caab75f 3352int spi_controller_suspend(struct spi_controller *ctlr)
ffbbdd21
LW		 3353{
3354 int ret;
3355
8caab75f
GU		 3356 /* Basically no-ops for non-queued controllers */
3357 if (!ctlr->queued)
ffbbdd21
LW		 3358 return 0;
3359
8caab75f 3360 ret = spi_stop_queue(ctlr);
ffbbdd21 3361 if (ret)
8caab75f 3362 dev_err(&ctlr->dev, "queue stop failed\n");
ffbbdd21
LW		 3363
3364 return ret;
3365}
8caab75f 3366EXPORT_SYMBOL_GPL(spi_controller_suspend);
ffbbdd21 3367
8caab75f 3368int spi_controller_resume(struct spi_controller *ctlr)
ffbbdd21
LW		 3369{
3370 int ret;
3371
8caab75f 3372 if (!ctlr->queued)
ffbbdd21
LW		 3373 return 0;
3374
8caab75f 3375 ret = spi_start_queue(ctlr);
ffbbdd21 3376 if (ret)
8caab75f 3377 dev_err(&ctlr->dev, "queue restart failed\n");
ffbbdd21
LW		 3378
3379 return ret;
3380}
8caab75f 3381EXPORT_SYMBOL_GPL(spi_controller_resume);
ffbbdd21 3382
8ae12a0d
DB		 3383/*-------------------------------------------------------------------------*/
3384
523baf5a
MS		 3385/* Core methods for spi_message alterations */
3386
8caab75f 3387static void __spi_replace_transfers_release(struct spi_controller *ctlr,
523baf5a
MS		 3388 struct spi_message *msg,
3389 void *res)
3390{
3391 struct spi_replaced_transfers *rxfer = res;
3392 size_t i;
3393
95c8222f 3394 /* Call extra callback if requested */
523baf5a 3395 if (rxfer->release)
8caab75f 3396 rxfer->release(ctlr, msg, res);
523baf5a 3397
95c8222f 3398 /* Insert replaced transfers back into the message */
523baf5a
MS		 3399 list_splice(&rxfer->replaced_transfers, rxfer->replaced_after);
3400
95c8222f 3401 /* Remove the formerly inserted entries */
523baf5a
MS		 3402 for (i = 0; i < rxfer->inserted; i++)
3403 list_del(&rxfer->inserted_transfers[i].transfer_list);
3404}
3405
3406/**
3407 * spi_replace_transfers - replace transfers with several transfers
3408 * and register change with spi_message.resources
3409 * @msg: the spi_message we work upon
3410 * @xfer_first: the first spi_transfer we want to replace
3411 * @remove: number of transfers to remove
3412 * @insert: the number of transfers we want to insert instead
3413 * @release: extra release code necessary in some circumstances
3414 * @extradatasize: extra data to allocate (with alignment guarantees
3415 * of struct @spi_transfer)
05885397 3416 * @gfp: gfp flags
523baf5a
MS		 3417 *
3418 * Returns: pointer to @spi_replaced_transfers,
3419 * PTR_ERR(...) in case of errors.
3420 */
da21fde0 3421static struct spi_replaced_transfers *spi_replace_transfers(
523baf5a
MS		 3422 struct spi_message *msg,
3423 struct spi_transfer *xfer_first,
3424 size_t remove,
3425 size_t insert,
3426 spi_replaced_release_t release,
3427 size_t extradatasize,
3428 gfp_t gfp)
3429{
3430 struct spi_replaced_transfers *rxfer;
3431 struct spi_transfer *xfer;
3432 size_t i;
3433
95c8222f 3434 /* Allocate the structure using spi_res */
523baf5a 3435 rxfer = spi_res_alloc(msg->spi, __spi_replace_transfers_release,
aef97522 3436 struct_size(rxfer, inserted_transfers, insert)
523baf5a
MS		 3437 + extradatasize,
3438 gfp);
3439 if (!rxfer)
3440 return ERR_PTR(-ENOMEM);
3441
95c8222f 3442 /* The release code to invoke before running the generic release */
523baf5a
MS		 3443 rxfer->release = release;
3444
95c8222f 3445 /* Assign extradata */
523baf5a
MS		 3446 if (extradatasize)
3447 rxfer->extradata =
3448 &rxfer->inserted_transfers[insert];
3449
95c8222f 3450 /* Init the replaced_transfers list */
523baf5a
MS		 3451 INIT_LIST_HEAD(&rxfer->replaced_transfers);
3452
350de7ce
AS		 3453 /*
3454 * Assign the list_entry after which we should reinsert
523baf5a
MS		 3455 * the @replaced_transfers - it may be spi_message.messages!
3456 */
3457 rxfer->replaced_after = xfer_first->transfer_list.prev;
3458
95c8222f 3459 /* Remove the requested number of transfers */
523baf5a 3460 for (i = 0; i < remove; i++) {
350de7ce
AS		 3461 /*
3462 * If the entry after replaced_after it is msg->transfers
523baf5a 3463 * then we have been requested to remove more transfers
350de7ce 3464 * than are in the list.
523baf5a
MS		 3465 */
3466 if (rxfer->replaced_after->next == &msg->transfers) {
3467 dev_err(&msg->spi->dev,
3468 "requested to remove more spi_transfers than are available\n");
95c8222f 3469 /* Insert replaced transfers back into the message */
523baf5a
MS		 3470 list_splice(&rxfer->replaced_transfers,
3471 rxfer->replaced_after);
3472
95c8222f 3473 /* Free the spi_replace_transfer structure... */
523baf5a
MS		 3474 spi_res_free(rxfer);
3475
95c8222f 3476 /* ...and return with an error */
523baf5a
MS		 3477 return ERR_PTR(-EINVAL);
3478 }
3479
350de7ce
AS		 3480 /*
3481 * Remove the entry after replaced_after from list of
3482 * transfers and add it to list of replaced_transfers.
523baf5a
MS		 3483 */
3484 list_move_tail(rxfer->replaced_after->next,
3485 &rxfer->replaced_transfers);
3486 }
3487
350de7ce
AS		 3488 /*
3489 * Create copy of the given xfer with identical settings
3490 * based on the first transfer to get removed.
523baf5a
MS		 3491 */
3492 for (i = 0; i < insert; i++) {
95c8222f 3493 /* We need to run in reverse order */
523baf5a
MS		 3494 xfer = &rxfer->inserted_transfers[insert - 1 - i];
3495
95c8222f 3496 /* Copy all spi_transfer data */
523baf5a
MS		 3497 memcpy(xfer, xfer_first, sizeof(*xfer));
3498
95c8222f 3499 /* Add to list */
523baf5a
MS		 3500 list_add(&xfer->transfer_list, rxfer->replaced_after);
3501
95c8222f 3502 /* Clear cs_change and delay for all but the last */
523baf5a
MS		 3503 if (i) {
3504 xfer->cs_change = false;
bebcfd27 3505 xfer->delay.value = 0;
523baf5a
MS		 3506 }
3507 }
3508
95c8222f 3509 /* Set up inserted... */
523baf5a
MS		 3510 rxfer->inserted = insert;
3511
95c8222f 3512 /* ...and register it with spi_res/spi_message */
523baf5a
MS		 3513 spi_res_add(msg, rxfer);
3514
3515 return rxfer;
3516}
523baf5a 3517
8caab75f 3518static int __spi_split_transfer_maxsize(struct spi_controller *ctlr,
08933418
FE		 3519 struct spi_message *msg,
3520 struct spi_transfer **xferp,
3521 size_t maxsize,
3522 gfp_t gfp)
d9f12122
MS		 3523{
3524 struct spi_transfer *xfer = *xferp, *xfers;
3525 struct spi_replaced_transfers *srt;
3526 size_t offset;
3527 size_t count, i;
3528
95c8222f 3529 /* Calculate how many we have to replace */
d9f12122
MS		 3530 count = DIV_ROUND_UP(xfer->len, maxsize);
3531
95c8222f 3532 /* Create replacement */
d9f12122 3533 srt = spi_replace_transfers(msg, xfer, 1, count, NULL, 0, gfp);
657d32ef
DC		 3534 if (IS_ERR(srt))
3535 return PTR_ERR(srt);
d9f12122
MS		 3536 xfers = srt->inserted_transfers;
3537
350de7ce
AS		 3538 /*
3539 * Now handle each of those newly inserted spi_transfers.
3540 * Note that the replacements spi_transfers all are preset
d9f12122
MS		 3541 * to the same values as *xferp, so tx_buf, rx_buf and len
3542 * are all identical (as well as most others)
3543 * so we just have to fix up len and the pointers.
3544 *
350de7ce
AS		 3545 * This also includes support for the depreciated
3546 * spi_message.is_dma_mapped interface.
d9f12122
MS		 3547 */
3548
350de7ce
AS		 3549 /*
3550 * The first transfer just needs the length modified, so we
3551 * run it outside the loop.
d9f12122 3552 */
c8dab77a 3553 xfers[0].len = min_t(size_t, maxsize, xfer[0].len);
d9f12122 3554
95c8222f 3555 /* All the others need rx_buf/tx_buf also set */
d9f12122 3556 for (i = 1, offset = maxsize; i < count; offset += maxsize, i++) {
95c8222f 3557 /* Update rx_buf, tx_buf and dma */
d9f12122
MS		 3558 if (xfers[i].rx_buf)
3559 xfers[i].rx_buf += offset;
3560 if (xfers[i].rx_dma)
3561 xfers[i].rx_dma += offset;
3562 if (xfers[i].tx_buf)
3563 xfers[i].tx_buf += offset;
3564 if (xfers[i].tx_dma)
3565 xfers[i].tx_dma += offset;
3566
95c8222f 3567 /* Update length */
d9f12122
MS		 3568 xfers[i].len = min(maxsize, xfers[i].len - offset);
3569 }
3570
350de7ce
AS		 3571 /*
3572 * We set up xferp to the last entry we have inserted,
3573 * so that we skip those already split transfers.
d9f12122
MS		 3574 */
3575 *xferp = &xfers[count - 1];
3576
95c8222f 3577 /* Increment statistics counters */
6598b91b 3578 SPI_STATISTICS_INCREMENT_FIELD(ctlr->pcpu_statistics,
d9f12122 3579 transfers_split_maxsize);
6598b91b 3580 SPI_STATISTICS_INCREMENT_FIELD(msg->spi->pcpu_statistics,
d9f12122
MS		 3581 transfers_split_maxsize);
3582
3583 return 0;
3584}
3585
3586/**
ce2424d7
MCC		 3587 * spi_split_transfers_maxsize - split spi transfers into multiple transfers
3588 * when an individual transfer exceeds a
3589 * certain size
8caab75f 3590 * @ctlr: the @spi_controller for this transfer
3700ce95
MI		 3591 * @msg: the @spi_message to transform
3592 * @maxsize: the maximum when to apply this
10f11a22 3593 * @gfp: GFP allocation flags
d9f12122
MS		 3594 *
3595 * Return: status of transformation
3596 */
8caab75f 3597int spi_split_transfers_maxsize(struct spi_controller *ctlr,
d9f12122
MS		 3598 struct spi_message *msg,
3599 size_t maxsize,
3600 gfp_t gfp)
3601{
3602 struct spi_transfer *xfer;
3603 int ret;
3604
350de7ce
AS		 3605 /*
3606 * Iterate over the transfer_list,
d9f12122
MS		 3607 * but note that xfer is advanced to the last transfer inserted
3608 * to avoid checking sizes again unnecessarily (also xfer does
350de7ce
AS		 3609 * potentially belong to a different list by the time the
3610 * replacement has happened).
d9f12122
MS		 3611 */
3612 list_for_each_entry(xfer, &msg->transfers, transfer_list) {
3613 if (xfer->len > maxsize) {
8caab75f
GU		 3614 ret = __spi_split_transfer_maxsize(ctlr, msg, &xfer,
3615 maxsize, gfp);
d9f12122
MS		 3616 if (ret)
3617 return ret;
3618 }
3619 }
3620
3621 return 0;
3622}
3623EXPORT_SYMBOL_GPL(spi_split_transfers_maxsize);
8ae12a0d
DB		 3624
3625/*-------------------------------------------------------------------------*/
3626
8caab75f 3627/* Core methods for SPI controller protocol drivers. Some of the
7d077197
DB		 3628 * other core methods are currently defined as inline functions.
3629 */
3630
8caab75f
GU		 3631static int __spi_validate_bits_per_word(struct spi_controller *ctlr,
3632 u8 bits_per_word)
63ab645f 3633{
8caab75f 3634 if (ctlr->bits_per_word_mask) {
63ab645f
SB		 3635 /* Only 32 bits fit in the mask */
3636 if (bits_per_word > 32)
3637 return -EINVAL;
8caab75f 3638 if (!(ctlr->bits_per_word_mask & SPI_BPW_MASK(bits_per_word)))
63ab645f
SB		 3639 return -EINVAL;
3640 }
3641
3642 return 0;
3643}
3644
684a4784
TA		 3645/**
3646 * spi_set_cs_timing - configure CS setup, hold, and inactive delays
3647 * @spi: the device that requires specific CS timing configuration
3648 *
3649 * Return: zero on success, else a negative error code.
3650 */
3651static int spi_set_cs_timing(struct spi_device *spi)
3652{
3653 struct device *parent = spi->controller->dev.parent;
3654 int status = 0;
3655
303feb3c 3656 if (spi->controller->set_cs_timing && !spi_get_csgpiod(spi, 0)) {
684a4784
TA		 3657 if (spi->controller->auto_runtime_pm) {
3658 status = pm_runtime_get_sync(parent);
3659 if (status < 0) {
3660 pm_runtime_put_noidle(parent);
3661 dev_err(&spi->controller->dev, "Failed to power device: %d\n",
3662 status);
3663 return status;
3664 }
3665
3666 status = spi->controller->set_cs_timing(spi);
3667 pm_runtime_mark_last_busy(parent);
3668 pm_runtime_put_autosuspend(parent);
3669 } else {
3670 status = spi->controller->set_cs_timing(spi);
3671 }
3672 }
3673 return status;
3674}
3675
7d077197
DB		 3676/**
3677 * spi_setup - setup SPI mode and clock rate
3678 * @spi: the device whose settings are being modified
3679 * Context: can sleep, and no requests are queued to the device
3680 *
3681 * SPI protocol drivers may need to update the transfer mode if the
3682 * device doesn't work with its default. They may likewise need
3683 * to update clock rates or word sizes from initial values. This function
3684 * changes those settings, and must be called from a context that can sleep.
3685 * Except for SPI_CS_HIGH, which takes effect immediately, the changes take
3686 * effect the next time the device is selected and data is transferred to
3687 * or from it. When this function returns, the spi device is deselected.
3688 *
3689 * Note that this call will fail if the protocol driver specifies an option
3690 * that the underlying controller or its driver does not support. For
3691 * example, not all hardware supports wire transfers using nine bit words,
3692 * LSB-first wire encoding, or active-high chipselects.
97d56dc6
JMC		 3693 *
3694 * Return: zero on success, else a negative error code.
7d077197
DB		 3695 */
3696int spi_setup(struct spi_device *spi)
3697{
83596fbe 3698 unsigned bad_bits, ugly_bits;
73f93db5 3699 int status = 0;
7d077197 3700
d962608c 3701 /*
350de7ce
AS		 3702 * Check mode to prevent that any two of DUAL, QUAD and NO_MOSI/MISO
3703 * are set at the same time.
f477b7fb 3704 */
d962608c
DB		 3705 if ((hweight_long(spi->mode &
3706 (SPI_TX_DUAL | SPI_TX_QUAD | SPI_NO_TX)) > 1) ||
3707 (hweight_long(spi->mode &
3708 (SPI_RX_DUAL | SPI_RX_QUAD | SPI_NO_RX)) > 1)) {
f477b7fb 3709 dev_err(&spi->dev,
d962608c 3710 "setup: can not select any two of dual, quad and no-rx/tx at the same time\n");
f477b7fb 3711 return -EINVAL;
3712 }
350de7ce 3713 /* If it is SPI_3WIRE mode, DUAL and QUAD should be forbidden */
f477b7fb 3714 if ((spi->mode & SPI_3WIRE) && (spi->mode &
6b03061f
YNG		 3715 (SPI_TX_DUAL | SPI_TX_QUAD | SPI_TX_OCTAL |
3716 SPI_RX_DUAL | SPI_RX_QUAD | SPI_RX_OCTAL)))
f477b7fb 3717 return -EINVAL;
350de7ce
AS		 3718 /*
3719 * Help drivers fail *cleanly* when they need options
3720 * that aren't supported with their current controller.
cbaa62e0
DL		 3721 * SPI_CS_WORD has a fallback software implementation,
3722 * so it is ignored here.
e7db06b5 3723 */
d962608c
DB		 3724 bad_bits = spi->mode & ~(spi->controller->mode_bits | SPI_CS_WORD |
3725 SPI_NO_TX | SPI_NO_RX);
83596fbe 3726 ugly_bits = bad_bits &
6b03061f
YNG		 3727 (SPI_TX_DUAL | SPI_TX_QUAD | SPI_TX_OCTAL |
3728 SPI_RX_DUAL | SPI_RX_QUAD | SPI_RX_OCTAL);
83596fbe
GU		 3729 if (ugly_bits) {
3730 dev_warn(&spi->dev,
3731 "setup: ignoring unsupported mode bits %x\n",
3732 ugly_bits);
3733 spi->mode &= ~ugly_bits;
3734 bad_bits &= ~ugly_bits;
3735 }
e7db06b5 3736 if (bad_bits) {
eb288a1f 3737 dev_err(&spi->dev, "setup: unsupported mode bits %x\n",
e7db06b5
DB		 3738 bad_bits);
3739 return -EINVAL;
3740 }
3741
b3fe2e51 3742 if (!spi->bits_per_word) {
7d077197 3743 spi->bits_per_word = 8;
b3fe2e51
PK		 3744 } else {
3745 /*
3746 * Some controllers may not support the default 8 bits-per-word
3747 * so only perform the check when this is explicitly provided.
3748 */
3749 status = __spi_validate_bits_per_word(spi->controller,
3750 spi->bits_per_word);
3751 if (status)
3752 return status;
3753 }
63ab645f 3754
6820e812
TA		 3755 if (spi->controller->max_speed_hz &&
3756 (!spi->max_speed_hz ||
3757 spi->max_speed_hz > spi->controller->max_speed_hz))
8caab75f 3758 spi->max_speed_hz = spi->controller->max_speed_hz;
052eb2d4 3759
4fae3a58
SS		 3760 mutex_lock(&spi->controller->io_mutex);
3761
c914dbf8 3762 if (spi->controller->setup) {
8caab75f 3763 status = spi->controller->setup(spi);
c914dbf8
JB		 3764 if (status) {
3765 mutex_unlock(&spi->controller->io_mutex);
3766 dev_err(&spi->controller->dev, "Failed to setup device: %d\n",
3767 status);
3768 return status;
3769 }
3770 }
7d077197 3771
684a4784
TA		 3772 status = spi_set_cs_timing(spi);
3773 if (status) {
3774 mutex_unlock(&spi->controller->io_mutex);
3775 return status;
3776 }
3777
d948e6ca 3778 if (spi->controller->auto_runtime_pm && spi->controller->set_cs) {
dd769f15 3779 status = pm_runtime_resume_and_get(spi->controller->dev.parent);
d948e6ca 3780 if (status < 0) {
4fae3a58 3781 mutex_unlock(&spi->controller->io_mutex);
d948e6ca
LX		 3782 dev_err(&spi->controller->dev, "Failed to power device: %d\n",
3783 status);
3784 return status;
3785 }
57a94607
TL		 3786
3787 /*
3788 * We do not want to return positive value from pm_runtime_get,
3789 * there are many instances of devices calling spi_setup() and
3790 * checking for a non-zero return value instead of a negative
3791 * return value.
3792 */
3793 status = 0;
3794
d347b4aa 3795 spi_set_cs(spi, false, true);
d948e6ca
LX		 3796 pm_runtime_mark_last_busy(spi->controller->dev.parent);
3797 pm_runtime_put_autosuspend(spi->controller->dev.parent);
3798 } else {
d347b4aa 3799 spi_set_cs(spi, false, true);
d948e6ca 3800 }
abeedb01 3801
4fae3a58
SS		 3802 mutex_unlock(&spi->controller->io_mutex);
3803
924b5867
DA		 3804 if (spi->rt && !spi->controller->rt) {
3805 spi->controller->rt = true;
3806 spi_set_thread_rt(spi->controller);
3807 }
3808
5cb4e1f3
AS		 3809 trace_spi_setup(spi, status);
3810
40b82c2d
AS		 3811 dev_dbg(&spi->dev, "setup mode %lu, %s%s%s%s%u bits/w, %u Hz max --> %d\n",
3812 spi->mode & SPI_MODE_X_MASK,
7d077197
DB		 3813 (spi->mode & SPI_CS_HIGH) ? "cs_high, " : "",
3814 (spi->mode & SPI_LSB_FIRST) ? "lsb, " : "",
3815 (spi->mode & SPI_3WIRE) ? "3wire, " : "",
3816 (spi->mode & SPI_LOOP) ? "loopback, " : "",
3817 spi->bits_per_word, spi->max_speed_hz,
3818 status);
3819
3820 return status;
3821}
3822EXPORT_SYMBOL_GPL(spi_setup);
3823
6c613f68
AA		 3824static int _spi_xfer_word_delay_update(struct spi_transfer *xfer,
3825 struct spi_device *spi)
3826{
3827 int delay1, delay2;
3828
3984d39b 3829 delay1 = spi_delay_to_ns(&xfer->word_delay, xfer);
6c613f68
AA		 3830 if (delay1 < 0)
3831 return delay1;
3832
3984d39b 3833 delay2 = spi_delay_to_ns(&spi->word_delay, xfer);
6c613f68
AA		 3834 if (delay2 < 0)
3835 return delay2;
3836
3837 if (delay1 < delay2)
3838 memcpy(&xfer->word_delay, &spi->word_delay,
3839 sizeof(xfer->word_delay));
3840
3841 return 0;
3842}
3843
90808738 3844static int __spi_validate(struct spi_device *spi, struct spi_message *message)
cf32b71e 3845{
8caab75f 3846 struct spi_controller *ctlr = spi->controller;
e6811d1d 3847 struct spi_transfer *xfer;
6ea31293 3848 int w_size;
cf32b71e 3849
24a0013a
MB		 3850 if (list_empty(&message->transfers))
3851 return -EINVAL;
24a0013a 3852
350de7ce
AS		 3853 /*
3854 * If an SPI controller does not support toggling the CS line on each
71388b21
DL		 3855 * transfer (indicated by the SPI_CS_WORD flag) or we are using a GPIO
3856 * for the CS line, we can emulate the CS-per-word hardware function by
cbaa62e0
DL		 3857 * splitting transfers into one-word transfers and ensuring that
3858 * cs_change is set for each transfer.
3859 */
71388b21 3860 if ((spi->mode & SPI_CS_WORD) && (!(ctlr->mode_bits & SPI_CS_WORD) ||
303feb3c 3861 spi_get_csgpiod(spi, 0))) {
cbaa62e0
DL		 3862 size_t maxsize;
3863 int ret;
3864
3865 maxsize = (spi->bits_per_word + 7) / 8;
3866
3867 /* spi_split_transfers_maxsize() requires message->spi */
3868 message->spi = spi;
3869
3870 ret = spi_split_transfers_maxsize(ctlr, message, maxsize,
3871 GFP_KERNEL);
3872 if (ret)
3873 return ret;
3874
3875 list_for_each_entry(xfer, &message->transfers, transfer_list) {
95c8222f 3876 /* Don't change cs_change on the last entry in the list */
cbaa62e0
DL		 3877 if (list_is_last(&xfer->transfer_list, &message->transfers))
3878 break;
3879 xfer->cs_change = 1;
3880 }
3881 }
3882
350de7ce
AS		 3883 /*
3884 * Half-duplex links include original MicroWire, and ones with
cf32b71e
ES		 3885 * only one data pin like SPI_3WIRE (switches direction) or where
3886 * either MOSI or MISO is missing. They can also be caused by
3887 * software limitations.
3888 */
8caab75f
GU		 3889 if ((ctlr->flags & SPI_CONTROLLER_HALF_DUPLEX) ||
3890 (spi->mode & SPI_3WIRE)) {
3891 unsigned flags = ctlr->flags;
cf32b71e
ES		 3892
3893 list_for_each_entry(xfer, &message->transfers, transfer_list) {
3894 if (xfer->rx_buf && xfer->tx_buf)
3895 return -EINVAL;
8caab75f 3896 if ((flags & SPI_CONTROLLER_NO_TX) && xfer->tx_buf)
cf32b71e 3897 return -EINVAL;
8caab75f 3898 if ((flags & SPI_CONTROLLER_NO_RX) && xfer->rx_buf)
cf32b71e
ES		 3899 return -EINVAL;
3900 }
3901 }
3902
350de7ce 3903 /*
059b8ffe
LD		 3904 * Set transfer bits_per_word and max speed as spi device default if
3905 * it is not set for this transfer.
f477b7fb 3906 * Set transfer tx_nbits and rx_nbits as single transfer default
3907 * (SPI_NBITS_SINGLE) if it is not set for this transfer.
b7bb367a
JB		 3908 * Ensure transfer word_delay is at least as long as that required by
3909 * device itself.
e6811d1d 3910 */
77e80588 3911 message->frame_length = 0;
e6811d1d 3912 list_for_each_entry(xfer, &message->transfers, transfer_list) {
5d7e2b5e 3913 xfer->effective_speed_hz = 0;
078726ce 3914 message->frame_length += xfer->len;
e6811d1d
LD		 3915 if (!xfer->bits_per_word)
3916 xfer->bits_per_word = spi->bits_per_word;
a6f87fad
AL		 3917
3918 if (!xfer->speed_hz)
059b8ffe 3919 xfer->speed_hz = spi->max_speed_hz;
a6f87fad 3920
8caab75f
GU		 3921 if (ctlr->max_speed_hz && xfer->speed_hz > ctlr->max_speed_hz)
3922 xfer->speed_hz = ctlr->max_speed_hz;
56ede94a 3923
8caab75f 3924 if (__spi_validate_bits_per_word(ctlr, xfer->bits_per_word))
63ab645f 3925 return -EINVAL;
a2fd4f9f 3926
4d94bd21
II		 3927 /*
3928 * SPI transfer length should be multiple of SPI word size
350de7ce 3929 * where SPI word size should be power-of-two multiple.
4d94bd21
II		 3930 */
3931 if (xfer->bits_per_word <= 8)
3932 w_size = 1;
3933 else if (xfer->bits_per_word <= 16)
3934 w_size = 2;
3935 else
3936 w_size = 4;
3937
4d94bd21 3938 /* No partial transfers accepted */
6ea31293 3939 if (xfer->len % w_size)
4d94bd21
II		 3940 return -EINVAL;
3941
8caab75f
GU		 3942 if (xfer->speed_hz && ctlr->min_speed_hz &&
3943 xfer->speed_hz < ctlr->min_speed_hz)
a2fd4f9f 3944 return -EINVAL;
f477b7fb 3945
3946 if (xfer->tx_buf && !xfer->tx_nbits)
3947 xfer->tx_nbits = SPI_NBITS_SINGLE;
3948 if (xfer->rx_buf && !xfer->rx_nbits)
3949 xfer->rx_nbits = SPI_NBITS_SINGLE;
350de7ce
AS		 3950 /*
3951 * Check transfer tx/rx_nbits:
1afd9989
GU		 3952 * 1. check the value matches one of single, dual and quad
3953 * 2. check tx/rx_nbits match the mode in spi_device
f477b7fb 3954 */
db90a441 3955 if (xfer->tx_buf) {
d962608c
DB		 3956 if (spi->mode & SPI_NO_TX)
3957 return -EINVAL;
db90a441
SP		 3958 if (xfer->tx_nbits != SPI_NBITS_SINGLE &&
3959 xfer->tx_nbits != SPI_NBITS_DUAL &&
3960 xfer->tx_nbits != SPI_NBITS_QUAD)
3961 return -EINVAL;
3962 if ((xfer->tx_nbits == SPI_NBITS_DUAL) &&
3963 !(spi->mode & (SPI_TX_DUAL | SPI_TX_QUAD)))
3964 return -EINVAL;
3965 if ((xfer->tx_nbits == SPI_NBITS_QUAD) &&
3966 !(spi->mode & SPI_TX_QUAD))
3967 return -EINVAL;
db90a441 3968 }
95c8222f 3969 /* Check transfer rx_nbits */
db90a441 3970 if (xfer->rx_buf) {
d962608c
DB		 3971 if (spi->mode & SPI_NO_RX)
3972 return -EINVAL;
db90a441
SP		 3973 if (xfer->rx_nbits != SPI_NBITS_SINGLE &&
3974 xfer->rx_nbits != SPI_NBITS_DUAL &&
3975 xfer->rx_nbits != SPI_NBITS_QUAD)
3976 return -EINVAL;
3977 if ((xfer->rx_nbits == SPI_NBITS_DUAL) &&
3978 !(spi->mode & (SPI_RX_DUAL | SPI_RX_QUAD)))
3979 return -EINVAL;
3980 if ((xfer->rx_nbits == SPI_NBITS_QUAD) &&
3981 !(spi->mode & SPI_RX_QUAD))
3982 return -EINVAL;
db90a441 3983 }
b7bb367a 3984
6c613f68
AA		 3985 if (_spi_xfer_word_delay_update(xfer, spi))
3986 return -EINVAL;
e6811d1d
LD		 3987 }
3988
cf32b71e 3989 message->status = -EINPROGRESS;
90808738
MB		 3990
3991 return 0;
3992}
3993
3994static int __spi_async(struct spi_device *spi, struct spi_message *message)
3995{
8caab75f 3996 struct spi_controller *ctlr = spi->controller;
b42faeee 3997 struct spi_transfer *xfer;
90808738 3998
b5932f5c
BB		 3999 /*
4000 * Some controllers do not support doing regular SPI transfers. Return
4001 * ENOTSUPP when this is the case.
4002 */
4003 if (!ctlr->transfer)
4004 return -ENOTSUPP;
4005
90808738
MB		 4006 message->spi = spi;
4007
6598b91b
DJ		 4008 SPI_STATISTICS_INCREMENT_FIELD(ctlr->pcpu_statistics, spi_async);
4009 SPI_STATISTICS_INCREMENT_FIELD(spi->pcpu_statistics, spi_async);
eca2ebc7 4010
90808738
MB		 4011 trace_spi_message_submit(message);
4012
b42faeee
VO		 4013 if (!ctlr->ptp_sts_supported) {
4014 list_for_each_entry(xfer, &message->transfers, transfer_list) {
4015 xfer->ptp_sts_word_pre = 0;
4016 ptp_read_system_prets(xfer->ptp_sts);
4017 }
4018 }
4019
8caab75f 4020 return ctlr->transfer(spi, message);
cf32b71e
ES		 4021}
4022
568d0697
DB		 4023/**
4024 * spi_async - asynchronous SPI transfer
4025 * @spi: device with which data will be exchanged
4026 * @message: describes the data transfers, including completion callback
4027 * Context: any (irqs may be blocked, etc)
4028 *
4029 * This call may be used in_irq and other contexts which can't sleep,
4030 * as well as from task contexts which can sleep.
4031 *
4032 * The completion callback is invoked in a context which can't sleep.
4033 * Before that invocation, the value of message->status is undefined.
4034 * When the callback is issued, message->status holds either zero (to
4035 * indicate complete success) or a negative error code. After that
4036 * callback returns, the driver which issued the transfer request may
4037 * deallocate the associated memory; it's no longer in use by any SPI
4038 * core or controller driver code.
4039 *
4040 * Note that although all messages to a spi_device are handled in
4041 * FIFO order, messages may go to different devices in other orders.
4042 * Some device might be higher priority, or have various "hard" access
4043 * time requirements, for example.
4044 *
4045 * On detection of any fault during the transfer, processing of
4046 * the entire message is aborted, and the device is deselected.
4047 * Until returning from the associated message completion callback,
4048 * no other spi_message queued to that device will be processed.
4049 * (This rule applies equally to all the synchronous transfer calls,
4050 * which are wrappers around this core asynchronous primitive.)
97d56dc6
JMC		 4051 *
4052 * Return: zero on success, else a negative error code.
568d0697
DB		 4053 */
4054int spi_async(struct spi_device *spi, struct spi_message *message)
4055{
8caab75f 4056 struct spi_controller *ctlr = spi->controller;
cf32b71e
ES		 4057 int ret;
4058 unsigned long flags;
568d0697 4059
90808738
MB		 4060 ret = __spi_validate(spi, message);
4061 if (ret != 0)
4062 return ret;
4063
8caab75f 4064 spin_lock_irqsave(&ctlr->bus_lock_spinlock, flags);
568d0697 4065
8caab75f 4066 if (ctlr->bus_lock_flag)
cf32b71e
ES		 4067 ret = -EBUSY;
4068 else
4069 ret = __spi_async(spi, message);
568d0697 4070
8caab75f 4071 spin_unlock_irqrestore(&ctlr->bus_lock_spinlock, flags);
cf32b71e
ES		 4072
4073 return ret;
568d0697
DB		 4074}
4075EXPORT_SYMBOL_GPL(spi_async);
4076
cf32b71e
ES		 4077/**
4078 * spi_async_locked - version of spi_async with exclusive bus usage
4079 * @spi: device with which data will be exchanged
4080 * @message: describes the data transfers, including completion callback
4081 * Context: any (irqs may be blocked, etc)
4082 *
4083 * This call may be used in_irq and other contexts which can't sleep,
4084 * as well as from task contexts which can sleep.
4085 *
4086 * The completion callback is invoked in a context which can't sleep.
4087 * Before that invocation, the value of message->status is undefined.
4088 * When the callback is issued, message->status holds either zero (to
4089 * indicate complete success) or a negative error code. After that
4090 * callback returns, the driver which issued the transfer request may
4091 * deallocate the associated memory; it's no longer in use by any SPI
4092 * core or controller driver code.
4093 *
4094 * Note that although all messages to a spi_device are handled in
4095 * FIFO order, messages may go to different devices in other orders.
4096 * Some device might be higher priority, or have various "hard" access
4097 * time requirements, for example.
4098 *
4099 * On detection of any fault during the transfer, processing of
4100 * the entire message is aborted, and the device is deselected.
4101 * Until returning from the associated message completion callback,
4102 * no other spi_message queued to that device will be processed.
4103 * (This rule applies equally to all the synchronous transfer calls,
4104 * which are wrappers around this core asynchronous primitive.)
97d56dc6
JMC		 4105 *
4106 * Return: zero on success, else a negative error code.
cf32b71e 4107 */
da21fde0 4108static int spi_async_locked(struct spi_device *spi, struct spi_message *message)
cf32b71e 4109{
8caab75f 4110 struct spi_controller *ctlr = spi->controller;
cf32b71e
ES		 4111 int ret;
4112 unsigned long flags;
4113
90808738
MB		 4114 ret = __spi_validate(spi, message);
4115 if (ret != 0)
4116 return ret;
4117
8caab75f 4118 spin_lock_irqsave(&ctlr->bus_lock_spinlock, flags);
cf32b71e
ES		 4119
4120 ret = __spi_async(spi, message);
4121
8caab75f 4122 spin_unlock_irqrestore(&ctlr->bus_lock_spinlock, flags);
cf32b71e
ES		 4123
4124 return ret;
4125
4126}
cf32b71e 4127
ae7d2346
DJ		 4128static void __spi_transfer_message_noqueue(struct spi_controller *ctlr, struct spi_message *msg)
4129{
4130 bool was_busy;
4131 int ret;
4132
4133 mutex_lock(&ctlr->io_mutex);
4134
1a9cafcb 4135 was_busy = ctlr->busy;
ae7d2346 4136
72c5c59b 4137 ctlr->cur_msg = msg;
ae7d2346
DJ		 4138 ret = __spi_pump_transfer_message(ctlr, msg, was_busy);
4139 if (ret)
4140 goto out;
4141
69fa9590
DJ		 4142 ctlr->cur_msg = NULL;
4143 ctlr->fallback = false;
4144
ae7d2346
DJ		 4145 if (!was_busy) {
4146 kfree(ctlr->dummy_rx);
4147 ctlr->dummy_rx = NULL;
4148 kfree(ctlr->dummy_tx);
4149 ctlr->dummy_tx = NULL;
4150 if (ctlr->unprepare_transfer_hardware &&
4151 ctlr->unprepare_transfer_hardware(ctlr))
4152 dev_err(&ctlr->dev,
4153 "failed to unprepare transfer hardware\n");
4154 spi_idle_runtime_pm(ctlr);
4155 }
4156
4157out:
4158 mutex_unlock(&ctlr->io_mutex);
4159}
4160
7d077197
DB		 4161/*-------------------------------------------------------------------------*/
4162
350de7ce
AS		 4163/*
4164 * Utility methods for SPI protocol drivers, layered on
7d077197
DB		 4165 * top of the core. Some other utility methods are defined as
4166 * inline functions.
4167 */
4168
5d870c8e
AM		 4169static void spi_complete(void *arg)
4170{
4171 complete(arg);
4172}
4173
ef4d96ec 4174static int __spi_sync(struct spi_device *spi, struct spi_message *message)
cf32b71e
ES		 4175{
4176 DECLARE_COMPLETION_ONSTACK(done);
4177 int status;
8caab75f 4178 struct spi_controller *ctlr = spi->controller;
0461a414
MB		 4179
4180 status = __spi_validate(spi, message);
4181 if (status != 0)
4182 return status;
cf32b71e 4183
0461a414 4184 message->spi = spi;
cf32b71e 4185
6598b91b
DJ		 4186 SPI_STATISTICS_INCREMENT_FIELD(ctlr->pcpu_statistics, spi_sync);
4187 SPI_STATISTICS_INCREMENT_FIELD(spi->pcpu_statistics, spi_sync);
eca2ebc7 4188
350de7ce 4189 /*
ae7d2346
DJ		 4190 * Checking queue_empty here only guarantees async/sync message
4191 * ordering when coming from the same context. It does not need to
4192 * guard against reentrancy from a different context. The io_mutex
4193 * will catch those cases.
0461a414 4194 */
b30f7c8e 4195 if (READ_ONCE(ctlr->queue_empty) && !ctlr->must_async) {
ae7d2346
DJ		 4196 message->actual_length = 0;
4197 message->status = -EINPROGRESS;
0461a414
MB		 4198
4199 trace_spi_message_submit(message);
4200
ae7d2346
DJ		 4201 SPI_STATISTICS_INCREMENT_FIELD(ctlr->pcpu_statistics, spi_sync_immediate);
4202 SPI_STATISTICS_INCREMENT_FIELD(spi->pcpu_statistics, spi_sync_immediate);
0461a414 4203
ae7d2346
DJ		 4204 __spi_transfer_message_noqueue(ctlr, message);
4205
4206 return message->status;
0461a414 4207 }
cf32b71e 4208
ae7d2346
DJ		 4209 /*
4210 * There are messages in the async queue that could have originated
4211 * from the same context, so we need to preserve ordering.
4212 * Therefor we send the message to the async queue and wait until they
4213 * are completed.
4214 */
4215 message->complete = spi_complete;
4216 message->context = &done;
4217 status = spi_async_locked(spi, message);
cf32b71e
ES		 4218 if (status == 0) {
4219 wait_for_completion(&done);
4220 status = message->status;
4221 }
4222 message->context = NULL;
ae7d2346 4223
cf32b71e
ES		 4224 return status;
4225}
4226
8ae12a0d
DB		 4227/**
4228 * spi_sync - blocking/synchronous SPI data transfers
4229 * @spi: device with which data will be exchanged
4230 * @message: describes the data transfers
33e34dc6 4231 * Context: can sleep
8ae12a0d
DB		 4232 *
4233 * This call may only be used from a context that may sleep. The sleep
4234 * is non-interruptible, and has no timeout. Low-overhead controller
4235 * drivers may DMA directly into and out of the message buffers.
4236 *
4237 * Note that the SPI device's chip select is active during the message,
4238 * and then is normally disabled between messages. Drivers for some
4239 * frequently-used devices may want to minimize costs of selecting a chip,
4240 * by leaving it selected in anticipation that the next message will go
4241 * to the same chip. (That may increase power usage.)
4242 *
0c868461
DB		 4243 * Also, the caller is guaranteeing that the memory associated with the
4244 * message will not be freed before this call returns.
4245 *
97d56dc6 4246 * Return: zero on success, else a negative error code.
8ae12a0d
DB		 4247 */
4248int spi_sync(struct spi_device *spi, struct spi_message *message)
4249{
ef4d96ec
MB		 4250 int ret;
4251
8caab75f 4252 mutex_lock(&spi->controller->bus_lock_mutex);
ef4d96ec 4253 ret = __spi_sync(spi, message);
8caab75f 4254 mutex_unlock(&spi->controller->bus_lock_mutex);
ef4d96ec
MB		 4255
4256 return ret;
8ae12a0d
DB		 4257}
4258EXPORT_SYMBOL_GPL(spi_sync);
4259
cf32b71e
ES		 4260/**
4261 * spi_sync_locked - version of spi_sync with exclusive bus usage
4262 * @spi: device with which data will be exchanged
4263 * @message: describes the data transfers
4264 * Context: can sleep
4265 *
4266 * This call may only be used from a context that may sleep. The sleep
4267 * is non-interruptible, and has no timeout. Low-overhead controller
4268 * drivers may DMA directly into and out of the message buffers.
4269 *
4270 * This call should be used by drivers that require exclusive access to the
25985edc 4271 * SPI bus. It has to be preceded by a spi_bus_lock call. The SPI bus must
cf32b71e
ES		 4272 * be released by a spi_bus_unlock call when the exclusive access is over.
4273 *
97d56dc6 4274 * Return: zero on success, else a negative error code.
cf32b71e
ES		 4275 */
4276int spi_sync_locked(struct spi_device *spi, struct spi_message *message)
4277{
ef4d96ec 4278 return __spi_sync(spi, message);
cf32b71e
ES		 4279}
4280EXPORT_SYMBOL_GPL(spi_sync_locked);
4281
4282/**
4283 * spi_bus_lock - obtain a lock for exclusive SPI bus usage
8caab75f 4284 * @ctlr: SPI bus master that should be locked for exclusive bus access
cf32b71e
ES		 4285 * Context: can sleep
4286 *
4287 * This call may only be used from a context that may sleep. The sleep
4288 * is non-interruptible, and has no timeout.
4289 *
4290 * This call should be used by drivers that require exclusive access to the
4291 * SPI bus. The SPI bus must be released by a spi_bus_unlock call when the
4292 * exclusive access is over. Data transfer must be done by spi_sync_locked
4293 * and spi_async_locked calls when the SPI bus lock is held.
4294 *
97d56dc6 4295 * Return: always zero.
cf32b71e 4296 */
8caab75f 4297int spi_bus_lock(struct spi_controller *ctlr)
cf32b71e
ES		 4298{
4299 unsigned long flags;
4300
8caab75f 4301 mutex_lock(&ctlr->bus_lock_mutex);
cf32b71e 4302
8caab75f
GU		 4303 spin_lock_irqsave(&ctlr->bus_lock_spinlock, flags);
4304 ctlr->bus_lock_flag = 1;
4305 spin_unlock_irqrestore(&ctlr->bus_lock_spinlock, flags);
cf32b71e 4306
95c8222f 4307 /* Mutex remains locked until spi_bus_unlock() is called */
cf32b71e
ES		 4308
4309 return 0;
4310}
4311EXPORT_SYMBOL_GPL(spi_bus_lock);
4312
4313/**
4314 * spi_bus_unlock - release the lock for exclusive SPI bus usage
8caab75f 4315 * @ctlr: SPI bus master that was locked for exclusive bus access
cf32b71e
ES		 4316 * Context: can sleep
4317 *
4318 * This call may only be used from a context that may sleep. The sleep
4319 * is non-interruptible, and has no timeout.
4320 *
4321 * This call releases an SPI bus lock previously obtained by an spi_bus_lock
4322 * call.
4323 *
97d56dc6 4324 * Return: always zero.
cf32b71e 4325 */
8caab75f 4326int spi_bus_unlock(struct spi_controller *ctlr)
cf32b71e 4327{
8caab75f 4328 ctlr->bus_lock_flag = 0;
cf32b71e 4329
8caab75f 4330 mutex_unlock(&ctlr->bus_lock_mutex);
cf32b71e
ES		 4331
4332 return 0;
4333}
4334EXPORT_SYMBOL_GPL(spi_bus_unlock);
4335
95c8222f 4336/* Portable code must never pass more than 32 bytes */
5fe5f05e 4337#define SPI_BUFSIZ max(32, SMP_CACHE_BYTES)
8ae12a0d
DB		 4338
4339static u8 *buf;
4340
4341/**
4342 * spi_write_then_read - SPI synchronous write followed by read
4343 * @spi: device with which data will be exchanged
4344 * @txbuf: data to be written (need not be dma-safe)
4345 * @n_tx: size of txbuf, in bytes
27570497
JP		 4346 * @rxbuf: buffer into which data will be read (need not be dma-safe)
4347 * @n_rx: size of rxbuf, in bytes
33e34dc6 4348 * Context: can sleep
8ae12a0d
DB		 4349 *
4350 * This performs a half duplex MicroWire style transaction with the
4351 * device, sending txbuf and then reading rxbuf. The return value
4352 * is zero for success, else a negative errno status code.
b885244e 4353 * This call may only be used from a context that may sleep.
8ae12a0d 4354 *
c373643b 4355 * Parameters to this routine are always copied using a small buffer.
33e34dc6 4356 * Performance-sensitive or bulk transfer code should instead use
0c868461 4357 * spi_{async,sync}() calls with dma-safe buffers.
97d56dc6
JMC		 4358 *
4359 * Return: zero on success, else a negative error code.
8ae12a0d
DB		 4360 */
4361int spi_write_then_read(struct spi_device *spi,
0c4a1590
MB		 4362 const void *txbuf, unsigned n_tx,
4363 void *rxbuf, unsigned n_rx)
8ae12a0d 4364{
068f4070 4365 static DEFINE_MUTEX(lock);
8ae12a0d
DB		 4366
4367 int status;
4368 struct spi_message message;
bdff549e 4369 struct spi_transfer x[2];
8ae12a0d
DB		 4370 u8 *local_buf;
4371
350de7ce
AS		 4372 /*
4373 * Use preallocated DMA-safe buffer if we can. We can't avoid
b3a223ee
MB		 4374 * copying here, (as a pure convenience thing), but we can
4375 * keep heap costs out of the hot path unless someone else is
4376 * using the pre-allocated buffer or the transfer is too large.
8ae12a0d 4377 */
b3a223ee 4378 if ((n_tx + n_rx) > SPI_BUFSIZ || !mutex_trylock(&lock)) {
2cd94c8a
MB		 4379 local_buf = kmalloc(max((unsigned)SPI_BUFSIZ, n_tx + n_rx),
4380 GFP_KERNEL | GFP_DMA);
b3a223ee
MB		 4381 if (!local_buf)
4382 return -ENOMEM;
4383 } else {
4384 local_buf = buf;
4385 }
8ae12a0d 4386
8275c642 4387 spi_message_init(&message);
5fe5f05e 4388 memset(x, 0, sizeof(x));
bdff549e
DB		 4389 if (n_tx) {
4390 x[0].len = n_tx;
4391 spi_message_add_tail(&x[0], &message);
4392 }
4393 if (n_rx) {
4394 x[1].len = n_rx;
4395 spi_message_add_tail(&x[1], &message);
4396 }
8275c642 4397
8ae12a0d 4398 memcpy(local_buf, txbuf, n_tx);
bdff549e
DB		 4399 x[0].tx_buf = local_buf;
4400 x[1].rx_buf = local_buf + n_tx;
8ae12a0d 4401
95c8222f 4402 /* Do the i/o */
8ae12a0d 4403 status = spi_sync(spi, &message);
9b938b74 4404 if (status == 0)
bdff549e 4405 memcpy(rxbuf, x[1].rx_buf, n_rx);
8ae12a0d 4406
bdff549e 4407 if (x[0].tx_buf == buf)
068f4070 4408 mutex_unlock(&lock);
8ae12a0d
DB		 4409 else
4410 kfree(local_buf);
4411
4412 return status;
4413}
4414EXPORT_SYMBOL_GPL(spi_write_then_read);
4415
4416/*-------------------------------------------------------------------------*/
4417
da21fde0 4418#if IS_ENABLED(CONFIG_OF_DYNAMIC)
95c8222f 4419/* Must call put_device() when done with returned spi_device device */
da21fde0 4420static struct spi_device *of_find_spi_device_by_node(struct device_node *node)
ce79d54a 4421{
cfba5de9
SP		 4422 struct device *dev = bus_find_device_by_of_node(&spi_bus_type, node);
4423
ce79d54a
PA		 4424 return dev ? to_spi_device(dev) : NULL;
4425}
4426
95c8222f 4427/* The spi controllers are not using spi_bus, so we find it with another way */
8caab75f 4428static struct spi_controller *of_find_spi_controller_by_node(struct device_node *node)
ce79d54a
PA		 4429{
4430 struct device *dev;
4431
cfba5de9 4432 dev = class_find_device_by_of_node(&spi_master_class, node);
6c364062 4433 if (!dev && IS_ENABLED(CONFIG_SPI_SLAVE))
cfba5de9 4434 dev = class_find_device_by_of_node(&spi_slave_class, node);
ce79d54a
PA		 4435 if (!dev)
4436 return NULL;
4437
95c8222f 4438 /* Reference got in class_find_device */
8caab75f 4439 return container_of(dev, struct spi_controller, dev);
ce79d54a
PA		 4440}
4441
4442static int of_spi_notify(struct notifier_block *nb, unsigned long action,
4443 void *arg)
4444{
4445 struct of_reconfig_data *rd = arg;
8caab75f 4446 struct spi_controller *ctlr;
ce79d54a
PA		 4447 struct spi_device *spi;
4448
4449 switch (of_reconfig_get_state_change(action, arg)) {
4450 case OF_RECONFIG_CHANGE_ADD:
8caab75f
GU		 4451 ctlr = of_find_spi_controller_by_node(rd->dn->parent);
4452 if (ctlr == NULL)
95c8222f 4453 return NOTIFY_OK; /* Not for us */
ce79d54a 4454
bd6c1644 4455 if (of_node_test_and_set_flag(rd->dn, OF_POPULATED)) {
8caab75f 4456 put_device(&ctlr->dev);
bd6c1644
GU		 4457 return NOTIFY_OK;
4458 }
4459
8caab75f
GU		 4460 spi = of_register_spi_device(ctlr, rd->dn);
4461 put_device(&ctlr->dev);
ce79d54a
PA		 4462
4463 if (IS_ERR(spi)) {
25c56c88
RH		 4464 pr_err("%s: failed to create for '%pOF'\n",
4465 __func__, rd->dn);
e0af98a7 4466 of_node_clear_flag(rd->dn, OF_POPULATED);
ce79d54a
PA		 4467 return notifier_from_errno(PTR_ERR(spi));
4468 }
4469 break;
4470
4471 case OF_RECONFIG_CHANGE_REMOVE:
95c8222f 4472 /* Already depopulated? */
bd6c1644
GU		 4473 if (!of_node_check_flag(rd->dn, OF_POPULATED))
4474 return NOTIFY_OK;
4475
95c8222f 4476 /* Find our device by node */
ce79d54a
PA		 4477 spi = of_find_spi_device_by_node(rd->dn);
4478 if (spi == NULL)
95c8222f 4479 return NOTIFY_OK; /* No? not meant for us */
ce79d54a 4480
95c8222f 4481 /* Unregister takes one ref away */
ce79d54a
PA		 4482 spi_unregister_device(spi);
4483
95c8222f 4484 /* And put the reference of the find */
ce79d54a
PA		 4485 put_device(&spi->dev);
4486 break;
4487 }
4488
4489 return NOTIFY_OK;
4490}
4491
4492static struct notifier_block spi_of_notifier = {
4493 .notifier_call = of_spi_notify,
4494};
4495#else /* IS_ENABLED(CONFIG_OF_DYNAMIC) */
4496extern struct notifier_block spi_of_notifier;
4497#endif /* IS_ENABLED(CONFIG_OF_DYNAMIC) */
4498
7f24467f 4499#if IS_ENABLED(CONFIG_ACPI)
8caab75f 4500static int spi_acpi_controller_match(struct device *dev, const void *data)
7f24467f
OP		 4501{
4502 return ACPI_COMPANION(dev->parent) == data;
4503}
4504
8caab75f 4505static struct spi_controller *acpi_spi_find_controller_by_adev(struct acpi_device *adev)
7f24467f
OP		 4506{
4507 struct device *dev;
4508
4509 dev = class_find_device(&spi_master_class, NULL, adev,
8caab75f 4510 spi_acpi_controller_match);
6c364062
GU		 4511 if (!dev && IS_ENABLED(CONFIG_SPI_SLAVE))
4512 dev = class_find_device(&spi_slave_class, NULL, adev,
8caab75f 4513 spi_acpi_controller_match);
7f24467f
OP		 4514 if (!dev)
4515 return NULL;
4516
8caab75f 4517 return container_of(dev, struct spi_controller, dev);
7f24467f
OP		 4518}
4519
4520static struct spi_device *acpi_spi_find_device_by_adev(struct acpi_device *adev)
4521{
4522 struct device *dev;
4523
00500147 4524 dev = bus_find_device_by_acpi_dev(&spi_bus_type, adev);
5b16668e 4525 return to_spi_device(dev);
7f24467f
OP		 4526}
4527
4528static int acpi_spi_notify(struct notifier_block *nb, unsigned long value,
4529 void *arg)
4530{
4531 struct acpi_device *adev = arg;
8caab75f 4532 struct spi_controller *ctlr;
7f24467f
OP		 4533 struct spi_device *spi;
4534
4535 switch (value) {
4536 case ACPI_RECONFIG_DEVICE_ADD:
62fcb99b 4537 ctlr = acpi_spi_find_controller_by_adev(acpi_dev_parent(adev));
8caab75f 4538 if (!ctlr)
7f24467f
OP		 4539 break;
4540
8caab75f
GU		 4541 acpi_register_spi_device(ctlr, adev);
4542 put_device(&ctlr->dev);
7f24467f
OP		 4543 break;
4544 case ACPI_RECONFIG_DEVICE_REMOVE:
4545 if (!acpi_device_enumerated(adev))
4546 break;
4547
4548 spi = acpi_spi_find_device_by_adev(adev);
4549 if (!spi)
4550 break;
4551
4552 spi_unregister_device(spi);
4553 put_device(&spi->dev);
4554 break;
4555 }
4556
4557 return NOTIFY_OK;
4558}
4559
4560static struct notifier_block spi_acpi_notifier = {
4561 .notifier_call = acpi_spi_notify,
4562};
4563#else
4564extern struct notifier_block spi_acpi_notifier;
4565#endif
4566
8ae12a0d
DB		 4567static int __init spi_init(void)
4568{
b885244e
DB		 4569 int status;
4570
e94b1766 4571 buf = kmalloc(SPI_BUFSIZ, GFP_KERNEL);
b885244e
DB		 4572 if (!buf) {
4573 status = -ENOMEM;
4574 goto err0;
4575 }
4576
4577 status = bus_register(&spi_bus_type);
4578 if (status < 0)
4579 goto err1;
8ae12a0d 4580
b885244e
DB		 4581 status = class_register(&spi_master_class);
4582 if (status < 0)
4583 goto err2;
ce79d54a 4584
6c364062
GU		 4585 if (IS_ENABLED(CONFIG_SPI_SLAVE)) {
4586 status = class_register(&spi_slave_class);
4587 if (status < 0)
4588 goto err3;
4589 }
4590
5267720e 4591 if (IS_ENABLED(CONFIG_OF_DYNAMIC))
ce79d54a 4592 WARN_ON(of_reconfig_notifier_register(&spi_of_notifier));
7f24467f
OP		 4593 if (IS_ENABLED(CONFIG_ACPI))
4594 WARN_ON(acpi_reconfig_notifier_register(&spi_acpi_notifier));
ce79d54a 4595
8ae12a0d 4596 return 0;
b885244e 4597
6c364062
GU		 4598err3:
4599 class_unregister(&spi_master_class);
b885244e
DB		 4600err2:
4601 bus_unregister(&spi_bus_type);
4602err1:
4603 kfree(buf);
4604 buf = NULL;
4605err0:
4606 return status;
8ae12a0d 4607}
b885244e 4608
350de7ce
AS		 4609/*
4610 * A board_info is normally registered in arch_initcall(),
4611 * but even essential drivers wait till later.
b885244e 4612 *
350de7ce
AS		 4613 * REVISIT only boardinfo really needs static linking. The rest (device and
4614 * driver registration) _could_ be dynamically linked (modular) ... Costs
b885244e 4615 * include needing to have boardinfo data structures be much more public.
8ae12a0d 4616 */
673c0c00 4617postcore_initcall(spi_init);
Linux 6.x block layer and io_uring tree(s)